From ffec377dd2d8b3d091dcc20c33c1354eed89836d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 10 Jan 2020 14:44:00 -0600 Subject: [PATCH 1/4] MQE-1918: MFTF AWS Secrets Manager - Local Use --- composer.json | 3 + composer.lock | 148 ++++++++++++++++- .../AwsSecretManagerStorageTest.php | 65 ++++++++ docs/configuration.md | 22 +++ docs/credentials.md | 67 +++++++- etc/config/.env.example | 6 +- .../Handlers/CredentialStore.php | 27 ++- .../SecretStorage/AwsSecretManagerStorage.php | 155 ++++++++++++++++++ .../Handlers/SecretStorage/VaultStorage.php | 2 +- 9 files changed, 483 insertions(+), 12 deletions(-) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php diff --git a/composer.json b/composer.json index 158287fbf..ca1c02572 100755 --- a/composer.json +++ b/composer.json @@ -11,7 +11,10 @@ "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*", "allure-framework/allure-codeception": "~1.3.0", + "aws/aws-sdk-php": "^3.132", "codeception/codeception": "~2.4.5", "composer/composer": "^1.4", "consolidation/robo": "^1.0.0", diff --git a/composer.lock b/composer.lock index 8f2fcb8e9..6bfc30ab9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59e95cc1ae6311e93111bd7ced180d29", + "content-hash": "2325a3a38edb33b24f6e33bd3009fd8a", "packages": [ { "name": "allure-framework/allure-codeception", @@ -109,6 +109,90 @@ ], "time": "2016-12-07T12:15:46+00:00" }, + { + "name": "aws/aws-sdk-php", + "version": "3.132.2", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f890689c6db27625522ea2e7e9b8420b6fccb063", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^5.3.3|^6.2.1|^7.0", + "guzzlehttp/promises": "^1.0", + "guzzlehttp/psr7": "^1.4.1", + "mtdowling/jmespath.php": "^2.5", + "php": ">=5.5" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-pcntl": "*", + "ext-sockets": "*", + "nette/neon": "^2.3", + "phpunit/phpunit": "^4.8.35|^5.4.3", + "psr/cache": "^1.0", + "psr/simple-cache": "^1.0", + "sebastian/comparator": "^1.2.3" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Aws\\": "src/" + }, + "files": [ + "src/functions.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "http://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "http://aws.amazon.com/sdkforphp", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "time": "2020-01-09T19:09:31+00:00" + }, { "name": "behat/gherkin", "version": "v4.4.5", @@ -2542,8 +2626,66 @@ "bcmath", "math" ], + "abandoned": "brick/math", "time": "2017-02-16T16:54:46+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.5.0", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "52168cb9472de06979613d365c7f1ab8798be895" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/52168cb9472de06979613d365c7f1ab8798be895", + "reference": "52168cb9472de06979613d365c7f1ab8798be895", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "symfony/polyfill-mbstring": "^1.4" + }, + "require-dev": { + "composer/xdebug-handler": "^1.2", + "phpunit/phpunit": "^4.8.36|^7.5.15" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.5-dev" + } + }, + "autoload": { + "psr-4": { + "JmesPath\\": "src/" + }, + "files": [ + "src/JmesPath.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "time": "2019-12-30T18:03:34+00:00" + }, { "name": "mustache/mustache", "version": "v2.12.0", @@ -6542,7 +6684,9 @@ "prefer-lowest": false, "platform": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "ext-curl": "*" + "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*" }, "platform-dev": [] } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php new file mode 100644 index 000000000..015c7a009 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php @@ -0,0 +1,65 @@ + 'mftf/magento/' . $testShortKey, + 'SecretString' => json_encode([$testShortKey => $testValue]) + ]; + /** @var Result */ + $result = new Result($data); + + $mockClient = $this->getMockBuilder(SecretsManagerClient::class) + ->disableOriginalConstructor() + ->setMethods(['__call']) + ->getMock(); + + $mockClient->expects($this->once()) + ->method('__call') + ->willReturnCallback(function($name, $args) use ($result) { + return $result; + }); + + /** @var SecretsManagerClient */ + $credentialStorage = new AwsSecretManagerStorage($testRegion, $testProfile); + $reflection = new ReflectionClass($credentialStorage); + $reflection_property = $reflection->getProperty('client'); + $reflection_property->setAccessible(true); + $reflection_property->setValue($credentialStorage, $mockClient); + + // Test getEncryptedValue() + $encryptedCred = $credentialStorage->getEncryptedValue($testLongKey); + + // Assert the value we've gotten is in fact not identical to our test value + $this->assertNotEquals($testValue, $encryptedCred); + + // Test getDecryptedValue() + $actualValue = $credentialStorage->getDecryptedValue($encryptedCred); + + // Assert that we are able to successfully decrypt our secret value + $this->assertEquals($testValue, $actualValue); + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 5140c01e7..f89090c06 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,28 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` +### CREDENTIAL_AWS_SECRET_MANAGER_REGION + +The region that Aws Secret Manager is located. + +Example: + +```conf +# Region of Aws Secret Manager +CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +``` + +### CREDENTIAL_AWS_SECRET_MANAGER_PROFILE + +The profile used to connect to Aws Secret Manager. + +Example: + +```conf +# Profile used to connect to Aws Secret Manager. +CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +``` + ### ENABLE_BROWSER_LOG Enables addition of browser logs to Allure steps diff --git a/docs/credentials.md b/docs/credentials.md index a2850cfe8..6ca9900cb 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -3,10 +3,11 @@ When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. -Currently the MFTF supports two types of credential storage: +Currently the MFTF supports three types of credential storage: - **.credentials file** -- **HashiCorp vault** +- **HashiCorp Vault** +- **Aws Secret Manager** ## Configure File Storage @@ -135,11 +136,64 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure both File Storage and Vault Storage +## Configure Aws Secret Manager -It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. -In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. +Aws Secrets Manager offers secret management that supports: +- Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB +- Fine-grained policies and permissions +- Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises +### Prerequisites +- AWS account +- AWS Secret Manger is created and configured +- IAM User or Role is created + +### Store secrets in Aws Secret Manager + +#### Secrets format +`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an Aws Secret. +`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: + +```conf +mftf// +``` + +```conf +# Secret name for carriers_usps_userid +mftf/magento/carriers_usps_userid + +# Secret key for carriers_usps_userid +carriers_usps_userid + +# Secret name for carriers_usps_password +mftf/magento/carriers_usps_password + +# Secret key for carriers_usps_password +carriers_usps_password +``` + +### Setup MFTF to use Aws Secret Manager + +To use Aws Secret Manager, the Aws region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_REGION`][] in `.env`. + +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to Aws Secret Manager service. +You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. +Optionally, however, you can explicitly set Aws profile through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`][] in `.env`. + +```conf +# Sample Aws Secret Manager configuration +CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +``` + +## Configure multiple credential storage + +It is possible and sometimes useful to setup and use multiple credential storage at the same time. +In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: + +``` +.credentials File > HashiCorp Vault > Aws Secret Manager +``` ## Use credentials in a test @@ -183,3 +237,6 @@ The MFTF tests delivered with Magento application do not use credentials and do [Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path +[credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html +[`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`]: configuration.md#credential_aws_secret_manager_profile +[`CREDENTIAL_AWS_SECRET_MANAGER_REGION`]: configuration.md#credential_aws_secret_manager_region \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 7320d8b8b..a772b1e9e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,10 +30,14 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#*** To use HashiCorp Vault to manage _CREDS secrets, uncomment and set vault address and secret base path ***# #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +#*** To use AWS Secret Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 94ff40069..bff3e9b00 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,12 +9,18 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; + const ARRAY_KEY_FOR_AWS_SECRET_MANAGER = 'aws'; + + const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' + . '.credentials file, HashiCorp Vault and AWS Secret Manager. ' + . 'You need to configure at least one to use _CREDS in tests.'; /** * Credential storage array @@ -71,9 +77,25 @@ private function __construct() } } + // Initialize AWS secret manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRET_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRET_MANAGER_PROFILE'); + if ($awsRegion !== false) { + if ($awsProfile === false) { + $awsProfile = null; + } + try { + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRET_MANAGER] = new AwsSecretManagerStorage( + $awsRegion, + $awsProfile + ); + } catch (TestFrameworkException $e) { + } + } + if (empty($this->credStorage)) { throw new TestFrameworkException( - "No credential storage is properly configured. Please configure vault or .credentials file." + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO ); } } @@ -97,8 +119,7 @@ public function getSecret($key) } throw new TestFrameworkException( - "\"{$key}\" not defined in vault or .credentials file, " - . "please provide a value in order to use this secret in a test." + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO . ' And make sure key/value exists.' ); } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php new file mode 100644 index 000000000..c617f1cac --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php @@ -0,0 +1,155 @@ +createAwsSecretManagerClient($region, $profile); + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + * @throws Exception + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Retrieving secret for key name {$key} from AWS Secret Manager" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret id + list($vendor, $key) = explode('/', trim($key, '/'), 2); + $secretId = self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by id from AWS Secret Manager, and parse the result + $value = $this->parseAwsSecretResult( + $this->client->getSecretValue(['SecretId' => $secretId]), + $key + ); + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (AwsException $e) { + $error = $e->getAwsErrorCode(); + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "AWS error code: {$error}. Unable to read secret for key {$key} from AWS Secret Manager" + ); + } + } catch (\Exception $e) { + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Unable to read secret for key {$key} from AWS Secret Manager" + ); + } + } + return $reValue; + } + + /** + * Parse AWS result object and return secret for key + * + * @param Result $awsResult + * @param string $key + * @return string + * @throws TestFrameworkException + */ + private function parseAwsSecretResult($awsResult, $key) + { + // Return secret from the associated KMS CMK + if (isset($awsResult['SecretString'])) { + $rawSecret = $awsResult['SecretString']; + } else { + throw new TestFrameworkException("Error parsing AWS secret result"); + } + $secret = json_decode($rawSecret, true); + if (isset($secret[$key])) { + return $secret[$key]; + } + throw new TestFrameworkException("Error parsing AWS secret result"); + } + + /** + * Create Aws Secret Manager client + * + * @param string $region + * @param string $profile + * @throws TestFrameworkException + * @throws InvalidArgumentException + */ + private function createAwsSecretManagerClient($region, $profile) + { + if (null !== $this->client) { + return; + } + + // Create AWS Secret Manager client + $this->client = new SecretsManagerClient([ + 'profile' => $profile, + 'region' => $region, + 'version' => self::LATEST_VERSION + ]); + + if ($this->client === null) { + throw new TestFrameworkException("Unable to create AWS Secret Manager client"); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 6a9e9f0cf..798ac660c 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -67,7 +67,7 @@ class VaultStorage extends BaseStorage private $secretBasePath; /** - * CredentialVault constructor + * VaultStorage constructor * * @param string $baseUrl * @param string $secretBasePath From 579c96d82c9786392cf85b78162947d42f686802 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 10 Jan 2020 14:49:15 -0600 Subject: [PATCH 2/4] MQE-1918: MFTF AWS Secrets Manager - Local Use --- .../Handlers/SecretStorage/AwsSecretManagerStorageTest.php | 2 +- .../Handlers/SecretStorage/AwsSecretManagerStorage.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php index 015c7a009..f11d14920 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php @@ -39,7 +39,7 @@ public function testEncryptAndDecrypt() $mockClient->expects($this->once()) ->method('__call') - ->willReturnCallback(function($name, $args) use ($result) { + ->willReturnCallback(function ($name, $args) use ($result) { return $result; }); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php index c617f1cac..0c10c53c4 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php @@ -132,6 +132,7 @@ private function parseAwsSecretResult($awsResult, $key) * * @param string $region * @param string $profile + * @return void * @throws TestFrameworkException * @throws InvalidArgumentException */ From d0d98051488658d8325b7e2e0fe3a969c9cd45ee Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 14 Jan 2020 15:36:30 -0600 Subject: [PATCH 3/4] MQE-1918: MFTF AWS Secrets Manager - Local Use --- ...t.php => AwsSecretsManagerStorageTest.php} | 8 ++--- docs/configuration.md | 16 ++++----- docs/credentials.md | 34 +++++++++---------- etc/config/.env.example | 6 ++-- .../Handlers/CredentialStore.php | 18 +++++----- ...orage.php => AwsSecretsManagerStorage.php} | 34 +++++++++---------- 6 files changed, 58 insertions(+), 58 deletions(-) rename dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/{AwsSecretManagerStorageTest.php => AwsSecretsManagerStorageTest.php} (88%) rename src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/{AwsSecretManagerStorage.php => AwsSecretsManagerStorage.php} (75%) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php similarity index 88% rename from dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php rename to dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php index f11d14920..e1f4e4879 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php @@ -7,15 +7,15 @@ namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers\SecretStorage; use Aws\SecretsManager\SecretsManagerClient; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Aws\Result; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use ReflectionClass; -class AwsSecretManagerStorageTest extends MagentoTestCase +class AwsSecretsManagerStorageTest extends MagentoTestCase { /** - * Test encryption/decryption functionality in AwsSecretManagerStorage class. + * Test encryption/decryption functionality in AwsSecretsManagerStorage class. */ public function testEncryptAndDecrypt() { @@ -44,7 +44,7 @@ public function testEncryptAndDecrypt() }); /** @var SecretsManagerClient */ - $credentialStorage = new AwsSecretManagerStorage($testRegion, $testProfile); + $credentialStorage = new AwsSecretsManagerStorage($testRegion, $testProfile); $reflection = new ReflectionClass($credentialStorage); $reflection_property = $reflection->getProperty('client'); $reflection_property->setAccessible(true); diff --git a/docs/configuration.md b/docs/configuration.md index f89090c06..9466f2bcc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,26 +277,26 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -### CREDENTIAL_AWS_SECRET_MANAGER_REGION +### CREDENTIAL_AWS_SECRETS_MANAGER_REGION -The region that Aws Secret Manager is located. +The region that AWS Secrets Manager is located. Example: ```conf -# Region of Aws Secret Manager -CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +# Region of AWS Secrets Manager +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 ``` -### CREDENTIAL_AWS_SECRET_MANAGER_PROFILE +### CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE -The profile used to connect to Aws Secret Manager. +The profile used to connect to AWS Secrets Manager. Example: ```conf -# Profile used to connect to Aws Secret Manager. -CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +# Profile used to connect to AWS Secrets Manager. +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` ### ENABLE_BROWSER_LOG diff --git a/docs/credentials.md b/docs/credentials.md index 6ca9900cb..4b20e68dc 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -7,7 +7,7 @@ Currently the MFTF supports three types of credential storage: - **.credentials file** - **HashiCorp Vault** -- **Aws Secret Manager** +- **AWS Secrets Manager** ## Configure File Storage @@ -136,22 +136,22 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure Aws Secret Manager +## Configure AWS Secrets Manager -Aws Secrets Manager offers secret management that supports: +AWS Secrets Manager offers secret management that supports: - Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB - Fine-grained policies and permissions - Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises ### Prerequisites - AWS account -- AWS Secret Manger is created and configured -- IAM User or Role is created +- AWS Secrets Manger is created and configured +- IAM User or Role is created with appropriate AWS Secrets Manger access permission -### Store secrets in Aws Secret Manager +### Store secrets in AWS Secrets Manager #### Secrets format -`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an Aws Secret. +`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an AWS Secret. `Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: ```conf @@ -172,18 +172,18 @@ mftf/magento/carriers_usps_password carriers_usps_password ``` -### Setup MFTF to use Aws Secret Manager +### Setup MFTF to use AWS Secrets Manager -To use Aws Secret Manager, the Aws region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_REGION`][] in `.env`. +To use AWS Secrets Manager, the AWS region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`][] in `.env`. -MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to Aws Secret Manager service. +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to AWS Secrets Manager service. You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. -Optionally, however, you can explicitly set Aws profile through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`][] in `.env`. +Optionally, however, you can explicitly set AWS profile through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`][] in `.env`. ```conf -# Sample Aws Secret Manager configuration -CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 -CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +# Sample AWS Secrets Manager configuration +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` ## Configure multiple credential storage @@ -192,7 +192,7 @@ It is possible and sometimes useful to setup and use multiple credential storage In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: ``` -.credentials File > HashiCorp Vault > Aws Secret Manager +.credentials File > HashiCorp Vault > AWS Secrets Manager ``` @@ -238,5 +238,5 @@ The MFTF tests delivered with Magento application do not use credentials and do [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path [credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html -[`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`]: configuration.md#credential_aws_secret_manager_profile -[`CREDENTIAL_AWS_SECRET_MANAGER_REGION`]: configuration.md#credential_aws_secret_manager_region \ No newline at end of file +[`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile +[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index a772b1e9e..f5b6ef40e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -34,9 +34,9 @@ BROWSER=chrome #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret -#*** To use AWS Secret Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# -#CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default -#CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +#*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index bff3e9b00..76560bcf1 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,17 +9,17 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; - const ARRAY_KEY_FOR_AWS_SECRET_MANAGER = 'aws'; + const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' - . '.credentials file, HashiCorp Vault and AWS Secret Manager. ' + . '.credentials file, HashiCorp Vault and AWS Secrets Manager. ' . 'You need to configure at least one to use _CREDS in tests.'; /** @@ -77,15 +77,15 @@ private function __construct() } } - // Initialize AWS secret manager storage - $awsRegion = getenv('CREDENTIAL_AWS_SECRET_MANAGER_REGION'); - $awsProfile = getenv('CREDENTIAL_AWS_SECRET_MANAGER_PROFILE'); + // Initialize AWS Secrets Manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); if ($awsRegion !== false) { if ($awsProfile === false) { $awsProfile = null; } try { - $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRET_MANAGER] = new AwsSecretManagerStorage( + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( $awsRegion, $awsProfile ); @@ -109,8 +109,8 @@ private function __construct() */ public function getSecret($key) { - // Get secret data from storage according to the order they are stored - // File storage is preferred over vault storage to allow local secret value overriding remote secret value + // Get secret data from storage according to the order they are stored which follows this precedence: + // FileStorage > VaultStorage > AwsSecretsManagerStorage foreach ($this->credStorage as $storage) { $value = $storage->getEncryptedValue($key); if (null !== $value) { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php similarity index 75% rename from src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php rename to src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php index 0c10c53c4..e7a858254 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -15,7 +15,7 @@ use InvalidArgumentException; use Exception; -class AwsSecretManagerStorage extends BaseStorage +class AwsSecretsManagerStorage extends BaseStorage { /** * Mftf project path @@ -23,7 +23,7 @@ class AwsSecretManagerStorage extends BaseStorage const MFTF_PATH = 'mftf'; /** - * AWS Secret Manager version + * AWS Secrets Manager version * * Last tested version '2017-10-17' */ @@ -37,7 +37,7 @@ class AwsSecretManagerStorage extends BaseStorage private $client = null; /** - * AwsSecretManagerStorage constructor + * AwsSecretsManagerStorage constructor * * @param string $region * @param string $profile @@ -47,7 +47,7 @@ class AwsSecretManagerStorage extends BaseStorage public function __construct($region, $profile = null) { parent::__construct(); - $this->createAwsSecretManagerClient($region, $profile); + $this->createAwsSecretsManagerClient($region, $profile); } /** @@ -65,8 +65,8 @@ public function getEncryptedValue($key) } if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Retrieving secret for key name {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Retrieving value for key name {$key} from AWS Secrets Manager" ); } @@ -79,7 +79,7 @@ public function getEncryptedValue($key) . $vendor . '/' . $key; - // Read value by id from AWS Secret Manager, and parse the result + // Read value by id from AWS Secrets Manager, and parse the result $value = $this->parseAwsSecretResult( $this->client->getSecretValue(['SecretId' => $secretId]), $key @@ -90,14 +90,14 @@ public function getEncryptedValue($key) } catch (AwsException $e) { $error = $e->getAwsErrorCode(); if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "AWS error code: {$error}. Unable to read secret for key {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "AWS error code: {$error}. Unable to read value for key {$key} from AWS Secrets Manager" ); } } catch (\Exception $e) { if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Unable to read secret for key {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Unable to read value for key {$key} from AWS Secrets Manager" ); } } @@ -118,17 +118,17 @@ private function parseAwsSecretResult($awsResult, $key) if (isset($awsResult['SecretString'])) { $rawSecret = $awsResult['SecretString']; } else { - throw new TestFrameworkException("Error parsing AWS secret result"); + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); } $secret = json_decode($rawSecret, true); if (isset($secret[$key])) { return $secret[$key]; } - throw new TestFrameworkException("Error parsing AWS secret result"); + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); } /** - * Create Aws Secret Manager client + * Create Aws Secrets Manager client * * @param string $region * @param string $profile @@ -136,13 +136,13 @@ private function parseAwsSecretResult($awsResult, $key) * @throws TestFrameworkException * @throws InvalidArgumentException */ - private function createAwsSecretManagerClient($region, $profile) + private function createAwsSecretsManagerClient($region, $profile) { if (null !== $this->client) { return; } - // Create AWS Secret Manager client + // Create AWS Secrets Manager client $this->client = new SecretsManagerClient([ 'profile' => $profile, 'region' => $region, @@ -150,7 +150,7 @@ private function createAwsSecretManagerClient($region, $profile) ]); if ($this->client === null) { - throw new TestFrameworkException("Unable to create AWS Secret Manager client"); + throw new TestFrameworkException("Unable to create AWS Secrets Manager client"); } } } From bb62cfe317f88c7e18df7039d72c2bbbb78b1bf9 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 15 Jan 2020 10:07:44 -0600 Subject: [PATCH 4/4] MQE-1918: MFTF AWS Secrets Manager - Local Use --- .../DataGenerator/Handlers/CredentialStore.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 76560bcf1..84d58ade8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -18,9 +18,8 @@ class CredentialStore const ARRAY_KEY_FOR_FILE = 'file'; const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; - const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' - . '.credentials file, HashiCorp Vault and AWS Secrets Manager. ' - . 'You need to configure at least one to use _CREDS in tests.'; + const CREDENTIAL_STORAGE_INFO = 'You need to configure at least one of these options: ' + . '.credentials file, HashiCorp Vault or AWS Secrets Manager correctly'; /** * Credential storage array @@ -95,7 +94,7 @@ private function __construct() if (empty($this->credStorage)) { throw new TestFrameworkException( - 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO . '.' ); } } @@ -119,7 +118,8 @@ public function getSecret($key) } throw new TestFrameworkException( - "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO . ' And make sure key/value exists.' + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO + . ' and ensure key, value exists to use _CREDS in tests.' ); }