diff --git a/composer.json b/composer.json index 158287fbf..ca1c02572 100755 --- a/composer.json +++ b/composer.json @@ -11,7 +11,10 @@ "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*", "allure-framework/allure-codeception": "~1.3.0", + "aws/aws-sdk-php": "^3.132", "codeception/codeception": "~2.4.5", "composer/composer": "^1.4", "consolidation/robo": "^1.0.0", diff --git a/composer.lock b/composer.lock index 8f2fcb8e9..6bfc30ab9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59e95cc1ae6311e93111bd7ced180d29", + "content-hash": "2325a3a38edb33b24f6e33bd3009fd8a", "packages": [ { "name": "allure-framework/allure-codeception", @@ -109,6 +109,90 @@ ], "time": "2016-12-07T12:15:46+00:00" }, + { + "name": "aws/aws-sdk-php", + "version": "3.132.2", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f890689c6db27625522ea2e7e9b8420b6fccb063", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^5.3.3|^6.2.1|^7.0", + "guzzlehttp/promises": "^1.0", + "guzzlehttp/psr7": "^1.4.1", + "mtdowling/jmespath.php": "^2.5", + "php": ">=5.5" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-pcntl": "*", + "ext-sockets": "*", + "nette/neon": "^2.3", + "phpunit/phpunit": "^4.8.35|^5.4.3", + "psr/cache": "^1.0", + "psr/simple-cache": "^1.0", + "sebastian/comparator": "^1.2.3" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Aws\\": "src/" + }, + "files": [ + "src/functions.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "http://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "http://aws.amazon.com/sdkforphp", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "time": "2020-01-09T19:09:31+00:00" + }, { "name": "behat/gherkin", "version": "v4.4.5", @@ -2542,8 +2626,66 @@ "bcmath", "math" ], + "abandoned": "brick/math", "time": "2017-02-16T16:54:46+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.5.0", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "52168cb9472de06979613d365c7f1ab8798be895" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/52168cb9472de06979613d365c7f1ab8798be895", + "reference": "52168cb9472de06979613d365c7f1ab8798be895", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "symfony/polyfill-mbstring": "^1.4" + }, + "require-dev": { + "composer/xdebug-handler": "^1.2", + "phpunit/phpunit": "^4.8.36|^7.5.15" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.5-dev" + } + }, + "autoload": { + "psr-4": { + "JmesPath\\": "src/" + }, + "files": [ + "src/JmesPath.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "time": "2019-12-30T18:03:34+00:00" + }, { "name": "mustache/mustache", "version": "v2.12.0", @@ -6542,7 +6684,9 @@ "prefer-lowest": false, "platform": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "ext-curl": "*" + "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*" }, "platform-dev": [] } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php new file mode 100644 index 000000000..e1f4e4879 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php @@ -0,0 +1,65 @@ + 'mftf/magento/' . $testShortKey, + 'SecretString' => json_encode([$testShortKey => $testValue]) + ]; + /** @var Result */ + $result = new Result($data); + + $mockClient = $this->getMockBuilder(SecretsManagerClient::class) + ->disableOriginalConstructor() + ->setMethods(['__call']) + ->getMock(); + + $mockClient->expects($this->once()) + ->method('__call') + ->willReturnCallback(function ($name, $args) use ($result) { + return $result; + }); + + /** @var SecretsManagerClient */ + $credentialStorage = new AwsSecretsManagerStorage($testRegion, $testProfile); + $reflection = new ReflectionClass($credentialStorage); + $reflection_property = $reflection->getProperty('client'); + $reflection_property->setAccessible(true); + $reflection_property->setValue($credentialStorage, $mockClient); + + // Test getEncryptedValue() + $encryptedCred = $credentialStorage->getEncryptedValue($testLongKey); + + // Assert the value we've gotten is in fact not identical to our test value + $this->assertNotEquals($testValue, $encryptedCred); + + // Test getDecryptedValue() + $actualValue = $credentialStorage->getDecryptedValue($encryptedCred); + + // Assert that we are able to successfully decrypt our secret value + $this->assertEquals($testValue, $actualValue); + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 5140c01e7..9466f2bcc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,28 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` +### CREDENTIAL_AWS_SECRETS_MANAGER_REGION + +The region that AWS Secrets Manager is located. + +Example: + +```conf +# Region of AWS Secrets Manager +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +``` + +### CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE + +The profile used to connect to AWS Secrets Manager. + +Example: + +```conf +# Profile used to connect to AWS Secrets Manager. +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +``` + ### ENABLE_BROWSER_LOG Enables addition of browser logs to Allure steps diff --git a/docs/credentials.md b/docs/credentials.md index a2850cfe8..4b20e68dc 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -3,10 +3,11 @@ When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. -Currently the MFTF supports two types of credential storage: +Currently the MFTF supports three types of credential storage: - **.credentials file** -- **HashiCorp vault** +- **HashiCorp Vault** +- **AWS Secrets Manager** ## Configure File Storage @@ -135,11 +136,64 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure both File Storage and Vault Storage +## Configure AWS Secrets Manager -It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. -In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. +AWS Secrets Manager offers secret management that supports: +- Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB +- Fine-grained policies and permissions +- Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises +### Prerequisites +- AWS account +- AWS Secrets Manger is created and configured +- IAM User or Role is created with appropriate AWS Secrets Manger access permission + +### Store secrets in AWS Secrets Manager + +#### Secrets format +`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an AWS Secret. +`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: + +```conf +mftf// +``` + +```conf +# Secret name for carriers_usps_userid +mftf/magento/carriers_usps_userid + +# Secret key for carriers_usps_userid +carriers_usps_userid + +# Secret name for carriers_usps_password +mftf/magento/carriers_usps_password + +# Secret key for carriers_usps_password +carriers_usps_password +``` + +### Setup MFTF to use AWS Secrets Manager + +To use AWS Secrets Manager, the AWS region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`][] in `.env`. + +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to AWS Secrets Manager service. +You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. +Optionally, however, you can explicitly set AWS profile through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`][] in `.env`. + +```conf +# Sample AWS Secrets Manager configuration +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +``` + +## Configure multiple credential storage + +It is possible and sometimes useful to setup and use multiple credential storage at the same time. +In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: + +``` +.credentials File > HashiCorp Vault > AWS Secrets Manager +``` ## Use credentials in a test @@ -183,3 +237,6 @@ The MFTF tests delivered with Magento application do not use credentials and do [Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path +[credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html +[`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile +[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 7320d8b8b..f5b6ef40e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,10 +30,14 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#*** To use HashiCorp Vault to manage _CREDS secrets, uncomment and set vault address and secret base path ***# #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +#*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 94ff40069..84d58ade8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,12 +9,17 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; + const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; + + const CREDENTIAL_STORAGE_INFO = 'You need to configure at least one of these options: ' + . '.credentials file, HashiCorp Vault or AWS Secrets Manager correctly'; /** * Credential storage array @@ -71,9 +76,25 @@ private function __construct() } } + // Initialize AWS Secrets Manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); + if ($awsRegion !== false) { + if ($awsProfile === false) { + $awsProfile = null; + } + try { + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( + $awsRegion, + $awsProfile + ); + } catch (TestFrameworkException $e) { + } + } + if (empty($this->credStorage)) { throw new TestFrameworkException( - "No credential storage is properly configured. Please configure vault or .credentials file." + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO . '.' ); } } @@ -87,8 +108,8 @@ private function __construct() */ public function getSecret($key) { - // Get secret data from storage according to the order they are stored - // File storage is preferred over vault storage to allow local secret value overriding remote secret value + // Get secret data from storage according to the order they are stored which follows this precedence: + // FileStorage > VaultStorage > AwsSecretsManagerStorage foreach ($this->credStorage as $storage) { $value = $storage->getEncryptedValue($key); if (null !== $value) { @@ -97,8 +118,8 @@ public function getSecret($key) } throw new TestFrameworkException( - "\"{$key}\" not defined in vault or .credentials file, " - . "please provide a value in order to use this secret in a test." + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO + . ' and ensure key, value exists to use _CREDS in tests.' ); } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php new file mode 100644 index 000000000..e7a858254 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -0,0 +1,156 @@ +createAwsSecretsManagerClient($region, $profile); + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + * @throws Exception + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Retrieving value for key name {$key} from AWS Secrets Manager" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret id + list($vendor, $key) = explode('/', trim($key, '/'), 2); + $secretId = self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by id from AWS Secrets Manager, and parse the result + $value = $this->parseAwsSecretResult( + $this->client->getSecretValue(['SecretId' => $secretId]), + $key + ); + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (AwsException $e) { + $error = $e->getAwsErrorCode(); + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "AWS error code: {$error}. Unable to read value for key {$key} from AWS Secrets Manager" + ); + } + } catch (\Exception $e) { + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Unable to read value for key {$key} from AWS Secrets Manager" + ); + } + } + return $reValue; + } + + /** + * Parse AWS result object and return secret for key + * + * @param Result $awsResult + * @param string $key + * @return string + * @throws TestFrameworkException + */ + private function parseAwsSecretResult($awsResult, $key) + { + // Return secret from the associated KMS CMK + if (isset($awsResult['SecretString'])) { + $rawSecret = $awsResult['SecretString']; + } else { + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); + } + $secret = json_decode($rawSecret, true); + if (isset($secret[$key])) { + return $secret[$key]; + } + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); + } + + /** + * Create Aws Secrets Manager client + * + * @param string $region + * @param string $profile + * @return void + * @throws TestFrameworkException + * @throws InvalidArgumentException + */ + private function createAwsSecretsManagerClient($region, $profile) + { + if (null !== $this->client) { + return; + } + + // Create AWS Secrets Manager client + $this->client = new SecretsManagerClient([ + 'profile' => $profile, + 'region' => $region, + 'version' => self::LATEST_VERSION + ]); + + if ($this->client === null) { + throw new TestFrameworkException("Unable to create AWS Secrets Manager client"); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 6a9e9f0cf..798ac660c 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -67,7 +67,7 @@ class VaultStorage extends BaseStorage private $secretBasePath; /** - * CredentialVault constructor + * VaultStorage constructor * * @param string $baseUrl * @param string $secretBasePath