chore: performance tests for ESDK-python

.gitignore

@@ -33,6 +33,12 @@ __pycache__
 .pytest_cache
 # Ignore key materials generated by examples or tests
 test_keyrings/
+# Ignore results of performance test
+performance_tests/results/*.csv
+performance_tests/results/*.pstats
+performance_tests/results/*.png
+# Ignore the memory profile logs
+mprofile_*
 
 # PyCharm
 .idea/

buildspec.yml

@@ -164,6 +164,10 @@ batch:
       buildspec: codebuild/py311/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py311_performance_tests_mpl
+      buildspec: codebuild/py311/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py311_examples
       buildspec: codebuild/py311/examples.yml
       env:
@@ -250,6 +254,10 @@ batch:
       buildspec: codebuild/py312/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py312_performance_tests_mpl
+      buildspec: codebuild/py312/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py312_examples
       buildspec: codebuild/py312/examples.yml
       env:

codebuild/py311/performance_tests_mpl.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.11.0
+      - pyenv local 3.11.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py311-performance_tests-mpl

codebuild/py312/performance_tests_mpl.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.12
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.12.0
+      - pyenv local 3.12.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py312-performance_tests-mpl

performance_tests/README.rst

@@ -0,0 +1,219 @@
+#####################################
+aws-encryption-sdk performance tests
+#####################################
+
+This module runs performance tests for the `AWS Encryption SDK Python`_.
+
+********
+Overview
+********
+
+This module tests the following keyrings / master key providers:
+
+1. KMS Keyring / KMS Master Key Provider
+2. Raw AES Keyring / AES Master Key Provider
+3. Raw RSA Keyring / RSA Master Key Provider
+4. Hierarchy Keyring
+5. Caching CMM
+
+For each test on the above keyrings / master key providers, this package measures:
+
+1. Execution time
+2. Total memory consumption
+
+For each keyring / master key provider, the execution time and memory consumption
+is measured for three operations:
+
+1. Create keyring / master key provider
+2. Encrypt
+3. Decrypt
+
+The usage of the performance tests is demonstrated through an `AWS KMS Keyring`_.
+However, the procedure is the same for any keyring / master key provider, with slight
+changes in the input arguments.
+
+The results for the performance test will be available in the results folder in the
+performance_tests directory.
+
+**********************
+Required Prerequisites
+**********************
+
+* Python 3.8+
+* aws-encryption-sdk
+* boto3 >= 1.10.0
+* click
+* tqdm
+* pytest
+
+Recommended Prerequisites
+=========================
+
+* aws-cryptographic-material-providers: >= 1.0.0
+  * Requires Python 3.11+.
+
+*****
+Usage
+*****
+
+Execution Time
+==============
+
+Create Keyring
+--------------
+To run the performance test for execution time, please use the
+following commands in the performance_tests directory.
+
+.. code::
+
+    usage: python test/keyrings/test_aws_kms_keyring.py create
+
+    Create a keyring to use for encryption and decryption.
+
+    optional arguments:
+      -h, --help                    show this help message and exit.
+      --kms_key_id KMS_KEY_ID       The KMS key ID you want to use.
+      --n_iters N_ITERS             Number of iterations you want to
+                                    run the test for. For instance,
+                                    if n_iters = 100, this performance
+                                    test script will run the create_keyring
+                                    method 100 times and report the
+                                    execution time of each of the calls.
+      --output_file OUTPUT_FILE     The output file for execution times
+                                    for each function call,
+                                    default='kms_keyring_create' in the
+                                    results folder.
+
+Encrypt
+-------
+
+To run the performance test for execution time, please use the following
+commands in the performance_tests directory:
+
+.. code::
+
+    usage: python test/keyrings/test_aws_kms_keyring.py encrypt
+
+    optional arguments:
+      -h, --help                                            show this help message and exit.
+      --plaintext_data_filename PLAINTEXT_DATA_FILENAME     Filename containing plaintext data
+                                                            you want to encrypt.
+                                                            default='test/resources/plaintext/plaintext-data-medium.dat'.
+                                                            You can choose to use any other plaintext
+                                                            file as well. Some example plaintext
+                                                            data files are present in the
+                                                            'test/resources' directory.
+      --kms_key_id KMS_KEY_ID                               The KMS key ID you want to use.
+      --n_iters N_ITERS                                     Number of iterations you want to
+                                                            run the test for. For instance,
+                                                            if n_iters = 100, this performance
+                                                            test script will run the create_keyring
+                                                            method 100 times and report the
+                                                            execution time of each of the calls.
+      --output_file OUTPUT_FILE                             The output file for execution times
+                                                            for each function call,
+                                                            default='kms_keyring_create' in the
+                                                            results folder.
+
+Decrypt
+-------
+
+To run the performance test for execution time, please use the
+following commands in the performance_tests directory
+
+.. code::
+
+    usage: python test/keyrings/test_aws_kms_keyring.py decrypt
+
+    optional arguments:
+      -h, --help                                            show this help message and exit.
+      --ciphertext_data_filename CIPHERTEXT_DATA_FILENAME   Filename containing ciphertext data
+                                                            you want to decrypt.
+                                                            default='test/resources/ciphertext/kms/ciphertext-data-medium.ct'.
+                                                            You can choose to use any other
+                                                            ciphertext file as well. Some example
+                                                            ciphertext data files are present in
+                                                            the 'test/resources' directory.
+      --kms_key_id KMS_KEY_ID                               The KMS key ID you want to use.
+      --n_iters N_ITERS                                     Number of iterations you want to
+                                                            run the test for. For instance,
+                                                            if n_iters = 100, this performance
+                                                            test script will run the create_keyring
+                                                            method 100 times and report the
+                                                            execution time of each of the calls.
+      --output_file OUTPUT_FILE                             The output file for execution times
+                                                            for each function call,
+                                                            default='kms_keyring_create' in the
+                                                            results folder.
+
+Consolidate Time Results
+========================
+
+In order to find the minimum, maximum, average, 99th percentile and bottom
+99th percentile trimmed average times from the n_iters runs, please use the
+following script from the performance_tests directory with the csv file
+containing times for each of the n_iters runs generated in the previous
+"Execution Time" section:
+
+.. code::
+
+    usage: python consolidate_results.py results/kms_keyring_decrypt.csv
+
+Memory Consumption
+==================
+
+To get the memory consumption, simply replace 'python'
+with 'mprof run' in the previously mentioned commands.
+
+For example, if you want to calculate the memory consumption
+of the encrypt function of a AWS KMS Keyring, simply write:
+
+.. code::
+
+    usage: mprof run test/keyrings/test_aws_kms_keyring.py encrypt
+
+
+This should generate an mprofile log file in your current directory.
+This mprofile log file contains the total memory consumed by the program
+with respect to time elapsed.
+To plot the memory consumption with respect to time, please use the following
+command from the same directory
+
+.. code::
+
+    usage: mprof plot
+
+
+This 'mprof plot' command will plot the most recent mprofile log file.
+
+
+Performance Graph
+=================
+
+To generate a performance graph, please use the following command
+to generate the pstats log file by specifying the output pstats file
+path. Here, 'results/kms_keyring_create.pstats' is set as the default
+output file.
+
+.. code::
+
+    usage: python -m cProfile -o results/kms_keyring_create.pstats test/keyrings/test_aws_kms_keyring.py create
+
+
+After generating the pstats file, please run the following command
+to generate the performance graph. The output performance graph will
+be a .png file that you specify. Here, 'results/kms_keyring_create.png'
+is set as the default output file.
+
+.. code::
+
+    usage: gprof2dot -f pstats results/kms_keyring_create.pstats | dot -Tpng -o results/kms_keyring_create.png && eog results/kms_keyring_create.png
+
+
+Note: This project does not adhere to semantic versioning; as such it
+makes no guarantees that functionality will persist across major,
+minor, or patch versions.
+**DO NOT** take a standalone dependency on this library.
+
+.. _AWS Encryption SDK Python: https://github.com/aws/aws-encryption-sdk-python/
+.. _AWS KMS Keyring: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html

performance_tests/__init__.py

@@ -0,0 +1,3 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Stub module indicator to make linter configuration simpler."""

performance_tests/consolidate_results.py

@@ -0,0 +1,49 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Script for consolidating results for execution times"""
+
+import argparse
+import csv
+
+import numpy as np
+
+
+def calculate_statistics(_csv_file):
+    """Calculate average, trimmed average, minimum, maximum and p99 statistics for execution times in a CSV file."""
+    with open(_csv_file, 'r', encoding='utf-8') as file:
+        reader = csv.reader(file)
+        data = [float(row[0]) for row in reader]
+
+    output_stats = {}
+
+    # Calculate statistics
+    if data:
+        data = np.sort(data)
+        output_stats['total_entries'] = len(data)
+        output_stats['average'] = np.mean(data)
+        output_stats['trimmed_average_99_bottom'] = np.mean(data[0:int(0.99 * len(data))])
+        output_stats['minimum'] = min(data)
+        output_stats['maximum'] = max(data)
+        output_stats['perc_99'] = np.percentile(data, 99)
+        return output_stats
+
+    return None
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('csv_file',
+                        help='csv file containing the outputs of execution times for n_iter iterations')
+    args = parser.parse_args()
+
+    statistics = calculate_statistics(args.csv_file)
+    if statistics:
+        print("CSV File:", args.csv_file)
+        print("Total Entries:", statistics['total_entries'])
+        print("Average:", statistics['average'])
+        print("Bottom 99th percentile trimmed average:", statistics['trimmed_average_99_bottom'])
+        print("Minimum:", statistics['minimum'])
+        print("Maximum:", statistics['maximum'])
+        print("99th percentile:", statistics['perc_99'])
+    else:
+        print("No data found in the CSV file.")