Make intrinsic functions configurable (#4129)

* Make intrinsic functions configurable
* Add jsonschema keyword to cfnContext to flip context information
* Add jsonschema keyword to dynamicValidation to validate values against dynamic information

Author: kddejong

docs/cfn-schema-specification.md

@@ -214,3 +214,80 @@ is equivalent to the JSON schema
   }
 }
 ```
+
+### CloudFormation Context-Aware Validation
+
+To support CloudFormation's unique validation requirements, cfn-lint extends JSON Schema with context-aware validation capabilities.
+
+#### cfnContext
+
+_cfnContext_ provides a way to specify which CloudFormation intrinsic functions are allowed in a specific context and define the schema for validating the value.
+
+```json
+{
+  "cfnContext": {
+    "functions": ["Ref", "Fn::GetAtt"],
+    "schema": {
+      "type": "string"
+    }
+  }
+}
+```
+
+The `functions` array specifies which intrinsic functions are allowed in this context. The `schema` object defines the validation rules for the value.
+
+For example, to specify that only `Ref` is allowed in a parameter reference:
+
+```json
+{
+  "cfnContext": {
+    "functions": ["Ref"],
+    "schema": {
+      "type": "string"
+    }
+  }
+}
+```
+
+#### dynamicValidation
+
+_dynamicValidation_ enables validation against dynamic sources from the template context, such as parameter names, condition names, or resource IDs.
+
+```json
+{
+  "dynamicValidation": {
+    "context": "conditions"
+  }
+}
+```
+
+This validates that the value exists in the specified context. Available contexts include:
+- `conditions`: Condition names defined in the template
+- `mappings`: Mapping names defined in the template
+- `refs`: CloudFormation valid refs
+
+_dynamicValidation_ can also check if a specific transform is present in the template:
+
+```json
+{
+  "dynamicValidation": {
+    "transformCheck": "AWS::LanguageExtensions"
+  }
+}
+```
+
+This will validate that the specified transform is included in the template.
+
+_dynamicValidation_ can also validate based on the current path in the template:
+
+```json
+{
+  "dynamicValidation": {
+    "pathCheck": "Resources/MyResource/Properties"
+  }
+}
+```
+
+This checks if the current path in the template matches the specified pattern.
+
+These context-aware validation features allow for more precise validation of CloudFormation templates, ensuring that references are valid and that template elements are used in the appropriate contexts.

scripts/update_schemas_from_aws_api.py

@@ -64,7 +64,7 @@ def configure_logging():
 
 def write_output(resource, filename, obj):
     filename = f"src/cfnlint/data/schemas/extensions/{resource}/{filename}.json"
-    obj["_description"] = ("Automatically updated using aws api",)
+    obj["description"] = "Automatically updated using aws api"
 
     with open(filename, "w+", encoding="utf-8") as f:
         json.dump(obj, f, indent=1, sort_keys=True, separators=(",", ": "))

scripts/update_specs_from_pricing.py

@@ -237,14 +237,12 @@ def get_rds_pricing():
     specs = {}
     cluster_specs = {}
     for product_region, product_values in rds_details.items():
-        if product_region not in specs:
-            specs[product_region] = {"allOf": []}
-        if product_region not in cluster_specs:
-            cluster_specs[product_region] = {"allOf": []}
         for deployment_option, deployment_values in product_values.items():
             if deployment_option in ["Single-AZ", "Multi-AZ"]:
                 for license_name, license_values in deployment_values.items():
                     for product_name, instance_types in license_values.items():
+                        if product_region not in specs:
+                            specs[product_region] = {"allOf": []}
                         if license_name == "general-public-license":
                             specs[product_region]["allOf"].append(
                                 {
@@ -297,6 +295,8 @@ def get_rds_pricing():
             else:
                 for license_name, license_values in deployment_values.items():
                     for product_name, instance_types in license_values.items():
+                        if product_region not in cluster_specs:
+                            cluster_specs[product_region] = {"allOf": []}
                         cluster_specs[product_region]["allOf"].append(
                             {
                                 "if": {
@@ -364,7 +364,7 @@ def get_results(service, product_families, default=None):
 def write_output(resource, filename, obj):
     filename = f"src/cfnlint/data/schemas/extensions/{resource}/{filename}.json"
     output = {
-        "_description": "Automatically updated using update_specs_from_pricing",
+        "description": "Automatically updated using update_specs_from_pricing",
     }
     for region, values in obj.items():
         output[region] = {"enum": sorted(list(values))}

src/cfnlint/config.py

@@ -515,6 +515,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         advanced.add_argument(
             "-s",
             "--registry-schemas",
+            default=[],
             help="one or more directories of CloudFormation Registry Schemas",
             action="extend",
             type=comma_separated_arg,
@@ -616,6 +617,35 @@ class ManualArgs(TypedDict, total=False):
     non_zero_exit_code: str
     output_file: str
     regions: list
+    registry_schemas: list[str]
+
+
+def _merge_configs(
+    cli_value: Any, template_value: Any, file_value: Any, manual_value: Any
+) -> Any:
+    # the CLI will always have an empty list when the item is a list
+    # we will use that to evaluate if we need to merge the lists
+    if isinstance(cli_value, list):
+        merged_list = cli_value.copy()
+        if isinstance(template_value, list):
+            merged_list.extend(template_value)
+        if isinstance(file_value, list):
+            merged_list.extend(file_value)
+        if isinstance(manual_value, list):
+            merged_list.extend(manual_value)
+        return merged_list
+
+    elif isinstance(cli_value, dict):
+        merged_dict = cli_value.copy()
+        if isinstance(template_value, dict):
+            merged_dict.update(template_value)
+        if isinstance(file_value, dict):
+            merged_dict.update(file_value)
+        if isinstance(manual_value, dict):
+            merged_dict.update(manual_value)
+        return merged_dict
+
+    return None
 
 
 # pylint: disable=too-many-public-methods
@@ -639,6 +669,7 @@ def __repr__(self):
                 "mandatory_checks": self.mandatory_checks,
                 "include_experimental": self.include_experimental,
                 "configure_rules": self.configure_rules,
+                "registry_schemas": self.registry_schemas,
                 "regions": self.regions,
                 "ignore_bad_template": self.ignore_bad_template,
                 "debug": self.debug,
@@ -651,34 +682,28 @@ def __repr__(self):
                 "config_file": self.config_file,
                 "merge_configs": self.merge_configs,
                 "non_zero_exit_code": self.non_zero_exit_code,
+                "patch_specs": self.patch_specs,
             }
         )
 
     def _get_argument_value(self, arg_name, is_template, is_config_file):
         cli_value = getattr(self.cli_args, arg_name)
         template_value = self.template_args.get(arg_name)
         file_value = self.file_args.get(arg_name)
+        manual_value = self._manual_args.get(arg_name)
 
         # merge list configurations
         # make sure we don't do an infinite loop so skip this check for merge_configs
         if arg_name != "merge_configs":
             if self.merge_configs:
-                # the CLI will always have an empty list when the item is a list
-                # we will use that to evaluate if we need to merge the lists
-                if isinstance(cli_value, list):
-                    # Use a copy here, otherwise we will
-                    # accumulate template level config
-                    # into the cli_value which will persist between template files
-                    result = cli_value.copy()
-                    if isinstance(template_value, list):
-                        result.extend(template_value)
-                    if isinstance(file_value, list):
-                        result.extend(file_value)
-                    return result
+                if isinstance(cli_value, (list, dict)):
+                    return _merge_configs(
+                        cli_value, template_value, file_value, manual_value
+                    )
 
         # return individual items
-        if arg_name in self._manual_args:
-            return self._manual_args[arg_name]
+        if manual_value:
+            return manual_value
         if cli_value:
             return cli_value
         if template_value and is_template:

src/cfnlint/context/context.py

@@ -33,16 +33,16 @@
 @dataclass
 class Transforms:
     # Template level parameters
-    transforms: InitVar[str | list[str] | None]
+    obj: InitVar[str | list[str] | None]
     _transforms: list[str] = field(init=False, default_factory=list)
 
-    def __post_init__(self, transforms) -> None:
-        if transforms is None:
+    def __post_init__(self, obj) -> None:
+        if obj is None:
             return
-        if not isinstance(transforms, list):
-            transforms = [transforms]
+        if not isinstance(obj, list):
+            obj = [obj]
 
-        for transform in transforms:
+        for transform in obj:
             if not isinstance(transform, str):
                 continue
             self._transforms.append(transform)
@@ -52,6 +52,10 @@ def __post_init__(self, transforms) -> None:
             self.has_language_extensions_transform
         )
 
+    @property
+    def transforms(self):
+        return self._transforms
+
     def has_language_extensions_transform(self):
         return bool(TRANSFORM_LANGUAGE_EXTENSION in self._transforms)
 

src/cfnlint/data/schemas/extensions/aws_amazonmq_broker/instancetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "mq.m5.2xlarge",
@@ -177,6 +176,7 @@
    "mq.t3.micro"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "mq.m4.large",

src/cfnlint/data/schemas/extensions/aws_appstream_fleet/instancetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "ap-northeast-1": {
   "enum": [
    "stream.compute.2xlarge",
@@ -270,6 +269,7 @@
    "stream.standard.xlarge"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "stream.compute.2xlarge",

src/cfnlint/data/schemas/extensions/aws_dax_cluster/nodetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "ap-northeast-1": {
   "enum": [
    "dax.r3.2xlarge",
@@ -146,6 +145,7 @@
    "dax.t3.small"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "dax.r4.16xlarge",

src/cfnlint/data/schemas/extensions/aws_docdb_dbinstance/dbinstanceclass_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "db.r4.16xlarge",
@@ -363,6 +362,7 @@
    "db.t4g.medium"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "db.r4.16xlarge",

src/cfnlint/data/schemas/extensions/aws_ec2_instance/instancetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "a1.2xlarge",
@@ -15929,6 +15928,7 @@
    "z1d.xlarge"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "a1.2xlarge",

src/cfnlint/data/schemas/extensions/aws_elasticache_cachecluster/cachenodetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "cache.c1.xlarge",
@@ -1615,6 +1614,7 @@
    "cache.t4g.small"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "cache.c1.xlarge",

src/cfnlint/data/schemas/extensions/aws_elasticache_cachecluster/engine_version.json

@@ -1,7 +1,4 @@
 {
- "_description": [
-  "Automatically updated using aws api"
- ],
  "allOf": [
   {
    "if": {
@@ -126,5 +123,6 @@
     }
    }
   }
- ]
+ ],
+ "description": "Automatically updated using aws api"
 }

src/cfnlint/data/schemas/extensions/aws_elasticsearch_domain/elasticsearchclusterconfig_instancetype_enum.json

@@ -1,3 +1,3 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing"
+ "description": "Automatically updated using update_specs_from_pricing"
 }

src/cfnlint/data/schemas/extensions/aws_emr_cluster/instancetypeconfig_instancetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "c1.medium",
@@ -10591,6 +10590,7 @@
    "z1d.xlarge"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "c1.medium",

src/cfnlint/data/schemas/extensions/aws_gamelift_fleet/ec2instancetype_enum.json

@@ -1,5 +1,4 @@
 {
- "_description": "Automatically updated using update_specs_from_pricing",
  "af-south-1": {
   "enum": [
    "c3.2xlarge",
@@ -6060,6 +6059,7 @@
    "r8g.xlarge"
   ]
  },
+ "description": "Automatically updated using update_specs_from_pricing",
  "eu-central-1": {
   "enum": [
    "c3.2xlarge",