From dd8831b6847ba092945807c9326cce8c54133494 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Sep 2024 16:59:01 -0500 Subject: [PATCH 1/6] feat: codify the infra --- infra/.gitignore | 15 ++ infra/.terraform.lock.hcl | 24 +++ infra/Makefile | 12 ++ infra/cdn/README.md | 49 ++++++ infra/cdn/certs/psf.io.pem | 25 +++ infra/cdn/main.tf | 346 +++++++++++++++++++++++++++++++++++++ infra/cdn/providers.tf | 4 + infra/cdn/variables.tf | 39 +++++ infra/cdn/versions.tf | 8 + infra/config.tf | 9 + infra/main.tf | 30 ++++ infra/variables.tf | 20 +++ 12 files changed, 581 insertions(+) create mode 100644 infra/.gitignore create mode 100644 infra/.terraform.lock.hcl create mode 100644 infra/Makefile create mode 100644 infra/cdn/README.md create mode 100644 infra/cdn/certs/psf.io.pem create mode 100644 infra/cdn/main.tf create mode 100644 infra/cdn/providers.tf create mode 100644 infra/cdn/variables.tf create mode 100644 infra/cdn/versions.tf create mode 100644 infra/config.tf create mode 100644 infra/main.tf create mode 100644 infra/variables.tf diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 000000000..e61ff928d --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,15 @@ + +**/.terraform/* +*.tfstate +*.tfstate.* +crash.log +crash.*.log +*.tfvars +*.tfvars.json +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraform.tfstate.lock.info +.terraformrc +terraform.rc \ No newline at end of file diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 000000000..552e40475 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/fastly/fastly" { + version = "5.13.0" + constraints = "5.13.0" + hashes = [ + "h1:op/7hntTRkfZFIZ5xLNtLb7eBY155ywQIVSy56XCmBE=", + "zh:04f7405ee22a8ace546b90cc3a08d81f1a49dae8b1050500398d4b0244dcbc86", + "zh:0e0c48aca34a1fc7ed7382c8e85b5da770f63f3c9aa79bc2c3c55ed570f9d0ab", + "zh:302d2b9872ab8ffee2082291cc2cfec487633e22c7970b2c9d22268d6b5f7624", + "zh:346ea021dbe2c7128cddc2c9e01a95242b8bceeda20d9d1b00ae09ee90e3962b", + "zh:41fbe18f63154a6a1a46e1b1cc909bfe90f5bba7f5cfab0a80d15be7eceec4c3", + "zh:524c2a54282a92d0d7633bfd511427f6d9aa6b6a52b7d9f71cf5206dedab381e", + "zh:721fe08bfb1b85f8946aeba3bdb7e0de3d74fce94c8657d0086b153f58558d89", + "zh:9c627b3170a5505c73455e6c2a99d2ce4187e225130e12aececdc808357f8b66", + "zh:a61a62cec9612358b08ef1895277a37d4d4ec134972991fa414255ef95683dba", + "zh:bde1a51553c15d333140c2b77481ee668c4af8de93a968d869c02a736db460c4", + "zh:c2683862bd0e9633d3800503a71b3aab51ec8e3aac3f6ef6b71831efe81a2afd", + "zh:dff5ad3766432550974d2f0c24535c572fee5eeb0dce7befeaa97cb6ca3d8443", + "zh:ec3c56fc43344a07b0eef5158df6dd50e68bdcee1b03299bb2acd502d11582d5", + "zh:ec8d899cafd925d3492f00c6523c90599aebc43c1373ad4bd6c55f12d2376230", + ] +} diff --git a/infra/Makefile b/infra/Makefile new file mode 100644 index 000000000..ae0f85b41 --- /dev/null +++ b/infra/Makefile @@ -0,0 +1,12 @@ +.PHONY: fmt +fmt: + @terraform fmt ./**/*.tf + +.PHONY: check +check: + @terraform validate + +.PHONY: yolo +yolo: + @echo "Wise, you are not..." + @terraform apply -auto-approve \ No newline at end of file diff --git a/infra/cdn/README.md b/infra/cdn/README.md new file mode 100644 index 000000000..51b5bbfaa --- /dev/null +++ b/infra/cdn/README.md @@ -0,0 +1,49 @@ +# Fastly CDN Config + +This module creates Fastly services for the Python.org CDN. + +## Usage +# Fastly VCL Terraform Module + +This Terraform module configures a Fastly service using VCL (Varnish Configuration Language) for Python.org. It sets up a robust CDN configuration with various features to optimize performance, security, and logging. + +## Features + +- Configures Fastly VCL service for Python.org +- Supports multiple domains (primary and extra domains) +- Sets up backends with health checks +- Implements caching strategies and TTL configurations +- Configures HTTPS and HSTS +- Implements rate limiting +- Sets up logging to Datadog and S3 +- Configures various headers and request/response manipulations +- Implements IP blocking capabilities + +## Usage + +```hcl +module "fastly_production" { + source = "./cdn" + + name = "CoolPythonApp.org" + domain = "CoolPythonApp.org" + extra_domains = ["www.CoolPythonApp.org"] + backend_address = "service.CoolPythonApp.org" + default_ttl = 3600 + + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + fastly_s3_logging = var.fastly_s3_logging +} +``` + +## Outputs + +N/A + +## Requirements + +Tested on +- Tested on Terraform 1.8.5 +- Fastly provider 5.13.0 \ No newline at end of file diff --git a/infra/cdn/certs/psf.io.pem b/infra/cdn/certs/psf.io.pem new file mode 100644 index 000000000..7952bb36b --- /dev/null +++ b/infra/cdn/certs/psf.io.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEQzCCAyugAwIBAgIUYH38nEb2KLRgscKhjcNpBLRUz+UwDQYJKoZIhvcNAQEL +BQAwgbAxCzAJBgNVBAYTAlVTMQ8wDQYDVQQIDAZPcmVnb24xEjAQBgNVBAcMCUJl +YXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xHDAa +BgNVBAsME0luZnJhc3RydWN0dXJlIFRlYW0xDzANBgNVBAMMBlBTRl9DQTEoMCYG +CSqGSIb3DQEJARYZaW5mcmFzdHJ1Y3R1cmVAcHl0aG9uLm9yZzAeFw0yNDAyMTIx +NzU0MDZaFw0yOTAyMTAxNzU0MDZaMIGwMQswCQYDVQQGEwJVUzEPMA0GA1UECAwG +T3JlZ29uMRIwEAYDVQQHDAlCZWF2ZXJ0b24xIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 +d2FyZSBGb3VuZGF0aW9uMRwwGgYDVQQLDBNJbmZyYXN0cnVjdHVyZSBUZWFtMQ8w +DQYDVQQDDAZQU0ZfQ0ExKDAmBgkqhkiG9w0BCQEWGWluZnJhc3RydWN0dXJlQHB5 +dGhvbi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCXAZagv2UK +AEnnnnrK/WWcZIKo/l+HTgL01XhReu9CDNs3f3ESlRT3Y4Hbla/pYRu9VM8tMGYS +xG5FGJQ2JPVnKCb3mIEC7wy9+VOaQIp3l8+o0lDQhsOZs78ZA8XQpNLD5OURsUHJ +re1U6WOTryMJwxpO+DzSBU+oSwfdn2k0BAJqSeIU45hHXeHO24z7GePuk3I1wb+E +vfhtdIF/tHvF1I6h7ntmHUeUWYrTKXKB9meMAFwEC1ZNoN1z05X68cSeK8dAsxYh +ghmQnUZ4hHH8pLlhYW/QBTol0nutwgHPyC9FIJnZzX50xAMRx3TKP1IbIehWBwF2 +CYJq6pRBZ1mfAgMBAAGjUzBRMB0GA1UdDgQWBBQrAQVRNWd6eVr6ZGn8vshzgS09 +qDAfBgNVHSMEGDAWgBQrAQVRNWd6eVr6ZGn8vshzgS09qDAPBgNVHRMBAf8EBTAD +AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBmtyljZ1q2manMvIMEtXtc9lq3gwxIP4Pq +ic5hKuEHDSy5iN0vZRhoqfgPzXMy61zCrvLmvxv8nN2B4Us44KQRzWwDvi8SavfQ +LxRZ4KLe5Bg7MNfIKM/ZqYqHIt1FtVFYR7UyEILN/yDCyQC+8n6s8RLmT5OtZHPL +0YAyHgdao4qCICkZShbCukq81ULvkq7i6QvHWZrVGAIc/1nN71QNEUMr9KtlTKO3 +TeSd+l13+CDGwMXUpglDiFL329TmG5pKr/zoTCGDmRvEfRPtICwY3FgqGDpmIwhw +dXq0JPGHrFODeVrchUMSGqXhAZ+k/9YdJlGLbv3WJmD1GwFTs3Wf +-----END CERTIFICATE----- \ No newline at end of file diff --git a/infra/cdn/main.tf b/infra/cdn/main.tf new file mode 100644 index 000000000..d5731940d --- /dev/null +++ b/infra/cdn/main.tf @@ -0,0 +1,346 @@ +resource "fastly_service_vcl" "python_org" { + name = var.name + default_ttl = var.default_ttl + http3 = false + stale_if_error = false + stale_if_error_ttl = 43200 + activate = true + + domain { + name = var.domain + } + + dynamic "domain" { + for_each = var.extra_domains + content { + name = domain.value + } + } + + backend { + name = "cabotage" + address = var.backend_address + port = 443 + shield = "iad-va-us" + auto_loadbalance = false + use_ssl = true + ssl_check_cert = true + ssl_cert_hostname = var.backend_address + ssl_sni_hostname = var.backend_address + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 30000 + between_bytes_timeout = 10000 + override_host = "www.python.org" + } + + backend { + name = "loadbalancer" + address = "lb.nyc1.psf.io" + shield = "iad-va-us" + healthcheck = "HAProxy Status" + auto_loadbalance = false + use_ssl = true + ssl_check_cert = true + ssl_cert_hostname = "lb.psf.io" + ssl_sni_hostname = "lb.psf.io" + ssl_ca_cert = file("${path.module}/cdn/certs/psf.io.pem") + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 15000 + between_bytes_timeout = 10000 + override_host = var.domain == "test.python.org" ? "www.python.org" : null + } + + acl { + name = "Generated_by_IP_block_list" + force_destroy = false + } + + cache_setting { + action = "pass" + cache_condition = "Force Pass No-Cache No-Store" + name = "Pass No-Cache No-Store" + stale_ttl = 0 + ttl = 0 + } + + condition { + name = "Force Pass No-Cache No-Store" + priority = 10 + statement = "beresp.http.Cache-Control ~ \"(no-cache|no-store)\"" + type = "CACHE" + } + condition { + name = "Generated by IP block list" + priority = 0 + statement = "client.ip ~ Generated_by_IP_block_list" + type = "REQUEST" + } + condition { + name = "HSTS w/ subdomains" + priority = 10 + statement = "req.http.host == \"${var.domain}\"" + type = "RESPONSE" + } + condition { + name = "HSTS w/o subdomain" + priority = 10 + statement = "req.http.host == \"${var.domain}\"" + type = "RESPONSE" + } + condition { + name = "Homepage" + priority = 10 + statement = "req.url.path ~ \"^/$\"" + type = "REQUEST" + } + condition { + name = "Is Download" + priority = 10 + statement = "req.url ~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Is Not Download" + priority = 5 + statement = "req.url !~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Uncacheable URLs" + priority = 10 + statement = "req.url ~ \"^/(api|admin)/\"" + type = "REQUEST" + } + condition { + name = "apex redirect" + priority = 10 + statement = "req.http.Host == \"python.org\"" + type = "RESPONSE" + } + condition { + name = "apex" + priority = 1 + statement = "req.http.host == \"python.org\"" + type = "REQUEST" + } + + gzip { + name = "Default rules" + content_types = [ + "application/javascript", + "text/css", + "application/javascript", + "text/javascript", + "application/json", + "application/vnd.ms-fontobject", + "application/x-font-opentype", + "application/x-font-truetype", + "application/x-font-ttf", + "application/xml", + "font/eot", + "font/opentype", + "font/otf", + "image/svg+xml", + "image/vnd.microsoft.icon", + "text/plain", + "text/xml", + ] + } + + header { + action = "delete" + destination = "http.Cookie" + name = "Remove cookies" + priority = 10 + request_condition = "Is Download" + type = "request" + } + header { + action = "set" + destination = "backend" + name = "Is Download Director" + priority = 10 + request_condition = "Is Download" + source = "F_loadbalancer" + type = "request" + } + header { + action = "set" + destination = "backend" + name = "Is Not Download Backend" + priority = 10 + request_condition = "Is Not Download" + source = "F_cabotage" + type = "request" + } + header { + action = "set" + destination = "http.Fastly-Token" + name = "Fastly Token" + priority = 10 + source = "\"${var.fastly_header_token}\"" + type = "request" + } + header { + action = "set" + destination = "http.Location" + name = "www redirect" + priority = 10 + response_condition = "apex redirect" + source = "\"https://www.\" + req.http.host + req.url" + type = "response" + } + header { + action = "set" + destination = "http.Strict-Transport-Security" + name = "HSTS w/ subdomains" + priority = 10 + response_condition = "HSTS w/ subdomains" + source = "\"max-age=63072000; includeSubDomains; preload\"" + type = "response" + } + header { + action = "set" + destination = "http.Strict-Transport-Security" + name = "HSTS w/o subdomains" + priority = 10 + response_condition = "HSTS w/o subdomain" + source = "\"max-age=315360000; preload\"" + type = "response" + } + header { + action = "set" + destination = "url" + name = "Chop off query string" + priority = 10 + request_condition = "Is Download" + source = "regsub(req.url, \"\\?.*$\", \"\")" + type = "request" + } + header { + action = "set" + destination = "url" + name = "Strip Query Strings" + priority = 10 + request_condition = "Homepage" + source = "req.url.path" + type = "request" + } + + healthcheck { + check_interval = 15000 + expected_response = 200 + host = var.domain + http_version = "1.1" + initial = 4 + method = "HEAD" + name = "HAProxy Status" + path = "/_haproxy_status" + threshold = 3 + timeout = 5000 + window = 5 + } + + logging_datadog { + name = "ratelimit-debug" + token = var.datadog_key + region = "US" + } + + logging_s3 { + name = "psf-fastly-logs" + bucket_name = "psf-fastly-logs-eu-west-1" + domain = "s3-eu-west-1.amazonaws.com" + path = "/${replace(var.domain, ".", "-")}/%Y/%m/%d/" + period = 3600 + gzip_level = 9 + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + timestamp_format = "%Y-%m-%dT%H:%M:%S.000" + redundancy = "standard" + format_version = 2 + message_type = "classic" + s3_access_key = var.s3_logging_keys["access_key"] + s3_secret_key = var.s3_logging_keys["secret_key"] + } + + logging_syslog { + name = "syslog" + address = "cdn-logs.nyc1.psf.io" + port = 514 + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + } + + product_enablement { + bot_management = true + brotli_compression = false + domain_inspector = true + image_optimizer = false + origin_inspector = true + websockets = false + } + + rate_limiter { + action = "log_only" + client_key = "client.ip" + feature_revision = 1 + http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" + logger_type = "datadog" + name = "${var.domain} backends" + penalty_box_duration = 2 + rps_limit = 10 + window_size = 10 + + response { + content = <<-EOT + + + Too Many Requests + + +

Too Many Requests

+ + + EOT + content_type = "text/html" + status = 429 + } + } + + request_setting { + action = null + bypass_busy_wait = false + force_ssl = true + max_stale_age = 86400 + name = "Default cache policy" + xff = "append" + } + request_setting { + action = "pass" + bypass_busy_wait = false + force_ssl = false + max_stale_age = 60 + name = "Force Pass" + request_condition = "Uncacheable URLs" + xff = "append" + } + + response_object { + name = "www redirect" + request_condition = "apex" + response = "Moved Permanently" + status = 301 + } + response_object { + content_type = "text/html" + name = "Generated by IP block list" + request_condition = "Generated by IP block list" + response = "Forbidden" + status = 403 + } + + force_destroy = true +} diff --git a/infra/cdn/providers.tf b/infra/cdn/providers.tf new file mode 100644 index 000000000..201f5de4a --- /dev/null +++ b/infra/cdn/providers.tf @@ -0,0 +1,4 @@ +provider "fastly" { + alias = "cdn" + api_key = var.fastly_key +} diff --git a/infra/cdn/variables.tf b/infra/cdn/variables.tf new file mode 100644 index 000000000..ccde39eeb --- /dev/null +++ b/infra/cdn/variables.tf @@ -0,0 +1,39 @@ +variable "fastly_key" { + type = string + description = "API key for the Fastly VCL edge configuration." +} +variable "fastly_header_token" { + description = "Fastly header token ensure we only allow Fastly to access the service" + type = string + sensitive = true +} +variable "datadog_key" { + type = string + description = "API key for Datadog logging" + sensitive = true +} +variable "s3_logging_keys" { + type = map(string) + description = "S3 bucket keys for Fastly logging" + sensitive = true +} +variable "name" { + type = string + description = "The name of the Fastly service." +} +variable "domain" { + type = string + description = "The domain name of the service." +} +variable "extra_domains" { + type = list(string) + description = "Extra domains to add to the service." +} +variable "backend_address" { + type = string + description = "The hostname of the backend service." +} +variable "default_ttl" { + type = number + description = "The default TTL for the service." +} \ No newline at end of file diff --git a/infra/cdn/versions.tf b/infra/cdn/versions.tf new file mode 100644 index 000000000..da9c01f79 --- /dev/null +++ b/infra/cdn/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "5.13.0" + } + } +} diff --git a/infra/config.tf b/infra/config.tf new file mode 100644 index 000000000..65b1a5210 --- /dev/null +++ b/infra/config.tf @@ -0,0 +1,9 @@ +# Connect us to TF Cloud for remote deploys +terraform { + cloud { + organization = "psf" + workspaces { + name = "pythondotorg-infra" + } + } +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 000000000..d760cc019 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,30 @@ +module "fastly_production" { + source = "./cdn" + + name = "Python.org" + domain = "python.org" + extra_domains = ["www.python.org"] + backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + s3_logging_keys = var.fastly_s3_logging +} + +module "fastly_staging" { + source = "./cdn" + + name = "test.Python.org" + domain = "test.python.org" + extra_domains = [] + # TODO: adjust to test-pythondotorg when done testing NGWAF + backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + s3_logging_keys = var.fastly_s3_logging +} diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 000000000..15cea38af --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,20 @@ +variable "FASTLY_API_KEY" { + type = string + description = "API key for the Fastly VCL edge configuration." + sensitive = true +} +variable "FASTLY_HEADER_TOKEN" { + description = "Fastly Token for authentication" + type = string + sensitive = true +} +variable "DATADOG_API_KEY" { + type = string + description = "API key for Datadog logging" + sensitive = true +} +variable "fastly_s3_logging" { + type = string + description = "S3 bucket keys for Fastly logging" + sensitive = true +} \ No newline at end of file From 21ec9814cc19697d894fdc0e85a5bfa47a7c3198 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 4 Sep 2024 12:27:50 -0500 Subject: [PATCH 2/6] feat: working tf plana --- infra/Makefile | 5 ----- infra/cdn/README.md | 2 +- infra/cdn/main.tf | 2 +- infra/main.tf | 2 +- infra/variables.tf | 2 +- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/infra/Makefile b/infra/Makefile index ae0f85b41..bee74cdc1 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -5,8 +5,3 @@ fmt: .PHONY: check check: @terraform validate - -.PHONY: yolo -yolo: - @echo "Wise, you are not..." - @terraform apply -auto-approve \ No newline at end of file diff --git a/infra/cdn/README.md b/infra/cdn/README.md index 51b5bbfaa..e0645c3ef 100644 --- a/infra/cdn/README.md +++ b/infra/cdn/README.md @@ -34,7 +34,7 @@ module "fastly_production" { datadog_key = var.DATADOG_API_KEY fastly_key = var.FASTLY_API_KEY fastly_header_token = var.FASTLY_HEADER_TOKEN - fastly_s3_logging = var.fastly_s3_logging + s3_logging_keys = var.fastly_s3_logging } ``` diff --git a/infra/cdn/main.tf b/infra/cdn/main.tf index d5731940d..472d05213 100644 --- a/infra/cdn/main.tf +++ b/infra/cdn/main.tf @@ -45,7 +45,7 @@ resource "fastly_service_vcl" "python_org" { ssl_check_cert = true ssl_cert_hostname = "lb.psf.io" ssl_sni_hostname = "lb.psf.io" - ssl_ca_cert = file("${path.module}/cdn/certs/psf.io.pem") + ssl_ca_cert = file("${path.module}/certs/psf.io.pem") weight = 100 max_conn = 200 connect_timeout = 1000 diff --git a/infra/main.tf b/infra/main.tf index d760cc019..bb6ceb19c 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -16,7 +16,7 @@ module "fastly_production" { module "fastly_staging" { source = "./cdn" - name = "test.Python.org" + name = "test.python.org" domain = "test.python.org" extra_domains = [] # TODO: adjust to test-pythondotorg when done testing NGWAF diff --git a/infra/variables.tf b/infra/variables.tf index 15cea38af..ec23b23ec 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -14,7 +14,7 @@ variable "DATADOG_API_KEY" { sensitive = true } variable "fastly_s3_logging" { - type = string + type = map(string) description = "S3 bucket keys for Fastly logging" sensitive = true } \ No newline at end of file From 5bda1329d1d2689569950df95de4ceb5bdda003b Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 4 Sep 2024 12:31:16 -0500 Subject: [PATCH 3/6] chore(docs): fix garbage --- infra/cdn/README.md | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/infra/cdn/README.md b/infra/cdn/README.md index e0645c3ef..029bb14c5 100644 --- a/infra/cdn/README.md +++ b/infra/cdn/README.md @@ -1,23 +1,6 @@ # Fastly CDN Config -This module creates Fastly services for the Python.org CDN. - -## Usage -# Fastly VCL Terraform Module - -This Terraform module configures a Fastly service using VCL (Varnish Configuration Language) for Python.org. It sets up a robust CDN configuration with various features to optimize performance, security, and logging. - -## Features - -- Configures Fastly VCL service for Python.org -- Supports multiple domains (primary and extra domains) -- Sets up backends with health checks -- Implements caching strategies and TTL configurations -- Configures HTTPS and HSTS -- Implements rate limiting -- Sets up logging to Datadog and S3 -- Configures various headers and request/response manipulations -- Implements IP blocking capabilities +This module creates Fastly services for the Python.org staging and production instances. ## Usage From 1b88cfbd8133f7b57db086551c31a43740d5ecfa Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 5 Sep 2024 15:07:54 -0500 Subject: [PATCH 4/6] feat: latest most minimal diff :D --- infra/.terraform.lock.hcl | 2 +- infra/cdn/main.tf | 39 +++++++++++++++++++-------------------- infra/cdn/variables.tf | 4 ++++ infra/main.tf | 6 ++++-- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 552e40475..165cd9357 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -3,7 +3,7 @@ provider "registry.terraform.io/fastly/fastly" { version = "5.13.0" - constraints = "5.13.0" + constraints = ">= 5.13.0" hashes = [ "h1:op/7hntTRkfZFIZ5xLNtLb7eBY155ywQIVSy56XCmBE=", "zh:04f7405ee22a8ace546b90cc3a08d81f1a49dae8b1050500398d4b0244dcbc86", diff --git a/infra/cdn/main.tf b/infra/cdn/main.tf index 472d05213..37c6410b1 100644 --- a/infra/cdn/main.tf +++ b/infra/cdn/main.tf @@ -4,7 +4,7 @@ resource "fastly_service_vcl" "python_org" { http3 = false stale_if_error = false stale_if_error_ttl = 43200 - activate = true + activate = false domain { name = var.domain @@ -32,13 +32,14 @@ resource "fastly_service_vcl" "python_org" { connect_timeout = 1000 first_byte_timeout = 30000 between_bytes_timeout = 10000 - override_host = "www.python.org" + override_host = var.subdomain == "test.python.org" ? "www.python.org" : null } backend { name = "loadbalancer" address = "lb.nyc1.psf.io" - shield = "iad-va-us" + port = 20004 + shield = "lga-ny-us" healthcheck = "HAProxy Status" auto_loadbalance = false use_ssl = true @@ -51,7 +52,7 @@ resource "fastly_service_vcl" "python_org" { connect_timeout = 1000 first_byte_timeout = 15000 between_bytes_timeout = 10000 - override_host = var.domain == "test.python.org" ? "www.python.org" : null + override_host = var.subdomain == "test.python.org" ? "www.python.org" : null } acl { @@ -82,7 +83,7 @@ resource "fastly_service_vcl" "python_org" { condition { name = "HSTS w/ subdomains" priority = 10 - statement = "req.http.host == \"${var.domain}\"" + statement = "req.http.host == \"${var.subdomain}\"" type = "RESPONSE" } condition { @@ -191,7 +192,7 @@ resource "fastly_service_vcl" "python_org" { name = "www redirect" priority = 10 response_condition = "apex redirect" - source = "\"https://www.\" + req.http.host + req.url" + source = "\"https://${var.subdomain}\" + req.url" type = "response" } header { @@ -255,10 +256,10 @@ resource "fastly_service_vcl" "python_org" { name = "psf-fastly-logs" bucket_name = "psf-fastly-logs-eu-west-1" domain = "s3-eu-west-1.amazonaws.com" - path = "/${replace(var.domain, ".", "-")}/%Y/%m/%d/" + path = "/${replace(var.subdomain, ".", "-")}/%Y/%m/%d/" period = 3600 gzip_level = 9 - format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + format = "%h \"%%{now}V\" %l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" timestamp_format = "%Y-%m-%dT%H:%M:%S.000" redundancy = "standard" format_version = 2 @@ -271,7 +272,7 @@ resource "fastly_service_vcl" "python_org" { name = "syslog" address = "cdn-logs.nyc1.psf.io" port = 514 - format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + format = "%h \"%%{now}V\" %l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" } product_enablement { @@ -296,15 +297,15 @@ resource "fastly_service_vcl" "python_org" { response { content = <<-EOT - - - Too Many Requests - - -

Too Many Requests

- - - EOT + + + Too Many Requests + + +

Too Many Requests

+ + + EOT content_type = "text/html" status = 429 } @@ -341,6 +342,4 @@ resource "fastly_service_vcl" "python_org" { response = "Forbidden" status = 403 } - - force_destroy = true } diff --git a/infra/cdn/variables.tf b/infra/cdn/variables.tf index ccde39eeb..4cbf6db6e 100644 --- a/infra/cdn/variables.tf +++ b/infra/cdn/variables.tf @@ -25,6 +25,10 @@ variable "domain" { type = string description = "The domain name of the service." } +variable "subdomain" { + type = string + description = "The subdomain of the service." +} variable "extra_domains" { type = list(string) description = "Extra domains to add to the service." diff --git a/infra/main.tf b/infra/main.tf index bb6ceb19c..b3ec26a77 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,8 +1,9 @@ module "fastly_production" { source = "./cdn" - name = "Python.org" + name = "www.python.org" domain = "python.org" + subdomain = "www.python.org" extra_domains = ["www.python.org"] backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" default_ttl = 3600 @@ -18,7 +19,8 @@ module "fastly_staging" { name = "test.python.org" domain = "test.python.org" - extra_domains = [] + subdomain = "www.test.python.org" + extra_domains = ["www.test.python.org"] # TODO: adjust to test-pythondotorg when done testing NGWAF backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" default_ttl = 3600 From 74113f0d5cad689f1a752fc88f9c6b546dbb4b3a Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 5 Sep 2024 15:09:15 -0500 Subject: [PATCH 5/6] Update infra/cdn/README.md --- infra/cdn/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/cdn/README.md b/infra/cdn/README.md index 029bb14c5..6ebe5a637 100644 --- a/infra/cdn/README.md +++ b/infra/cdn/README.md @@ -10,6 +10,7 @@ module "fastly_production" { name = "CoolPythonApp.org" domain = "CoolPythonApp.org" + subdomain = "www.CoolPythonApp.org" extra_domains = ["www.CoolPythonApp.org"] backend_address = "service.CoolPythonApp.org" default_ttl = 3600 From 77e7c82d48506d8a669d87eb3e9cde94e9fe1a39 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 5 Sep 2024 15:13:24 -0500 Subject: [PATCH 6/6] run tf