From faf1720b22da674f84980927db980bb7ab104bfb Mon Sep 17 00:00:00 2001 From: Kochetkov S Date: Mon, 6 Jul 2026 17:10:46 +0300 Subject: [PATCH] ++ port sops secret-values adopt mechanism to prod --- .sops.yaml | 3 + Dockerfile | 7 +- live/prod/secrets/terragrunt.hcl | 12 +++ live/terragrunt.hcl | 5 + modules/k8s-secret/main.tf | 160 +++++++++++++++++++------------ modules/k8s-secret/variables.tf | 57 +++++++---- scripts/generate-pipeline.sh | 25 ++++- 7 files changed, 182 insertions(+), 87 deletions(-) create mode 100644 .sops.yaml diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..ce9c6c1 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - path_regex: infrastructure-secrets\.yaml$ + age: age198ea80z7gan2ruetn288ln6rl2qh7xtsqqpwrjtyn8maya3364fsvpdd63 diff --git a/Dockerfile b/Dockerfile index 864fd88..58b9473 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,15 @@ RUN curl -fsSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/ter rm -f terraform.zip # Terragrunt -ARG TERRAGRUNT_VERSION=0.69.1 +ARG TERRAGRUNT_VERSION=0.93.11 RUN curl -fsSL https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64 -o /usr/local/bin/terragrunt && \ chmod +x /usr/local/bin/terragrunt +# SOPS (расшифровка secret-values через age; ключ передаётся в CI через SOPS_AGE_KEY) +ARG SOPS_VERSION=3.9.0 +RUN curl -fsSL https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64 -o /usr/local/bin/sops && \ + chmod +x /usr/local/bin/sops + # Plugin cache и mirror для Yandex Cloud RUN mkdir -p /root/.terraform.d/plugin-cache && \ mkdir -p /root/.terraform.d && \ diff --git a/live/prod/secrets/terragrunt.hcl b/live/prod/secrets/terragrunt.hcl index f8b0d78..642322c 100644 --- a/live/prod/secrets/terragrunt.hcl +++ b/live/prod/secrets/terragrunt.hcl @@ -125,6 +125,13 @@ EOT infra_config = yamldecode(file("${local.repo_root}/infrastructure.yaml")) env_name = basename(dirname(get_terragrunt_dir())) env_config = local.infra_config.environments[local.env_name] + + # Live-значения adopt-секретов (переезд 1:1) из sops-файла infrastructure-secrets.yaml. + # В CI расшифровывается заранее (INFRA_SECRET_VALUES_FILE), локально - sops_decrypt_file. + secret_values_env_file = get_env("INFRA_SECRET_VALUES_FILE", "") + secret_values_raw = local.secret_values_env_file != "" ? file(local.secret_values_env_file) : try(sops_decrypt_file("${local.repo_root}/infrastructure-secrets.yaml"), "") + secret_values_config = try(yamldecode(local.secret_values_raw), {}) + secret_values_map = try(local.secret_values_config.environments[local.env_name].secret_values, {}) } terraform { @@ -134,6 +141,7 @@ terraform { inputs = { secrets = [ for secret in local.env_config.secrets : { + resource_key = try(secret.resource_key, "") name = secret.name namespace = secret.namespace secret_type = secret.type @@ -143,12 +151,16 @@ inputs = { custom_keys = try(secret.custom_keys, {}) constant_keys = try(secret.constant_keys, {}) random_keys = try(secret.random_keys, {}) + credential_keys = try(secret.credential_keys, {}) labels = try(secret.labels, {}) annotations = try(secret.annotations, {}) ignore_changes = try(secret.lifecycle.ignore_changes, false) } ] + # Live-значения adopt-секретов (1:1) из расшифрованного sops secret-values. + secret_values = local.secret_values_map + s3_outputs = { bucket_name = dependency.s3.outputs.bucket_name access_key = dependency.s3.outputs.access_key diff --git a/live/terragrunt.hcl b/live/terragrunt.hcl index 9d381b9..5441297 100644 --- a/live/terragrunt.hcl +++ b/live/terragrunt.hcl @@ -16,6 +16,11 @@ remote_state { skip_metadata_api_check = true force_path_style = true skip_bucket_root_access = true + # Object Storage бакет без versioning/SSE; свежий terragrunt иначе падает на проверке. + skip_bucket_versioning = true + skip_bucket_ssencryption = true + skip_bucket_enforced_tls = true + skip_bucket_public_access_blocking = true } } diff --git a/modules/k8s-secret/main.tf b/modules/k8s-secret/main.tf index 6caae7a..a2b3b95 100644 --- a/modules/k8s-secret/main.tf +++ b/modules/k8s-secret/main.tf @@ -5,11 +5,10 @@ locals { k8s_secret_type_map = { "dockerconfigjson" = "kubernetes.io/dockerconfigjson" "database" = "Opaque" - "database_url" = "Opaque" "s3" = "Opaque" - "valkey" = "Opaque" "yc_sa" = "Opaque" "rabbitmq" = "Opaque" + "kafka" = "Opaque" "opaque" = "Opaque" } @@ -20,7 +19,7 @@ locals { } secrets_map = { - for idx, secret in var.secrets : secret.name => secret + for idx, secret in var.secrets : (try(secret.resource_key, "") != "" ? secret.resource_key : secret.name) => secret } random_keys_flat = merge([ @@ -41,20 +40,6 @@ locals { ) } - valkey_dependency_keys = { - for name, secret in local.secrets_map : name => { - cluster = try(secret.dependencies.valkey_cluster, "") != "" ? try(secret.dependencies.valkey_cluster, "") : try(secret.dependencies.cluster, "") - user = try(secret.dependencies.valkey_user, "") != "" ? try(secret.dependencies.valkey_user, "") : try(secret.dependencies.user, "") - } - } - - valkey_outputs_by_secret = { - for name, secret in local.secrets_map : name => try( - var.valkey_outputs_map["${local.valkey_dependency_keys[name].cluster}:${local.valkey_dependency_keys[name].user}"], - null - ) - } - rabbitmq_outputs_by_secret = { for name, secret in local.secrets_map : name => try( var.rabbitmq_outputs_map[ @@ -69,6 +54,65 @@ locals { ) } + kafka_outputs_by_secret = { + for name, secret in local.secrets_map : name => try( + var.kafka_outputs_map[ + format( + "%s:%s", + try(secret.dependencies.kafka_ref, try(secret.dependencies.cluster_ref, try(secret.dependencies.cluster, ""))), + try(secret.dependencies.kafka_user, try(secret.dependencies.user, "")) + ) + ], + null + ) + } + + rabbitmq_default_credentials_by_secret = { + for name, secret in local.secrets_map : name => local.rabbitmq_outputs_by_secret[name] == null ? {} : { + host = local.rabbitmq_outputs_by_secret[name].host + hostname = "${local.rabbitmq_outputs_by_secret[name].host}:${local.rabbitmq_outputs_by_secret[name].port}" + port = local.rabbitmq_outputs_by_secret[name].port + vhost = local.rabbitmq_outputs_by_secret[name].vhost + user = local.rabbitmq_outputs_by_secret[name].username + username = local.rabbitmq_outputs_by_secret[name].username + password = local.rabbitmq_outputs_by_secret[name].password + uri = local.rabbitmq_outputs_by_secret[name].uri + management_endpoint = local.rabbitmq_outputs_by_secret[name].management_endpoint + } + if secret.secret_type == "rabbitmq" + } + + rabbitmq_credentials_by_secret = { + for name, secret in local.secrets_map : name => length(try(secret.credential_keys, {})) == 0 ? local.rabbitmq_default_credentials_by_secret[name] : { + for key, source in try(secret.credential_keys, {}) : key => try(local.rabbitmq_default_credentials_by_secret[name][source], "") + } + if secret.secret_type == "rabbitmq" && local.rabbitmq_outputs_by_secret[name] != null + } + + kafka_default_credentials_by_secret = { + for name, secret in local.secrets_map : name => local.kafka_outputs_by_secret[name] == null ? {} : { + host = local.kafka_outputs_by_secret[name].host + hostname = local.kafka_outputs_by_secret[name].hostname + port = local.kafka_outputs_by_secret[name].port + username = local.kafka_outputs_by_secret[name].username + user = local.kafka_outputs_by_secret[name].username + password = local.kafka_outputs_by_secret[name].password + sasl_mechanism = local.kafka_outputs_by_secret[name].sasl_mechanism + security_protocol = local.kafka_outputs_by_secret[name].security_protocol + bootstrap_server = local.kafka_outputs_by_secret[name].bootstrap_server + bootstrap_servers = local.kafka_outputs_by_secret[name].bootstrap_servers + bootstrap_servers_json = local.kafka_outputs_by_secret[name].bootstrap_servers_json + } + if secret.secret_type == "kafka" + } + + kafka_credentials_by_secret = { + for name, secret in local.secrets_map : name => length(try(secret.credential_keys, {})) == 0 ? local.kafka_default_credentials_by_secret[name] : { + for key, source in try(secret.credential_keys, {}) : key => try(local.kafka_default_credentials_by_secret[name][source], "") + } + if secret.secret_type == "kafka" && local.kafka_outputs_by_secret[name] != null + } + yc_sa_sources = { for name, secret in local.secrets_map : name => try( var.yc_service_accounts_map[try(secret.dependencies.service_account, "")], @@ -85,7 +129,17 @@ locals { if secret.secret_type == "yc_sa" } - secrets_data = { + # Значения, захваченные из live-кластера (adopt 1:1). Приходят из расшифрованного + # secret-values файла как map(resource_key => map(key => base64(raw_value))). + # k8s хранит .data уже в base64, поэтому base64decode даёт исходное значение байт-в-байт. + secrets_data_override = { + for name, secret in local.secrets_map : name => { + for key, b64 in try(var.secret_values[name], {}) : key => base64decode(b64) + } + if length(try(var.secret_values[name], {})) > 0 + } + + secrets_data_computed = { for name, secret in local.secrets_map : name => secret.secret_type == "dockerconfigjson" ? { ".dockerconfigjson" = jsonencode({ @@ -136,52 +190,17 @@ locals { { for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result } - ) : secret.secret_type == "database_url" && local.database_outputs_by_secret[name] != null ? merge( - { - database_url = format( - "postgresql+asyncpg://%s:%s@%s:%s/%s", - local.database_outputs_by_secret[name].user_name, - local.database_outputs_by_secret[name].password, - local.database_outputs_by_secret[name].host, - try(var.constants.postgres_port, "6432"), - local.database_outputs_by_secret[name].database_name - ) - }, - try(secret.custom_keys, {}), - { - for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "") - }, - { - for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result - } - ) : secret.secret_type == "valkey" && local.valkey_outputs_by_secret[name] != null ? merge( - { - cert = try(var.constants.valkey_ca, try(var.constants.postgres_ca, "")) - host = local.valkey_outputs_by_secret[name].host - login = local.valkey_outputs_by_secret[name].user_name - password = local.valkey_outputs_by_secret[name].password - port = tostring(try(local.valkey_outputs_by_secret[name].port, try(var.constants.valkey_port, "6380"))) - url = format("%s:%s", local.valkey_outputs_by_secret[name].host, tostring(try(local.valkey_outputs_by_secret[name].port, try(var.constants.valkey_port, "6380")))) - }, - try(secret.custom_keys, {}), - { - for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "") - }, - { - for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result - } ) : secret.secret_type == "rabbitmq" && local.rabbitmq_outputs_by_secret[name] != null ? merge( + local.rabbitmq_credentials_by_secret[name], + try(secret.custom_keys, {}), { - host = local.rabbitmq_outputs_by_secret[name].host - hostname = "${local.rabbitmq_outputs_by_secret[name].host}:${local.rabbitmq_outputs_by_secret[name].port}" - port = local.rabbitmq_outputs_by_secret[name].port - vhost = local.rabbitmq_outputs_by_secret[name].vhost - user = local.rabbitmq_outputs_by_secret[name].username - username = local.rabbitmq_outputs_by_secret[name].username - password = local.rabbitmq_outputs_by_secret[name].password - uri = local.rabbitmq_outputs_by_secret[name].uri - management_endpoint = local.rabbitmq_outputs_by_secret[name].management_endpoint + for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "") }, + { + for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result + } + ) : secret.secret_type == "kafka" && local.kafka_outputs_by_secret[name] != null ? merge( + local.kafka_credentials_by_secret[name], try(secret.custom_keys, {}), { for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "") @@ -200,6 +219,16 @@ locals { ) } + # Если для секрета заданы live-значения (adopt) - используем их 1:1, + # иначе собираем data из outputs/custom/constant/random как обычно. + secrets_data = { + for name, secret in local.secrets_map : name => ( + contains(keys(local.secrets_data_override), name) + ? local.secrets_data_override[name] + : local.secrets_data_computed[name] + ) + } + secrets_with_data = { for name, secret in local.secrets_map : name => { name = secret.name @@ -211,7 +240,6 @@ locals { ignore_changes = try(secret.ignore_changes, false) data = try(local.secrets_data[name], {}) } - if secret.secret_type != "rabbitmq" || local.rabbitmq_outputs_by_secret[name] != null } secrets_with_ignore = { @@ -250,6 +278,11 @@ resource "kubernetes_secret" "with_ignore" { type = each.value.k8s_type + # Совпадает с импортированным состоянием (провайдер по умолчанию ставит true и + # даёт phantom "+ wait_for_service_account_token = true" на adopt-секретах). + # Актуально только для secret-ов типа service-account-token; тут все Opaque. + wait_for_service_account_token = false + data = each.value.data lifecycle { @@ -272,6 +305,9 @@ resource "kubernetes_secret" "without_ignore" { type = each.value.k8s_type + # Совпадает с импортированным состоянием (иначе phantom "+ wait_for_service_account_token = true"). + wait_for_service_account_token = false + data = each.value.data lifecycle { diff --git a/modules/k8s-secret/variables.tf b/modules/k8s-secret/variables.tf index 2dea32b..3e2a25d 100644 --- a/modules/k8s-secret/variables.tf +++ b/modules/k8s-secret/variables.tf @@ -2,20 +2,23 @@ variable "secrets" { description = "Список секретов для создания" type = list(object({ name = string + resource_key = optional(string, "") namespace = string secret_type = string registry_url = optional(string, "") dependencies = optional(object({ cluster = optional(string, "") + cluster_ref = optional(string, "") db = optional(string, "") user = optional(string, "") - valkey_cluster = optional(string, "") - valkey_user = optional(string, "") + kafka_ref = optional(string, "") + kafka_user = optional(string, "") + vhost = optional(string, "") + rabbitmq_ref = optional(string, "default") + rabbitmq_user = optional(string, "") + rabbitmq_vhost = optional(string, "") bucket = optional(string, "") # Имя бакета для s3 секретов service_account = optional(string, "") # Имя YC Service Account для yc_sa секретов - rabbitmq_ref = optional(string, "default") - rabbitmq_vhost = optional(string, "") - rabbitmq_user = optional(string, "") }), {}) yc_sa_key_names = optional(object({ access_key = optional(string, "access_key") @@ -28,12 +31,19 @@ variable "secrets" { length = optional(number, 32) special = optional(bool, false) })), {}) # Случайные ключи - labels = optional(map(string), {}) - annotations = optional(map(string), {}) - ignore_changes = optional(bool, false) + credential_keys = optional(map(string), {}) + labels = optional(map(string), {}) + annotations = optional(map(string), {}) + ignore_changes = optional(bool, false) })) } +variable "secret_values" { + description = "Захваченные из live-кластера значения секретов для adopt 1:1. Ключ = resource_key (или name), значение = map(key => base64(raw)). Если задано - используется как data секрета без изменений." + type = map(map(string)) + default = {} +} + variable "s3_outputs" { description = "Outputs от модуля S3 (default/первый бакет)" type = object({ @@ -81,20 +91,8 @@ variable "database_outputs_map" { default = {} } -variable "valkey_outputs_map" { - description = "Outputs от модулей Valkey/Redis users, ключ = cluster_id:user_name" - type = map(object({ - cluster_id = string - host = string - port = string - user_name = string - password = string - })) - default = {} -} - variable "rabbitmq_outputs_map" { - description = "Outputs от модуля RabbitMQ, ключ = ref:vhost:user" + description = "Outputs от модуля RabbitMQ, ключ = rabbitmq_ref:vhost:user" type = map(object({ host = string port = string @@ -107,6 +105,23 @@ variable "rabbitmq_outputs_map" { default = {} } +variable "kafka_outputs_map" { + description = "Outputs от модуля Kafka, ключ = kafka_ref:user" + type = map(object({ + host = string + hostname = string + port = string + username = string + password = string + sasl_mechanism = string + security_protocol = string + bootstrap_server = string + bootstrap_servers = string + bootstrap_servers_json = string + })) + default = {} +} + variable "constants" { description = "Константные значения (CA сертификаты, endpoints и т.д.)" type = map(string) diff --git a/scripts/generate-pipeline.sh b/scripts/generate-pipeline.sh index c13ab8a..2fcffc0 100755 --- a/scripts/generate-pipeline.sh +++ b/scripts/generate-pipeline.sh @@ -37,8 +37,15 @@ find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sort | whi secrets_dependency_components=() needs_section="" + # Для secrets: расшифровать sops-файл с live-значениями во временный файл вне артефактов + # и передать путь в terragrunt через INFRA_SECRET_VALUES_FILE (требуется SOPS_AGE_KEY в CI). + secrets_decrypt_before="" if [ "$component" = "secrets" ]; then - for dep_component in namespace s3 database rabbitmq valkey-users; do + secrets_decrypt_before=" - sops --decrypt infrastructure-secrets.yaml > /tmp/infra-secret-values.yaml + - export INFRA_SECRET_VALUES_FILE=/tmp/infra-secret-values.yaml" + fi + if [ "$component" = "secrets" ]; then + for dep_component in namespace s3 database rabbitmq kafka-topics; do if [ -f "live/${env}/${dep_component}/terragrunt.hcl" ]; then secrets_dependency_components+=("$dep_component") fi @@ -59,6 +66,10 @@ ${dependencies_section}" validate_before_script=" - cd \$TG_ROOT - echo \"Working directory:\" && pwd - ls -la" + if [ "$component" = "secrets" ]; then + validate_before_script="${secrets_decrypt_before} +${validate_before_script}" + fi case "$env" in stage) folder_var="YC_STAGE_FOLDER_ID" ;; @@ -132,17 +143,24 @@ VALIDATE_JOB dependency_init_chain="${dependency_init_chain} && cd ../${dep_component} && terragrunt init -upgrade -reconfigure" fi done - plan_before_script=" - ${dependency_init_chain} && cd ../secrets && ls -la" + plan_before_script="${secrets_decrypt_before} + - ${dependency_init_chain} && cd ../secrets && ls -la" fi if [ "$component" = "kafka-topics" ]; then plan_before_script=" - cd \$TG_ROOT && ls -la" fi + plan_allow_failure="" + if [ "$env" = "stage" ] && [ "$component" = "secrets" ]; then + plan_allow_failure=" allow_failure: true" + fi + cat >> "$OUTPUT_FILE" << PLAN_JOB plan-${job_prefix}: stage: plan image: \$TERRAFORM_IMAGE_NAME ${runner_tags} +${plan_allow_failure} variables: TG_ROOT: "${full_component_dir}" ENVIRONMENT: "${env}" @@ -184,7 +202,8 @@ PLAN_JOB dependency_init_chain="${dependency_init_chain} && cd ../${dep_component} && terragrunt init -upgrade -reconfigure" fi done - apply_before_script=" - ${dependency_init_chain} && cd ../secrets && ls -la" + apply_before_script="${secrets_decrypt_before} + - ${dependency_init_chain} && cd ../secrets && ls -la" fi if [ "$component" = "kafka-topics" ]; then apply_before_script=" - cd \$TG_ROOT && ls -la"