sarex/terraform-contour-mirror
1
0
Fork 0
mirror of https://gitlab.sarex.io/infra/terraform-contour-mirror.git synced 2026-08-05 18:31:00 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

add sarex-agents infra prod

Browse Source
This commit is contained in:
Kochetkov S 2026-06-10 11:47:30 +03:00
parent 42bcbb6f3a
commit fad9234ae4
11 changed files with 432 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
README.md
View File

@ -20,6 +20,7 @@ terraform/
├── k8s-secret/ # Создание Kubernetes secrets (с зависимостями)
├── k8s-secrets/ # Простое создание secrets
├── yc-database/ # PostgreSQL users и databases в Yandex Cloud
├── yc-valkey-user/ # Valkey/Redis users в Yandex Cloud
└── yc-s3/ # S3 бакеты с изолированным доступом
```
@ -57,10 +58,19 @@ environments:
password_length: 32
conn_limit: 10
valkey_users: # Valkey/Redis users
- cluster_id: "xxx"
host: rc1a-example.mdb.yandexcloud.net
user:
name: myuser
permissions:
patterns: allkeys
pubSubChannels: allchannels
secrets: # Kubernetes secrets
- name: my-secret
namespace: pulse
type: database # database | s3 | dockerconfigjson | opaque
type: database # database | database_url | valkey | s3 | dockerconfigjson | opaque
dependencies:
cluster: "xxx"
db: mydb
@ -87,7 +97,7 @@ inputs = {
Модули имеют зависимости и применяются в порядке:
```
namespace → database → s3 → secrets
namespace → database / s3 / valkey-users → secrets
```
## Модули
@ -139,12 +149,40 @@ databases:
permissions: [other_db]
```
### yc-valkey-user
Создание Valkey/Redis users в существующем кластере Yandex Cloud.
**Особенности:**
- Пароль генерируется автоматически
- Поддержка ACL permissions
- `ignore_changes` на password
**Пример:**
```yaml
valkey_users:
- cluster_id: "c9qa2coo5ukgcg93fldm"
host: rc1a-example.mdb.yandexcloud.net
port: "6380"
user:
name: myuser
password_length: 32
permissions:
patterns: allkeys
pubSubChannels: allchannels
categories: "+@read +@write"
commands: "+GET -FLUSHALL"
sanitizePayload: sanitize-payload
```
### k8s-secret
Создание Kubernetes secrets с поддержкой зависимостей.
**Типы секретов:**
- `database` — host, port, database, username, password, ca.crt
- `database_url` — database_url в формате postgresql+asyncpg
- `valkey` — cert, host, login, password, port, url
- `s3` — access_key, secret_key, bucket, endpoint
- `dockerconfigjson` — credentials для container registry
- `opaque` — произвольные данные

126
infrastructure.yaml
View File

@ -50,6 +50,11 @@ environments:
istio-injection: enabled
annotations:
managed-by: terraform
- name: ai
labels:
istio-injection: enabled
annotations:
managed-by: terraform
secrets:
- name: dockerhub
@ -57,6 +62,94 @@ environments:
type: dockerconfigjson
registry_url: cr.yandex
- name: sarex-agents-valkey-secret
namespace: ai
type: valkey
dependencies:
valkey_cluster: "c9qb9l18fvb0vg2nrcq4"
valkey_user: sarex-agents
lifecycle:
ignore_changes: true
- name: ai-agents-s3-secret
namespace: ai
type: s3
dependencies:
bucket: ai-agents
lifecycle:
ignore_changes: true
- name: postgresql-secret
namespace: ai
type: database_url
dependencies:
cluster: "c9quu72dj2ibu8dk54q5"
db: document-understanding
user: document-understanding
lifecycle:
ignore_changes: true
- name: gemini-api-key
namespace: ai
type: opaque
random_keys:
gemini_api_key:
length: 32
special: false
lifecycle:
ignore_changes: true
- name: mvp-agents-secrets
namespace: ai
type: opaque
random_keys:
GEMINI_API_KEY:
length: 32
special: false
LANGSMITH_API_KEY:
length: 32
special: false
POSTGRES_URI:
length: 32
special: false
REDIS_URI:
length: 32
special: false
lifecycle:
ignore_changes: true
- name: openai-api-key
namespace: ai
type: opaque
random_keys:
openai_api_key:
length: 32
special: false
lifecycle:
ignore_changes: true
- name: sarex-agents-secrets
namespace: ai
type: opaque
random_keys:
GEMINI_API_KEY:
length: 32
special: false
LANGSMITH_API_KEY:
length: 32
special: false
OPENAI_API_KEY:
length: 32
special: false
POSTGRES_URI:
length: 32
special: false
REDIS_URI:
length: 32
special: false
lifecycle:
ignore_changes: true
- name: pulse-s3-secret
namespace: pulse
type: s3
@ -347,6 +440,13 @@ environments:
cors:
enabled: false
- name: ai-agents
acl: private
versioning:
enabled: false
cors:
enabled: false
# yc_service_accounts:
# - name: dwg-sa
# description: "DWG SA Uploader"
@ -378,3 +478,29 @@ environments:
password_special: false
conn_limit: 10
- cluster_id: "c9quu72dj2ibu8dk54q5"
database:
name: document-understanding
host: rc1b-384bzx2drqoaxt4d.mdb.yandexcloud.net
user:
name: document-understanding
password_length: 32
password_special: false
conn_limit: 10
valkey_users:
- cluster_id: "c9qb9l18fvb0vg2nrcq4"
host: rc1d-qk3kd08nre0ntdu9.mdb.yandexcloud.net
port: "6380"
user:
name: sarex-agents
password_length: 32
password_special: false
enabled: true
permissions:
patterns: allkeys
pubSubChannels: allchannels
categories: "+@read +@write"
commands: "+GET -FLUSHALL"
sanitizePayload: sanitize-payload

12
live/prod/secrets/terragrunt.hcl
View File

@ -40,6 +40,15 @@ dependency "database" {
skip_outputs = get_env("TG_SKIP_DEPENDENCY_OUTPUTS", "false") == "true"
}
dependency "valkey_users" {
config_path = "../valkey-users"
mock_outputs = {
valkey_outputs_map = {}
}
mock_outputs_allowed_terraform_commands = ["init", "validate", "plan"]
skip_outputs = get_env("TG_SKIP_DEPENDENCY_OUTPUTS", "false") == "true"
}
locals {
yc_postgresql_ca_cert = <<-EOT
-----BEGIN CERTIFICATE-----
@ -141,11 +150,14 @@ inputs = {
yc_service_accounts_map = try(dependency.s3.outputs.service_accounts, {})
database_outputs_map = dependency.database.outputs.database_outputs_map
valkey_outputs_map = dependency.valkey_users.outputs.valkey_outputs_map
constants = {
s3_endpoint = "https://storage.yandexcloud.net"
postgres_port = "6432"
postgres_ca = local.yc_postgresql_ca_cert
valkey_port = "6380"
valkey_ca = local.yc_postgresql_ca_cert
}
env_vars = {

31
live/prod/valkey-users/terragrunt.hcl Normal file
View File

@ -0,0 +1,31 @@
include "root" {
path = find_in_parent_folders()
}
include "env" {
path = find_in_parent_folders("env.hcl")
expose = true
merge_strategy = "deep"
}
locals {
repo_root = try(get_repo_root(), "${get_terragrunt_dir()}/../../..")
infra_config = yamldecode(file("${local.repo_root}/infrastructure.yaml"))
env_name = basename(dirname(get_terragrunt_dir()))
env_config = local.infra_config.environments[local.env_name]
}
terraform {
source = "${get_terragrunt_dir()}/../../../modules//yc-valkey-user"
}
inputs = {
valkey_users = try(local.env_config.valkey_users, [])
yc_token = get_env("YC_TOKEN", "")
yc_service_account_key_file = get_env("YC_SERVICE_ACCOUNT_KEY_FILE", "")
yc_cloud_id = get_env("YC_CLOUD_ID", "")
yc_folder_id = get_env("YC_PROD_FOLDER_ID", get_env("YC_FOLDER_ID", ""))
kubeconfig_path = get_env("KUBECONFIG", "")
kube_context = get_env("KUBE_CONTEXT", "")
}

50
modules/k8s-secret/main.tf
View File

@ -5,7 +5,9 @@ locals {
k8s_secret_type_map = {
"dockerconfigjson" = "kubernetes.io/dockerconfigjson"
"database" = "Opaque"
"database_url" = "Opaque"
"s3" = "Opaque"
"valkey" = "Opaque"
"yc_sa" = "Opaque"
"opaque" = "Opaque"
}
@ -38,6 +40,20 @@ locals {
)
}
valkey_dependency_keys = {
for name, secret in local.secrets_map : name => {
cluster = try(secret.dependencies.valkey_cluster, "") != "" ? try(secret.dependencies.valkey_cluster, "") : try(secret.dependencies.cluster, "")
user = try(secret.dependencies.valkey_user, "") != "" ? try(secret.dependencies.valkey_user, "") : try(secret.dependencies.user, "")
}
}
valkey_outputs_by_secret = {
for name, secret in local.secrets_map : name => try(
var.valkey_outputs_map["${local.valkey_dependency_keys[name].cluster}:${local.valkey_dependency_keys[name].user}"],
null
)
}
yc_sa_sources = {
for name, secret in local.secrets_map : name => try(
var.yc_service_accounts_map[try(secret.dependencies.service_account, "")],
@ -105,6 +121,40 @@ locals {
{
for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result
}
) : secret.secret_type == "database_url" && local.database_outputs_by_secret[name] != null ? merge(
{
database_url = format(
"postgresql+asyncpg://%s:%s@%s:%s/%s",
local.database_outputs_by_secret[name].user_name,
local.database_outputs_by_secret[name].password,
local.database_outputs_by_secret[name].host,
try(var.constants.postgres_port, "6432"),
local.database_outputs_by_secret[name].database_name
)
},
try(secret.custom_keys, {}),
{
for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "")
},
{
for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result
}
) : secret.secret_type == "valkey" && local.valkey_outputs_by_secret[name] != null ? merge(
{
cert = try(var.constants.valkey_ca, try(var.constants.postgres_ca, ""))
host = local.valkey_outputs_by_secret[name].host
login = local.valkey_outputs_by_secret[name].user_name
password = local.valkey_outputs_by_secret[name].password
port = tostring(try(local.valkey_outputs_by_secret[name].port, try(var.constants.valkey_port, "6380")))
url = format("%s:%s", local.valkey_outputs_by_secret[name].host, tostring(try(local.valkey_outputs_by_secret[name].port, try(var.constants.valkey_port, "6380"))))
},
try(secret.custom_keys, {}),
{
for key, constant_name in try(secret.constant_keys, {}) : key => try(var.constants[constant_name], "")
},
{
for key, config in try(secret.random_keys, {}) : key => random_password.secrets["${name}:${key}"].result
}
) : merge(
try(secret.custom_keys, {}),
{

14
modules/k8s-secret/variables.tf
View File

@ -9,6 +9,8 @@ variable "secrets" {
cluster = optional(string, "")
db = optional(string, "")
user = optional(string, "")
valkey_cluster = optional(string, "")
valkey_user = optional(string, "")
bucket = optional(string, "") # Имя бакета для s3 секретов
service_account = optional(string, "") # Имя YC Service Account для yc_sa секретов
}), {})
@ -76,6 +78,18 @@ variable "database_outputs_map" {
default = {}
}
variable "valkey_outputs_map" {
description = "Outputs от модулей Valkey/Redis users, ключ = cluster_id:user_name"
type = map(object({
cluster_id = string
host = string
port = string
user_name = string
password = string
}))
default = {}
}
variable "constants" {
description = "Константные значения (CA сертификаты, endpoints и т.д.)"
type = map(string)

56
modules/yc-valkey-user/main.tf Normal file
View File

@ -0,0 +1,56 @@
locals {
valkey_users_map = {
for valkey_user in var.valkey_users : "${valkey_user.cluster_id}:${valkey_user.user.name}" => valkey_user
}
valkey_user_permissions = {
for key, valkey_user in local.valkey_users_map : key => merge(
try(valkey_user.permissions.commands, null) != null ? {
commands = valkey_user.permissions.commands
} : {},
try(valkey_user.permissions.categories, null) != null ? {
categories = valkey_user.permissions.categories
} : {},
try(valkey_user.permissions.patterns, null) != null ? {
patterns = valkey_user.permissions.patterns
} : {},
try(valkey_user.permissions.pub_sub_channels, try(valkey_user.permissions.pubSubChannels, null)) != null ? {
pub_sub_channels = try(valkey_user.permissions.pub_sub_channels, valkey_user.permissions.pubSubChannels)
} : {},
try(valkey_user.permissions.sanitize_payload, try(valkey_user.permissions.sanitizePayload, null)) != null ? {
sanitize_payload = try(valkey_user.permissions.sanitize_payload, valkey_user.permissions.sanitizePayload)
} : {},
try(valkey_user.permissions.databases, null) != null ? {
databases = valkey_user.permissions.databases
} : {}
)
}
}
resource "random_password" "user_password" {
for_each = local.valkey_users_map
length = try(each.value.user.password_length, 32)
special = try(each.value.user.password_special, false)
upper = true
lower = true
numeric = true
lifecycle {
ignore_changes = all
}
}
resource "yandex_mdb_redis_user" "this" {
for_each = local.valkey_users_map
cluster_id = each.value.cluster_id
name = each.value.user.name
passwords = [random_password.user_password[each.key].result]
enabled = try(each.value.user.enabled, true)
permissions = local.valkey_user_permissions[each.key]
lifecycle {
ignore_changes = [passwords]
}
}

29
modules/yc-valkey-user/outputs.tf Normal file
View File

@ -0,0 +1,29 @@
output "valkey_users" {
description = "Map of created Valkey/Redis users (cluster_id:user_name => data)"
value = {
for key, valkey_user in local.valkey_users_map : key => {
cluster_id = valkey_user.cluster_id
host = try(valkey_user.host, "")
port = tostring(try(valkey_user.port, "6380"))
user_name = yandex_mdb_redis_user.this[key].name
password = random_password.user_password[key].result
permissions = local.valkey_user_permissions[key]
user_enabled = try(valkey_user.user.enabled, true)
}
}
sensitive = true
}
output "valkey_outputs_map" {
description = "Map for secrets module (cluster_id:user_name => credentials)"
value = {
for key, valkey_user in local.valkey_users_map : key => {
cluster_id = valkey_user.cluster_id
host = try(valkey_user.host, "")
port = tostring(try(valkey_user.port, "6380"))
user_name = yandex_mdb_redis_user.this[key].name
password = random_password.user_password[key].result
}
}
sensitive = true
}

25
modules/yc-valkey-user/variables.tf Normal file
View File

@ -0,0 +1,25 @@
variable "valkey_users" {
description = "List of Valkey/Redis users to create"
type = list(object({
cluster_id = string
host = optional(string, "")
port = optional(string, "6380")
user = object({
name = string
password_length = optional(number, 32)
password_special = optional(bool, false)
enabled = optional(bool, true)
})
permissions = optional(object({
commands = optional(string)
categories = optional(string)
patterns = optional(string)
pub_sub_channels = optional(string)
pubSubChannels = optional(string)
sanitize_payload = optional(string)
sanitizePayload = optional(string)
databases = optional(list(number))
}), {})
}))
default = []
}

4
modules/yc-valkey-user/versions.tf Normal file
View File

@ -0,0 +1,4 @@
terraform {
required_version = ">= 1.0"
}

63
scripts/generate-pipeline.sh
View File

@ -35,16 +35,25 @@ find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sort | whi
job_prefix="${env}-${component}"
full_component_dir="${component_dir}"
secrets_dependency_components=()
needs_section=""
if [ "$component" = "secrets" ]; then
needs_section=" needs:
- validate-${env}-namespace
- validate-${env}-s3
- validate-${env}-database
dependencies:
- validate-${env}-namespace
- validate-${env}-s3
- validate-${env}-database"
for dep_component in namespace s3 database valkey-users; do
if [ -f "live/${env}/${dep_component}/terragrunt.hcl" ]; then
secrets_dependency_components+=("$dep_component")
fi
done
needs_section=" needs:"
dependencies_section=" dependencies:"
for dep_component in "${secrets_dependency_components[@]}"; do
needs_section="${needs_section}
- validate-${env}-${dep_component}"
dependencies_section="${dependencies_section}
- validate-${env}-${dep_component}"
done
needs_section="${needs_section}
${dependencies_section}"
fi
validate_before_script=" - cd \$TG_ROOT
@ -110,11 +119,20 @@ VALIDATE_JOB
plan_needs="validate-${job_prefix}"
plan_before_script=" - cd \$TG_ROOT && ls -la"
if [ "$component" = "secrets" ]; then
plan_needs="validate-${job_prefix}
- plan-${env}-namespace
- plan-${env}-s3
- plan-${env}-database"
plan_before_script=" - cd live/${env}/namespace && terragrunt init -upgrade -reconfigure && cd ../s3 && terragrunt init -upgrade -reconfigure && cd ../database && terragrunt init -upgrade -reconfigure && cd ../secrets && ls -la"
for dep_component in "${secrets_dependency_components[@]}"; do
plan_needs="${plan_needs}
- plan-${env}-${dep_component}"
done
dependency_init_chain=""
for dep_component in "${secrets_dependency_components[@]}"; do
if [ -z "$dependency_init_chain" ]; then
dependency_init_chain="cd live/${env}/${dep_component} && terragrunt init -upgrade -reconfigure"
else
dependency_init_chain="${dependency_init_chain} && cd ../${dep_component} && terragrunt init -upgrade -reconfigure"
fi
done
plan_before_script=" - ${dependency_init_chain} && cd ../secrets && ls -la"
fi
if [ "$component" = "kafka-topics" ]; then
plan_before_script=" - cd \$TG_ROOT && ls -la"
@ -153,11 +171,20 @@ PLAN_JOB
apply_needs="plan-${job_prefix}"
apply_before_script=" - cd \$TG_ROOT && ls -la"
if [ "$component" = "secrets" ]; then
apply_needs="plan-${job_prefix}
- apply-${env}-namespace
- apply-${env}-s3
- apply-${env}-database"
apply_before_script=" - cd live/${env}/namespace && terragrunt init -upgrade -reconfigure && cd ../s3 && terragrunt init -upgrade -reconfigure && cd ../database && terragrunt init -upgrade -reconfigure && cd ../secrets && ls -la"
for dep_component in "${secrets_dependency_components[@]}"; do
apply_needs="${apply_needs}
- apply-${env}-${dep_component}"
done
dependency_init_chain=""
for dep_component in "${secrets_dependency_components[@]}"; do
if [ -z "$dependency_init_chain" ]; then
dependency_init_chain="cd live/${env}/${dep_component} && terragrunt init -upgrade -reconfigure"
else
dependency_init_chain="${dependency_init_chain} && cd ../${dep_component} && terragrunt init -upgrade -reconfigure"
fi
done
apply_before_script=" - ${dependency_init_chain} && cd ../secrets && ls -la"
fi
if [ "$component" = "kafka-topics" ]; then
apply_before_script=" - cd \$TG_ROOT && ls -la"