From 858fc2f232ac5f9d90831cd13c0269559e235d47 Mon Sep 17 00:00:00 2001 From: Work Date: Wed, 28 Jan 2026 13:30:50 +0300 Subject: [PATCH] fixes --- .gitlab-ci.yml | 22 +++- Dockerfile | 9 +- live/stage/database/.terraform.lock.hcl | 26 ++++ live/stage/namespace/.terraform.lock.hcl | 26 ++++ live/stage/s3/.terraform.lock.hcl | 26 ++++ live/terragrunt.hcl | 2 +- scripts/generate-pipeline.sh | 148 +++++++++-------------- 7 files changed, 159 insertions(+), 100 deletions(-) create mode 100644 live/stage/database/.terraform.lock.hcl create mode 100644 live/stage/namespace/.terraform.lock.hcl create mode 100644 live/stage/s3/.terraform.lock.hcl diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5924945..b012216 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ stages: - trigger variables: - TERRAFORM_IMAGE_NAME: cr.yandex/crp3ccidau046kdj8g9q/terraform/terragrunt:v9.9 + TERRAFORM_IMAGE_NAME: cr.yandex/crp3ccidau046kdj8g9q/terraform/terragrunt:v9.10 generate-pipeline: stage: generate @@ -11,10 +11,13 @@ generate-pipeline: before_script: - apk add --no-cache bash findutils script: + - cd terraform + - chmod +x ./scripts/generate-pipeline.sh - ./scripts/generate-pipeline.sh + - cat .gitlab-ci.generated.yml artifacts: paths: - - .gitlab-ci.generated.yml + - terraform/.gitlab-ci.generated.yml expire_in: 1 hour rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -27,17 +30,24 @@ trigger-downstream: - generate-pipeline trigger: include: - - artifact: .gitlab-ci.generated.yml + - artifact: terraform/.gitlab-ci.generated.yml job: generate-pipeline + strategy: depend variables: + TERRAFORM_IMAGE_NAME: ${TERRAFORM_IMAGE_NAME} YC_IAM_TOKEN: ${YC_IAM_TOKEN} YC_TOKEN: ${YC_IAM_TOKEN} + YC_CLOUD_ID: ${YC_CLOUD_ID} YC_STAGE_FOLDER_ID: ${YC_STAGE_FOLDER_ID} + YC_PROD_FOLDER_ID: ${YC_PROD_FOLDER_ID} + YC_PREPROD_FOLDER_ID: ${YC_PREPROD_FOLDER_ID} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_STATE_BUCKET: ${S3_STATE_BUCKET} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + S3_STATE_BUCKET: ${S3_STATE_BUCKET} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + KUBECONFIG: ${KUBECONFIG} + KUBE_CONTEXT: ${KUBE_CONTEXT} rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main"' diff --git a/Dockerfile b/Dockerfile index b6417ad..5520c3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,20 @@ FROM alpine:latest -RUN apk add --no-cache curl unzip bash git +RUN apk add --no-cache curl unzip bash git jq +# Terraform ARG TERRAFORM_VERSION=1.5.7 RUN curl -fsSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o terraform.zip && \ unzip terraform.zip -d /usr/local/bin/ && \ chmod +x /usr/local/bin/terraform && \ rm -f terraform.zip +# Terragrunt ARG TERRAGRUNT_VERSION=0.93.11 RUN curl -fsSL https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64 -o /usr/local/bin/terragrunt && \ chmod +x /usr/local/bin/terragrunt +# Plugin cache и mirror для Yandex Cloud RUN mkdir -p /root/.terraform.d/plugin-cache && \ mkdir -p /root/.terraform.d && \ echo 'provider_installation {' > /root/.terraformrc && \ @@ -24,8 +27,10 @@ RUN mkdir -p /root/.terraform.d/plugin-cache && \ echo ' }' >> /root/.terraformrc && \ echo '}' >> /root/.terraformrc +# Environment variables ENV TF_PLUGIN_CACHE_DIR=/root/.terraform.d/plugin-cache - ENV TG_TF_PATH=/usr/local/bin/terraform +ENV AWS_EC2_METADATA_DISABLED=true +ENV TF_CLI_ARGS="-no-color" WORKDIR /workspace diff --git a/live/stage/database/.terraform.lock.hcl b/live/stage/database/.terraform.lock.hcl new file mode 100644 index 0000000..63698be --- /dev/null +++ b/live/stage/database/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.23" + hashes = [ + "h1:ems+O2dA7atxMWpbtqIrsH7Oa+u+ERWSfpMaFnZPbh0=", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.8.1" + constraints = "~> 3.1" + hashes = [ + "h1:LsYuJLZcYl1RiH7Hd3w90Ra5+k5cNqfdRUQXItkTI8Y=", + ] +} + +provider "registry.opentofu.org/yandex-cloud/yandex" { + version = "0.127.0" + constraints = "~> 0.100" + hashes = [ + "h1:tms24VWADoryHdkGdH5A76fWG2k+9CrE9Je9yZMXUoI=", + ] +} diff --git a/live/stage/namespace/.terraform.lock.hcl b/live/stage/namespace/.terraform.lock.hcl new file mode 100644 index 0000000..63698be --- /dev/null +++ b/live/stage/namespace/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.23" + hashes = [ + "h1:ems+O2dA7atxMWpbtqIrsH7Oa+u+ERWSfpMaFnZPbh0=", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.8.1" + constraints = "~> 3.1" + hashes = [ + "h1:LsYuJLZcYl1RiH7Hd3w90Ra5+k5cNqfdRUQXItkTI8Y=", + ] +} + +provider "registry.opentofu.org/yandex-cloud/yandex" { + version = "0.127.0" + constraints = "~> 0.100" + hashes = [ + "h1:tms24VWADoryHdkGdH5A76fWG2k+9CrE9Je9yZMXUoI=", + ] +} diff --git a/live/stage/s3/.terraform.lock.hcl b/live/stage/s3/.terraform.lock.hcl new file mode 100644 index 0000000..63698be --- /dev/null +++ b/live/stage/s3/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.23" + hashes = [ + "h1:ems+O2dA7atxMWpbtqIrsH7Oa+u+ERWSfpMaFnZPbh0=", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.8.1" + constraints = "~> 3.1" + hashes = [ + "h1:LsYuJLZcYl1RiH7Hd3w90Ra5+k5cNqfdRUQXItkTI8Y=", + ] +} + +provider "registry.opentofu.org/yandex-cloud/yandex" { + version = "0.127.0" + constraints = "~> 0.100" + hashes = [ + "h1:tms24VWADoryHdkGdH5A76fWG2k+9CrE9Je9yZMXUoI=", + ] +} diff --git a/live/terragrunt.hcl b/live/terragrunt.hcl index e03c29c..462bd07 100644 --- a/live/terragrunt.hcl +++ b/live/terragrunt.hcl @@ -14,7 +14,7 @@ remote_state { skip_region_validation = true skip_credentials_validation = true skip_metadata_api_check = true - use_path_style = true + force_path_style = true skip_bucket_root_access = true } } diff --git a/scripts/generate-pipeline.sh b/scripts/generate-pipeline.sh index c41a758..49fd5bc 100755 --- a/scripts/generate-pipeline.sh +++ b/scripts/generate-pipeline.sh @@ -3,12 +3,12 @@ set -e OUTPUT_FILE=".gitlab-ci.generated.yml" -yaml_escape() { - echo "$1" | sed 's/"/\\"/g' -} - -cat > "$OUTPUT_FILE" << 'EOF' +# Определяем базовую директорию terraform (где находится этот скрипт) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TERRAFORM_DIR="$(dirname "$SCRIPT_DIR")" +TERRAFORM_PREFIX="terraform" +cat > "$OUTPUT_FILE" << 'HEADER' workflow: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' @@ -21,27 +21,30 @@ variables: TERRAFORM_IMAGE_NAME: ${TERRAFORM_IMAGE_NAME} YC_IAM_TOKEN: ${YC_IAM_TOKEN} YC_TOKEN: ${YC_IAM_TOKEN} + YC_CLOUD_ID: ${YC_CLOUD_ID} YC_STAGE_FOLDER_ID: ${YC_STAGE_FOLDER_ID} YC_PROD_FOLDER_ID: ${YC_PROD_FOLDER_ID} YC_PREPROD_FOLDER_ID: ${YC_PREPROD_FOLDER_ID} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} S3_STATE_BUCKET: ${S3_STATE_BUCKET} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + AWS_EC2_METADATA_DISABLED: "true" + TF_CLI_ARGS: "-no-color" + TG_LOG_LEVEL: "error" stages: - validate - plan - apply -EOF +HEADER find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sort | while read -r config_file; do component_dir=$(dirname "$config_file") relative_path=$(echo "$component_dir" | sed 's|^live/||') - # live/{env}/{component}/terragrunt.hcl env=$(echo "$relative_path" | cut -d'/' -f1) component=$(echo "$relative_path" | cut -d'/' -f2) @@ -51,55 +54,41 @@ find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sort | whi job_prefix="${env}-${component}" + # Полный путь от корня репо + full_component_dir="${TERRAFORM_PREFIX}/${component_dir}" - needs_deps="" + needs_section="" if [ "$component" = "secrets" ]; then - needs_deps="validate-${env}-namespace validate-${env}-s3 validate-${env}-database" + needs_section=" needs: + - validate-${env}-namespace + - validate-${env}-s3 + - validate-${env}-database + dependencies: + - validate-${env}-namespace + - validate-${env}-s3 + - validate-${env}-database" fi case "$env" in - stage) - folder_var="YC_STAGE_FOLDER_ID" - ;; - prod) - folder_var="YC_PROD_FOLDER_ID" - ;; - preprod) - folder_var="YC_PREPROD_FOLDER_ID" - ;; - *) - folder_var="YC_FOLDER_ID" - ;; + stage) folder_var="YC_STAGE_FOLDER_ID" ;; + prod) folder_var="YC_PROD_FOLDER_ID" ;; + preprod) folder_var="YC_PREPROD_FOLDER_ID" ;; + *) folder_var="YC_FOLDER_ID" ;; esac - - cat >> "$OUTPUT_FILE" << EOF + + cat >> "$OUTPUT_FILE" << VALIDATE_JOB validate-${job_prefix}: stage: validate image: \${TERRAFORM_IMAGE_NAME} variables: - TG_ROOT: "${component_dir}" + TG_ROOT: "${full_component_dir}" ENVIRONMENT: "${env}" YC_FOLDER_ID: "\${${folder_var}}" - YC_TOKEN: "\${YC_IAM_TOKEN}" - S3_ACCESS_KEY: "\${S3_ACCESS_KEY}" - S3_SECRET_KEY: "\${S3_SECRET_KEY}" - S3_STATE_BUCKET: "\${S3_STATE_BUCKET}" - TG_TF_PATH: "/usr/local/bin/terraform" - AWS_ACCESS_KEY_ID: "\${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "\${AWS_SECRET_ACCESS_KEY}" before_script: - - echo "TG_ROOT=\${TG_ROOT}" - - echo "CI_PROJECT_DIR=\${CI_PROJECT_DIR}" - - export AWS_ACCESS_KEY_ID="\${AWS_ACCESS_KEY_ID}" - - export AWS_SECRET_ACCESS_KEY="\${AWS_SECRET_ACCESS_KEY}" - - export S3_ACCESS_KEY="\${S3_ACCESS_KEY}" - - export S3_SECRET_KEY="\${S3_SECRET_KEY}" - cd \${TG_ROOT} + - echo "Working directory:" && pwd - ls -la - - export TF_PLUGIN_CACHE_DIR=/root/.terraform.d/plugin-cache script: - - env | grep -E "(S3_|AWS_)" || echo "No S3/AWS vars found" - - rm -f .terraform.lock.hcl - terragrunt init -upgrade -reconfigure - terragrunt validate rules: @@ -110,46 +99,24 @@ validate-${job_prefix}: interruptible: true artifacts: paths: - - "\${TG_ROOT}/.terragrunt-cache/**/*" + - ${full_component_dir}/.terragrunt-cache/ expire_in: 1 hour when: always -EOF +${needs_section} - if [ -n "$needs_deps" ]; then - cat >> "$OUTPUT_FILE" << EOF - needs: -EOF - for dep in $needs_deps; do - echo " - ${dep}" >> "$OUTPUT_FILE" - done - cat >> "$OUTPUT_FILE" << EOF - dependencies: -EOF - for dep in $needs_deps; do - echo " - ${dep}" >> "$OUTPUT_FILE" - done - fi - - cat >> "$OUTPUT_FILE" << EOF +VALIDATE_JOB + cat >> "$OUTPUT_FILE" << PLAN_JOB plan-${job_prefix}: stage: plan image: \${TERRAFORM_IMAGE_NAME} variables: - TG_ROOT: "${component_dir}" + TG_ROOT: "${full_component_dir}" ENVIRONMENT: "${env}" YC_FOLDER_ID: "\${${folder_var}}" - YC_TOKEN: "\${YC_IAM_TOKEN}" - S3_ACCESS_KEY: "\${S3_ACCESS_KEY}" - S3_SECRET_KEY: "\${S3_SECRET_KEY}" - S3_STATE_BUCKET: "\${S3_STATE_BUCKET}" - TG_TF_PATH: "/usr/local/bin/terraform" - AWS_ACCESS_KEY_ID: "\${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "\${AWS_SECRET_ACCESS_KEY}" + needs: + - validate-${job_prefix} before_script: - - echo "${TG_ROOT}" - - export AWS_ACCESS_KEY_ID="\${AWS_ACCESS_KEY_ID}" - - export AWS_SECRET_ACCESS_KEY="\${AWS_SECRET_ACCESS_KEY}" - cd \${TG_ROOT} - ls -la script: @@ -163,12 +130,12 @@ plan-${job_prefix}: interruptible: true artifacts: paths: - - "\${TG_ROOT}/tfplan" - - "\${TG_ROOT}/.terragrunt-cache/**/*" + - ${full_component_dir}/tfplan + - ${full_component_dir}/.terragrunt-cache/ expire_in: 1 week when: always -EOF +PLAN_JOB if [ "$env" = "prod" ]; then when_clause="manual" @@ -176,30 +143,31 @@ EOF when_clause="on_success" fi - cat >> "$OUTPUT_FILE" << EOF + # Apply dependencies для secrets + apply_needs="plan-${job_prefix}" + if [ "$component" = "secrets" ]; then + apply_needs="plan-${job_prefix} + - apply-${env}-namespace + - apply-${env}-s3 + - apply-${env}-database" + fi + + cat >> "$OUTPUT_FILE" << APPLY_JOB apply-${job_prefix}: stage: apply image: \${TERRAFORM_IMAGE_NAME} variables: - TG_ROOT: "${component_dir}" + TG_ROOT: "${full_component_dir}" ENVIRONMENT: "${env}" YC_FOLDER_ID: "\${${folder_var}}" - YC_TOKEN: "\${YC_IAM_TOKEN}" - S3_ACCESS_KEY: "\${S3_ACCESS_KEY}" - S3_SECRET_KEY: "\${S3_SECRET_KEY}" - S3_STATE_BUCKET: "\${S3_STATE_BUCKET}" - TG_TF_PATH: "/usr/local/bin/terraform" - AWS_ACCESS_KEY_ID: "\${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "\${AWS_SECRET_ACCESS_KEY}" + needs: + - ${apply_needs} before_script: - - echo "${TG_ROOT}" - - export AWS_ACCESS_KEY_ID="\${AWS_ACCESS_KEY_ID}" - - export AWS_SECRET_ACCESS_KEY="\${AWS_SECRET_ACCESS_KEY}" - cd \${TG_ROOT} - ls -la script: - terragrunt init -upgrade -reconfigure - - terragrunt apply tfplan + - terragrunt apply -auto-approve rules: - if: '\$CI_COMMIT_BRANCH == "master" || \$CI_COMMIT_BRANCH == "main"' when: ${when_clause} @@ -208,13 +176,11 @@ apply-${job_prefix}: - if: '\$CI_PIPELINE_SOURCE == "pipeline"' when: ${when_clause} interruptible: true - dependencies: - - plan-${job_prefix} -EOF +APPLY_JOB done echo "Pipeline generated successfully: $OUTPUT_FILE" echo "Found components:" -find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sed 's|live/||; s|/terragrunt.hcl||' | sort +find live -name "terragrunt.hcl" -not -path "*/.terragrunt-cache/*" | sed 's|live/||; s|/terragrunt.hcl||' | grep '/' | sort