add kafka+minio

clusters/yc-k8s-test/infrastructure/bootstrap-jobs/service-bootstrap-jobs.yaml

@@ -148,11 +148,24 @@ spec:
                     app_json="$(curl -sS -H "X-Vault-Token: ${VAULT_TOKEN}" "${VAULT_ADDR}/v1/secrets/data/minio/apps/${app}")"
                     username="$(echo "${app_json}" | jq -r '.data.data.username')"
                     password="$(echo "${app_json}" | jq -r '.data.data.password')"
+                    access_key="$(echo "${app_json}" | jq -r '.data.data.access_key // .data.data.username // empty')"
+                    secret_key="$(echo "${app_json}" | jq -r '.data.data.secret_key // .data.data.password // empty')"
+                    policy="$(echo "${app_json}" | jq -r '.data.data.policy // empty')"
                     [ -z "${username}" ] && username="${app}"
-                    [ -z "${password}" ] && continue
+                    [ -z "${access_key}" ] && access_key="${username}"
+                    [ -z "${secret_key}" ] && secret_key="${password}"
+                    [ -z "${secret_key}" ] && continue
 
-                    /usr/local/bin/mc admin user add local "${username}" "${password}" >/dev/null 2>&1 || true
+                    /usr/local/bin/mc admin user add local "${access_key}" "${secret_key}" >/dev/null 2>&1 || true
-                    /usr/local/bin/mc admin user enable local "${username}" >/dev/null 2>&1 || true
+                    /usr/local/bin/mc admin user enable local "${access_key}" >/dev/null 2>&1 || true
+                    if [ -n "${policy}" ]; then
+                      /usr/local/bin/mc admin policy attach local "${policy}" --user "${access_key}" >/dev/null 2>&1 || true
+                    fi
+
+                    echo "${app_json}" | jq -r '.data.data.buckets[]? | if type=="string" then . else .name // empty end' | while read -r bucket; do
+                      [ -z "${bucket}" ] && continue
+                      /usr/local/bin/mc mb --ignore-existing "local/${bucket}" >/dev/null 2>&1 || true
+                    done
                   done
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -249,12 +262,32 @@ spec:
 
                     kubectl -n kafka exec "${target_pod}" -c kafka -- /bin/bash -lc "\
                       cat >/tmp/admin.properties <<EOF
-                      security.protocol=SASL_SSL
+                      security.protocol=SASL_PLAINTEXT
                       sasl.mechanism=PLAIN
                       sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='inter_broker_user' password='${inter_broker_password}';
                       EOF
-                      kafka-configs.sh --bootstrap-server localhost:9092 --command-config /tmp/admin.properties \
+                      kafka-configs.sh --bootstrap-server localhost:9094 --command-config /tmp/admin.properties \
                         --alter --add-config 'SCRAM-SHA-512=[password=${password}],SCRAM-SHA-256=[password=${password}]' \
                         --entity-type users --entity-name '${username}'
                     " >/dev/null
+
+                    echo "${app_json}" | jq -c '.data.data.topics[]?' | while read -r topic_item; do
+                      topic_name="$(echo "${topic_item}" | jq -r '.name // empty')"
+                      partitions="$(echo "${topic_item}" | jq -r '.partitions // 3')"
+                      replication_factor="$(echo "${topic_item}" | jq -r '.replication_factor // 1')"
+                      topic_configs="$(echo "${topic_item}" | jq -r '(.configs // {}) | to_entries | map("\(.key)=\(.value|tostring)") | join(",")')"
+                      [ -z "${topic_name}" ] && continue
+
+                      kubectl -n kafka exec "${target_pod}" -c kafka -- /bin/bash -lc "\
+                        kafka-topics.sh --bootstrap-server localhost:9094 --command-config /tmp/admin.properties \
+                          --create --if-not-exists --topic '${topic_name}' --partitions '${partitions}' --replication-factor '${replication_factor}'
+                      " >/dev/null
+
+                      if [ -n "${topic_configs}" ]; then
+                        kubectl -n kafka exec "${target_pod}" -c kafka -- /bin/bash -lc "\
+                          kafka-configs.sh --bootstrap-server localhost:9094 --command-config /tmp/admin.properties \
+                            --alter --entity-type topics --entity-name '${topic_name}' --add-config '${topic_configs}'
+                        " >/dev/null
+                      fi
+                    done
                   done