.github/scripts/functional_container_test.sh93 of 93 statements covered (100.00%).
coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.
| 1 | 11 | #!/usr/bin/env bash |
| 2 | # ============================================================================= | |
| 3 | # functional_container_test.sh — boot a distroless service image under the | |
| 4 | # deployment manifests' pod-equivalent runtime constraints and assert its | |
| 5 | # serving contract. | |
| 6 | # ============================================================================= | |
| 7 | # | |
| 8 | # What it asserts, in order: | |
| 9 | # 1. The container boots to a serving state under the same constraints the | |
| 10 | # pod securityContext enforces: read-only root filesystem, tmpfs /tmp, | |
| 11 | # uid:gid 1000:1000, all capabilities dropped, no-new-privileges. | |
| 12 | # 2. Each --probe "path=code[=body-substring]" returns the expected HTTP | |
| 13 | # status (and body substring when given) — covering liveness/readiness | |
| 14 | # endpoints, auth fail-closed 503s, and degraded-dependency 503s. | |
| 15 | # 3. Each --exec-python payload runs inside the live container via | |
| 16 | # `docker exec` — the same mechanism kubelet uses for exec probes and | |
| 17 | # preStop hooks. The images ship no shell, so python is the only | |
| 18 | # executable; this proves the manifests' ["python", "-c", ...] command | |
| 19 | # shapes actually work against the running distroless container. | |
| 20 | # 4. Optional --min-uptime N: the service must still be serving N seconds | |
| 21 | # after boot (guards "boots then crash-loops on unreachable deps"). | |
| 22 | # 5. `docker stop` (SIGTERM, then SIGKILL after --stop-timeout) exits with | |
| 23 | # --expect-stop-exit: 0 for the uvicorn services' graceful shutdown, | |
| 24 | # 143 for inference-monitor which has no SIGTERM handler today. | |
| 25 | # | |
| 26 | # Kubernetes-dependent behavior (RBAC, reconciliation, NetworkPolicy) is out | |
| 27 | # of scope here — integration:kind:cluster-e2e owns it. This script owns the | |
| 28 | # container-level contract that kind deliberately does not probe. | |
| 29 | # | |
| 30 | # Usage: | |
| 31 | # functional_container_test.sh --image svc:ci --name svc-fn --host-port 18080 \ | |
| 32 | # [--container-port 8080] [--wait-path /healthz] [--env K=V]... \ | |
| 33 | # [--kubeconfig FILE] [--probe "path=code[=substring]"]... \ | |
| 34 | # [--exec-python "code"]... [--min-uptime N] \ | |
| 35 | # [--expect-stop-exit 0] [--stop-timeout 30] [--boot-timeout 60] | |
| 36 | # ============================================================================= | |
| 37 | ||
| 38 | 13 | set -euo pipefail |
| 39 | ||
| 40 | 13 | IMAGE="" |
| 41 | 13 | NAME="" |
| 42 | 13 | HOST_PORT="" |
| 43 | 13 | CONTAINER_PORT="8080" |
| 44 | 13 | WAIT_PATH="/healthz" |
| 45 | 13 | KUBECONFIG_FILE="" |
| 46 | 13 | MIN_UPTIME="0" |
| 47 | 13 | EXPECT_STOP_EXIT="0" |
| 48 | 13 | STOP_TIMEOUT="30" |
| 49 | 13 | BOOT_TIMEOUT="60" |
| 50 | 13 | ENVS=() |
| 51 | 13 | PROBES=() |
| 52 | 13 | EXEC_SNIPPETS=() |
| 53 | ||
| 54 | 69 | while [ $# -gt 0 ]; do |
| 55 | 57 | case "$1" in |
| 56 | 26 | --image) IMAGE="$2"; shift 2 ;; |
| 57 | 26 | --name) NAME="$2"; shift 2 ;; |
| 58 | 24 | --host-port) HOST_PORT="$2"; shift 2 ;; |
| 59 | 2 | --container-port) CONTAINER_PORT="$2"; shift 2 ;; |
| 60 | 2 | --wait-path) WAIT_PATH="$2"; shift 2 ;; |
| 61 | 4 | --env) ENVS+=("$2"); shift 2 ;; |
| 62 | 2 | --kubeconfig) KUBECONFIG_FILE="$2"; shift 2 ;; |
| 63 | 8 | --probe) PROBES+=("$2"); shift 2 ;; |
| 64 | 4 | --exec-python) EXEC_SNIPPETS+=("$2"); shift 2 ;; |
| 65 | 6 | --min-uptime) MIN_UPTIME="$2"; shift 2 ;; |
| 66 | 4 | --expect-stop-exit) EXPECT_STOP_EXIT="$2"; shift 2 ;; |
| 67 | 2 | --stop-timeout) STOP_TIMEOUT="$2"; shift 2 ;; |
| 68 | 2 | --boot-timeout) BOOT_TIMEOUT="$2"; shift 2 ;; |
| 69 | 2 | *) echo "unknown argument: $1" >&2; exit 2 ;; |
| 70 | esac | |
| 71 | done | |
| 72 | ||
| 73 | 36 | if [ -z "${IMAGE}" ] || [ -z "${NAME}" ] || [ -z "${HOST_PORT}" ]; then |
| 74 | 1 | echo "required: --image, --name, --host-port" >&2 |
| 75 | 1 | exit 2 |
| 76 | fi | |
| 77 | ||
| 78 | fail() { | |
| 79 | 8 | echo "::error::${NAME}: $1" |
| 80 | 8 | echo "===== ${NAME}: container logs =====" |
| 81 | 8 | docker logs "${NAME}" 2>&1 || true |
| 82 | 8 | exit 1 |
| 83 | } | |
| 84 | ||
| 85 | 11 | cleanup() { docker rm -f "${NAME}" >/dev/null 2>&1 || true; } |
| 86 | 11 | trap cleanup EXIT |
| 87 | ||
| 88 | # Pod-equivalent runtime constraints, mirroring the container securityContext | |
| 89 | # in lambda/kubectl-applier-simple/manifests/3*.yaml: readOnlyRootFilesystem, | |
| 90 | # runAsUser/runAsGroup 1000, capabilities drop ALL, allowPrivilegeEscalation | |
| 91 | # false, and the /tmp emptyDir the manifests mount. | |
| 92 | 11 | run_args=( |
| 93 | -d --name "${NAME}" | |
| 94 | --read-only | |
| 95 | --tmpfs "/tmp:rw,size=64m,mode=1777" | |
| 96 | --user 1000:1000 | |
| 97 | --cap-drop ALL | |
| 98 | --security-opt no-new-privileges | |
| 99 | -p "127.0.0.1:${HOST_PORT}:${CONTAINER_PORT}" | |
| 100 | ) | |
| 101 | 2 | for pair in ${ENVS[@]+"${ENVS[@]}"}; do |
| 102 | 2 | run_args+=(-e "${pair}") |
| 103 | done | |
| 104 | 11 | if [ -n "${KUBECONFIG_FILE}" ]; then |
| 105 | # A static kubeconfig pointing at an unreachable apiserver: the kubernetes | |
| 106 | # client parses it at startup without connecting, which is exactly what | |
| 107 | # lets these services reach their serving state outside a cluster. | |
| 108 | 1 | run_args+=(-v "${KUBECONFIG_FILE}:/kubeconfig:ro" -e KUBECONFIG=/kubeconfig) |
| 109 | fi | |
| 110 | ||
| 111 | 11 | docker run "${run_args[@]}" "${IMAGE}" |
| 112 | ||
| 113 | # --- 1. Wait for the serving state -------------------------------------- | |
| 114 | 11 | base_url="http://127.0.0.1:${HOST_PORT}" |
| 115 | 11 | deadline=$((SECONDS + BOOT_TIMEOUT)) |
| 116 | 13 | while :; do |
| 117 | 30 | code="$(curl -s -o /dev/null -w '%{http_code}' "${base_url}${WAIT_PATH}" || true)" |
| 118 | 22 | [ "${code}" = "200" ] && break |
| 119 | 8 | if [ "$(docker inspect -f '{{.State.Running}}' "${NAME}" 2>/dev/null)" != "true" ]; then |
| 120 | 1 | fail "container exited during startup (last ${WAIT_PATH} code: ${code})" |
| 121 | fi | |
| 122 | 4 | [ "${SECONDS}" -ge "${deadline}" ] && fail "${WAIT_PATH} never returned 200 within ${BOOT_TIMEOUT}s (last: ${code})" |
| 123 | 2 | sleep 2 |
| 124 | done | |
| 125 | 9 | echo "${NAME}: serving (${WAIT_PATH} -> 200)" |
| 126 | ||
| 127 | # --- 2. HTTP contract probes --------------------------------------------- | |
| 128 | 4 | for spec in ${PROBES[@]+"${PROBES[@]}"}; do |
| 129 | 4 | path="${spec%%=*}" |
| 130 | 4 | rest="${spec#*=}" |
| 131 | 4 | expected_code="${rest%%=*}" |
| 132 | 4 | expected_body="" |
| 133 | 6 | [ "${rest}" != "${expected_code}" ] && expected_body="${rest#*=}" |
| 134 | ||
| 135 | 8 | body_file="$(mktemp)" |
| 136 | 8 | actual_code="$(curl -s -o "${body_file}" -w '%{http_code}' "${base_url}${path}" || true)" |
| 137 | 4 | if [ "${actual_code}" != "${expected_code}" ]; then |
| 138 | 2 | echo "response body: $(cat "${body_file}")" |
| 139 | 1 | rm -f "${body_file}" |
| 140 | 1 | fail "GET ${path}: expected HTTP ${expected_code}, got ${actual_code}" |
| 141 | fi | |
| 142 | 5 | if [ -n "${expected_body}" ] && ! grep -q "${expected_body}" "${body_file}"; then |
| 143 | 2 | echo "response body: $(cat "${body_file}")" |
| 144 | 1 | rm -f "${body_file}" |
| 145 | 1 | fail "GET ${path}: body does not contain '${expected_body}'" |
| 146 | fi | |
| 147 | 2 | rm -f "${body_file}" |
| 148 | 2 | if [ -n "${expected_body}" ]; then |
| 149 | 1 | echo "${NAME}: GET ${path} -> ${actual_code} (body matches \"${expected_body}\") OK" |
| 150 | else | |
| 151 | 1 | echo "${NAME}: GET ${path} -> ${actual_code} OK" |
| 152 | fi | |
| 153 | done | |
| 154 | ||
| 155 | # --- 3. Kubelet exec-command shapes (probes / preStop hooks) -------------- | |
| 156 | 2 | for snippet in ${EXEC_SNIPPETS[@]+"${EXEC_SNIPPETS[@]}"}; do |
| 157 | 2 | if ! docker exec "${NAME}" python -c "${snippet}"; then |
| 158 | 1 | fail "exec command failed in live container: python -c '${snippet}'" |
| 159 | fi | |
| 160 | 1 | echo "${NAME}: exec python -c '${snippet}' OK" |
| 161 | done | |
| 162 | ||
| 163 | # --- 4. Stability under unreachable dependencies -------------------------- | |
| 164 | 6 | if [ "${MIN_UPTIME}" -gt 0 ]; then |
| 165 | 3 | sleep "${MIN_UPTIME}" |
| 166 | 6 | if [ "$(docker inspect -f '{{.State.Running}}' "${NAME}")" != "true" ]; then |
| 167 | 1 | fail "container died within ${MIN_UPTIME}s of becoming healthy" |
| 168 | fi | |
| 169 | 4 | code="$(curl -s -o /dev/null -w '%{http_code}' "${base_url}${WAIT_PATH}" || true)" |
| 170 | 3 | [ "${code}" = "200" ] || fail "${WAIT_PATH} degraded to ${code} after ${MIN_UPTIME}s" |
| 171 | 1 | echo "${NAME}: still serving after ${MIN_UPTIME}s with unreachable dependencies OK" |
| 172 | fi | |
| 173 | ||
| 174 | # --- 5. Shutdown contract -------------------------------------------------- | |
| 175 | 4 | docker stop -t "${STOP_TIMEOUT}" "${NAME}" >/dev/null |
| 176 | 8 | exit_code="$(docker inspect -f '{{.State.ExitCode}}' "${NAME}")" |
| 177 | 4 | if [ "${exit_code}" != "${EXPECT_STOP_EXIT}" ]; then |
| 178 | 1 | fail "SIGTERM shutdown: expected exit ${EXPECT_STOP_EXIT}, got ${exit_code}" |
| 179 | fi | |
| 180 | 3 | echo "${NAME}: SIGTERM shutdown exited ${exit_code} OK" |
| 181 | 3 | echo "${NAME}: functional container test PASSED" |