diff --git a/Jenkinsfile b/Jenkinsfile index 4af148f..e411759 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,10 +9,9 @@ pipeline { environment { PYENVPIPELINE_VIRTUALENV = '1' GIT_SSH_COMMAND = 'ssh -o StrictHostKeyChecking=accept-new' - STAGING_AUDIT_HOST = 'root@welkombij.mandelblog.com' + STAGING_AUDIT_MINION = 'welkombij.mandelblog.com' STAGING_AUDIT_PROJECT_DIR = '/home/www-mandelstudio/mandelstudio' STAGING_AUDIT_MANAGE = '/var/lib/virtualenv/mandelstudio/bin/manage.py' - STAGING_AUDIT_SSH_CREDENTIALS_ID = 'staging-root-ssh' } stages { @@ -108,9 +107,7 @@ pipeline { deleteDir() checkout scm sh 'mkdir -p artifacts && chmod +x scripts/run_remote_multilingual_audit.sh' - withCredentials([sshUserPrivateKey(credentialsId: env.STAGING_AUDIT_SSH_CREDENTIALS_ID, keyFileVariable: 'STAGING_SSH_KEYFILE')]) { - sh './scripts/run_remote_multilingual_audit.sh' - } + sh './scripts/run_remote_multilingual_audit.sh' script { int status = sh(script: 'python3 scripts/multilingual_audit_ci.py --json artifacts/multilingual-audit.json', returnStatus: true) if (status == 2) { diff --git a/Jenkinsfile.multilingual-nightly b/Jenkinsfile.multilingual-nightly index 25da60f..b8b672f 100644 --- a/Jenkinsfile.multilingual-nightly +++ b/Jenkinsfile.multilingual-nightly @@ -10,10 +10,9 @@ pipeline { skipDefaultCheckout(true) } environment { - STAGING_AUDIT_HOST = 'root@welkombij.mandelblog.com' + STAGING_AUDIT_MINION = 'welkombij.mandelblog.com' STAGING_AUDIT_PROJECT_DIR = '/home/www-mandelstudio/mandelstudio' STAGING_AUDIT_MANAGE = '/var/lib/virtualenv/mandelstudio/bin/manage.py' - STAGING_AUDIT_SSH_CREDENTIALS_ID = 'staging-root-ssh' } stages { stage('Checkout') { @@ -42,9 +41,7 @@ pipeline { checkout scm sh 'mkdir -p artifacts && [ -f artifacts/multilingual-audit.json ] && cp artifacts/multilingual-audit.json artifacts/previous-multilingual-audit.json || true' sh 'chmod +x scripts/run_remote_multilingual_audit.sh' - withCredentials([sshUserPrivateKey(credentialsId: env.STAGING_AUDIT_SSH_CREDENTIALS_ID, keyFileVariable: 'STAGING_SSH_KEYFILE')]) { - sh './scripts/run_remote_multilingual_audit.sh' - } + sh './scripts/run_remote_multilingual_audit.sh' script { int status = sh(script: 'python3 scripts/multilingual_audit_ci.py --json artifacts/multilingual-audit.json --previous-json artifacts/previous-multilingual-audit.json', returnStatus: true) if (status == 2) { diff --git a/docs/CI_MULTILINGUAL_AUDIT.md b/docs/CI_MULTILINGUAL_AUDIT.md index 0edcc5d..dd4ea4a 100644 --- a/docs/CI_MULTILINGUAL_AUDIT.md +++ b/docs/CI_MULTILINGUAL_AUDIT.md @@ -55,18 +55,13 @@ The audit summary is interpreted as follows: This keeps deploys safe without making warning-level cleanup a hard blocker. -## Required Jenkins credential -Credential location: -- `Manage Jenkins -> Credentials -> System -> Global credentials` +## Jenkins requirements +No dedicated staging SSH credential is required for the multilingual audit stage. -Credential to add: -- `Kind`: `SSH Username with private key` -- `ID`: `staging-root-ssh` -- `Username`: `root` -- `Private key`: staging SSH key +The audit runs from the Jenkins built-in node through the same `serverpillar` / Salt transport used by staging deployment. Current implementation uses the following environment defaults: -- `STAGING_AUDIT_HOST=root@welkombij.mandelblog.com` +- `STAGING_AUDIT_MINION=welkombij.mandelblog.com` - `STAGING_AUDIT_PROJECT_DIR=/home/www-mandelstudio/mandelstudio` - `STAGING_AUDIT_MANAGE=/var/lib/virtualenv/mandelstudio/bin/manage.py` @@ -106,7 +101,7 @@ This happens when the remote audit times out or fails, and is intentional so Jen ## Local rerun To rerun the same remote audit flow locally: ```bash -export STAGING_AUDIT_HOST='root@welkombij.mandelblog.com' +export STAGING_AUDIT_MINION='welkombij.mandelblog.com' export STAGING_AUDIT_PROJECT_DIR='/home/www-mandelstudio/mandelstudio' export STAGING_AUDIT_MANAGE='/var/lib/virtualenv/mandelstudio/bin/manage.py' ./scripts/run_remote_multilingual_audit.sh diff --git a/scripts/run_remote_multilingual_audit.sh b/scripts/run_remote_multilingual_audit.sh index 90cfd32..4d458b3 100755 --- a/scripts/run_remote_multilingual_audit.sh +++ b/scripts/run_remote_multilingual_audit.sh @@ -1,72 +1,97 @@ #!/usr/bin/env bash set -euo pipefail -: "${STAGING_AUDIT_HOST:?STAGING_AUDIT_HOST is required}" +: "${STAGING_AUDIT_MINION:?STAGING_AUDIT_MINION is required}" : "${STAGING_AUDIT_PROJECT_DIR:?STAGING_AUDIT_PROJECT_DIR is required}" : "${STAGING_AUDIT_MANAGE:?STAGING_AUDIT_MANAGE is required}" -mkdir -p artifacts -SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=accept-new"} -if [[ -n "${STAGING_SSH_KEYFILE:-}" ]]; then - SSH_OPTS="$SSH_OPTS -i ${STAGING_SSH_KEYFILE}" -fi AUDIT_TIMEOUT_SECONDS=${AUDIT_TIMEOUT_SECONDS:-300} -OUT_FILE="artifacts/multilingual-audit.json" -TMP_FILE="${OUT_FILE}.tmp" - -write_failure_json() { - python3 - < "$OUT_FILE" -import json -print(json.dumps({ - "run_id": None, - "total_urls_checked": 0, - "issues_found": 0, - "summary": {}, - "issues": {}, - "error": ${1@Q} -}, indent=2)) -PY -} +ARTIFACT_DIR=${ARTIFACT_DIR:-artifacts} +OUTPUT_JSON=${OUTPUT_JSON:-${ARTIFACT_DIR}/multilingual-audit.json} +mkdir -p "${ARTIFACT_DIR}" +TMP_FILE=$(mktemp) +trap 'rm -f "$TMP_FILE"' EXIT REMOTE_CMD="cd '${STAGING_AUDIT_PROJECT_DIR}' && '${STAGING_AUDIT_MANAGE}' audit_locales --format=json" -set +e -SSH_OPTS="$SSH_OPTS" STAGING_AUDIT_HOST="$STAGING_AUDIT_HOST" REMOTE_CMD="$REMOTE_CMD" AUDIT_TIMEOUT_SECONDS="$AUDIT_TIMEOUT_SECONDS" python3 - <<'PY' > "$TMP_FILE" + +STAGING_AUDIT_MINION="$STAGING_AUDIT_MINION" REMOTE_CMD="$REMOTE_CMD" AUDIT_TIMEOUT_SECONDS="$AUDIT_TIMEOUT_SECONDS" python3 - <<'PY2' > "$TMP_FILE" +import json import os -import shlex import subprocess import sys -ssh_opts = shlex.split(os.environ["SSH_OPTS"]) -cmd = ["ssh", *ssh_opts, os.environ["STAGING_AUDIT_HOST"], os.environ["REMOTE_CMD"]] +minion = os.environ["STAGING_AUDIT_MINION"] +remote_cmd = os.environ["REMOTE_CMD"] +timeout_seconds = int(os.environ["AUDIT_TIMEOUT_SECONDS"]) +cmd = [ + "sudo", "-n", "-u", "mandel", "-g", "www-data", + "/usr/bin/salt", "--out=json", minion, + "cmd.run_all", remote_cmd, "python_shell=True", +] try: - proc = subprocess.run( + result = subprocess.run( cmd, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, text=True, - timeout=int(os.environ["AUDIT_TIMEOUT_SECONDS"]), + timeout=timeout_seconds, + check=False, ) - sys.stdout.write(proc.stdout) - sys.stderr.write(proc.stderr) -except subprocess.TimeoutExpired as exc: - sys.stderr.write(exc.stderr or "") - raise SystemExit(124) -except subprocess.CalledProcessError as exc: - sys.stdout.write(exc.stdout or "") - sys.stderr.write(exc.stderr or "") - raise SystemExit(exc.returncode) -PY -rc=$? -set -e -if [[ $rc -eq 0 ]]; then - mv "$TMP_FILE" "$OUT_FILE" - exit 0 -fi -rm -f "$TMP_FILE" -if [[ $rc -eq 124 ]]; then - write_failure_json "Remote multilingual audit timed out after ${AUDIT_TIMEOUT_SECONDS}s" -else - write_failure_json "Remote multilingual audit failed with exit status ${rc}" -fi -exit $rc +except subprocess.TimeoutExpired: + print(json.dumps({ + "run_id": None, + "total_urls_checked": 0, + "issues_found": 0, + "summary": {}, + "issues": {}, + "error": f"Salt multilingual audit timed out after {timeout_seconds} seconds", + }, indent=2)) + sys.exit(2) + +if result.returncode != 0: + print(json.dumps({ + "run_id": None, + "total_urls_checked": 0, + "issues_found": 0, + "summary": {}, + "issues": {}, + "error": f"Salt audit transport failed with exit status {result.returncode}: {(result.stderr or result.stdout).strip()}", + }, indent=2)) + sys.exit(2) + +try: + payload = json.loads(result.stdout) + if not isinstance(payload, dict) or minion not in payload: + raise ValueError("Missing minion payload") + minion_payload = payload[minion] + if not isinstance(minion_payload, dict): + raise ValueError("Unexpected minion payload type") + retcode = int(minion_payload.get("retcode", 1)) + stdout = minion_payload.get("stdout", "") + stderr = minion_payload.get("stderr", "") + if retcode != 0: + print(json.dumps({ + "run_id": None, + "total_urls_checked": 0, + "issues_found": 0, + "summary": {}, + "issues": {}, + "error": f"Remote multilingual audit failed with exit status {retcode}: {(stderr or stdout).strip()}", + }, indent=2)) + sys.exit(2) + audit = json.loads(stdout) +except Exception as exc: + print(json.dumps({ + "run_id": None, + "total_urls_checked": 0, + "issues_found": 0, + "summary": {}, + "issues": {}, + "error": f"Unable to parse salt audit response: {exc}", + }, indent=2)) + sys.exit(2) + +print(json.dumps(audit, indent=2, sort_keys=True)) +PY2 +status=$? +cp "$TMP_FILE" "$OUTPUT_JSON" +exit $status