What Is QAOps? The Policy Behind the Buzzword

# Gating policy: the human-readable source of truth for what this pipeline is
# allowed to stop.
#
# One rule underneath every classification below: block on checks that are fast,
# deterministic, and whose failure means the change is wrong; report on checks
# that are slow, flaky, or whose failure means the change is merely suspicious.
#
# gate values:
#   blocks_merge  - a red result stops the pull request from merging
#   blocks_deploy - a red result stops the deploy, not the merge
#   reports_only  - a red result is a signal for a human, and stops nothing
#
# budget_seconds is a wall-clock ceiling, not an average. A check that routinely
# exceeds its budget is a policy problem, not a patience problem.

version: 1
owner: platform-quality
last_reviewed: 2026-08-20

checks:
  # --- blocks_merge -------------------------------------------------------
  - name: lint-and-format
    gate: blocks_merge
    owner: platform-quality
    budget_seconds: 60
    why: Fails fast and deterministically; a red result is never ambiguous.

  - name: type-check
    gate: blocks_merge
    owner: platform-quality
    budget_seconds: 90
    why: A type error is a defect, not a suggestion, and it is cheap to catch.

  - name: unit-tests
    gate: blocks_merge
    owner: application-teams
    budget_seconds: 180
    why: Deterministic, and isolates the exact break rather than hinting at it.

  - name: build
    gate: blocks_merge
    owner: platform-quality
    budget_seconds: 240
    why: Unmergeable code is not a judgment call.

  - name: secret-scanning
    gate: blocks_merge
    owner: security
    budget_seconds: 45
    why: A leaked credential is never marginal.

  - name: contract-tests
    gate: blocks_merge
    owner: api-platform
    budget_seconds: 120
    why: A schema break is deterministic and cheap to detect at this layer.

  - name: dependency-cve-audit
    gate: blocks_merge
    owner: security
    budget_seconds: 60
    why: A known-bad version is a fact, not an interpretation.

  # --- blocks_deploy ------------------------------------------------------
  - name: build-verification-suite
    gate: blocks_deploy
    owner: quality-engineering
    budget_seconds: 270
    why: >-
      Four flows plus one shared setup step, budgeted at four and a half minutes.
      Confirms this build is worth testing further. Needs a real build and a real
      environment, so it cannot sit on the merge gate.

  - name: performance-budget
    gate: blocks_deploy
    owner: web-performance
    budget_seconds: 420
    why: A performance regression should not reach users, but it needs a build to measure.

  # --- reports_only -------------------------------------------------------
  - name: unit-coverage-delta
    gate: reports_only
    owner: application-teams
    budget_seconds: 20
    why: A drop is a signal, not proof that the change is wrong.

  - name: full-regression-e2e
    gate: reports_only
    owner: quality-engineering
    budget_seconds: 1800
    why: >-
      Comprehensive, and far too slow to gate every merge. Promote to
      blocks_deploy only once its failures are reliably about the product.

  - name: visual-regression
    gate: reports_only
    owner: design-systems
    budget_seconds: 600
    why: Pixel diffs need a human reviewer, not a gate.

  - name: accessibility-scan
    gate: reports_only
    owner: design-systems
    budget_seconds: 300
    why: Flags candidates worth reviewing; false positives are common.

  - name: load-test
    gate: reports_only
    owner: platform-quality
    budget_seconds: 1800
    why: Too slow and too costly to run per commit.

  - name: bundle-size
    gate: reports_only
    owner: web-performance
    budget_seconds: 45
    why: Worth tracking on every pull request, rarely worth blocking on alone.

# Every gate needs a documented escape hatch, or it quietly becomes a
# suggestion. Records live in overrides/ and are validated by
# scripts/validate-override.js in CI.
overrides:
  directory: overrides
  approver_role: release-captain
  required_fields:
    - check
    - requested_by
    - reason
    - follow_up_issue
    - expires_on
  review_cadence: monthly
  review_rule: >-
    A check overridden more than twice in a review window is misclassified.
    Move it to a lower gate or fix what makes it fail for reasons unrelated to
    the product. The log should shrink over time, not grow.

name: Gating Policy

# The enforcement half of gating-policy.yml at the repository root. Every job
# below maps to one gate class in that file:
#   blocks_merge  -> a required job with no continue-on-error
#   blocks_deploy -> a required job the deploy job declares in `needs`
#   reports_only  -> continue-on-error: true, uploads a report, never blocks
#
# The `on` key is quoted so this file parses identically under YAML 1.1
# loaders (which read a bare `on` as the boolean true) and YAML 1.2 loaders.
"on":
  pull_request:
    branches: [main]
  push:
    branches: [main]

concurrency:
  group: gating-policy-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  NODE_VERSION: "20"

jobs:
  # ---------------------------------------------------------------------------
  # BLOCKS MERGE
  # Fast and deterministic. A red result here means the change is wrong, not
  # merely suspicious, so nothing in this job is allowed to soft-fail.
  # Budget: 10 minutes, matching the sum of the blocks_merge budgets in
  # gating-policy.yml.
  # ---------------------------------------------------------------------------
  fast-checks:
    name: Fast checks (blocks merge)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Lint and format
        run: npm run lint

      - name: Type check
        run: npm run typecheck

      - name: Unit tests
        run: npm test

      - name: Build
        run: npm run build

      - name: Secret scan
        uses: gitleaks/gitleaks-action@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Dependency and CVE audit
        run: npm audit --audit-level=high

  # ---------------------------------------------------------------------------
  # BLOCKS DEPLOY
  # Four flows, one shared setup, budgeted at 4m30s end to end. It answers a
  # narrower question than "did we break anything": is this build worth testing
  # further? It gates the deploy job, and only the deploy job.
  # ---------------------------------------------------------------------------
  build-verification:
    name: Build verification suite (blocks deploy)
    needs: fast-checks
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Install Playwright browsers
        run: npx playwright install --with-deps chromium

      - name: Run the build verification suite
        run: npx playwright test tests/bvt --reporter=line,html

      - name: Upload the build verification report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: build-verification-report
          path: playwright-report
          retention-days: 14
          if-no-files-found: warn

  # ---------------------------------------------------------------------------
  # REPORTS ONLY
  # Comprehensive but too slow to gate every merge. continue-on-error keeps a
  # red result from stopping the pipeline; the uploaded report is the deliverable.
  # ---------------------------------------------------------------------------
  full-regression:
    name: Full regression E2E (reports only)
    needs: fast-checks
    runs-on: ubuntu-latest
    continue-on-error: true
    timeout-minutes: 45
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Install Playwright browsers
        run: npx playwright install --with-deps

      - name: Run the full regression suite
        run: npx playwright test --reporter=line,html

      - name: Upload the regression report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: full-regression-report
          path: playwright-report
          retention-days: 14
          if-no-files-found: warn

  # ---------------------------------------------------------------------------
  # REPORTS ONLY
  # Pixel diffs need a human, not a gate. The report is the point.
  # ---------------------------------------------------------------------------
  visual-regression:
    name: Visual regression (reports only)
    needs: fast-checks
    runs-on: ubuntu-latest
    continue-on-error: true
    timeout-minutes: 20
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Install Playwright browsers
        run: npx playwright install --with-deps chromium

      - name: Run the visual regression suite
        run: npx playwright test tests/visual --reporter=line,html

      - name: Upload the visual diff report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: visual-regression-report
          path: playwright-report
          retention-days: 14
          if-no-files-found: warn

  # ---------------------------------------------------------------------------
  # DEPLOY
  # Gated behind build-verification, and behind main. The reports-only jobs are
  # deliberately absent from `needs`: a red visual diff must not hold a deploy.
  # ---------------------------------------------------------------------------
  deploy:
    name: Deploy
    needs: build-verification
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment:
      name: production
      url: https://example.com
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      - name: Deploy
        run: npm run deploy
        env:
          DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}

# One override record. Five fields, all required, no exceptions:
# drop any one of them and this stops being a record and becomes a bypass
# with a timestamp.
#
# `check` must match a check name in gating-policy.yml.

check: performance-budget
requested_by: r.okafor (release captain, on call 2026-08-17 to 2026-08-23)
reason: >-
  Lighthouse largest-contentful-paint regressed from 2.1s to 2.6s against a
  budget of 2.5s. The regression traces to the vendored charting bundle loading
  eagerly on the dashboard route, not to this change, which only touches the
  billing settings form. Confirmed by re-running the budget against main at the
  same commit depth, where it reports 2.6s as well. Shipping this deploy does
  not make the measured regression worse.
follow_up_issue: https://github.com/Autonoma-Tools/qaops/issues/42
expires_on: 2026-12-31

#!/usr/bin/env node
"use strict";

/**
 * Validates gating-policy override records.
 *
 * Usage:
 *   node scripts/validate-override.js <directory>
 *
 * Reads every .yml / .yaml file in <directory> and enforces the two rules that
 * keep an override a record instead of a bypass with a timestamp:
 *
 *   1. All five required fields are present and non-empty.
 *   2. expires_on parses as a date and is not already in the past.
 *
 * Exits 0 if every record passes, 1 if any record fails.
 */

const fs = require("node:fs");
const path = require("node:path");
const yaml = require("js-yaml");

const REQUIRED_FIELDS = [
  "check",
  "requested_by",
  "reason",
  "follow_up_issue",
  "expires_on",
];

function usage() {
  return "Usage: node scripts/validate-override.js <directory>";
}

/** Returns the .yml/.yaml files in dir, sorted, so output is deterministic. */
function listRecordFiles(dir) {
  return fs
    .readdirSync(dir, { withFileTypes: true })
    .filter((entry) => entry.isFile() && /\.ya?ml$/i.test(entry.name))
    .map((entry) => path.join(dir, entry.name))
    .sort();
}

/** True when a field is absent, null, or an empty/whitespace-only string. */
function isBlank(value) {
  if (value === undefined || value === null) return true;
  if (typeof value === "string") return value.trim() === "";
  return false;
}

/**
 * js-yaml resolves a bare `2026-12-31` to a Date via the core timestamp type,
 * but a quoted date arrives as a string, so both shapes are handled here.
 * Returns a Date, or null when the value is not a usable date.
 */
function parseExpiry(value) {
  if (value instanceof Date) {
    return Number.isNaN(value.getTime()) ? null : value;
  }
  if (typeof value !== "string") return null;

  const trimmed = value.trim();
  if (!/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return null;

  const parsed = new Date(trimmed);
  return Number.isNaN(parsed.getTime()) ? null : parsed;
}

/** Midnight-UTC epoch for a date, so comparison is day-granular. */
function toUtcDay(date) {
  return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
}

function formatDay(date) {
  return date.toISOString().slice(0, 10);
}

/** Returns an array of human-readable failure strings. Empty means valid. */
function validateRecord(record, today) {
  if (record === null || record === undefined) {
    return ["file is empty; expected one override record"];
  }
  if (typeof record !== "object" || Array.isArray(record)) {
    return ["expected a single YAML mapping of override fields"];
  }

  const failures = [];

  for (const field of REQUIRED_FIELDS) {
    if (isBlank(record[field])) {
      failures.push(`missing required field: ${field}`);
    }
  }

  if (!isBlank(record.expires_on)) {
    const expiry = parseExpiry(record.expires_on);
    if (expiry === null) {
      failures.push(
        `expires_on is not a parseable date: ${JSON.stringify(record.expires_on)}`
      );
    } else if (toUtcDay(expiry) < today) {
      failures.push(
        `expires_on is in the past: ${formatDay(expiry)} (today is ${formatDay(new Date(today))})`
      );
    }
  }

  return failures;
}

function main(argv) {
  const dir = argv[0];

  if (!dir || argv.length > 1) {
    console.error(usage());
    return 1;
  }

  let stats;
  try {
    stats = fs.statSync(dir);
  } catch (error) {
    console.error(`Cannot read ${dir}: ${error.message}`);
    return 1;
  }
  if (!stats.isDirectory()) {
    console.error(`Not a directory: ${dir}`);
    console.error(usage());
    return 1;
  }

  const files = listRecordFiles(dir);
  if (files.length === 0) {
    console.log(`No override records found in ${dir}. Nothing to validate.`);
    return 0;
  }

  const today = toUtcDay(new Date());
  let failed = 0;

  for (const file of files) {
    let record;
    try {
      record = yaml.load(fs.readFileSync(file, "utf8"));
    } catch (error) {
      failed += 1;
      console.log(`FAIL ${file}`);
      console.log(`  - YAML did not parse: ${error.message.split("\n")[0]}`);
      continue;
    }

    const failures = validateRecord(record, today);
    if (failures.length === 0) {
      console.log(`PASS ${file}`);
    } else {
      failed += 1;
      console.log(`FAIL ${file}`);
      for (const failure of failures) {
        console.log(`  - ${failure}`);
      }
    }
  }

  console.log("");
  console.log(
    `${files.length - failed} of ${files.length} override record(s) passed.`
  );

  if (failed > 0) {
    console.log(
      "An override missing a field, or past its expiry, is a bypass rather than a record."
    );
    return 1;
  }
  return 0;
}

process.exit(main(process.argv.slice(2)));