Kubernetes PVC 一直 Terminating？移除 finalizer 前先稽核所有儲存引用

finalizer 是待履行的約定，不是根因

刪除 PVC 後，Kubernetes 寫入 deletionTimestamp；所有 finalizer 完成後物件才消失。官方提醒，不了解目的或尚未完成職責時不要手動移除。

Pod 物件仍宣告 PVC 時仍屬使用中，綁定 PV 也有自己的保護與回收週期。先保存關係，再處理中繼資料。

先收集五種 API 檢視

稽核器讀取 PVC、命名空間內所有 Pod、綁定 PV、StorageClass 與 VolumeAttachment，記錄 UID、resourceVersion、刪除時間、finalizer、claimRef、回收策略與 CSI 識別。

讀取失敗是未解決證據。RBAC 拒絕不能變成空清單，錯誤 JSON 也不能視為零引用。

儲存並執行唯讀收集器

把完整程式儲存為 audit_kubernetes_pvc.mjs；先用 node --check 檢查，再執行 node audit_kubernetes_pvc.mjs 命名空間 PVC名 > pvc-before.json。

#!/usr/bin/env node import { spawnSync } from 'node:child_process'; const SAFE_NAME = /^(?=.{1,253}$)[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$/; export function validateName(value, label) { if (typeof value !== 'string' || !SAFE_NAME.test(value) || value.startsWith('-')) { throw new Error(`${label} must be a safe lowercase Kubernetes name`); } } function summarizeObject(object) { if (!object) return null; return { apiVersion: object.apiVersion ?? null, kind: object.kind ?? null, name: object.metadata?.name ?? null, namespace: object.metadata?.namespace ?? null, uid: object.metadata?.uid ?? null, resourceVersion: object.metadata?.resourceVersion ?? null, deletionTimestamp: object.metadata?.deletionTimestamp ?? null, finalizers: object.metadata?.finalizers ?? [], }; } export function spawnKubectl(args) { const result = spawnSync('kubectl', args, { encoding: 'utf8', shell: false, timeout: 20_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true, }); return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '', error: result.error?.message ?? null, }; } function readJson(run, args, label, unresolved, { required = false } = {}) { const result = run(args); if (result.error || result.status !== 0) { unresolved.push({ check: label, reason: result.error ?? `kubectl exited ${result.status}`, stderr: String(result.stderr ?? '').trim().slice(0, 1000), }); if (required) throw new Error(`${label} could not be read`); return null; } try { return JSON.parse(result.stdout); } catch (error) { unresolved.push({ check: label, reason: `malformed JSON: ${error.message}` }); if (required) throw new Error(`${label} returned malformed JSON`); return null; } } export function auditPVC(namespace, pvcName, run = spawnKubectl) { validateName(namespace, 'namespace'); validateName(pvcName, 'PVC name'); const unresolved = []; const commands = []; const read = (args, label, options) => { commands.push(['kubectl', ...args]); return readJson(run, args, label, unresolved, options); }; const pvc = read(['get', 'pvc', pvcName, '-n', namespace, '-o', 'json'], 'PVC', { required: true }); const pods = read(['get', 'pods', '-n', namespace, '-o', 'json'], 'Pods'); const pvName = pvc.spec?.volumeName || null; const pv = pvName ? read(['get', 'pv', pvName, '-o', 'json'], 'PV') : null; const storageClassName = pvc.spec?.storageClassName || pv?.spec?.storageClassName || null; const storageClass = storageClassName ? read(['get', 'storageclass', storageClassName, '-o', 'json'], 'StorageClass') : null; const volumeAttachments = read( ['get', 'volumeattachments.storage.k8s.io', '-o', 'json'], 'VolumeAttachments', ); const podReferences = (pods?.items ?? []).flatMap((pod) => { const matchingVolumes = (pod.spec?.volumes ?? []) .filter((volume) => volume.persistentVolumeClaim?.claimName === pvcName) .map((volume) => ({ volumeName: volume.name, readOnly: Boolean(volume.persistentVolumeClaim?.readOnly) })); return matchingVolumes.length ? [{ ...summarizeObject(pod), phase: pod.status?.phase ?? null, nodeName: pod.spec?.nodeName ?? null, matchingVolumes, }] : []; }); const matchingAttachments = (volumeAttachments?.items ?? []) .filter((attachment) => attachment.spec?.source?.persistentVolumeName === pvName) .map((attachment) => ({ ...summarizeObject(attachment), attacher: attachment.spec?.attacher ?? null, nodeName: attachment.spec?.nodeName ?? null, attached: attachment.status?.attached ?? null, attachError: attachment.status?.attachError?.message ?? null, detachError: attachment.status?.detachError?.message ?? null, })); return { schemaVersion: 1, generatedAt: new Date().toISOString(), target: { namespace, pvcName, pvName, storageClassName }, safety: { mode: 'read-only', mutatingCommandsRun: false, notRun: ['patch', 'delete', 'replace', 'edit', 'scale', 'drain', 'detach', 'finalizer removal'], backendContacted: false, }, commands, pvc: { ...summarizeObject(pvc), phase: pvc.status?.phase ?? null, accessModes: pvc.spec?.accessModes ?? [], requestedStorage: pvc.spec?.resources?.requests?.storage ?? null, volumeName: pvName, storageClassName, }, pv: pv ? { ...summarizeObject(pv), phase: pv.status?.phase ?? null, reclaimPolicy: pv.spec?.persistentVolumeReclaimPolicy ?? null, storageClassName: pv.spec?.storageClassName ?? null, claimRef: pv.spec?.claimRef ? { namespace: pv.spec.claimRef.namespace ?? null, name: pv.spec.claimRef.name ?? null, uid: pv.spec.claimRef.uid ?? null, } : null, csi: pv.spec?.csi ? { driver: pv.spec.csi.driver ?? null, volumeHandle: pv.spec.csi.volumeHandle ?? null, fsType: pv.spec.csi.fsType ?? null, } : null, } : null, storageClass: storageClass ? { ...summarizeObject(storageClass), provisioner: storageClass.provisioner ?? null, reclaimPolicy: storageClass.reclaimPolicy ?? null, volumeBindingMode: storageClass.volumeBindingMode ?? null, allowVolumeExpansion: storageClass.allowVolumeExpansion ?? null, } : null, podReferences, volumeAttachments: matchingAttachments, unresolved, decisionBoundary: { stopBeforeMutation: true, reasons: [ ...(podReferences.length ? ['one or more Pods still reference the PVC'] : []), ...(matchingAttachments.length ? ['one or more VolumeAttachments still reference the PV'] : []), ...(unresolved.length ? ['one or more required API reads are unresolved'] : []), ...(pvc.metadata?.finalizers?.length ? ['PVC finalizers require controller-purpose investigation'] : []), ...(pv?.metadata?.finalizers?.length ? ['PV finalizers require controller-purpose investigation'] : []), ], }, }; } if (import.meta.url === `file://${process.argv[1]}`) { const [namespace, pvcName] = process.argv.slice(2); try { const report = auditPVC(namespace, pvcName); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); if (report.unresolved.length) process.exitCode = 2; } catch (error) { process.stderr.write(`${error.message}\n`); process.exitCode = 1; } }

程式通過 8 個合成情境，涵蓋 Pod、PV、附著、無 PV、RBAC 拒絕、錯誤 JSON、不安全名稱與禁止變更動詞；沒有連線真實叢集。

先看 Pod 引用

報告只列出 spec.volumes 確實指向目標 claim 的 Pod。上層工作流程完成，不代表保留物件已停止引用。

不要因為 Pod 出現在台帳就直接刪除；先找出控制器、應用程式負責人和資料策略。

分開追蹤 PV、回收策略與 CSI

比較 Bound PV 的 claimRef 與 PVC UID，並記錄 StorageClass provisioner 與回收策略。Delete 或 Retain 不證明後端已刪除。

VolumeAttachment 是重要 API 訊號但不是後端直接證據。不可讀或錯誤時，應轉交 CSI 與供應商支援流程。

使用停止矩陣，而非強刪配方

Pod 或附著仍引用、關鍵讀取未知、finalizer 責任不明時都應停止。評審代表找出下一位責任人，不代表自動移除。

不同叢集可能卡在工作負載、provisioner、附著或舊物件；照抄 patch 會抹去辨識失敗控制器的證據。

保存一次獲准變更前後的台帳

責任人批准最小動作後，保留舊 JSON、記錄動作，再產生新檔並比較 UID、resourceVersion、引用、finalizer 與附著狀態。

物件消失不證明資料安全、已分離或可復原；應用程式與後端須另行驗證。