Kubernetes PVC 一直 Terminating？移除 finalizer 前先审计全部存储引用

finalizer 是待完成的约定，不是根因本身

删除 PVC 后，Kubernetes 会写入 deletionTimestamp；只有所有 finalizer 的职责完成，对象才真正消失。官方明确提醒：不了解用途、未完成职责时不要手工移除 finalizer。

存储保护可能对应真实引用。Pod 对象仍声明该 PVC 时，它仍属于使用中；绑定 PV 还有独立保护与回收生命周期。先保留关系，再考虑元数据。

先采集五类 API 视图

审计器读取目标 PVC、命名空间内全部 Pod、已绑定 PV、StorageClass 与 VolumeAttachment，记录 UID、resourceVersion、删除时间、finalizer、claimRef、回收策略和 CSI 标识。

读取失败必须算未解决证据。VolumeAttachment 被 RBAC 拒绝不能当成空列表，坏 JSON 也不能当作零引用。

保存并运行只读采集器

把下列完整程序保存为 audit_kubernetes_pvc.mjs。先运行 node --check audit_kubernetes_pvc.mjs，再运行 node audit_kubernetes_pvc.mjs 命名空间 PVC名 > pvc-before.json。参数只接受安全的小写 Kubernetes 名称。

#!/usr/bin/env node import { spawnSync } from 'node:child_process'; const SAFE_NAME = /^(?=.{1,253}$)[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$/; export function validateName(value, label) { if (typeof value !== 'string' || !SAFE_NAME.test(value) || value.startsWith('-')) { throw new Error(`${label} must be a safe lowercase Kubernetes name`); } } function summarizeObject(object) { if (!object) return null; return { apiVersion: object.apiVersion ?? null, kind: object.kind ?? null, name: object.metadata?.name ?? null, namespace: object.metadata?.namespace ?? null, uid: object.metadata?.uid ?? null, resourceVersion: object.metadata?.resourceVersion ?? null, deletionTimestamp: object.metadata?.deletionTimestamp ?? null, finalizers: object.metadata?.finalizers ?? [], }; } export function spawnKubectl(args) { const result = spawnSync('kubectl', args, { encoding: 'utf8', shell: false, timeout: 20_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true, }); return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '', error: result.error?.message ?? null, }; } function readJson(run, args, label, unresolved, { required = false } = {}) { const result = run(args); if (result.error || result.status !== 0) { unresolved.push({ check: label, reason: result.error ?? `kubectl exited ${result.status}`, stderr: String(result.stderr ?? '').trim().slice(0, 1000), }); if (required) throw new Error(`${label} could not be read`); return null; } try { return JSON.parse(result.stdout); } catch (error) { unresolved.push({ check: label, reason: `malformed JSON: ${error.message}` }); if (required) throw new Error(`${label} returned malformed JSON`); return null; } } export function auditPVC(namespace, pvcName, run = spawnKubectl) { validateName(namespace, 'namespace'); validateName(pvcName, 'PVC name'); const unresolved = []; const commands = []; const read = (args, label, options) => { commands.push(['kubectl', ...args]); return readJson(run, args, label, unresolved, options); }; const pvc = read(['get', 'pvc', pvcName, '-n', namespace, '-o', 'json'], 'PVC', { required: true }); const pods = read(['get', 'pods', '-n', namespace, '-o', 'json'], 'Pods'); const pvName = pvc.spec?.volumeName || null; const pv = pvName ? read(['get', 'pv', pvName, '-o', 'json'], 'PV') : null; const storageClassName = pvc.spec?.storageClassName || pv?.spec?.storageClassName || null; const storageClass = storageClassName ? read(['get', 'storageclass', storageClassName, '-o', 'json'], 'StorageClass') : null; const volumeAttachments = read( ['get', 'volumeattachments.storage.k8s.io', '-o', 'json'], 'VolumeAttachments', ); const podReferences = (pods?.items ?? []).flatMap((pod) => { const matchingVolumes = (pod.spec?.volumes ?? []) .filter((volume) => volume.persistentVolumeClaim?.claimName === pvcName) .map((volume) => ({ volumeName: volume.name, readOnly: Boolean(volume.persistentVolumeClaim?.readOnly) })); return matchingVolumes.length ? [{ ...summarizeObject(pod), phase: pod.status?.phase ?? null, nodeName: pod.spec?.nodeName ?? null, matchingVolumes, }] : []; }); const matchingAttachments = (volumeAttachments?.items ?? []) .filter((attachment) => attachment.spec?.source?.persistentVolumeName === pvName) .map((attachment) => ({ ...summarizeObject(attachment), attacher: attachment.spec?.attacher ?? null, nodeName: attachment.spec?.nodeName ?? null, attached: attachment.status?.attached ?? null, attachError: attachment.status?.attachError?.message ?? null, detachError: attachment.status?.detachError?.message ?? null, })); return { schemaVersion: 1, generatedAt: new Date().toISOString(), target: { namespace, pvcName, pvName, storageClassName }, safety: { mode: 'read-only', mutatingCommandsRun: false, notRun: ['patch', 'delete', 'replace', 'edit', 'scale', 'drain', 'detach', 'finalizer removal'], backendContacted: false, }, commands, pvc: { ...summarizeObject(pvc), phase: pvc.status?.phase ?? null, accessModes: pvc.spec?.accessModes ?? [], requestedStorage: pvc.spec?.resources?.requests?.storage ?? null, volumeName: pvName, storageClassName, }, pv: pv ? { ...summarizeObject(pv), phase: pv.status?.phase ?? null, reclaimPolicy: pv.spec?.persistentVolumeReclaimPolicy ?? null, storageClassName: pv.spec?.storageClassName ?? null, claimRef: pv.spec?.claimRef ? { namespace: pv.spec.claimRef.namespace ?? null, name: pv.spec.claimRef.name ?? null, uid: pv.spec.claimRef.uid ?? null, } : null, csi: pv.spec?.csi ? { driver: pv.spec.csi.driver ?? null, volumeHandle: pv.spec.csi.volumeHandle ?? null, fsType: pv.spec.csi.fsType ?? null, } : null, } : null, storageClass: storageClass ? { ...summarizeObject(storageClass), provisioner: storageClass.provisioner ?? null, reclaimPolicy: storageClass.reclaimPolicy ?? null, volumeBindingMode: storageClass.volumeBindingMode ?? null, allowVolumeExpansion: storageClass.allowVolumeExpansion ?? null, } : null, podReferences, volumeAttachments: matchingAttachments, unresolved, decisionBoundary: { stopBeforeMutation: true, reasons: [ ...(podReferences.length ? ['one or more Pods still reference the PVC'] : []), ...(matchingAttachments.length ? ['one or more VolumeAttachments still reference the PV'] : []), ...(unresolved.length ? ['one or more required API reads are unresolved'] : []), ...(pvc.metadata?.finalizers?.length ? ['PVC finalizers require controller-purpose investigation'] : []), ...(pv?.metadata?.finalizers?.length ? ['PV finalizers require controller-purpose investigation'] : []), ], }, }; } if (import.meta.url === `file://${process.argv[1]}`) { const [namespace, pvcName] = process.argv.slice(2); try { const report = auditPVC(namespace, pvcName); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); if (report.unresolved.length) process.exitCode = 2; } catch (error) { process.stderr.write(`${error.message}\n`); process.exitCode = 1; } }

程序通过 8 个合成场景，覆盖绑定卷、Pod 与附着引用、删除时间、无 PV、RBAC 拒绝、坏 JSON、非法名称与禁止变更动词；没有连接真实集群或存储后端。

先读 Pod 引用，再碰 PVC

报告只列出 spec.volumes 中确实指向目标 claim 的 Pod。上层工作流显示完成，不代表保存下来的对象不再引用 PVC；Argo 案例说明控制器对象也可能延长引用。

不要因为 Pod 出现在台账就直接删除。先识别 Deployment、StatefulSet 或工作流控制器，确认应用停机和数据策略；程序不会缩容、驱逐或删除。

分别追踪 PV、回收策略与 CSI 附着

把 Bound PV 的 claimRef 与 PVC UID 对照，并记录 StorageClass provisioner 和回收策略。Delete 或 Retain 只描述释放后的策略，不证明后端卷已经分离或删除。

VolumeAttachment 可显示 CSI attacher、节点和 attached 状态，但仍只是 API 证据。若不可读或出现 attach/detach 错误，应保留并转交 CSI 与存储供应商流程。

使用停止矩阵，不使用强删配方

只要 Pod 或 VolumeAttachment 仍有引用、任一关键读取未知、或 finalizer 的控制器职责不清楚，就应停止。评审只是选择下一位责任人，不是自动删 finalizer。

社区答案之所以冲突，是因为不同集群可能卡在工作负载、provisioner、附着或陈旧对象。照抄别人的 patch 可能抹掉定位失败控制器的唯一证据。

围绕一次获批变更保存前后台账

责任人批准一个最小动作后，保存旧 JSON、记录动作，再用相同程序生成新文件；比较 UID、resourceVersion、引用、finalizer 与附着状态。

对象从 API 消失并不证明数据安全、已分离或可恢复。应用数据与存储后端仍需分别按支持流程验证。