diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index c6160c9f..5b8694bd 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -143,7 +143,7 @@ namespace KubernetesWorkflow log.Debug(); if (IsNamespaceOnline(K8sNamespace)) { - DeleteAllPvcsInNamespace(K8sNamespace); + PrepareNamespaceForDeletion(K8sNamespace); client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0)); if (wait) WaitUntilNamespaceDeleted(K8sNamespace); @@ -155,12 +155,74 @@ namespace KubernetesWorkflow log.Debug(); if (IsNamespaceOnline(ns)) { - DeleteAllPvcsInNamespace(ns); + PrepareNamespaceForDeletion(ns); client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0)); if (wait) WaitUntilNamespaceDeleted(ns); } } + /// + /// Ensures all pods are stopped and all PVCs are deleted before the namespace + /// is removed. This guarantees the underlying GCE PDs are released by the time + /// the namespace deletion is issued. + /// + private void PrepareNamespaceForDeletion(string ns) + { + ForceDeleteAllPodsInNamespace(ns); + WaitUntilAllPodsInNamespaceAreGone(ns); + DeleteAllPvcsInNamespace(ns); + } + + /// + /// Sends a SIGKILL (gracePeriodSeconds=0) delete request for every pod in the + /// namespace. Individual failures are logged and skipped so one stuck pod does + /// not prevent the rest from being force-deleted. + /// + private void ForceDeleteAllPodsInNamespace(string ns) + { + var pods = client.Run(c => c.ListNamespacedPod(ns)); + foreach (var pod in pods.Items) + { + try + { + client.Run(c => c.DeleteNamespacedPod(pod.Name(), ns, gracePeriodSeconds: 0)); + log.Debug($"Force-deleted pod '{pod.Name()}' in namespace '{ns}'."); + } + catch (k8s.Autorest.HttpOperationException ex) + { + log.Error($"Failed to force-delete pod '{pod.Name()}' in namespace '{ns}': {ex.Response.ReasonPhrase}"); + } + } + } + + /// + /// Polls until no pods remain in the namespace, or until the 2-minute timeout + /// expires. On timeout the error is logged and execution continues: PVC deletion + /// will still be attempted with the finalizer removed (see DeleteAllPvcsInNamespace). + /// + private void WaitUntilAllPodsInNamespaceAreGone(string ns) + { + try + { + Time.WaitUntil( + () => !client.Run(c => c.ListNamespacedPod(ns)).Items.Any(), + TimeSpan.FromMinutes(2), + TimeSpan.FromSeconds(5), + $"WaitUntilAllPodsInNamespaceAreGone:{ns}"); + } + catch (TimeoutException) + { + log.Error($"Timed out waiting for pods in namespace '{ns}' to stop. Proceeding with PVC deletion."); + } + } + + /// + /// Removes the kubernetes.io/pvc-protection finalizer from each PVC then deletes + /// it. The finalizer is always patched off unconditionally: it is a no-op when + /// already absent, and it ensures deletion succeeds even when pods are still + /// present after a force-delete timeout (which would otherwise leave the PVC + /// stuck in Terminating and the backing GCE disk allocated indefinitely). + /// private void DeleteAllPvcsInNamespace(string ns) { var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(ns)); @@ -168,6 +230,8 @@ namespace KubernetesWorkflow { try { + var patch = new V1Patch("{\"metadata\":{\"finalizers\":[]}}", V1Patch.PatchType.MergePatch); + client.Run(c => c.PatchNamespacedPersistentVolumeClaim(patch, pvc.Name(), ns)); client.Run(c => c.DeleteNamespacedPersistentVolumeClaim(pvc.Name(), ns)); log.Debug($"Deleted PVC '{pvc.Name()}' in namespace '{ns}'."); }