From 6b4521b38a688cf6072d6fc849171022af1e1490 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 12:02:19 +0800 Subject: [PATCH 1/7] [vm]: use max of virtual and actual size for root disk when no disk offering Resolves: ZSTAC-74683 Change-Id: Id0339ed0221e92e506f60745cde972cc3ee6d9ae --- header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java index 7007c592aea..99ee2173b98 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java @@ -847,7 +847,9 @@ public void setBootMode(String bootMode) { public long getRootDiskAllocateSize() { if (rootDiskOffering == null) { - return this.getImageSpec().getInventory().getSize(); + long virtualSize = this.getImageSpec().getInventory().getSize(); + long actualSize = this.getImageSpec().getInventory().getActualSize(); + return Math.max(virtualSize, actualSize); } return rootDiskOffering.getDiskSize(); } From 3b5bda3b76aef968a911d18e35b3b30bd0cab803 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 13:52:13 +0800 Subject: [PATCH 2/7] [zbs]: enable tryNext and 30s timeout for getActiveClients MDS call When anti-split-brain check selects a disconnected MDS node, the HTTP call now times out after 30s instead of 5+ minutes, and automatically retries the next available MDS via tryNext mechanism. Resolves: ZSTAC-80595 Change-Id: I1be80f1b70cad1606eb38d1f0078c8f2781e6941 --- .../org/zstack/storage/zbs/ZbsStorageController.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index db06239acb3..276ab367ba1 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -179,7 +179,10 @@ public List getActiveClients(String installPath, String prot if (VolumeProtocol.CBD.toString().equals(protocol)) { GetVolumeClientsCmd cmd = new GetVolumeClientsCmd(); cmd.setPath(installPath); - GetVolumeClientsRsp rsp = syncHttpCall(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class); + GetVolumeClientsRsp rsp = new HttpCaller<>(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class, + null, TimeUnit.SECONDS, 30, true) + .setTryNext(true) + .syncCall(); List clients = new ArrayList<>(); if (!rsp.isSuccess()) { @@ -1411,6 +1414,11 @@ public class HttpCaller { private boolean tryNext = false; + HttpCaller setTryNext(boolean tryNext) { + this.tryNext = tryNext; + return this; + } + public HttpCaller(String path, AgentCommand cmd, Class retClass, ReturnValueCompletion callback) { this(path, cmd, retClass, callback, null, 0, false); } From 80df074f8dd1140b278ce0979f2068d5c271d8e5 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 14:22:40 +0800 Subject: [PATCH 3/7] [vm]: add Destroying->Stopped state transition When MN restarts during a destroy operation, the hypervisor may report the VM as Stopped. Without this transition, the state machine throws an exception and the VM stays stuck in Destroying state forever. Resolves: ZSTAC-80620 Change-Id: I037edba70d145a44a88ce0d3573089182fedb162 --- header/src/main/java/org/zstack/header/vm/VmInstanceState.java | 1 + 1 file changed, 1 insertion(+) diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceState.java b/header/src/main/java/org/zstack/header/vm/VmInstanceState.java index 8a755b52fda..49303e23252 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceState.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceState.java @@ -168,6 +168,7 @@ public enum VmInstanceState { new Transaction(VmInstanceStateEvent.destroyed, VmInstanceState.Destroyed), new Transaction(VmInstanceStateEvent.destroying, VmInstanceState.Destroying), new Transaction(VmInstanceStateEvent.running, VmInstanceState.Running), + new Transaction(VmInstanceStateEvent.stopped, VmInstanceState.Stopped), new Transaction(VmInstanceStateEvent.expunging, VmInstanceState.Expunging) ); Destroyed.transactions( From 24d4f3b4870ea72fce77bcf64980d0b70b868502 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 13 Feb 2026 13:32:45 +0800 Subject: [PATCH 4/7] [i18n]: improve snapshot error message for unattached volume Resolves: ZSTAC-82153 Change-Id: Ib51c2e21553277416d1a9444be55aca2aa4b2fc4 --- conf/i18n/globalErrorCodeMapping/global-error-en_US.json | 2 +- conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/i18n/globalErrorCodeMapping/global-error-en_US.json b/conf/i18n/globalErrorCodeMapping/global-error-en_US.json index 32eb4c8f056..715e823d95e 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-en_US.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-en_US.json @@ -3374,7 +3374,7 @@ "ORG_ZSTACK_NETWORK_HUAWEI_IMASTER_10019": "delete token of SDN controller [IP:%s] failed because %s", "ORG_ZSTACK_STORAGE_PRIMARY_BLOCK_10004": "Cannot execute volume mapping to host flow due to invalid volume ID.%s", "ORG_ZSTACK_NETWORK_SERVICE_PORTFORWARDING_10007": "port forwarding rule [uuid:%s] has not been attached to any virtual machine network interface, cannot detach", - "ORG_ZSTACK_MEVOCO_10088": "cannot take a snapshot for volumes[%s] when volume[uuid: %s] is not attached", + "ORG_ZSTACK_MEVOCO_10088": "cannot create snapshot for volume[uuid:%s] because it is not attached to any VM instance. Please attach the volume to a VM first. Affected volumes: %s", "ORG_ZSTACK_STORAGE_PRIMARY_BLOCK_10005": "Cannot execute map LUN to host flow due to invalid LUN type: %s", "ORG_ZSTACK_NETWORK_SERVICE_PORTFORWARDING_10008": "port forwarding rule [uuid:%s] has been associated with vm nic [uuid:%s], cannot be reassigned again", "ORG_ZSTACK_MEVOCO_10087": "A Running VM[uuid:%s] has no associated Host UUID.", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json b/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json index 84609838ddc..01960e8eb45 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json @@ -3374,7 +3374,7 @@ "ORG_ZSTACK_NETWORK_HUAWEI_IMASTER_10019": "删除 SDN 控制器 [IP：%s] 的令牌失败，因为 %s", "ORG_ZSTACK_STORAGE_PRIMARY_BLOCK_10004": "无法执行映射LUN到主机流程，无效的LUN ID", "ORG_ZSTACK_NETWORK_SERVICE_PORTFORWARDING_10007": "端口转发规则 rule[uuid:%s] 没有绑定到任何 VM 的网卡上，无法解除绑定", - "ORG_ZSTACK_MEVOCO_10088": "无法为挂载状态以外的卷[%s]创建快照", + "ORG_ZSTACK_MEVOCO_10088": "无法为云盘[uuid:%s]创建快照，因为该云盘未挂载到任何云主机。请先将云盘挂载到云主机后再创建快照。相关云盘: %s", "ORG_ZSTACK_STORAGE_PRIMARY_BLOCK_10005": "无法执行映射LUN到主机流程，无效的LUN类型", "ORG_ZSTACK_NETWORK_SERVICE_PORTFORWARDING_10008": "端口转发规则[uuid:%s]已绑定到VM网卡[uuid:%s]，无法再次绑定", "ORG_ZSTACK_MEVOCO_10087": "如何一个运行中的VM[uuid:%s]没有宿主机uuid?", From 42ff6142c64c5eb57a21ab0355d6f4326638d4ae Mon Sep 17 00:00:00 2001 From: shenjin Date: Fri, 13 Feb 2026 16:21:47 +0800 Subject: [PATCH 5/7] [compute]: add syncVmDeviceInfo when vm is running add syncVmDeviceInfo when vm is running Resolves: ZSTAC-67275 Change-Id: I746e63786a677676676b6d6265657063666b7677 --- .../org/zstack/compute/vm/VmInstanceBase.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index d8c8be65325..80ba38486fc 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -7226,6 +7226,7 @@ protected void scripts() { self.setZoneUuid(spec.getDestHost().getZoneUuid()); } }.execute()); + syncVmDevicesAddressInfo(self.getUuid()); logger.debug(String.format("vm[uuid:%s] is running ..", self.getUuid())); VmInstanceInventory inv = VmInstanceInventory.valueOf(self); extEmitter.afterStartVm(inv); @@ -7495,6 +7496,9 @@ public void run(FlowTrigger trigger, Map data) { self.setHypervisorType(spec.getDestHost().getHypervisorType()); self.setRootVolumeUuid(spec.getDestRootVolume().getUuid()); }); + if (struct.getStrategy() == VmCreationStrategy.InstantStart) { + syncVmDevicesAddressInfo(self.getUuid()); + } logger.debug(String.format("vm[uuid:%s] is started ..", self.getUuid())); VmInstanceInventory inv = VmInstanceInventory.valueOf(self); extEmitter.afterStartNewCreatedVm(inv); @@ -7931,6 +7935,7 @@ public void handle(Map data) { public void done() { self = changeVmStateInDb(VmInstanceStateEvent.running, () -> self.setHostUuid(originalCopy.getHostUuid())); + syncVmDevicesAddressInfo(self.getUuid()); VmInstanceInventory inv = VmInstanceInventory.valueOf(self); extEmitter.afterRebootVm(inv); new StaticIpOperator().deleteIpChange(self.getUuid()); @@ -8347,6 +8352,7 @@ protected void resumeVm(final Message msg, Completion completion) { @Override public void handle(Map Data) { self = changeVmStateInDb(VmInstanceStateEvent.running); + syncVmDevicesAddressInfo(self.getUuid()); completion.success(); } }).error(new FlowErrorHandler(completion) { @@ -8467,6 +8473,28 @@ public String getName() { }); } + private void syncVmDevicesAddressInfo(String vmUuid) { + if (self.getHostUuid() == null) { + return; + } + SyncVmDeviceInfoMsg msg = new SyncVmDeviceInfoMsg(); + msg.setVmInstanceUuid(vmUuid); + msg.setHostUuid(self.getHostUuid()); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(msg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format("Failed to sync vm device info for vm[uuid:%s], %s", + vmUuid, reply.getError())); + } else { + logger.debug(String.format("Sent SyncVmDeviceInfoMsg for vm[uuid:%s] on host[uuid:%s]", + vmUuid, self.getHostUuid())); + } + } + }); + } + private void deleteVmCdRom(String cdRomUuid, Completion completion) { boolean exist = dbf.isExist(cdRomUuid, VmCdRomVO.class); if (!exist) { From b8cd68ee782bf402f98b3c792d6b000443ee308b Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 13:29:00 +0800 Subject: [PATCH 6/7] [ha]: defer skip-trace list cleanup on MN departure to prevent split-brain When a management node departs, its VM skip-trace entries were immediately removed. If VMs were still being started by kvmagent, the next VM sync would falsely detect them as Stopped and trigger HA, causing split-brain. Fix: transfer departed MN skip-trace entries to an orphaned set with 10-minute TTL instead of immediate deletion. VMs in the orphaned set remain skip-traced until the TTL expires or they are explicitly continued, preventing false HA triggers during MN restart scenarios. Resolves: ZSTAC-80821 Change-Id: I3222e260b2d7b33dc43aba0431ce59a788566b34 --- .../org/zstack/kvm/KvmVmSyncPingTask.java | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index b02525b99d2..2e8d3ff9e32 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -69,6 +69,12 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + // Orphaned skip entries from departed MN nodes. Key=vmUuid, Value=timestamp when orphaned. + // These VMs remain in skip-trace state for ORPHAN_TTL_MS to avoid false HA triggers + // when a MN restarts and its in-flight VM operations haven't completed yet. See ZSTAC-80821. + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + private static final long ORPHAN_TTL_MS = 10 * 60 * 1000; // 10 minutes + { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { skipVmTracerMessages.add(clz.asSubclass(Message.class)); @@ -196,8 +202,13 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + // ZSTAC-80821: clean up expired orphaned entries each sync cycle + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + // ZSTAC-80821: also skip VMs from departed MN nodes that are still within TTL + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -228,6 +239,8 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + // ZSTAC-80821: include orphaned entries from departed MN nodes + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -446,7 +459,19 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + + // ZSTAC-80821: Instead of immediately removing skip list entries, move them + // to the orphaned set with a TTL. This prevents false HA triggers for VMs that + // are still being started by kvmagent but whose controlling MN has restarted. + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), ORPHAN_TTL_MS / 60000)); + } + } } @Override @@ -460,6 +485,41 @@ public void iJoin(ManagementNodeInventory inv) { } public boolean isVmDoNotNeedToTrace(String vmUuid) { - return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid)); + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + // ZSTAC-80821: Also check orphaned skip entries from departed MN nodes + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < ORPHAN_TTL_MS) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } else { + // Expired, clean up + orphanedSkipVms.remove(vmUuid); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, ORPHAN_TTL_MS / 60000)); + } + } + + return false; + } + + // Periodically clean up expired orphaned entries. Called from VM sync cycle. + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + Iterator> it = orphanedSkipVms.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (now - entry.getValue() >= ORPHAN_TTL_MS) { + it.remove(); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } } } From de83221e3e7b9b492ec654b0579f4ec8a2b5b2dd Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 16:47:28 +0800 Subject: [PATCH 7/7] [kvm]: use CAS remove to fix TOCTOU race in orphaned skip VM cleanup Resolves: ZSTAC-80821 Change-Id: I59284c4e69f5d2ee357b1836b7c243200e30949a --- .../src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index 2e8d3ff9e32..d1bb9133903 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -497,7 +497,7 @@ public boolean isVmDoNotNeedToTrace(String vmUuid) { return true; } else { // Expired, clean up - orphanedSkipVms.remove(vmUuid); + orphanedSkipVms.remove(vmUuid, orphanedAt); logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", vmUuid, ORPHAN_TTL_MS / 60000)); } @@ -513,11 +513,9 @@ private void cleanupExpiredOrphanedSkipVms() { } long now = System.currentTimeMillis(); - Iterator> it = orphanedSkipVms.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = it.next(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { if (now - entry.getValue() >= ORPHAN_TTL_MS) { - it.remove(); + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); } }