<fix>[storage]: when the snapshot group creation fails, delete the successfully created snapshots

VERSION

@@ -1,3 +1,3 @@
 MAJOR=4
 MINOR=10
-UPDATE=16
+UPDATE=20

build/pom.xml

@@ -292,11 +292,6 @@
                     <artifactId>cloudformation</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>hybrid</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>zwatch</artifactId>
@@ -317,21 +312,11 @@
                     <artifactId>sharedblock</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>ministorage</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>storage-device</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>mini</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>autoscaling</artifactId>
@@ -357,11 +342,6 @@
                     <artifactId>xdragon</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>aliyun-storage</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>block-primary-storage</artifactId>
@@ -402,21 +382,6 @@
                     <artifactId>baremetal</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>baremetal2</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>external-api-adapter</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>aliyun-proxy</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>upgrade-hack</artifactId>
@@ -468,11 +433,6 @@
                     <artifactId>vpcFirewall</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-                <dependency>
-                    <groupId>org.zstack</groupId>
-                    <artifactId>sns-aliyun-sms</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
                 <dependency>
                     <groupId>org.zstack</groupId>
                     <artifactId>guesttools</artifactId>
@@ -578,6 +538,11 @@
                     <artifactId>hostNetworkInterface</artifactId>
                     <version>${project.version}</version>
                 </dependency>
+                <dependency>
+                    <groupId>org.zstack</groupId>
+                    <artifactId>managements</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
             </dependencies>
         </profile>
     </profiles>

compute/src/main/java/org/zstack/compute/allocator/HostOsVersionAllocatorFlow.java

@@ -4,7 +4,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Configurable;
 import org.zstack.compute.host.HostManager;
-import org.zstack.core.Platform;
 import org.zstack.core.db.Q;
 import org.zstack.header.allocator.AbstractHostAllocatorFlow;
 import org.zstack.header.allocator.HostCandidate;
@@ -72,7 +71,8 @@ public void allocate() {
 
     private Map<String, HostOperationSystem> generateHostUuidOsMap(List<HostVO> hostList) {
         final Map<String, String> hostHypervisorTypeMap = hostList.stream()
-                .collect(Collectors.toMap(ResourceVO::getUuid, HostAO::getHypervisorType));
+                .collect(Collectors.toMap(ResourceVO::getUuid, HostAO::getHypervisorType,
+                        (existing, replacement) -> replacement));
         final Set<String> hypervisorTypeSet = new HashSet<>(hostHypervisorTypeMap.values());
 
         final Map<String, HostOperationSystem> results = new HashMap<>(hostList.size());

compute/src/main/java/org/zstack/compute/host/HostApiInterceptor.java

@@ -1,6 +1,7 @@
 package org.zstack.compute.host;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.zstack.compute.vm.VmHostnameUtils;
 import org.zstack.core.CoreGlobalProperty;
 import org.zstack.core.componentloader.PluginRegistry;
 import org.zstack.core.upgrade.UpgradeGlobalConfig;
@@ -17,6 +18,7 @@
 import org.zstack.header.apimediator.ApiMessageInterceptor;
 import org.zstack.header.apimediator.StopRoutingException;
 import org.zstack.header.host.*;
+import org.zstack.header.image.ImagePlatform;
 import org.zstack.header.message.APIMessage;
 import org.zstack.utils.ShellResult;
 import org.zstack.utils.ShellUtils;
@@ -73,6 +75,8 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti
             validate((APIGetPhysicalMachineBlockDevicesMsg) msg);
         } else if (msg instanceof APIMountBlockDeviceMsg) {
             validate((APIMountBlockDeviceMsg) msg);
+        } else if (msg instanceof APIUpdateHostnameMsg) {
+            validate((APIUpdateHostnameMsg) msg);
         }
 
         return msg;
@@ -169,6 +173,10 @@ private void validate(APIMountBlockDeviceMsg msg) {
         validateMountPoint(msg.getMountPoint());
     }
 
+    private void validate(APIUpdateHostnameMsg msg) {
+        VmHostnameUtils.validateHostname(msg.getHostname(), false);
+    }
+
     private void validatePath(String path) {
         if (path == null || path.isEmpty()) {
             throw new ApiMessageInterceptionException(operr("path cannot be empty"));

compute/src/main/java/org/zstack/compute/host/HostBase.java

@@ -186,11 +186,34 @@ protected void handleApiMessage(APIMessage msg) {
             handle((APIGetHostPowerStatusMsg) msg);
         } else if (msg instanceof APIUpdateHostNqnMsg) {
             handle((APIUpdateHostNqnMsg) msg);
+        } else if (msg instanceof APIUpdateHostnameMsg) {
+            handle((APIUpdateHostnameMsg) msg);
         } else {
             bus.dealWithUnknownMessage(msg);
         }
     }
 
+    private void handle(APIUpdateHostnameMsg msg) {
+        APIUpdateHostnameEvent event = new APIUpdateHostnameEvent(msg.getId());
+        UpdateHostnameMsg umsg = new UpdateHostnameMsg();
+        umsg.setUuid(msg.getUuid());
+        umsg.setHostname(msg.getHostname());
+        bus.makeTargetServiceIdByResourceUuid(umsg, HostConstant.SERVICE_ID, msg.getHostUuid());
+        bus.send(umsg, new CloudBusCallBack(msg) {
+            @Override
+            public void run(MessageReply reply) {
+                UpdateHostnameReply r = reply.castReply();
+                if (!r.isSuccess()) {
+                    event.setSuccess(false);
+                    event.setError(r.getError());
+                } else {
+                    event.setInventory(r.getInventory());
+                }
+                bus.publish(event);
+            }
+        });
+    }
+
     private void handle(APIUpdateHostNqnMsg msg) {
         APIUpdateHostNqnEvent event = new APIUpdateHostNqnEvent(msg.getId());
         UpdateHostNqnMsg umsg = new UpdateHostNqnMsg();

compute/src/main/java/org/zstack/compute/vm/ApplianceVmAllocatePrimaryStorageFlow.java

@@ -34,6 +34,8 @@
 
 import java.util.*;
 
+import static org.zstack.utils.CollectionUtils.isEmpty;
+
 @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
 public class ApplianceVmAllocatePrimaryStorageFlow implements Flow {
     protected static final CLogger logger = Utils.getLogger(ApplianceVmAllocatePrimaryStorageFlow.class);
@@ -64,7 +66,7 @@ public void run(final FlowTrigger trigger, final Map data) {
         List<String> primaryStorageTypes = hostAllocatorMgr.getBackupStoragePrimaryStorageMetrics().get(bsType);
         DebugUtils.Assert(primaryStorageTypes != null, "why primaryStorageTypes is null");
 
-        DebugUtils.Assert(spec.getDataDiskOfferings().size() == 0, "create appliance vm can not with data volume");
+        DebugUtils.Assert(isEmpty(spec.getDeprecatedDisksSpecs()), "create appliance vm can not with data volume");
 
         for (PrimaryStorageAllocatorStrategyExtensionPoint ext : pluginRgty.getExtensionList(PrimaryStorageAllocatorStrategyExtensionPoint.class)) {
             String allocatorStrategyType = ext.getAllocatorStrategy(destHost);

compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java

@@ -1,11 +1,8 @@
 package org.zstack.compute.vm;
 
+import org.zstack.header.configuration.VmCustomSpecificationStruct;
 import org.zstack.header.host.CpuArchitecture;
-import org.zstack.header.vm.CreateVmInstanceMsg;
-import org.zstack.header.vm.DiskAO;
-import org.zstack.header.vm.InstantiateNewCreatedVmInstanceMsg;
-import org.zstack.header.vm.VmCreationStrategy;
-import org.zstack.header.vm.VmNicSpec;
+import org.zstack.header.vm.*;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -16,7 +13,6 @@
  * Created by xing5 on 2016/9/13.
  */
 public class InstantiateVmFromNewCreatedStruct {
-    private List<String> dataDiskOfferingUuids;
     private List<String> dataVolumeTemplateUuids;
     private Map<String, List<String>> dataVolumeFromTemplateSystemTags;
     private List<VmNicSpec> l3NetworkUuids;
@@ -28,12 +24,14 @@ public class InstantiateVmFromNewCreatedStruct {
     private String requiredHostUuid;
     private List<String> softAvoidHostUuids;
     private List<String> avoidHostUuids;
-    private Map<String, List<String>> dataVolumeSystemTagsOnIndex;
     private List<String> disableL3Networks;
     private List<String> sshKeyPairUuids;
     private final List<String> candidatePrimaryStorageUuidsForRootVolume = new ArrayList<>();
     private final List<String> candidatePrimaryStorageUuidsForDataVolume = new ArrayList<>();
-    private List<DiskAO> diskAOs;
+    private DiskAO rootDisk;
+    private List<DiskAO> dataDisks;
+    private List<DiskAO> deprecatedDataVolumeSpecs = new ArrayList<>();
+    private VmCustomSpecificationStruct vmCustomSpecification;
 
     public List<String> getCandidatePrimaryStorageUuidsForRootVolume() {
         return candidatePrimaryStorageUuidsForRootVolume;
@@ -57,13 +55,36 @@ public void setCandidatePrimaryStorageUuidsForDataVolume(List<String> candidateP
         }
     }
 
+    public DiskAO getRootDisk() {
+        return rootDisk;
+    }
+
+    public void setRootDisk(DiskAO rootDisk) {
+        this.rootDisk = rootDisk;
+    }
+
+    public List<DiskAO> getDataDisks() {
+        return dataDisks;
+    }
+
+    public void setDataDisks(List<DiskAO> dataDisks) {
+        this.dataDisks = dataDisks;
+    }
+
+    public List<DiskAO> getDeprecatedDataVolumeSpecs() {
+        return deprecatedDataVolumeSpecs;
+    }
 
-    public List<DiskAO> getDiskAOs() {
-        return diskAOs;
+    public void setDeprecatedDataVolumeSpecs(List<DiskAO> deprecatedDataVolumeSpecs) {
+        this.deprecatedDataVolumeSpecs = deprecatedDataVolumeSpecs;
     }
 
-    public void setDiskAOs(List<DiskAO> diskAOs) {
-        this.diskAOs = diskAOs;
+    public VmCustomSpecificationStruct getVmCustomSpecification() {
+        return vmCustomSpecification;
+    }
+
+    public void setVmCustomSpecification(VmCustomSpecificationStruct vmCustomSpecification) {
+        this.vmCustomSpecification = vmCustomSpecification;
     }
 
     public List<String> getRootVolumeSystemTags() {
@@ -86,14 +107,6 @@ public static String makeLabelKey(String vmUuid) {
         return String.format("not-start-vm-%s", vmUuid);
     }
 
-    public List<String> getDataDiskOfferingUuids() {
-        return dataDiskOfferingUuids;
-    }
-
-    public void setDataDiskOfferingUuids(List<String> dataDiskOfferingUuids) {
-        this.dataDiskOfferingUuids = dataDiskOfferingUuids;
-    }
-
     public List<String> getDataVolumeTemplateUuids() {
         return dataVolumeTemplateUuids;
     }
@@ -136,7 +149,6 @@ public CpuArchitecture getArchitecture() {
 
     public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreatedVmInstanceMsg msg) {
         InstantiateVmFromNewCreatedStruct struct = new InstantiateVmFromNewCreatedStruct();
-        struct.setDataDiskOfferingUuids(msg.getDataDiskOfferingUuids());
         struct.setDataVolumeTemplateUuids(msg.getDataVolumeTemplateUuids());
         struct.setDataVolumeFromTemplateSystemTags(msg.getDataVolumeFromTemplateSystemTags());
         struct.setL3NetworkUuids(msg.getL3NetworkUuids());
@@ -150,15 +162,16 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreate
         struct.setRequiredHostUuid(msg.getHostUuid());
         struct.setSoftAvoidHostUuids(msg.getSoftAvoidHostUuids());
         struct.setAvoidHostUuids(msg.getAvoidHostUuids());
-        struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
         struct.setDisableL3Networks(msg.getDisableL3Networks());
-        struct.setDiskAOs(msg.getDiskAOs());
+        struct.setRootDisk(msg.getRootDisk());
+        struct.setDataDisks(msg.getDataDisks());
+        struct.setDeprecatedDataVolumeSpecs(msg.getDeprecatedDataVolumeSpecs());
+        struct.setVmCustomSpecification(msg.getVmCustomSpecification());
         return struct;
     }
 
     public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg msg) {
         InstantiateVmFromNewCreatedStruct struct = new InstantiateVmFromNewCreatedStruct();
-        struct.setDataDiskOfferingUuids(msg.getDataDiskOfferingUuids());
         struct.setDataVolumeTemplateUuids(msg.getDataVolumeTemplateUuids());
         struct.setDataVolumeFromTemplateSystemTags(msg.getDataVolumeFromTemplateSystemTags());
         struct.setL3NetworkUuids(msg.getL3NetworkSpecs());
@@ -169,9 +182,11 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg
         struct.setRootVolumeSystemTags(msg.getRootVolumeSystemTags());
         struct.setDataVolumeSystemTags(msg.getDataVolumeSystemTags());
         struct.setRequiredHostUuid(msg.getHostUuid());
-        struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
         struct.setDisableL3Networks(msg.getDisableL3Networks());
-        struct.setDiskAOs(msg.getDiskAOs());
+        struct.setRootDisk(msg.getRootDisk());
+        struct.setDataDisks(msg.getDataDisks());
+        struct.setDeprecatedDataVolumeSpecs(msg.getDeprecatedDataVolumeSpecs());
+        struct.setVmCustomSpecification(msg.getVmCustomSpecification());
         return struct;
     }
 
@@ -231,12 +246,29 @@ public void setDataVolumeFromTemplateSystemTags(Map<String, List<String>> dataVo
         this.dataVolumeFromTemplateSystemTags = dataVolumeFromTemplateSystemTags;
     }
 
-    public Map<String, List<String>> getDataVolumeSystemTagsOnIndex() {
-        return dataVolumeSystemTagsOnIndex;
-    }
-
+    @Deprecated
     public void setDataVolumeSystemTagsOnIndex(Map<String, List<String>> dataVolumeSystemTagsOnIndex) {
-        this.dataVolumeSystemTagsOnIndex = dataVolumeSystemTagsOnIndex;
+        if (dataVolumeSystemTagsOnIndex == null) {
+            return;
+        }
+
+        int maxIndex = dataVolumeSystemTagsOnIndex.keySet().stream().mapToInt(Integer::parseInt).max().orElse(-1);
+        for (int i = deprecatedDataVolumeSpecs.size(); i <= maxIndex; i++) {
+            deprecatedDataVolumeSpecs.add(DiskAO.nonRootDisk());
+        }
+
+        for (int i = 0; i <= maxIndex; i++) {
+            String key = i + "";
+            if (!dataVolumeSystemTagsOnIndex.containsKey(key)) {
+                continue;
+            }
+
+            final DiskAO diskAO = deprecatedDataVolumeSpecs.get(i);
+            if (diskAO.getSystemTags() == null) {
+                diskAO.setSystemTags(new ArrayList<>());
+            }
+            diskAO.getSystemTags().addAll(dataVolumeSystemTagsOnIndex.get(key));
+        }
     }
 
     public List<String> getDisableL3Networks() {

compute/src/main/java/org/zstack/compute/vm/VmAllocateHostAndPrimaryStorageFlow.java

@@ -385,7 +385,7 @@ private boolean dataVolumePsUnique(VmInstanceSpec spec) {
     }
 
     private boolean needCreateDataVolume(VmInstanceSpec spec) {
-        return !CollectionUtils.isEmpty(spec.getDataDiskOfferings());
+        return !CollectionUtils.isEmpty(spec.getDeprecatedDisksSpecs());
     }
 
     private FlowChain buildAllocateHostAndPrimaryStorageFlowChain(final FlowTrigger trigger, VmInstanceSpec spec) {
@@ -451,8 +451,7 @@ private List<List<String>> getPrimaryStorageCombinationFromSpec(VmInstanceSpec s
                 autoAllocateDataVolumePs = true;
                 dataPs.addAll(availPsForDataVolume);
             }
-            String dataVolumeStrategy = spec.getDataDiskOfferings().get(0).getAllocatorStrategy();
-            sortPrimaryStorages(dataPs, dataVolumeStrategy, null);
+            sortPrimaryStorages(dataPs, PrimaryStorageConstant.DEFAULT_PRIMARY_STORAGE_ALLOCATION_STRATEGY_TYPE, null);
         } else {
             dataPs.add(null);
         }