diff --git a/conf/globalConfig/primaryStorage.xml b/conf/globalConfig/primaryStorage.xml index 033b82a04b..13d78277f7 100755 --- a/conf/globalConfig/primaryStorage.xml +++ b/conf/globalConfig/primaryStorage.xml @@ -71,4 +71,12 @@ java.lang.Integer primaryStorage + + + primarystorage.host.status.refresh.interval + primaryStorage + The interval to refresh ps and host connection status, in seconds + 60 + java.lang.Integer + diff --git a/header/src/main/java/org/zstack/header/storage/primary/PrimaryStorageType.java b/header/src/main/java/org/zstack/header/storage/primary/PrimaryStorageType.java index 10bdc11b25..cd18db5f20 100755 --- a/header/src/main/java/org/zstack/header/storage/primary/PrimaryStorageType.java +++ b/header/src/main/java/org/zstack/header/storage/primary/PrimaryStorageType.java @@ -24,6 +24,8 @@ public class PrimaryStorageType { private PrimaryStorageFindBackupStorage primaryStorageFindBackupStorage; private boolean supportCreateVolumeSnapshotCheckCapacity = true; + private boolean supportCheckHostStatus=false; + public boolean isSupportSharedVolume() { return supportSharedVolume; } @@ -200,4 +202,12 @@ public boolean isSupportStorageTrash() { public void setSupportStorageTrash(boolean supportStorageTrash) { this.supportStorageTrash = supportStorageTrash; } + + public boolean isSupportCheckHostStatus(){ + return supportCheckHostStatus; + } + + public void setSupportCheckHostStatus(boolean supportCheckHostStatus) { + this.supportCheckHostStatus = supportCheckHostStatus; + } } diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephKvmExtension.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephKvmExtension.java index 81ef29976f..099b3e9e20 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephKvmExtension.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephKvmExtension.java @@ -25,6 +25,7 @@ import org.zstack.kvm.KVMHostConnectedContext; import org.zstack.kvm.KVMHostFactory; import org.zstack.storage.ceph.CephConstants; +import org.zstack.storage.primary.CheckHostStorageConnectionMsg; import javax.persistence.TypedQuery; import java.util.List; diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java index f9cf2cb705..f2fd14187d 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java @@ -97,6 +97,7 @@ public class CephPrimaryStorageFactory implements PrimaryStorageFactory, CephCap { type.setSupportSharedVolume(true); + type.setSupportCheckHostStatus(true); } @Autowired diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionMsg.java b/storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionMsg.java similarity index 91% rename from plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionMsg.java rename to storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionMsg.java index 68be16ff74..de2e9a1034 100644 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionMsg.java +++ b/storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionMsg.java @@ -1,28 +1,28 @@ -package org.zstack.storage.ceph.primary; - -import org.zstack.header.message.NeedReplyMessage; -import org.zstack.header.storage.primary.PrimaryStorageMessage; - -import java.util.List; - -public class CheckHostStorageConnectionMsg extends NeedReplyMessage implements PrimaryStorageMessage { - private String primaryStorageUuid; - private List hostUuids; - - @Override - public String getPrimaryStorageUuid() { - return primaryStorageUuid; - } - - public void setPrimaryStorageUuid(String primaryStorageUuid) { - this.primaryStorageUuid = primaryStorageUuid; - } - - public List getHostUuids() { - return hostUuids; - } - - public void setHostUuids(List hostUuids) { - this.hostUuids = hostUuids; - } +package org.zstack.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.storage.primary.PrimaryStorageMessage; + +import java.util.List; + +public class CheckHostStorageConnectionMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + private List hostUuids; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public List getHostUuids() { + return hostUuids; + } + + public void setHostUuids(List hostUuids) { + this.hostUuids = hostUuids; + } } \ No newline at end of file diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionReply.java b/storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionReply.java similarity index 73% rename from plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionReply.java rename to storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionReply.java index 7766a0eea9..8aca9b7a6d 100644 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CheckHostStorageConnectionReply.java +++ b/storage/src/main/java/org/zstack/storage/primary/CheckHostStorageConnectionReply.java @@ -1,6 +1,6 @@ -package org.zstack.storage.ceph.primary; - -import org.zstack.header.message.MessageReply; - -public class CheckHostStorageConnectionReply extends MessageReply { -} +package org.zstack.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class CheckHostStorageConnectionReply extends MessageReply { +} diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageGlobalConfig.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageGlobalConfig.java index ffcf041d6d..251c331819 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageGlobalConfig.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageGlobalConfig.java @@ -56,4 +56,6 @@ public class PrimaryStorageGlobalConfig { @GlobalConfigDef(defaultValue = "1", type = Long.class) public static GlobalConfig COLLECT_AND_FORECAST_INTERVAL = new GlobalConfig(CATEGORY, "collect.forecast.interval"); + @GlobalConfigValidation(numberGreaterThan = 0) + public static GlobalConfig PRIMARY_STORAGE_HOST_STATUS_REFRESH_INTERVAL = new GlobalConfig(CATEGORY, "primarystorage.host.status.refresh.interval"); } diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageManagerImpl.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageManagerImpl.java index a84ed42f7b..1d63d14eb2 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageManagerImpl.java @@ -9,6 +9,7 @@ import org.zstack.configuration.InstanceOfferingSystemTags; import org.zstack.configuration.OfferingUserConfigUtils; import org.zstack.core.Platform; +import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.core.cloudbus.MessageSafe; @@ -35,6 +36,7 @@ import org.zstack.header.configuration.userconfig.InstanceOfferingUserConfig; import org.zstack.header.configuration.userconfig.InstanceOfferingUserConfigValidator; import org.zstack.header.core.NoErrorCompletion; +import org.zstack.header.core.NopeWhileDoneCompletion; import org.zstack.header.core.trash.InstallPathRecycleVO; import org.zstack.header.core.trash.InstallPathRecycleVO_; import org.zstack.header.errorcode.ErrorCode; @@ -109,6 +111,8 @@ public class PrimaryStorageManagerImpl extends AbstractService implements Primar private static final Map> extensionFactories = Maps.newConcurrentMap(); private AutoDeleteTrashTask globalTrashTask; + private Future refreshPrimaryStorageHostStatusTask; + static { allowedMessageAfterSoftDeletion.add(PrimaryStorageDeletionMsg.class); } @@ -1158,6 +1162,78 @@ public boolean stop() { return true; } + private void startPeriodTasks() { + PrimaryStorageGlobalConfig.PRIMARY_STORAGE_HOST_STATUS_REFRESH_INTERVAL.installUpdateExtension((oldConfig, newConfig) -> startRefreshPrimaryStorageHostStatusTask()); + startRefreshPrimaryStorageHostStatusTask(); + } + + private synchronized void startRefreshPrimaryStorageHostStatusTask() { + if (refreshPrimaryStorageHostStatusTask != null) { + refreshPrimaryStorageHostStatusTask.cancel(true); + } + + refreshPrimaryStorageHostStatusTask = thdf.submitPeriodicTask(new PeriodicTask() { + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + public long getInterval() { + return PrimaryStorageGlobalConfig.PRIMARY_STORAGE_HOST_STATUS_REFRESH_INTERVAL.value(Integer.class).longValue(); + } + + @Override + public String getName() { + return "update-host-storage-connection-status"; + } + + @Override + public void run() { + Map> disconnectedHostsByPsUuid = new HashMap<>(); + + List refs = Q.New(PrimaryStorageHostRefVO.class) + .eq(PrimaryStorageHostRefVO_.status, PrimaryStorageHostStatus.Disconnected) + .list(); + + refs.forEach(ref -> { + disconnectedHostsByPsUuid.computeIfAbsent(ref.getPrimaryStorageUuid(), key -> new ArrayList<>()).add(ref.getHostUuid()); + }); + + + List msgs = new ArrayList<>(); + for (Map.Entry> entry : disconnectedHostsByPsUuid.entrySet()) { + PrimaryStorageVO storageVO = dbf.findByUuid(entry.getKey(), PrimaryStorageVO.class); + if (!PrimaryStorageType.valueOf(storageVO.getType()).isSupportCheckHostStatus()){ + continue; + } + CheckHostStorageConnectionMsg msg = new CheckHostStorageConnectionMsg(); + msg.setPrimaryStorageUuid(entry.getKey()); + msg.setHostUuids(entry.getValue()); + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, entry.getKey()); + msgs.add(msg); + } + + new While<>(msgs).step((msg, comp) -> { + bus.send(msg, new CloudBusCallBack(comp) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.error(String.format("Failed to check host storage connection for primary storage %s and host %s due to: %s", + msg.getPrimaryStorageUuid(), msg.getHostUuids().get(0), reply.getError())); + } + comp.done(); + } + }); + }, 10).run(new NopeWhileDoneCompletion()); + } + + + }); + + } + + private void populateExtensions() { for (PrimaryStorageAllocatorStrategyFactory f : pluginRgty.getExtensionList(PrimaryStorageAllocatorStrategyFactory.class)) { PrimaryStorageAllocatorStrategyFactory old = allocatorFactories.get(f.getPrimaryStorageAllocatorStrategyType().toString()); @@ -1264,6 +1340,7 @@ public void managementNodeReady() { Platform.getManagementServerId())); loadPrimaryStorage(false); initResourcePrimaryStorageAutoDeleteTrash(); + startPeriodTasks(); } private void checkVmAllVolumePrimaryStorageState(String vmUuid) {