diff --git a/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostMsg.java b/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostMsg.java new file mode 100644 index 00000000000..931da330c79 --- /dev/null +++ b/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostMsg.java @@ -0,0 +1,46 @@ +package org.zstack.header.image; + +import org.zstack.header.log.NoLogging; +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.storage.backup.BackupStorageMessage; + +public class UploadFileToBackupStorageHostMsg extends NeedReplyMessage implements BackupStorageMessage { + private String backupStorageUuid; + private String taskUuid; + @NoLogging(type = NoLogging.Type.Uri) + private String url; + private String installPath; + + @Override + public String getBackupStorageUuid() { + return backupStorageUuid; + } + + public void setBackupStorageUuid(String backupStorageUuid) { + this.backupStorageUuid = backupStorageUuid; + } + + public String getTaskUuid() { + return taskUuid; + } + + public void setTaskUuid(String taskUuid) { + this.taskUuid = taskUuid; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } +} diff --git a/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostReply.java b/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostReply.java new file mode 100644 index 00000000000..58afbeab944 --- /dev/null +++ b/header/src/main/java/org/zstack/header/image/UploadFileToBackupStorageHostReply.java @@ -0,0 +1,35 @@ +package org.zstack.header.image; + +import org.zstack.header.log.NoLogging; +import org.zstack.header.message.MessageReply; + +public class UploadFileToBackupStorageHostReply extends MessageReply { + private String md5sum; + private long size; + @NoLogging(type = NoLogging.Type.Uri) + private String directUploadUrl; + + public String getMd5sum() { + return md5sum; + } + + public void setMd5sum(String md5sum) { + this.md5sum = md5sum; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public String getDirectUploadUrl() { + return directUploadUrl; + } + + public void setDirectUploadUrl(String directUploadUrl) { + this.directUploadUrl = directUploadUrl; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressMsg.java b/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressMsg.java new file mode 100644 index 00000000000..1b8517923da --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressMsg.java @@ -0,0 +1,34 @@ +package org.zstack.header.storage.backup; + +import org.zstack.header.message.NeedReplyMessage; + +public class GetFileDownloadProgressMsg extends NeedReplyMessage implements BackupStorageMessage { + private String backupStorageUuid; + private String taskUuid; + private String hostname; + + @Override + public String getBackupStorageUuid() { + return backupStorageUuid; + } + + public void setBackupStorageUuid(String backupStorageUuid) { + this.backupStorageUuid = backupStorageUuid; + } + + public String getTaskUuid() { + return taskUuid; + } + + public void setTaskUuid(String taskUuid) { + this.taskUuid = taskUuid; + } + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressReply.java b/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressReply.java new file mode 100644 index 00000000000..61e2934243f --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/backup/GetFileDownloadProgressReply.java @@ -0,0 +1,92 @@ +package org.zstack.header.storage.backup; + +import org.zstack.header.message.MessageReply; + +public class GetFileDownloadProgressReply extends MessageReply { + private boolean completed; + private int progress; + + private long size; + private long actualSize; + private long downloadSize; + private String installPath; + private String format; + private long lastOpTime; + private boolean supportSuspend; + + public boolean isCompleted() { + return completed; + } + + public void setCompleted(boolean completed) { + this.completed = completed; + } + + public int getProgress() { + return progress; + } + + public void setProgress(int progress) { + this.progress = progress; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public long getActualSize() { + return actualSize; + } + + public void setActualSize(long actualSize) { + this.actualSize = actualSize; + } + + public boolean isDownloadComplete() { + return actualSize > 0 && actualSize == downloadSize; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public long getLastOpTime() { + return lastOpTime; + } + + public void setLastOpTime(long lastOpTime) { + this.lastOpTime = lastOpTime; + } + + public long getDownloadSize() { + return downloadSize; + } + + public void setDownloadSize(long downloadSize) { + this.downloadSize = downloadSize; + } + + public boolean isSupportSuspend() { + return supportSuspend; + } + + public void setSupportSuspend(boolean supportSuspend) { + this.supportSuspend = supportSuspend; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java index 87c501b580c..1456f64bdb6 100755 --- a/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/APICreateDataVolumeMsg.java @@ -1,5 +1,6 @@ package org.zstack.header.volume; + import org.springframework.http.HttpMethod; import org.zstack.header.configuration.DiskOfferingVO; import org.zstack.header.message.*; diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/backup/CephBackupStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/backup/CephBackupStorageBase.java index e24fdc21e11..1743f1744a4 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/backup/CephBackupStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/backup/CephBackupStorageBase.java @@ -19,6 +19,7 @@ import org.zstack.core.thread.ChainTask; import org.zstack.core.thread.SyncTaskChain; import org.zstack.core.thread.SyncThread; +import org.zstack.core.timeout.ApiTimeoutManager; import org.zstack.core.workflow.FlowChainBuilder; import org.zstack.core.workflow.ShareFlow; import org.zstack.header.Constants; @@ -30,6 +31,8 @@ import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.errorcode.SysErrors; import org.zstack.header.exception.CloudRuntimeException; +import org.zstack.header.host.GetFileDownloadProgressReply; +import org.zstack.header.host.UploadFileToHostReply; import org.zstack.header.image.*; import org.zstack.header.log.NoLogging; import org.zstack.header.message.APIMessage; @@ -37,6 +40,8 @@ import org.zstack.header.message.MessageReply; import org.zstack.header.rest.RESTFacade; import org.zstack.header.storage.backup.*; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMHost; import org.zstack.storage.backup.BackupStorageBase; import org.zstack.storage.ceph.*; import org.zstack.storage.ceph.CephMonBase.PingResult; @@ -51,6 +56,7 @@ import javax.persistence.Tuple; import javax.persistence.TypedQuery; import java.io.Serializable; +import java.net.URI; import java.net.URISyntaxException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -91,6 +97,8 @@ void unlock() { protected RESTFacade restf; @Autowired protected CephBackupStorageMetaDataMaker metaDataMaker; + @Autowired + private ApiTimeoutManager timeoutManager; public enum PingOperationFailure { UnableToCreateFile, @@ -368,6 +376,52 @@ public void setFormat(String format) { } } + public static class DownloadFileCmd extends AgentCommand implements HasThreadContext, Serializable { + public String taskUuid; + public String installPath; + @NoLogging(type = NoLogging.Type.Uri) + public String url; + @NoLogging(type = NoLogging.Type.Uri) + public String urlScheme; + public long timeout; + @NoLogging(type = NoLogging.Type.Uri) + public String sendCommandUrl; + } + + public static class DownloadFileResponse extends AgentResponse { + public String md5sum; + public long size; + } + + public static class UploadFileCmd extends AgentCommand implements HasThreadContext, Serializable { + public String taskUuid; + public String installPath; + @NoLogging(type = NoLogging.Type.Uri) + public String url; + public long timeout; + } + + public static class UploadFileResponse extends AgentResponse { + public String directUploadPath; + } + + public static class GetDownloadFileProgressCmd extends AgentCommand { + public String taskUuid; + } + + public static class GetDownloadFileProgressResponse extends AgentResponse { + public boolean completed; + public int progress; + public long size; + public long actualSize; + public String installPath; + public String format; + public long lastOpTime; + public long downloadSize; + public String md5sum; + public boolean supportSuspend; + } + public static class DeleteCmd extends AgentCommand { String installPath; @@ -680,6 +734,10 @@ public static class StorageMigrationRsp extends AgentResponse { public static final String GET_LOCAL_FILE_SIZE = "/ceph/backupstorage/getlocalfilesize"; public static final String CEPH_TO_CEPH_MIGRATE_IMAGE_PATH = "/ceph/backupstorage/image/migrate"; + public static final String FILE_DOWNLOAD_PATH = "/ceph/file/download"; + public static final String FILE_UPLOAD_PATH = "/ceph/file/upload"; + public static final String FILE_UPLOAD_PROGRESS_PATH = "/ceph/file/progress"; + protected String makeImageInstallPath(String imageUuid) { return String.format("ceph://%s/%s", getSelf().getPoolName(), imageUuid); } @@ -2021,4 +2079,99 @@ protected void handle(CalculateImageHashOnBackupStorageMsg msg) { private void doRestoreImagesBackupStorageMetadataToDatabase(RestoreImagesBackupStorageMetadataToDatabaseMsg msg) { metaDataMaker.restoreImagesBackupStorageMetadataToDatabase(msg.getImagesMetadata(), msg.getBackupStorageUuid()); } + + @Override + protected void handle(final UploadFileToBackupStorageHostMsg msg) { + UploadFileToBackupStorageHostReply reply = new UploadFileToBackupStorageHostReply(); + + if (msg.getUrl().startsWith("upload://")) { + UploadFileCmd cmd = new UploadFileCmd(); + cmd.url = msg.getUrl(); + cmd.installPath = msg.getInstallPath(); + cmd.timeout = timeoutManager.getTimeout(); + cmd.taskUuid = msg.getTaskUuid(); + httpCall(FILE_UPLOAD_PATH, cmd, UploadFileResponse.class, new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(UploadFileResponse rsp) { + reply.setDirectUploadUrl(rsp.directUploadPath); + bus.reply(msg, reply); + } + }); + return; + } + + DownloadFileCmd cmd = new DownloadFileCmd(); + cmd.url = msg.getUrl(); + cmd.installPath = msg.getInstallPath(); + cmd.timeout = timeoutManager.getTimeout(); + cmd.taskUuid = msg.getTaskUuid(); + cmd.sendCommandUrl = restf.getSendCommandUrl(); + + String scheme; + try { + URI uri = new URI(msg.getUrl()); + scheme = uri.getScheme(); + } catch (URISyntaxException e) { + reply.setError(operr("failed to parse upload URL [%s]: %s", msg.getUrl(), e.getMessage())); + bus.reply(msg, reply); + return; + } + if (scheme == null) { + reply.setError(operr("upload URL [%s] is missing a protocol prefix", msg.getUrl())); + bus.reply(msg, reply); + return; + } + cmd.urlScheme = scheme; + + httpCall(FILE_DOWNLOAD_PATH, cmd, DownloadFileResponse.class, new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(DownloadFileResponse rsp) { + reply.setMd5sum(rsp.md5sum); + reply.setSize(rsp.size); + bus.reply(msg, reply); + } + }); + } + + @Override + protected void handle(GetFileDownloadProgressMsg msg) { + GetFileDownloadProgressReply reply = new GetFileDownloadProgressReply(); + + GetDownloadFileProgressCmd cmd = new GetDownloadFileProgressCmd(); + cmd.taskUuid = msg.getTaskUuid(); + + httpCall(FILE_UPLOAD_PROGRESS_PATH, cmd, GetDownloadFileProgressResponse.class, new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(GetDownloadFileProgressResponse rsp) { + reply.setCompleted(rsp.completed); + reply.setProgress(rsp.progress); + reply.setActualSize(rsp.actualSize); + reply.setSize(rsp.size); + reply.setInstallPath(rsp.installPath); + reply.setDownloadSize(rsp.downloadSize); + reply.setLastOpTime(rsp.lastOpTime); + reply.setMd5sum(rsp.md5sum); + reply.setSupportSuspend(rsp.supportSuspend); + bus.reply(msg, reply); + } + }); + } } diff --git a/plugin/sftpBackupStorage/src/main/java/org/zstack/storage/backup/sftp/SftpBackupStorage.java b/plugin/sftpBackupStorage/src/main/java/org/zstack/storage/backup/sftp/SftpBackupStorage.java index e6cc338873a..a34b6d851e6 100755 --- a/plugin/sftpBackupStorage/src/main/java/org/zstack/storage/backup/sftp/SftpBackupStorage.java +++ b/plugin/sftpBackupStorage/src/main/java/org/zstack/storage/backup/sftp/SftpBackupStorage.java @@ -705,4 +705,14 @@ protected void handle(GetBackupStorageManagerHostnameMsg msg) { private void doRestoreImagesBackupStorageMetadataToDatabase(RestoreImagesBackupStorageMetadataToDatabaseMsg msg) { metaDataMaker.restoreImagesBackupStorageMetadataToDatabase(msg.getImagesMetadata(), msg.getBackupStorageUuid()); } + + @Override + protected void handle(UploadFileToBackupStorageHostMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } + + @Override + protected void handle(GetFileDownloadProgressMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } } diff --git a/pom.xml b/pom.xml index 8d763b2470b..1a1c22fd344 100755 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ resourceconfig plugin/iscsi abstraction + premium/plugin-premium/thirdPartySoftwarePackage diff --git a/simulator/simulatorImpl/src/main/java/org/zstack/simulator/storage/backup/SimulatorBackupStorage.java b/simulator/simulatorImpl/src/main/java/org/zstack/simulator/storage/backup/SimulatorBackupStorage.java index 755fbf5ca7d..7efe1aac3c2 100755 --- a/simulator/simulatorImpl/src/main/java/org/zstack/simulator/storage/backup/SimulatorBackupStorage.java +++ b/simulator/simulatorImpl/src/main/java/org/zstack/simulator/storage/backup/SimulatorBackupStorage.java @@ -6,6 +6,7 @@ import org.zstack.header.image.CancelAddImageReply; import org.zstack.header.image.CancelDownloadImageMsg; import org.zstack.header.image.ImageInventory; +import org.zstack.header.image.UploadFileToBackupStorageHostMsg; import org.zstack.header.storage.backup.*; import org.zstack.storage.backup.BackupStorageBase; import org.zstack.utils.Utils; @@ -137,4 +138,14 @@ protected void handle(CalculateImageHashOnBackupStorageMsg msg) { CalculateImageHashOnBackupStorageReply reply = new CalculateImageHashOnBackupStorageReply(); bus.reply(msg, reply); } + + @Override + protected void handle(UploadFileToBackupStorageHostMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } + + @Override + protected void handle(GetFileDownloadProgressMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } } diff --git a/storage/src/main/java/org/zstack/storage/addon/backup/ExternalBackupStorage.java b/storage/src/main/java/org/zstack/storage/addon/backup/ExternalBackupStorage.java index de4e84d1b4e..4b913c08901 100755 --- a/storage/src/main/java/org/zstack/storage/addon/backup/ExternalBackupStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/backup/ExternalBackupStorage.java @@ -7,6 +7,7 @@ import org.zstack.header.image.CancelDownloadImageMsg; import org.zstack.header.image.CancelDownloadImageReply; import org.zstack.header.image.ImageInventory; +import org.zstack.header.image.UploadFileToBackupStorageHostMsg; import org.zstack.header.storage.addon.ImageDescriptor; import org.zstack.header.storage.addon.StorageHealthy; import org.zstack.header.storage.addon.backup.BackupStorageController; @@ -162,4 +163,14 @@ protected void handle(RestoreImagesBackupStorageMetadataToDatabaseMsg msg) { protected void handle(CalculateImageHashOnBackupStorageMsg msg) { bus.replyErrorByMessageType(msg, "not supported"); } + + @Override + protected void handle(UploadFileToBackupStorageHostMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } + + @Override + protected void handle(GetFileDownloadProgressMsg msg) { + bus.replyErrorByMessageType(msg, "not supported"); + } } diff --git a/storage/src/main/java/org/zstack/storage/backup/BackupStorageBase.java b/storage/src/main/java/org/zstack/storage/backup/BackupStorageBase.java index 7fcdba39d76..26a1583aa62 100755 --- a/storage/src/main/java/org/zstack/storage/backup/BackupStorageBase.java +++ b/storage/src/main/java/org/zstack/storage/backup/BackupStorageBase.java @@ -36,6 +36,7 @@ import org.zstack.header.image.CancelDownloadImageMsg; import org.zstack.header.image.ImageConstant; import org.zstack.header.image.ImageVO; +import org.zstack.header.image.UploadFileToBackupStorageHostMsg; import org.zstack.header.message.APIDeleteMessage; import org.zstack.header.message.APIMessage; import org.zstack.header.message.Message; @@ -121,6 +122,10 @@ public abstract class BackupStorageBase extends AbstractBackupStorage { abstract protected void pingHook(Completion completion); + abstract protected void handle(UploadFileToBackupStorageHostMsg msg); + + abstract protected void handle(GetFileDownloadProgressMsg msg); + protected void handle(GetBackupStorageManagerHostnameMsg msg) { bus.dealWithUnknownMessage(msg); }