From 6d6f0ed891d6b0cb3899eae3146bd710be689948 Mon Sep 17 00:00:00 2001 From: Isaac Parenteau Date: Wed, 25 Nov 2020 00:49:36 -0600 Subject: [PATCH] Added logic to remove output dir and changed logging --- .../locusworks/s3sync/client/S3Client.java | 15 ++++++++-- .../s3sync/scp/AbstractSshMessage.java | 28 +++++++++++++++++-- .../locusworks/s3sync/scp/ScpFromMessage.java | 10 +++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/locusworks/s3sync/client/S3Client.java b/src/main/java/net/locusworks/s3sync/client/S3Client.java index ef7f91a..0a90d32 100644 --- a/src/main/java/net/locusworks/s3sync/client/S3Client.java +++ b/src/main/java/net/locusworks/s3sync/client/S3Client.java @@ -1,8 +1,10 @@ package net.locusworks.s3sync.client; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.HashSet; import java.util.Set; import com.amazonaws.AmazonClientException; @@ -138,12 +140,15 @@ public class S3Client implements AutoCloseable { jsch.addIdentity(identity); Session session = jsch.getSession(user, host, port); session.connect(); - try(ScpFromMessage msg = new ScpFromMessage(true, session, remoteFolder, localFolder, true); - FileManager manager = FileManager.getInstance()) { + try(ScpFromMessage msg = new ScpFromMessage(true, session, remoteFolder, localFolder, true); FileManager manager = FileManager.getInstance()) { msg.fileCallback(p -> { - logger.info(p+""); if (Files.isRegularFile(p)) { uploadFile(xferMgr, p); + try { + Files.delete(p); + } catch (Exception ex) { + logger.warn("Unable to delete local file %s: %s", p, ex.getMessage()); + } } }); msg.execute(); @@ -152,6 +157,10 @@ public class S3Client implements AutoCloseable { } xferMgr.shutdownNow(false); + Files.walk(localFolder) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); } public S3Object getObject(String bucketName, String key) { diff --git a/src/main/java/net/locusworks/s3sync/scp/AbstractSshMessage.java b/src/main/java/net/locusworks/s3sync/scp/AbstractSshMessage.java index b0685d4..214baf4 100644 --- a/src/main/java/net/locusworks/s3sync/scp/AbstractSshMessage.java +++ b/src/main/java/net/locusworks/s3sync/scp/AbstractSshMessage.java @@ -13,6 +13,7 @@ import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import com.jcraft.jsch.SftpProgressMonitor; import net.locusworks.logger.ApplicationLogger; +import net.locusworks.logger.LogLevel; public abstract class AbstractSshMessage { @@ -138,8 +139,29 @@ public abstract class AbstractSshMessage { * Log a message to the log listener. * @param message the message to log */ - protected void log(final String message) { - logger.info(message); + protected void log(final String message, LogLevel logLevel) { + switch (logLevel) { + case DEBUG: + logger.debug(message); + break; + case ERROR: + case FATAL: + logger.error(message); + break; + case INFO: + logger.info(message); + break; + case WARN: + logger.warn(message); + break; + case TRACE: + logger.trace(message); + break; + case OFF: + default: + logger.trace(message); + break; + } } /** @@ -153,7 +175,7 @@ public abstract class AbstractSshMessage { final NumberFormat format = NumberFormat.getNumberInstance(); format.setMaximumFractionDigits(2); format.setMinimumFractionDigits(1); - log("File transfer time: " + format.format(duration) + " Average Rate: " + format.format(totalLength / duration) + " B/s"); + log("File transfer time: " + format.format(duration) + " Average Rate: " + format.format(totalLength / duration) + " B/s", LogLevel.DEBUG); } /** diff --git a/src/main/java/net/locusworks/s3sync/scp/ScpFromMessage.java b/src/main/java/net/locusworks/s3sync/scp/ScpFromMessage.java index 333a105..f8c07bd 100644 --- a/src/main/java/net/locusworks/s3sync/scp/ScpFromMessage.java +++ b/src/main/java/net/locusworks/s3sync/scp/ScpFromMessage.java @@ -36,6 +36,7 @@ import com.jcraft.jsch.Session; import com.jcraft.jsch.SftpATTRS; import com.jcraft.jsch.SftpException; import net.locusworks.logger.ApplicationLoggerFactory; +import net.locusworks.logger.LogLevel; import net.locusworks.logger.ApplicationLogger; public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable { @@ -153,9 +154,8 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable command += "-C "; } command += remoteFile; - log(command); + log("Executing command: " + command, LogLevel.DEBUG); final Channel channel = openExecChannel(command); - log(localFile + ""); try { // get I/O streams for remote scp final OutputStream out = channel.getOutputStream(); @@ -170,7 +170,7 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable channel.disconnect(); } } - log("done\n"); + log("Done\n", LogLevel.INFO); } protected boolean getPreserveLastModified() { @@ -218,7 +218,7 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable if (Files.isDirectory(localFile)) { final Path dir = localFile.resolve(directoryName); Files.createDirectories(dir); - log("Creating: " + dir); + log("Creating: " + dir, LogLevel.DEBUG); return dir; } return null; @@ -231,8 +231,8 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable end = serverResponse.indexOf(' ', start + 1); final long filesize = Long.parseLong(serverResponse.substring(start, end)); final String filename = serverResponse.substring(end + 1); - log("Receiving: " + filename + " : " + filesize); final Path transferFile = Files.isDirectory(localFile) ? localFile.resolve(filename) : localFile; + log("Receiving: " + transferFile + " : " + filesize, LogLevel.INFO); fetchFile(transferFile, filesize, out, in); waitForAck(in); sendAck(out);