locusworks/aws-s3-sync
3
0
Fork 0
Code Issues 2 Pull Requests Releases 2 Wiki Activity

Added logic to remove output dir and changed logging

Browse Source
This commit is contained in:
Isaac Parenteau
2020-11-25 00:49:36 -06:00
parent 4b61c811be
commit 6d6f0ed891
3 changed files with 42 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/net/locusworks/s3sync/client/S3Client.java
View File

@ -1,8 +1,10 @@
package net.locusworks.s3sync.client;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import com.amazonaws.AmazonClientException;
@ -138,12 +140,15 @@ public class S3Client implements AutoCloseable {
jsch.addIdentity(identity);
Session session = jsch.getSession(user, host, port);
session.connect();
try(ScpFromMessage msg = new ScpFromMessage(true, session, remoteFolder, localFolder, true);
FileManager manager = FileManager.getInstance()) {
try(ScpFromMessage msg = new ScpFromMessage(true, session, remoteFolder, localFolder, true); FileManager manager = FileManager.getInstance()) {
msg.fileCallback(p -> {
logger.info(p+"");
if (Files.isRegularFile(p)) {
uploadFile(xferMgr, p);
try {
Files.delete(p);
} catch (Exception ex) {
logger.warn("Unable to delete local file %s: %s", p, ex.getMessage());
}
}
});
msg.execute();
@ -152,6 +157,10 @@ public class S3Client implements AutoCloseable {
}
xferMgr.shutdownNow(false);
Files.walk(localFolder)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
public S3Object getObject(String bucketName, String key) {

28
src/main/java/net/locusworks/s3sync/scp/AbstractSshMessage.java
View File

@ -13,6 +13,7 @@ import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpProgressMonitor;
import net.locusworks.logger.ApplicationLogger;
import net.locusworks.logger.LogLevel;
public abstract class AbstractSshMessage {
@ -138,8 +139,29 @@ public abstract class AbstractSshMessage {
* Log a message to the log listener.
* @param message the message to log
*/
protected void log(final String message) {
logger.info(message);
protected void log(final String message, LogLevel logLevel) {
switch (logLevel) {
case DEBUG:
logger.debug(message);
break;
case ERROR:
case FATAL:
logger.error(message);
break;
case INFO:
logger.info(message);
break;
case WARN:
logger.warn(message);
break;
case TRACE:
logger.trace(message);
break;
case OFF:
default:
logger.trace(message);
break;
}
}
/**
@ -153,7 +175,7 @@ public abstract class AbstractSshMessage {
final NumberFormat format = NumberFormat.getNumberInstance();
format.setMaximumFractionDigits(2);
format.setMinimumFractionDigits(1);
log("File transfer time: " + format.format(duration) + " Average Rate: " + format.format(totalLength / duration) + " B/s");
log("File transfer time: " + format.format(duration) + " Average Rate: " + format.format(totalLength / duration) + " B/s", LogLevel.DEBUG);
}
/**

10
src/main/java/net/locusworks/s3sync/scp/ScpFromMessage.java
View File

@ -36,6 +36,7 @@ import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpATTRS;
import com.jcraft.jsch.SftpException;
import net.locusworks.logger.ApplicationLoggerFactory;
import net.locusworks.logger.LogLevel;
import net.locusworks.logger.ApplicationLogger;
public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable {
@ -153,9 +154,8 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable
command += "-C ";
}
command += remoteFile;
log(command);
log("Executing command: " + command, LogLevel.DEBUG);
final Channel channel = openExecChannel(command);
log(localFile + "");
try {
// get I/O streams for remote scp
final OutputStream out = channel.getOutputStream();
@ -170,7 +170,7 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable
channel.disconnect();
}
}
log("done\n");
log("Done\n", LogLevel.INFO);
}
protected boolean getPreserveLastModified() {
@ -218,7 +218,7 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable
if (Files.isDirectory(localFile)) {
final Path dir = localFile.resolve(directoryName);
Files.createDirectories(dir);
log("Creating: " + dir);
log("Creating: " + dir, LogLevel.DEBUG);
return dir;
}
return null;
@ -231,8 +231,8 @@ public class ScpFromMessage extends AbstractSshMessage implements AutoCloseable
end = serverResponse.indexOf(' ', start + 1);
final long filesize = Long.parseLong(serverResponse.substring(start, end));
final String filename = serverResponse.substring(end + 1);
log("Receiving: " + filename + " : " + filesize);
final Path transferFile = Files.isDirectory(localFile) ? localFile.resolve(filename) : localFile;
log("Receiving: " + transferFile + " : " + filesize, LogLevel.INFO);
fetchFile(transferFile, filesize, out, in);
waitForAck(in);
sendAck(out);