From 572c067b9d27a366fe2cd29c8e9bd1b8a03fdb19 Mon Sep 17 00:00:00 2001 From: Isaac Parenteau Date: Mon, 2 Nov 2020 20:35:40 -0600 Subject: [PATCH 1/3] added sync folder and file manager --- .gitignore | 2 + .../java/net/locusworks/s3sync/Entry.java | 3 +- .../locusworks/s3sync/client/FileDetail.java | 10 +- .../locusworks/s3sync/client/FileManager.java | 98 +++++++ .../locusworks/s3sync/client/S3Client.java | 84 +++--- .../s3sync/client/XferMgrProgress.java | 270 ------------------ 6 files changed, 157 insertions(+), 310 deletions(-) create mode 100644 src/main/java/net/locusworks/s3sync/client/FileManager.java delete mode 100644 src/main/java/net/locusworks/s3sync/client/XferMgrProgress.java diff --git a/.gitignore b/.gitignore index 0a06aad..ac68908 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ /logs/ .project /.settings/ +**/*.csv +*.properties diff --git a/src/main/java/net/locusworks/s3sync/Entry.java b/src/main/java/net/locusworks/s3sync/Entry.java index 0a19d94..7ea0044 100644 --- a/src/main/java/net/locusworks/s3sync/Entry.java +++ b/src/main/java/net/locusworks/s3sync/Entry.java @@ -1,6 +1,5 @@ package net.locusworks.s3sync; -import java.nio.file.Paths; import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; import net.locusworks.logger.ApplicationLoggerInitializer; @@ -38,7 +37,7 @@ public class Entry { try { S3Client client = new S3Client(ConfigurationManager.getInstance()); - client.uploadFile(Paths.get("D:\\OneDrive\\Documents\\config.docx")); + client.syncFolder(); } catch (Exception | Error e) { logger.error(e); System.exit(-1); diff --git a/src/main/java/net/locusworks/s3sync/client/FileDetail.java b/src/main/java/net/locusworks/s3sync/client/FileDetail.java index 0b3ed5b..658195e 100644 --- a/src/main/java/net/locusworks/s3sync/client/FileDetail.java +++ b/src/main/java/net/locusworks/s3sync/client/FileDetail.java @@ -1,10 +1,8 @@ package net.locusworks.s3sync.client; -import java.nio.file.Path; - public class FileDetail { - private Path file; + private String file; private String hash; private boolean uploaded; @@ -14,7 +12,7 @@ public class FileDetail { * @param hash * @param uploaded */ - public FileDetail(Path file, String hash, boolean uploaded) { + public FileDetail(String file, String hash, boolean uploaded) { this.file = file; this.hash = hash; this.uploaded = uploaded; @@ -22,13 +20,13 @@ public class FileDetail { /** * @return the file */ - public synchronized final Path getFile() { + public synchronized final String getFile() { return file; } /** * @param file the file to set */ - public synchronized final void setFile(Path file) { + public synchronized final void setFile(String file) { this.file = file; } /** diff --git a/src/main/java/net/locusworks/s3sync/client/FileManager.java b/src/main/java/net/locusworks/s3sync/client/FileManager.java new file mode 100644 index 0000000..abe2ee4 --- /dev/null +++ b/src/main/java/net/locusworks/s3sync/client/FileManager.java @@ -0,0 +1,98 @@ +package net.locusworks.s3sync.client; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.commons.codec.digest.DigestUtils; +import net.locusworks.logger.ApplicationLogger; +import net.locusworks.logger.ApplicationLoggerFactory; + +public class FileManager implements AutoCloseable { + + private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(FileManager.class); + + public static final String FILE_CSV = "upload.csv"; + + private Map detailMap; + + private static FileManager instance; + + private FileManager() throws IOException { + detailMap = new LinkedHashMap(); + readFile(); + } + + private void readFile() throws IOException { + Path file = Paths.get(FILE_CSV); + if (Files.notExists(file)) return; + + try(BufferedReader reader = Files.newBufferedReader(file)) { + String line = null; + while((line = reader.readLine()) != null) { + String[] values = line.split(","); + if (values.length != 3) { + logger.warn("Invalid entry detected: " + line); + continue; + } + + FileDetail fd = new FileDetail(values[0], values[1], Boolean.valueOf(values[2])); + detailMap.put(values[0], fd); + } + } + } + + private void saveFile() throws IOException { + try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(FILE_CSV))) { + writer.write("FILE_NAME,HASH,STATUS\n"); + for(FileDetail v : detailMap.values()) { + writer.write(String.format("%s,%s,%s%n", v.getFile(), v.getHash(), v.isUploaded())); + }; + } + } + + public boolean addEntry(String file, String hash, boolean uploaded) { + return addEntry(new FileDetail(file, hash, uploaded)); + } + + public boolean addEntry(FileDetail fd) { + return detailMap.put(fd.getFile(), fd) != null; + } + + public FileDetail uploadFile(Path path) throws IOException { + boolean newFile = false; + String file = path.toString(); + FileDetail fd = null; + if (detailMap.containsKey(file)) { + fd = detailMap.get(file); + } else { + newFile = true; + fd = new FileDetail(file, DigestUtils.sha1Hex(Files.newInputStream(path)), false); + } + + String sha1 = newFile ? fd.getHash() : DigestUtils.sha1Hex(Files.newInputStream(path)); + + if (!sha1.equals(fd.getHash())) { + fd.setUploaded(false); + } + + return fd; + } + + @Override + public void close() throws Exception { + saveFile(); + } + + public static FileManager getInstance() throws IOException { + if (instance == null) { + instance = new FileManager(); + } + return instance; + } + +} diff --git a/src/main/java/net/locusworks/s3sync/client/S3Client.java b/src/main/java/net/locusworks/s3sync/client/S3Client.java index 61a4fe8..626c8b0 100644 --- a/src/main/java/net/locusworks/s3sync/client/S3Client.java +++ b/src/main/java/net/locusworks/s3sync/client/S3Client.java @@ -1,14 +1,13 @@ package net.locusworks.s3sync.client; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.client.builder.ExecutorFactory; +import com.amazonaws.AmazonClientException; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.transfer.MultipleFileUpload; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; @@ -22,7 +21,7 @@ public class S3Client { private AmazonS3 s3Client; private String bucket; - private Path synchFolder; + private Path syncFolder; public S3Client(ConfigurationManager conf) { String region = conf.getRegion(); @@ -36,41 +35,62 @@ public class S3Client { } logger.info("Found Bucket: %s", bucket); this.bucket = conf.getBucketName(); - this.synchFolder = conf.getSyncFolder(); + this.syncFolder = conf.getSyncFolder(); } public void uploadFile(Path file) { - logger.info("Uploading file: %s", file); TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - try { - Upload xfer = xferMgr.upload(bucket, file.getFileName().toString(), file.toFile()); - // loop with Transfer.isDone() - XferMgrProgress.showTransferProgress(xfer); - // or block with Transfer.waitForCompletion() - XferMgrProgress.waitForCompletion(xfer); - logger.info("Done uploading %s", file); - } catch (AmazonServiceException e) { - logger.error(e.getErrorMessage()); - System.exit(1); - } + uploadFile(xferMgr, file); xferMgr.shutdownNow(); } - public void syncFolder() { - - - TransferManager xferMgr = TransferManagerBuilder.standard() - .withS3Client(s3Client) - .build(); - - MultipleFileUpload xfer = xferMgr.uploadDirectory(bucket, null, synchFolder.toFile(), true); - - - - - - + public void uploadFile(TransferManager xferMgr, Path file) { + boolean xferMgrNull = xferMgr == null; + xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); + FileDetail fd = null; + try { + fd = FileManager.getInstance().uploadFile(file); + if (fd.isUploaded()) return; + logger.info("Uploading file: %s", file); + Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile()); + xfer.waitForCompletion(); + fd.setUploaded(true); + FileManager.getInstance().addEntry(fd); + logger.info("Done uploading %s", file); + } catch (AmazonClientException | InterruptedException | IOException e) { + if (fd != null) { + fd.setUploaded(false); + try { + FileManager.getInstance().addEntry(fd); + } catch (IOException e1) { + logger.error("Unable to save file to file manager: " + e1.getMessage()); + } + } + logger.error(e.getMessage()); + } finally { + if (xferMgrNull) { + xferMgr.shutdownNow(); + } + } } + public void syncFolder() throws IOException { + TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); + try (FileManager manager = FileManager.getInstance()) { + Files.walk(syncFolder) + .filter(f -> Files.isRegularFile(f)) + .forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f))); + } catch (Exception e) { + logger.error("Unable to load file Manager: " + e.getMessage()); + } + + xferMgr.shutdownNow(); + } + + private String getPath(Path file) { + if (file.getParent() == null) return file.getFileName().toString(); + Path relative = syncFolder.relativize(file); + return relative.toString().replace("\\", "/"); + } } diff --git a/src/main/java/net/locusworks/s3sync/client/XferMgrProgress.java b/src/main/java/net/locusworks/s3sync/client/XferMgrProgress.java deleted file mode 100644 index dc46f0f..0000000 --- a/src/main/java/net/locusworks/s3sync/client/XferMgrProgress.java +++ /dev/null @@ -1,270 +0,0 @@ -//snippet-sourcedescription:[XferMgrProgress.java demonstrates how to use the S3 transfermanager to upload files to a bucket and show progress of the upload.] -//snippet-keyword:[Java] -//snippet-sourcesyntax:[java] -//snippet-keyword:[Code Sample] -//snippet-keyword:[Amazon S3] -//snippet-keyword:[TransferProgress] -//snippet-keyword:[TransferManager] -//snippet-service:[s3] -//snippet-sourcetype:[full-example] -//snippet-sourcedate:[] -//snippet-sourceauthor:[soo-aws] -/* - Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - - This file is licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. A copy of - the License is located at - - http://aws.amazon.com/apache2.0/ - - This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - CONDITIONS OF ANY KIND, either express or implied. See the License for the - specific language governing permissions and limitations under the License. -*/ -package net.locusworks.s3sync.client; -// snippet-start:[s3.java1.s3_xfer_mgr_progress.import] - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.event.ProgressEvent; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.services.s3.transfer.*; -import com.amazonaws.services.s3.transfer.Transfer.TransferState; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -// snippet-end:[s3.java1.s3_xfer_mgr_progress.import] - -// snippet-start:[s3.java1.s3_xfer_mgr_progress.complete] -public class XferMgrProgress { - // waits for the transfer to complete, catching any exceptions that occur. - public static void waitForCompletion(Transfer xfer) { - // snippet-start:[s3.java1.s3_xfer_mgr_progress.wait_for_transfer] - try { - xfer.waitForCompletion(); - } catch (AmazonServiceException e) { - System.err.println("Amazon service error: " + e.getMessage()); - System.exit(1); - } catch (AmazonClientException e) { - System.err.println("Amazon client error: " + e.getMessage()); - System.exit(1); - } catch (InterruptedException e) { - System.err.println("Transfer interrupted: " + e.getMessage()); - System.exit(1); - } - // snippet-end:[s3.java1.s3_xfer_mgr_progress.wait_for_transfer] - } - - // Prints progress while waiting for the transfer to finish. - public static void showTransferProgress(Transfer xfer) { - // snippet-start:[s3.java1.s3_xfer_mgr_progress.poll] - // print the transfer's human-readable description - System.out.println(xfer.getDescription()); - // print an empty progress bar... - printProgressBar(0.0); - // update the progress bar while the xfer is ongoing. - do { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - return; - } - // Note: so_far and total aren't used, they're just for - // documentation purposes. - TransferProgress progress = xfer.getProgress(); - long so_far = progress.getBytesTransferred(); - long total = progress.getTotalBytesToTransfer(); - double pct = progress.getPercentTransferred(); - eraseProgressBar(); - printProgressBar(pct); - } while (xfer.isDone() == false); - // print the final state of the transfer. - TransferState xfer_state = xfer.getState(); - System.out.println(": " + xfer_state); - // snippet-end:[s3.java1.s3_xfer_mgr_progress.poll] - } - - // Prints progress of a multiple file upload while waiting for it to finish. - public static void showMultiUploadProgress(MultipleFileUpload multi_upload) { - // print the upload's human-readable description - System.out.println(multi_upload.getDescription()); - - // snippet-start:[s3.java1.s3_xfer_mgr_progress.substranferes] - Collection sub_xfers = new ArrayList(); - sub_xfers = multi_upload.getSubTransfers(); - - do { - System.out.println("\nSubtransfer progress:\n"); - for (Upload u : sub_xfers) { - System.out.println(" " + u.getDescription()); - if (u.isDone()) { - TransferState xfer_state = u.getState(); - System.out.println(" " + xfer_state); - } else { - TransferProgress progress = u.getProgress(); - double pct = progress.getPercentTransferred(); - printProgressBar(pct); - System.out.println(); - } - } - - // wait a bit before the next update. - try { - Thread.sleep(200); - } catch (InterruptedException e) { - return; - } - } while (multi_upload.isDone() == false); - // print the final state of the transfer. - TransferState xfer_state = multi_upload.getState(); - System.out.println("\nMultipleFileUpload " + xfer_state); - // snippet-end:[s3.java1.s3_xfer_mgr_progress.substranferes] - } - - // prints a simple text progressbar: [##### ] - public static void printProgressBar(double pct) { - // if bar_size changes, then change erase_bar (in eraseProgressBar) to - // match. - final int bar_size = 40; - final String empty_bar = " "; - final String filled_bar = "########################################"; - int amt_full = (int) (bar_size * (pct / 100.0)); - System.out.format(" [%s%s]", filled_bar.substring(0, amt_full), - empty_bar.substring(0, bar_size - amt_full)); - } - - // erases the progress bar. - public static void eraseProgressBar() { - // erase_bar is bar_size (from printProgressBar) + 4 chars. - final String erase_bar = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"; - System.out.format(erase_bar); - } - - public static void uploadFileWithListener(String file_path, - String bucket_name, String key_prefix, boolean pause) { - System.out.println("file: " + file_path + - (pause ? " (pause)" : "")); - - String key_name = null; - if (key_prefix != null) { - key_name = key_prefix + '/' + file_path; - } else { - key_name = file_path; - } - - // snippet-start:[s3.java1.s3_xfer_mgr_progress.progress_listener] - File f = new File(file_path); - TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); - try { - Upload u = xfer_mgr.upload(bucket_name, key_name, f); - // print an empty progress bar... - printProgressBar(0.0); - u.addProgressListener(new ProgressListener() { - public void progressChanged(ProgressEvent e) { - double pct = e.getBytesTransferred() * 100.0 / e.getBytes(); - eraseProgressBar(); - printProgressBar(pct); - } - }); - // block with Transfer.waitForCompletion() - XferMgrProgress.waitForCompletion(u); - // print the final state of the transfer. - TransferState xfer_state = u.getState(); - System.out.println(": " + xfer_state); - } catch (AmazonServiceException e) { - System.err.println(e.getErrorMessage()); - System.exit(1); - } - xfer_mgr.shutdownNow(); - // snippet-end:[s3.java1.s3_xfer_mgr_progress.progress_listener] - } - - public static void uploadDirWithSubprogress(String dir_path, - String bucket_name, String key_prefix, boolean recursive, - boolean pause) { - System.out.println("directory: " + dir_path + (recursive ? - " (recursive)" : "") + (pause ? " (pause)" : "")); - - TransferManager xfer_mgr = new TransferManager(); - try { - MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory( - bucket_name, key_prefix, new File(dir_path), recursive); - // loop with Transfer.isDone() - XferMgrProgress.showMultiUploadProgress(multi_upload); - // or block with Transfer.waitForCompletion() - XferMgrProgress.waitForCompletion(multi_upload); - } catch (AmazonServiceException e) { - System.err.println(e.getErrorMessage()); - System.exit(1); - } - xfer_mgr.shutdownNow(); - } - - public static void main(String[] args) { - final String USAGE = "\n" + - "Usage:\n" + - " XferMgrProgress [--recursive] [--pause] \n\n" + - "Where:\n" + - " --recursive - Only applied if local_path is a directory.\n" + - " Copies the contents of the directory recursively.\n\n" + - " --pause - Attempt to pause+resume the upload. This may not work for\n" + - " small files.\n\n" + - " s3_path - The S3 destination (bucket/path) to upload the file(s) to.\n\n" + - " local_path - The path to a local file or directory path to upload to S3.\n\n" + - "Examples:\n" + - " XferMgrProgress public_photos/cat_happy.png my_photos/funny_cat.png\n" + - " XferMgrProgress public_photos my_photos/cat_sad.png\n" + - " XferMgrProgress public_photos my_photos\n\n"; - - if (args.length < 2) { - System.out.println(USAGE); - System.exit(1); - } - - int cur_arg = 0; - boolean recursive = false; - boolean pause = false; - - // first, parse any switches - while (args[cur_arg].startsWith("--")) { - if (args[cur_arg].equals("--recursive")) { - recursive = true; - } else if (args[cur_arg].equals("--pause")) { - pause = true; - } else { - System.out.println("Unknown argument: " + args[cur_arg]); - System.out.println(USAGE); - System.exit(1); - } - cur_arg += 1; - } - - // only the first '/' character is of interest to get the bucket name. - // Subsequent ones are part of the key name. - String[] s3_path = args[cur_arg].split("/", 2); - cur_arg += 1; - - String bucket_name = s3_path[0]; - String key_prefix = null; - if (s3_path.length > 1) { - key_prefix = s3_path[1]; - } - - String local_path = args[cur_arg]; - - // check to see if local path is a directory or file... - File f = new File(args[cur_arg]); - if (f.exists() == false) { - System.out.println("Input path doesn't exist: " + args[cur_arg]); - System.exit(1); - } else if (f.isDirectory()) { - uploadDirWithSubprogress(local_path, bucket_name, key_prefix, - recursive, pause); - } else { - uploadFileWithListener(local_path, bucket_name, key_prefix, pause); - } - } -} -// snippet-end:[s3.java1.s3_xfer_mgr_progress.complete] \ No newline at end of file From 70f54ea2eec8064908c3a9246d4088f19975ebcc Mon Sep 17 00:00:00 2001 From: Isaac Parenteau Date: Tue, 3 Nov 2020 22:32:58 -0600 Subject: [PATCH 2/3] Updated how the files are handled --- .../java/net/locusworks/s3sync/Entry.java | 5 +- .../locusworks/s3sync/client/FileManager.java | 61 ++++++---- .../locusworks/s3sync/client/S3Client.java | 112 +++++++++++++++++- 3 files changed, 148 insertions(+), 30 deletions(-) diff --git a/src/main/java/net/locusworks/s3sync/Entry.java b/src/main/java/net/locusworks/s3sync/Entry.java index 7ea0044..bcae437 100644 --- a/src/main/java/net/locusworks/s3sync/Entry.java +++ b/src/main/java/net/locusworks/s3sync/Entry.java @@ -35,14 +35,11 @@ public class Entry { logger.info("Starting S3 Sync"); - try { - S3Client client = new S3Client(ConfigurationManager.getInstance()); + try (S3Client client = new S3Client(ConfigurationManager.getInstance())) { client.syncFolder(); } catch (Exception | Error e) { logger.error(e); System.exit(-1); } - } - } diff --git a/src/main/java/net/locusworks/s3sync/client/FileManager.java b/src/main/java/net/locusworks/s3sync/client/FileManager.java index abe2ee4..d37a490 100644 --- a/src/main/java/net/locusworks/s3sync/client/FileManager.java +++ b/src/main/java/net/locusworks/s3sync/client/FileManager.java @@ -9,28 +9,42 @@ import java.nio.file.Paths; import java.util.LinkedHashMap; import java.util.Map; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import com.amazonaws.services.s3.model.S3Object; import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; public class FileManager implements AutoCloseable { - + private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(FileManager.class); - + public static final String FILE_CSV = "upload.csv"; - + private Map detailMap; - + + private S3Client client; + + private String bucket; + private static FileManager instance; - - private FileManager() throws IOException { + + private FileManager(S3Client client) throws IOException { detailMap = new LinkedHashMap(); + this.client = client; + this.bucket = client.getBucket(); readFile(); } private void readFile() throws IOException { + + S3Object hashFile = client.getObject(bucket, FILE_CSV); + if (hashFile == null) return; Path file = Paths.get(FILE_CSV); - if (Files.notExists(file)) return; + client.downloadFile(FILE_CSV, file); + + if (Files.notExists(file)) return; + try(BufferedReader reader = Files.newBufferedReader(file)) { String line = null; while((line = reader.readLine()) != null) { @@ -39,13 +53,13 @@ public class FileManager implements AutoCloseable { logger.warn("Invalid entry detected: " + line); continue; } - + FileDetail fd = new FileDetail(values[0], values[1], Boolean.valueOf(values[2])); detailMap.put(values[0], fd); } } } - + private void saveFile() throws IOException { try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(FILE_CSV))) { writer.write("FILE_NAME,HASH,STATUS\n"); @@ -54,16 +68,16 @@ public class FileManager implements AutoCloseable { }; } } - + public boolean addEntry(String file, String hash, boolean uploaded) { return addEntry(new FileDetail(file, hash, uploaded)); } - + public boolean addEntry(FileDetail fd) { return detailMap.put(fd.getFile(), fd) != null; } - - public FileDetail uploadFile(Path path) throws IOException { + + public FileDetail getFileDetail(Path path) throws IOException { boolean newFile = false; String file = path.toString(); FileDetail fd = null; @@ -73,26 +87,33 @@ public class FileManager implements AutoCloseable { newFile = true; fd = new FileDetail(file, DigestUtils.sha1Hex(Files.newInputStream(path)), false); } - + String sha1 = newFile ? fd.getHash() : DigestUtils.sha1Hex(Files.newInputStream(path)); - + if (!sha1.equals(fd.getHash())) { fd.setUploaded(false); } - + return fd; } @Override public void close() throws Exception { saveFile(); + client.uploadFile(Paths.get(FILE_CSV)); + Files.deleteIfExists(Paths.get(FILE_CSV)); } - + + public static FileManager newInstance(S3Client client) throws IOException { + instance = new FileManager(client); + return instance; + } + public static FileManager getInstance() throws IOException { - if (instance == null) { - instance = new FileManager(); + if (instance == null || instance.client == null || StringUtils.isBlank(instance.bucket)) { + throw new IOException("a call to newInstance() was not made. Please call newInstance first"); } return instance; } - + } diff --git a/src/main/java/net/locusworks/s3sync/client/S3Client.java b/src/main/java/net/locusworks/s3sync/client/S3Client.java index 626c8b0..1763271 100644 --- a/src/main/java/net/locusworks/s3sync/client/S3Client.java +++ b/src/main/java/net/locusworks/s3sync/client/S3Client.java @@ -4,10 +4,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.SdkClientException; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.transfer.Download; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; @@ -15,7 +21,7 @@ import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; import net.locusworks.s3sync.conf.ConfigurationManager; -public class S3Client { +public class S3Client implements AutoCloseable { private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(S3Client.class); @@ -37,11 +43,15 @@ public class S3Client { this.bucket = conf.getBucketName(); this.syncFolder = conf.getSyncFolder(); } + + public String getBucket() { + return this.bucket; + } public void uploadFile(Path file) { TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); uploadFile(xferMgr, file); - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); } public void uploadFile(TransferManager xferMgr, Path file) { @@ -49,7 +59,7 @@ public class S3Client { xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); FileDetail fd = null; try { - fd = FileManager.getInstance().uploadFile(file); + fd = FileManager.getInstance().getFileDetail(file); if (fd.isUploaded()) return; logger.info("Uploading file: %s", file); Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile()); @@ -69,14 +79,37 @@ public class S3Client { logger.error(e.getMessage()); } finally { if (xferMgrNull) { - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); + } + } + } + + public void downloadFile(String key, Path file) { + TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); + downloadFile(xferMgr, key, file); + xferMgr.shutdownNow(false); + } + + public void downloadFile(TransferManager xferMgr, String key, Path file) { + boolean xferMgrNull = xferMgr == null; + xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); + try { + logger.info("Downloading file: %s", file); + Download xfer = xferMgr.download(bucket, key, file.toFile()); + xfer.waitForCompletion(); + logger.info("Done downloading %s", file); + } catch (AmazonClientException | InterruptedException e) { + logger.error(e.getMessage()); + } finally { + if (xferMgrNull) { + xferMgr.shutdownNow(false); } } } public void syncFolder() throws IOException { TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - try (FileManager manager = FileManager.getInstance()) { + try (FileManager manager = FileManager.newInstance(this)) { Files.walk(syncFolder) .filter(f -> Files.isRegularFile(f)) .forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f))); @@ -84,7 +117,67 @@ public class S3Client { logger.error("Unable to load file Manager: " + e.getMessage()); } - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); + } + + /** + *

+ * Gets the object stored in Amazon S3 under the specified bucket and key. + *

+ *

+ * Be extremely careful when using this method; the returned Amazon S3 + * object contains a direct stream of data from the HTTP connection. The + * underlying HTTP connection cannot be reused until the user finishes + * reading the data and closes the stream. Also note that if not all data + * is read from the stream then the SDK will abort the underlying connection, + * this may have a negative impact on performance. Therefore: + *

+ *
    + *
  • Use the data from the input stream in Amazon S3 object as soon as possible
    • + *
  • Read all data from the stream (use {@link GetObjectRequest#setRange(long, long)} to request only the bytes you need)
    • + *
  • Close the input stream in Amazon S3 object as soon as possible
    • + *
+ * If these rules are not followed, the client can run out of resources by + * allocating too many open, but unused, HTTP connections.

+ *

+ * To get an object from Amazon S3, the caller must have + * {@link Permission#Read} access to the object. + *

+ *

+ * If the object fetched is publicly readable, it can also read it by + * pasting its URL into a browser. + *

+ *

+ * For more advanced options (such as downloading only a range of an + * object's content, or placing constraints on when the object should be + * downloaded) callers can use {@link #getObject(GetObjectRequest)}. + *

+ *

+ * If you are accessing AWS + * KMS-encrypted objects, you need to specify the correct region of the + * bucket on your client and configure AWS Signature Version 4 for added + * security. For more information on how to do this, see + * http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html# + * specify-signature-version + *

+ * + * @param bucketName + * The name of the bucket containing the desired object. + * @param key + * The key under which the desired object is stored. + * + * @return The object stored in Amazon S3 in the specified bucket and key. + * + * @throws SdkClientException + * If any errors are encountered in the client while making the + * request or handling the response. + * + */ + public S3Object getObject(String bucketName, String key) { + try { + return s3Client.getObject(bucketName, key); + } catch (AmazonServiceException ex) {} + return null; } private String getPath(Path file) { @@ -92,5 +185,12 @@ public class S3Client { Path relative = syncFolder.relativize(file); return relative.toString().replace("\\", "/"); } + + @Override + public void close() throws Exception { + if (s3Client != null) { + s3Client.shutdown(); + } + } } From e12e45d2aab327698bba3d4111d3598f5371d57c Mon Sep 17 00:00:00 2001 From: Isaac Parenteau Date: Wed, 11 Nov 2020 23:15:23 -0600 Subject: [PATCH 3/3] Added logic to check the s3 bucket to make sure the file exists and delete orphan files --- pom.xml | 74 ++++++++++++++++ .../java/net/locusworks/s3sync/Entry.java | 3 + .../locusworks/s3sync/client/FileManager.java | 29 +++++-- .../locusworks/s3sync/client/S3Client.java | 85 ++++++------------- 4 files changed, 123 insertions(+), 68 deletions(-) diff --git a/pom.xml b/pom.xml index 9e3145e..71c374c 100644 --- a/pom.xml +++ b/pom.xml @@ -12,8 +12,82 @@ https://nexus.locusworks.net 1.8 1.8 + 6.0.2 + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M2 + + + + + + + + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + + + + org.owasp + dependency-check-maven + ${dep.check.version} + + true + true + false + true + + + + + check + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + S3Sync-${project.version} + + + net.locusworks.s3sync.Entry + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + com.amazonaws diff --git a/src/main/java/net/locusworks/s3sync/Entry.java b/src/main/java/net/locusworks/s3sync/Entry.java index bcae437..99acab5 100644 --- a/src/main/java/net/locusworks/s3sync/Entry.java +++ b/src/main/java/net/locusworks/s3sync/Entry.java @@ -4,6 +4,7 @@ import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; import net.locusworks.logger.ApplicationLoggerInitializer; import net.locusworks.logger.LogLevel; +import net.locusworks.s3sync.client.FileManager; import net.locusworks.s3sync.client.S3Client; import net.locusworks.s3sync.conf.ConfigurationManager; @@ -36,7 +37,9 @@ public class Entry { logger.info("Starting S3 Sync"); try (S3Client client = new S3Client(ConfigurationManager.getInstance())) { + FileManager manager = FileManager.newInstance(client); client.syncFolder(); + manager.removeOrphanedFiles(); } catch (Exception | Error e) { logger.error(e); System.exit(-1); diff --git a/src/main/java/net/locusworks/s3sync/client/FileManager.java b/src/main/java/net/locusworks/s3sync/client/FileManager.java index d37a490..18b027b 100644 --- a/src/main/java/net/locusworks/s3sync/client/FileManager.java +++ b/src/main/java/net/locusworks/s3sync/client/FileManager.java @@ -6,8 +6,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import com.amazonaws.services.s3.model.S3Object; @@ -21,6 +23,8 @@ public class FileManager implements AutoCloseable { public static final String FILE_CSV = "upload.csv"; private Map detailMap; + + private Set s3Files; private S3Client client; @@ -30,19 +34,20 @@ public class FileManager implements AutoCloseable { private FileManager(S3Client client) throws IOException { detailMap = new LinkedHashMap(); + s3Files = new HashSet(); this.client = client; this.bucket = client.getBucket(); readFile(); } private void readFile() throws IOException { - + S3Object hashFile = client.getObject(bucket, FILE_CSV); if (hashFile == null) return; Path file = Paths.get(FILE_CSV); client.downloadFile(FILE_CSV, file); - + s3Files = client.getFileList(); if (Files.notExists(file)) return; try(BufferedReader reader = Files.newBufferedReader(file)) { @@ -77,25 +82,31 @@ public class FileManager implements AutoCloseable { return detailMap.put(fd.getFile(), fd) != null; } - public FileDetail getFileDetail(Path path) throws IOException { + public FileDetail getFileDetail(Path path, String key) throws IOException { boolean newFile = false; - String file = path.toString(); FileDetail fd = null; - if (detailMap.containsKey(file)) { - fd = detailMap.get(file); + if (detailMap.containsKey(key)) { + fd = detailMap.get(key); } else { newFile = true; - fd = new FileDetail(file, DigestUtils.sha1Hex(Files.newInputStream(path)), false); + fd = new FileDetail(key, DigestUtils.sha1Hex(Files.newInputStream(path)), false); } String sha1 = newFile ? fd.getHash() : DigestUtils.sha1Hex(Files.newInputStream(path)); - - if (!sha1.equals(fd.getHash())) { + + if (sha1.equals(fd.getHash()) && s3Files.contains(key)) { + fd.setUploaded(true); + s3Files.remove(key); + } else { fd.setUploaded(false); } return fd; } + + public void removeOrphanedFiles() { + client.removeFiles(s3Files); + } @Override public void close() throws Exception { diff --git a/src/main/java/net/locusworks/s3sync/client/S3Client.java b/src/main/java/net/locusworks/s3sync/client/S3Client.java index 1763271..e2c01f1 100644 --- a/src/main/java/net/locusworks/s3sync/client/S3Client.java +++ b/src/main/java/net/locusworks/s3sync/client/S3Client.java @@ -3,16 +3,16 @@ package net.locusworks.s3sync.client; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkClientException; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.Permission; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.transfer.Download; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -59,10 +59,11 @@ public class S3Client implements AutoCloseable { xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); FileDetail fd = null; try { - fd = FileManager.getInstance().getFileDetail(file); + String key = getPath(file); + fd = FileManager.getInstance().getFileDetail(file, key); if (fd.isUploaded()) return; logger.info("Uploading file: %s", file); - Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile()); + Upload xfer = xferMgr.upload(bucket, key, file.toFile()); xfer.waitForCompletion(); fd.setUploaded(true); FileManager.getInstance().addEntry(fd); @@ -109,7 +110,7 @@ public class S3Client implements AutoCloseable { public void syncFolder() throws IOException { TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - try (FileManager manager = FileManager.newInstance(this)) { + try (FileManager manager = FileManager.getInstance()) { Files.walk(syncFolder) .filter(f -> Files.isRegularFile(f)) .forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f))); @@ -120,65 +121,31 @@ public class S3Client implements AutoCloseable { xferMgr.shutdownNow(false); } - /** - *

- * Gets the object stored in Amazon S3 under the specified bucket and key. - *

- *

- * Be extremely careful when using this method; the returned Amazon S3 - * object contains a direct stream of data from the HTTP connection. The - * underlying HTTP connection cannot be reused until the user finishes - * reading the data and closes the stream. Also note that if not all data - * is read from the stream then the SDK will abort the underlying connection, - * this may have a negative impact on performance. Therefore: - *

- *
    - *
  • Use the data from the input stream in Amazon S3 object as soon as possible
    • - *
  • Read all data from the stream (use {@link GetObjectRequest#setRange(long, long)} to request only the bytes you need)
    • - *
  • Close the input stream in Amazon S3 object as soon as possible
    • - *
- * If these rules are not followed, the client can run out of resources by - * allocating too many open, but unused, HTTP connections.

- *

- * To get an object from Amazon S3, the caller must have - * {@link Permission#Read} access to the object. - *

- *

- * If the object fetched is publicly readable, it can also read it by - * pasting its URL into a browser. - *

- *

- * For more advanced options (such as downloading only a range of an - * object's content, or placing constraints on when the object should be - * downloaded) callers can use {@link #getObject(GetObjectRequest)}. - *

- *

- * If you are accessing AWS - * KMS-encrypted objects, you need to specify the correct region of the - * bucket on your client and configure AWS Signature Version 4 for added - * security. For more information on how to do this, see - * http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html# - * specify-signature-version - *

- * - * @param bucketName - * The name of the bucket containing the desired object. - * @param key - * The key under which the desired object is stored. - * - * @return The object stored in Amazon S3 in the specified bucket and key. - * - * @throws SdkClientException - * If any errors are encountered in the client while making the - * request or handling the response. - * - */ public S3Object getObject(String bucketName, String key) { try { return s3Client.getObject(bucketName, key); } catch (AmazonServiceException ex) {} return null; } + + public Set getFileList() { + Set fileList = new HashSet(); + for(S3ObjectSummary os: s3Client.listObjectsV2(getBucket()).getObjectSummaries()) { + fileList.add(os.getKey()); + } + return fileList; + } + + public void removeFiles(Set s3Files) { + for(String key : s3Files) { + try { + logger.info("Removing file: " + key); + s3Client.deleteObject(getBucket(), key); + } catch (AmazonServiceException ex) { + logger.warn(String.format("Unable to delete %s: %s", key, ex.getMessage()), ex); + } + } + } private String getPath(Path file) { if (file.getParent() == null) return file.getFileName().toString();