diff --git a/src/main/java/net/locusworks/s3sync/Entry.java b/src/main/java/net/locusworks/s3sync/Entry.java index 7ea0044..bcae437 100644 --- a/src/main/java/net/locusworks/s3sync/Entry.java +++ b/src/main/java/net/locusworks/s3sync/Entry.java @@ -35,14 +35,11 @@ public class Entry { logger.info("Starting S3 Sync"); - try { - S3Client client = new S3Client(ConfigurationManager.getInstance()); + try (S3Client client = new S3Client(ConfigurationManager.getInstance())) { client.syncFolder(); } catch (Exception | Error e) { logger.error(e); System.exit(-1); } - } - } diff --git a/src/main/java/net/locusworks/s3sync/client/FileManager.java b/src/main/java/net/locusworks/s3sync/client/FileManager.java index abe2ee4..d37a490 100644 --- a/src/main/java/net/locusworks/s3sync/client/FileManager.java +++ b/src/main/java/net/locusworks/s3sync/client/FileManager.java @@ -9,28 +9,42 @@ import java.nio.file.Paths; import java.util.LinkedHashMap; import java.util.Map; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import com.amazonaws.services.s3.model.S3Object; import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; public class FileManager implements AutoCloseable { - + private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(FileManager.class); - + public static final String FILE_CSV = "upload.csv"; - + private Map detailMap; - + + private S3Client client; + + private String bucket; + private static FileManager instance; - - private FileManager() throws IOException { + + private FileManager(S3Client client) throws IOException { detailMap = new LinkedHashMap(); + this.client = client; + this.bucket = client.getBucket(); readFile(); } private void readFile() throws IOException { + + S3Object hashFile = client.getObject(bucket, FILE_CSV); + if (hashFile == null) return; Path file = Paths.get(FILE_CSV); - if (Files.notExists(file)) return; + client.downloadFile(FILE_CSV, file); + + if (Files.notExists(file)) return; + try(BufferedReader reader = Files.newBufferedReader(file)) { String line = null; while((line = reader.readLine()) != null) { @@ -39,13 +53,13 @@ public class FileManager implements AutoCloseable { logger.warn("Invalid entry detected: " + line); continue; } - + FileDetail fd = new FileDetail(values[0], values[1], Boolean.valueOf(values[2])); detailMap.put(values[0], fd); } } } - + private void saveFile() throws IOException { try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(FILE_CSV))) { writer.write("FILE_NAME,HASH,STATUS\n"); @@ -54,16 +68,16 @@ public class FileManager implements AutoCloseable { }; } } - + public boolean addEntry(String file, String hash, boolean uploaded) { return addEntry(new FileDetail(file, hash, uploaded)); } - + public boolean addEntry(FileDetail fd) { return detailMap.put(fd.getFile(), fd) != null; } - - public FileDetail uploadFile(Path path) throws IOException { + + public FileDetail getFileDetail(Path path) throws IOException { boolean newFile = false; String file = path.toString(); FileDetail fd = null; @@ -73,26 +87,33 @@ public class FileManager implements AutoCloseable { newFile = true; fd = new FileDetail(file, DigestUtils.sha1Hex(Files.newInputStream(path)), false); } - + String sha1 = newFile ? fd.getHash() : DigestUtils.sha1Hex(Files.newInputStream(path)); - + if (!sha1.equals(fd.getHash())) { fd.setUploaded(false); } - + return fd; } @Override public void close() throws Exception { saveFile(); + client.uploadFile(Paths.get(FILE_CSV)); + Files.deleteIfExists(Paths.get(FILE_CSV)); } - + + public static FileManager newInstance(S3Client client) throws IOException { + instance = new FileManager(client); + return instance; + } + public static FileManager getInstance() throws IOException { - if (instance == null) { - instance = new FileManager(); + if (instance == null || instance.client == null || StringUtils.isBlank(instance.bucket)) { + throw new IOException("a call to newInstance() was not made. Please call newInstance first"); } return instance; } - + } diff --git a/src/main/java/net/locusworks/s3sync/client/S3Client.java b/src/main/java/net/locusworks/s3sync/client/S3Client.java index 626c8b0..1763271 100644 --- a/src/main/java/net/locusworks/s3sync/client/S3Client.java +++ b/src/main/java/net/locusworks/s3sync/client/S3Client.java @@ -4,10 +4,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.SdkClientException; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.transfer.Download; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; @@ -15,7 +21,7 @@ import net.locusworks.logger.ApplicationLogger; import net.locusworks.logger.ApplicationLoggerFactory; import net.locusworks.s3sync.conf.ConfigurationManager; -public class S3Client { +public class S3Client implements AutoCloseable { private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(S3Client.class); @@ -37,11 +43,15 @@ public class S3Client { this.bucket = conf.getBucketName(); this.syncFolder = conf.getSyncFolder(); } + + public String getBucket() { + return this.bucket; + } public void uploadFile(Path file) { TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); uploadFile(xferMgr, file); - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); } public void uploadFile(TransferManager xferMgr, Path file) { @@ -49,7 +59,7 @@ public class S3Client { xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); FileDetail fd = null; try { - fd = FileManager.getInstance().uploadFile(file); + fd = FileManager.getInstance().getFileDetail(file); if (fd.isUploaded()) return; logger.info("Uploading file: %s", file); Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile()); @@ -69,14 +79,37 @@ public class S3Client { logger.error(e.getMessage()); } finally { if (xferMgrNull) { - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); + } + } + } + + public void downloadFile(String key, Path file) { + TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); + downloadFile(xferMgr, key, file); + xferMgr.shutdownNow(false); + } + + public void downloadFile(TransferManager xferMgr, String key, Path file) { + boolean xferMgrNull = xferMgr == null; + xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build(); + try { + logger.info("Downloading file: %s", file); + Download xfer = xferMgr.download(bucket, key, file.toFile()); + xfer.waitForCompletion(); + logger.info("Done downloading %s", file); + } catch (AmazonClientException | InterruptedException e) { + logger.error(e.getMessage()); + } finally { + if (xferMgrNull) { + xferMgr.shutdownNow(false); } } } public void syncFolder() throws IOException { TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - try (FileManager manager = FileManager.getInstance()) { + try (FileManager manager = FileManager.newInstance(this)) { Files.walk(syncFolder) .filter(f -> Files.isRegularFile(f)) .forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f))); @@ -84,7 +117,67 @@ public class S3Client { logger.error("Unable to load file Manager: " + e.getMessage()); } - xferMgr.shutdownNow(); + xferMgr.shutdownNow(false); + } + + /** + *

+ * Gets the object stored in Amazon S3 under the specified bucket and key. + *

+ *

+ * Be extremely careful when using this method; the returned Amazon S3 + * object contains a direct stream of data from the HTTP connection. The + * underlying HTTP connection cannot be reused until the user finishes + * reading the data and closes the stream. Also note that if not all data + * is read from the stream then the SDK will abort the underlying connection, + * this may have a negative impact on performance. Therefore: + *

+ * + * If these rules are not followed, the client can run out of resources by + * allocating too many open, but unused, HTTP connections.

+ *

+ * To get an object from Amazon S3, the caller must have + * {@link Permission#Read} access to the object. + *

+ *

+ * If the object fetched is publicly readable, it can also read it by + * pasting its URL into a browser. + *

+ *

+ * For more advanced options (such as downloading only a range of an + * object's content, or placing constraints on when the object should be + * downloaded) callers can use {@link #getObject(GetObjectRequest)}. + *

+ *

+ * If you are accessing AWS + * KMS-encrypted objects, you need to specify the correct region of the + * bucket on your client and configure AWS Signature Version 4 for added + * security. For more information on how to do this, see + * http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html# + * specify-signature-version + *

+ * + * @param bucketName + * The name of the bucket containing the desired object. + * @param key + * The key under which the desired object is stored. + * + * @return The object stored in Amazon S3 in the specified bucket and key. + * + * @throws SdkClientException + * If any errors are encountered in the client while making the + * request or handling the response. + * + */ + public S3Object getObject(String bucketName, String key) { + try { + return s3Client.getObject(bucketName, key); + } catch (AmazonServiceException ex) {} + return null; } private String getPath(Path file) { @@ -92,5 +185,12 @@ public class S3Client { Path relative = syncFolder.relativize(file); return relative.toString().replace("\\", "/"); } + + @Override + public void close() throws Exception { + if (s3Client != null) { + s3Client.shutdown(); + } + } }