Updated how the files are handled

src/main/java/net/locusworks/s3sync/Entry.java

@@ -35,14 +35,11 @@ public class Entry {
     
     logger.info("Starting S3 Sync");
     
-    try {
-      S3Client client = new S3Client(ConfigurationManager.getInstance());
+    try (S3Client client = new S3Client(ConfigurationManager.getInstance())) {
       client.syncFolder();
     } catch (Exception | Error e) {
       logger.error(e);
       System.exit(-1);
     } 
-
   }
-
 }

src/main/java/net/locusworks/s3sync/client/FileManager.java

@@ -9,28 +9,42 @@ import java.nio.file.Paths;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.services.s3.model.S3Object;
 import net.locusworks.logger.ApplicationLogger;
 import net.locusworks.logger.ApplicationLoggerFactory;
 
 public class FileManager implements AutoCloseable {
-  
+
   private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(FileManager.class);
-  
+
   public static final String FILE_CSV = "upload.csv";
-  
+
   private Map<String, FileDetail> detailMap;
-  
+
+  private S3Client client;
+
+  private String bucket;
+
   private static FileManager instance;
-  
-  private FileManager() throws IOException {
+
+  private FileManager(S3Client client) throws IOException {
     detailMap = new LinkedHashMap<String, FileDetail>();
+    this.client = client;
+    this.bucket = client.getBucket();
     readFile();
   }
 
   private void readFile() throws IOException {
+
+    S3Object hashFile = client.getObject(bucket, FILE_CSV);
+    if (hashFile == null) return;
     Path file = Paths.get(FILE_CSV);
-    if (Files.notExists(file)) return;
     
+    client.downloadFile(FILE_CSV, file);
+
+    if (Files.notExists(file)) return;
+
     try(BufferedReader reader = Files.newBufferedReader(file)) {
       String line = null;
       while((line = reader.readLine()) != null) {
@@ -39,13 +53,13 @@ public class FileManager implements AutoCloseable {
           logger.warn("Invalid entry detected: " + line);
           continue;
         }
-        
+
         FileDetail fd = new FileDetail(values[0], values[1], Boolean.valueOf(values[2]));
         detailMap.put(values[0], fd);
       }
     }
   }
-  
+
   private void saveFile() throws IOException {
     try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(FILE_CSV))) {
       writer.write("FILE_NAME,HASH,STATUS\n");
@@ -54,16 +68,16 @@ public class FileManager implements AutoCloseable {
       };
     }
   }
-  
+
   public boolean addEntry(String file, String hash, boolean uploaded) {
     return addEntry(new FileDetail(file, hash, uploaded));
   }
-  
+
   public boolean addEntry(FileDetail fd) {
     return detailMap.put(fd.getFile(), fd) != null;
   }
-  
-  public FileDetail uploadFile(Path path) throws IOException {
+
+  public FileDetail getFileDetail(Path path) throws IOException {
     boolean newFile = false;
     String file = path.toString();
     FileDetail fd = null;
@@ -73,26 +87,33 @@ public class FileManager implements AutoCloseable {
       newFile = true;
       fd = new FileDetail(file, DigestUtils.sha1Hex(Files.newInputStream(path)), false);
     }
-    
+
     String sha1 = newFile ? fd.getHash() : DigestUtils.sha1Hex(Files.newInputStream(path));
-    
+
     if (!sha1.equals(fd.getHash())) {
       fd.setUploaded(false);
     }
-    
+
     return fd;
   }
 
   @Override
   public void close() throws Exception {
     saveFile();
+    client.uploadFile(Paths.get(FILE_CSV));
+    Files.deleteIfExists(Paths.get(FILE_CSV));
   }
-  
+
+  public static FileManager newInstance(S3Client client) throws IOException {
+    instance = new FileManager(client);
+    return instance;
+  }
+
   public static FileManager getInstance() throws IOException {
-    if (instance == null) {
-      instance = new FileManager();
+    if (instance == null || instance.client == null || StringUtils.isBlank(instance.bucket)) {
+      throw new IOException("a call to newInstance() was not made. Please call newInstance first");
     }
     return instance;
   }
-  
+
 }

src/main/java/net/locusworks/s3sync/client/S3Client.java

@@ -4,10 +4,16 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.regions.Regions;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.Permission;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.transfer.Download;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
 import com.amazonaws.services.s3.transfer.Upload;
@@ -15,7 +21,7 @@ import net.locusworks.logger.ApplicationLogger;
 import net.locusworks.logger.ApplicationLoggerFactory;
 import net.locusworks.s3sync.conf.ConfigurationManager;
 
-public class S3Client {
+public class S3Client implements AutoCloseable {
 
   private ApplicationLogger logger = ApplicationLoggerFactory.getLogger(S3Client.class);
 
@@ -37,11 +43,15 @@ public class S3Client {
     this.bucket = conf.getBucketName();
     this.syncFolder = conf.getSyncFolder();
   }
+  
+  public String getBucket() {
+    return this.bucket;
+  }
 
   public void uploadFile(Path file) {
     TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
     uploadFile(xferMgr, file);
-    xferMgr.shutdownNow();
+    xferMgr.shutdownNow(false);
   }
 
   public void uploadFile(TransferManager xferMgr, Path file) {
@@ -49,7 +59,7 @@ public class S3Client {
     xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build();
     FileDetail fd = null;
     try {
-      fd = FileManager.getInstance().uploadFile(file);
+      fd = FileManager.getInstance().getFileDetail(file);
       if (fd.isUploaded()) return;
       logger.info("Uploading file: %s", file);
       Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile());
@@ -69,14 +79,37 @@ public class S3Client {
       logger.error(e.getMessage());
     } finally {
       if (xferMgrNull) {
-        xferMgr.shutdownNow();
+        xferMgr.shutdownNow(false);
+      }
+    }
+  }
+  
+  public void downloadFile(String key, Path file) {
+    TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
+    downloadFile(xferMgr, key, file);
+    xferMgr.shutdownNow(false);
+  }
+
+  public void downloadFile(TransferManager xferMgr, String key, Path file) {
+    boolean xferMgrNull = xferMgr == null;
+    xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build();
+    try {
+      logger.info("Downloading file: %s", file);
+      Download xfer = xferMgr.download(bucket, key, file.toFile());
+      xfer.waitForCompletion();
+      logger.info("Done downloading %s", file);
+    } catch (AmazonClientException | InterruptedException e) {
+      logger.error(e.getMessage());
+    } finally {
+      if (xferMgrNull) {
+        xferMgr.shutdownNow(false);
       }
     }
   }
 
   public void syncFolder() throws IOException {
     TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
-    try (FileManager manager = FileManager.getInstance()) {
+    try (FileManager manager = FileManager.newInstance(this)) {
       Files.walk(syncFolder)
       .filter(f -> Files.isRegularFile(f))
       .forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f)));
@@ -84,7 +117,67 @@ public class S3Client {
       logger.error("Unable to load file Manager: " + e.getMessage());
     } 
 
-    xferMgr.shutdownNow();
+    xferMgr.shutdownNow(false);
+  }
+  
+  /**
+   * <p>
+   * Gets the object stored in Amazon S3 under the specified bucket and key.
+   * </p>
+   * <p>
+   * Be extremely careful when using this method; the returned Amazon S3
+   * object contains a direct stream of data from the HTTP connection. The
+   * underlying HTTP connection cannot be reused until the user finishes
+   * reading the data and closes the stream. Also note that if not all data
+   * is read from the stream then the SDK will abort the underlying connection,
+   * this may have a negative impact on performance. Therefore:
+   * </p>
+   * <ul>
+   * <li>Use the data from the input stream in Amazon S3 object as soon as possible</li>
+   * <li>Read all data from the stream (use {@link GetObjectRequest#setRange(long, long)} to request only the bytes you need)</li>
+   * <li>Close the input stream in Amazon S3 object as soon as possible</li>
+   * </ul>
+   * If these rules are not followed, the client can run out of resources by
+   * allocating too many open, but unused, HTTP connections. </p>
+   * <p>
+   * To get an object from Amazon S3, the caller must have
+   * {@link Permission#Read} access to the object.
+   * </p>
+   * <p>
+   * If the object fetched is publicly readable, it can also read it by
+   * pasting its URL into a browser.
+   * </p>
+   * <p>
+   * For more advanced options (such as downloading only a range of an
+   * object's content, or placing constraints on when the object should be
+   * downloaded) callers can use {@link #getObject(GetObjectRequest)}.
+   * </p>
+   * <p>
+   * If you are accessing <a href="http://aws.amazon.com/kms/">AWS
+   * KMS</a>-encrypted objects, you need to specify the correct region of the
+   * bucket on your client and configure AWS Signature Version 4 for added
+   * security. For more information on how to do this, see
+   * http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#
+   * specify-signature-version
+   * </p>
+   *
+   * @param bucketName
+   *            The name of the bucket containing the desired object.
+   * @param key
+   *            The key under which the desired object is stored.
+   *
+   * @return The object stored in Amazon S3 in the specified bucket and key.
+   *
+   * @throws SdkClientException
+   *             If any errors are encountered in the client while making the
+   *             request or handling the response.
+   *
+   */
+  public S3Object getObject(String bucketName, String key) {
+    try {
+      return s3Client.getObject(bucketName, key);
+    } catch (AmazonServiceException ex) {}
+    return null;
   }
  
   private String getPath(Path file) {
@@ -92,5 +185,12 @@ public class S3Client {
     Path relative = syncFolder.relativize(file);
     return relative.toString().replace("\\", "/");
   }
+  
+  @Override
+  public void close() throws Exception {
+    if (s3Client != null) {
+      s3Client.shutdown();
+    }
+  }
 
 }