added sync folder and file manager
All checks were successful
Locusworks Team/aws-s3-sync/pipeline/head This commit looks good

This commit is contained in:
2020-11-02 20:35:40 -06:00
parent b2a8a83988
commit 572c067b9d
6 changed files with 157 additions and 310 deletions

View File

@@ -1,14 +1,13 @@
package net.locusworks.s3sync.client;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.ExecutorService;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.client.builder.ExecutorFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.transfer.MultipleFileUpload;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
@@ -22,7 +21,7 @@ public class S3Client {
private AmazonS3 s3Client;
private String bucket;
private Path synchFolder;
private Path syncFolder;
public S3Client(ConfigurationManager conf) {
String region = conf.getRegion();
@@ -36,41 +35,62 @@ public class S3Client {
}
logger.info("Found Bucket: %s", bucket);
this.bucket = conf.getBucketName();
this.synchFolder = conf.getSyncFolder();
this.syncFolder = conf.getSyncFolder();
}
public void uploadFile(Path file) {
logger.info("Uploading file: %s", file);
TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
try {
Upload xfer = xferMgr.upload(bucket, file.getFileName().toString(), file.toFile());
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
logger.info("Done uploading %s", file);
} catch (AmazonServiceException e) {
logger.error(e.getErrorMessage());
System.exit(1);
}
uploadFile(xferMgr, file);
xferMgr.shutdownNow();
}
public void syncFolder() {
TransferManager xferMgr = TransferManagerBuilder.standard()
.withS3Client(s3Client)
.build();
MultipleFileUpload xfer = xferMgr.uploadDirectory(bucket, null, synchFolder.toFile(), true);
public void uploadFile(TransferManager xferMgr, Path file) {
boolean xferMgrNull = xferMgr == null;
xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build();
FileDetail fd = null;
try {
fd = FileManager.getInstance().uploadFile(file);
if (fd.isUploaded()) return;
logger.info("Uploading file: %s", file);
Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile());
xfer.waitForCompletion();
fd.setUploaded(true);
FileManager.getInstance().addEntry(fd);
logger.info("Done uploading %s", file);
} catch (AmazonClientException | InterruptedException | IOException e) {
if (fd != null) {
fd.setUploaded(false);
try {
FileManager.getInstance().addEntry(fd);
} catch (IOException e1) {
logger.error("Unable to save file to file manager: " + e1.getMessage());
}
}
logger.error(e.getMessage());
} finally {
if (xferMgrNull) {
xferMgr.shutdownNow();
}
}
}
public void syncFolder() throws IOException {
TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
try (FileManager manager = FileManager.getInstance()) {
Files.walk(syncFolder)
.filter(f -> Files.isRegularFile(f))
.forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f)));
} catch (Exception e) {
logger.error("Unable to load file Manager: " + e.getMessage());
}
xferMgr.shutdownNow();
}
private String getPath(Path file) {
if (file.getParent() == null) return file.getFileName().toString();
Path relative = syncFolder.relativize(file);
return relative.toString().replace("\\", "/");
}
}