added sync folder and file manager
All checks were successful
Locusworks Team/aws-s3-sync/pipeline/head This commit looks good
All checks were successful
Locusworks Team/aws-s3-sync/pipeline/head This commit looks good
This commit is contained in:
@@ -1,14 +1,13 @@
|
||||
package net.locusworks.s3sync.client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.client.builder.ExecutorFactory;
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.Bucket;
|
||||
import com.amazonaws.services.s3.transfer.MultipleFileUpload;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
@@ -22,7 +21,7 @@ public class S3Client {
|
||||
|
||||
private AmazonS3 s3Client;
|
||||
private String bucket;
|
||||
private Path synchFolder;
|
||||
private Path syncFolder;
|
||||
|
||||
public S3Client(ConfigurationManager conf) {
|
||||
String region = conf.getRegion();
|
||||
@@ -36,41 +35,62 @@ public class S3Client {
|
||||
}
|
||||
logger.info("Found Bucket: %s", bucket);
|
||||
this.bucket = conf.getBucketName();
|
||||
this.synchFolder = conf.getSyncFolder();
|
||||
this.syncFolder = conf.getSyncFolder();
|
||||
}
|
||||
|
||||
public void uploadFile(Path file) {
|
||||
logger.info("Uploading file: %s", file);
|
||||
TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
|
||||
try {
|
||||
Upload xfer = xferMgr.upload(bucket, file.getFileName().toString(), file.toFile());
|
||||
// loop with Transfer.isDone()
|
||||
XferMgrProgress.showTransferProgress(xfer);
|
||||
// or block with Transfer.waitForCompletion()
|
||||
XferMgrProgress.waitForCompletion(xfer);
|
||||
logger.info("Done uploading %s", file);
|
||||
} catch (AmazonServiceException e) {
|
||||
logger.error(e.getErrorMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
uploadFile(xferMgr, file);
|
||||
xferMgr.shutdownNow();
|
||||
}
|
||||
|
||||
public void syncFolder() {
|
||||
|
||||
|
||||
TransferManager xferMgr = TransferManagerBuilder.standard()
|
||||
.withS3Client(s3Client)
|
||||
.build();
|
||||
|
||||
MultipleFileUpload xfer = xferMgr.uploadDirectory(bucket, null, synchFolder.toFile(), true);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public void uploadFile(TransferManager xferMgr, Path file) {
|
||||
boolean xferMgrNull = xferMgr == null;
|
||||
xferMgr = !xferMgrNull ? xferMgr : TransferManagerBuilder.standard().withS3Client(s3Client).build();
|
||||
FileDetail fd = null;
|
||||
try {
|
||||
fd = FileManager.getInstance().uploadFile(file);
|
||||
if (fd.isUploaded()) return;
|
||||
logger.info("Uploading file: %s", file);
|
||||
Upload xfer = xferMgr.upload(bucket, getPath(file), file.toFile());
|
||||
xfer.waitForCompletion();
|
||||
fd.setUploaded(true);
|
||||
FileManager.getInstance().addEntry(fd);
|
||||
logger.info("Done uploading %s", file);
|
||||
} catch (AmazonClientException | InterruptedException | IOException e) {
|
||||
if (fd != null) {
|
||||
fd.setUploaded(false);
|
||||
try {
|
||||
FileManager.getInstance().addEntry(fd);
|
||||
} catch (IOException e1) {
|
||||
logger.error("Unable to save file to file manager: " + e1.getMessage());
|
||||
}
|
||||
}
|
||||
logger.error(e.getMessage());
|
||||
} finally {
|
||||
if (xferMgrNull) {
|
||||
xferMgr.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void syncFolder() throws IOException {
|
||||
TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
|
||||
try (FileManager manager = FileManager.getInstance()) {
|
||||
Files.walk(syncFolder)
|
||||
.filter(f -> Files.isRegularFile(f))
|
||||
.forEach(f -> uploadFile(xferMgr, syncFolder.resolve(f)));
|
||||
} catch (Exception e) {
|
||||
logger.error("Unable to load file Manager: " + e.getMessage());
|
||||
}
|
||||
|
||||
xferMgr.shutdownNow();
|
||||
}
|
||||
|
||||
private String getPath(Path file) {
|
||||
if (file.getParent() == null) return file.getFileName().toString();
|
||||
Path relative = syncFolder.relativize(file);
|
||||
return relative.toString().replace("\\", "/");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user