Local S3 Bucket Development Setup With Docker

docker-compose.yaml

version: '2.2'
services:
  local-s3:
    image: localstack/localstack:1.4.0
    environment:
      SERVICES: s3
      START_WEB: 0
    ports:
      - 4566:4566

Start docker-compose and then to list buckets, enter:

aws --profile localstack --endpoint-url https://localhost:4566  s3 ls

# **Create bucket**

aws s3api create-bucket --bucket conductor --profile localstack --endpoint-url=http://localhost:4566    
```

# **Post file to s3**


curl --location 'http://localhost:13300/api/v1/file-storage/test-filename1' \
--form 'file=@"/location-to/test-file.png"'

Java/Kotlin Code

@Singleton
class S3FileStorage(private val config: S3FileStorageConfig, private val monitoring: FileMetrics) : FileStorage {

    private val logger = KotlinLogging.logger {}
    private val s3Client = S3Client.builder()
        .region(Region.of(config.region()))
        .credentialsProvider(
            StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    config.accessKey(),
                    config.secretKey()
                )
            )
        )
        .let {
            if (config.endpoint() != null) {
                it.endpointOverride(URI.create(config.endpoint()))
                    .forcePathStyle(true)

            } else
                it
        }
        .build()

    init {
        logger.info { "Starting S3FileService with configuration ${config.toStringWithoutSecretKey()}" }
    }

    override fun createFile(filePath: String, content: ByteBuffer) {
        try {
            s3Client.putObject(
                PutObjectRequest.builder().bucket(config.bucketName()).key(filePath).build(),
                RequestBody.fromByteBuffer(content)
            )
            logger.info("File $filePath created successfully.")
            monitoring.onCreateFile(filePath)
        } catch (e: S3Exception) {
            monitoring.onCreateFileFailed(e)
            logger.error("File creation failed", e)
            throw e
        }
    }

    override fun createFile(filePath: String, content: List<InputPart>) {
        try {
            multipartUpload(filePath, content)
            logger.info("File $filePath created successfully.")
            monitoring.onCreateFile(filePath)
        } catch (e: S3Exception) {
            monitoring.onCreateFileFailed(e)
            logger.error("File creation failed", e)
            throw e
        }
    }

    private fun multipartUpload(filePath: String, content: List<InputPart>) {
        istr.use { content ->
            // Initiate a multipart upload
            val createRequest = CreateMultipartUploadRequest.builder()
                .bucket(config.bucketName())
                .key(filePath)
                .build()

            val createResponse: CreateMultipartUploadResponse = s3Client.createMultipartUpload(createRequest)
            val uploadId = createResponse.uploadId()
            val completedParts: MutableList<CompletedPart> = mutableListOf()
            var partNumber = 1
            val buffer = ByteBuffer.allocate(5 * 1024 * 1024) // Set your preferred part size (5 MB in this example)


            while (true){
                val readBytes: Int = content.read (buffer.array()) // readBytes in [-1 .. partSize]!
                if (readBytes == -1) { //EOF
                    break
                }
                val uploadRequest =
                    UploadPartRequest.builder()
                        .bucket(config.bucketName())
                        .key(filePath)
                        .uploadId(uploadId)
                        .partNumber(partNumber)
                        .contentLength(readBytes.toLong())
                        .build()

                val uploadResponse = s3Client.uploadPart(uploadRequest, RequestBody.fromByteBuffer(buffer))

                completedParts.add(CompletedPart.builder()
                    .partNumber(partNumber)
                    .eTag(uploadResponse.eTag())
                    .build());

                partNumber++
                buffer.clear()
            }

            val completedUpload = CompletedMultipartUpload.builder()
                .parts(completedParts)
                .build()

            val completeRequest = CompleteMultipartUploadRequest.builder()
                .bucket(config.bucketName())
                .key(filePath)
                .uploadId(uploadId)
                .multipartUpload(completedUpload)
                .build()

            val completeResponse = s3Client.completeMultipartUpload(completeRequest)

            // Print the object's URL
            val objectUrl: String = s3Client.utilities().getUrl(
                GetUrlRequest.builder()
                    .bucket(config.bucketName())
                    .key(filePath)
                    .build()
            ).toExternalForm()
            logger.info { "Uploaded object URL: $objectUrl s3 response $completeResponse" }
        }
    }

    override fun getFile(filePath: String): InputStream? {
        return try {
            val response =
                s3Client.getObject(GetObjectRequest.builder().bucket(config.bucketName()).key(filePath).build())
            if (response == null) {
                monitoring.onFileNotFound(filePath)
                logger.warn { "File '$filePath' not found" }
                return null
            }
            logger.info("File $filePath retrieved successfully.")
            monitoring.onGetFile(filePath)
            response
        } catch (e: S3Exception) {
            logger.error("File retrieval failed", e)
            monitoring.onGetFileFailed(e)
            throw e
        }
    }

    override fun deleteFile(filePath: String) {
        try {
            s3Client.deleteObject(DeleteObjectRequest.builder().bucket(config.bucketName()).key(filePath).build())
            logger.info("File $filePath deleted successfully.")
            monitoring.onDeleteFile(filePath)
        } catch (e: S3Exception) {
            logger.error("File deletion failed", e)
            monitoring.onDeleteFileFailed(e)
            throw e
        }
    }
}

Connecting S3FS FUSE to Docker S3 TestContainer

s3fs -o use_path_request_style -o host=http://localhost:4566  conductor  mount_dir_path