mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2024-12-21 10:13:11 +00:00
464 lines
15 KiB
Go
464 lines
15 KiB
Go
/*
|
|
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
|
|
* Copyright 2015-2017 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package minio
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/minio/minio-go/v7/pkg/encrypt"
|
|
"github.com/minio/minio-go/v7/pkg/s3utils"
|
|
)
|
|
|
|
func (c *Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64,
|
|
opts PutObjectOptions,
|
|
) (info UploadInfo, err error) {
|
|
info, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts)
|
|
if err != nil {
|
|
errResp := ToErrorResponse(err)
|
|
// Verify if multipart functionality is not available, if not
|
|
// fall back to single PutObject operation.
|
|
if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") {
|
|
// Verify if size of reader is greater than '5GiB'.
|
|
if size > maxSinglePutObjectSize {
|
|
return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
|
}
|
|
// Fall back to uploading as single PutObject operation.
|
|
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
|
|
}
|
|
}
|
|
return info, err
|
|
}
|
|
|
|
func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) {
|
|
// Input validation.
|
|
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
if err = s3utils.CheckValidObjectName(objectName); err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
|
|
// Total data read and written to server. should be equal to
|
|
// 'size' at the end of the call.
|
|
var totalUploadedSize int64
|
|
|
|
// Complete multipart upload.
|
|
var complMultipartUpload completeMultipartUpload
|
|
|
|
// Calculate the optimal parts info for a given size.
|
|
totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
|
|
// Choose hash algorithms to be calculated by hashCopyN,
|
|
// avoid sha256 with non-v4 signature request or
|
|
// HTTPS connection.
|
|
hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5, !opts.DisableContentSha256)
|
|
if len(hashSums) == 0 {
|
|
if opts.UserMetadata == nil {
|
|
opts.UserMetadata = make(map[string]string, 1)
|
|
}
|
|
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C"
|
|
}
|
|
|
|
// Initiate a new multipart upload.
|
|
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
|
|
|
|
defer func() {
|
|
if err != nil {
|
|
c.abortMultipartUpload(ctx, bucketName, objectName, uploadID)
|
|
}
|
|
}()
|
|
|
|
// Part number always starts with '1'.
|
|
partNumber := 1
|
|
|
|
// Initialize parts uploaded map.
|
|
partsInfo := make(map[int]ObjectPart)
|
|
|
|
// Create a buffer.
|
|
buf := make([]byte, partSize)
|
|
|
|
// Create checksums
|
|
// CRC32C is ~50% faster on AMD64 @ 30GB/s
|
|
var crcBytes []byte
|
|
customHeader := make(http.Header)
|
|
crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
|
for partNumber <= totalPartsCount {
|
|
length, rErr := readFull(reader, buf)
|
|
if rErr == io.EOF && partNumber > 1 {
|
|
break
|
|
}
|
|
|
|
if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF {
|
|
return UploadInfo{}, rErr
|
|
}
|
|
|
|
// Calculates hash sums while copying partSize bytes into cw.
|
|
for k, v := range hashAlgos {
|
|
v.Write(buf[:length])
|
|
hashSums[k] = v.Sum(nil)
|
|
v.Close()
|
|
}
|
|
|
|
// Update progress reader appropriately to the latest offset
|
|
// as we read from the source.
|
|
rd := newHook(bytes.NewReader(buf[:length]), opts.Progress)
|
|
|
|
// Checksums..
|
|
var (
|
|
md5Base64 string
|
|
sha256Hex string
|
|
)
|
|
|
|
if hashSums["md5"] != nil {
|
|
md5Base64 = base64.StdEncoding.EncodeToString(hashSums["md5"])
|
|
}
|
|
if hashSums["sha256"] != nil {
|
|
sha256Hex = hex.EncodeToString(hashSums["sha256"])
|
|
}
|
|
if len(hashSums) == 0 {
|
|
crc.Reset()
|
|
crc.Write(buf[:length])
|
|
cSum := crc.Sum(nil)
|
|
customHeader.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(cSum))
|
|
crcBytes = append(crcBytes, cSum...)
|
|
}
|
|
|
|
p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, sha256Hex: sha256Hex, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
|
|
// Proceed to upload the part.
|
|
objPart, uerr := c.uploadPart(ctx, p)
|
|
if uerr != nil {
|
|
return UploadInfo{}, uerr
|
|
}
|
|
|
|
// Save successfully uploaded part metadata.
|
|
partsInfo[partNumber] = objPart
|
|
|
|
// Save successfully uploaded size.
|
|
totalUploadedSize += int64(length)
|
|
|
|
// Increment part number.
|
|
partNumber++
|
|
|
|
// For unknown size, Read EOF we break away.
|
|
// We do not have to upload till totalPartsCount.
|
|
if rErr == io.EOF {
|
|
break
|
|
}
|
|
}
|
|
|
|
// Loop over total uploaded parts to save them in
|
|
// Parts array before completing the multipart request.
|
|
for i := 1; i < partNumber; i++ {
|
|
part, ok := partsInfo[i]
|
|
if !ok {
|
|
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
|
|
}
|
|
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
|
|
ETag: part.ETag,
|
|
PartNumber: part.PartNumber,
|
|
ChecksumCRC32: part.ChecksumCRC32,
|
|
ChecksumCRC32C: part.ChecksumCRC32C,
|
|
ChecksumSHA1: part.ChecksumSHA1,
|
|
ChecksumSHA256: part.ChecksumSHA256,
|
|
})
|
|
}
|
|
|
|
// Sort all completed parts.
|
|
sort.Sort(completedParts(complMultipartUpload.Parts))
|
|
opts = PutObjectOptions{
|
|
ServerSideEncryption: opts.ServerSideEncryption,
|
|
}
|
|
if len(crcBytes) > 0 {
|
|
// Add hash of hashes.
|
|
crc.Reset()
|
|
crc.Write(crcBytes)
|
|
opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
|
|
}
|
|
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
|
|
uploadInfo.Size = totalUploadedSize
|
|
return uploadInfo, nil
|
|
}
|
|
|
|
// initiateMultipartUpload - Initiates a multipart upload and returns an upload ID.
|
|
func (c *Client) initiateMultipartUpload(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (initiateMultipartUploadResult, error) {
|
|
// Input validation.
|
|
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
|
return initiateMultipartUploadResult{}, err
|
|
}
|
|
if err := s3utils.CheckValidObjectName(objectName); err != nil {
|
|
return initiateMultipartUploadResult{}, err
|
|
}
|
|
|
|
// Initialize url queries.
|
|
urlValues := make(url.Values)
|
|
urlValues.Set("uploads", "")
|
|
|
|
if opts.Internal.SourceVersionID != "" {
|
|
if opts.Internal.SourceVersionID != nullVersionID {
|
|
if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil {
|
|
return initiateMultipartUploadResult{}, errInvalidArgument(err.Error())
|
|
}
|
|
}
|
|
urlValues.Set("versionId", opts.Internal.SourceVersionID)
|
|
}
|
|
|
|
// Set ContentType header.
|
|
customHeader := opts.Header()
|
|
|
|
reqMetadata := requestMetadata{
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
queryValues: urlValues,
|
|
customHeader: customHeader,
|
|
}
|
|
|
|
// Execute POST on an objectName to initiate multipart upload.
|
|
resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata)
|
|
defer closeResponse(resp)
|
|
if err != nil {
|
|
return initiateMultipartUploadResult{}, err
|
|
}
|
|
if resp != nil {
|
|
if resp.StatusCode != http.StatusOK {
|
|
return initiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
|
}
|
|
}
|
|
// Decode xml for new multipart upload.
|
|
initiateMultipartUploadResult := initiateMultipartUploadResult{}
|
|
err = xmlDecoder(resp.Body, &initiateMultipartUploadResult)
|
|
if err != nil {
|
|
return initiateMultipartUploadResult, err
|
|
}
|
|
return initiateMultipartUploadResult, nil
|
|
}
|
|
|
|
type uploadPartParams struct {
|
|
bucketName string
|
|
objectName string
|
|
uploadID string
|
|
reader io.Reader
|
|
partNumber int
|
|
md5Base64 string
|
|
sha256Hex string
|
|
size int64
|
|
sse encrypt.ServerSide
|
|
streamSha256 bool
|
|
customHeader http.Header
|
|
trailer http.Header
|
|
}
|
|
|
|
// uploadPart - Uploads a part in a multipart upload.
|
|
func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart, error) {
|
|
// Input validation.
|
|
if err := s3utils.CheckValidBucketName(p.bucketName); err != nil {
|
|
return ObjectPart{}, err
|
|
}
|
|
if err := s3utils.CheckValidObjectName(p.objectName); err != nil {
|
|
return ObjectPart{}, err
|
|
}
|
|
if p.size > maxPartSize {
|
|
return ObjectPart{}, errEntityTooLarge(p.size, maxPartSize, p.bucketName, p.objectName)
|
|
}
|
|
if p.size <= -1 {
|
|
return ObjectPart{}, errEntityTooSmall(p.size, p.bucketName, p.objectName)
|
|
}
|
|
if p.partNumber <= 0 {
|
|
return ObjectPart{}, errInvalidArgument("Part number cannot be negative or equal to zero.")
|
|
}
|
|
if p.uploadID == "" {
|
|
return ObjectPart{}, errInvalidArgument("UploadID cannot be empty.")
|
|
}
|
|
|
|
// Get resources properly escaped and lined up before using them in http request.
|
|
urlValues := make(url.Values)
|
|
// Set part number.
|
|
urlValues.Set("partNumber", strconv.Itoa(p.partNumber))
|
|
// Set upload id.
|
|
urlValues.Set("uploadId", p.uploadID)
|
|
|
|
// Set encryption headers, if any.
|
|
if p.customHeader == nil {
|
|
p.customHeader = make(http.Header)
|
|
}
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
|
|
// Server-side encryption is supported by the S3 Multipart Upload actions.
|
|
// Unless you are using a customer-provided encryption key, you don't need
|
|
// to specify the encryption parameters in each UploadPart request.
|
|
if p.sse != nil && p.sse.Type() == encrypt.SSEC {
|
|
p.sse.Marshal(p.customHeader)
|
|
}
|
|
|
|
reqMetadata := requestMetadata{
|
|
bucketName: p.bucketName,
|
|
objectName: p.objectName,
|
|
queryValues: urlValues,
|
|
customHeader: p.customHeader,
|
|
contentBody: p.reader,
|
|
contentLength: p.size,
|
|
contentMD5Base64: p.md5Base64,
|
|
contentSHA256Hex: p.sha256Hex,
|
|
streamSha256: p.streamSha256,
|
|
trailer: p.trailer,
|
|
}
|
|
|
|
// Execute PUT on each part.
|
|
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
|
|
defer closeResponse(resp)
|
|
if err != nil {
|
|
return ObjectPart{}, err
|
|
}
|
|
if resp != nil {
|
|
if resp.StatusCode != http.StatusOK {
|
|
return ObjectPart{}, httpRespToErrorResponse(resp, p.bucketName, p.objectName)
|
|
}
|
|
}
|
|
// Once successfully uploaded, return completed part.
|
|
h := resp.Header
|
|
objPart := ObjectPart{
|
|
ChecksumCRC32: h.Get("x-amz-checksum-crc32"),
|
|
ChecksumCRC32C: h.Get("x-amz-checksum-crc32c"),
|
|
ChecksumSHA1: h.Get("x-amz-checksum-sha1"),
|
|
ChecksumSHA256: h.Get("x-amz-checksum-sha256"),
|
|
}
|
|
objPart.Size = p.size
|
|
objPart.PartNumber = p.partNumber
|
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
|
objPart.ETag = trimEtag(h.Get("ETag"))
|
|
return objPart, nil
|
|
}
|
|
|
|
// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts.
|
|
func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string,
|
|
complete completeMultipartUpload, opts PutObjectOptions,
|
|
) (UploadInfo, error) {
|
|
// Input validation.
|
|
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
if err := s3utils.CheckValidObjectName(objectName); err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
|
|
// Initialize url queries.
|
|
urlValues := make(url.Values)
|
|
urlValues.Set("uploadId", uploadID)
|
|
// Marshal complete multipart body.
|
|
completeMultipartUploadBytes, err := xml.Marshal(complete)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
|
|
headers := opts.Header()
|
|
if s3utils.IsAmazonEndpoint(*c.endpointURL) {
|
|
headers.Del(encrypt.SseKmsKeyID) // Remove X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id not supported in CompleteMultipartUpload
|
|
headers.Del(encrypt.SseGenericHeader) // Remove X-Amz-Server-Side-Encryption not supported in CompleteMultipartUpload
|
|
}
|
|
|
|
// Instantiate all the complete multipart buffer.
|
|
completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
|
|
reqMetadata := requestMetadata{
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
queryValues: urlValues,
|
|
contentBody: completeMultipartUploadBuffer,
|
|
contentLength: int64(len(completeMultipartUploadBytes)),
|
|
contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
|
|
customHeader: headers,
|
|
}
|
|
|
|
// Execute POST to complete multipart upload for an objectName.
|
|
resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata)
|
|
defer closeResponse(resp)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
if resp != nil {
|
|
if resp.StatusCode != http.StatusOK {
|
|
return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
|
}
|
|
}
|
|
|
|
// Read resp.Body into a []bytes to parse for Error response inside the body
|
|
var b []byte
|
|
b, err = io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return UploadInfo{}, err
|
|
}
|
|
// Decode completed multipart upload response on success.
|
|
completeMultipartUploadResult := completeMultipartUploadResult{}
|
|
err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult)
|
|
if err != nil {
|
|
// xml parsing failure due to presence an ill-formed xml fragment
|
|
return UploadInfo{}, err
|
|
} else if completeMultipartUploadResult.Bucket == "" {
|
|
// xml's Decode method ignores well-formed xml that don't apply to the type of value supplied.
|
|
// In this case, it would leave completeMultipartUploadResult with the corresponding zero-values
|
|
// of the members.
|
|
|
|
// Decode completed multipart upload response on failure
|
|
completeMultipartUploadErr := ErrorResponse{}
|
|
err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadErr)
|
|
if err != nil {
|
|
// xml parsing failure due to presence an ill-formed xml fragment
|
|
return UploadInfo{}, err
|
|
}
|
|
return UploadInfo{}, completeMultipartUploadErr
|
|
}
|
|
|
|
// extract lifecycle expiry date and rule ID
|
|
expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration))
|
|
|
|
return UploadInfo{
|
|
Bucket: completeMultipartUploadResult.Bucket,
|
|
Key: completeMultipartUploadResult.Key,
|
|
ETag: trimEtag(completeMultipartUploadResult.ETag),
|
|
VersionID: resp.Header.Get(amzVersionID),
|
|
Location: completeMultipartUploadResult.Location,
|
|
Expiration: expTime,
|
|
ExpirationRuleID: ruleID,
|
|
|
|
ChecksumSHA256: completeMultipartUploadResult.ChecksumSHA256,
|
|
ChecksumSHA1: completeMultipartUploadResult.ChecksumSHA1,
|
|
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
|
|
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
|
|
}, nil
|
|
}
|