diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index 7a2bf2db9..6d9f8492c 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -24,6 +24,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/minio/minio/pkg/env" "io" "io/ioutil" "net/http" @@ -48,16 +49,23 @@ import ( minio "github.com/minio/minio/cmd" ) +var ( + azureUploadChunkSize = getUploadChunkSizeFromEnv(azureChunkSizeEnvVar, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte)) + azureSdkTimeout = time.Duration(azureUploadChunkSize/humanize.MiByte) * azureSdkTimeoutPerMb + azureUploadConcurrency = azureUploadMaxMemoryUsage / azureUploadChunkSize +) + const ( // The defaultDialTimeout for communicating with the cloud backends is set // to 30 seconds in utils.go; the Azure SDK recommends to set a timeout of 60 // seconds per MB of data a client expects to upload so we must transfer less // than 0.5 MB per chunk to stay within the defaultDialTimeout tolerance. // See https://github.com/Azure/azure-storage-blob-go/blob/fc70003/azblob/zc_policy_retry.go#L39-L44 for more details. - azureUploadChunkSize = 0.25 * humanize.MiByte - azureSdkTimeout = (azureUploadChunkSize / humanize.MiByte) * 60 * time.Second - azureUploadMaxMemoryUsage = 10 * humanize.MiByte - azureUploadConcurrency = azureUploadMaxMemoryUsage / azureUploadChunkSize + // To change the upload chunk size, set the environmental variable MINIO_AZURE_CHUNK_SIZE_MB with a (float) value between 0 and 100 + azureDefaultUploadChunkSize = 25 * humanize.MiByte + azureSdkTimeoutPerMb = 60 * time.Second + azureUploadMaxMemoryUsage = 100 * humanize.MiByte + azureChunkSizeEnvVar = "MINIO_AZURE_CHUNK_SIZE_MB" azureDownloadRetryAttempts = 5 azureBlockSize = 100 * humanize.MiByte @@ -86,6 +94,7 @@ EXAMPLES: 1. Start minio gateway server for Azure Blob Storage backend on custom endpoint. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}0.25 {{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint 2. Start minio gateway server for Azure Blob Storage backend with edge caching enabled. @@ -97,6 +106,7 @@ EXAMPLES: {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85 + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}25 {{.Prompt}} {{.HelpName}} ` @@ -125,6 +135,27 @@ func azureGatewayMain(ctx *cli.Context) { minio.StartGateway(ctx, &Azure{host}) } +// getUploadChunkSizeFromEnv returns the parsed chunk size from the environmental variable 'MINIO_AZURE_CHUNK_SIZE_MB' +// The environmental variable should be a floating point number between 0 and 100 representing the MegaBytes +// The returned value is an int representing the size in bytes +func getUploadChunkSizeFromEnv(envvar string, defaultValue string) int { + envChunkSize := env.Get(envvar, defaultValue) + + i, err := strconv.ParseFloat(envChunkSize, 64) + if err != nil { + logger.LogIf(context.Background(), err) + return azureDefaultUploadChunkSize + } + + if i <= 0 || i > 100 { + logger.LogIf(context.Background(), fmt.Errorf("ENV '%v' should be a floating point value between 0 and 100.\n"+ + "The upload chunk size is set to its default: %s\n", azureChunkSizeEnvVar, defaultValue)) + return azureDefaultUploadChunkSize + } + + return int(i * humanize.MiByte) +} + // Azure implements Gateway. type Azure struct { host string diff --git a/cmd/gateway/azure/gateway-azure_test.go b/cmd/gateway/azure/gateway-azure_test.go index b23812f21..c7813dc9c 100644 --- a/cmd/gateway/azure/gateway-azure_test.go +++ b/cmd/gateway/azure/gateway-azure_test.go @@ -19,8 +19,11 @@ package azure import ( "encoding/base64" "fmt" + "github.com/dustin/go-humanize" "net/http" + "os" "reflect" + "strconv" "testing" "github.com/Azure/azure-storage-blob-go/azblob" @@ -301,3 +304,36 @@ func TestCheckAzureUploadID(t *testing.T) { } } } + +func TestParsingUploadChunkSize(t *testing.T) { + key := "MINIO_AZURE_CHUNK_SIZE_MB" + invalidValues := []string{ + "", + "0,3", + "100.1", + "-1", + } + + for i, chunkValue := range invalidValues { + os.Setenv(key, chunkValue) + result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte)) + if result != azureDefaultUploadChunkSize { + t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result) + } + } + + validValues := []string{ + "1", + "1.25", + "50", + "99", + } + for i, chunkValue := range validValues { + os.Setenv(key, chunkValue) + result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte)) + if result == azureDefaultUploadChunkSize { + t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result) + } + } + +} diff --git a/docs/gateway/azure.md b/docs/gateway/azure.md index f7a87e64c..648ff3455 100644 --- a/docs/gateway/azure.md +++ b/docs/gateway/azure.md @@ -7,6 +7,7 @@ MinIO Gateway adds Amazon S3 compatibility to Microsoft Azure Blob Storage. docker run -p 9000:9000 --name azure-s3 \ -e "MINIO_ACCESS_KEY=azurestorageaccountname" \ -e "MINIO_SECRET_KEY=azurestorageaccountkey" \ + -e "MINIO_AZURE_CHUNK_SIZE_MB=0.25" \ minio/minio gateway azure ``` @@ -14,6 +15,7 @@ docker run -p 9000:9000 --name azure-s3 \ ``` export MINIO_ACCESS_KEY=azureaccountname export MINIO_SECRET_KEY=azureaccountkey +export MINIO_AZURE_CHUNK_SIZE_MB=0.25 minio gateway azure ``` ## Test using MinIO Browser