Move etcd, logger, crypto into their own packages (#8366)
- Deprecates _MINIO_PROFILER, `mc admin profile` does the job - Move ENVs to common location in cmd/config/master
Harshavardhana
5 years ago
committed by
kannappanr
parent
bffc378a4f
commit
290ad0996f
@ -0,0 +1,35 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package config |
||||
|
||||
// Config value separator
|
||||
const ( |
||||
ValueSeparator = "," |
||||
) |
||||
|
||||
// Top level common ENVs
|
||||
const ( |
||||
EnvAccessKey = "MINIO_ACCESS_KEY" |
||||
EnvSecretKey = "MINIO_SECRET_KEY" |
||||
EnvBrowser = "MINIO_BROWSER" |
||||
EnvDomain = "MINIO_DOMAIN" |
||||
EnvPublicIPs = "MINIO_PUBLIC_IPS" |
||||
EnvEndpoints = "MINIO_ENDPOINTS" |
||||
|
||||
EnvUpdate = "MINIO_UPDATE" |
||||
EnvWorm = "MINIO_WORM" |
||||
) |
||||
@ -0,0 +1,96 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package etcd |
||||
|
||||
import ( |
||||
"crypto/tls" |
||||
"crypto/x509" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/coreos/etcd/clientv3" |
||||
"github.com/minio/minio/cmd/config" |
||||
"github.com/minio/minio/pkg/env" |
||||
xnet "github.com/minio/minio/pkg/net" |
||||
) |
||||
|
||||
const ( |
||||
// Default values used while communicating with etcd.
|
||||
defaultDialTimeout = 30 * time.Second |
||||
defaultDialKeepAlive = 30 * time.Second |
||||
) |
||||
|
||||
// etcd environment values
|
||||
const ( |
||||
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS" |
||||
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT" |
||||
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY" |
||||
) |
||||
|
||||
// New - Initialize new etcd client
|
||||
func New(rootCAs *x509.CertPool) (*clientv3.Client, error) { |
||||
envEndpoints := env.Get(EnvEtcdEndpoints, "") |
||||
if envEndpoints == "" { |
||||
// etcd is not configured, nothing to do.
|
||||
return nil, nil |
||||
} |
||||
|
||||
etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator) |
||||
|
||||
var etcdSecure bool |
||||
for _, endpoint := range etcdEndpoints { |
||||
u, err := xnet.ParseURL(endpoint) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// If one of the endpoint is https, we will use https directly.
|
||||
etcdSecure = etcdSecure || u.Scheme == "https" |
||||
} |
||||
|
||||
var err error |
||||
var etcdClnt *clientv3.Client |
||||
if etcdSecure { |
||||
// This is only to support client side certificate authentication
|
||||
// https://coreos.com/etcd/docs/latest/op-guide/security.html
|
||||
etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert) |
||||
etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey) |
||||
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) |
||||
if ok1 && ok2 { |
||||
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) { |
||||
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey) |
||||
return &cert, terr |
||||
} |
||||
} |
||||
|
||||
etcdClnt, err = clientv3.New(clientv3.Config{ |
||||
Endpoints: etcdEndpoints, |
||||
DialTimeout: defaultDialTimeout, |
||||
DialKeepAliveTime: defaultDialKeepAlive, |
||||
TLS: &tls.Config{ |
||||
RootCAs: rootCAs, |
||||
GetClientCertificate: getClientCertificate, |
||||
}, |
||||
}) |
||||
} else { |
||||
etcdClnt, err = clientv3.New(clientv3.Config{ |
||||
Endpoints: etcdEndpoints, |
||||
DialTimeout: defaultDialTimeout, |
||||
DialKeepAliveTime: defaultDialKeepAlive, |
||||
}) |
||||
} |
||||
return etcdClnt, err |
||||
} |
||||
@ -0,0 +1,42 @@ |
||||
// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto |
||||
|
||||
import ( |
||||
"encoding/hex" |
||||
"fmt" |
||||
"strings" |
||||
) |
||||
|
||||
// ParseMasterKey parses the value of the environment variable
|
||||
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
|
||||
func ParseMasterKey(envArg string) (KMS, error) { |
||||
values := strings.SplitN(envArg, ":", 2) |
||||
if len(values) != 2 { |
||||
return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg) |
||||
} |
||||
var ( |
||||
keyID = values[0] |
||||
hexKey = values[1] |
||||
) |
||||
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
|
||||
return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey) |
||||
} |
||||
var masterKey [32]byte |
||||
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil { |
||||
return nil, err |
||||
} |
||||
return NewMasterKey(keyID, masterKey), nil |
||||
} |
||||
@ -0,0 +1,59 @@ |
||||
// MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto |
||||
|
||||
import "testing" |
||||
|
||||
func TestParseMasterKey(t *testing.T) { |
||||
tests := []struct { |
||||
envValue string |
||||
expectedKeyID string |
||||
success bool |
||||
}{ |
||||
{ |
||||
envValue: "invalid-value", |
||||
success: false, |
||||
}, |
||||
{ |
||||
envValue: "too:many:colons", |
||||
success: false, |
||||
}, |
||||
{ |
||||
envValue: "myminio-key:not-a-hex", |
||||
success: false, |
||||
}, |
||||
{ |
||||
envValue: "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574", |
||||
expectedKeyID: "my-minio-key", |
||||
success: true, |
||||
}, |
||||
} |
||||
|
||||
for _, tt := range tests { |
||||
tt := tt |
||||
t.Run(tt.envValue, func(t *testing.T) { |
||||
kms, err := ParseMasterKey(tt.envValue) |
||||
if tt.success && err != nil { |
||||
t.Error(err) |
||||
} |
||||
if !tt.success && err == nil { |
||||
t.Error("Unexpected failure") |
||||
} |
||||
if err == nil && kms.KeyID() != tt.expectedKeyID { |
||||
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID()) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -1,158 +0,0 @@ |
||||
// MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"encoding/hex" |
||||
"errors" |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
"github.com/minio/minio/cmd/crypto" |
||||
"github.com/minio/minio/pkg/env" |
||||
) |
||||
|
||||
const ( |
||||
// EnvKMSMasterKey is the environment variable used to specify
|
||||
// a KMS master key used to protect SSE-S3 per-object keys.
|
||||
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
|
||||
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY" |
||||
|
||||
// EnvAutoEncryption is the environment variable used to en/disable
|
||||
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
|
||||
// requires a valid KMS configuration and turns any non-SSE-C
|
||||
// request into an SSE-S3 request.
|
||||
// If present EnvAutoEncryption must be either "on" or "off".
|
||||
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION" |
||||
) |
||||
|
||||
const ( |
||||
// EnvVaultEndpoint is the environment variable used to specify
|
||||
// the vault HTTPS endpoint.
|
||||
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT" |
||||
|
||||
// EnvVaultAuthType is the environment variable used to specify
|
||||
// the authentication type for vault.
|
||||
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE" |
||||
|
||||
// EnvVaultAppRoleID is the environment variable used to specify
|
||||
// the vault AppRole ID.
|
||||
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID" |
||||
|
||||
// EnvVaultAppSecretID is the environment variable used to specify
|
||||
// the vault AppRole secret corresponding to the AppRole ID.
|
||||
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET" |
||||
|
||||
// EnvVaultKeyVersion is the environment variable used to specify
|
||||
// the vault key version.
|
||||
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION" |
||||
|
||||
// EnvVaultKeyName is the environment variable used to specify
|
||||
// the vault named key-ring. In the S3 context it's referred as
|
||||
// customer master key ID (CMK-ID).
|
||||
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME" |
||||
|
||||
// EnvVaultCAPath is the environment variable used to specify the
|
||||
// path to a directory of PEM-encoded CA cert files. These CA cert
|
||||
// files are used to authenticate MinIO to Vault over mTLS.
|
||||
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH" |
||||
|
||||
// EnvVaultNamespace is the environment variable used to specify
|
||||
// vault namespace. The vault namespace is used if the enterprise
|
||||
// version of Hashicorp Vault is used.
|
||||
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE" |
||||
) |
||||
|
||||
// LookupKMSConfig extracts the KMS configuration provided by environment
|
||||
// variables and merge them with the provided KMS configuration. The
|
||||
// merging follows the following rules:
|
||||
//
|
||||
// 1. A valid value provided as environment variable is higher prioritized
|
||||
// than the provided configuration and overwrites the value from the
|
||||
// configuration file.
|
||||
//
|
||||
// 2. A value specified as environment variable never changes the configuration
|
||||
// file. So it is never made a persistent setting.
|
||||
//
|
||||
// It sets the global KMS configuration according to the merged configuration
|
||||
// on success.
|
||||
func LookupKMSConfig(config crypto.KMSConfig) (err error) { |
||||
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
|
||||
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint) |
||||
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath) |
||||
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type) |
||||
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID) |
||||
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret) |
||||
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name) |
||||
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace) |
||||
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version)) |
||||
config.Vault.Key.Version, err = strconv.Atoi(keyVersion) |
||||
if err != nil { |
||||
return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion) |
||||
} |
||||
if err = config.Vault.Verify(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Lookup KMS master keys - only available through ENV.
|
||||
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok { |
||||
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
|
||||
return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time") |
||||
} |
||||
globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
if !config.Vault.IsEmpty() { |
||||
GlobalKMS, err = crypto.NewVault(config.Vault) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
globalKMSKeyID = config.Vault.Key.Name |
||||
} |
||||
|
||||
autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
globalAutoEncryption = bool(autoEncryption) |
||||
if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
|
||||
return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present") |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// parseKMSMasterKey parses the value of the environment variable
|
||||
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
|
||||
func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) { |
||||
values := strings.SplitN(envArg, ":", 2) |
||||
if len(values) != 2 { |
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg) |
||||
} |
||||
var ( |
||||
keyID = values[0] |
||||
hexKey = values[1] |
||||
) |
||||
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey) |
||||
} |
||||
var masterKey [32]byte |
||||
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil { |
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey) |
||||
} |
||||
return keyID, crypto.NewKMS(masterKey), nil |
||||
} |
||||
@ -0,0 +1,86 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package logger |
||||
|
||||
import ( |
||||
"strings" |
||||
|
||||
"github.com/minio/minio/pkg/env" |
||||
) |
||||
|
||||
// Console logger target
|
||||
type Console struct { |
||||
Enabled bool `json:"enabled"` |
||||
} |
||||
|
||||
// HTTP logger target
|
||||
type HTTP struct { |
||||
Enabled bool `json:"enabled"` |
||||
Endpoint string `json:"endpoint"` |
||||
} |
||||
|
||||
// Config console and http logger targets
|
||||
type Config struct { |
||||
Console Console `json:"console"` |
||||
HTTP map[string]HTTP `json:"http"` |
||||
Audit map[string]HTTP `json:"audit"` |
||||
} |
||||
|
||||
// HTTP endpoint logger
|
||||
const ( |
||||
EnvLoggerHTTPEndpoint = "MINIO_LOGGER_HTTP_ENDPOINT" |
||||
EnvAuditLoggerHTTPEndpoint = "MINIO_AUDIT_LOGGER_HTTP_ENDPOINT" |
||||
) |
||||
|
||||
// Default target name when no targets are found
|
||||
const ( |
||||
defaultTarget = "_" |
||||
) |
||||
|
||||
// LookupConfig - lookup logger config, override with ENVs if set.
|
||||
func LookupConfig(cfg Config) (Config, error) { |
||||
envs := env.List(EnvLoggerHTTPEndpoint) |
||||
for _, e := range envs { |
||||
target := strings.TrimPrefix(e, EnvLoggerHTTPEndpoint) |
||||
if target == "" { |
||||
target = defaultTarget |
||||
} |
||||
_, ok := cfg.HTTP[target] |
||||
if ok { |
||||
cfg.HTTP[target] = HTTP{ |
||||
Enabled: true, |
||||
Endpoint: env.Get(e, cfg.HTTP[target].Endpoint), |
||||
} |
||||
} |
||||
} |
||||
aenvs := env.List(EnvAuditLoggerHTTPEndpoint) |
||||
for _, e := range aenvs { |
||||
target := strings.TrimPrefix(e, EnvAuditLoggerHTTPEndpoint) |
||||
if target == "" { |
||||
target = defaultTarget |
||||
} |
||||
_, ok := cfg.Audit[target] |
||||
if ok { |
||||
cfg.Audit[target] = HTTP{ |
||||
Enabled: true, |
||||
Endpoint: env.Get(e, cfg.Audit[target].Endpoint), |
||||
} |
||||
} |
||||
} |
||||
|
||||
return cfg, nil |
||||
} |
||||
Loading…
Reference in new issue