oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Migrate minio etcd config to backend config (#7751)

Browse Source 
etcd when used in federated setups, currently
mandates that all clusters should have same
config.json, which is too restrictive and makes
federation a restrictive environment.

This change makes it apparent that each cluster
needs to be independently managed if necessary
from `mc admin info` command line.

Each cluster with in federation can have their
own root credentials and as well as separate
regions. This way buckets get further restrictions
and allows for root creds to be not common
across clusters/data centers.

Existing data in etcd gets migrated to backend
on each clusters, upon start. Once done
users can change their config entries
independently.
master
Harshavardhana 6 years ago
parent da2887f914
commit 4a4048fe27
4 changed files with 57 additions and 114 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 69
      cmd/config-common.go
  2. 35
      cmd/config-migrate.go
  3. 63
      cmd/config.go
  4. 4
      docs/federation/lookup/README.md

69
cmd/config-common.go
Unescape View File

@ -21,7 +21,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"time"
etcd "github.com/coreos/etcd/clientv3" etcd "github.com/coreos/etcd/clientv3"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
@ -53,8 +52,19 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
} }
func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error { func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
_, err := client.Delete(ctx, configFile) timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
return err defer cancel()
_, err := client.Delete(timeoutCtx, configFile)
if err != nil {
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
client.Endpoints())
}
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
err, client.Endpoints())
}
return nil
} }
func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error { func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
@ -89,9 +99,11 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
resp, err := client.Get(timeoutCtx, configFile) resp, err := client.Get(timeoutCtx, configFile)
if err != nil { if err != nil {
if err == context.DeadlineExceeded { if err == context.DeadlineExceeded {
return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints()) return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
client.Endpoints())
} }
return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints()) return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
err, client.Endpoints())
} }
if resp.Count == 0 { if resp.Count == 0 {
return nil, errConfigNotFound return nil, errConfigNotFound
@ -104,54 +116,7 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
return nil, errConfigNotFound return nil, errConfigNotFound
} }
// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
for {
watchCh := globalEtcdClient.Watch(context.Background(), configFile)
select {
case <-GlobalServiceDoneCh:
return
case watchResp, ok := <-watchCh:
if !ok {
time.Sleep(1 * time.Second)
continue
}
if err := watchResp.Err(); err != nil {
logger.LogIf(context.Background(), err)
// log and retry.
time.Sleep(1 * time.Second)
continue
}
for _, event := range watchResp.Events {
if event.IsModify() || event.IsCreate() {
loadCfgFn(objAPI)
}
}
}
}
}
func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
resp, err := client.Get(timeoutCtx, configFile)
if err != nil {
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
}
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
if resp.Count == 0 {
return errConfigNotFound
}
return nil
}
func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error { func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
if globalEtcdClient != nil {
return checkConfigEtcd(ctx, globalEtcdClient, configFile)
}
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil { if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
// Treat object not found as config not found. // Treat object not found as config not found.
if isErrObjectNotFound(err) { if isErrObjectNotFound(err) {

35
cmd/config-migrate.go
Unescape View File

@ -29,7 +29,7 @@ import (
"github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/event/target" "github.com/minio/minio/pkg/event/target"
"github.com/minio/minio/pkg/iam/policy" iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/iam/validator" "github.com/minio/minio/pkg/iam/validator"
xnet "github.com/minio/minio/pkg/net" xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/quick" "github.com/minio/minio/pkg/quick"
@ -2413,6 +2413,7 @@ func migrateV27ToV28() error {
} }
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json' // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
// if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) { func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
// Construct path to config.json for the given bucket. // Construct path to config.json for the given bucket.
configFile := path.Join(minioConfigPrefix, minioConfigFile) configFile := path.Join(minioConfigPrefix, minioConfigFile)
@ -2423,10 +2424,14 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
} }
defer func() { defer func() {
// Rename config.json to config.json.deprecated only upon
// success of this function.
if err == nil { if err == nil {
os.Rename(getConfigFile(), getConfigFile()+".deprecated") if globalEtcdClient != nil {
deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
} else {
// Rename config.json to config.json.deprecated only upon
// success of this function.
os.Rename(getConfigFile(), getConfigFile()+".deprecated")
}
} }
}() }()
@ -2446,20 +2451,24 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
return err return err
} // if errConfigNotFound proceed to migrate.. } // if errConfigNotFound proceed to migrate..
var configFiles = []string{
getConfigFile(),
getConfigFile() + ".deprecated",
configFile,
}
var config = &serverConfig{} var config = &serverConfig{}
if _, err = Load(getConfigFile(), config); err != nil { for _, cfgFile := range configFiles {
if !os.IsNotExist(err) { if _, err = Load(cfgFile, config); err != nil {
return err
}
// Read from deprecate file as well if necessary.
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
return err return err
} }
// If all else fails simply initialize the server config. continue
return newSrvConfig(objAPI)
} }
break
}
if os.IsNotExist(err) {
// Initialize the server config, if no config exists.
return newSrvConfig(objAPI)
} }
return saveServerConfig(context.Background(), objAPI, config) return saveServerConfig(context.Background(), objAPI, config)
} }

63
cmd/config.go
Unescape View File

@ -49,10 +49,6 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
} }
configFile := path.Join(minioConfigPrefix, minioConfigFile) configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
}
// Create a backup of the current config // Create a backup of the current config
oldData, err := readConfig(ctx, objAPI, configFile) oldData, err := readConfig(ctx, objAPI, configFile)
if err == nil { if err == nil {
@ -71,15 +67,8 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
} }
func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) { func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
var configData []byte
var err error
configFile := path.Join(minioConfigPrefix, minioConfigFile) configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil { configData, err := readConfig(ctx, objAPI, configFile)
configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
} else {
configData, err = readConfig(ctx, objAPI, configFile)
}
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -152,44 +141,24 @@ func initConfig(objAPI ObjectLayer) error {
return errServerNotInitialized return errServerNotInitialized
} }
configFile := path.Join(minioConfigPrefix, minioConfigFile) if isFile(getConfigFile()) {
if err := migrateConfig(); err != nil {
if globalEtcdClient != nil {
if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
if err == errConfigNotFound {
// Migrates all configs at old location.
if err = migrateConfig(); err != nil {
return err
}
// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
if err = migrateConfigToMinioSys(objAPI); err != nil {
return err
}
} else {
return err
}
}
// Watch config for changes and reloads them.
go watchConfigEtcd(objAPI, configFile, loadConfig)
} else {
if isFile(getConfigFile()) {
if err := migrateConfig(); err != nil {
return err
}
}
// Migrates ${HOME}/.minio/config.json or config.json.deprecated
// to '<export_path>/.minio.sys/config/config.json'
// ignore if the file doesn't exist.
if err := migrateConfigToMinioSys(objAPI); err != nil {
return err return err
} }
}
// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version. // Migrates ${HOME}/.minio/config.json or config.json.deprecated
if err := migrateMinioSysConfig(objAPI); err != nil { // to '<export_path>/.minio.sys/config/config.json'
return err // ignore if the file doesn't exist.
} // If etcd is set then migrates /config/config.json
// to '<export_path>/.minio.sys/config/config.json'
if err := migrateConfigToMinioSys(objAPI); err != nil {
return err
}
// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
if err := migrateMinioSysConfig(objAPI); err != nil {
return err
} }
return loadConfig(objAPI) return loadConfig(objAPI)

4
docs/federation/lookup/README.md
Unescape View File

@ -9,8 +9,8 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta
### 2. Run MinIO in federated mode ### 2. Run MinIO in federated mode
Bucket lookup from DNS federation requires two dependencies Bucket lookup from DNS federation requires two dependencies
- etcd (for config, bucket SRV records) - etcd (for bucket DNS service records)
- CoreDNS (for DNS management based on populated bucket SRV records, optional) - CoreDNS (for DNS management based on populated bucket DNS service records, optional)
## Architecture ## Architecture

Write Preview
Loading…
Cancel
Save