Add Kubernetes operator webook server as DNS target (#10404)

This PR adds a DNS target that ensures to update an entry
into Kubernetes operator when a bucket is created or deleted.

See minio/operator#264 for details.

Co-authored-by: Harshavardhana <harsha@minio.io>
master
Nitish Tiwari 4 years ago committed by GitHub
parent 52e21bc853
commit eaaf05a7cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 58
      cmd/bucket-handlers.go
  2. 9
      cmd/common-main.go
  3. 1
      cmd/config/constants.go
  4. 200
      cmd/config/etcd/dns/operator_dns.go
  5. 4
      cmd/object-handlers.go
  6. 2
      cmd/server-main.go
  7. 4
      cmd/web-handlers.go
  8. 4
      pkg/env/env.go
  9. 34
      pkg/env/web_env.go
  10. 10
      pkg/env/web_env_test.go

@ -72,7 +72,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
// Get buckets in the DNS // Get buckets in the DNS
dnsBuckets, err := globalDNSConfig.List() dnsBuckets, err := globalDNSConfig.List()
if err != nil && err != dns.ErrNoEntriesFound { if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
logger.LogIf(GlobalContext, err) logger.LogIf(GlobalContext, err)
return return
} }
@ -80,33 +80,35 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
bucketsSet := set.NewStringSet() bucketsSet := set.NewStringSet()
bucketsToBeUpdated := set.NewStringSet() bucketsToBeUpdated := set.NewStringSet()
bucketsInConflict := set.NewStringSet() bucketsInConflict := set.NewStringSet()
for _, bucket := range buckets { if dnsBuckets != nil {
bucketsSet.Add(bucket.Name) for _, bucket := range buckets {
r, ok := dnsBuckets[bucket.Name] bucketsSet.Add(bucket.Name)
if !ok { r, ok := dnsBuckets[bucket.Name]
bucketsToBeUpdated.Add(bucket.Name) if !ok {
continue bucketsToBeUpdated.Add(bucket.Name)
}
if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
// No difference in terms of domainIPs and nothing
// has changed so we don't change anything on the etcd.
continue continue
} }
// if domain IPs intersect then it won't be an empty set. if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
// such an intersection means that bucket exists on etcd. if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
// but if we do see a difference with local domain IPs with // No difference in terms of domainIPs and nothing
// hostSlice from etcd then we should update with newer // has changed so we don't change anything on the etcd.
// domainIPs, we proceed to do that here. continue
bucketsToBeUpdated.Add(bucket.Name) }
continue // if domain IPs intersect then it won't be an empty set.
// such an intersection means that bucket exists on etcd.
// but if we do see a difference with local domain IPs with
// hostSlice from etcd then we should update with newer
// domainIPs, we proceed to do that here.
bucketsToBeUpdated.Add(bucket.Name)
continue
}
// No IPs seem to intersect, this means that bucket exists but has
// different IP addresses perhaps from a different deployment.
// bucket names are globally unique in federation at a given
// path prefix, name collision is not allowed. We simply log
// an error and continue.
bucketsInConflict.Add(bucket.Name)
} }
// No IPs seem to intersect, this means that bucket exists but has
// different IP addresses perhaps from a different deployment.
// bucket names are globally unique in federation at a given
// path prefix, name collision is not allowed. We simply log
// an error and continue.
bucketsInConflict.Add(bucket.Name)
} }
// Add/update buckets that are not registered with the DNS // Add/update buckets that are not registered with the DNS
@ -562,7 +564,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
if globalDNSConfig != nil { if globalDNSConfig != nil {
sr, err := globalDNSConfig.Get(bucket) sr, err := globalDNSConfig.Get(bucket)
if err != nil { if err != nil {
if err == dns.ErrNoEntriesFound { // ErrNotImplemented indicates a DNS backend that doesn't need to check if bucket already
// exists elsewhere
if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
// Proceed to creating a bucket. // Proceed to creating a bucket.
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil { if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@ -1000,7 +1004,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
if globalDNSConfig != nil { if globalDNSConfig != nil {
if err := globalDNSConfig.Delete(bucket); err != nil { if err := globalDNSConfig.Delete(bucket); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err)) logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

@ -33,6 +33,7 @@ import (
"github.com/minio/cli" "github.com/minio/cli"
"github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd/dns"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/certs" "github.com/minio/minio/pkg/certs"
@ -285,6 +286,14 @@ func handleCommonEnvVars() {
os.Unsetenv(config.EnvAccessKeyOld) os.Unsetenv(config.EnvAccessKeyOld)
os.Unsetenv(config.EnvSecretKeyOld) os.Unsetenv(config.EnvSecretKeyOld)
} }
url, user, pwd, ok := env.LookupEnv(config.EnvDNSWebhook)
if ok {
globalDNSConfig, err = dns.NewOperatorDNS(url, user, pwd)
if err != nil {
logger.Fatal(err, "Unable to fetch the value of "+config.EnvDNSWebhook)
}
}
} }
func logStartupMessage(msg string) { func logStartupMessage(msg string) {

@ -33,6 +33,7 @@ const (
EnvPublicIPs = "MINIO_PUBLIC_IPS" EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvFSOSync = "MINIO_FS_OSYNC" EnvFSOSync = "MINIO_FS_OSYNC"
EnvArgs = "MINIO_ARGS" EnvArgs = "MINIO_ARGS"
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
EnvUpdate = "MINIO_UPDATE" EnvUpdate = "MINIO_UPDATE"

@ -0,0 +1,200 @@
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dns
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"net/url"
"time"
"github.com/dgrijalva/jwt-go"
"github.com/minio/minio/cmd/config"
)
var (
defaultOperatorContextTimeout = 10 * time.Second
// ErrNotImplemented - Indicates no entries were found for the given key (directory)
ErrNotImplemented = errors.New("The method is not implemented")
globalRootCAs *x509.CertPool
)
// RegisterGlobalCAs register the global root CAs
func RegisterGlobalCAs(CAs *x509.CertPool) {
globalRootCAs = CAs
}
func (c *OperatorDNS) addAuthHeader(r *http.Request) (*http.Request, error) {
claims := &jwt.StandardClaims{
ExpiresAt: int64(15 * time.Minute),
Issuer: c.Username,
Subject: config.EnvDNSWebhook,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
ss, err := token.SignedString([]byte(c.Password))
if err != nil {
return r, err
}
r.Header.Set("Authorization", "Bearer "+ss)
return r, nil
}
func (c *OperatorDNS) endpoint(bucket string, delete bool) (string, error) {
u, err := url.Parse(c.Endpoint)
if err != nil {
return "", err
}
q := u.Query()
q.Add("bucket", bucket)
if delete {
q.Add("delete", "true")
} else {
q.Add("delete", "false")
}
u.RawQuery = q.Encode()
return u.String(), nil
}
// Put - Adds DNS entries into operator webhook server
func (c *OperatorDNS) Put(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, false)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return err
}
req, err = c.addAuthHeader(req)
if err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if err != nil {
if err := c.Delete(bucket); err != nil {
return err
}
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("request to create the service for bucket %s, failed with status %s", bucket, resp.Status)
}
return nil
}
// Delete - Removes DNS entries added in Put().
func (c *OperatorDNS) Delete(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, true)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return err
}
req, err = c.addAuthHeader(req)
if err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
}
return nil
}
// DeleteRecord - Removes a specific DNS entry
// No Op for Operator because operator deals on with bucket entries
func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
return ErrNotImplemented
}
// Close closes the internal http client
func (c *OperatorDNS) Close() error {
c.httpClient.CloseIdleConnections()
return nil
}
// List - Retrieves list of DNS entries for the domain.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
return nil, ErrNotImplemented
}
// Get - Retrieves DNS records for a bucket.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
return nil, ErrNotImplemented
}
// OperatorDNS - represents dns config for MinIO k8s operator.
type OperatorDNS struct {
httpClient *http.Client
Endpoint string
Username string
Password string
}
// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
func NewOperatorDNS(endpoint, user, pwd string) (Store, error) {
if endpoint == "" || user == "" || pwd == "" {
return nil, errors.New("invalid argument")
}
args := &OperatorDNS{
Username: user,
Password: pwd,
Endpoint: endpoint,
httpClient: &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 3 * time.Second,
KeepAlive: 5 * time.Second,
}).DialContext,
ResponseHeaderTimeout: 3 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 3 * time.Second,
TLSClientConfig: &tls.Config{
RootCAs: globalRootCAs,
},
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
},
},
}
return args, nil
}

@ -764,7 +764,7 @@ func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo.
// when federation is enabled, ie when globalDNSConfig is non 'nil'. // when federation is enabled, ie when globalDNSConfig is non 'nil'.
// //
// This function is similar to isRemoteCallRequired but specifically for COPY object API // This function is similar to isRemoteCallRequired but specifically for COPY object API
// if destination and source are same we do not need to check for destnation bucket // if destination and source are same we do not need to check for destination bucket
// to exist locally. // to exist locally.
func isRemoteCopyRequired(ctx context.Context, srcBucket, dstBucket string, objAPI ObjectLayer) bool { func isRemoteCopyRequired(ctx context.Context, srcBucket, dstBucket string, objAPI ObjectLayer) bool {
if srcBucket == dstBucket { if srcBucket == dstBucket {
@ -2713,7 +2713,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
if globalDNSConfig != nil { if globalDNSConfig != nil {
_, err := globalDNSConfig.Get(bucket) _, err := globalDNSConfig.Get(bucket)
if err != nil { if err != nil && err != dns.ErrNotImplemented {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

@ -32,6 +32,7 @@ import (
"github.com/minio/cli" "github.com/minio/cli"
"github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd/dns"
xhttp "github.com/minio/minio/cmd/http" xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/auth"
@ -129,6 +130,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
// Register root CAs for remote ENVs // Register root CAs for remote ENVs
env.RegisterGlobalCAs(globalRootCAs) env.RegisterGlobalCAs(globalRootCAs)
dns.RegisterGlobalCAs(globalRootCAs)
globalMinioAddr = globalCLIContext.Addr globalMinioAddr = globalCLIContext.Addr

@ -179,7 +179,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
if globalDNSConfig != nil { if globalDNSConfig != nil {
if _, err := globalDNSConfig.Get(args.BucketName); err != nil { if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
if err == dns.ErrNoEntriesFound { if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
// Proceed to creating a bucket. // Proceed to creating a bucket.
if err = objectAPI.MakeBucketWithLocation(ctx, args.BucketName, opts); err != nil { if err = objectAPI.MakeBucketWithLocation(ctx, args.BucketName, opts); err != nil {
return toJSONError(ctx, err) return toJSONError(ctx, err)
@ -281,7 +281,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
if globalDNSConfig != nil { if globalDNSConfig != nil {
if err := globalDNSConfig.Delete(args.BucketName); err != nil { if err := globalDNSConfig.Delete(args.BucketName); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err)) logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
return toJSONError(ctx, err) return toJSONError(ctx, err)
} }
} }

4
pkg/env/env.go vendored

@ -45,7 +45,7 @@ func SetEnvOn() {
// IsSet returns if the given env key is set. // IsSet returns if the given env key is set.
func IsSet(key string) bool { func IsSet(key string) bool {
_, ok := LookupEnv(key) _, _, _, ok := LookupEnv(key)
return ok return ok
} }
@ -60,7 +60,7 @@ func Get(key, defaultValue string) string {
if ok { if ok {
return defaultValue return defaultValue
} }
if v, ok := LookupEnv(key); ok { if v, _, _, ok := LookupEnv(key); ok {
return v return v
} }
return defaultValue return defaultValue

34
pkg/env/web_env.go vendored

@ -72,10 +72,10 @@ func fetchHTTPConstituentParts(u *url.URL) (username string, password string, en
return username, password, envURL, nil return username, password, envURL, nil
} }
func getEnvValueFromHTTP(urlStr, envKey string) (string, error) { func getEnvValueFromHTTP(urlStr, envKey string) (string, string, string, error) {
u, err := url.Parse(urlStr) u, err := url.Parse(urlStr)
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
switch u.Scheme { switch u.Scheme {
@ -84,12 +84,12 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
case webEnvSchemeSecure: case webEnvSchemeSecure:
u.Scheme = "https" u.Scheme = "https"
default: default:
return "", errors.New("invalid arguments") return "", "", "", errors.New("invalid arguments")
} }
username, password, envURL, err := fetchHTTPConstituentParts(u) username, password, envURL, err := fetchHTTPConstituentParts(u)
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
@ -97,7 +97,7 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, envURL+"?key="+envKey, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, envURL+"?key="+envKey, nil)
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
claims := &jwt.StandardClaims{ claims := &jwt.StandardClaims{
@ -109,7 +109,7 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims) token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
ss, err := token.SignedString([]byte(password)) ss, err := token.SignedString([]byte(password))
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
req.Header.Set("Authorization", "Bearer "+ss) req.Header.Set("Authorization", "Bearer "+ss)
@ -136,15 +136,15 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
resp, err := clnt.Do(req) resp, err := clnt.Do(req)
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
envValueBytes, err := ioutil.ReadAll(resp.Body) envValueBytes, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", err return "", "", "", err
} }
return string(envValueBytes), nil return string(envValueBytes), username, password, nil
} }
// Environ returns a copy of strings representing the // Environ returns a copy of strings representing the
@ -161,23 +161,27 @@ func Environ() []string {
// //
// Additionally if the input is env://username:password@remote:port/ // Additionally if the input is env://username:password@remote:port/
// to fetch ENV values for the env value from a remote server. // to fetch ENV values for the env value from a remote server.
func LookupEnv(key string) (string, bool) { // In this case, it also returns the credentials username and password
func LookupEnv(key string) (string, string, string, bool) {
v, ok := os.LookupEnv(key) v, ok := os.LookupEnv(key)
if ok && strings.HasPrefix(v, webEnvScheme) { if ok && strings.HasPrefix(v, webEnvScheme) {
// If env value starts with `env*://` // If env value starts with `env*://`
// continue to parse and fetch from remote // continue to parse and fetch from remote
var err error var err error
v, err = getEnvValueFromHTTP(strings.TrimSpace(v), key) v, user, pwd, err := getEnvValueFromHTTP(strings.TrimSpace(v), key)
if err != nil { if err != nil {
// fallback to cached value if-any. env, eok := os.LookupEnv("_" + key)
return os.LookupEnv("_" + key) if eok {
// fallback to cached value if-any.
return env, user, pwd, eok
}
} }
// Set the ENV value to _env value, // Set the ENV value to _env value,
// this value is a fallback in-case of // this value is a fallback in-case of
// server restarts when webhook server // server restarts when webhook server
// is down. // is down.
os.Setenv("_"+key, v) os.Setenv("_"+key, v)
return v, true return v, user, pwd, true
} }
return v, ok return v, "", "", ok
} }

@ -67,7 +67,7 @@ func TestWebEnv(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
v, err := getEnvValueFromHTTP( v, user, pwd, err := getEnvValueFromHTTP(
fmt.Sprintf("env://minio:minio123@%s/webhook/v1/getenv/default/minio", fmt.Sprintf("env://minio:minio123@%s/webhook/v1/getenv/default/minio",
u.Host), u.Host),
"MINIO_ARGS") "MINIO_ARGS")
@ -78,4 +78,12 @@ func TestWebEnv(t *testing.T) {
if v != "http://127.0.0.{1..4}:9000/data{1...4}" { if v != "http://127.0.0.{1..4}:9000/data{1...4}" {
t.Fatalf("Unexpected value %s", v) t.Fatalf("Unexpected value %s", v)
} }
if user != "minio" {
t.Fatalf("Unexpected value %s", v)
}
if pwd != "minio123" {
t.Fatalf("Unexpected value %s", v)
}
} }

Loading…
Cancel
Save