From a4383051d941417cf8e508b943f6fa95e3cb4263 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 4 Jan 2021 09:43:31 -0800 Subject: [PATCH] remove/deprecate crawler disable environment (#11214) with changes present to automatically throttle crawler at runtime, there is no need to have an environment value to disable crawling. crawling is a fundamental piece for healing, lifecycle and many other features there is no good reason anyone would need to disable this on a production system. * Apply suggestions from code review --- cmd/admin-bucket-handlers.go | 8 -------- cmd/admin-router.go | 40 ++++++++++++++++-------------------- cmd/api-errors.go | 12 ----------- cmd/bucket-handlers.go | 7 ------- cmd/data-crawler.go | 8 ++------ cmd/data-usage.go | 1 - cmd/metrics.go | 7 ------- 7 files changed, 20 insertions(+), 63 deletions(-) diff --git a/cmd/admin-bucket-handlers.go b/cmd/admin-bucket-handlers.go index ac8ddf574..a640eba2a 100644 --- a/cmd/admin-bucket-handlers.go +++ b/cmd/admin-bucket-handlers.go @@ -23,9 +23,7 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/env" iampolicy "github.com/minio/minio/pkg/iam/policy" "github.com/minio/minio/pkg/madmin" ) @@ -54,12 +52,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r * vars := mux.Vars(r) bucket := vars["bucket"] - // Turn off quota commands if data usage info is unavailable. - if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff { - writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL) - return - } - if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil { writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) return diff --git a/cmd/admin-router.go b/cmd/admin-router.go index c82d3752e..3ca06cca4 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -20,8 +20,6 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/minio/minio/cmd/config" - "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/madmin" ) @@ -172,27 +170,25 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) } if globalIsDistErasure || globalIsErasure { - // Quota operations - if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn { - // GetBucketQuotaConfig - adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc( - httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") - // PutBucketQuotaConfig - adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc( - httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") - - // Bucket replication operations - // GetBucketTargetHandler - adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc( - httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}") - // SetRemoteTargetHandler - adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc( - httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}") - // RemoveRemoteTargetHandler - adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc( - httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}") - } + // GetBucketQuotaConfig + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc( + httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") + // PutBucketQuotaConfig + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc( + httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") + + // Bucket replication operations + // GetBucketTargetHandler + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc( + httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}") + // SetRemoteTargetHandler + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc( + httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}") + // RemoveRemoteTargetHandler + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc( + httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}") } + // -- Top APIs -- // Top locks if globalIsDistErasure { diff --git a/cmd/api-errors.go b/cmd/api-errors.go index c35af92f6..50391a04c 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -120,7 +120,6 @@ const ( ErrReplicationSourceNotVersionedError ErrReplicationNeedsVersioningError ErrReplicationBucketNeedsVersioningError - ErrBucketReplicationDisabledError ErrObjectRestoreAlreadyInProgress ErrNoSuchKey ErrNoSuchUpload @@ -263,7 +262,6 @@ const ( // Bucket Quota error codes ErrAdminBucketQuotaExceeded ErrAdminNoSuchQuotaConfiguration - ErrAdminBucketQuotaDisabled ErrHealNotImplemented ErrHealNoSuchProcess @@ -889,11 +887,6 @@ var errorCodes = errorCodeMap{ Description: "Versioning must be 'Enabled' on the bucket to add a replication target", HTTPStatusCode: http.StatusBadRequest, }, - ErrBucketReplicationDisabledError: { - Code: "XMinioAdminBucketReplicationDisabled", - Description: "Replication specified but disk usage crawl is disabled on MinIO server", - HTTPStatusCode: http.StatusBadRequest, - }, ErrNoSuchObjectLockConfiguration: { Code: "NoSuchObjectLockConfiguration", Description: "The specified object does not have a ObjectLock configuration", @@ -1215,11 +1208,6 @@ var errorCodes = errorCodeMap{ Description: "The quota configuration does not exist", HTTPStatusCode: http.StatusNotFound, }, - ErrAdminBucketQuotaDisabled: { - Code: "XMinioAdminBucketQuotaDisabled", - Description: "Quota specified but disk usage crawl is disabled on MinIO server", - HTTPStatusCode: http.StatusBadRequest, - }, ErrInsecureClientRequest: { Code: "XMinioInsecureClientRequest", Description: "Cannot respond to plain-text request from TLS-encrypted server", diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 441db5189..f7c558b19 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -33,7 +33,6 @@ import ( "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/tags" - "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config/dns" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" @@ -42,7 +41,6 @@ import ( objectlock "github.com/minio/minio/pkg/bucket/object/lock" "github.com/minio/minio/pkg/bucket/policy" "github.com/minio/minio/pkg/bucket/replication" - "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/hash" @@ -1336,11 +1334,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) return } - // Turn off replication if disk crawl is unavailable. - if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff { - writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL) - return - } if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return diff --git a/cmd/data-crawler.go b/cmd/data-crawler.go index bdd55f7c7..00570975f 100644 --- a/cmd/data-crawler.go +++ b/cmd/data-crawler.go @@ -29,14 +29,12 @@ import ( "sync" "time" - "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config/heal" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/bucket/lifecycle" "github.com/minio/minio/pkg/bucket/replication" "github.com/minio/minio/pkg/color" "github.com/minio/minio/pkg/console" - "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/madmin" @@ -62,11 +60,9 @@ var ( crawlerSleeper = newDynamicSleeper(10, 10*time.Second) ) -// initDataCrawler will start the crawler unless disabled. +// initDataCrawler will start the crawler in the background. func initDataCrawler(ctx context.Context, objAPI ObjectLayer) { - if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn { - go runDataCrawler(ctx, objAPI) - } + go runDataCrawler(ctx, objAPI) } // runDataCrawler will start a data crawler. diff --git a/cmd/data-usage.go b/cmd/data-usage.go index 88df443ec..87e6e9851 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -27,7 +27,6 @@ import ( ) const ( - envDataUsageCrawlConf = "MINIO_DISK_USAGE_CRAWL_ENABLE" envDataUsageCrawlDebug = "MINIO_DISK_USAGE_CRAWL_DEBUG" dataUsageRoot = SlashSeparator diff --git a/cmd/metrics.go b/cmd/metrics.go index 912cd46e9..b85d46a4e 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -22,9 +22,7 @@ import ( "sync/atomic" "time" - "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/env" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -402,11 +400,6 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) { return } - // Crawler disabled, nothing to do. - if env.Get(envDataUsageCrawlConf, config.EnableOn) != config.EnableOn { - return - } - dataUsageInfo, err := loadDataUsageFromBackend(GlobalContext, objLayer) if err != nil { return