From 52bdbcd0460c73f2705f31d1657ac70be4637256 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Wed, 5 Feb 2020 03:20:39 +0100 Subject: [PATCH] Add new admin API to return Accounting Usage (#8689) --- cmd/admin-handlers.go | 61 +++++++++++ cmd/admin-router.go | 2 + cmd/config/policy/opa/config.go | 1 + cmd/iam.go | 104 +++++++++++++++++++ pkg/iam/policy/actionset.go | 5 + pkg/iam/policy/admin-action.go | 2 + pkg/madmin/examples/accounting-usage-info.go | 45 ++++++++ pkg/madmin/info-commands.go | 44 ++++++++ 8 files changed, 264 insertions(+) create mode 100644 pkg/madmin/examples/accounting-usage-info.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 409c685eb..f311e526d 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -316,6 +316,67 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re writeSuccessResponseJSON(w, dataUsageInfoJSON) } +func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AccountingUsageInfo") + objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction) + if objectAPI == nil { + return + } + + var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage) + + buckets, err := objectAPI.ListBuckets(ctx) + if err != nil { + // writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + users, err := globalIAMSys.ListUsers() + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Load the latest calculated data usage + dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) + if err != nil { + logger.LogIf(ctx, err) + } + + // Calculate for each bucket, which users are allowed to access to it + for _, bucket := range buckets { + bucketUsageInfo := madmin.BucketAccountingUsage{} + + // Fetch the data usage of the current bucket + if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil { + bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name] + } + + for user := range users { + rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name) + if rd || wr || custom { + bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{ + AccountName: user, + Read: rd, + Write: wr, + Custom: custom, + }) + } + } + + accountingUsageInfo[bucket.Name] = bucketUsageInfo + } + + usageInfoJSON, err := json.Marshal(accountingUsageInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, usageInfoJSON) +} + // ServerCPULoadInfo holds informantion about cpu utilization // of one minio node. It also reports any errors if encountered // while trying to reach this server. diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 1c3313d13..b2cb97533 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -56,6 +56,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) // DataUsageInfo operations adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler)) + adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler)) + if globalIsDistXL || globalIsXL { /// Heal operations diff --git a/cmd/config/policy/opa/config.go b/cmd/config/policy/opa/config.go index a7c6f702f..05745e2c7 100644 --- a/cmd/config/policy/opa/config.go +++ b/cmd/config/policy/opa/config.go @@ -223,5 +223,6 @@ func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) { } return resultAllow.Result.Allow, nil } + return result.Result, nil } diff --git a/cmd/iam.go b/cmd/iam.go index fc244a59c..be18521e2 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -26,6 +26,7 @@ import ( "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/bucket/policy" iampolicy "github.com/minio/minio/pkg/iam/policy" "github.com/minio/minio/pkg/madmin" ) @@ -1130,6 +1131,109 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error { return nil } +var iamAccountReadAccessActions = iampolicy.NewActionSet( + iampolicy.ListMultipartUploadPartsAction, + iampolicy.ListBucketMultipartUploadsAction, + iampolicy.ListBucketAction, + iampolicy.HeadBucketAction, + iampolicy.GetObjectAction, + iampolicy.GetBucketLocationAction, + + // iampolicy.ListAllMyBucketsAction, +) + +var iamAccountWriteAccessActions = iampolicy.NewActionSet( + iampolicy.AbortMultipartUploadAction, + iampolicy.CreateBucketAction, + iampolicy.PutObjectAction, + iampolicy.DeleteObjectAction, + iampolicy.DeleteBucketAction, +) + +var iamAccountOtherAccessActions = iampolicy.NewActionSet( + iampolicy.BypassGovernanceModeAction, + iampolicy.BypassGovernanceRetentionAction, + iampolicy.PutObjectRetentionAction, + iampolicy.GetObjectRetentionAction, + iampolicy.GetObjectLegalHoldAction, + iampolicy.PutObjectLegalHoldAction, + iampolicy.GetBucketObjectLockConfigurationAction, + iampolicy.PutBucketObjectLockConfigurationAction, + + iampolicy.ListenBucketNotificationAction, + + iampolicy.PutBucketLifecycleAction, + iampolicy.GetBucketLifecycleAction, + + iampolicy.PutBucketNotificationAction, + iampolicy.GetBucketNotificationAction, + + iampolicy.PutBucketPolicyAction, + iampolicy.DeleteBucketPolicyAction, + iampolicy.GetBucketPolicyAction, +) + +// GetAccountAccess iterates over all policies documents associated to a user +// and returns if the user has read and/or write access to any resource. +func (sys *IAMSys) GetAccountAccess(accountName, bucket string) (rd, wr, o bool) { + policies, err := sys.PolicyDBGet(accountName, false) + if err != nil { + logger.LogIf(context.Background(), err) + return false, false, false + } + + if len(policies) == 0 { + // No policy found. + return false, false, false + } + + // Policies were found, evaluate all of them. + sys.RLock() + defer sys.RUnlock() + + var availablePolicies []iampolicy.Policy + for _, pname := range policies { + p, found := sys.iamPolicyDocsMap[pname] + if found { + availablePolicies = append(availablePolicies, p) + } + } + + if len(availablePolicies) == 0 { + return false, false, false + } + + combinedPolicy := availablePolicies[0] + for i := 1; i < len(availablePolicies); i++ { + combinedPolicy.Statements = append(combinedPolicy.Statements, + availablePolicies[i].Statements...) + } + + allActions := iampolicy.NewActionSet(iampolicy.AllActions) + for _, st := range combinedPolicy.Statements { + // Ignore if this is not an allow policy statement + if st.Effect != policy.Allow { + continue + } + // Fast calculation if there is s3:* permissions to any resource + if !st.Actions.Intersection(allActions).IsEmpty() { + rd, wr, o = true, true, true + break + } + if !st.Actions.Intersection(iamAccountReadAccessActions).IsEmpty() { + rd = true + } + if !st.Actions.Intersection(iamAccountWriteAccessActions).IsEmpty() { + wr = true + } + if !st.Actions.Intersection(iamAccountOtherAccessActions).IsEmpty() { + o = true + } + } + + return +} + // PolicyDBGet - gets policy set on a user or group. Since a user may // be a member of multiple groups, this function returns an array of // applicable policies (each group is mapped to at most one policy). diff --git a/pkg/iam/policy/actionset.go b/pkg/iam/policy/actionset.go index fc09209e1..e670c7d6c 100644 --- a/pkg/iam/policy/actionset.go +++ b/pkg/iam/policy/actionset.go @@ -32,6 +32,11 @@ func (actionSet ActionSet) Add(action Action) { actionSet[action] = struct{}{} } +// IsEmpty - returns if the current action set is empty +func (actionSet ActionSet) IsEmpty() bool { + return len(actionSet) == 0 +} + // Match - matches object name with anyone of action pattern in action set. func (actionSet ActionSet) Match(action Action) bool { for r := range actionSet { diff --git a/pkg/iam/policy/admin-action.go b/pkg/iam/policy/admin-action.go index 6f8af81ac..cb14dca12 100644 --- a/pkg/iam/policy/admin-action.go +++ b/pkg/iam/policy/admin-action.go @@ -31,6 +31,8 @@ const ( // StorageInfoAdminAction - allow listing server info StorageInfoAdminAction = "admin:StorageInfo" + // AccountingUsageInfoAdminAction - allow listing accounting usage info + AccountingUsageInfoAdminAction = "admin:AccountingUsageInfo" // DataUsageInfoAdminAction - allow listing data usage info DataUsageInfoAdminAction = "admin:DataUsageInfo" // PerfInfoAdminAction - allow listing performance info diff --git a/pkg/madmin/examples/accounting-usage-info.go b/pkg/madmin/examples/accounting-usage-info.go new file mode 100644 index 000000000..ca2a38466 --- /dev/null +++ b/pkg/madmin/examples/accounting-usage-info.go @@ -0,0 +1,45 @@ +// +build ignore + +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "log" + + "github.com/minio/minio/pkg/madmin" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise. + // New returns an MinIO Admin client object. + madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + accountingUsageInfo, err := madmClnt.AccountingUsageInfo() + if err != nil { + log.Fatalln(err) + } + + log.Println(accountingUsageInfo) +} diff --git a/pkg/madmin/info-commands.go b/pkg/madmin/info-commands.go index 6b5c5c011..54911f877 100644 --- a/pkg/madmin/info-commands.go +++ b/pkg/madmin/info-commands.go @@ -198,6 +198,50 @@ func (adm *AdminClient) DataUsageInfo() (DataUsageInfo, error) { return dataUsageInfo, nil } +// AccountAccess contains information about +type AccountAccess struct { + AccountName string `json:"accountName"` + Read bool `json:"read"` + Write bool `json:"write"` + Custom bool `json:"custom"` +} + +// BucketAccountingUsage represents the accounting usage of a particular bucket +type BucketAccountingUsage struct { + Size uint64 `json:"size"` + AccessList []AccountAccess `json:"accessList"` +} + +// AccountingUsageInfo returns the accounting usage info, currently it returns +// the type of access of different accounts to the different buckets. +func (adm *AdminClient) AccountingUsageInfo() (map[string]BucketAccountingUsage, error) { + resp, err := adm.executeMethod(http.MethodGet, requestData{relPath: adminAPIPrefix + "/accountingusageinfo"}) + defer closeResponse(resp) + if err != nil { + return nil, err + } + + // Check response http status code + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp) + } + + // Unmarshal the server's json response + var accountingUsageInfo map[string]BucketAccountingUsage + + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(respBytes, &accountingUsageInfo) + if err != nil { + return nil, err + } + + return accountingUsageInfo, nil +} + // ServerDrivesPerfInfo holds informantion about address and write speed of // all drives in a single server node type ServerDrivesPerfInfo struct {