Add new admin API to return Accounting Usage (#8689)

cmd/admin-handlers.go

@@ -316,6 +316,67 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
+func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountingUsageInfo")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	users, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	// Calculate for each bucket, which users are allowed to access to it
+	for _, bucket := range buckets {
+		bucketUsageInfo := madmin.BucketAccountingUsage{}
+
+		// Fetch the data usage of the current bucket
+		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
+			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
+		}
+
+		for user := range users {
+			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
+			if rd || wr || custom {
+				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
+					AccountName: user,
+					Read:        rd,
+					Write:       wr,
+					Custom:      custom,
+				})
+			}
+		}
+
+		accountingUsageInfo[bucket.Name] = bucketUsageInfo
+	}
+
+	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
+}
+
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.

cmd/admin-router.go

@@ -56,6 +56,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// DataUsageInfo operations
 	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
+
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
 

cmd/config/policy/opa/config.go

@@ -223,5 +223,6 @@ func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) {
 		}
 		return resultAllow.Result.Allow, nil
 	}
+
 	return result.Result, nil
 }

cmd/iam.go

@@ -26,6 +26,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -1130,6 +1131,109 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 	return nil
 }
 
+var iamAccountReadAccessActions = iampolicy.NewActionSet(
+	iampolicy.ListMultipartUploadPartsAction,
+	iampolicy.ListBucketMultipartUploadsAction,
+	iampolicy.ListBucketAction,
+	iampolicy.HeadBucketAction,
+	iampolicy.GetObjectAction,
+	iampolicy.GetBucketLocationAction,
+
+	// iampolicy.ListAllMyBucketsAction,
+)
+
+var iamAccountWriteAccessActions = iampolicy.NewActionSet(
+	iampolicy.AbortMultipartUploadAction,
+	iampolicy.CreateBucketAction,
+	iampolicy.PutObjectAction,
+	iampolicy.DeleteObjectAction,
+	iampolicy.DeleteBucketAction,
+)
+
+var iamAccountOtherAccessActions = iampolicy.NewActionSet(
+	iampolicy.BypassGovernanceModeAction,
+	iampolicy.BypassGovernanceRetentionAction,
+	iampolicy.PutObjectRetentionAction,
+	iampolicy.GetObjectRetentionAction,
+	iampolicy.GetObjectLegalHoldAction,
+	iampolicy.PutObjectLegalHoldAction,
+	iampolicy.GetBucketObjectLockConfigurationAction,
+	iampolicy.PutBucketObjectLockConfigurationAction,
+
+	iampolicy.ListenBucketNotificationAction,
+
+	iampolicy.PutBucketLifecycleAction,
+	iampolicy.GetBucketLifecycleAction,
+
+	iampolicy.PutBucketNotificationAction,
+	iampolicy.GetBucketNotificationAction,
+
+	iampolicy.PutBucketPolicyAction,
+	iampolicy.DeleteBucketPolicyAction,
+	iampolicy.GetBucketPolicyAction,
+)
+
+// GetAccountAccess iterates over all policies documents associated to a user
+// and returns if the user has read and/or write access to any resource.
+func (sys *IAMSys) GetAccountAccess(accountName, bucket string) (rd, wr, o bool) {
+	policies, err := sys.PolicyDBGet(accountName, false)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false, false, false
+	}
+
+	if len(policies) == 0 {
+		// No policy found.
+		return false, false, false
+	}
+
+	// Policies were found, evaluate all of them.
+	sys.RLock()
+	defer sys.RUnlock()
+
+	var availablePolicies []iampolicy.Policy
+	for _, pname := range policies {
+		p, found := sys.iamPolicyDocsMap[pname]
+		if found {
+			availablePolicies = append(availablePolicies, p)
+		}
+	}
+
+	if len(availablePolicies) == 0 {
+		return false, false, false
+	}
+
+	combinedPolicy := availablePolicies[0]
+	for i := 1; i < len(availablePolicies); i++ {
+		combinedPolicy.Statements = append(combinedPolicy.Statements,
+			availablePolicies[i].Statements...)
+	}
+
+	allActions := iampolicy.NewActionSet(iampolicy.AllActions)
+	for _, st := range combinedPolicy.Statements {
+		// Ignore if this is not an allow policy statement
+		if st.Effect != policy.Allow {
+			continue
+		}
+		// Fast calculation if there is s3:* permissions to any resource
+		if !st.Actions.Intersection(allActions).IsEmpty() {
+			rd, wr, o = true, true, true
+			break
+		}
+		if !st.Actions.Intersection(iamAccountReadAccessActions).IsEmpty() {
+			rd = true
+		}
+		if !st.Actions.Intersection(iamAccountWriteAccessActions).IsEmpty() {
+			wr = true
+		}
+		if !st.Actions.Intersection(iamAccountOtherAccessActions).IsEmpty() {
+			o = true
+		}
+	}
+
+	return
+}
+
 // PolicyDBGet - gets policy set on a user or group. Since a user may
 // be a member of multiple groups, this function returns an array of
 // applicable policies (each group is mapped to at most one policy).

pkg/iam/policy/actionset.go

@@ -32,6 +32,11 @@ func (actionSet ActionSet) Add(action Action) {
 	actionSet[action] = struct{}{}
 }
 
+// IsEmpty - returns if the current action set is empty
+func (actionSet ActionSet) IsEmpty() bool {
+	return len(actionSet) == 0
+}
+
 // Match - matches object name with anyone of action pattern in action set.
 func (actionSet ActionSet) Match(action Action) bool {
 	for r := range actionSet {

pkg/iam/policy/admin-action.go

@@ -31,6 +31,8 @@ const (
 
 	// StorageInfoAdminAction - allow listing server info
 	StorageInfoAdminAction = "admin:StorageInfo"
+	// AccountingUsageInfoAdminAction - allow listing accounting usage info
+	AccountingUsageInfoAdminAction = "admin:AccountingUsageInfo"
 	// DataUsageInfoAdminAction - allow listing data usage info
 	DataUsageInfoAdminAction = "admin:DataUsageInfo"
 	// PerfInfoAdminAction - allow listing performance info

pkg/madmin/examples/accounting-usage-info.go

@@ -0,0 +1,45 @@
+// +build ignore
+
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	accountingUsageInfo, err := madmClnt.AccountingUsageInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	log.Println(accountingUsageInfo)
+}

pkg/madmin/info-commands.go

@@ -198,6 +198,50 @@ func (adm *AdminClient) DataUsageInfo() (DataUsageInfo, error) {
 	return dataUsageInfo, nil
 }
 
+// AccountAccess contains information about
+type AccountAccess struct {
+	AccountName string `json:"accountName"`
+	Read        bool   `json:"read"`
+	Write       bool   `json:"write"`
+	Custom      bool   `json:"custom"`
+}
+
+// BucketAccountingUsage represents the accounting usage of a particular bucket
+type BucketAccountingUsage struct {
+	Size       uint64          `json:"size"`
+	AccessList []AccountAccess `json:"accessList"`
+}
+
+// AccountingUsageInfo returns the accounting usage info, currently it returns
+// the type of access of different accounts to the different buckets.
+func (adm *AdminClient) AccountingUsageInfo() (map[string]BucketAccountingUsage, error) {
+	resp, err := adm.executeMethod(http.MethodGet, requestData{relPath: adminAPIPrefix + "/accountingusageinfo"})
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var accountingUsageInfo map[string]BucketAccountingUsage
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &accountingUsageInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	return accountingUsageInfo, nil
+}
+
 // ServerDrivesPerfInfo holds informantion about address and write speed of
 // all drives in a single server node
 type ServerDrivesPerfInfo struct {