bucket-policy: Migrate bucket policy to minioMetaBuket/buckets (#2321)

master
Krishnan Parthasarathi 8 years ago committed by Harshavardhana
parent 14cefd352c
commit 50dae0ab04
  1. 4
      bucket-handlers-listobjects.go
  2. 14
      bucket-handlers.go
  3. 6
      bucket-policy-handlers.go
  4. 83
      bucket-policy-migrate.go
  5. 79
      bucket-policy.go
  6. 28
      object-handlers.go
  7. 7
      routers.go

@ -71,7 +71,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -130,7 +130,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}

@ -32,9 +32,9 @@ import (
)
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
func enforceBucketPolicy(api objectAPIHandlers, action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
// Read saved bucket policy.
policy, err := readBucketPolicy(bucket)
policy, err := readBucketPolicy(api, bucket)
if err != nil {
errorIf(err, "Unable read bucket policy.")
switch err.(type) {
@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -154,7 +154,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -232,7 +232,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -436,7 +436,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -474,7 +474,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
}
// Delete bucket access policy, if present - ignore any errors.
removeBucketPolicy(bucket)
removeBucketPolicy(api, bucket)
// Write success response.
writeSuccessNoContent(w)

@ -215,7 +215,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
}
// Save bucket policy.
if err := writeBucketPolicy(bucket, bucketPolicyBuf); err != nil {
if err := writeBucketPolicy(api, bucket, bucketPolicyBuf); err != nil {
errorIf(err, "Unable to write bucket policy.")
switch err.(type) {
case BucketNameInvalid:
@ -249,7 +249,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
}
// Delete bucket access policy.
if err := removeBucketPolicy(bucket); err != nil {
if err := removeBucketPolicy(api, bucket); err != nil {
errorIf(err, "Unable to remove bucket policy.")
switch err.(type) {
case BucketNameInvalid:
@ -285,7 +285,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
}
// Read bucket access policy.
p, err := readBucketPolicy(bucket)
p, err := readBucketPolicy(api, bucket)
if err != nil {
errorIf(err, "Unable to read bucket policy.")
switch err.(type) {

@ -0,0 +1,83 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
const policyJSON = "policy.json"
func getBucketFromPolicyPath(oldPolicyPath string) string {
bucketPrefix, _ := filepath.Split(oldPolicyPath)
_, bucketName := filepath.Split(strings.TrimSuffix(bucketPrefix, slashSeparator))
return bucketName
}
func cleanupOldBucketPolicyConfigs() error {
// Get old bucket policy config directory.
oldBucketsConfigDir, err := getOldBucketsConfigPath()
fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
// Recursively remove configDir/buckets/ - old bucket policy config location.
// N B This is called only if all bucket policies were successfully migrated.
return os.RemoveAll(oldBucketsConfigDir)
}
func migrateBucketPolicyConfig(objAPI ObjectLayer) error {
// Get old bucket policy config directory.
oldBucketsConfigDir, err := getOldBucketsConfigPath()
fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
// Check if config directory holding bucket policy exists before
// migration.
_, err = os.Stat(oldBucketsConfigDir)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
// WalkFunc that migrates access-policy.json to
// .minio.sys/buckets/bucketName/policy.json on all disks.
migrateBucketPolicy := func(policyPath string, fileInfo os.FileInfo, err error) error {
// policyFile - e.g /configDir/sample-bucket/access-policy.json
if err != nil {
return err
}
// Skip entries that aren't bucket policy files.
if fileInfo.Name() != "access-policy.json" {
return nil
}
// Get bucketName from old policy file path.
bucketName := getBucketFromPolicyPath(policyPath)
// Read bucket policy config from old location.
policyBytes, err := ioutil.ReadFile(policyPath)
fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath)
newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON
// Erasure code the policy config to all the disks.
_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil)
fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath)
return nil
}
return filepath.Walk(oldBucketsConfigDir, migrateBucketPolicy)
}

@ -17,13 +17,12 @@
package main
import (
"io/ioutil"
"os"
"bytes"
"path/filepath"
)
// getBucketsConfigPath - get buckets path.
func getBucketsConfigPath() (string, error) {
func getOldBucketsConfigPath() (string, error) {
configPath, err := getConfigPath()
if err != nil {
return "", err
@ -32,97 +31,65 @@ func getBucketsConfigPath() (string, error) {
}
// getBucketConfigPath - get bucket config path.
func getBucketConfigPath(bucket string) (string, error) {
bucketsConfigPath, err := getBucketsConfigPath()
func getOldBucketConfigPath(bucket string) (string, error) {
bucketsConfigPath, err := getOldBucketsConfigPath()
if err != nil {
return "", err
}
return filepath.Join(bucketsConfigPath, bucket), nil
}
// createBucketConfigPath - create bucket config directory.
func createBucketConfigPath(bucket string) error {
bucketConfigPath, err := getBucketConfigPath(bucket)
if err != nil {
return err
}
return os.MkdirAll(bucketConfigPath, 0700)
}
// readBucketPolicy - read bucket policy.
func readBucketPolicy(bucket string) ([]byte, error) {
func readBucketPolicy(api objectAPIHandlers, bucket string) ([]byte, error) {
// Verify bucket is valid.
if !IsValidBucketName(bucket) {
return nil, BucketNameInvalid{Bucket: bucket}
}
bucketConfigPath, err := getBucketConfigPath(bucket)
policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
objInfo, err := api.ObjectAPI.GetObjectInfo(minioMetaBucket, policyPath)
if err != nil {
if _, ok := err.(ObjectNotFound); ok {
return nil, BucketPolicyNotFound{Bucket: bucket}
}
return nil, err
}
// Get policy file.
bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
if _, err = os.Stat(bucketPolicyFile); err != nil {
if os.IsNotExist(err) {
var buffer bytes.Buffer
err = api.ObjectAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer)
if err != nil {
if _, ok := err.(ObjectNotFound); ok {
return nil, BucketPolicyNotFound{Bucket: bucket}
}
return nil, err
}
return ioutil.ReadFile(bucketPolicyFile)
return buffer.Bytes(), nil
}
// removeBucketPolicy - remove bucket policy.
func removeBucketPolicy(bucket string) error {
func removeBucketPolicy(api objectAPIHandlers, bucket string) error {
// Verify bucket is valid.
if !IsValidBucketName(bucket) {
return BucketNameInvalid{Bucket: bucket}
}
bucketConfigPath, err := getBucketConfigPath(bucket)
if err != nil {
return err
}
// Get policy file.
bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
if _, err = os.Stat(bucketPolicyFile); err != nil {
if os.IsNotExist(err) {
policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
if err := api.ObjectAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
if _, ok := err.(ObjectNotFound); ok {
return BucketPolicyNotFound{Bucket: bucket}
}
return err
}
if err := os.Remove(bucketPolicyFile); err != nil {
return err
}
return nil
}
// writeBucketPolicy - save bucket policy.
func writeBucketPolicy(bucket string, accessPolicyBytes []byte) error {
func writeBucketPolicy(api objectAPIHandlers, bucket string, accessPolicyBytes []byte) error {
// Verify if bucket path legal
if !IsValidBucketName(bucket) {
return BucketNameInvalid{Bucket: bucket}
}
// Create bucket config path.
if err := createBucketConfigPath(bucket); err != nil {
return err
}
bucketConfigPath, err := getBucketConfigPath(bucket)
if err != nil {
return err
}
// Get policy file.
bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
if _, err := os.Stat(bucketPolicyFile); err != nil {
if !os.IsNotExist(err) {
return err
}
}
// Write bucket policy.
return ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600)
policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
_, err := api.ObjectAPI.PutObject(minioMetaBucket, policyPath, int64(len(accessPolicyBytes)), bytes.NewReader(accessPolicyBytes), nil)
return err
}

@ -53,13 +53,13 @@ func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
// this is in keeping with the permissions sections of the docs of both:
// HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
// GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode {
func errAllowableObjectNotFound(api objectAPIHandlers, bucket string, r *http.Request) APIErrorCode {
if getRequestAuthType(r) == authTypeAnonymous {
//we care about the bucket as a whole, not a particular resource
url := *r.URL
url.Path = "/" + bucket
if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, &url); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListBucket", bucket, &url); s3Error != ErrNone {
return ErrAccessDenied
}
}
@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
errorIf(err, "Unable to fetch object info.")
apiErr := toAPIErrorCode(err)
if apiErr == ErrNoSuchKey {
apiErr = errAllowableObjectNotFound(bucket, r)
apiErr = errAllowableObjectNotFound(api, bucket, r)
}
writeErrorResponse(w, r, apiErr, r.URL.Path)
return
@ -196,7 +196,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:GetObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:GetObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -212,7 +212,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
errorIf(err, "Unable to fetch object info.")
apiErr := toAPIErrorCode(err)
if apiErr == ErrNoSuchKey {
apiErr = errAllowableObjectNotFound(bucket, r)
apiErr = errAllowableObjectNotFound(api, bucket, r)
}
writeErrorResponse(w, r, apiErr, r.URL.Path)
return
@ -246,7 +246,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -418,7 +418,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -479,7 +479,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -558,7 +558,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -594,7 +594,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -640,7 +640,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -691,7 +691,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:PutObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}
@ -815,7 +815,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
return
case authTypeAnonymous:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
if s3Error := enforceBucketPolicy(api, "s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path)
return
}

@ -45,6 +45,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
objAPI, err := newObjectLayer(srvCmdConfig.disks, srvCmdConfig.ignoredDisks)
fatalIf(err, "Unable to intialize object layer.")
// Migrate bucket policy from configDir to .minio.sys/buckets/
err = migrateBucketPolicyConfig(objAPI)
fatalIf(err, "Unable to migrate bucket policy from config directory")
err = cleanupOldBucketPolicyConfigs()
fatalIf(err, "Unable to clean up bucket policy from config directory.")
// Initialize storage rpc server.
storageRPC, err := newRPCServer(srvCmdConfig.disks[0]) // FIXME: should only have one path.
fatalIf(err, "Unable to initialize storage RPC server.")

Loading…
Cancel
Save