Add bucket lifecycle expiry feature (#7834)
parent
a8296445ad
commit
1ce8d2c476
@ -0,0 +1,164 @@ |
|||||||
|
/* |
||||||
|
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package cmd |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/minio/minio/cmd/logger" |
||||||
|
"github.com/minio/minio/pkg/lifecycle" |
||||||
|
) |
||||||
|
|
||||||
|
const ( |
||||||
|
bgLifecycleInterval = 24 * time.Hour |
||||||
|
bgLifecycleTick = time.Hour |
||||||
|
) |
||||||
|
|
||||||
|
type lifecycleOps struct { |
||||||
|
LastActivity time.Time |
||||||
|
} |
||||||
|
|
||||||
|
// Register to the daily objects listing
|
||||||
|
var globalLifecycleOps = &lifecycleOps{} |
||||||
|
|
||||||
|
func getLocalBgLifecycleOpsStatus() BgLifecycleOpsStatus { |
||||||
|
return BgLifecycleOpsStatus{ |
||||||
|
LastActivity: globalLifecycleOps.LastActivity, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// initDailyLifecycle starts the routine that receives the daily
|
||||||
|
// listing of all objects and applies any matching bucket lifecycle
|
||||||
|
// rules.
|
||||||
|
func initDailyLifecycle() { |
||||||
|
go startDailyLifecycle() |
||||||
|
} |
||||||
|
|
||||||
|
func startDailyLifecycle() { |
||||||
|
var objAPI ObjectLayer |
||||||
|
var ctx = context.Background() |
||||||
|
|
||||||
|
// Wait until the object API is ready
|
||||||
|
for { |
||||||
|
objAPI = newObjectLayerFn() |
||||||
|
if objAPI == nil { |
||||||
|
time.Sleep(time.Second) |
||||||
|
continue |
||||||
|
} |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
// Calculate the time of the last lifecycle operation in all peers node of the cluster
|
||||||
|
computeLastLifecycleActivity := func(status []BgOpsStatus) time.Time { |
||||||
|
var lastAct time.Time |
||||||
|
for _, st := range status { |
||||||
|
if st.LifecycleOps.LastActivity.After(lastAct) { |
||||||
|
lastAct = st.LifecycleOps.LastActivity |
||||||
|
} |
||||||
|
} |
||||||
|
return lastAct |
||||||
|
} |
||||||
|
|
||||||
|
for { |
||||||
|
// Check if we should perform lifecycle ops based on the last lifecycle activity, sleep one hour otherwise
|
||||||
|
allLifecycleStatus := []BgOpsStatus{ |
||||||
|
{LifecycleOps: getLocalBgLifecycleOpsStatus()}, |
||||||
|
} |
||||||
|
if globalIsDistXL { |
||||||
|
allLifecycleStatus = append(allLifecycleStatus, globalNotificationSys.BackgroundOpsStatus()...) |
||||||
|
} |
||||||
|
lastAct := computeLastLifecycleActivity(allLifecycleStatus) |
||||||
|
if !lastAct.IsZero() && time.Since(lastAct) < bgLifecycleInterval { |
||||||
|
time.Sleep(bgLifecycleTick) |
||||||
|
} |
||||||
|
|
||||||
|
// Perform one lifecycle operation
|
||||||
|
err := lifecycleRound(ctx, objAPI) |
||||||
|
switch err.(type) { |
||||||
|
// Unable to hold a lock means there is another
|
||||||
|
// instance doing the lifecycle round round
|
||||||
|
case OperationTimedOut: |
||||||
|
time.Sleep(bgLifecycleTick) |
||||||
|
default: |
||||||
|
logger.LogIf(ctx, err) |
||||||
|
time.Sleep(time.Minute) |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error { |
||||||
|
|
||||||
|
zeroDuration := time.Millisecond |
||||||
|
zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration) |
||||||
|
|
||||||
|
// Lock to avoid concurrent lifecycle ops from other nodes
|
||||||
|
sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-lifecycle-ops") |
||||||
|
if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
defer sweepLock.Unlock() |
||||||
|
|
||||||
|
buckets, err := objAPI.ListBuckets(ctx) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
for _, bucket := range buckets { |
||||||
|
// Check if the current bucket has a configured lifecycle policy, skip otherwise
|
||||||
|
l, ok := globalLifecycleSys.Get(bucket.Name) |
||||||
|
if !ok { |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
// Calculate the common prefix of all lifecycle rules
|
||||||
|
var prefixes []string |
||||||
|
for _, rule := range l.Rules { |
||||||
|
prefixes = append(prefixes, rule.Filter.Prefix) |
||||||
|
} |
||||||
|
commonPrefix := lcp(prefixes) |
||||||
|
|
||||||
|
// List all objects and calculate lifecycle action based on object name & object modtime
|
||||||
|
marker := "" |
||||||
|
for { |
||||||
|
res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", 1000) |
||||||
|
if err != nil { |
||||||
|
continue |
||||||
|
} |
||||||
|
for _, obj := range res.Objects { |
||||||
|
// Find the action that need to be executed
|
||||||
|
action := l.ComputeAction(obj.Name, obj.ModTime) |
||||||
|
switch action { |
||||||
|
case lifecycle.DeleteAction: |
||||||
|
objAPI.DeleteObject(ctx, bucket.Name, obj.Name) |
||||||
|
default: |
||||||
|
// Nothing
|
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
if !res.IsTruncated { |
||||||
|
break |
||||||
|
} else { |
||||||
|
marker = res.NextMarker |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
} |
@ -0,0 +1,53 @@ |
|||||||
|
# Object Lifecycle Configuration Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) |
||||||
|
|
||||||
|
Enable object lifecycle configuration on buckets to setup automatic deletion of objects after a specified number of days or a specified date. |
||||||
|
|
||||||
|
## 1. Prerequisites |
||||||
|
- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide). |
||||||
|
- Install AWS Cli - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) |
||||||
|
|
||||||
|
|
||||||
|
## 2. Enable bucket lifecycle configuration |
||||||
|
|
||||||
|
1. Create a bucket lifecycle configuration which expires the objects under the prefix `uploads/2015` on `2020-01-01T00:00:00.000Z` date and the objects under `temporary-uploads/` after 7 days. Generate it as shown below: |
||||||
|
|
||||||
|
```sh |
||||||
|
$ cat >bucket-lifecycle.json << EOF |
||||||
|
{ |
||||||
|
"Rules": [ |
||||||
|
{ |
||||||
|
"Expiration": { |
||||||
|
"Date": "2020-01-01T00:00:00.000Z" |
||||||
|
}, |
||||||
|
"ID": "Delete very old messenger pictures", |
||||||
|
"Filter": { |
||||||
|
"Prefix": "uploads/2015/" |
||||||
|
}, |
||||||
|
"Status": "Enabled" |
||||||
|
}, |
||||||
|
{ |
||||||
|
"Expiration": { |
||||||
|
"Days": 7 |
||||||
|
}, |
||||||
|
"ID": "Delete temporary uploads", |
||||||
|
"Filter": { |
||||||
|
"Prefix": "temporary-uploads/" |
||||||
|
}, |
||||||
|
"Status": "Enabled" |
||||||
|
} |
||||||
|
] |
||||||
|
} |
||||||
|
EOF |
||||||
|
``` |
||||||
|
|
||||||
|
2. Enable bucket lifecycle configuration using `aws-cli`: |
||||||
|
|
||||||
|
```sh |
||||||
|
$ export AWS_ACCESS_KEY_ID="your-access-key" |
||||||
|
$ export AWS_SECRET_ACCESS_KEY="your-secret-key" |
||||||
|
$ aws s3api put-bucket-lifecycle-configuration --bucket your-bucket --endpoint-url http://minio-server-address:port --lifecycle-configuration file://bucket-lifecycle.json |
||||||
|
``` |
||||||
|
|
||||||
|
## Explore Further |
||||||
|
- [MinIO | Golang Client API Reference](https://docs.min.io/docs/golang-client-api-reference.html#SetBucketLifecycle) |
||||||
|
- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) |
@ -1,33 +0,0 @@ |
|||||||
/* |
|
||||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
|
||||||
* |
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
* you may not use this file except in compliance with the License. |
|
||||||
* You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package lifecycle |
|
||||||
|
|
||||||
// Action - policy action.
|
|
||||||
// Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html
|
|
||||||
// for more information about available actions.
|
|
||||||
type Action string |
|
||||||
|
|
||||||
const ( |
|
||||||
// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
|
|
||||||
PutBucketLifecycleAction = "s3:PutBucketLifecycle" |
|
||||||
|
|
||||||
// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
|
|
||||||
GetBucketLifecycleAction = "s3:GetBucketLifecycle" |
|
||||||
|
|
||||||
// DeleteBucketLifecycleAction - DeleteBucketLifecycleAction Rest API action.
|
|
||||||
DeleteBucketLifecycleAction = "s3:DeleteBucketLifecycle" |
|
||||||
) |
|
Loading…
Reference in new issue