diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 68d33a41d..1bd0014f5 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -222,21 +222,26 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA versionPurgeStatus = Complete } } + var eventName = event.ObjectReplicationComplete if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed { - objInfo := ObjectInfo{ - Name: dobj.ObjectName, - DeleteMarker: dobj.DeleteMarker, - VersionID: versionID, - ReplicationStatus: replication.StatusType(dobj.DeleteMarkerReplicationStatus), - } - eventArg := &eventArgs{ - BucketName: bucket, - Object: objInfo, - Host: "Internal: [Replication]", - EventName: event.ObjectReplicationFailed, - } - sendEvent(*eventArg) + eventName = event.ObjectReplicationFailed + } + objInfo := ObjectInfo{ + Name: dobj.ObjectName, + DeleteMarker: dobj.DeleteMarker, + VersionID: versionID, + ReplicationStatus: replication.StatusType(dobj.DeleteMarkerReplicationStatus), + VersionPurgeStatus: versionPurgeStatus, + } + + eventArg := &eventArgs{ + BucketName: bucket, + Object: objInfo, + Host: "Internal: [Replication]", + EventName: eventName, } + sendEvent(*eventArg) + // Update metadata on the delete marker or purge permanent delete if replication success. if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{ VersionID: versionID, diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index e88688140..efdfa22db 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -415,23 +415,26 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin if err != nil { return objInfo, toObjectErr(err, bucket, object) } - + objInfo = fi.ToObjectInfo(bucket, object) + if objInfo.TransitionStatus == lifecycle.TransitionComplete { + // overlay storage class for transitioned objects with transition tier SC Label + if sc := transitionSC(ctx, bucket); sc != "" { + objInfo.StorageClass = sc + } + } + if !fi.VersionPurgeStatus.Empty() { + // Make sure to return object info to provide extra information. + return objInfo, toObjectErr(errMethodNotAllowed, bucket, object) + } if fi.Deleted { - objInfo = fi.ToObjectInfo(bucket, object) if opts.VersionID == "" || opts.DeleteMarker { return objInfo, toObjectErr(errFileNotFound, bucket, object) } // Make sure to return object info to provide extra information. return objInfo, toObjectErr(errMethodNotAllowed, bucket, object) } - oi := fi.ToObjectInfo(bucket, object) - if oi.TransitionStatus == lifecycle.TransitionComplete { - // overlay storage class for transitioned objects with transition tier SC Label - if sc := transitionSC(ctx, bucket); sc != "" { - oi.StorageClass = sc - } - } - return oi, nil + + return objInfo, nil } func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) { diff --git a/cmd/http/headers.go b/cmd/http/headers.go index 68a2ab52d..5c4cea2ef 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -144,6 +144,11 @@ const ( // Header indicates if the delete marker version needs to be purged. MinIOSourceDeleteMarkerDelete = "x-minio-source-deletemarker-delete" + + // Header indicates permanent delete replication status. + MinIODeleteReplicationStatus = "X-Minio-Replication-Delete-Status" + // Header indicates delete-marker replication status. + MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status" ) // Common http query params S3 API diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index 39e31f18e..2481e8716 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -151,7 +151,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro }, }, nil } - return getFileInfoVersions(e.metadata, bucket, e.name, false) + return getFileInfoVersions(e.metadata, bucket, e.name) } // metaCacheEntries is a slice of metacache entries. diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index c5c944bb3..979099f39 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -575,6 +575,13 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { if globalBucketVersioningSys.Enabled(bucket) { + if !objInfo.VersionPurgeStatus.Empty() { + // Shows the replication status of a permanent delete of a version + w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(objInfo.VersionPurgeStatus)} + } + if !objInfo.ReplicationStatus.Empty() && objInfo.DeleteMarker { + w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(objInfo.ReplicationStatus)} + } // Versioning enabled quite possibly object is deleted might be delete-marker // if present set the headers, no idea why AWS S3 sets these headers. if objInfo.VersionID != "" && objInfo.DeleteMarker { diff --git a/cmd/xl-storage-format-utils.go b/cmd/xl-storage-format-utils.go index 98ebffb95..c24a78b91 100644 --- a/cmd/xl-storage-format-utils.go +++ b/cmd/xl-storage-format-utils.go @@ -34,13 +34,13 @@ func (v versionsSorter) Less(i, j int) bool { return v[i].ModTime.After(v[j].ModTime) } -func getFileInfoVersions(xlMetaBuf []byte, volume, path string, showPendingDeletes bool) (FileInfoVersions, error) { +func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) { if isXL2V1Format(xlMetaBuf) { var xlMeta xlMetaV2 if err := xlMeta.Load(xlMetaBuf); err != nil { return FileInfoVersions{}, err } - versions, latestModTime, err := xlMeta.ListVersions(volume, path, showPendingDeletes) + versions, latestModTime, err := xlMeta.ListVersions(volume, path) if err != nil { return FileInfoVersions{}, err } diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 0d51ad70b..f3d6a5ca3 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -598,7 +598,7 @@ func (z xlMetaV2) TotalSize() int64 { // versions returns error for unexpected entries. // showPendingDeletes is set to true if ListVersions needs to list objects marked deleted // but waiting to be replicated -func (z xlMetaV2) ListVersions(volume, path string, showPendingDeletes bool) (versions []FileInfo, modTime time.Time, err error) { +func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTime time.Time, err error) { var latestModTime time.Time var latestVersionID string for _, version := range z.Versions { @@ -611,9 +611,6 @@ func (z xlMetaV2) ListVersions(volume, path string, showPendingDeletes bool) (ve fi, err = version.ObjectV2.ToFileInfo(volume, path) case DeleteType: fi, err = version.DeleteMarker.ToFileInfo(volume, path) - if !fi.VersionPurgeStatus.Empty() && !showPendingDeletes { - continue - } case LegacyType: fi, err = version.ObjectV1.ToFileInfo(volume, path) } diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 5bdb317f4..d5fbe01ec 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -380,7 +380,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac // Remove filename which is the meta file. item.transformMetaDir() - fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), true) + fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath()) if err != nil { return 0, errSkipFile } @@ -898,7 +898,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st continue } - fiv, err = getFileInfoVersions(xlMetaBuf, volume, walkResult.entry, false) + fiv, err = getFileInfoVersions(xlMetaBuf, volume, walkResult.entry) if err != nil { continue } diff --git a/docs/bucket/replication/DELETE_bucket-replication.png b/docs/bucket/replication/DELETE_bucket-replication.png deleted file mode 100644 index 19cdebe6d..000000000 Binary files a/docs/bucket/replication/DELETE_bucket-replication.png and /dev/null differ diff --git a/docs/bucket/replication/DELETE_bucket_replication.png b/docs/bucket/replication/DELETE_bucket_replication.png index 64b17f3bc..19cdebe6d 100644 Binary files a/docs/bucket/replication/DELETE_bucket_replication.png and b/docs/bucket/replication/DELETE_bucket_replication.png differ diff --git a/docs/bucket/replication/README.md b/docs/bucket/replication/README.md index a550b731b..72edffed3 100644 --- a/docs/bucket/replication/README.md +++ b/docs/bucket/replication/README.md @@ -11,6 +11,8 @@ To replicate objects in a bucket to a destination bucket on a target site either - Active-Active replication ## How to use? +Ensure that versioning is enabled on the source and target buckets with `mc version` command. If object locking is required, the buckets should have been created with `mc mb --with-lock` + Create a replication target on the source cluster as shown below: ``` @@ -115,7 +117,7 @@ The replication configuration can now be added to the source bucket by applying The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely. -When object locking is used in conjunction with replication, both source and destination buckets needs to have object locking enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption. +When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://docs.min.io/docs/minio-bucket-object-lock-guide.html) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption. Replication status can be seen in the metadata on the source and destination objects. On the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETE` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk crawl cycle. @@ -128,9 +130,9 @@ It is recommended that replication be run in a system with atleast two CPU's ava ![head](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/HEAD_bucket_replication.png) ## MinIO Extension -Delete marker replication is allowed in [AWS V1 Configuration](https://aws.amazon.com/blogs/storage/managing-delete-marker-replication-in-amazon-s3/) but not in V2 configuration. The MinIO implementation above is based on V2 configuration, however it has been extended to allow both DeleteMarker replication and replication of versioned deletes with the `DeleteMarkerReplication` and `DeleteReplication` fields in the replication configuration above. By default, this is set to `Disabled` unless the user specifies it while adding a replication rule. +Delete marker replication is allowed in [AWS V1 Configuration](https://aws.amazon.com/blogs/storage/managing-delete-marker-replication-in-amazon-s3/) but not in V2 configuration. The MinIO implementation above is based on V2 configuration, however it has been extended to allow both DeleteMarker replication and replication of versioned deletes with the `DeleteMarkerReplication` and `DeleteReplication` fields in the replication configuration above. By default, this is set to `Disabled` unless the user specifies it while adding a replication rule. -When an object is deleted from the source bucket, the corresponding replica version will be marked deleted if delete marker replication is enabled in the replication configuration.Replication of deletes that specify a version id (a.k.a hard deletes) can be enabled by setting the `DeleteReplication` status to enabled in the replication configuration. This is a MinIO specific extension that can be enabled using the `mc replicate add` or `mc replicate edit` command with the --replicate "delete" flag. +When an object is deleted from the source bucket, the corresponding replica version will be marked deleted if delete marker replication is enabled in the replication configuration. Replication of deletes that specify a version id (a.k.a hard deletes) can be enabled by setting the `DeleteReplication` status to enabled in the replication configuration. This is a MinIO specific extension that can be enabled using the `mc replicate add` or `mc replicate edit` command with the --replicate "delete" flag. Note that due to this extension behavior, AWS SDK's may not support the extension functionality pertaining to replicating versioned deletes. @@ -143,8 +145,15 @@ Replication configuration applied successfully to myminio/srcbucket. ``` Note that both source and target instance need to be upgraded to latest release to take advantage of Delete marker replication. +Status of delete marker replication can be viewed by doing a GET/HEAD on the object version - it will return a `X-Minio-Replication-DeleteMarker-Status` header and http response code of `405`. In the case of permanent deletes, if the delete replication is pending or failed to propagate to the target cluster, GET/HEAD will return additional `X-Minio-Replication-Delete-Status` header and a http response code of `405`. + ![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/DELETE_bucket_replication.png) +The status of replication can be monitored by configuring event notifications on the source and target buckets using `mc event add`.On the source side, the `s3:PutObject`, `s3:Replication:OperationCompletedReplication` and `s3:Replication:OperationFailedReplication` events show the status of replication in the `X-Amz-Replication-Status` metadata. + +On the target bucket, `s3:PutObject` event shows `X-Amz-Replication-Status` status of `REPLICA` in the metadata. Additional metrics to monitor backlog state for the purpose of bandwidth management and resource allocation are +an upcoming feature. + ## Explore Further - [MinIO Bucket Versioning Implementation](https://docs.minio.io/docs/minio-bucket-versioning-guide.html) - [MinIO Client Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide.html)