From 61b08137b00c465c872c7b1327cbc38d906a01af Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Wed, 29 Mar 2017 21:25:53 +0530 Subject: [PATCH] Add `access` format support for Redis notification target (#3989) This change adds `access` format support for notifications to a Redis server, and it refactors `namespace` format support. In the case of `access` format, a list is used to store Minio operations in Redis. Each entry in the list is a JSON encoded list of two items - the first is the Minio server timestamp of the event, and the second is an object describing the operation that created/replaced the object in the server. In the case of `namespace` format, a hash is used. Entries in the hash may be updated or removed if objects in Minio are updated or deleted respectively. The field values in the Redis hash are JSON encoded. Also updates documentation on Redis notification target usage. Towards resolving #3928 --- cmd/notify-redis.go | 94 ++++++++++++++++++++++++----- docs/bucket/notifications/README.md | 56 +++++++++++++---- 2 files changed, 122 insertions(+), 28 deletions(-) diff --git a/cmd/notify-redis.go b/cmd/notify-redis.go index c25f0ba71..249eb446d 100644 --- a/cmd/notify-redis.go +++ b/cmd/notify-redis.go @@ -17,6 +17,7 @@ package cmd import ( + "encoding/json" "fmt" "io/ioutil" "time" @@ -25,6 +26,16 @@ import ( "github.com/garyburd/redigo/redis" ) +func makeRedisError(msg string, a ...interface{}) error { + s := fmt.Sprintf(msg, a...) + return fmt.Errorf("Redis Notifier Error: %s", s) +} + +var ( + rdNFormatError = makeRedisError(`"format" value is invalid - it must be one of "access" or "namespace".`) + rdNKeyError = makeRedisError("Key was not specified in the configuration.") +) + // redisNotify to send logs to Redis server type redisNotify struct { Enable bool `json:"enable"` @@ -38,14 +49,15 @@ func (r *redisNotify) Validate() error { if !r.Enable { return nil } - if r.Format != formatNamespace { - return fmt.Errorf( - "Redis Notifier Error: \"format\" must be \"%s\"", - formatNamespace) + if r.Format != formatNamespace && r.Format != formatAccess { + return rdNFormatError } if _, err := checkURL(r.Addr); err != nil { return err } + if r.Key == "" { + return rdNKeyError + } return nil } @@ -92,7 +104,25 @@ func dialRedis(rNotify redisNotify) (*redis.Pool, error) { // Check connection. _, err := rConn.Do("PING") if err != nil { - return nil, err + return nil, makeRedisError("Error connecting to server: %v", err) + } + + // Test that Key is of desired type + reply, err := redis.String(rConn.Do("TYPE", rNotify.Key)) + if err != nil { + return nil, makeRedisError("Error getting type of Key=%s: %v", + rNotify.Key, err) + } + if reply != "none" { + expectedType := "hash" + if rNotify.Format == formatAccess { + expectedType = "list" + } + if reply != expectedType { + return nil, makeRedisError( + "Key=%s has type %s, but we expect it to be a %s", + rNotify.Key, reply, expectedType) + } } // Return pool. @@ -137,17 +167,51 @@ func (r redisConn) Fire(entry *logrus.Entry) error { return nil } - // Match the event if its a delete request, attempt to delete the key - if eventMatch(entryStr, []string{"s3:ObjectRemoved:*"}) { - if _, err := rConn.Do("DEL", entry.Data["Key"]); err != nil { - return err + switch r.params.Format { + case formatNamespace: + // Match the event if its a delete request, attempt to delete the key + if eventMatch(entryStr, []string{"s3:ObjectRemoved:*"}) { + _, err := rConn.Do("HDEL", r.params.Key, entry.Data["Key"]) + if err != nil { + return makeRedisError("Error deleting entry: %v", + err) + } + return nil + } // else save this as new entry or update any existing ones. + + value, err := json.Marshal(map[string]interface{}{ + "Records": entry.Data["Records"], + }) + if err != nil { + return makeRedisError( + "Unable to encode event %v to JSON: %v", + entry.Data["Records"], err) + } + _, err = rConn.Do("HSET", r.params.Key, entry.Data["Key"], + value) + if err != nil { + return makeRedisError("Error updating hash entry: %v", + err) + } + case formatAccess: + // eventTime is taken from the first entry in the + // records. + events, ok := entry.Data["Records"].([]NotificationEvent) + if !ok { + return makeRedisError("unable to extract event time due to conversion error of entry.Data[\"Records\"]=%v", entry.Data["Records"]) + } + eventTime := events[0].EventTime + + listEntry := []interface{}{eventTime, entry.Data["Records"]} + jsonValue, err := json.Marshal(listEntry) + if err != nil { + return makeRedisError("JSON encoding error: %v", err) + } + _, err = rConn.Do("RPUSH", r.params.Key, jsonValue) + if err != nil { + return makeRedisError("Error appending to Redis list: %v", + err) } - return nil - } // else save this as new entry or update any existing ones. - if _, err := rConn.Do("SET", entry.Data["Key"], map[string]interface{}{ - "Records": entry.Data["Records"], - }); err != nil { - return err } return nil } diff --git a/docs/bucket/notifications/README.md b/docs/bucket/notifications/README.md index 441348704..d7172b85f 100644 --- a/docs/bucket/notifications/README.md +++ b/docs/bucket/notifications/README.md @@ -223,28 +223,53 @@ curl -XGET '127.0.0.1:9200/bucketevents/_search?pretty=1' ## Publish Minio events via Redis -Install Redis from [here](http://redis.io/download). +Install [Redis](http://redis.io/download) server. For illustrative purposes, we have set the database password as "yoursecret". + +This notification target supports two formats: _namespace_ and _access_. + +When the _namespace_ format is used, Minio synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in Minio. When objects are updated or deleted, the corresponding entry int he hash is updated or deleted respectively. + +When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and second item is a JSON object containing evnet data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format. + +The steps below show how to use this notification target in `namespace` and `access` format. ### Step 1: Add Redis endpoint to Minio -The default location of Minio server configuration file is ``~/.minio/config.json``. Update the Redis configuration block in ``config.json`` as follows: +The default location of Minio server configuration file is ``~/.minio/config.json``. The Redis configuration is located in the `redis` key under the `notify` top-level key. Create a configuration key-value pair here for your Redis instance. The key is a name for your Redis endpoint, and the value is a collection of key-value parameters described in the table below. + +| Parameter | Type | Description | +|:---|:---|:---| +| `enable` | _bool_ | (Required) Is this server endpoint configuration active/enabled? | +| `format` | _string_ | (Required) Either `namespace` or `access`. | +| `address` | _string_ | (Required) The Redis server's address. For example: `localhost:6379`. | +| `password` | _string_ | (Optional) The Redis server's password. | +| `key` | _string_ | (Required) The name of the redis key under which events are stored. A hash is used in case of `namespace` format and a list in case of `access` format.| + +An example of Redis configuration is as follows: + ```json "redis": { "1": { - "enable": true, - "address": "127.0.0.1:6379", - "password": "yoursecret", - "key": "bucketevents" + "enable": true, + "address": "127.0.0.1:6379", + "password": "yoursecret", + "key": "bucketevents" } } ``` -Restart Minio server to reflect config changes. ``bucketevents`` is the key used by Redis in this example. +After updating the configuration file, restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs:us-east-1:1:redis` at start-up if there were no errors. + +Note that, you can add as many Redis server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Redis instance and an object of per-server configuration parameters. ### Step 2: Enable bucket notification using Minio client -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from ``images`` bucket on ``myminio`` server. Here ARN value is ``arn:minio:sqs:us-east-1:1:redis``. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. +We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new key is added or an existing key is updated in the Redis hash configured above. When an existing object is deleted, the corresponding key is deleted from the Redis hash. Thus, the rows in the Redis hash, reflect the `.jpg` objects in the `images` bucket. + +To configure this bucket notification, we need the ARN printed by Minio in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). + +With the `mc` tool, the configuration is very simple to add. Let us say that the Minio server is aliased as `myminio` in our mc configuration. Execute the following: ``` mc mb myminio/images @@ -255,10 +280,12 @@ arn:minio:sqs:us-east-1:1:redis s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: su ### Step 3: Test on Redis -Redis comes with a handy command line interface ``redis-cli`` to print all notifications on the console. +Start the `redis-cli` Redis client program to inspect the contents in Redis. Run the `monitor` Redis command. This prints each operation performed on Redis as it occurs. ``` redis-cli -a yoursecret +127.0.0.1:6379> monitor +OK ``` Open another terminal and upload a JPEG image into ``images`` bucket. @@ -267,16 +294,19 @@ Open another terminal and upload a JPEG image into ``images`` bucket. mc cp myphoto.jpg myminio/images ``` -``redis-cli`` prints event notification to the console. +In the previous terminal, you will now see the operation that Minio performs on Redis: ``` -redis-cli -a yoursecret 127.0.0.1:6379> monitor OK -1474321638.556108 [0 127.0.0.1:40190] "AUTH" "yoursecret" -1474321638.556477 [0 127.0.0.1:40190] "RPUSH" "bucketevents" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2016-09-19T21:47:18.555Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"[::1]:39250\"},\"responseElements\":{},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":23745,\"sequencer\":\"1475D7B80ECBD853\"}}}],\"level\":\"info\",\"msg\":\"\",\"time\":\"2016-09-19T14:47:18-07:00\"}\n" +1490686879.650649 [0 172.17.0.1:44710] "PING" +1490686879.651061 [0 172.17.0.1:44710] "HSET" "minio_events" "images/myphoto.jpg" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"minio:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2017-03-28T07:41:19Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"127.0.0.1:52234\"},\"responseElements\":{\"x-amz-request-id\":\"14AFFBD1ACE5F632\",\"x-minio-origin-endpoint\":\"http://192.168.86.115:9000\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":2586,\"eTag\":\"5d284463f9da279f060f0ea4d11af098\",\"sequencer\":\"14AFFBD1ACE5F632\"}},\"source\":{\"host\":\"127.0.0.1\",\"port\":\"52234\",\"userAgent\":\"Minio (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z\"}}]}" ``` +Here we see that Minio performed `HSET` on `minio_events` key. + +In case, `access` format was used, then `minio_events` would be a list, and the Minio server would have performed an `RPUSH` to append to the list. A consumer of this list would ideally use `BLPOP` to remove list items from the left-end of the list. + ## Publish Minio events via NATS