Enable event persistence in MySQL and PostgreSQL (#7629)

cmd/admin-handlers_test.go

@@ -139,7 +139,9 @@ var (
         "port": "",
         "user": "",
         "password": "",
-        "database": ""
+        "database": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "nats": {
@@ -185,7 +187,9 @@ var (
         "port": "",
         "user": "",
         "password": "",
-        "database": ""
+        "database": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "redis": {

cmd/config-current.go

@@ -347,7 +347,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewMySQLTarget(k, v)
+		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("mysql(%s): %s", k, err.Error())
 		}
@@ -380,7 +380,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewPostgreSQLTarget(k, v)
+		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
 		}
@@ -696,7 +696,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.MySQL {
 		if args.Enable {
-			newTarget, err := target.NewMySQLTarget(id, args)
+			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -738,7 +738,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.PostgreSQL {
 		if args.Enable {
-			newTarget, err := target.NewPostgreSQLTarget(id, args)
+			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -197,7 +197,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
 
 		// Test 16 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
@@ -206,7 +206,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "", "queueDir": "", "queueLimit": 0} }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},

docs/bucket/notifications/README.md

@@ -789,11 +789,15 @@ An example of PostgreSQL configuration is as follows:
         "port": "5432",
         "user": "postgres",
         "password": "password",
-        "database": "minio_events"
+        "database": "minio_events",
+        "queueDir": "",
+        "queueLimit": 0
     }
 }
 ```
 
+MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended.
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
 
@@ -884,21 +888,27 @@ The MinIO server configuration file is stored on the backend in json format. The
 
 An example of MySQL configuration is as follows:
 
-```
+```json
 "mysql": {
-        "1": {
+       "1": {
-                "enable": true,
+           "enable": true,
-                "dsnString": "",
+           "dsnString": "",
-                "table": "minio_images",
+           "format": "namespace",
-                "host": "172.17.0.1",
+           "table": "minio_images",
-                "port": "3306",
+           "host": "172.17.0.1",
-                "user": "root",
+           "port": "3306",
-                "password": "password",
+           "user": "root",
-                "database": "miniodb"
+           "password": "password",
-        }
+           "database": "miniodb",
+           "queueDir": "",
+           "queueLimit": 0
+       }
 }
 ```
 
+
+MinIO supports persistent event store. The persistent store will backup events when the MySQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
 
 ```sh

docs/config/config.sample.json

@@ -104,7 +104,9 @@
 				"port": "",
 				"user": "",
 				"password": "",
-				"database": ""
+				"database": "",
+                               "queueDir": "",
+                               "queueLimit": 0
 			}
 		},
 		"nats": {
@@ -150,7 +152,9 @@
 				"port": "",
 				"user": "",
 				"password": "",
-				"database": ""
+				"database": "",
+                               "queueDir": "",
+                               "queueLimit": 0
 			}
 		},
 		"redis": {

pkg/event/target/mysql.go

@@ -56,8 +56,11 @@ package target
 import (
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -79,15 +82,17 @@ const (
 
 // MySQLArgs - MySQL target arguments.
 type MySQLArgs struct {
-	Enable   bool     `json:"enable"`
+	Enable     bool     `json:"enable"`
-	Format   string   `json:"format"`
+	Format     string   `json:"format"`
-	DSN      string   `json:"dsnString"`
+	DSN        string   `json:"dsnString"`
-	Table    string   `json:"table"`
+	Table      string   `json:"table"`
-	Host     xnet.URL `json:"host"`
+	Host       xnet.URL `json:"host"`
-	Port     string   `json:"port"`
+	Port       string   `json:"port"`
-	User     string   `json:"user"`
+	User       string   `json:"user"`
-	Password string   `json:"password"`
+	Password   string   `json:"password"`
-	Database string   `json:"database"`
+	Database   string   `json:"database"`
+	QueueDir   string   `json:"queueDir"`
+	QueueLimit uint64   `json:"queueLimit"`
 }
 
 // Validate MySQLArgs fields
@@ -123,6 +128,16 @@ func (m MySQLArgs) Validate() error {
 			return fmt.Errorf("database unspecified")
 		}
 	}
+
+	if m.QueueDir != "" {
+		if !filepath.IsAbs(m.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+	if m.QueueLimit > 10000 {
+		return errors.New("queueLimit should not exceed 10000")
+	}
+
 	return nil
 }
 
@@ -134,6 +149,8 @@ type MySQLTarget struct {
 	deleteStmt *sql.Stmt
 	insertStmt *sql.Stmt
 	db         *sql.DB
+	store      Store
+	firstPing  bool
 }
 
 // ID - returns target ID.
@@ -141,11 +158,21 @@ func (target *MySQLTarget) ID() event.TargetID {
 	return target.id
 }
 
-// Save - Sends event directly without persisting.
+// Save - saves the events to the store which will be replayed when the SQL connection is active.
 func (target *MySQLTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		return target.store.Put(eventData)
+	}
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
 	return target.send(eventData)
 }
 
+// send - sends an event to the mysql.
 func (target *MySQLTarget) send(eventData event.Event) error {
 	if target.args.Format == event.NamespaceFormat {
 		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
@@ -164,6 +191,7 @@ func (target *MySQLTarget) send(eventData event.Event) error {
 
 			_, err = target.updateStmt.Exec(key, data)
 		}
+
 		return err
 	}
 
@@ -179,15 +207,51 @@ func (target *MySQLTarget) send(eventData event.Event) error {
 		}
 
 		_, err = target.insertStmt.Exec(eventTime, data)
+
 		return err
 	}
 
 	return nil
 }
 
-// Send - interface compatible method does no-op.
+// Send - reads an event from store and sends it to MySQL.
 func (target *MySQLTarget) Send(eventKey string) error {
-	return nil
+
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
+
+	if !target.firstPing {
+		if err := target.executeStmts(); err != nil {
+			if IsConnErr(err) {
+				return errNotConnected
+			}
+			return err
+		}
+	}
+
+	eventData, eErr := target.store.Get(eventKey)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(eventKey)
 }
 
 // Close - closes underneath connections to MySQL database.
@@ -210,8 +274,45 @@ func (target *MySQLTarget) Close() error {
 	return target.db.Close()
 }
 
+// Executes the table creation statements.
+func (target *MySQLTarget) executeStmts() error {
+
+	_, err := target.db.Exec(fmt.Sprintf(mysqlTableExists, target.args.Table))
+	if err != nil {
+		createStmt := mysqlCreateNamespaceTable
+		if target.args.Format == event.AccessFormat {
+			createStmt = mysqlCreateAccessTable
+		}
+
+		if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil {
+			return dbErr
+		}
+	}
+
+	switch target.args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(mysqlUpdateRow, target.args.Table)); err != nil {
+			return err
+		}
+		// delete statement
+		if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(mysqlDeleteRow, target.args.Table)); err != nil {
+			return err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(mysqlInsertRow, target.args.Table)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
 // NewMySQLTarget - creates new MySQL target.
-func NewMySQLTarget(id string, args MySQLArgs) (*MySQLTarget, error) {
+func NewMySQLTarget(id string, args MySQLArgs, doneCh <-chan struct{}) (*MySQLTarget, error) {
+	var firstPing bool
 	if args.DSN == "" {
 		config := mysql.Config{
 			User:                 args.User,
@@ -230,45 +331,42 @@ func NewMySQLTarget(id string, args MySQLArgs) (*MySQLTarget, error) {
 		return nil, err
 	}
 
-	if err = db.Ping(); err != nil {
+	var store Store
-		return nil, err
-	}
 
-	if _, err = db.Exec(fmt.Sprintf(mysqlTableExists, args.Table)); err != nil {
+	if args.QueueDir != "" {
-		createStmt := mysqlCreateNamespaceTable
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-mysql-"+id)
-		if args.Format == event.AccessFormat {
+		store = NewQueueStore(queueDir, args.QueueLimit)
-			createStmt = mysqlCreateAccessTable
+		if oErr := store.Open(); oErr != nil {
+			return nil, oErr
 		}
+	}
 
-		if _, err = db.Exec(fmt.Sprintf(createStmt, args.Table)); err != nil {
+	target := &MySQLTarget{
-			return nil, err
+		id:        event.TargetID{ID: id, Name: "mysql"},
-		}
+		args:      args,
+		db:        db,
+		store:     store,
+		firstPing: firstPing,
 	}
 
-	var updateStmt, deleteStmt, insertStmt *sql.Stmt
+	err = target.db.Ping()
-	switch args.Format {
+	if err != nil {
-	case event.NamespaceFormat:
+		if target.store == nil || !IsConnRefusedErr(err) {
-		// insert or update statement
-		if updateStmt, err = db.Prepare(fmt.Sprintf(mysqlUpdateRow, args.Table)); err != nil {
-			return nil, err
-		}
-		// delete statement
-		if deleteStmt, err = db.Prepare(fmt.Sprintf(mysqlDeleteRow, args.Table)); err != nil {
 			return nil, err
 		}
-	case event.AccessFormat:
+	} else {
-		// insert statement
+		if err = target.executeStmts(); err != nil {
-		if insertStmt, err = db.Prepare(fmt.Sprintf(mysqlInsertRow, args.Table)); err != nil {
 			return nil, err
 		}
+		target.firstPing = true
+	}
+
+	if target.store != nil {
+		// Replays the events from the store.
+		eventKeyCh := replayEvents(target.store, doneCh)
+		// Start replaying events from the store.
+		go sendEvents(target, eventKeyCh, doneCh)
 	}
 
-	return &MySQLTarget{
+	return target, nil
-		id:         event.TargetID{ID: id, Name: "mysql"},
-		args:       args,
-		updateStmt: updateStmt,
-		deleteStmt: deleteStmt,
-		insertStmt: insertStmt,
-		db:         db,
-	}, nil
 }

pkg/event/target/postgresql.go

@@ -56,8 +56,11 @@ package target
 import (
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -89,6 +92,8 @@ type PostgreSQLArgs struct {
 	User             string    `json:"user"`     // default: user running minio
 	Password         string    `json:"password"` // default: no password
 	Database         string    `json:"database"` // default: same as user
+	QueueDir         string    `json:"queueDir"`
+	QueueLimit       uint64    `json:"queueLimit"`
 }
 
 // Validate PostgreSQLArgs fields
@@ -122,6 +127,15 @@ func (p PostgreSQLArgs) Validate() error {
 		}
 	}
 
+	if p.QueueDir != "" {
+		if !filepath.IsAbs(p.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+	if p.QueueLimit > 10000 {
+		return errors.New("queueLimit should not exceed 10000")
+	}
+
 	return nil
 }
 
@@ -133,6 +147,8 @@ type PostgreSQLTarget struct {
 	deleteStmt *sql.Stmt
 	insertStmt *sql.Stmt
 	db         *sql.DB
+	store      Store
+	firstPing  bool
 }
 
 // ID - returns target ID.
@@ -140,11 +156,26 @@ func (target *PostgreSQLTarget) ID() event.TargetID {
 	return target.id
 }
 
-// Save - Sends event directly without persisting.
+// Save - saves the events to the store if questore is configured, which will be replayed when the PostgreSQL connection is active.
 func (target *PostgreSQLTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		return target.store.Put(eventData)
+	}
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
 	return target.send(eventData)
 }
 
+// IsConnErr - To detect a connection error.
+func IsConnErr(err error) bool {
+	return IsConnRefusedErr(err) || err.Error() == "sql: database is closed" || err.Error() == "sql: statement is closed" || err.Error() == "invalid connection"
+}
+
+// send - sends an event to the PostgreSQL.
 func (target *PostgreSQLTarget) send(eventData event.Event) error {
 	if target.args.Format == event.NamespaceFormat {
 		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
@@ -177,16 +208,52 @@ func (target *PostgreSQLTarget) send(eventData event.Event) error {
 			return err
 		}
 
-		_, err = target.insertStmt.Exec(eventTime, data)
+		if _, err = target.insertStmt.Exec(eventTime, data); err != nil {
-		return err
+			return err
+		}
 	}
 
 	return nil
 }
 
-// Send - interface compatible method does no-op.
+// Send - reads an event from store and sends it to PostgreSQL.
 func (target *PostgreSQLTarget) Send(eventKey string) error {
-	return nil
+
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
+
+	if !target.firstPing {
+		if err := target.executeStmts(); err != nil {
+			if IsConnErr(err) {
+				return errNotConnected
+			}
+			return err
+		}
+	}
+
+	eventData, eErr := target.store.Get(eventKey)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if IsConnErr(err) {
+			return errNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(eventKey)
 }
 
 // Close - closes underneath connections to PostgreSQL database.
@@ -209,8 +276,45 @@ func (target *PostgreSQLTarget) Close() error {
 	return target.db.Close()
 }
 
+// Executes the table creation statements.
+func (target *PostgreSQLTarget) executeStmts() error {
+
+	_, err := target.db.Exec(fmt.Sprintf(psqlTableExists, target.args.Table))
+	if err != nil {
+		createStmt := psqlCreateNamespaceTable
+		if target.args.Format == event.AccessFormat {
+			createStmt = psqlCreateAccessTable
+		}
+
+		if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil {
+			return dbErr
+		}
+	}
+
+	switch target.args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(psqlUpdateRow, target.args.Table)); err != nil {
+			return err
+		}
+		// delete statement
+		if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(psqlDeleteRow, target.args.Table)); err != nil {
+			return err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(psqlInsertRow, target.args.Table)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // NewPostgreSQLTarget - creates new PostgreSQL target.
-func NewPostgreSQLTarget(id string, args PostgreSQLArgs) (*PostgreSQLTarget, error) {
+func NewPostgreSQLTarget(id string, args PostgreSQLArgs, doneCh <-chan struct{}) (*PostgreSQLTarget, error) {
+	var firstPing bool
+
 	params := []string{args.ConnectionString}
 	if !args.Host.IsEmpty() {
 		params = append(params, "host="+args.Host.String())
@@ -234,45 +338,42 @@ func NewPostgreSQLTarget(id string, args PostgreSQLArgs) (*PostgreSQLTarget, err
 		return nil, err
 	}
 
-	if err = db.Ping(); err != nil {
+	var store Store
-		return nil, err
-	}
 
-	if _, err = db.Exec(fmt.Sprintf(psqlTableExists, args.Table)); err != nil {
+	if args.QueueDir != "" {
-		createStmt := psqlCreateNamespaceTable
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-postgresql-"+id)
-		if args.Format == event.AccessFormat {
+		store = NewQueueStore(queueDir, args.QueueLimit)
-			createStmt = psqlCreateAccessTable
+		if oErr := store.Open(); oErr != nil {
+			return nil, oErr
 		}
+	}
 
-		if _, err = db.Exec(fmt.Sprintf(createStmt, args.Table)); err != nil {
+	target := &PostgreSQLTarget{
-			return nil, err
+		id:        event.TargetID{ID: id, Name: "postgresql"},
-		}
+		args:      args,
+		db:        db,
+		store:     store,
+		firstPing: firstPing,
 	}
 
-	var updateStmt, deleteStmt, insertStmt *sql.Stmt
+	err = target.db.Ping()
-	switch args.Format {
+	if err != nil {
-	case event.NamespaceFormat:
+		if target.store == nil || !IsConnRefusedErr(err) {
-		// insert or update statement
-		if updateStmt, err = db.Prepare(fmt.Sprintf(psqlUpdateRow, args.Table)); err != nil {
-			return nil, err
-		}
-		// delete statement
-		if deleteStmt, err = db.Prepare(fmt.Sprintf(psqlDeleteRow, args.Table)); err != nil {
 			return nil, err
 		}
-	case event.AccessFormat:
+	} else {
-		// insert statement
+		if err = target.executeStmts(); err != nil {
-		if insertStmt, err = db.Prepare(fmt.Sprintf(psqlInsertRow, args.Table)); err != nil {
 			return nil, err
 		}
+		target.firstPing = true
+	}
+
+	if target.store != nil {
+		// Replays the events from the store.
+		eventKeyCh := replayEvents(target.store, doneCh)
+		// Start replaying events from the store.
+		go sendEvents(target, eventKeyCh, doneCh)
 	}
 
-	return &PostgreSQLTarget{
+	return target, nil
-		id:         event.TargetID{ID: id, Name: "postgresql"},
-		args:       args,
-		updateStmt: updateStmt,
-		deleteStmt: deleteStmt,
-		insertStmt: insertStmt,
-		db:         db,
-	}, nil
 }