From b880796aef3394ecf00e43fdfcaff5145eb348b5 Mon Sep 17 00:00:00 2001 From: Praveen raj Mani Date: Fri, 25 Sep 2020 10:50:30 +0530 Subject: [PATCH] Set the maximum open connections limit in PG and MySQL target configs (#10558) As the bulk/recursive delete will require multiple connections to open at an instance, The default open connections limit will be reached which results in the following error ```FATAL: sorry, too many clients already``` By setting the open connections to a reasonable value - `2`, We ensure that the max open connections will not be exhausted and lie under bounds. The queries are simple inserts/updates/deletes which is operational and sufficient with the the maximum open connection limit is 2. Fixes #10553 Allow user configuration for MaxOpenConnections --- cmd/config/notify/help.go | 12 +++++ cmd/config/notify/legacy.go | 8 +++ cmd/config/notify/parse.go | 55 ++++++++++++++----- docs/bucket/notifications/README.md | 64 ++++++++++++---------- go.mod | 2 +- go.sum | 2 + pkg/event/target/mysql.go | 82 +++++++++++++++++------------ pkg/event/target/postgresql.go | 82 +++++++++++++++++------------ ruleguard.rules.go | 1 - 9 files changed, 201 insertions(+), 107 deletions(-) diff --git a/cmd/config/notify/help.go b/cmd/config/notify/help.go index 948db538d..c2e56bb64 100644 --- a/cmd/config/notify/help.go +++ b/cmd/config/notify/help.go @@ -340,6 +340,12 @@ var ( Optional: true, Type: "sentence", }, + config.HelpKV{ + Key: target.PostgresMaxOpenConnections, + Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.", + Optional: true, + Type: "number", + }, } HelpMySQL = config.HelpKVS{ @@ -377,6 +383,12 @@ var ( Optional: true, Type: "sentence", }, + config.HelpKV{ + Key: target.MySQLMaxOpenConnections, + Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.", + Optional: true, + Type: "number", + }, } HelpNATS = config.HelpKVS{ diff --git a/cmd/config/notify/legacy.go b/cmd/config/notify/legacy.go index 269ed5ca6..89e6c095a 100644 --- a/cmd/config/notify/legacy.go +++ b/cmd/config/notify/legacy.go @@ -357,6 +357,10 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg Key: target.PostgresQueueLimit, Value: strconv.Itoa(int(cfg.QueueLimit)), }, + config.KV{ + Key: target.PostgresMaxOpenConnections, + Value: strconv.Itoa(cfg.MaxOpenConnections), + }, } return nil @@ -554,6 +558,10 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error Key: target.MySQLQueueLimit, Value: strconv.Itoa(int(cfg.QueueLimit)), }, + config.KV{ + Key: target.MySQLMaxOpenConnections, + Value: strconv.Itoa(cfg.MaxOpenConnections), + }, } return nil diff --git a/cmd/config/notify/parse.go b/cmd/config/notify/parse.go index 25764af38..88adbbf2c 100644 --- a/cmd/config/notify/parse.go +++ b/cmd/config/notify/parse.go @@ -807,6 +807,10 @@ var ( Key: target.MySQLQueueLimit, Value: "0", }, + config.KV{ + Key: target.MySQLMaxOpenConnections, + Value: "2", + }, } ) @@ -855,13 +859,25 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs if k != config.Default { queueDirEnv = queueDirEnv + config.Default + k } + + maxOpenConnectionsEnv := target.EnvMySQLMaxOpenConnections + if k != config.Default { + maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k + } + + maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.MySQLMaxOpenConnections))) + if cErr != nil { + return nil, cErr + } + mysqlArgs := target.MySQLArgs{ - Enable: enabled, - Format: env.Get(formatEnv, kv.Get(target.MySQLFormat)), - DSN: env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)), - Table: env.Get(tableEnv, kv.Get(target.MySQLTable)), - QueueDir: env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)), - QueueLimit: queueLimit, + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.MySQLFormat)), + DSN: env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)), + Table: env.Get(tableEnv, kv.Get(target.MySQLTable)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)), + QueueLimit: queueLimit, + MaxOpenConnections: maxOpenConnections, } if err = mysqlArgs.Validate(); err != nil { return nil, err @@ -1235,6 +1251,10 @@ var ( Key: target.PostgresQueueLimit, Value: "0", }, + config.KV{ + Key: target.PostgresMaxOpenConnections, + Value: "2", + }, } ) @@ -1285,13 +1305,24 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos queueDirEnv = queueDirEnv + config.Default + k } + maxOpenConnectionsEnv := target.EnvPostgresMaxOpenConnections + if k != config.Default { + maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k + } + + maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.PostgresMaxOpenConnections))) + if cErr != nil { + return nil, cErr + } + psqlArgs := target.PostgreSQLArgs{ - Enable: enabled, - Format: env.Get(formatEnv, kv.Get(target.PostgresFormat)), - ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)), - Table: env.Get(tableEnv, kv.Get(target.PostgresTable)), - QueueDir: env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)), - QueueLimit: uint64(queueLimit), + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.PostgresFormat)), + ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)), + Table: env.Get(tableEnv, kv.Get(target.PostgresTable)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)), + QueueLimit: uint64(queueLimit), + MaxOpenConnections: maxOpenConnections, } if err = psqlArgs.Validate(); err != nil { return nil, err diff --git a/docs/bucket/notifications/README.md b/docs/bucket/notifications/README.md index 68919fdc6..f61ff08c4 100644 --- a/docs/bucket/notifications/README.md +++ b/docs/bucket/notifications/README.md @@ -856,12 +856,13 @@ KEY: notify_postgres[:name] publish bucket notifications to Postgres databases ARGS: -connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" -table* (string) DB table name to store/update events, table is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting +connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" +table* (string) DB table name to store/update events, table is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +max_open_connections (number) maximum number of open connections to the database, defaults to '2' +comment (sentence) optionally add a comment to this setting ``` or environment variables @@ -870,15 +871,19 @@ KEY: notify_postgres[:name] publish bucket notifications to Postgres databases ARGS: -MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off' -MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" -MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created -MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting +MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off' +MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" +MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created +MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting +MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2' ``` +> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of +> open connections to the database. This setting is generally NOT recommended as the behaviour may be inconsistent during recursive deletes in `namespace` format. + MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended. @@ -985,12 +990,13 @@ KEY: notify_mysql[:name] publish bucket notifications to MySQL databases. When multiple MySQL server endpoints are needed, a user specified "name" can be added for each configuration, (e.g."notify_mysql:myinstance"). ARGS: -dsn_string* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" -table* (string) DB table name to store/update events, table is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting +dsn_string* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" +table* (string) DB table name to store/update events, table is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +max_open_connections (number) maximum number of open connections to the database, defaults to '2' +comment (sentence) optionally add a comment to this setting ``` or environment variables @@ -999,14 +1005,18 @@ KEY: notify_mysql[:name] publish bucket notifications to MySQL databases ARGS: -MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off' -MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" -MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created -MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting -``` +MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off' +MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" +MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created +MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2' +MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting +``` + +> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of +> open connections to the database. This setting is generally NOT recommended as the behaviour may be inconsistent during recursive deletes in `namespace` format. `dsn_string` is required and is of form `":@tcp(:)/"` diff --git a/go.mod b/go.mod index e700b1010..3feff9dae 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/klauspost/pgzip v1.2.1 github.com/klauspost/readahead v1.3.1 github.com/klauspost/reedsolomon v1.9.9 - github.com/lib/pq v1.7.0 + github.com/lib/pq v1.8.0 github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.8 diff --git a/go.sum b/go.sum index 4c78503df..757a58253 100644 --- a/go.sum +++ b/go.sum @@ -280,6 +280,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= diff --git a/pkg/event/target/mysql.go b/pkg/event/target/mysql.go index 969697eae..7eb94517b 100644 --- a/pkg/event/target/mysql.go +++ b/pkg/event/target/mysql.go @@ -83,43 +83,46 @@ const ( // MySQL related constants const ( - MySQLFormat = "format" - MySQLDSNString = "dsn_string" - MySQLTable = "table" - MySQLHost = "host" - MySQLPort = "port" - MySQLUsername = "username" - MySQLPassword = "password" - MySQLDatabase = "database" - MySQLQueueLimit = "queue_limit" - MySQLQueueDir = "queue_dir" - - EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE" - EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT" - EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING" - EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE" - EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST" - EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT" - EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME" - EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD" - EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE" - EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT" - EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR" + MySQLFormat = "format" + MySQLDSNString = "dsn_string" + MySQLTable = "table" + MySQLHost = "host" + MySQLPort = "port" + MySQLUsername = "username" + MySQLPassword = "password" + MySQLDatabase = "database" + MySQLQueueLimit = "queue_limit" + MySQLQueueDir = "queue_dir" + MySQLMaxOpenConnections = "max_open_connections" + + EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE" + EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT" + EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING" + EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE" + EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST" + EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT" + EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME" + EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD" + EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE" + EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT" + EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR" + EnvMySQLMaxOpenConnections = "MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS" ) // MySQLArgs - MySQL target arguments. type MySQLArgs struct { - Enable bool `json:"enable"` - Format string `json:"format"` - DSN string `json:"dsnString"` - Table string `json:"table"` - Host xnet.URL `json:"host"` - Port string `json:"port"` - User string `json:"user"` - Password string `json:"password"` - Database string `json:"database"` - QueueDir string `json:"queueDir"` - QueueLimit uint64 `json:"queueLimit"` + Enable bool `json:"enable"` + Format string `json:"format"` + DSN string `json:"dsnString"` + Table string `json:"table"` + Host xnet.URL `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Database string `json:"database"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + MaxOpenConnections int `json:"maxOpenConnections"` } // Validate MySQLArgs fields @@ -162,6 +165,10 @@ func (m MySQLArgs) Validate() error { } } + if m.MaxOpenConnections < 0 { + return errors.New("maxOpenConnections cannot be less than zero") + } + return nil } @@ -196,6 +203,10 @@ func (target *MySQLTarget) IsActive() (bool, error) { return false, sErr } target.db = db + if target.args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(target.args.MaxOpenConnections) + } } if err := target.db.Ping(); err != nil { if IsConnErr(err) { @@ -384,6 +395,11 @@ func NewMySQLTarget(id string, args MySQLArgs, doneCh <-chan struct{}, loggerOnc } target.db = db + if args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(args.MaxOpenConnections) + } + var store Store if args.QueueDir != "" { diff --git a/pkg/event/target/postgresql.go b/pkg/event/target/postgresql.go index bf30438e4..c1a10a0f7 100644 --- a/pkg/event/target/postgresql.go +++ b/pkg/event/target/postgresql.go @@ -84,43 +84,46 @@ const ( // Postgres constants const ( - PostgresFormat = "format" - PostgresConnectionString = "connection_string" - PostgresTable = "table" - PostgresHost = "host" - PostgresPort = "port" - PostgresUsername = "username" - PostgresPassword = "password" - PostgresDatabase = "database" - PostgresQueueDir = "queue_dir" - PostgresQueueLimit = "queue_limit" - - EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE" - EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT" - EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING" - EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE" - EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST" - EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT" - EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME" - EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD" - EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE" - EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR" - EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT" + PostgresFormat = "format" + PostgresConnectionString = "connection_string" + PostgresTable = "table" + PostgresHost = "host" + PostgresPort = "port" + PostgresUsername = "username" + PostgresPassword = "password" + PostgresDatabase = "database" + PostgresQueueDir = "queue_dir" + PostgresQueueLimit = "queue_limit" + PostgresMaxOpenConnections = "max_open_connections" + + EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE" + EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT" + EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING" + EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE" + EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST" + EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT" + EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME" + EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD" + EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE" + EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR" + EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT" + EnvPostgresMaxOpenConnections = "MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS" ) // PostgreSQLArgs - PostgreSQL target arguments. type PostgreSQLArgs struct { - Enable bool `json:"enable"` - Format string `json:"format"` - ConnectionString string `json:"connectionString"` - Table string `json:"table"` - Host xnet.Host `json:"host"` // default: localhost - Port string `json:"port"` // default: 5432 - Username string `json:"username"` // default: user running minio - Password string `json:"password"` // default: no password - Database string `json:"database"` // default: same as user - QueueDir string `json:"queueDir"` - QueueLimit uint64 `json:"queueLimit"` + Enable bool `json:"enable"` + Format string `json:"format"` + ConnectionString string `json:"connectionString"` + Table string `json:"table"` + Host xnet.Host `json:"host"` // default: localhost + Port string `json:"port"` // default: 5432 + Username string `json:"username"` // default: user running minio + Password string `json:"password"` // default: no password + Database string `json:"database"` // default: same as user + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + MaxOpenConnections int `json:"maxOpenConnections"` } // Validate PostgreSQLArgs fields @@ -160,6 +163,10 @@ func (p PostgreSQLArgs) Validate() error { } } + if p.MaxOpenConnections < 0 { + return errors.New("maxOpenConnections cannot be less than zero") + } + return nil } @@ -195,6 +202,10 @@ func (target *PostgreSQLTarget) IsActive() (bool, error) { return false, err } target.db = db + if target.args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(target.args.MaxOpenConnections) + } } if err := target.db.Ping(); err != nil { if IsConnErr(err) { @@ -391,6 +402,11 @@ func NewPostgreSQLTarget(id string, args PostgreSQLArgs, doneCh <-chan struct{}, } target.db = db + if args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(args.MaxOpenConnections) + } + var store Store if args.QueueDir != "" { diff --git a/ruleguard.rules.go b/ruleguard.rules.go index 3f94fa44d..50aa4a010 100644 --- a/ruleguard.rules.go +++ b/ruleguard.rules.go @@ -438,7 +438,6 @@ func httpheaderadd(m fluent.Matcher) { Suggest(`$H.Set($KEY, $VALUE)`) } - func hmacnew(m fluent.Matcher) { m.Match("hmac.New(func() hash.Hash { return $x }, $_)", `$f := func() hash.Hash { return $x }