Event persistence for MQTT (#7268)
- The events will be persisted in queueStore if `queueDir` is set. - Else, if queueDir is not set events persist in memory. The events are replayed back when the mqtt broker is back online.master
Praveen raj Mani
6 years ago
committed by
Harshavardhana
parent
2fc341394d
commit
78d116c487
@ -0,0 +1,104 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package target |
||||
|
||||
import ( |
||||
"sync" |
||||
|
||||
"github.com/minio/minio/pkg/event" |
||||
) |
||||
|
||||
const ( |
||||
maxStoreLimit = 10000 |
||||
) |
||||
|
||||
// MemoryStore persists events in memory.
|
||||
type MemoryStore struct { |
||||
sync.RWMutex |
||||
events map[string]event.Event |
||||
eC uint16 |
||||
limit uint16 |
||||
} |
||||
|
||||
// NewMemoryStore creates a memory store instance.
|
||||
func NewMemoryStore(limit uint16) *MemoryStore { |
||||
if limit == 0 || limit > maxStoreLimit { |
||||
limit = maxStoreLimit |
||||
} |
||||
memoryStore := &MemoryStore{ |
||||
events: make(map[string]event.Event), |
||||
limit: limit, |
||||
} |
||||
return memoryStore |
||||
} |
||||
|
||||
// Open is in-effective here.
|
||||
// Implemented for interface compatibility.
|
||||
func (store *MemoryStore) Open() error { |
||||
return nil |
||||
} |
||||
|
||||
// Put - puts the event in store.
|
||||
func (store *MemoryStore) Put(e event.Event) error { |
||||
store.Lock() |
||||
defer store.Unlock() |
||||
if store.eC == store.limit { |
||||
return ErrLimitExceeded |
||||
} |
||||
key, kErr := getNewUUID() |
||||
if kErr != nil { |
||||
return kErr |
||||
} |
||||
store.events[key] = e |
||||
store.eC++ |
||||
return nil |
||||
} |
||||
|
||||
// Get - retrieves the event from store.
|
||||
func (store *MemoryStore) Get(key string) (event.Event, error) { |
||||
store.RLock() |
||||
defer store.RUnlock() |
||||
|
||||
if event, exist := store.events[key]; exist { |
||||
return event, nil |
||||
} |
||||
|
||||
return event.Event{}, ErrNoSuchKey |
||||
} |
||||
|
||||
// Del - deletes the event from store.
|
||||
func (store *MemoryStore) Del(key string) { |
||||
store.Lock() |
||||
defer store.Unlock() |
||||
|
||||
delete(store.events, key) |
||||
|
||||
store.eC-- |
||||
} |
||||
|
||||
// ListAll - lists all the keys in the store.
|
||||
func (store *MemoryStore) ListAll() []string { |
||||
store.RLock() |
||||
defer store.RUnlock() |
||||
|
||||
keys := []string{} |
||||
for k := range store.events { |
||||
keys = append(keys, k) |
||||
} |
||||
|
||||
return keys |
||||
} |
||||
@ -0,0 +1,106 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package target |
||||
|
||||
import ( |
||||
"reflect" |
||||
"testing" |
||||
) |
||||
|
||||
// TestMemoryStorePut - Tests for store.Put
|
||||
func TestMemoryStorePut(t *testing.T) { |
||||
store := NewMemoryStore(1000) |
||||
defer func() { |
||||
store = nil |
||||
}() |
||||
for i := 0; i < 100; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
if len(store.ListAll()) != 100 { |
||||
t.Fatalf("ListAll() Expected: 100, got %d", len(store.ListAll())) |
||||
} |
||||
} |
||||
|
||||
// TestMemoryStoreGet - Tests for store.Get.
|
||||
func TestMemoryStoreGet(t *testing.T) { |
||||
store := NewMemoryStore(1000) |
||||
defer func() { |
||||
store = nil |
||||
}() |
||||
for i := 0; i < 10; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
eventKeys := store.ListAll() |
||||
if len(eventKeys) == 10 { |
||||
for _, key := range eventKeys { |
||||
event, eErr := store.Get(key) |
||||
if eErr != nil { |
||||
t.Fatal("Failed to Get the event from the queue store ", eErr) |
||||
} |
||||
if !reflect.DeepEqual(testEvent, event) { |
||||
t.Fatalf("Failed to read the event: error: expected = %v, got = %v", testEvent, event) |
||||
} |
||||
} |
||||
} else { |
||||
t.Fatalf("ListAll() Expected: 10, got %d", len(eventKeys)) |
||||
} |
||||
} |
||||
|
||||
// TestMemoryStoreDel - Tests for store.Del.
|
||||
func TestMemoryStoreDel(t *testing.T) { |
||||
store := NewMemoryStore(1000) |
||||
defer func() { |
||||
store = nil |
||||
}() |
||||
for i := 0; i < 20; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
eventKeys := store.ListAll() |
||||
if len(eventKeys) == 20 { |
||||
for _, key := range eventKeys { |
||||
store.Del(key) |
||||
} |
||||
} else { |
||||
t.Fatalf("ListAll() Expected: 20, got %d", len(eventKeys)) |
||||
} |
||||
|
||||
if len(store.ListAll()) != 0 { |
||||
t.Fatalf("ListAll() Expected: 0, got %d", len(store.ListAll())) |
||||
} |
||||
} |
||||
|
||||
// TestMemoryStoreLimit - tests for store limit.
|
||||
func TestMemoryStoreLimit(t *testing.T) { |
||||
store := NewMemoryStore(5) |
||||
defer func() { |
||||
store = nil |
||||
}() |
||||
for i := 0; i < 5; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
if err := store.Put(testEvent); err == nil { |
||||
t.Fatalf("Expected to fail with %s, but passes", ErrLimitExceeded) |
||||
} |
||||
} |
||||
@ -0,0 +1,178 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package target |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"github.com/minio/minio/pkg/event" |
||||
) |
||||
|
||||
const ( |
||||
maxLimit = 10000 // Max store limit.
|
||||
eventExt = ".event" |
||||
) |
||||
|
||||
// QueueStore - Filestore for persisting events.
|
||||
type QueueStore struct { |
||||
sync.RWMutex |
||||
directory string |
||||
eC uint16 |
||||
limit uint16 |
||||
} |
||||
|
||||
// NewQueueStore - Creates an instance for QueueStore.
|
||||
func NewQueueStore(directory string, limit uint16) *QueueStore { |
||||
if limit == 0 { |
||||
limit = maxLimit |
||||
} |
||||
queueStore := &QueueStore{ |
||||
directory: directory, |
||||
limit: limit, |
||||
} |
||||
return queueStore |
||||
} |
||||
|
||||
// Open - Creates the directory if not present.
|
||||
func (store *QueueStore) Open() error { |
||||
store.Lock() |
||||
defer store.Unlock() |
||||
|
||||
if terr := os.MkdirAll(store.directory, os.FileMode(0770)); terr != nil { |
||||
return terr |
||||
} |
||||
|
||||
eCount := uint16(len(store.listAll())) |
||||
if eCount >= store.limit { |
||||
return ErrLimitExceeded |
||||
} |
||||
|
||||
store.eC = eCount |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// write - writes event to the directory.
|
||||
func (store *QueueStore) write(directory string, key string, e event.Event) error { |
||||
|
||||
// Marshalls the event.
|
||||
eventData, err := json.Marshal(e) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
path := filepath.Join(store.directory, key+eventExt) |
||||
if err := ioutil.WriteFile(path, eventData, os.FileMode(0770)); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Increment the event count.
|
||||
store.eC++ |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Put - puts a event to the store.
|
||||
func (store *QueueStore) Put(e event.Event) error { |
||||
store.Lock() |
||||
defer store.Unlock() |
||||
if store.eC >= store.limit { |
||||
return ErrLimitExceeded |
||||
} |
||||
key, kErr := getNewUUID() |
||||
if kErr != nil { |
||||
return kErr |
||||
} |
||||
return store.write(store.directory, key, e) |
||||
} |
||||
|
||||
// Get - gets a event from the store.
|
||||
func (store *QueueStore) Get(key string) (event.Event, error) { |
||||
store.RLock() |
||||
defer store.RUnlock() |
||||
|
||||
var event event.Event |
||||
|
||||
filepath := filepath.Join(store.directory, key+eventExt) |
||||
|
||||
eventData, rerr := ioutil.ReadFile(filepath) |
||||
if rerr != nil { |
||||
store.del(key) |
||||
return event, rerr |
||||
} |
||||
|
||||
if len(eventData) == 0 { |
||||
store.del(key) |
||||
} |
||||
|
||||
uerr := json.Unmarshal(eventData, &event) |
||||
if uerr != nil { |
||||
store.del(key) |
||||
return event, uerr |
||||
} |
||||
|
||||
return event, nil |
||||
} |
||||
|
||||
// Del - Deletes an entry from the store.
|
||||
func (store *QueueStore) Del(key string) { |
||||
store.Lock() |
||||
defer store.Unlock() |
||||
store.del(key) |
||||
} |
||||
|
||||
// lockless call
|
||||
func (store *QueueStore) del(key string) { |
||||
p := filepath.Join(store.directory, key+eventExt) |
||||
|
||||
rerr := os.Remove(p) |
||||
if rerr != nil { |
||||
return |
||||
} |
||||
|
||||
// Decrement the event count.
|
||||
store.eC-- |
||||
} |
||||
|
||||
// ListAll - lists all the keys in the directory.
|
||||
func (store *QueueStore) ListAll() []string { |
||||
store.RLock() |
||||
defer store.RUnlock() |
||||
return store.listAll() |
||||
} |
||||
|
||||
// lockless call.
|
||||
func (store *QueueStore) listAll() []string { |
||||
var err error |
||||
var keys []string |
||||
var files []os.FileInfo |
||||
|
||||
files, err = ioutil.ReadDir(store.directory) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
|
||||
for _, f := range files { |
||||
keys = append(keys, strings.TrimSuffix(f.Name(), eventExt)) |
||||
} |
||||
return keys |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package target |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
"reflect" |
||||
"testing" |
||||
|
||||
"github.com/minio/minio/pkg/event" |
||||
) |
||||
|
||||
// TestDir
|
||||
var queueDir = filepath.Join(os.TempDir(), "minio_test") |
||||
|
||||
// Sample test event.
|
||||
var testEvent = event.Event{EventVersion: "1.0", EventSource: "test_source", AwsRegion: "test_region", EventTime: "test_time", EventName: event.ObjectAccessedGet} |
||||
|
||||
// Initialize the store.
|
||||
func setUpStore(directory string, limit uint16) (Store, error) { |
||||
store := NewQueueStore(queueDir, limit) |
||||
if oErr := store.Open(); oErr != nil { |
||||
return nil, oErr |
||||
} |
||||
return store, nil |
||||
} |
||||
|
||||
// Tear down store
|
||||
func tearDownStore() error { |
||||
if err := os.RemoveAll(queueDir); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// TestQueueStorePut - tests for store.Put
|
||||
func TestQueueStorePut(t *testing.T) { |
||||
defer func() { |
||||
if err := tearDownStore(); err != nil { |
||||
t.Fatal("Failed to tear down store ", err) |
||||
} |
||||
}() |
||||
store, err := setUpStore(queueDir, 10000) |
||||
if err != nil { |
||||
t.Fatal("Failed to create a queue store ", err) |
||||
|
||||
} |
||||
// Put 100 events.
|
||||
for i := 0; i < 100; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
// Count the events.
|
||||
if len(store.ListAll()) != 100 { |
||||
t.Fatalf("ListAll() Expected: 100, got %d", len(store.ListAll())) |
||||
} |
||||
} |
||||
|
||||
// TestQueueStoreGet - tests for store.Get
|
||||
func TestQueueStoreGet(t *testing.T) { |
||||
defer func() { |
||||
if err := tearDownStore(); err != nil { |
||||
t.Fatal("Failed to tear down store ", err) |
||||
} |
||||
}() |
||||
store, err := setUpStore(queueDir, 10000) |
||||
if err != nil { |
||||
t.Fatal("Failed to create a queue store ", err) |
||||
} |
||||
// Put 10 events
|
||||
for i := 0; i < 10; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
eventKeys := store.ListAll() |
||||
// Get 10 events.
|
||||
if len(eventKeys) == 10 { |
||||
for _, key := range eventKeys { |
||||
event, eErr := store.Get(key) |
||||
if eErr != nil { |
||||
t.Fatal("Failed to Get the event from the queue store ", eErr) |
||||
} |
||||
if !reflect.DeepEqual(testEvent, event) { |
||||
t.Fatalf("Failed to read the event: error: expected = %v, got = %v", testEvent, event) |
||||
} |
||||
} |
||||
} else { |
||||
t.Fatalf("ListAll() Expected: 10, got %d", len(eventKeys)) |
||||
} |
||||
} |
||||
|
||||
// TestQueueStoreDel - tests for store.Del
|
||||
func TestQueueStoreDel(t *testing.T) { |
||||
defer func() { |
||||
if err := tearDownStore(); err != nil { |
||||
t.Fatal("Failed to tear down store ", err) |
||||
} |
||||
}() |
||||
store, err := setUpStore(queueDir, 10000) |
||||
if err != nil { |
||||
t.Fatal("Failed to create a queue store ", err) |
||||
} |
||||
// Put 20 events.
|
||||
for i := 0; i < 20; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
eventKeys := store.ListAll() |
||||
// Remove all the events.
|
||||
if len(eventKeys) == 20 { |
||||
for _, key := range eventKeys { |
||||
store.Del(key) |
||||
} |
||||
} else { |
||||
t.Fatalf("ListAll() Expected: 20, got %d", len(eventKeys)) |
||||
} |
||||
|
||||
if len(store.ListAll()) != 0 { |
||||
t.Fatalf("ListAll() Expected: 0, got %d", len(store.ListAll())) |
||||
} |
||||
} |
||||
|
||||
// TestQueueStoreLimit - tests the event limit for the store.
|
||||
func TestQueueStoreLimit(t *testing.T) { |
||||
defer func() { |
||||
if err := tearDownStore(); err != nil { |
||||
t.Fatal("Failed to tear down store ", err) |
||||
} |
||||
}() |
||||
// The max limit is set to 5.
|
||||
store, err := setUpStore(queueDir, 5) |
||||
if err != nil { |
||||
t.Fatal("Failed to create a queue store ", err) |
||||
} |
||||
for i := 0; i < 5; i++ { |
||||
if err := store.Put(testEvent); err != nil { |
||||
t.Fatal("Failed to put to queue store ", err) |
||||
} |
||||
} |
||||
// Should not allow 6th Put.
|
||||
if err := store.Put(testEvent); err == nil { |
||||
t.Fatalf("Expected to fail with %s, but passes", ErrLimitExceeded) |
||||
} |
||||
} |
||||
@ -0,0 +1,37 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package target |
||||
|
||||
import ( |
||||
"errors" |
||||
"github.com/minio/minio/pkg/event" |
||||
) |
||||
|
||||
// ErrLimitExceeded error is sent when the maximum limit is reached.
|
||||
var ErrLimitExceeded = errors.New("[Store] The maximum limit reached") |
||||
|
||||
// ErrNoSuchKey error is sent in Get when the key is not found.
|
||||
var ErrNoSuchKey = errors.New("[Store] No such key found") |
||||
|
||||
// Store - To persist the events.
|
||||
type Store interface { |
||||
Put(event event.Event) error |
||||
Get(key string) (event.Event, error) |
||||
ListAll() []string |
||||
Del(key string) |
||||
Open() error |
||||
} |
||||
Loading…
Reference in new issue