oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Quotas are more accurate, occur on Read()

Browse Source
master
Frederick F. Kautz IV 10 years ago
parent f94d9db4f7
commit e2403cbc56
4 changed files with 174 additions and 59 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      pkg/api/api_router.go
  2. 83
      pkg/api/quota/bandwidth_cap.go
  3. 91
      pkg/api/quota/quota_handler.go
  4. 53
      pkg/api/quota/request_limit.go

6
pkg/api/api_router.go
Unescape View File

@ -25,6 +25,7 @@ import (
"github.com/minio-io/minio/pkg/api/quota"
"github.com/minio-io/minio/pkg/iodine"
"github.com/minio-io/minio/pkg/storage/drivers"
"time"
)
// private use
@ -91,6 +92,7 @@ func HTTPHandler(domain string, driver drivers.Driver) http.Handler {
}
h := validateHandler(conf, ignoreResourcesHandler(mux))
// quota handler is always last
return quota.BandwidthCap(h, int64(100*1024*1024))
h = quota.BandwidthCap(h, 250*1024*1024, time.Duration(30*time.Minute))
h = quota.RequestLimit(h, 100, time.Duration(30*time.Minute))
return h
}

83
pkg/api/quota/bandwidth_cap.go
Unescape View File

@ -0,0 +1,83 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quota
import (
"errors"
"io"
"net"
"net/http"
"time"
)
// bandwidthQuotaHandler
type bandwidthQuotaHandler struct {
handler http.Handler
quotas *quotaMap
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *bandwidthQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
req.Body = quotaReader{
ReadCloser: req.Body,
quotas: h.quotas,
ip: longIP,
}
h.handler.ServeHTTP(w, req)
}
// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64, duration time.Duration) http.Handler {
return &bandwidthQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]int64),
limit: int64(limit),
duration: duration,
segmentSize: segmentSize(duration),
},
}
}
type quotaReader struct {
io.ReadCloser
quotas *quotaMap
ip uint32
}
func (q quotaReader) Read(b []byte) (int, error) {
if q.quotas.IsQuotaMet(q.ip) {
return 0, errors.New("Quota Met")
}
n, err := q.ReadCloser.Read(b)
q.quotas.Add(q.ip, int64(n))
return n, err
}
func (q quotaReader) Close() error {
return q.ReadCloser.Close()
}
func segmentSize(duration time.Duration) time.Duration {
var segmentSize time.Duration
for i := int64(1); i < duration.Nanoseconds(); i = i * 10 {
segmentSize = time.Duration(i)
}
return segmentSize
}

91
pkg/api/quota/quota_handler.go
Unescape View File

@ -19,7 +19,6 @@ package quota
import (
"encoding/binary"
"net"
"net/http"
"sync"
"time"
)
@ -27,38 +26,51 @@ import (
// map[minute][address] = current quota
type quotaMap struct {
sync.RWMutex
data map[int64]map[uint32]uint64
limit uint64
duration int64
data map[int64]map[uint32]int64
limit int64
duration time.Duration
segmentSize time.Duration
}
func (q *quotaMap) Add(ip uint32, size uint64) bool {
func (q *quotaMap) Add(ip uint32, size int64) {
q.Lock()
defer q.Unlock()
currentMinute := time.Now().Unix() / q.duration
expiredQuotas := (time.Now().Unix() / q.duration) - 5
for time := range q.data {
if time < expiredQuotas {
delete(q.data, time)
}
}
q.clean()
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
if _, ok := q.data[currentMinute]; !ok {
q.data[currentMinute] = make(map[uint32]uint64)
q.data[currentMinute] = make(map[uint32]int64)
}
currentData, _ := q.data[currentMinute][ip]
proposedDataSize := currentData + size
if proposedDataSize > q.limit {
return false
}
q.data[currentMinute][ip] = proposedDataSize
return true
}
// HttpQuotaHandler
type httpQuotaHandler struct {
handler http.Handler
quotas *quotaMap
adder func(uint64) uint64
func (q *quotaMap) IsQuotaMet(ip uint32) bool {
q.clean()
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
if _, ok := q.data[currentMinute]; !ok {
q.data[currentMinute] = make(map[uint32]int64)
}
var total int64
for _, segment := range q.data {
if used, ok := segment[ip]; ok {
total += used
}
}
if total >= q.limit {
return true
}
return false
}
func (q *quotaMap) clean() {
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
expiredQuotas := currentMinute - q.duration.Nanoseconds()
for time := range q.data {
if time < expiredQuotas {
delete(q.data, time)
}
}
}
type longIP struct {
@ -73,38 +85,3 @@ func (p longIP) IptoUint32() (result uint32) {
}
return binary.BigEndian.Uint32(ip)
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *httpQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
if h.quotas.Add(longIP, h.adder(uint64(req.ContentLength))) {
h.handler.ServeHTTP(w, req)
}
}
// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64) http.Handler {
return &httpQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]uint64),
limit: uint64(limit),
duration: int64(60),
},
adder: func(count uint64) uint64 { return count },
}
}
// RequestLimit sets a quota based upon request count
func RequestLimit(h http.Handler, limit int64) http.Handler {
return &httpQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]uint64),
limit: uint64(limit),
duration: int64(60),
},
adder: func(count uint64) uint64 { return 1 },
}
}

53
pkg/api/quota/request_limit.go
Unescape View File

@ -0,0 +1,53 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quota
import (
"net"
"net/http"
"time"
)
// requestLimitHandler
type requestLimitHandler struct {
handler http.Handler
quotas *quotaMap
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *requestLimitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
if h.quotas.IsQuotaMet(longIP) {
return
}
h.quotas.Add(longIP, 1)
h.handler.ServeHTTP(w, req)
}
// RequestLimit sets a quote based upon number of requests allowed over a time period
func RequestLimit(h http.Handler, limit int64, duration time.Duration) http.Handler {
return &requestLimitHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]int64),
limit: int64(limit),
duration: duration,
segmentSize: segmentSize(duration),
},
}
}
Write Preview
Loading…
Cancel
Save