Quotas are more accurate, occur on Read()

master
Frederick F. Kautz IV 10 years ago
parent f94d9db4f7
commit e2403cbc56
  1. 6
      pkg/api/api_router.go
  2. 83
      pkg/api/quota/bandwidth_cap.go
  3. 91
      pkg/api/quota/quota_handler.go
  4. 53
      pkg/api/quota/request_limit.go

@ -25,6 +25,7 @@ import (
"github.com/minio-io/minio/pkg/api/quota"
"github.com/minio-io/minio/pkg/iodine"
"github.com/minio-io/minio/pkg/storage/drivers"
"time"
)
// private use
@ -91,6 +92,7 @@ func HTTPHandler(domain string, driver drivers.Driver) http.Handler {
}
h := validateHandler(conf, ignoreResourcesHandler(mux))
// quota handler is always last
return quota.BandwidthCap(h, int64(100*1024*1024))
h = quota.BandwidthCap(h, 250*1024*1024, time.Duration(30*time.Minute))
h = quota.RequestLimit(h, 100, time.Duration(30*time.Minute))
return h
}

@ -0,0 +1,83 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quota
import (
"errors"
"io"
"net"
"net/http"
"time"
)
// bandwidthQuotaHandler
type bandwidthQuotaHandler struct {
handler http.Handler
quotas *quotaMap
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *bandwidthQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
req.Body = quotaReader{
ReadCloser: req.Body,
quotas: h.quotas,
ip: longIP,
}
h.handler.ServeHTTP(w, req)
}
// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64, duration time.Duration) http.Handler {
return &bandwidthQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]int64),
limit: int64(limit),
duration: duration,
segmentSize: segmentSize(duration),
},
}
}
type quotaReader struct {
io.ReadCloser
quotas *quotaMap
ip uint32
}
func (q quotaReader) Read(b []byte) (int, error) {
if q.quotas.IsQuotaMet(q.ip) {
return 0, errors.New("Quota Met")
}
n, err := q.ReadCloser.Read(b)
q.quotas.Add(q.ip, int64(n))
return n, err
}
func (q quotaReader) Close() error {
return q.ReadCloser.Close()
}
func segmentSize(duration time.Duration) time.Duration {
var segmentSize time.Duration
for i := int64(1); i < duration.Nanoseconds(); i = i * 10 {
segmentSize = time.Duration(i)
}
return segmentSize
}

@ -19,7 +19,6 @@ package quota
import (
"encoding/binary"
"net"
"net/http"
"sync"
"time"
)
@ -27,38 +26,51 @@ import (
// map[minute][address] = current quota
type quotaMap struct {
sync.RWMutex
data map[int64]map[uint32]uint64
limit uint64
duration int64
data map[int64]map[uint32]int64
limit int64
duration time.Duration
segmentSize time.Duration
}
func (q *quotaMap) Add(ip uint32, size uint64) bool {
func (q *quotaMap) Add(ip uint32, size int64) {
q.Lock()
defer q.Unlock()
currentMinute := time.Now().Unix() / q.duration
expiredQuotas := (time.Now().Unix() / q.duration) - 5
for time := range q.data {
if time < expiredQuotas {
delete(q.data, time)
}
}
q.clean()
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
if _, ok := q.data[currentMinute]; !ok {
q.data[currentMinute] = make(map[uint32]uint64)
q.data[currentMinute] = make(map[uint32]int64)
}
currentData, _ := q.data[currentMinute][ip]
proposedDataSize := currentData + size
if proposedDataSize > q.limit {
return false
}
q.data[currentMinute][ip] = proposedDataSize
return true
}
// HttpQuotaHandler
type httpQuotaHandler struct {
handler http.Handler
quotas *quotaMap
adder func(uint64) uint64
func (q *quotaMap) IsQuotaMet(ip uint32) bool {
q.clean()
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
if _, ok := q.data[currentMinute]; !ok {
q.data[currentMinute] = make(map[uint32]int64)
}
var total int64
for _, segment := range q.data {
if used, ok := segment[ip]; ok {
total += used
}
}
if total >= q.limit {
return true
}
return false
}
func (q *quotaMap) clean() {
currentMinute := time.Now().UnixNano() / q.segmentSize.Nanoseconds()
expiredQuotas := currentMinute - q.duration.Nanoseconds()
for time := range q.data {
if time < expiredQuotas {
delete(q.data, time)
}
}
}
type longIP struct {
@ -73,38 +85,3 @@ func (p longIP) IptoUint32() (result uint32) {
}
return binary.BigEndian.Uint32(ip)
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *httpQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
if h.quotas.Add(longIP, h.adder(uint64(req.ContentLength))) {
h.handler.ServeHTTP(w, req)
}
}
// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64) http.Handler {
return &httpQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]uint64),
limit: uint64(limit),
duration: int64(60),
},
adder: func(count uint64) uint64 { return count },
}
}
// RequestLimit sets a quota based upon request count
func RequestLimit(h http.Handler, limit int64) http.Handler {
return &httpQuotaHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]uint64),
limit: uint64(limit),
duration: int64(60),
},
adder: func(count uint64) uint64 { return 1 },
}
}

@ -0,0 +1,53 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quota
import (
"net"
"net/http"
"time"
)
// requestLimitHandler
type requestLimitHandler struct {
handler http.Handler
quotas *quotaMap
}
// ServeHTTP is an http.Handler ServeHTTP method
func (h *requestLimitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
host, _, _ := net.SplitHostPort(req.RemoteAddr)
longIP := longIP{net.ParseIP(host)}.IptoUint32()
if h.quotas.IsQuotaMet(longIP) {
return
}
h.quotas.Add(longIP, 1)
h.handler.ServeHTTP(w, req)
}
// RequestLimit sets a quote based upon number of requests allowed over a time period
func RequestLimit(h http.Handler, limit int64, duration time.Duration) http.Handler {
return &requestLimitHandler{
handler: h,
quotas: &quotaMap{
data: make(map[int64]map[uint32]int64),
limit: int64(limit),
duration: duration,
segmentSize: segmentSize(duration),
},
}
}
Loading…
Cancel
Save