Add admin API to send trace notifications to registered (#7128)
Remove current functionality to log trace to file using MINIO_HTTP_TRACE env, and replace it with mc admin trace command on mc client.master
poornas
5 years ago
committed by
Harshavardhana
parent
fb531235de
commit
97090aa16c
@ -0,0 +1,115 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/json" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
"github.com/minio/minio/pkg/pubsub" |
||||
"github.com/minio/minio/pkg/trace" |
||||
) |
||||
|
||||
//HTTPTraceSys holds global trace state
|
||||
type HTTPTraceSys struct { |
||||
peers []*peerRESTClient |
||||
pubsub *pubsub.PubSub |
||||
} |
||||
|
||||
// NewTraceSys - creates new HTTPTraceSys with all nodes subscribed to
|
||||
// the trace pub sub system
|
||||
func NewTraceSys(ctx context.Context, endpoints EndpointList) *HTTPTraceSys { |
||||
remoteHosts := getRemoteHosts(endpoints) |
||||
remoteClients, err := getRestClients(remoteHosts) |
||||
if err != nil { |
||||
logger.FatalIf(err, "Unable to start httptrace sub system") |
||||
} |
||||
|
||||
ps := pubsub.New() |
||||
return &HTTPTraceSys{ |
||||
remoteClients, ps, |
||||
} |
||||
} |
||||
|
||||
// HasTraceListeners returns true if trace listeners are registered
|
||||
// for this node or peers
|
||||
func (sys *HTTPTraceSys) HasTraceListeners() bool { |
||||
return sys != nil && sys.pubsub.HasSubscribers() |
||||
} |
||||
|
||||
// Publish - publishes trace message to the http trace pubsub system
|
||||
func (sys *HTTPTraceSys) Publish(traceMsg trace.Info) { |
||||
sys.pubsub.Publish(traceMsg) |
||||
} |
||||
|
||||
// Trace writes http trace to writer
|
||||
func (sys *HTTPTraceSys) Trace(doneCh chan struct{}, trcAll bool) chan []byte { |
||||
traceCh := make(chan []byte) |
||||
go func() { |
||||
defer close(traceCh) |
||||
|
||||
var wg = &sync.WaitGroup{} |
||||
wg.Add(1) |
||||
go func() { |
||||
defer wg.Done() |
||||
|
||||
buf := &bytes.Buffer{} |
||||
ch := sys.pubsub.Subscribe() |
||||
defer sys.pubsub.Unsubscribe(ch) |
||||
for { |
||||
select { |
||||
case entry := <-ch: |
||||
trcInfo := entry.(trace.Info) |
||||
path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/") |
||||
// omit inter-node traffic if trcAll is false
|
||||
if !trcAll && strings.HasPrefix(path, minioReservedBucket) { |
||||
continue |
||||
} |
||||
buf.Reset() |
||||
enc := json.NewEncoder(buf) |
||||
enc.SetEscapeHTML(false) |
||||
if err := enc.Encode(trcInfo); err != nil { |
||||
continue |
||||
} |
||||
traceCh <- buf.Bytes() |
||||
case <-doneCh: |
||||
return |
||||
} |
||||
} |
||||
}() |
||||
|
||||
for _, peer := range sys.peers { |
||||
wg.Add(1) |
||||
go func(peer *peerRESTClient) { |
||||
defer wg.Done() |
||||
ch, err := peer.Trace(doneCh, trcAll) |
||||
if err != nil { |
||||
return |
||||
} |
||||
for entry := range ch { |
||||
traceCh <- entry |
||||
} |
||||
}(peer) |
||||
} |
||||
wg.Wait() |
||||
}() |
||||
return traceCh |
||||
} |
||||
@ -1,154 +0,0 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2017 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package handlers |
||||
|
||||
import ( |
||||
"bytes" |
||||
"io" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"net/http/httptest" |
||||
"regexp" |
||||
"testing" |
||||
) |
||||
|
||||
func wsTestSuccessHandler(w http.ResponseWriter, r *http.Request) { |
||||
// A very simple health check.
|
||||
w.WriteHeader(http.StatusOK) |
||||
w.Header().Set("Content-Type", "application/json") |
||||
|
||||
ioutil.ReadAll(r.Body) |
||||
|
||||
// In the future we could report back on the status of our DB, or our cache
|
||||
// (e.g. Redis) by performing a simple PING, and include them in the response.
|
||||
io.WriteString(w, `{"success": true}`) |
||||
} |
||||
|
||||
func wsTest404Handler(w http.ResponseWriter, r *http.Request) { |
||||
w.WriteHeader(http.StatusNotFound) |
||||
} |
||||
|
||||
func TestTraceHTTPHandler(t *testing.T) { |
||||
|
||||
logOutput := bytes.NewBuffer([]byte("")) |
||||
|
||||
testCases := []struct { |
||||
method string |
||||
path string |
||||
sentData string |
||||
headers map[string]string |
||||
handler http.HandlerFunc |
||||
expectedStatus int |
||||
expectedLogRegexp string |
||||
}{ |
||||
|
||||
{ |
||||
method: "PUT", |
||||
path: "/test-log", |
||||
sentData: "sending data", |
||||
headers: map[string]string{"Test-Header": "TestHeaderValue"}, |
||||
handler: TraceReqHandlerFunc(http.HandlerFunc(wsTestSuccessHandler), logOutput, true), |
||||
expectedStatus: http.StatusOK, |
||||
expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTestSuccessHandler\] \[[^\]]*\] \[[^\]]*\] |
||||
PUT /test-log |
||||
Host:\
|
||||
Test-Header: TestHeaderValue |
||||
|
||||
sending data |
||||
|
||||
\[RESPONSE\] \[[^\]]*\] \[[^\]]*\] |
||||
200 OK |
||||
|
||||
{"success": true} |
||||
|
||||
`, |
||||
}, |
||||
{ |
||||
method: "POST", |
||||
path: "/test-log", |
||||
handler: TraceReqHandlerFunc(http.HandlerFunc(wsTestSuccessHandler), logOutput, false), |
||||
headers: map[string]string{"Test-Header": "TestHeaderValue"}, |
||||
expectedStatus: http.StatusOK, |
||||
expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTestSuccessHandler\] \[[^\]]*\] \[[^\]]*\] |
||||
POST /test-log |
||||
Host:\
|
||||
Test-Header: TestHeaderValue |
||||
|
||||
<BODY> |
||||
|
||||
\[RESPONSE\] \[[^\]]*\] \[[^\]]*\] |
||||
200 OK |
||||
|
||||
<BODY> |
||||
|
||||
`, |
||||
}, |
||||
{ |
||||
method: "POST", |
||||
path: "/test-log", |
||||
handler: TraceReqHandlerFunc(http.HandlerFunc(wsTest404Handler), logOutput, false), |
||||
headers: map[string]string{"Test-Header": "TestHeaderValue"}, |
||||
expectedStatus: http.StatusNotFound, |
||||
expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTest404Handler\] \[[^\]]*\] \[[^\]]*\] |
||||
POST /test-log |
||||
Host:\
|
||||
Test-Header: TestHeaderValue |
||||
|
||||
<BODY> |
||||
|
||||
\[RESPONSE\] \[[^\]]*\] \[[^\]]*\] |
||||
404 Not Found |
||||
|
||||
<BODY> |
||||
|
||||
`, |
||||
}, |
||||
} |
||||
|
||||
for i, testCase := range testCases { |
||||
logOutput.Reset() |
||||
|
||||
req, err := http.NewRequest(testCase.method, testCase.path, bytes.NewBuffer([]byte(testCase.sentData))) |
||||
if err != nil { |
||||
t.Fatalf("Test %d: %v\n", i+1, err) |
||||
} |
||||
|
||||
for k, v := range testCase.headers { |
||||
req.Header.Set(k, v) |
||||
} |
||||
|
||||
rr := httptest.NewRecorder() |
||||
|
||||
handler := testCase.handler |
||||
handler.ServeHTTP(rr, req) |
||||
|
||||
// Check the status code is what we expect.
|
||||
if status := rr.Code; status != testCase.expectedStatus { |
||||
t.Errorf("Test %d: handler returned wrong status code: got %v want %v", i+1, |
||||
status, testCase.expectedStatus) |
||||
} |
||||
|
||||
matched, err := regexp.MatchString(testCase.expectedLogRegexp, logOutput.String()) |
||||
if err != nil { |
||||
t.Fatalf("Test %d: Incorrect regexp: %v", i+1, err) |
||||
} |
||||
|
||||
if !matched { |
||||
t.Fatalf("Test %d: Unexpected log content, found: `%s`", i+1, logOutput.String()) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,92 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package madmin |
||||
|
||||
import ( |
||||
"bufio" |
||||
"encoding/json" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"strconv" |
||||
|
||||
trace "github.com/minio/minio/pkg/trace" |
||||
) |
||||
|
||||
// TraceInfo holds http trace
|
||||
type TraceInfo struct { |
||||
Trace trace.Info |
||||
Err error `json:"-"` |
||||
} |
||||
|
||||
// Trace - listen on http trace notifications.
|
||||
func (adm AdminClient) Trace(allTrace bool, doneCh <-chan struct{}) <-chan TraceInfo { |
||||
traceInfoCh := make(chan TraceInfo, 1) |
||||
// Only success, start a routine to start reading line by line.
|
||||
go func(traceInfoCh chan<- TraceInfo) { |
||||
defer close(traceInfoCh) |
||||
for { |
||||
urlValues := make(url.Values) |
||||
urlValues.Set("all", strconv.FormatBool(allTrace)) |
||||
reqData := requestData{ |
||||
relPath: "/v1/trace", |
||||
queryValues: urlValues, |
||||
} |
||||
// Execute GET to call trace handler
|
||||
resp, err := adm.executeMethod("GET", reqData) |
||||
if err != nil { |
||||
closeResponse(resp) |
||||
return |
||||
} |
||||
|
||||
if resp.StatusCode != http.StatusOK { |
||||
traceInfoCh <- TraceInfo{Err: httpRespToErrorResponse(resp)} |
||||
return |
||||
} |
||||
|
||||
// Initialize a new bufio scanner, to read line by line.
|
||||
bio := bufio.NewScanner(resp.Body) |
||||
|
||||
// Close the response body.
|
||||
defer resp.Body.Close() |
||||
|
||||
// Unmarshal each line, returns marshaled values.
|
||||
for bio.Scan() { |
||||
var traceRec trace.Info |
||||
if err = json.Unmarshal(bio.Bytes(), &traceRec); err != nil { |
||||
continue |
||||
} |
||||
select { |
||||
case <-doneCh: |
||||
return |
||||
case traceInfoCh <- TraceInfo{Trace: traceRec}: |
||||
} |
||||
} |
||||
// Look for any underlying errors.
|
||||
if err = bio.Err(); err != nil { |
||||
// For an unexpected connection drop from server, we close the body
|
||||
// and re-connect.
|
||||
if err == io.ErrUnexpectedEOF { |
||||
resp.Body.Close() |
||||
} |
||||
} |
||||
} |
||||
}(traceInfoCh) |
||||
|
||||
// Returns the trace info channel, for caller to start reading from.
|
||||
return traceInfoCh |
||||
} |
||||
@ -0,0 +1,50 @@ |
||||
// +build ignore
|
||||
|
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"log" |
||||
|
||||
"github.com/minio/minio/pkg/madmin" |
||||
) |
||||
|
||||
func main() { |
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||
// dummy values, please replace them with original values.
|
||||
|
||||
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
|
||||
// New returns an MinIO Admin client object.
|
||||
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) |
||||
if err != nil { |
||||
log.Fatalln(err) |
||||
} |
||||
doneCh := make(chan struct{}) |
||||
defer close(doneCh) |
||||
|
||||
// Start listening on all http trace activity from all servers
|
||||
// in the minio cluster.
|
||||
traceCh := madmClnt.Trace(false, doneCh) |
||||
for traceInfo := range traceCh { |
||||
if traceInfo.Err != nil { |
||||
fmt.Println(traceInfo.Err) |
||||
} |
||||
fmt.Println(traceInfo) |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package pubsub |
||||
|
||||
import ( |
||||
"sync" |
||||
) |
||||
|
||||
// PubSub holds publishers and subscribers
|
||||
type PubSub struct { |
||||
subs []chan interface{} |
||||
pub chan interface{} |
||||
mutex sync.Mutex |
||||
} |
||||
|
||||
// process item to subscribers.
|
||||
func (ps *PubSub) process() { |
||||
for item := range ps.pub { |
||||
ps.mutex.Lock() |
||||
for _, sub := range ps.subs { |
||||
go func(s chan interface{}) { |
||||
s <- item |
||||
}(sub) |
||||
} |
||||
ps.mutex.Unlock() |
||||
} |
||||
} |
||||
|
||||
// Publish message to pubsub system
|
||||
func (ps *PubSub) Publish(item interface{}) { |
||||
ps.pub <- item |
||||
} |
||||
|
||||
// Subscribe - Adds a subscriber to pubsub system
|
||||
func (ps *PubSub) Subscribe() chan interface{} { |
||||
ps.mutex.Lock() |
||||
defer ps.mutex.Unlock() |
||||
ch := make(chan interface{}) |
||||
ps.subs = append(ps.subs, ch) |
||||
return ch |
||||
} |
||||
|
||||
// Unsubscribe removes current subscriber
|
||||
func (ps *PubSub) Unsubscribe(ch chan interface{}) { |
||||
ps.mutex.Lock() |
||||
defer ps.mutex.Unlock() |
||||
|
||||
for i, sub := range ps.subs { |
||||
if sub == ch { |
||||
close(ch) |
||||
ps.subs = append(ps.subs[:i], ps.subs[i+1:]...) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// HasSubscribers returns true if pubsub system has subscribers
|
||||
func (ps *PubSub) HasSubscribers() bool { |
||||
ps.mutex.Lock() |
||||
defer ps.mutex.Unlock() |
||||
return len(ps.subs) > 0 |
||||
} |
||||
|
||||
// New inits a PubSub system
|
||||
func New() *PubSub { |
||||
ps := &PubSub{} |
||||
ps.pub = make(chan interface{}) |
||||
go ps.process() |
||||
return ps |
||||
} |
||||
@ -0,0 +1,66 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package pubsub |
||||
|
||||
import ( |
||||
"fmt" |
||||
"testing" |
||||
) |
||||
|
||||
func TestSubscribe(t *testing.T) { |
||||
ps := New() |
||||
ps.Subscribe() |
||||
ps.Subscribe() |
||||
if len(ps.subs) != 2 { |
||||
t.Errorf("expected 2 subscribers") |
||||
} |
||||
} |
||||
|
||||
func TestUnsubscribe(t *testing.T) { |
||||
ps := New() |
||||
c1 := ps.Subscribe() |
||||
ps.Subscribe() |
||||
ps.Unsubscribe(c1) |
||||
if len(ps.subs) != 1 { |
||||
t.Errorf("expected 1 subscriber") |
||||
} |
||||
} |
||||
|
||||
func TestPubSub(t *testing.T) { |
||||
ps := New() |
||||
c1 := ps.Subscribe() |
||||
val := "hello" |
||||
ps.Publish(val) |
||||
msg := <-c1 |
||||
if msg != "hello" { |
||||
t.Errorf(fmt.Sprintf("expected %s , found %s", val, msg)) |
||||
} |
||||
} |
||||
|
||||
func TestMultiPubSub(t *testing.T) { |
||||
ps := New() |
||||
c1 := ps.Subscribe() |
||||
c2 := ps.Subscribe() |
||||
val := "hello" |
||||
ps.Publish(val) |
||||
|
||||
msg1 := <-c1 |
||||
msg2 := <-c2 |
||||
if msg1 != "hello" && msg2 != "hello" { |
||||
t.Errorf(fmt.Sprintf("expected both subscribers to have%s , found %s and %s", val, msg1, msg2)) |
||||
} |
||||
} |
||||
@ -0,0 +1,49 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package trace |
||||
|
||||
import ( |
||||
"net/http" |
||||
"time" |
||||
) |
||||
|
||||
// Info - represents a trace record, additionally
|
||||
// also reports errors if any while listening on trace.
|
||||
type Info struct { |
||||
NodeName string `json:"nodename"` |
||||
FuncName string `json:"funcname"` |
||||
ReqInfo RequestInfo `json:"request"` |
||||
RespInfo ResponseInfo `json:"response"` |
||||
} |
||||
|
||||
// RequestInfo represents trace of http request
|
||||
type RequestInfo struct { |
||||
Time time.Time `json:"time"` |
||||
Method string `json:"method"` |
||||
Path string `json:"path,omitempty"` |
||||
RawQuery string `json:"rawquery,omitempty"` |
||||
Headers http.Header `json:"headers,omitempty"` |
||||
Body []byte `json:"body,omitempty"` |
||||
} |
||||
|
||||
// ResponseInfo represents trace of http request
|
||||
type ResponseInfo struct { |
||||
Time time.Time `json:"time"` |
||||
Headers http.Header `json:"headers,omitempty"` |
||||
Body []byte `json:"body,omitempty"` |
||||
StatusCode int `json:"statuscode,omitempty"` |
||||
} |
||||
Loading…
Reference in new issue