From d955ce412339136b5093338922552ae8f5d7eb63 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 28 Dec 2015 17:43:28 -0800 Subject: [PATCH] s3cmd: Fix signature issues related to s3cmd. Support regions both 'us-east-1' and 'US' (short hand for US Standard) honored by S3. --- README.md | 4 +- api-signature.go | 84 ++++++++++++++++++++++++--------- pkg/fs/fs-bucket-listobjects.go | 15 ++++-- pkg/fs/signature.go | 5 +- 4 files changed, 79 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index b65525ced..6881c9d73 100644 --- a/README.md +++ b/README.md @@ -138,13 +138,13 @@ secret_key = YOUR_SECRET_KEY_HERE To list your buckets. ``` -$ s3cmd --region us-east-1 ls s3:// +$ s3cmd ls s3:// 2015-12-09 16:12 s3://testbbucket ``` To list contents inside buckets. ``` -$ s3cmd --region us-east-1 ls s3://testbucket/ +$ s3cmd ls s3://testbucket/ DIR s3://testbucket/test/ 2015-12-09 16:05 138504 s3://testbucket/newfile ``` diff --git a/api-signature.go b/api-signature.go index 5e41e4d6d..3bdb947b0 100644 --- a/api-signature.go +++ b/api-signature.go @@ -57,12 +57,6 @@ func getCredentialsFromAuth(authValue string) ([]string, *probe.Error) { if len(credentials) != 2 { return nil, probe.NewError(errMissingFieldsCredentialTag) } - if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 { - return nil, probe.NewError(errMissingFieldsSignedHeadersTag) - } - if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 { - return nil, probe.NewError(errMissingFieldsSignatureTag) - } credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/") if len(credentialElements) != 5 { return nil, probe.NewError(errCredentialTagMalformed) @@ -70,24 +64,55 @@ func getCredentialsFromAuth(authValue string) ([]string, *probe.Error) { return credentialElements, nil } -// verify if authHeader value has valid region -func isValidRegion(authHeaderValue string) *probe.Error { +func getSignatureFromAuth(authHeaderValue string) (string, *probe.Error) { + authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix) + authFields := strings.Split(strings.TrimSpace(authValue), ",") + if len(authFields) != 3 { + return "", probe.NewError(errInvalidAuthHeaderValue) + } + if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 { + return "", probe.NewError(errMissingFieldsSignatureTag) + } + signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1] + return signature, nil +} + +func getSignedHeadersFromAuth(authHeaderValue string) ([]string, *probe.Error) { + authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix) + authFields := strings.Split(strings.TrimSpace(authValue), ",") + if len(authFields) != 3 { + return nil, probe.NewError(errInvalidAuthHeaderValue) + } + if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 { + return nil, probe.NewError(errMissingFieldsSignedHeadersTag) + } + signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";") + return signedHeaders, nil +} + +// verify if region value is valid. +func isValidRegion(region string) *probe.Error { + if region != "us-east-1" && region != "US" { + return probe.NewError(errInvalidRegion) + } + return nil +} + +// stripRegion - strip only region from auth header. +func stripRegion(authHeaderValue string) (string, *probe.Error) { credentialElements, err := getCredentialsFromAuth(authHeaderValue) if err != nil { - return err.Trace() + return "", err.Trace(authHeaderValue) } region := credentialElements[2] - if region != "us-east-1" { - return probe.NewError(errInvalidRegion) + if err = isValidRegion(region); err != nil { + return "", err.Trace(authHeaderValue) } - return nil + return region, nil } -// stripAccessKeyID - strip only access key id from auth header +// stripAccessKeyID - strip only access key id from auth header. func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) { - if err := isValidRegion(authHeaderValue); err != nil { - return "", err.Trace() - } credentialElements, err := getCredentialsFromAuth(authHeaderValue) if err != nil { return "", err.Trace() @@ -99,25 +124,36 @@ func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) { return accessKeyID, nil } -// initSignatureV4 initializing signature verification +// initSignatureV4 initializing signature verification. func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) { - // strip auth from authorization header + // strip auth from authorization header. authHeaderValue := req.Header.Get("Authorization") + + region, err := stripRegion(authHeaderValue) + if err != nil { + return nil, err.Trace(authHeaderValue) + } accessKeyID, err := stripAccessKeyID(authHeaderValue) if err != nil { - return nil, err.Trace() + return nil, err.Trace(authHeaderValue) + } + signature, err := getSignatureFromAuth(authHeaderValue) + if err != nil { + return nil, err.Trace(authHeaderValue) + } + signedHeaders, err := getSignedHeadersFromAuth(authHeaderValue) + if err != nil { + return nil, err.Trace(authHeaderValue) } config, err := loadConfigV2() if err != nil { return nil, err.Trace() } - authFields := strings.Split(strings.TrimSpace(authHeaderValue), ",") - signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";") - signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1] if config.Credentials.AccessKeyID == accessKeyID { signature := &fs.Signature{ AccessKeyID: config.Credentials.AccessKeyID, SecretAccessKey: config.Credentials.SecretAccessKey, + Region: region, Signature: signature, SignedHeaders: signedHeaders, Request: req, @@ -216,10 +252,12 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr if perr != nil { return nil, perr.Trace() } + region := credentialElements[2] if config.Credentials.AccessKeyID == accessKeyID { signature := &fs.Signature{ AccessKeyID: config.Credentials.AccessKeyID, SecretAccessKey: config.Credentials.SecretAccessKey, + Region: region, Signature: formValues["X-Amz-Signature"], PresignedPolicy: formValues["Policy"], } @@ -242,12 +280,14 @@ func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) { if err != nil { return nil, err.Trace() } + region := credentialElements[2] signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";") signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature")) if config.Credentials.AccessKeyID == accessKeyID { signature := &fs.Signature{ AccessKeyID: config.Credentials.AccessKeyID, SecretAccessKey: config.Credentials.SecretAccessKey, + Region: region, Signature: signature, SignedHeaders: signedHeaders, Presigned: true, diff --git a/pkg/fs/fs-bucket-listobjects.go b/pkg/fs/fs-bucket-listobjects.go index 323aba12c..1e32b4e03 100644 --- a/pkg/fs/fs-bucket-listobjects.go +++ b/pkg/fs/fs-bucket-listobjects.go @@ -100,10 +100,19 @@ func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadat FileInfo: fl, }) } else { - files, err := ioutil.ReadDir(filepath.Join(rootPrefix, resources.Prefix)) + var prefixPath string + if runtime.GOOS == "windows" { + prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix + } else { + prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix + } + files, err := ioutil.ReadDir(prefixPath) if err != nil { - if os.IsNotExist(err) { - return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) + switch err := err.(type) { + case *os.PathError: + if err.Op == "open" { + return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) + } } return nil, resources, probe.NewError(err) } diff --git a/pkg/fs/signature.go b/pkg/fs/signature.go index 355fd933c..ba40a5030 100644 --- a/pkg/fs/signature.go +++ b/pkg/fs/signature.go @@ -37,6 +37,7 @@ import ( type Signature struct { AccessKeyID string SecretAccessKey string + Region string Presigned bool PresignedPolicy string SignedHeaders []string @@ -210,7 +211,7 @@ func (r Signature) getPresignedCanonicalRequest(presignedQuery string) string { func (r Signature) getScope(t time.Time) string { scope := strings.Join([]string{ t.Format(yyyymmdd), - "us-east-1", + r.Region, "s3", "aws4_request", }, "/") @@ -229,7 +230,7 @@ func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string func (r Signature) getSigningKey(t time.Time) []byte { secret := r.SecretAccessKey date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) - region := sumHMAC(date, []byte("us-east-1")) + region := sumHMAC(date, []byte(r.Region)) service := sumHMAC(region, []byte("s3")) signingKey := sumHMAC(service, []byte("aws4_request")) return signingKey