You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
minio/mint/run/core/s3cmd/test.sh

422 lines
11 KiB

#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
if [ -n "$MINT_MODE" ]; then
if [ -z "${MINT_DATA_DIR+x}" ]; then
echo "MINT_DATA_DIR not defined"
exit 1
fi
if [ -z "${SERVER_ENDPOINT+x}" ]; then
echo "SERVER_ENDPOINT not defined"
exit 1
fi
if [ -z "${ACCESS_KEY+x}" ]; then
echo "ACCESS_KEY not defined"
exit 1
fi
if [ -z "${SECRET_KEY+x}" ]; then
echo "SECRET_KEY not defined"
exit 1
fi
fi
if [ -z "${SERVER_ENDPOINT+x}" ]; then
SERVER_ENDPOINT="play.minio.io:9000"
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
ENABLE_HTTPS=1
SERVER_REGION="us-east-1"
fi
WORK_DIR="$PWD"
DATA_DIR="$MINT_DATA_DIR"
if [ -z "$MINT_MODE" ]; then
WORK_DIR="$PWD/.run-$RANDOM"
DATA_DIR="$WORK_DIR/data"
fi
FILE_1_MB="$DATA_DIR/datafile-1-MB"
FILE_65_MB="$DATA_DIR/datafile-65-MB"
declare FILE_1_MB_MD5SUM
declare FILE_65_MB_MD5SUM
BUCKET_NAME="s3cmd-test-bucket-$RANDOM"
S3CMD=$(which s3cmd)
declare -a S3CMD_CMD
function get_md5sum()
{
filename="$FILE_1_MB"
out=$(md5sum "$filename" 2>/dev/null)
rv=$?
if [ "$rv" -eq 0 ]; then
awk '{ print $1 }' <<< "$out"
fi
return "$rv"
}
function get_time()
{
date +%s%N
}
function get_duration()
{
start_time=$1
end_time=$(get_time)
echo $(( (end_time - start_time) / 1000000 ))
}
function log_success()
{
if [ -n "$MINT_MODE" ]; then
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2"
fi
}
function show()
{
if [ -z "$MINT_MODE" ]; then
func_name="$1"
echo "Running $func_name()"
fi
}
function fail()
{
rv="$1"
shift
if [ "$rv" -ne 0 ]; then
echo "$@"
fi
return "$rv"
}
function assert()
{
expected_rv="$1"
shift
start_time="$1"
shift
func_name="$1"
shift
err=$("$@" 2>&1)
rv=$?
if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then
if [ -n "$MINT_MODE" ]; then
err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))')
## err is already JSON string, no need to double quote
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err"
else
echo "s3cmd: $func_name: $err"
fi
exit "$rv"
fi
return 0
}
function assert_success() {
assert 0 "$@"
}
function assert_failure() {
assert 1 "$@"
}
function s3cmd_cmd()
{
cmd=( "${S3CMD_CMD[@]}" "$@" )
"${cmd[@]}"
rv=$?
return "$rv"
}
function check_md5sum()
{
expected_checksum="$1"
shift
filename="$*"
checksum="$(get_md5sum "$filename")"
rv=$?
if [ "$rv" -ne 0 ]; then
echo "unable to get md5sum for $filename"
return "$rv"
fi
if [ "$checksum" != "$expected_checksum" ]; then
echo "$filename: md5sum mismatch"
return 1
fi
return 0
}
function test_make_bucket()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="s3cmd-test-bucket-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_make_bucket_error() {
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="S3CMD-test%bucket%$RANDOM"
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function setup()
{
start_time=$(get_time)
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}"
}
function teardown()
{
start_time=$(get_time)
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}"
}
function test_put_object()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_error()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_long_name=$(printf "s3cmd-test-object-%01100d" 1)
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_multipart()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object_error()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object_multipart()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_copy_object()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name1="s3cmd-test-object-$RANDOM"
object_name2="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name1}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd cp "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd info "s3://${BUCKET_NAME}/${object_name2}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_sync_list_objects()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="s3cmd-test-bucket-$RANDOM"
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}"
diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1
assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function run_test()
{
test_make_bucket
test_make_bucket_error
setup
test_put_object
test_put_object_error
test_put_object_multipart
test_get_object
test_get_object_multipart
test_copy_object
test_sync_list_objects
teardown
}
function __init__()
{
set -e
S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM"
mkdir -p $S3CMD_CONFIG_DIR
S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg"
# configure s3cmd
cat > $S3CMD_CONFIG_FILE <<EOF
signature_v2 = False
host_base = $SERVER_ENDPOINT
host_bucket = $SERVER_ENDPOINT
bucket_location = $SERVER_REGION
use_https = $ENABLE_HTTPS
access_key = $ACCESS_KEY
secret_key = $SECRET_KEY
EOF
# For Mint, setup is already done. For others, setup the environment
if [ -z "$MINT_MODE" ]; then
mkdir -p "$WORK_DIR"
mkdir -p "$DATA_DIR"
# If s3cmd executable binary is not available in current directory, use it in the path.
if [ ! -x "$S3CMD" ]; then
echo "'s3cmd' executable binary not found in current directory and in path"
exit 1
fi
fi
if [ ! -x "$S3CMD" ]; then
echo "$S3CMD executable binary not found"
exit 1
fi
S3CMD_CMD=( "${S3CMD}" --config "$S3CMD_CONFIG_FILE" )
if [ ! -e "$FILE_1_MB" ]; then
shred -n 1 -s 1MB - >"$FILE_1_MB"
fi
if [ ! -e "$FILE_65_MB" ]; then
shred -n 1 -s 65MB - >"$FILE_65_MB"
fi
set -E
set -o pipefail
FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")"
rv=$?
if [ $rv -ne 0 ]; then
echo "unable to get md5sum of $FILE_1_MB"
exit 1
fi
FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")"
rv=$?
if [ $rv -ne 0 ]; then
echo "unable to get md5sum of $FILE_65_MB"
exit 1
fi
set +e
}
function main()
{
( run_test )
rv=$?
rm -fr "$S3CMD_CONFIG_FILE"
if [ -z "$MINT_MODE" ]; then
rm -fr "$WORK_DIR" "$DATA_DIR"
fi
exit "$rv"
}
__init__ "$@"
main "$@"