diff --git a/tools/check_format_compatible.sh b/tools/check_format_compatible.sh index 39335446d..5959fb832 100755 --- a/tools/check_format_compatible.sh +++ b/tools/check_format_compatible.sh @@ -17,7 +17,7 @@ mkdir $input_data_path || true rm -rf $script_copy_dir cp $scriptpath $script_copy_dir -rf -# Generate four random files. +# Generate random files. for i in {1..6} do input_data[$i]=$input_data_path/data$i @@ -41,10 +41,22 @@ with open('${input_data[$i]}', 'w') as f: EOF done +# Generate file(s) with sorted keys. +sorted_input_data=$input_data_path/sorted_data +echo == Generating file with sorted keys ${sorted_input_data} +python - <> f, k + " ==> " + v +EOF + declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb") declare -a forward_compatible_checkout_objs=("3.10.fb" "3.11.fb" "3.12.fb" "3.13.fb" "4.0.fb" "4.1.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb") declare -a forward_compatible_with_options_checkout_objs=("5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb") declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]}) +declare -a extern_sst_ingestion_compatible_checkout_objs=("5.14.fb" "5.15.fb") generate_db() { @@ -68,6 +80,28 @@ compare_db() set -e } +write_external_sst() +{ + set +e + $script_copy_dir/write_external_sst.sh $1 $2 $3 + if [ $? -ne 0 ]; then + echo ==== Error writing external SST file using data from $1 to $3 ==== + exit 1 + fi + set -e +} + +ingest_external_sst() +{ + set +e + $script_copy_dir/ingest_external_sst.sh $1 $2 + if [ $? -ne 0 ]; then + echo ==== Error ingesting external SST in $2 to DB at $1 ==== + exit 1 + fi + set -e +} + # Sandcastle sets us up with a remote that is just another directory on the same # machine and doesn't have our branches. Need to fetch them so checkout works. # Remote add may fail if added previously (we don't cleanup). @@ -75,6 +109,41 @@ git remote add github_origin "https://github.com/facebook/rocksdb.git" set -e https_proxy="fwdproxy:8080" git fetch github_origin +# Compatibility test for external SST file ingestion +for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" +do + echo == Generating DB with extern SST file in "$checkout_obj" ... + https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj + make clean + make ldb -j32 + write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj + ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj +done + +checkout_flag=${1:-"master"} + +echo == Building $checkout_flag debug +https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag +make clean +make ldb -j32 +compare_base_db_dir=$test_dir"/base_db_dir" +write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir +ingest_external_sst $compare_base_db_dir $compare_base_db_dir + +for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" +do + echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag + git checkout $checkout_obj + make clean + make ldb -j32 + compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1 + git checkout tmp-$checkout_flag + # Clean up + git branch -D $checkout_obj +done + +echo == Finish compatibility test for SST ingestion. + for checkout_obj in "${checkout_objs[@]}" do echo == Generating DB from "$checkout_obj" ... @@ -87,7 +156,7 @@ done checkout_flag=${1:-"master"} echo == Building $checkout_flag debug -https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag +git checkout tmp-$checkout_flag make clean make ldb -j32 compare_base_db_dir=$test_dir"/base_db_dir" diff --git a/tools/ingest_external_sst.sh b/tools/ingest_external_sst.sh new file mode 100755 index 000000000..54ca3db3b --- /dev/null +++ b/tools/ingest_external_sst.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# + +if [ "$#" -lt 2 ]; then + echo "usage: $BASH_SOURCE " + exit 1 +fi + +db_dir=$1 +external_sst_dir=$2 + +for f in `find $external_sst_dir -name extern_sst*` +do + echo == Ingesting external SST file $f to DB at $db_dir + ./ldb --db=$db_dir --create_if_missing ingest_extern_sst $f +done diff --git a/tools/write_external_sst.sh b/tools/write_external_sst.sh new file mode 100755 index 000000000..d50447c52 --- /dev/null +++ b/tools/write_external_sst.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# +# + +if [ "$#" -lt 3 ]; then + echo "usagee: $BASH_SOURCE " + exit 1 +fi + +input_data_dir=$1 +db_dir=$2 +extern_sst_dir=$3 +rm -rf $db_dir + +set -e + +n=0 + +for f in `find $input_data_dir -name sorted_data*` +do + echo == Writing external SST file $f to $extern_sst_dir/extern_sst${n} + ./ldb --db=$db_dir --create_if_missing write_extern_sst $extern_sst_dir/extern_sst${n} < $f + let "n = n + 1" +done