From 5b123813f8ac912e0140c41c0b19250e72fb4c26 Mon Sep 17 00:00:00 2001 From: katherine Date: Wed, 9 Oct 2019 07:39:19 -0700 Subject: [PATCH] Remove deprecated RocksDBCommonHelper and cont_integration.sh (#5889) Summary: As titled. RocksDBCommonHelper contains references to legacy APIs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5889 Differential Revision: D17783179 fbshipit-source-id: dcde82a73a311bfa3300ad69189b3a32727134d1 --- build_tools/RocksDBCommonHelper.php | 377 ---------------------------- build_tools/cont_integration.sh | 137 ---------- 2 files changed, 514 deletions(-) delete mode 100644 build_tools/RocksDBCommonHelper.php delete mode 100755 build_tools/cont_integration.sh diff --git a/build_tools/RocksDBCommonHelper.php b/build_tools/RocksDBCommonHelper.php deleted file mode 100644 index e7bfb5203..000000000 --- a/build_tools/RocksDBCommonHelper.php +++ /dev/null @@ -1,377 +0,0 @@ - 0); - assert(is_numeric($diffID)); - assert(strlen($url) > 0); - - $cmd_args = array( - 'diff_id' => (int)$diffID, - 'name' => sprintf( - 'click here for sandcastle tests for D%d', - (int)$diffID - ), - 'link' => $url - ); - $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args)) - . ' | arc call-conduit differential.updateunitresults'; - - shell_exec($cmd); -} - -function buildUpdateTestStatusCmd($diffID, $test, $status) { - assert(strlen($diffID) > 0); - assert(is_numeric($diffID)); - assert(strlen($test) > 0); - assert(strlen($status) > 0); - - $cmd_args = array( - 'diff_id' => (int)$diffID, - 'name' => $test, - 'result' => $status - ); - - $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args)) - . ' | arc call-conduit differential.updateunitresults'; - - return $cmd; -} - -function updateTestStatus($diffID, $test) { - assert(strlen($diffID) > 0); - assert(is_numeric($diffID)); - assert(strlen($test) > 0); - - shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting")); -} - -function getSteps($applyDiff, $diffID, $username, $test) { - assert(strlen($username) > 0); - assert(strlen($test) > 0); - - if ($applyDiff) { - assert(strlen($diffID) > 0); - assert(is_numeric($diffID)); - - $arcrc_content = (PHP_OS == "Darwin" ? - exec("cat ~/.arcrc | gzip -f | base64") : - exec("cat ~/.arcrc | gzip -f | base64 -w0")); - assert(strlen($arcrc_content) > 0); - - // Sandcastle machines don't have arc setup. We copy the user certificate - // and authenticate using that in Sandcastle. - $setup = array( - "name" => "Setup arcrc", - "shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode" - . " | gzip -d > ~/.arcrc", - "user" => "root" - ); - - // arc demands certain permission on its config. - // also fix the sticky bit issue in sandcastle - $fix_permission = array( - "name" => "Fix environment", - "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm", - "user" => "root" - ); - - // Construct the steps in the order of execution. - $steps[] = $setup; - $steps[] = $fix_permission; - } - - // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise - // Git thinks it is an uncommitted change. - $fix_git_ignore = array( - "name" => "Fix git ignore", - "shell" => "echo fbcode >> .git/info/exclude", - "user" => "root" - ); - - // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory" - // Source: - // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ - $fix_kernel_issue = array( - "name" => "Fix kernel issue with tsan", - "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space", - "user" => "root" - ); - - $steps[] = $fix_git_ignore; - $steps[] = $fix_kernel_issue; - - // This will be the command used to execute particular type of tests. - $cmd = ""; - - if ($applyDiff) { - // Patch the code (keep your fingures crossed). - $patch = array( - "name" => "Patch " . $diffID, - "shell" => "arc --arcrc-file ~/.arcrc " - . "patch --nocommit --diff " . escapeshellarg($diffID), - "user" => "root" - ); - - $steps[] = $patch; - - updateTestStatus($diffID, $test); - $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; "; - } - - // Run the actual command. - $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . - escapeshellarg($test) . "; exit_code=$?; "; - - if ($applyDiff) { - $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&" - . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")" - . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail") - . "; "; - } - - // shell command to sort the tests based on exit code and print - // the output of the log files. - $cat_sorted_logs = " - while read code log_file; - do echo \"################ cat \$log_file [exit_code : \$code] ################\"; - cat \$log_file; - done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')"; - - // Shell command to cat all log files - $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done"; - - // If LOG file exist use it to cat log files sorted by exit code, otherwise - // cat everything - $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi"; - - $cmd = $cmd . " cat /tmp/precommit-check.log" - . "; shopt -s extglob; {$logs_cmd}" - . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]"; - assert(strlen($cmd) > 0); - - $run_test = array( - "name" => "Run " . $test, - "shell" => $cmd, - "user" => "root", - "parser" => "python build_tools/error_filter.py " . escapeshellarg($test), - ); - - $steps[] = $run_test; - - if ($applyDiff) { - // Clean up the user arc config we are using. - $cleanup = array( - "name" => "Arc cleanup", - "shell" => "rm -f ~/.arcrc", - "user" => "root" - ); - - $steps[] = $cleanup; - } - - assert(count($steps) > 0); - return $steps; -} - -function getSandcastleConfig() { - $sandcastle_config = array(); - - $cwd = getcwd(); - $cwd_token_file = "{$cwd}/.sandcastle"; - // This is a case when we're executed from a continuous run. Fetch the values - // from the environment. - if (getenv(ENV_POST_RECEIVE_HOOK)) { - $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE); - $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE); - } else { - // This is a typical `[p]arc diff` case. Fetch the values from the specific - // configuration files. - for ($i = 0; $i < 50; $i++) { - if (file_exists(PRIMARY_TOKEN_FILE) || - file_exists($cwd_token_file)) { - break; - } - // If we failed to fetch the tokens, sleep for 0.2 second and try again - usleep(200000); - } - assert(file_exists(PRIMARY_TOKEN_FILE) || - file_exists($cwd_token_file)); - - // Try the primary location first, followed by a secondary. - if (file_exists(PRIMARY_TOKEN_FILE)) { - $cmd = 'cat ' . PRIMARY_TOKEN_FILE; - } else { - $cmd = 'cat ' . escapeshellarg($cwd_token_file); - } - - assert(strlen($cmd) > 0); - $sandcastle_config = explode(':', rtrim(shell_exec($cmd))); - } - - // In this case be very explicit about the implications. - if (count($sandcastle_config) != 2) { - echo "Sandcastle configuration files don't contain valid information " . - "or the necessary environment variables aren't defined. Unable " . - "to validate the code changes."; - exit(1); - } - - assert(strlen($sandcastle_config[0]) > 0); - assert(strlen($sandcastle_config[1]) > 0); - assert(count($sandcastle_config) > 0); - - return $sandcastle_config; -} - -// This function can be called either from `[p]arc diff` command or during -// the Git post-receive hook. - function startTestsInSandcastle($applyDiff, $workflow, $diffID) { - // Default options don't terminate on failure, but that's what we want. In - // the current case we use assertions intentionally as "terminate on failure - // invariants". - assert_options(ASSERT_BAIL, true); - - // In case of a diff we'll send notificatios to the author. Else it'll go to - // the entire team because failures indicate that build quality has regressed. - $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS; - assert(strlen($username) > 0); - - if ($applyDiff) { - assert($workflow); - assert(strlen($diffID) > 0); - assert(is_numeric($diffID)); - } - - // List of tests we want to run in Sandcastle. - $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan", - "asan", "lite_test", "valgrind", "release", "release_481", - "clang_release", "clang_analyze", "code_cov", - "java_build", "no_compression", "unity", "ubsan"); - - $send_email_template = array( - 'type' => 'email', - 'triggers' => array('fail'), - 'emails' => array($username . '@fb.com'), - ); - - // Construct a job definition for each test and add it to the master plan. - foreach ($tests as $test) { - $stepName = "RocksDB diff " . $diffID . " test " . $test; - - if (!$applyDiff) { - $stepName = "RocksDB continuous integration test " . $test; - } - - $arg[] = array( - "name" => $stepName, - "report" => array($send_email_template), - "steps" => getSteps($applyDiff, $diffID, $username, $test) - ); - } - - // We cannot submit the parallel execution master plan to Sandcastle and - // need supply the job plan as a determinator. So we construct a small job - // that will spit out the master job plan which Sandcastle will parse and - // execute. Why compress the job definitions? Otherwise we run over the max - // string size. - $cmd = "echo " . base64_encode(json_encode($arg)) - . (PHP_OS == "Darwin" ? - " | gzip -f | base64" : - " | gzip -f | base64 -w0"); - assert(strlen($cmd) > 0); - - $arg_encoded = shell_exec($cmd); - assert(strlen($arg_encoded) > 0); - - $runName = "Run diff " . $diffID . "for user " . $username; - - if (!$applyDiff) { - $runName = "RocksDB continuous integration build and test run"; - } - - $command = array( - "name" => $runName, - "steps" => array() - ); - - $command["steps"][] = array( - "name" => "Generate determinator", - "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" - . " | base64 --decode", - "determinator" => true, - "user" => "root" - ); - - // Submit to Sandcastle. - $url = 'https://interngraph.intern.facebook.com/sandcastle/create'; - - $job = array( - 'command' => 'SandcastleUniversalCommand', - 'args' => $command, - 'capabilities' => array( - 'vcs' => 'rocksdb-int-git', - 'type' => 'lego', - ), - 'hash' => 'origin/master', - 'user' => $username, - 'alias' => 'rocksdb-precommit', - 'tags' => array('rocksdb'), - 'description' => 'Rocksdb precommit job', - ); - - // Fetch the configuration necessary to submit a successful HTTPS request. - $sandcastle_config = getSandcastleConfig(); - - $app = $sandcastle_config[0]; - $token = $sandcastle_config[1]; - - $cmd = 'curl -s -k ' - . ' -F app=' . escapeshellarg($app) - . ' -F token=' . escapeshellarg($token) - . ' -F job=' . escapeshellarg(json_encode($job)) - .' ' . escapeshellarg($url); - - $output = shell_exec($cmd); - assert(strlen($output) > 0); - - // Extract Sandcastle URL from the response. - preg_match('/url": "(.+)"/', $output, $sandcastle_url); - - assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request."); - assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL."); - - if ($applyDiff) { - echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; - // Ask Phabricator to display it on the diff UI. - postURL($diffID, $sandcastle_url[1]); - } else { - echo "Continuous integration started Sandcastle tests. You can look at "; - echo "the progress at:\n" . $sandcastle_url[1] . "\n"; - } -} - -// Continuous run cript will set the environment variable and based on that -// we'll trigger the execution of tests in Sandcastle. In that case we don't -// need to apply any diffs and there's no associated workflow either. -if (getenv(ENV_POST_RECEIVE_HOOK)) { - startTestsInSandcastle( - false /* $applyDiff */, - NULL /* $workflow */, - NULL /* $diffID */); -} diff --git a/build_tools/cont_integration.sh b/build_tools/cont_integration.sh deleted file mode 100755 index 66d255227..000000000 --- a/build_tools/cont_integration.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016, Facebook. All rights reserved. -# -# Overall wrapper script for RocksDB continuous builds. The implementation is a -# trivial pulling scheme. We loop infinitely, check if any new changes have been -# committed, if yes then trigger a Sandcastle run, and finally go to sleep again -# for a certain interval. -# - -SRC_GIT_REPO=/data/git/rocksdb-public -error=0 - -function log { - DATE=`date +%Y-%m-%d:%H:%M:%S` - # shellcheck disable=SC2068 - echo $DATE $@ -} - -function log_err { - # shellcheck disable=SC2145 - log "ERROR: $@ Error code: $error." -} - -function update_repo_status { - # Update the parent first. - pushd $SRC_GIT_REPO - - # This is a fatal error. Something in the environment isn't right and we will - # terminate the execution. - error=$? - if [ ! $error -eq 0 ]; then - log_err "Where is $SRC_GIT_REPO?" - exit $error - fi - - HTTPS_PROXY=fwdproxy:8080 git fetch -f - - error=$? - if [ ! $error -eq 0 ]; then - log_err "git fetch -f failed." - popd - return $error - fi - - git update-ref refs/heads/master refs/remotes/origin/master - - error=$? - if [ ! $error -eq 0 ]; then - log_err "git update-ref failed." - popd - return $error - fi - - popd - - # We're back in an instance-specific directory. Get the latest changes. - git pull --rebase - - error=$? - if [ ! $error -eq 0 ]; then - log_err "git pull --rebase failed." - return $error - fi -} - -# -# Execution starts here. -# - -# Path to the determinator from the root of the RocksDB repo. -CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php - -# Value of the previous commit. -PREV_COMMIT= - -log "Starting to monitor for new RocksDB changes ..." -log "Running under `pwd` as `whoami`." - -# Paranoia. Make sure that we're using the right branch. -git checkout master - -error=$? -if [ ! $error -eq 0 ]; then - log_err "This is not good. Can't checkout master. Bye-bye!" - exit 1 -fi - -# We'll run forever and let the execution environment terminate us if we'll -# exceed whatever timeout is set for the job. -while true; -do - # Get the latest changes committed. - update_repo_status - - error=$? - if [ $error -eq 0 ]; then - LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'` - - log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'." - - if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then - log "There were no changes since the last time I checked. Going to sleep." - else - if [ ! -z "$LAST_COMMIT" ]; then - log "New code has been committed or previous commit not known. " \ - "Will trigger the tests." - - PREV_COMMIT=$LAST_COMMIT - log "Updated previous commit to '$PREV_COMMIT'." - - # - # This is where we'll trigger the Sandcastle run. The values for - # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're - # running in. - # - POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR - - error=$? - if [ $error -eq 0 ]; then - log "Sandcastle run successfully triggered." - else - log_err "Failed to trigger Sandcastle run." - fi - else - log_err "Previous commit not updated. Don't know what the last one is." - fi - fi - else - log_err "Getting latest changes failed. Will skip running tests for now." - fi - - # Always sleep, even if errors happens while trying to determine the latest - # commit. This will prevent us terminating in case of transient errors. - log "Will go to sleep for 5 minutes." - sleep 5m -done