diff --git a/arcanist_util/config/FacebookArcanistConfiguration.php b/arcanist_util/config/FacebookArcanistConfiguration.php index f0f9653a2..d82f9a873 100644 --- a/arcanist_util/config/FacebookArcanistConfiguration.php +++ b/arcanist_util/config/FacebookArcanistConfiguration.php @@ -3,204 +3,34 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. + +require('RocksDBCommonHelper.php'); + +define("DIFF_COMMAND", "diff"); + class FacebookArcanistConfiguration extends ArcanistConfiguration { public function didRunWorkflow($command, ArcanistWorkflow $workflow, $error_code) { - if ($command == 'diff' && !$workflow->isRawDiffSource()) { - $this->startTestsInSandcastle($workflow); - } - } + // Default options don't terminate on failure, but that's what we want. In + // the current case we use assertions intentionally as "terminate on failure + // invariants". + assert_options(ASSERT_BAIL, true); - ////////////////////////////////////////////////////////////////////// - /* Run tests in sandcastle */ - function postURL($diffID, $url) { - $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' - . '"name":"click here for sandcastle tests for D' . $diffID . '", ' - . '"link":"' . $url . '"}\' | ' - . 'http_proxy=fwdproxy.any.facebook.com:8080 ' - . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' - . 'differential.updateunitresults'; - shell_exec($cmd); - } + assert($workflow); + assert(strlen($command) > 0); - function updateTestCommand($diffID, $test, $status) { - $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' - . '"name":"' . $test . '", ' - . '"result":"' . $status . '"}\' | ' - . 'http_proxy=fwdproxy.any.facebook.com:8080 ' - . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' - . 'differential.updateunitresults'; - return $cmd; - } + if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) { + $diffID = $workflow->getDiffId(); - function updateTest($diffID, $test) { - shell_exec($this->updateTestCommand($diffID, $test, "waiting")); - } - - function getSteps($diffID, $username, $test) { - $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0"); - - // Sandcastle machines don't have arc setup. We copy the user certificate - // and authenticate using that in sandcastle - $setup = array( - "name" => "Setup arcrc", - "shell" => "echo " . $arcrc_content . " | base64 --decode" - . " | gzip -d > ~/.arcrc", - "user" => "root" - ); - - // arc demands certain permission on its config - $fix_permission = array( - "name" => "Fix environment", - "shell" => "chmod 600 ~/.arcrc", - "user" => "root" - ); - - // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise - // git thinks it is uncommited change - $fix_git_ignore = array( - "name" => "Fix git ignore", - "shell" => "echo fbcode >> .git/info/exclude", - "user" => "root" - ); - - // Patch the code (keep your fingures crossed) - $patch = array( - "name" => "Patch " . $diffID, - "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc " - . "patch --nocommit --diff " . $diffID, - "user" => "root" - ); - - // Clean up the user arc config we are using - $cleanup = array( - "name" => "Arc cleanup", - "shell" => "rm -f ~/.arcrc", - "user" => "root" - ); - - // Construct the steps in the order of execution - $steps[] = $setup; - $steps[] = $fix_permission; - $steps[] = $fix_git_ignore; - $steps[] = $patch; - - // Run the actual command - $this->updateTest($diffID, $test); - $cmd = $this->updateTestCommand($diffID, $test, "running") . ";" - . "./build_tools/precommit_checker.py " . $test - . "; exit_code=$?; ([[ \$exit_code -eq 0 ]] &&" - . $this->updateTestCommand($diffID, $test, "pass") . ")" - . "||" . $this->updateTestCommand($diffID, $test, "fail") - . "; cat /tmp/precommit-check.log" - . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;" - . "[[ \$exit_code -eq 0 ]]"; - - $run_test = array( - "name" => "Run " . $test, - "shell" => $cmd, - "user" => "root", - ); - - $steps[] = $run_test; - $steps[] = $cleanup; - - return $steps; - } - - function startTestsInSandcastle($workflow) { - // extract information we need from workflow or CLI - $diffID = $workflow->getDiffId(); - $username = exec("whoami"); - - if ($diffID == null || $username == null) { - // there is no diff and we can't extract username - // we cannot schedule sandcasstle job - return; - } - - if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) { - // extract all tests from the CI definition - $output = file_get_contents("build_tools/rocksdb-lego-determinator"); - preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches); - $tests = $matches[1]; - } else { - // manually list of tests we want to run in sandcastle - $tests = array( - "unit", "unit_481", "clang_unit", "tsan", "asan", "lite_test", "valgrind" - ); - } - - // construct a job definition for each test and add it to the master plan - foreach ($tests as $test) { - $arg[] = array( - "name" => "RocksDB diff " . $diffID . " test " . $test, - "steps" => $this->getSteps($diffID, $username, $test) - ); - } - - // we cannot submit the parallel execution master plan to sandcastle - // we need supply the job plan as a determinator - // so we construct a small job that will spit out the master job plan - // which sandcastle will parse and execute - // Why compress ? Otherwise we run over the max string size. - $cmd = "echo " . base64_encode(json_encode($arg)) - . " | gzip -f | base64 -w0"; - $arg_encoded = shell_exec($cmd); - - $command = array( - "name" => "Run diff " . $diffID . "for user " . $username, - "steps" => array() - ); - - $command["steps"][] = array( - "name" => "Generate determinator", - "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" - . " | base64 --decode", - "determinator" => true, - "user" => "root" - ); - - // submit to sandcastle - $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?' - .'command=SandcastleUniversalCommand' - .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego' - .'&user=krad&alias=rocksdb-precommit' - .'&command-args=' . urlencode(json_encode($command)); - - if (file_exists('/home/krad/.sandcastle')) { - $cmd = 'cat /home/krad/.sandcastle'; - } else { - $cmd = 'cat ~/.sandcastle'; - } - $sandcastle_config = explode(':', rtrim(shell_exec($cmd))); - - if (count($sandcastle_config) != 2) { - print('.sandcastle does not contain valid configuration'); - return; - } - - $app = $sandcastle_config[0]; - $token = $sandcastle_config[1]; - - $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=' . $app . ' ' - . '-F token=' . $token . ' "' . $url . '"'; - - $output = shell_exec($cmd); - - // extract sandcastle URL from the response - preg_match('/url": "(.+)"/', $output, $sandcastle_url); - - if (count($sandcastle_url) > 1) { - echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; - - // Ask phabricator to display it on the diff UI - $this->postURL($diffID, $sandcastle_url[1]); - } else { - print("Error submitting job to sandcastle."); - print($output); + // When submitting a diff this code path gets executed multiple times in + // a row. We only care about the case when ID for the diff is provided + // because that's what we need to apply the diff and trigger the tests. + if (strlen($diffID) > 0) { + assert(is_numeric($diffID)); + startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID); + } } } } diff --git a/arcanist_util/config/FacebookOldArcanistConfiguration.php b/arcanist_util/config/FacebookOldArcanistConfiguration.php index 4e936ce7e..aaab040dd 100644 --- a/arcanist_util/config/FacebookOldArcanistConfiguration.php +++ b/arcanist_util/config/FacebookOldArcanistConfiguration.php @@ -4,184 +4,33 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +require('RocksDBCommonHelper.php'); + +define("DIFF_COMMAND", "diff"); + class FacebookArcanistConfiguration extends ArcanistConfiguration { public function didRunWorkflow($command, ArcanistBaseWorkflow $workflow, $error_code) { - if ($command == 'diff' && !$workflow->isRawDiffSource()) { - $this->startTestsInSandcastle($workflow); + // Default options don't terminate on failure, but that's what we want. In + // the current case we use assertions intentionally as "terminate on failure + // invariants". + assert_options(ASSERT_BAIL, true); + + assert($workflow); + assert(strlen($command) > 0); + + if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) { + $diffID = $workflow->getDiffId(); + + // When submitting a diff this code path gets executed multiple times in + // a row. We only care about the case when ID for the diff is provided + // because that's what we need to apply the diff and trigger the tests. + if (strlen($diffID) > 0) { + assert(is_numeric($diffID)); + startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID); + } } } - - ////////////////////////////////////////////////////////////////////// - /* Run tests in sandcastle */ - function postURL($diffID, $url) { - $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' - . '"name":"click here for sandcastle tests for D' . $diffID . '", ' - . '"link":"' . $url . '"}\' | ' - . 'http_proxy=fwdproxy.any.facebook.com:8080 ' - . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' - . 'differential.updateunitresults'; - shell_exec($cmd); - } - - function updateTestCommand($diffID, $test, $status) { - $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' - . '"name":"' . $test . '", ' - . '"result":"' . $status . '"}\' | ' - . 'http_proxy=fwdproxy.any.facebook.com:8080 ' - . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' - . 'differential.updateunitresults'; - return $cmd; - } - - function updateTest($diffID, $test) { - shell_exec($this->updateTestCommand($diffID, $test, "waiting")); - } - - function getSteps($diffID, $username, $test) { - $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0"); - - // Sandcastle machines don't have arc setup. We copy the user certificate - // and authenticate using that in sandcastle - $setup = array( - "name" => "Setup arcrc", - "shell" => "echo " . $arcrc_content . " | base64 --decode" - . " | gzip -d > ~/.arcrc", - "user" => "root" - ); - - // arc demands certain permission on its config - $fix_permission = array( - "name" => "Fix environment", - "shell" => "chmod 600 ~/.arcrc", - "user" => "root" - ); - - // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise - // git thinks it is uncommited change - $fix_git_ignore = array( - "name" => "Fix git ignore", - "shell" => "echo fbcode >> .git/info/exclude", - "user" => "root" - ); - - // Patch the code (keep your fingures crossed) - $patch = array( - "name" => "Patch " . $diffID, - "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc " - . "patch --nocommit --diff " . $diffID, - "user" => "root" - ); - - // Clean up the user arc config we are using - $cleanup = array( - "name" => "Arc cleanup", - "shell" => "rm -f ~/.arcrc", - "user" => "root" - ); - - // Construct the steps in the order of execution - $steps[] = $setup; - $steps[] = $fix_permission; - $steps[] = $fix_git_ignore; - $steps[] = $patch; - - // Run the actual command - $this->updateTest($diffID, $test); - $cmd = $this->updateTestCommand($diffID, $test, "running") . ";" - . "./build_tools/precommit_checker.py " . $test - . "; exit_code=$?; ([[ \$exit_code -eq 0 ]] &&" - . $this->updateTestCommand($diffID, $test, "pass") . ")" - . "||" . $this->updateTestCommand($diffID, $test, "fail") - . "; cat /tmp/precommit-check.log" - . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;" - . "[[ \$exit_code -eq 0 ]]"; - - $run_test = array( - "name" => "Run " . $test, - "shell" => $cmd, - "user" => "root", - ); - - $steps[] = $run_test; - $steps[] = $cleanup; - - return $steps; - } - - function startTestsInSandcastle($workflow) { - // extract information we need from workflow or CLI - $diffID = $workflow->getDiffId(); - $username = exec("whoami"); - - if ($diffID == null || $username == null) { - // there is no diff and we can't extract username - // we cannot schedule sandcasstle job - return; - } - - if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) { - // extract all tests from the CI definition - $output = file_get_contents("build_tools/rocksdb-lego-determinator"); - preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches); - $tests = $matches[1]; - } else { - // manually list of tests we want to run in sandcastle - $tests = array( - "unit", "unit_481", "clang_unit", "tsan", "asan", "lite_test", "valgrind" - ); - } - - // construct a job definition for each test and add it to the master plan - foreach ($tests as $test) { - $arg[] = array( - "name" => "RocksDB diff " . $diffID . " test " . $test, - "steps" => $this->getSteps($diffID, $username, $test) - ); - } - - // we cannot submit the parallel execution master plan to sandcastle - // we need supply the job plan as a determinator - // so we construct a small job that will spit out the master job plan - // which sandcastle will parse and execute - // Why compress ? Otherwise we run over the max string size. - $cmd = "echo " . base64_encode(json_encode($arg)) - . " | gzip -f | base64 -w0"; - $arg_encoded = shell_exec($cmd); - - $command = array( - "name" => "Run diff " . $diffID . "for user " . $username, - "steps" => array() - ); - - $command["steps"][] = array( - "name" => "Generate determinator", - "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" - . " | base64 --decode", - "determinator" => true, - "user" => "root" - ); - - // submit to sandcastle - $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?' - .'command=SandcastleUniversalCommand' - .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego' - .'&user=krad&alias=rocksdb-precommit' - .'&command-args=' . urlencode(json_encode($command)); - - $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=659387027470559 ' - . '-F token=AeO_3f2Ya3TujjnxGD4 "' . $url . '"'; - - $output = shell_exec($cmd); - - // extract sandcastle URL from the response - preg_match('/url": "(.+)"/', $output, $sandcastle_url); - - echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; - - // Ask phabricator to display it on the diff UI - $this->postURL($diffID, $sandcastle_url[1]); - } } diff --git a/arcanist_util/config/RocksDBCommonHelper.php b/arcanist_util/config/RocksDBCommonHelper.php new file mode 100644 index 000000000..072f3f64f --- /dev/null +++ b/arcanist_util/config/RocksDBCommonHelper.php @@ -0,0 +1,326 @@ + 0); + assert(is_numeric($diffID)); + assert(strlen($url) > 0); + + $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' + . '"name":"click here for sandcastle tests for D' . $diffID . '", ' + . '"link":"' . $url . '"}\' | ' + . 'http_proxy=fwdproxy.any.facebook.com:8080 ' + . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' + . 'differential.updateunitresults'; + shell_exec($cmd); +} + +function buildUpdateTestStatusCmd($diffID, $test, $status) { + assert(strlen($diffID) > 0); + assert(is_numeric($diffID)); + assert(strlen($test) > 0); + assert(strlen($status) > 0); + + $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' + . '"name":"' . $test . '", ' + . '"result":"' . $status . '"}\' | ' + . 'http_proxy=fwdproxy.any.facebook.com:8080 ' + . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' + . 'differential.updateunitresults'; + return $cmd; +} + +function updateTestStatus($diffID, $test) { + assert(strlen($diffID) > 0); + assert(is_numeric($diffID)); + assert(strlen($test) > 0); + + shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting")); +} + +function getSteps($applyDiff, $diffID, $username, $test) { + assert(strlen($username) > 0); + assert(strlen($test) > 0); + + if ($applyDiff) { + assert(strlen($diffID) > 0); + assert(is_numeric($diffID)); + + $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0"); + assert(strlen($arcrc_content) > 0); + + // Sandcastle machines don't have arc setup. We copy the user certificate + // and authenticate using that in Sandcastle. + $setup = array( + "name" => "Setup arcrc", + "shell" => "echo " . $arcrc_content . " | base64 --decode" + . " | gzip -d > ~/.arcrc", + "user" => "root" + ); + + // arc demands certain permission on its config. + $fix_permission = array( + "name" => "Fix environment", + "shell" => "chmod 600 ~/.arcrc", + "user" => "root" + ); + + // Construct the steps in the order of execution. + $steps[] = $setup; + $steps[] = $fix_permission; + } + + // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise + // Git thinks it is an uncommited change. + $fix_git_ignore = array( + "name" => "Fix git ignore", + "shell" => "echo fbcode >> .git/info/exclude", + "user" => "root" + ); + + $steps[] = $fix_git_ignore; + + // This will be the command used to execute particular type of tests. + $cmd = ""; + + if ($applyDiff) { + // Patch the code (keep your fingures crossed). + $patch = array( + "name" => "Patch " . $diffID, + "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc " + . "patch --nocommit --diff " . $diffID, + "user" => "root" + ); + + $steps[] = $patch; + + updateTestStatus($diffID, $test); + $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; "; + } + + // Run the actual command. + $cmd = $cmd . "./build_tools/precommit_checker.py " . $test + . "; exit_code=$?; "; + + if ($applyDiff) { + $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&" + . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")" + . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail") + . "; "; + } + + $cmd = $cmd . " cat /tmp/precommit-check.log" + . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;" + . "[[ \$exit_code -eq 0 ]]"; + assert(strlen($cmd) > 0); + + $run_test = array( + "name" => "Run " . $test, + "shell" => $cmd, + "user" => "root", + ); + + $steps[] = $run_test; + + if ($applyDiff) { + // Clean up the user arc config we are using. + $cleanup = array( + "name" => "Arc cleanup", + "shell" => "rm -f ~/.arcrc", + "user" => "root" + ); + + $steps[] = $cleanup; + } + + assert(count($steps) > 0); + return $steps; +} + +function getSandcastleConfig() { + $sandcastle_config = array(); + + // This is a case when we're executed from a continuous run. Fetch the values + // from the environment. + if (getenv(ENV_POST_RECEIVE_HOOK)) { + $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE); + $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE); + } else { + // This is a typical `[p]arc diff` case. Fetch the values from the specific + // configuration files. + assert(file_exists(PRIMARY_TOKEN_FILE) || + file_exists(SECONDARY_TOKEN_FILE)); + + // Try the primary location first, followed by a secondary. + if (file_exists(PRIMARY_TOKEN_FILE)) { + $cmd = 'cat ' . PRIMARY_TOKEN_FILE; + } else { + $cmd = 'cat ' . SECONDARY_TOKEN_FILE; + } + + assert(strlen($cmd) > 0); + $sandcastle_config = explode(':', rtrim(shell_exec($cmd))); + } + + // In this case be very explicit about the implications. + if (count($sandcastle_config) != 2) { + echo "Sandcastle configuration files don't contain valid information " . + "or the necessary environment variables aren't defined. Unable " . + "to validate the code changes."; + exit(1); + } + + assert(strlen($sandcastle_config[0]) > 0); + assert(strlen($sandcastle_config[1]) > 0); + assert(count($sandcastle_config) > 0); + + return $sandcastle_config; +} + +// This function can be called either from `[p]arc diff` command or during +// the Git post-receive hook. + function startTestsInSandcastle($applyDiff, $workflow, $diffID) { + // Default options don't terminate on failure, but that's what we want. In + // the current case we use assertions intentionally as "terminate on failure + // invariants". + assert_options(ASSERT_BAIL, true); + + // In case of a diff we'll send notificatios to the author. Else it'll go to + // the entire team because failures indicate that build quality has regressed. + $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS; + assert(strlen($username) > 0); + + if ($applyDiff) { + assert($workflow); + assert(strlen($diffID) > 0); + assert(is_numeric($diffID)); + } + + if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) { + // Extract all tests from the CI definition. + $output = file_get_contents("build_tools/rocksdb-lego-determinator"); + assert(strlen($output) > 0); + + preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches); + $tests = $matches[1]; + assert(count($tests) > 0); + } else { + // Manually list of tests we want to run in Sandcastle. + $tests = array( + "unit", "unit_481", "clang_unit", "tsan", "asan", "lite_test", + "valgrind" + ); + } + + $send_email_template = array( + 'type' => 'email', + 'triggers' => array('fail'), + 'emails' => array($username . '@fb.com'), + ); + + // Construct a job definition for each test and add it to the master plan. + foreach ($tests as $test) { + $stepName = "RocksDB diff " . $diffID . " test " . $test; + + if (!$applyDiff) { + $stepName = "RocksDB continuous integration test " . $test; + } + + $arg[] = array( + "name" => $stepName, + "report" => array($send_email_template), + "steps" => getSteps($applyDiff, $diffID, $username, $test) + ); + } + + // We cannot submit the parallel execution master plan to Sandcastle and + // need supply the job plan as a determinator. So we construct a small job + // that will spit out the master job plan which Sandcastle will parse and + // execute. Why compress the job definitions? Otherwise we run over the max + // string size. + $cmd = "echo " . base64_encode(json_encode($arg)) + . " | gzip -f | base64 -w0"; + assert(strlen($cmd) > 0); + + $arg_encoded = shell_exec($cmd); + assert(strlen($arg_encoded) > 0); + + $runName = "Run diff " . $diffID . "for user " . $username; + + if (!$applyDiff) { + $runName = "RocksDB continuous integration build and test run"; + } + + $command = array( + "name" => $runName, + "steps" => array() + ); + + $command["steps"][] = array( + "name" => "Generate determinator", + "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" + . " | base64 --decode", + "determinator" => true, + "user" => "root" + ); + + // Submit to Sandcastle. + $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?' + .'command=SandcastleUniversalCommand' + .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego' + .'&user=' . $username . '&alias=rocksdb-precommit' + .'&command-args=' . urlencode(json_encode($command)); + + // Fetch the configuration necessary to submit a successful HTTPS request. + $sandcastle_config = getSandcastleConfig(); + + $app = $sandcastle_config[0]; + $token = $sandcastle_config[1]; + + $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=' . $app . ' ' + . '-F token=' . $token . ' "' . $url . '"'; + + $output = shell_exec($cmd); + assert(strlen($output) > 0); + + // Extract Sandcastle URL from the response. + preg_match('/url": "(.+)"/', $output, $sandcastle_url); + + assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request."); + assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL."); + + if ($applyDiff) { + echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; + // Ask Phabricator to display it on the diff UI. + postURL($diffID, $sandcastle_url[1]); + } else { + echo "Continuous integration started Sandcastle tests. You can look at "; + echo "the progress at:\n" . $sandcastle_url[1] . "\n"; + } +} + +// Continuous run cript will set the environment variable and based on that +// we'll trigger the execution of tests in Sandcastle. In that case we don't +// need to apply any diffs and there's no associated workflow either. +if (getenv(ENV_POST_RECEIVE_HOOK)) { + startTestsInSandcastle( + false /* $applyDiff */, + NULL /* $workflow */, + NULL /* $diffID */); +} diff --git a/build_tools/cont_integration.sh b/build_tools/cont_integration.sh new file mode 100755 index 000000000..390f30c7e --- /dev/null +++ b/build_tools/cont_integration.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Copyright (c) 2016, Facebook. All rights reserved. +# +# Overall wrapper script for RocksDB continuous builds. The implementation is a +# trivial pulling scheme. We loop infinitely, check if any new changes have been +# committed, if yes then trigger a Sandcastle run, and finally go to sleep again +# for a certain interval. +# + +function log { + DATE=`date +%Y-%m-%d:%H:%M:%S` + echo $DATE $@ +} + +function log_err { + log "ERROR: $@. Error code: $?." +} + +# +# Execution starts here. +# + +# Path to the determinator from the root of the RocksDB repo. +CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonDeterminator.php + +# Value of the previous commit. +PREV_COMMIT= + +log "Starting to monitor for new RocksDB changes ..." +log "Running under `pwd` as `whoami`." + +# Paranoia. Make sure that we're using the right branch. +git checkout master + +if [ ! $? -eq 0 ]; then + log_err "This is not good. Can't checkout master. Bye-bye!" + exit 1 +fi + +# We'll run forever and let the execution environment terminate us if we'll +# exceed whatever timeout is set for the job. +while true; +do + # Get the latest changes committed. + git pull --rebase + + if [ $? -eq 0 ]; then + LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'` + + log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'." + + if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then + log "There were no changes since the last time I checked. Going to sleep." + else + if [ ! -z "$LAST_COMMIT" ]; then + log "New code has been committed or previous commit not know. " \ + "Will trigger the tests." + + PREV_COMMIT=$LAST_COMMIT + log "Updated previous commit to '$PREV_COMMIT'." + + # + # This is where we'll trigger the Sandcastle run. The values for + # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're + # running in. + # + POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR + + if [ $? -eq 0 ]; then + log "Sandcastle run successfully triggered." + else + log_err "Failed to trigger Sandcastle run." + fi + else + log_err "Previous commit not updated. Don't know what the last one is." + fi + fi + else + log_err "git pull --rebase failed. Will skip running tests for now." + fi + + # Always sleep, even if errors happens while trying to determine the latest + # commit. This will prevent us terminating in case of transient errors. + log "Will go to sleep for 5 minutes." + sleep 5m +done