andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Some more files.

Browse Source
This commit is contained in:
sdong 2017-07-18 12:21:24 -07:00
parent f63f26f842
commit 3327adcc72
16 changed files with 3 additions and 6055 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arcanist_util/__phutil_library_init__.php
View File

@ -1,3 +0,0 @@
<?php
phutil_register_library('arcanist_util', __FILE__);

71
arcanist_util/__phutil_library_map__.php
View File

@ -1,71 +0,0 @@
<?php
/**
* This file is automatically generated. Use 'arc liberate' to rebuild it.
* @generated
* @phutil-library-version 2
*/
if (class_exists('ArcanistWorkflow')) {
phutil_register_library_map(array(
'__library_version__' => 2,
'class' =>
array(
'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
),
'function' =>
array(
),
'xmap' =>
array(
'ArcanistCpplintLinter' => 'ArcanistLinter',
'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
'FacebookHowtoevenLinter' => 'ArcanistLinter',
'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
'FbcodeCppLinter' => 'ArcanistLinter',
),
));
} else {
phutil_register_library_map(array(
'__library_version__' => 2,
'class' =>
array(
'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php',
'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookOldFbcodeUnitTestEngine.php',
'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
),
'function' =>
array(
),
'xmap' =>
array(
'ArcanistCpplintLinter' => 'ArcanistLinter',
'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
'FacebookHowtoevenLinter' => 'ArcanistLinter',
'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
'FbcodeCppLinter' => 'ArcanistLinter',
),
));
}

43
arcanist_util/config/FacebookArcanistConfiguration.php
View File

@ -1,43 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
require('RocksDBCommonHelper.php');
define("DIFF_COMMAND", "diff");
class FacebookArcanistConfiguration extends ArcanistConfiguration {
public function getCustomArgumentsForCommand($command) {
if ($command == "land") {
return array(
'async' => array('help' => 'Just to make tools happy'));
}
return array();
}
public function didRunWorkflow($command,
ArcanistWorkflow $workflow,
$error_code) {
// Default options don't terminate on failure, but that's what we want. In
// the current case we use assertions intentionally as "terminate on failure
// invariants".
assert_options(ASSERT_BAIL, true);
assert($workflow);
assert(strlen($command) > 0);
if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
$diffID = $workflow->getDiffId();
// When submitting a diff this code path gets executed multiple times in
// a row. We only care about the case when ID for the diff is provided
// because that's what we need to apply the diff and trigger the tests.
if (strlen($diffID) > 0) {
assert(is_numeric($diffID));
startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
}
}
}
}

43
arcanist_util/config/FacebookOldArcanistConfiguration.php
View File

@ -1,43 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
require('RocksDBCommonHelper.php');
define("DIFF_COMMAND", "diff");
class FacebookArcanistConfiguration extends ArcanistConfiguration {
public function getCustomArgumentsForCommand($command) {
if ($command == "land") {
return array(
'async' => array('help' => 'Just to make tools happy'));
}
return array();
}
public function didRunWorkflow($command,
ArcanistBaseWorkflow $workflow,
$error_code) {
// Default options don't terminate on failure, but that's what we want. In
// the current case we use assertions intentionally as "terminate on failure
// invariants".
assert_options(ASSERT_BAIL, true);
assert($workflow);
assert(strlen($command) > 0);
if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
$diffID = $workflow->getDiffId();
// When submitting a diff this code path gets executed multiple times in
// a row. We only care about the case when ID for the diff is provided
// because that's what we need to apply the diff and trigger the tests.
if (strlen($diffID) > 0) {
assert(is_numeric($diffID));
startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
}
}
}
}

355
arcanist_util/config/RocksDBCommonHelper.php
View File

@ -1,355 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// Name of the environment variables which need to be set by the entity which
// triggers continuous runs so that code at the end of the file gets executed
// and Sandcastle run starts.
define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
define("CONT_RUN_ALIAS", "leveldb");
//////////////////////////////////////////////////////////////////////
/* Run tests in sandcastle */
function postURL($diffID, $url) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($url) > 0);
$cmd = 'echo \'{"diff_id": ' . $diffID . ', '
. '"name":"click here for sandcastle tests for D' . $diffID . '", '
. '"link":"' . $url . '"}\' | '
. 'arc call-conduit '
. 'differential.updateunitresults';
shell_exec($cmd);
}
function buildUpdateTestStatusCmd($diffID, $test, $status) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($test) > 0);
assert(strlen($status) > 0);
$cmd = 'echo \'{"diff_id": ' . $diffID . ', '
. '"name":"' . $test . '", '
. '"result":"' . $status . '"}\' | '
. 'arc call-conduit '
. 'differential.updateunitresults';
return $cmd;
}
function updateTestStatus($diffID, $test) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
assert(strlen($test) > 0);
shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
}
function getSteps($applyDiff, $diffID, $username, $test) {
assert(strlen($username) > 0);
assert(strlen($test) > 0);
if ($applyDiff) {
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
$arcrc_content = (PHP_OS == "Darwin" ?
exec("cat ~/.arcrc | gzip -f | base64") :
exec("cat ~/.arcrc | gzip -f | base64 -w0"));
assert(strlen($arcrc_content) > 0);
// Sandcastle machines don't have arc setup. We copy the user certificate
// and authenticate using that in Sandcastle.
$setup = array(
"name" => "Setup arcrc",
"shell" => "echo " . $arcrc_content . " | base64 --decode"
. " | gzip -d > ~/.arcrc",
"user" => "root"
);
// arc demands certain permission on its config.
// also fix the sticky bit issue in sandcastle
$fix_permission = array(
"name" => "Fix environment",
"shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
"user" => "root"
);
// Construct the steps in the order of execution.
$steps[] = $setup;
$steps[] = $fix_permission;
}
// fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
// Git thinks it is an uncommited change.
$fix_git_ignore = array(
"name" => "Fix git ignore",
"shell" => "echo fbcode >> .git/info/exclude",
"user" => "root"
);
$steps[] = $fix_git_ignore;
// This will be the command used to execute particular type of tests.
$cmd = "";
if ($applyDiff) {
// Patch the code (keep your fingures crossed).
$patch = array(
"name" => "Patch " . $diffID,
"shell" => "arc --arcrc-file ~/.arcrc "
. "patch --nocommit --diff " . $diffID,
"user" => "root"
);
$steps[] = $patch;
updateTestStatus($diffID, $test);
$cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
}
// Run the actual command.
$cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
. "; exit_code=$?; ";
if ($applyDiff) {
$cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
. buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
. "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
. "; ";
}
// shell command to sort the tests based on exit code and print
// the output of the log files.
$cat_sorted_logs = "
while read code log_file;
do echo \"################ cat \$log_file [exit_code : \$code] ################\";
cat \$log_file;
done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
// Shell command to cat all log files
$cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
// If LOG file exist use it to cat log files sorted by exit code, otherwise
// cat everything
$logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
$cmd = $cmd . " cat /tmp/precommit-check.log"
. "; shopt -s extglob; {$logs_cmd}"
. "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
assert(strlen($cmd) > 0);
$run_test = array(
"name" => "Run " . $test,
"shell" => $cmd,
"user" => "root",
"parser" => "python build_tools/error_filter.py " . $test,
);
$steps[] = $run_test;
if ($applyDiff) {
// Clean up the user arc config we are using.
$cleanup = array(
"name" => "Arc cleanup",
"shell" => "rm -f ~/.arcrc",
"user" => "root"
);
$steps[] = $cleanup;
}
assert(count($steps) > 0);
return $steps;
}
function getSandcastleConfig() {
$sandcastle_config = array();
$cwd = getcwd();
$cwd_token_file = "{$cwd}/.sandcastle";
// This is a case when we're executed from a continuous run. Fetch the values
// from the environment.
if (getenv(ENV_POST_RECEIVE_HOOK)) {
$sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
$sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
} else {
// This is a typical `[p]arc diff` case. Fetch the values from the specific
// configuration files.
for ($i = 0; $i < 50; $i++) {
if (file_exists(PRIMARY_TOKEN_FILE) ||
file_exists($cwd_token_file)) {
break;
}
// If we failed to fetch the tokens, sleep for 0.2 second and try again
usleep(200000);
}
assert(file_exists(PRIMARY_TOKEN_FILE) ||
file_exists($cwd_token_file));
// Try the primary location first, followed by a secondary.
if (file_exists(PRIMARY_TOKEN_FILE)) {
$cmd = 'cat ' . PRIMARY_TOKEN_FILE;
} else {
$cmd = 'cat ' . $cwd_token_file;
}
assert(strlen($cmd) > 0);
$sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
}
// In this case be very explicit about the implications.
if (count($sandcastle_config) != 2) {
echo "Sandcastle configuration files don't contain valid information " .
"or the necessary environment variables aren't defined. Unable " .
"to validate the code changes.";
exit(1);
}
assert(strlen($sandcastle_config[0]) > 0);
assert(strlen($sandcastle_config[1]) > 0);
assert(count($sandcastle_config) > 0);
return $sandcastle_config;
}
// This function can be called either from `[p]arc diff` command or during
// the Git post-receive hook.
function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
// Default options don't terminate on failure, but that's what we want. In
// the current case we use assertions intentionally as "terminate on failure
// invariants".
assert_options(ASSERT_BAIL, true);
// In case of a diff we'll send notificatios to the author. Else it'll go to
// the entire team because failures indicate that build quality has regressed.
$username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
assert(strlen($username) > 0);
if ($applyDiff) {
assert($workflow);
assert(strlen($diffID) > 0);
assert(is_numeric($diffID));
}
// List of tests we want to run in Sandcastle.
$tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
"asan", "lite_test", "valgrind", "release", "release_481",
"clang_release", "punit", "clang_analyze", "code_cov",
"java_build", "no_compression", "unity", "ubsan");
$send_email_template = array(
'type' => 'email',
'triggers' => array('fail'),
'emails' => array($username . '@fb.com'),
);
// Construct a job definition for each test and add it to the master plan.
foreach ($tests as $test) {
$stepName = "RocksDB diff " . $diffID . " test " . $test;
if (!$applyDiff) {
$stepName = "RocksDB continuous integration test " . $test;
}
$arg[] = array(
"name" => $stepName,
"report" => array($send_email_template),
"steps" => getSteps($applyDiff, $diffID, $username, $test)
);
}
// We cannot submit the parallel execution master plan to Sandcastle and
// need supply the job plan as a determinator. So we construct a small job
// that will spit out the master job plan which Sandcastle will parse and
// execute. Why compress the job definitions? Otherwise we run over the max
// string size.
$cmd = "echo " . base64_encode(json_encode($arg))
. (PHP_OS == "Darwin" ?
" | gzip -f | base64" :
" | gzip -f | base64 -w0");
assert(strlen($cmd) > 0);
$arg_encoded = shell_exec($cmd);
assert(strlen($arg_encoded) > 0);
$runName = "Run diff " . $diffID . "for user " . $username;
if (!$applyDiff) {
$runName = "RocksDB continuous integration build and test run";
}
$command = array(
"name" => $runName,
"steps" => array()
);
$command["steps"][] = array(
"name" => "Generate determinator",
"shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
. " | base64 --decode",
"determinator" => true,
"user" => "root"
);
// Submit to Sandcastle.
$url = 'https://interngraph.intern.facebook.com/sandcastle/create';
$job = array(
'command' => 'SandcastleUniversalCommand',
'args' => $command,
'capabilities' => array(
'vcs' => 'rocksdb-int-git',
'type' => 'lego',
),
'hash' => 'origin/master',
'user' => $username,
'alias' => 'rocksdb-precommit',
'tags' => array('rocksdb'),
'description' => 'Rocksdb precommit job',
);
// Fetch the configuration necessary to submit a successful HTTPS request.
$sandcastle_config = getSandcastleConfig();
$app = $sandcastle_config[0];
$token = $sandcastle_config[1];
$cmd = 'curl -s -k -F app=' . $app . ' '
. '-F token=' . $token . ' -F job=\'' . json_encode($job)
.'\' "' . $url . '"';
$output = shell_exec($cmd);
assert(strlen($output) > 0);
// Extract Sandcastle URL from the response.
preg_match('/url": "(.+)"/', $output, $sandcastle_url);
assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
if ($applyDiff) {
echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
// Ask Phabricator to display it on the diff UI.
postURL($diffID, $sandcastle_url[1]);
} else {
echo "Continuous integration started Sandcastle tests. You can look at ";
echo "the progress at:\n" . $sandcastle_url[1] . "\n";
}
}
// Continuous run cript will set the environment variable and based on that
// we'll trigger the execution of tests in Sandcastle. In that case we don't
// need to apply any diffs and there's no associated workflow either.
if (getenv(ENV_POST_RECEIVE_HOOK)) {
startTestsInSandcastle(
false /* $applyDiff */,
NULL /* $workflow */,
NULL /* $diffID */);
}

88
arcanist_util/cpp_linter/ArcanistCpplintLinter.php
View File

@ -1,88 +0,0 @@
<?php
/**
* Uses google's cpplint.py to check code. RocksDB team forked this file from
* phabricator's /src/lint/linter/ArcanistCpplintLinter.php, and customized it
* for its own use.
*
* You can get it here:
* http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
* @group linter
*/
final class ArcanistCpplintLinter extends ArcanistLinter {
public function willLintPaths(array $paths) {
return;
}
public function getLinterName() {
return 'cpplint.py';
}
public function getLintPath() {
$bin = 'cpplint.py';
// Search under current dir
list($err) = exec_manual('which %s/%s', $this->linterDir(), $bin);
if (!$err) {
return $this->linterDir().'/'.$bin;
}
// Look for globally installed cpplint.py
list($err) = exec_manual('which %s', $bin);
if ($err) {
throw new ArcanistUsageException(
"cpplint.py does not appear to be installed on this system. Install ".
"it (e.g., with 'wget \"http://google-styleguide.googlecode.com/".
"svn/trunk/cpplint/cpplint.py\"') ".
"in your .arcconfig to point to the directory where it resides. ".
"Also don't forget to chmod a+x cpplint.py!");
}
return $bin;
}
public function lintPath($path) {
$bin = $this->getLintPath();
$path = $this->rocksdbDir().'/'.$path;
$f = new ExecFuture("%C $path", $bin);
list($err, $stdout, $stderr) = $f->resolve();
if ($err === 2) {
throw new Exception("cpplint failed to run correctly:\n".$stderr);
}
$lines = explode("\n", $stderr);
$messages = array();
foreach ($lines as $line) {
$line = trim($line);
$matches = null;
$regex = '/^[^:]+:(\d+):\s*(.*)\s*\[(.*)\] \[(\d+)\]$/';
if (!preg_match($regex, $line, $matches)) {
continue;
}
foreach ($matches as $key => $match) {
$matches[$key] = trim($match);
}
$message = new ArcanistLintMessage();
$message->setPath($path);
$message->setLine($matches[1]);
$message->setCode($matches[3]);
$message->setName($matches[3]);
$message->setDescription($matches[2]);
$message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
$this->addLintMessage($message);
}
}
// The path of this linter
private function linterDir() {
return dirname(__FILE__);
}
// TODO(kaili) a quick and dirty way to figure out rocksdb's root dir.
private function rocksdbDir() {
return $this->linterDir()."/../..";
}
}

74
arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php
View File

@ -1,74 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
const LINT_FORMATTING = 1;
private $changedLines = array();
private $rawLintOutput = array();
abstract protected function getPathsToLint();
protected function shouldLintPath($path) {
foreach ($this->getPathsToLint() as $p) {
// check if $path starts with $p
if (strncmp($path, $p, strlen($p)) === 0) {
return true;
}
}
return false;
}
// API to tell this linter which lines were changed
final public function setPathChangedLines($path, $changed) {
$this->changedLines[$path] = $changed;
}
final public function willLintPaths(array $paths) {
$futures = array();
foreach ($paths as $path) {
if (!$this->shouldLintPath($path)) {
continue;
}
$changed = $this->changedLines[$path];
if (!isset($changed)) {
// do not run linter if there are no changes
continue;
}
$futures[$path] = $this->getFormatFuture($path, $changed);
}
foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) {
$this->rawLintOutput[$p] = $f->resolvex();
}
}
abstract protected function getFormatFuture($path, array $changed);
abstract protected function getLintMessage($diff);
final public function lintPath($path) {
if (!isset($this->rawLintOutput[$path])) {
return;
}
list($new_content) = $this->rawLintOutput[$path];
$old_content = $this->getData($path);
if ($new_content != $old_content) {
$diff = ArcanistDiffUtils::renderDifferences($old_content, $new_content);
$this->raiseLintAtOffset(
0,
self::LINT_FORMATTING,
$this->getLintMessage($diff),
$old_content,
$new_content);
}
}
}

223
arcanist_util/cpp_linter/FacebookHowtoevenLinter.php
View File

@ -1,223 +0,0 @@
<?php
// Copyright 2015-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
final class FacebookHowtoevenLinter extends ArcanistLinter {
const VERSION = 'fd9192f324c36d28136d14380f0b552a1385b59b';
private $parsedTargets = array();
public function getLinterName() {
return 'Howtoeven';
}
protected function getSeverity($code) {
$severities = array(
ArcanistLintSeverity::SEVERITY_DISABLED,
ArcanistLintSeverity::SEVERITY_ADVICE,
ArcanistLintSeverity::SEVERITY_WARNING,
ArcanistLintSeverity::SEVERITY_ERROR,
);
return idx($severities, $code, ArcanistLintSeverity::SEVERITY_WARNING);
}
public function willLintPaths(array $paths) {
// Cleanup previous runs.
$this->localExecx("rm -rf _build/_lint");
// Build compilation database.
$lintable_paths = $this->getLintablePaths($paths);
$interesting_paths = $this->getInterestingPaths($lintable_paths);
if (!$lintable_paths) {
return;
}
// Run lint.
try {
$this->localExecx(
"%C %C -p _build/dev/ %Ls",
$this->getBinaryPath(),
$this->getFilteredIssues(),
$lintable_paths);
} catch (CommandException $exception) {
PhutilConsole::getConsole()->writeErr($exception->getMessage());
}
// Load results.
$result = id(
new SQLite3(
$this->getProjectRoot().'/_build/_lint/lint.db',
SQLITE3_OPEN_READONLY))
->query("SELECT * FROM raised_issues");
while ($issue = $result->fetchArray(SQLITE3_ASSOC)) {
// Skip issues not part of the linted file.
if (in_array($issue['file'], $interesting_paths)) {
$this->addLintMessage(id(new ArcanistLintMessage())
->setPath($issue['file'])
->setLine($issue['line'])
->setChar($issue['column'])
->setCode('Howtoeven')
->setSeverity($this->getSeverity($issue['severity']))
->setName('Hte-'.$issue['name'])
->setDescription(
sprintf(
"%s\n\n%s",
($issue['message']) ? $issue['message'] : $issue['description'],
$issue['explanation']))
->setOriginalText(idx($issue, 'original', ''))
->setReplacementText(idx($issue, 'replacement', '')));
}
}
}
public function lintPath($path) {
}
/**
* Get the paths that we know how to lint.
*
* The strategy is to first look whether there's an existing compilation
* database and use that if it's exhaustive. We generate our own only if
* necessary.
*/
private function getLintablePaths($paths) {
// Replace headers with existing sources.
for ($i = 0; $i < count($paths); $i++) {
if (preg_match("/\.h$/", $paths[$i])) {
$header = preg_replace("/\.h$/", ".cpp", $paths[$i]);
if (file_exists($header)) {
$paths[$i] = $header;
}
}
}
// Check if database exists and is exhaustive.
$available_paths = $this->getAvailablePaths();
$lintable_paths = array_intersect($paths, $available_paths);
if ($paths === $lintable_paths) {
return $lintable_paths;
}
// Generate our own database.
$targets = $this->getTargetsFor($paths);
if (!$targets) {
PhutilConsole::getConsole()->writeErr(
"No build targets found for %s\n",
implode(', ', $paths));
return array();
}
$this->localExecx("./tools/build/bin/fbconfig.par -r %Ls", $targets);
$this->localExecx("./tools/build/bin/fbmake.par gen_cdb");
$available_paths = $this->getAvailablePaths();
$lintable_paths = array_intersect($paths, $available_paths);
if ($paths != $lintable_paths) {
PhutilConsole::getConsole()->writeErr(
"Can't lint %s\n",
implode(', ', array_diff($paths, $available_paths)));
}
// Return what we know how to lint.
return $lintable_paths;
}
/**
* Get the available paths in the current compilation database.
*/
private function getAvailablePaths() {
$database_path = $this->getProjectRoot()
.'/_build/dev/compile_commands.json';
if (!file_exists($database_path)) {
return array();
}
$entries = json_decode(file_get_contents($database_path), true);
$paths = array();
foreach ($entries as $entry) {
$paths[] = $entry['file'];
}
return $paths;
}
/**
* Search for the targets directories for the given files.
*/
private static function getTargetsFor($paths) {
$targets = array();
foreach ($paths as $path) {
while (($path = dirname($path)) !== '.') {
if (in_array('TARGETS', scandir($path))) {
$contents = file_get_contents($path.'/TARGETS');
if (strpos($contents, 'cpp_binary') !== false) {
$targets[] = $path;
break;
}
}
}
}
return array_unique($targets);
}
/**
* The paths that we actually want to report on.
*/
private function getInterestingPaths($paths) {
$headers = array();
foreach ($paths as $path) {
$headers[] = preg_replace("/\.cpp$/", ".h", $path);
}
return array_merge($paths, $headers);
}
/**
* The path where the binary is located. Will return the current dewey binary
* unless the `HOWTOEVEN_BUILD` environment variable is set.
*/
private function getBinaryPath() {
$path = sprintf(
"/mnt/dewey/fbcode/.commits/%s/builds/howtoeven/client",
self::VERSION);
$build = getenv('HOWTOEVEN_BUILD');
if ($build) {
$path = sprintf(
"./_build/%s/tools/howtoeven/client",
$build);
if (!file_exists($path)) {
PhutilConsole::getConsole()->writeErr(">> %s does not exist\n", $path);
exit(1);
}
}
return $path;
}
/**
* Execute the command in the root directory.
*/
private function localExecx($command /* , ... */) {
$arguments = func_get_args();
return newv('ExecFuture', $arguments)
->setCWD($this->getProjectRoot())
->resolvex();
}
/**
* The root of the project.
*/
private function getProjectRoot() {
return $this->getEngine()->getWorkingCopy()->getProjectRoot();
}
private function getFilteredIssues() {
$issues = getenv('HOWTOEVEN_ISSUES');
return ($issues) ? csprintf('-issues %s', $issues) : '';
}
}

58
arcanist_util/cpp_linter/FbcodeClangFormatLinter.php
View File

@ -1,58 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
final class FbcodeClangFormatLinter extends BaseDirectoryScopedFormatLinter {
const LINT_FORMATTING = 1;
const CLANG_FORMAT_BINARY = '/mnt/vol/engshare/admin/scripts/clang-format';
protected function getPathsToLint() {
return array('');
}
public function getLinterName() {
return 'CLANG_FORMAT';
}
public function getLintSeverityMap() {
return array(
self::LINT_FORMATTING => ArcanistLintSeverity::SEVERITY_ADVICE,
);
}
public function getLintNameMap() {
return array(
self::LINT_FORMATTING => pht('Changes are not clang-formatted'),
);
}
protected function getFormatFuture($path, array $changed) {
$args = "";
foreach ($changed as $key => $value) {
$args .= " --lines=$key:$key";
}
$binary = self::CLANG_FORMAT_BINARY;
if (!file_exists($binary)) {
// trust the $PATH
$binary = "clang-format";
}
return new ExecFuture(
"%s %s $args",
$binary,
$this->getEngine()->getFilePathOnDisk($path));
}
protected function getLintMessage($diff) {
$link_to_clang_format =
"[[ http://fburl.com/clang-format | clang-format ]]";
return <<<LINT_MSG
Changes in this file were not formatted using $link_to_clang_format.
Please run build_tools/format-diff.sh or `make format`
LINT_MSG;
}
}

126
arcanist_util/cpp_linter/FbcodeCppLinter.php
View File

@ -1,126 +0,0 @@
<?php
// Copyright 2004-present Facebook. All rights reserved.
class FbcodeCppLinter extends ArcanistLinter {
const FLINT = "/home/engshare/tools/flint";
const LINT_ERROR = 1;
const LINT_WARNING = 2;
const LINT_ADVICE = 3;
const C_FLAG = "--c_mode=true";
private $rawLintOutput = array();
public function willLintPaths(array $paths) {
if (!file_exists(self::FLINT)) {
return;
}
$futures = array();
foreach ($paths as $p) {
$lpath = $this->getEngine()->getFilePathOnDisk($p);
$lpath_file = file($lpath);
if (preg_match('/\.(c)$/', $lpath) ||
preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
) {
$futures[$p] = new ExecFuture("%s %s %s 2>&1",
self::FLINT, self::C_FLAG,
$this->getEngine()->getFilePathOnDisk($p));
} else {
$futures[$p] = new ExecFuture("%s %s 2>&1",
self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
}
}
foreach (Futures($futures)->limit(8) as $p => $f) {
$this->rawLintOutput[$p] = $f->resolvex();
}
return;
}
public function getLinterName() {
return "FBCPP";
}
public function lintPath($path) {
$this->runCppLint($path);
}
private function runCppLint($path) {
$msgs = $this->getCppLintOutput($path);
foreach ($msgs as $m) {
$this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
}
}
private function adviseOnEachPattern(
$path,
$regex,
$message,
$lint_type = self::LINT_ADVICE,
$match_idx = 0) {
$file_data = $this->getData($path);
$matches = array();
if (!preg_match_all($regex, $file_data, $matches, PREG_OFFSET_CAPTURE)) {
return;
}
foreach ($matches[$match_idx] as $match) {
list($match_str, $offset) = $match;
$this->raiseLintAtOffset($offset, $lint_type, $message, $match_str);
}
}
public function getLintSeverityMap() {
return array(
self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
self::LINT_ADVICE => ArcanistLintSeverity::SEVERITY_ADVICE,
self::LINT_ERROR => ArcanistLintSeverity::SEVERITY_ERROR
);
}
public function getLintNameMap() {
return array(
self::LINT_ADVICE => "CppLint Advice",
self::LINT_WARNING => "CppLint Warning",
self::LINT_ERROR => "CppLint Error"
);
}
private function getCppLintOutput($path) {
if (!array_key_exists($path, $this->rawLintOutput)) {
return array();
}
list($output) = $this->rawLintOutput[$path];
$msgs = array();
$current = null;
$matches = array();
foreach (explode("\n", $output) as $line) {
if (preg_match('/.*?:(\d+):(.*)/', $line, $matches)) {
if ($current) {
$msgs[] = $current;
}
$line = $matches[1];
$text = $matches[2];
if (preg_match('/.*Warning.*/', $text)) {
$sev = self::LINT_WARNING;
} else if (preg_match('/.*Advice.*/', $text)) {
$sev = self::LINT_ADVICE;
} else {
$sev = self::LINT_ERROR;
}
$current = array('line' => $line,
'msg' => $text,
'severity' => $sev);
} else if ($current) {
$current['msg'] .= ' ' . $line;
}
}
if ($current) {
$msgs[] = $current;
}
return $msgs;
}
}

4767
arcanist_util/cpp_linter/cpplint.py vendored
View File

File diff suppressed because it is too large Load Diff

138
arcanist_util/lint_engine/FacebookFbcodeLintEngine.php
View File

@ -1,138 +0,0 @@
<?php
// Copyright 2004-present Facebook. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
class FacebookFbcodeLintEngine extends ArcanistLintEngine {
public function buildLinters() {
$linters = array();
$paths = $this->getPaths();
// Remove all deleted files, which are not checked by the
// following linters.
foreach ($paths as $key => $path) {
if (!Filesystem::pathExists($this->getFilePathOnDisk($path))) {
unset($paths[$key]);
}
}
$generated_linter = new ArcanistGeneratedLinter();
$linters[] = $generated_linter;
$nolint_linter = new ArcanistNoLintLinter();
$linters[] = $nolint_linter;
$text_linter = new ArcanistTextLinter();
$text_linter->setCustomSeverityMap(array(
ArcanistTextLinter::LINT_LINE_WRAP
=> ArcanistLintSeverity::SEVERITY_ADVICE,
));
$linters[] = $text_linter;
$java_text_linter = new ArcanistTextLinter();
$java_text_linter->setMaxLineLength(100);
$java_text_linter->setCustomSeverityMap(array(
ArcanistTextLinter::LINT_LINE_WRAP
=> ArcanistLintSeverity::SEVERITY_ADVICE,
));
$linters[] = $java_text_linter;
$python_linter = new ArcanistPEP8Linter();
$linters[] = $python_linter;
$cpp_linters = array();
$cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
$cpp_linters[] = $linters[] = new FbcodeCppLinter();
$clang_format_linter = new FbcodeClangFormatLinter();
$linters[] = $clang_format_linter;
$spelling_linter = new ArcanistSpellingLinter();
$linters[] = $spelling_linter;
foreach ($paths as $path) {
$is_text = false;
$text_extensions = (
'/\.('.
'cpp|cxx|c|cc|h|hpp|hxx|tcc|'.
'py|rb|hs|pl|pm|tw|'.
'php|phpt|css|js|'.
'java|'.
'thrift|'.
'lua|'.
'siv|'.
'txt'.
')$/'
);
if (preg_match($text_extensions, $path)) {
$is_text = true;
}
if ($is_text) {
$nolint_linter->addPath($path);
$generated_linter->addPath($path);
$generated_linter->addData($path, $this->loadData($path));
if (preg_match('/\.java$/', $path)) {
$java_text_linter->addPath($path);
$java_text_linter->addData($path, $this->loadData($path));
} else {
$text_linter->addPath($path);
$text_linter->addData($path, $this->loadData($path));
}
$spelling_linter->addPath($path);
$spelling_linter->addData($path, $this->loadData($path));
}
if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)
&& !preg_match('/third-party/', $path)) {
foreach ($cpp_linters as &$linter) {
$linter->addPath($path);
$linter->addData($path, $this->loadData($path));
}
$clang_format_linter->addPath($path);
$clang_format_linter->addData($path, $this->loadData($path));
$clang_format_linter->setPathChangedLines(
$path, $this->getPathChangedLines($path));
}
// Match *.py and contbuild config files
if (preg_match('/(\.(py|tw|smcprops)|^contbuild\/configs\/[^\/]*)$/',
$path)) {
$space_count = 4;
$real_path = $this->getFilePathOnDisk($path);
$dir = dirname($real_path);
do {
if (file_exists($dir.'/.python2space')) {
$space_count = 2;
break;
}
$dir = dirname($dir);
} while ($dir != '/' && $dir != '.');
$cur_path_linter = $python_linter;
$cur_path_linter->addPath($path);
$cur_path_linter->addData($path, $this->loadData($path));
if (preg_match('/\.tw$/', $path)) {
$cur_path_linter->setCustomSeverityMap(array(
'E251' => ArcanistLintSeverity::SEVERITY_DISABLED,
));
}
}
}
$name_linter = new ArcanistFilenameLinter();
$linters[] = $name_linter;
foreach ($paths as $path) {
$name_linter->addPath($path);
}
return $linters;
}
}

27
arcanist_util/lint_engine/FacebookHowtoevenLintEngine.php
View File

@ -1,27 +0,0 @@
<?php
// Copyright 2015-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
final class FacebookHowtoevenLintEngine extends ArcanistLintEngine {
public function buildLinters() {
$paths = array();
foreach ($this->getPaths() as $path) {
// Don't try to lint deleted files or changed directories.
if (!Filesystem::pathExists($path) || is_dir($path)) {
continue;
}
if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
$paths[] = $path;
}
}
$howtoeven = new FacebookHowtoevenLinter();
$howtoeven->setPaths($paths);
return array($howtoeven);
}
}

17
arcanist_util/unit_engine/FacebookFbcodeUnitTestEngine.php
View File

@ -1,17 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
class FacebookFbcodeUnitTestEngine extends ArcanistUnitTestEngine {
public function run() {
// For a call to `arc call-conduit differential.updateunitresults` to
// succeed we need at least one entry here.
$result = new ArcanistUnitTestResult();
$result->setName("dummy_placeholder_entry");
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
return array($result);
}
}

17
arcanist_util/unit_engine/FacebookOldFbcodeUnitTestEngine.php
View File

@ -1,17 +0,0 @@
<?php
// Copyright 2004-present Facebook. All Rights Reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
public function run() {
// For a call to `arc call-conduit differential.updateunitresults` to
// succeed we need at least one entry here.
$result = new ArcanistUnitTestResult();
$result->setName("dummy_placeholder_entry");
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
return array($result);
}
}

8
memtable/memtable_allocator.h
View File

@ -1,9 +1,7 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// This source code is also licensed under the GPLv2 license found in the
// COPYING file in the root directory of this source tree.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be