rocksdb/build_tools/run_ci_db_test.ps1
Dmitri Smirnov e5a1372b24 Rework test running script.
Summary:
Rework test running script.
  New options SuiteRun - runs specified executables as google suite
  test cases in parallel.
  Run - this option now runs executables in parallel the same as 'tests'
  RunAll - scans for test executables and attempts to run them all
  as suites except those mentiones in RunOnly (hardcoded in the script)
  or specified either in $ExcludeTestCases $ExcludeTestExe
Closes https://github.com/facebook/rocksdb/pull/2089

Differential Revision: D4832212

Pulled By: yiwu-arbug

fbshipit-source-id: 954990c
2017-04-05 11:39:20 -07:00

457 lines
13 KiB
PowerShell

# This script enables you running RocksDB tests by running
# All the tests concurrently and utilizing all the cores
Param(
[switch]$EnableJE = $false, # Look for and use _je executable, append _je to listed exclusions
[switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all
# of them as Google suites. I.e. It will run test cases concurrently
# except those mentioned as $Run, those will run as individual test cases
# And any execlued with $ExcludeExes or $ExcludeCases
# It will also not run any individual test cases
# excluded but $ExcludeCasese
[string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll
[string]$Run = "", # Run specified executables in parallel but do not split to test cases
[string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces
# Takes effect when $RunAll or $SuiteRun is specified. Must have full
# Test cases name including a group and a parameter if any
[string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list,
# no spaces. Takes effect only when $RunAll is specified
[string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options.
# Number of async tasks that would run concurrently. Recommend a number below 64.
# However, CPU utlization really depends on the storage media. Recommend ram based disk.
# a value of 1 will run everything serially
[int]$Concurrency = 8,
[int]$Limit = -1 # -1 means do not limit for test purposes
)
# Folders and commands must be fullpath to run assuming
# the current folder is at the root of the git enlistment
$StartDate = (Get-Date)
$StartDate
$DebugPreference = "Continue"
# These tests are not google test suites and we should guard
# Against running them as suites
$RunOnly = New-Object System.Collections.Generic.HashSet[string]
$RunOnly.Add("c_test") | Out-Null
$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
$RunOnly.Add("merge_test") | Out-Null
$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
if($RunAll -and $SuiteRun -ne "") {
Write-Error "$RunAll and $SuiteRun are not compatible"
exit 1
}
# If running under Appveyor assume that root
[string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
if($Appveyor -ne "") {
$RootFolder = $Appveyor
} else {
$RootFolder = $PSScriptRoot -replace '\\build_tools', ''
}
$LogFolder = -Join($RootFolder, "\db_logs\")
$BinariesFolder = -Join($RootFolder, "\build\Debug\")
if($WorkFolder -eq "") {
# If TEST_TMPDIR is set use it
[string]$var = $Env:TEST_TMPDIR
if($var -eq "") {
$WorkFolder = -Join($RootFolder, "\db_tests\")
$Env:TEST_TMPDIR = $WorkFolder
} else {
$WorkFolder = $var
}
} else {
# Override from a command line
$Env:TEST_TMPDIR = $WorkFolder
}
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
# Create test directories in the current folder
md -Path $WorkFolder -ErrorAction Ignore | Out-Null
md -Path $LogFolder -ErrorAction Ignore | Out-Null
$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
if($ExcludeCases -ne "") {
Write-Host "ExcludeCases: $ExcludeCases"
$l = $ExcludeCases -split ' '
ForEach($t in $l) {
$ExcludeCasesSet.Add($t) | Out-Null
}
}
$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
if($ExcludeExes -ne "") {
Write-Host "ExcludeExe: $ExcludeExes"
$l = $ExcludeExes -split ' '
ForEach($t in $l) {
$ExcludeExesSet.Add($t) | Out-Null
}
}
# Extract the names of its tests by running db_test with --gtest_list_tests.
# This filter removes the "#"-introduced comments, and expands to
# fully-qualified names by changing input like this:
#
# DBTest.
# Empty
# WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.
# MultiThreaded/0 # GetParam() = 0
# MultiThreaded/1 # GetParam() = 1
#
# into this:
#
# DBTest.Empty
# DBTest.WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
#
# Output into the parameter in a form TestName -> Log File Name
function ExtractTestCases([string]$GTestExe, $HashTable) {
$Tests = @()
# Run db_test to get a list of tests and store it into $a array
&$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
# Current group
$Group=""
ForEach( $l in $Tests) {
# Leading whitespace is fine
$l = $l -replace '^\s+',''
# but no whitespace any other place
if($l -match "\s+") {
continue
}
# Trailing dot is a test group but no whitespace
elseif ( $l -match "\.$" ) {
$Group = $l
} else {
# Otherwise it is a test name, remove leading space
$test = $l
# remove trailing comment if any and create a log name
$test = $test -replace '\s+\#.*',''
$test = "$Group$test"
if($ExcludeCasesSet.Contains($test)) {
Write-Warning "$test case is excluded"
continue
}
$test_log = $test -replace '[\./]','_'
$test_log += ".log"
$log_path = -join ($LogFolder, $test_log)
# Add to a hashtable
$HashTable.Add($test, $log_path);
}
}
}
# The function removes trailing .exe siffix if any,
# creates a name for the log file
# Then adds the test name if it was not excluded into
# a HashTable in a form of test_name -> log_path
function MakeAndAdd([string]$token, $HashTable) {
$test_name = $token -replace '.exe$', ''
$log_name = -join ($test_name, ".log")
$log_path = -join ($LogFolder, $log_name)
$HashTable.Add($test_name, $log_path)
}
# This function takes a list of Suites to run
# Lists all the test cases in each of the suite
# and populates HashOfHashes
# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
function ProcessSuites($ListOfSuites, $HashOfHashes) {
$suite_list = $ListOfSuites
# Problem: if you run --gtest_list_tests on
# a non Google Test executable then it will start executing
# and we will get nowhere
ForEach($suite in $suite_list) {
if($RunOnly.Contains($suite)) {
Write-Warning "$suite is excluded from running as Google test suite"
continue
}
if($EnableJE) {
$suite += "_je"
}
$Cases = [ordered]@{}
$Cases.Clear()
$suite_exe = -Join ($BinariesFolder, $suite)
ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
if($Cases.Count -gt 0) {
$HashOfHashes.Add($suite, $Cases);
}
}
# Make logs and run
if($CasesToRun.Count -lt 1) {
Write-Error "Failed to extract tests from $SuiteRun"
exit 1
}
}
# This will contain all test executables to run
# Hash table that contains all non suite
# Test executable to run
$TestExes = [ordered]@{}
# Check for test exe that are not
# Google Test Suites
# Since this is explicitely mentioned it is not subject
# for exclusions
if($Run -ne "") {
$test_list = $Run -split ' '
ForEach($t in $test_list) {
if($EnableJE) {
$t += "_je"
}
MakeAndAdd -token $t -HashTable $TestExes
}
if($TestExes.Count -lt 1) {
Write-Error "Failed to extract tests from $Run"
exit 1
}
}
# Ordered by exe @{ Exe = @{ TestCase = LogName }}
$CasesToRun = [ordered]@{}
if($SuiteRun -ne "") {
$suite_list = $SuiteRun -split ' '
ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
}
if($RunAll) {
# Discover all the test binaries
if($EnableJE) {
$pattern = "*_test_je.exe"
} else {
$pattern = "*_test.exe"
}
$search_path = -join ($BinariesFolder, $pattern)
Write-Host "Binaries Search Path: $search_path"
$ListOfExe = @()
dir -Path $search_path | ForEach-Object {
$ListOfExe += ($_.Name)
}
# Exclude those in RunOnly from running as suites
$ListOfSuites = @()
ForEach($e in $ListOfExe) {
$e = $e -replace '.exe$', ''
$bare_name = $e -replace '_je$', ''
if($ExcludeExesSet.Contains($bare_name)) {
Write-Warning "Test $e is excluded"
continue
}
if($RunOnly.Contains($e)) {
MakeAndAdd -token $e -HashTable $TestExes
} else {
$ListOfSuites += $bare_name
}
}
ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
}
Write-Host "Attempting to start: $NumTestsToStart tests"
# Invoke a test with a filter and redirect all output
$InvokeTestCase = {
param($exe, $test, $log);
&$exe --gtest_filter=$test > $log 2>&1
}
# Invoke all tests and redirect output
$InvokeTestAsync = {
param($exe, $log)
&$exe > $log 2>&1
}
# Hash that contains tests to rerun if any failed
# Those tests will be rerun sequentially
# $Rerun = [ordered]@{}
# Test limiting factor here
[int]$count = 0
# Overall status
[bool]$success = $true;
function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
{
# Array to wait for any of the running jobs
$jobs = @()
# Hash JobToLog
$JobToLog = @{}
# Wait for all to finish and get the results
while(($JobToLog.Count -gt 0) -or
($TestCmds.Count -gt 0) -or
($Suites.Count -gt 0)) {
# Make sure we have maximum concurrent jobs running if anything
# and the $Limit either not set or allows to proceed
while(($JobToLog.Count -lt $ConcurrencyVal) -and
((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
(($Limit -lt 0) -or ($count -lt $Limit)))) {
# We always favore suites to run if available
[string]$exe_name = ""
[string]$log_path = ""
$Cases = @{}
if($Suites.Count -gt 0) {
# Will teh first one
ForEach($e in $Suites.Keys) {
$exe_name = $e
$Cases = $Suites[$e]
break
}
[string]$test_case = ""
[string]$log_path = ""
ForEach($c in $Cases.Keys) {
$test_case = $c
$log_path = $Cases[$c]
break
}
Write-Host "Starting $exe_name::$test_case"
[string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
$JobToLog.Add($job, $log_path)
$Cases.Remove($test_case)
if($Cases.Count -lt 1) {
$Suites.Remove($exe_name)
}
} elseif ($TestCmds.Count -gt 0) {
ForEach($e in $TestCmds.Keys) {
$exe_name = $e
$log_path = $TestCmds[$e]
break
}
[string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
$JobToLog.Add($job, $log_path)
$TestCmds.Remove($exe_name)
} else {
Write-Error "In the job loop but nothing to run"
exit 1
}
++$count
} # End of Job starting loop
if($JobToLog.Count -lt 1) {
break
}
$jobs = @()
foreach($k in $JobToLog.Keys) { $jobs += $k }
$completed = Wait-Job -Job $jobs -Any
$log = $JobToLog[$completed]
$JobToLog.Remove($completed)
$message = -join @($completed.Name, " State: ", ($completed.State))
$log_content = @(Get-Content $log)
if($completed.State -ne "Completed") {
$success = $false
Write-Warning $message
$log_content | Write-Warning
} else {
# Scan the log. If we find PASSED and no occurrence of FAILED
# then it is a success
[bool]$pass_found = $false
ForEach($l in $log_content) {
if(($l -match "^\[\s+FAILED") -or
($l -match "Assertion failed:")) {
$pass_found = $false
break
}
if(($l -match "^\[\s+PASSED") -or
($l -match " : PASSED$") -or
($l -match "^PASS$") -or # Special c_test case
($l -match "Passed all tests!") ) {
$pass_found = $true
}
}
if(!$pass_found) {
$success = $false;
Write-Warning $message
$log_content | Write-Warning
} else {
Write-Host $message
}
}
# Remove cached job info from the system
# Should be no output
Receive-Job -Job $completed | Out-Null
}
}
RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
$EndDate = (Get-Date)
New-TimeSpan -Start $StartDate -End $EndDate |
ForEach-Object {
"Elapsed time: {0:g}" -f $_
}
if(!$success) {
# This does not succeed killing off jobs quick
# So we simply exit
# Remove-Job -Job $jobs -Force
# indicate failure using this exit code
exit 1
}
exit 0