Run tests imporvements
Add sequential rerun for any failed tests. Add env_test case. Limit concurrency Allow to specify individual tests Take $Limit into account when displaying number of tests
This commit is contained in:
yuslepukhin
parent
7f59e33b19
commit
935d1495c5
12
appveyor.yml
12
appveyor.yml
@ -10,14 +10,6 @@ build:
|
||||
verbosity: minimal
|
||||
test:
|
||||
test_script:
|
||||
- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16
|
||||
notifications:
|
||||
- provider: Email
|
||||
to:
|
||||
- svmtrocksdb@microsoft.com
|
||||
subject: "Build {{status}}"
|
||||
message: "{{message}}, {{commitId}}, ..."
|
||||
on_build_success: false
|
||||
on_build_failure: true
|
||||
on_build_status_changed: true
|
||||
- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.Randomized -Concurrency 16
|
||||
- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1
|
||||
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
version: 1.0.{build}
|
||||
before_build:
|
||||
- md %APPVEYOR_BUILD_FOLDER%\build
|
||||
- cd %APPVEYOR_BUILD_FOLDER%\build
|
||||
- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
|
||||
- cd ..
|
||||
build:
|
||||
project: build\rocksdb.sln
|
||||
parallel: true
|
||||
verbosity: minimal
|
||||
test:
|
||||
test_script:
|
||||
- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16
|
||||
notifications:
|
||||
- provider: Email
|
||||
to:
|
||||
- svmtrocksdb@microsoft.com
|
||||
subject: "Build {{status}}"
|
||||
message: "{{message}}, {{commitId}}, ..."
|
||||
on_build_success: false
|
||||
on_build_failure: true
|
||||
on_build_status_changed: true
|
||||
@ -5,13 +5,15 @@
|
||||
# Run the script from the enlistment
|
||||
Param(
|
||||
[switch]$EnableJE = $false, # Use je executable
|
||||
[switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
|
||||
[string]$WorkFolder = "", # Direct tests to use that folder
|
||||
[int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
|
||||
[string]$Exclude = "", # Expect a comma separated list, no spaces
|
||||
[string]$Run = "db_test", # Run db_test|tests
|
||||
[string]$Run = "db_test", # Run db_test|tests|testname1,testname2...
|
||||
# Number of async tasks that would run concurrently. Recommend a number below 64.
|
||||
# However, CPU utlization really depends on the storage media. Recommend ram based disk.
|
||||
[int]$Concurrency = 62
|
||||
# a value of 1 will run everything serially
|
||||
[int]$Concurrency = 16
|
||||
)
|
||||
|
||||
# Folders and commands must be fullpath to run assuming
|
||||
@ -52,7 +54,7 @@ if($EnableJE) {
|
||||
}
|
||||
|
||||
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
|
||||
Write-Output "Binaries: $BinariesFolder exe: $db_test"
|
||||
Write-Output "Binaries: $BinariesFolder db_test: $db_test"
|
||||
|
||||
#Exclusions that we do not want to run
|
||||
$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
|
||||
@ -60,7 +62,7 @@ $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
|
||||
|
||||
if($Exclude -ne "") {
|
||||
Write-Host "Exclude: $Exclude"
|
||||
$l = $Exclude -split ','
|
||||
$l = $Exclude -split ' '
|
||||
ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
|
||||
}
|
||||
|
||||
@ -119,41 +121,60 @@ function Normalize-DbTests($HashTable) {
|
||||
}
|
||||
}
|
||||
|
||||
# The function removes trailing .exe siffix if any,
|
||||
# creates a name for the log file
|
||||
function MakeAndAdd([string]$token, $HashTable) {
|
||||
$test_name = $token -replace '.exe$', ''
|
||||
$log_name = -join ($test_name, ".log")
|
||||
if(!$ExcludeTests.Contains($test)) {
|
||||
$HashTable.Add($test_name, $log_name)
|
||||
} else {
|
||||
Write-Warning "Test $test_name is excluded"
|
||||
}
|
||||
}
|
||||
|
||||
# The function scans build\Debug folder to discover
|
||||
# Test executables. It then populates a table with
|
||||
# Test executable name -> Log file
|
||||
function Discover-TestBinaries($HashTable) {
|
||||
function Discover-TestBinaries([string]$Pattern, $HashTable) {
|
||||
|
||||
$Exclusions = @("db_test*", "db_sanity_test*")
|
||||
if($EnableJE) {
|
||||
$p = -join ($BinariesFolder, "*_test_je.exe")
|
||||
} else {
|
||||
$p = -join ($BinariesFolder, "*_test.exe")
|
||||
}
|
||||
|
||||
$p = -join ($BinariesFolder, $pattern)
|
||||
|
||||
Write-Host "Path: $p"
|
||||
|
||||
dir -Path $p -Exclude $Exclusions | ForEach-Object {
|
||||
$t = ($_.Name) -replace '.exe$', ''
|
||||
if($ExcludeTests.Contains($t)) {
|
||||
continue
|
||||
}
|
||||
$test_log = -join ($t, ".log")
|
||||
$HashTable.Add($t, $test_log)
|
||||
MakeAndAdd -token ($_.Name) -HashTable $HashTable
|
||||
}
|
||||
}
|
||||
|
||||
$TestToLog = [ordered]@{}
|
||||
$TestsToRun = [ordered]@{}
|
||||
|
||||
if($Run -ceq "db_test") {
|
||||
Normalize-DbTests -HashTable $TestToLog
|
||||
Normalize-DbTests -HashTable $TestsToRun
|
||||
} elseif($Run -ceq "tests") {
|
||||
Discover-TestBinaries -HashTable $TestToLog
|
||||
if($EnableJE) {
|
||||
$pattern = "*_test_je.exe"
|
||||
} else {
|
||||
$pattern = "*_test.exe"
|
||||
}
|
||||
Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
|
||||
} else {
|
||||
Write-Warning "Invalid -Run option value"
|
||||
exit 2
|
||||
|
||||
$test_list = $Run -split ' '
|
||||
|
||||
ForEach($t in $test_list) {
|
||||
MakeAndAdd -token $t -HashTable $TestsToRun
|
||||
}
|
||||
}
|
||||
|
||||
$NumTestsToStart = $TestsToRun.Count
|
||||
if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
|
||||
$NumTestsToStart = $Limit
|
||||
}
|
||||
|
||||
Write-Host "Attempting to start: " ($TestToLog.Count) " tests"
|
||||
Write-Host "Attempting to start: $NumTestsToStart tests"
|
||||
|
||||
# Invoke a test with a filter and redirect all output
|
||||
$InvokeTestCase = {
|
||||
@ -167,101 +188,123 @@ $InvokeTestAsync = {
|
||||
&$exe > $log 2>&1
|
||||
}
|
||||
|
||||
$jobs = @()
|
||||
$JobToLog = @{}
|
||||
# Hash that contains tests to rerun if any failed
|
||||
# Those tests will be rerun sequentially
|
||||
$Rerun = [ordered]@{}
|
||||
# Test limiting factor here
|
||||
$count = 0
|
||||
|
||||
# Overall status
|
||||
[bool]$success = $true;
|
||||
|
||||
# Wait for all to finish and get the results
|
||||
while(($JobToLog.Count -gt 0) -or
|
||||
($TestToLog.Count -gt 0)) {
|
||||
|
||||
# Make sure we have maximum concurrent jobs running if anything
|
||||
# and the $Limit either not set or allows to proceed
|
||||
while(($JobToLog.Count -lt $Concurrency) -and
|
||||
(($TestToLog.Count -gt 0) -and
|
||||
(($Limit -lt 0) -or ($count -lt $Limit)))) {
|
||||
|
||||
|
||||
# We only need the first key
|
||||
foreach($key in $TestToLog.keys) {
|
||||
$k = $key
|
||||
break
|
||||
}
|
||||
|
||||
Write-Host "Starting $k"
|
||||
$log_path = -join ($LogFolder, ($TestToLog.$k))
|
||||
|
||||
if($Run -ceq "db_test") {
|
||||
$job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
|
||||
} else {
|
||||
[string]$Exe = -Join ($BinariesFolder, $k)
|
||||
$job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
|
||||
}
|
||||
|
||||
$JobToLog.Add($job, $log_path)
|
||||
$TestToLog.Remove($k)
|
||||
|
||||
++$count
|
||||
}
|
||||
|
||||
if($JobToLog.Count -lt 1) {
|
||||
break
|
||||
}
|
||||
|
||||
function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
|
||||
{
|
||||
# Array to wait for any of the running jobs
|
||||
$jobs = @()
|
||||
foreach($k in $JobToLog.Keys) { $jobs += $k }
|
||||
# Hash JobToLog
|
||||
$JobToLog = @{}
|
||||
|
||||
$completed = Wait-Job -Job $jobs -Any
|
||||
$log = $JobToLog[$completed]
|
||||
$JobToLog.Remove($completed)
|
||||
# Wait for all to finish and get the results
|
||||
while(($JobToLog.Count -gt 0) -or
|
||||
($TestToLog.Count -gt 0)) {
|
||||
|
||||
$message = -join @($completed.Name, " State: ", ($completed.State))
|
||||
# Make sure we have maximum concurrent jobs running if anything
|
||||
# and the $Limit either not set or allows to proceed
|
||||
while(($JobToLog.Count -lt $ConcurrencyVal) -and
|
||||
(($TestToLog.Count -gt 0) -and
|
||||
(($Limit -lt 0) -or ($count -lt $Limit)))) {
|
||||
|
||||
$log_content = @(Get-Content $log)
|
||||
|
||||
if($completed.State -ne "Completed") {
|
||||
$success = $false
|
||||
Write-Warning $message
|
||||
$log_content | Write-Warning
|
||||
} else {
|
||||
# Scan the log. If we find PASSED and no occurence of FAILED
|
||||
# then it is a success
|
||||
[bool]$pass_found = $false
|
||||
ForEach($l in $log_content) {
|
||||
|
||||
if(($l -match "^\[\s+FAILED") -or
|
||||
($l -match "Assertion failed:")) {
|
||||
$pass_found = $false
|
||||
# We only need the first key
|
||||
foreach($key in $TestToLog.keys) {
|
||||
$k = $key
|
||||
break
|
||||
}
|
||||
|
||||
if(($l -match "^\[\s+PASSED") -or
|
||||
($l -match " : PASSED$") -or
|
||||
($l -match "^PASSED") -or
|
||||
($l -match "Passed all tests!") ) {
|
||||
$pass_found = $true
|
||||
Write-Host "Starting $k"
|
||||
$log_path = -join ($LogFolder, ($TestToLog.$k))
|
||||
|
||||
if($Run -ceq "db_test") {
|
||||
$job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
|
||||
} else {
|
||||
[string]$Exe = -Join ($BinariesFolder, $k)
|
||||
$job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
|
||||
}
|
||||
|
||||
$JobToLog.Add($job, $log_path)
|
||||
$TestToLog.Remove($k)
|
||||
|
||||
++$count
|
||||
}
|
||||
|
||||
if(!$pass_found) {
|
||||
$success = $false;
|
||||
if($JobToLog.Count -lt 1) {
|
||||
break
|
||||
}
|
||||
|
||||
$jobs = @()
|
||||
foreach($k in $JobToLog.Keys) { $jobs += $k }
|
||||
|
||||
$completed = Wait-Job -Job $jobs -Any
|
||||
$log = $JobToLog[$completed]
|
||||
$JobToLog.Remove($completed)
|
||||
|
||||
$message = -join @($completed.Name, " State: ", ($completed.State))
|
||||
|
||||
$log_content = @(Get-Content $log)
|
||||
|
||||
if($completed.State -ne "Completed") {
|
||||
$success = $false
|
||||
Write-Warning $message
|
||||
$log_content | Write-Warning
|
||||
} else {
|
||||
Write-Host $message
|
||||
}
|
||||
}
|
||||
# Scan the log. If we find PASSED and no occurence of FAILED
|
||||
# then it is a success
|
||||
[bool]$pass_found = $false
|
||||
ForEach($l in $log_content) {
|
||||
|
||||
# Remove cached job info from the system
|
||||
# Should be no output
|
||||
Receive-Job -Job $completed | Out-Null
|
||||
if(($l -match "^\[\s+FAILED") -or
|
||||
($l -match "Assertion failed:")) {
|
||||
$pass_found = $false
|
||||
break
|
||||
}
|
||||
|
||||
if(($l -match "^\[\s+PASSED") -or
|
||||
($l -match " : PASSED$") -or
|
||||
($l -match "^PASS$") -or # Special c_test case
|
||||
($l -match "Passed all tests!") ) {
|
||||
$pass_found = $true
|
||||
}
|
||||
}
|
||||
|
||||
if(!$pass_found) {
|
||||
$success = $false;
|
||||
Write-Warning $message
|
||||
$log_content | Write-Warning
|
||||
if($AddForRerun) {
|
||||
MakeAndAdd -token $completed.Name -HashTable $Rerun
|
||||
}
|
||||
} else {
|
||||
Write-Host $message
|
||||
}
|
||||
}
|
||||
|
||||
# Remove cached job info from the system
|
||||
# Should be no output
|
||||
Receive-Job -Job $completed | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun
|
||||
|
||||
if($Rerun.Count -gt 0) {
|
||||
Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
|
||||
$success = $true
|
||||
$count = 0
|
||||
RunJobs -TestToLog $Rerun -ConcurrencyVal $Concurrency -AddForRerun $false
|
||||
}
|
||||
|
||||
Get-Date
|
||||
|
||||
|
||||
if(!$success) {
|
||||
# This does not succeed killing off jobs quick
|
||||
# So we simply exit
|
||||
|
||||
Loading…
Reference in New Issue
Block a user