Improve CI build and build switches
Add an optimized build config switch for faster test runs Change compiler options to introduce more opitmizations and be more inline with MS internal switches. Make appveyor build to utilize all the avaiable cores on the VM (parallel) Introduce new appveyor configuration for daily test runs as it would take too long to run db_test after each checkin even in paralell. With some exclusions we make it in 38 minutes. We currently fail to install ramdisk during the build. Add a powershell script to faicilitate paralell run for db_test cases.
This commit is contained in:
Dmitri Smirnov
parent
48b4497f75
commit
2e7506d82c
@ -45,11 +45,26 @@ add_custom_command(OUTPUT ${BUILD_VERSION_CC}
|
|||||||
|
|
||||||
add_custom_target(GenerateBuildVersion DEPENDS ${BUILD_VERSION_CC})
|
add_custom_target(GenerateBuildVersion DEPENDS ${BUILD_VERSION_CC})
|
||||||
|
|
||||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /W3 /WX /EHsc /GS /fp:precise /Zc:wchar_t /Zc:forScope /Gd /TP /errorReport:queue")
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
||||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /wd4018 /wd4100 /wd4101 /wd4127 /wd4189 /wd4200 /wd4244 /wd4267 /wd4296 /wd4305 /wd4307 /wd4309 /wd4512 /wd4701 /wd4702 /wd4800 /wd4804 /wd4996")
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /WX /wd4018 /wd4100 /wd4101 /wd4127 /wd4189 /wd4200 /wd4244 /wd4267 /wd4296 /wd4305 /wd4307 /wd4309 /wd4512 /wd4701 /wd4702 /wd4800 /wd4804 /wd4996")
|
||||||
|
|
||||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
|
# Used to run CI build and tests so we can run faster
|
||||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2 /Oi /Gm- /Gy /MD")
|
set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
|
||||||
|
|
||||||
|
if(DEFINED OPTDBG)
|
||||||
|
set(OPTIMIZE_DEBUG ${OPTDBG})
|
||||||
|
else()
|
||||||
|
set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if((${OPTIMIZE_DEBUG} EQUAL 1))
|
||||||
|
message("Debug optimization is enabled")
|
||||||
|
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
|
||||||
|
else()
|
||||||
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
|
||||||
|
|
||||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
|
||||||
|
|
||||||
|
|||||||
@ -6,5 +6,6 @@ before_build:
|
|||||||
- cd ..
|
- cd ..
|
||||||
build:
|
build:
|
||||||
project: build\ALL_BUILD.vcxproj
|
project: build\ALL_BUILD.vcxproj
|
||||||
|
parallel: true
|
||||||
verbosity: minimal
|
verbosity: minimal
|
||||||
test: off
|
test: off
|
||||||
|
|||||||
13
appveyordailytests.yml
Normal file
13
appveyordailytests.yml
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
version: 1.0.{build}
|
||||||
|
before_build:
|
||||||
|
- md %APPVEYOR_BUILD_FOLDER%\build
|
||||||
|
- cd %APPVEYOR_BUILD_FOLDER%\build
|
||||||
|
- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
|
||||||
|
- cd ..
|
||||||
|
build:
|
||||||
|
project: build\ALL_BUILD.vcxproj
|
||||||
|
parallel: true
|
||||||
|
verbosity: minimal
|
||||||
|
test:
|
||||||
|
test_script:
|
||||||
|
- ps: build_tools\run_ci_db_test.ps1
|
||||||
180
build_tools/run_ci_db_test.ps1
Normal file
180
build_tools/run_ci_db_test.ps1
Normal file
@ -0,0 +1,180 @@
|
|||||||
|
# Extract the names of its tests by running db_test with --gtest_list_tests.
|
||||||
|
# This filter removes the "#"-introduced comments, and expands to
|
||||||
|
# fully-qualified names by changing input like this:
|
||||||
|
#
|
||||||
|
# DBTest.
|
||||||
|
# Empty
|
||||||
|
# WriteEmptyBatch
|
||||||
|
# MultiThreaded/MultiThreadedDBTest.
|
||||||
|
# MultiThreaded/0 # GetParam() = 0
|
||||||
|
# MultiThreaded/1 # GetParam() = 1
|
||||||
|
#
|
||||||
|
# into this:
|
||||||
|
#
|
||||||
|
# DBTest.Empty
|
||||||
|
# DBTest.WriteEmptyBatch
|
||||||
|
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
|
||||||
|
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
|
||||||
|
|
||||||
|
# Folders and commands must be fullpath to run assuming
|
||||||
|
# the current folder is at the root of the git enlistment
|
||||||
|
Get-Date
|
||||||
|
# Limit the number of tests to start for debugging purposes
|
||||||
|
$limit = -1
|
||||||
|
|
||||||
|
$RootFolder = $pwd -replace '\\build_tools', ''
|
||||||
|
$LogFolder = -Join($RootFolder, "\db_logs\")
|
||||||
|
$TmpFolder = -Join($RootFolder, "\db_tests\")
|
||||||
|
$Env:TEST_TMPDIR = $TmpFolder
|
||||||
|
$global:db_test = -Join ($RootFolder, "\build\Debug\db_test.exe")
|
||||||
|
|
||||||
|
#Exclusions that we do not want to run
|
||||||
|
$ExcludeTests = @{
|
||||||
|
<#
|
||||||
|
"DBTest.HugeNumberOfLevels" = ""
|
||||||
|
"DBTest.SparseMerge" = ""
|
||||||
|
"DBTest.RateLimitingTest" = ""
|
||||||
|
"DBTest.kAbsoluteConsistency" = ""
|
||||||
|
"DBTest.GroupCommitTest" = ""
|
||||||
|
"DBTest.FileCreationRandomFailure" = ""
|
||||||
|
"DBTest.kTolerateCorruptedTailRecords" = ""
|
||||||
|
"DBTest.kSkipAnyCorruptedRecords" = ""
|
||||||
|
"DBTest.kPointInTimeRecovery" = ""
|
||||||
|
"DBTest.Randomized" = ""
|
||||||
|
#>
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create test directories in the current folder
|
||||||
|
md -Path $TmpFolder -ErrorAction Ignore
|
||||||
|
md -Path $LogFolder -ErrorAction Ignore
|
||||||
|
|
||||||
|
function Normalize-Tests([System.Array]$Tests, $HashTable) {
|
||||||
|
# Current group
|
||||||
|
$Group=""
|
||||||
|
|
||||||
|
ForEach( $l in $tests) {
|
||||||
|
# Trailing dot is a test group
|
||||||
|
if( $l -match "\.$") {
|
||||||
|
$Group = $l
|
||||||
|
} else {
|
||||||
|
# Otherwise it is a test name, remove leading space
|
||||||
|
$test = $l -replace '^\s+',''
|
||||||
|
# remove trailing comment if any and create a log name
|
||||||
|
$test = $test -replace '\s+\#.*',''
|
||||||
|
$test = "$Group$test"
|
||||||
|
|
||||||
|
if($ExcludeTests.Contains($test)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$test_log = $test -replace '[./]','_'
|
||||||
|
$test_log += ".log"
|
||||||
|
|
||||||
|
# Add to a hashtable
|
||||||
|
$HashTable.Add($test, $test_log);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run db_test to get a list of tests and store it into $a array
|
||||||
|
&$db_test --gtest_list_tests | tee -Variable TestList | Out-Null
|
||||||
|
|
||||||
|
# Parse the tests and store along with the log name into a hash
|
||||||
|
$TestToLog = [ordered]@{}
|
||||||
|
|
||||||
|
Normalize-Tests -Tests $TestList -HashTable $TestToLog
|
||||||
|
|
||||||
|
Write-Host "Attempting to start: " ($TestToLog.Count) " tests"
|
||||||
|
|
||||||
|
# Start jobs async each running a separate test
|
||||||
|
$AsyncScript = {
|
||||||
|
param($exe, $test, $log);
|
||||||
|
&$exe --gtest_filter=$test > $log 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
$jobs = @()
|
||||||
|
$JobToLog = @{}
|
||||||
|
# Test limiting factor here
|
||||||
|
$count = 0
|
||||||
|
|
||||||
|
ForEach($k in $TestToLog.keys) {
|
||||||
|
|
||||||
|
Write-Host "Starting $k"
|
||||||
|
$log_path = -join ($LogFolder, ($TestToLog.$k))
|
||||||
|
$job = Start-Job -Name $k -ScriptBlock $AsyncScript -ArgumentList @($db_test,$k,$log_path)
|
||||||
|
$JobToLog.Add($job, $log_path)
|
||||||
|
|
||||||
|
# Limiting trial runs
|
||||||
|
if(($limit -gt 0) -and (++$count -ge $limit)) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
$success = 1;
|
||||||
|
|
||||||
|
# Wait for all to finish and get the results
|
||||||
|
while($JobToLog.Count -gt 0) {
|
||||||
|
|
||||||
|
$jobs = @()
|
||||||
|
foreach($k in $JobToLog.Keys) { $jobs += $k }
|
||||||
|
|
||||||
|
<#
|
||||||
|
if(!$success) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
#>
|
||||||
|
|
||||||
|
$completed = Wait-Job -Job $jobs -Any
|
||||||
|
$log = $JobToLog[$completed]
|
||||||
|
$JobToLog.Remove($completed)
|
||||||
|
|
||||||
|
$message = -join @($completed.Name, " State: ", ($completed.State))
|
||||||
|
|
||||||
|
$log_content = @(Get-Content $log)
|
||||||
|
|
||||||
|
if($completed.State -ne "Completed") {
|
||||||
|
$success = 0
|
||||||
|
Write-Warning $message
|
||||||
|
$log_content | Write-Warning
|
||||||
|
} else {
|
||||||
|
# Scan the log. If we find PASSED and no occurence of FAILED
|
||||||
|
# then it is a success
|
||||||
|
$pass_found = 0
|
||||||
|
ForEach($l in $log_content) {
|
||||||
|
|
||||||
|
if($l -match "^\[\s+FAILED") {
|
||||||
|
$pass_found = 0
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if($l -match "^\[\s+PASSED") {
|
||||||
|
$pass_found = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!$pass_found) {
|
||||||
|
$success = 0;
|
||||||
|
Write-Warning $message
|
||||||
|
$log_content | Write-Warning
|
||||||
|
} else {
|
||||||
|
Write-Host $message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove cached job info from the system
|
||||||
|
# Should be no output
|
||||||
|
Receive-Job -Job $completed | Out-Null
|
||||||
|
}
|
||||||
|
|
||||||
|
Get-Date
|
||||||
|
|
||||||
|
if(!$success) {
|
||||||
|
# This does not succeed killing off jobs quick
|
||||||
|
# So we simply exit
|
||||||
|
# Remove-Job -Job $jobs -Force
|
||||||
|
# indicate failure using this exit code
|
||||||
|
exit 12345
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user