Merge pull request #824 from yuslepukhin/try_ci_tests_on_daily

Try running db_test during integration build

CMakeLists.txt

@@ -383,7 +383,7 @@ set(TESTS
         utilities/geodb/geodb_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
-        utilities/options_util_test.cc
+        utilities/options/options_util_test.cc
         utilities/redis/redis_lists_test.cc
         utilities/spatialdb/spatial_db_test.cc
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc

appveyor.yml

@@ -2,10 +2,14 @@ version: 1.0.{build}
 before_build:
 - md %APPVEYOR_BUILD_FOLDER%\build
 - cd %APPVEYOR_BUILD_FOLDER%\build
-- cmake -G "Visual Studio 12 Win64" ..
+- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
 - cd ..
 build:
   project: build\rocksdb.sln
   parallel: true
   verbosity: minimal
-test: off
+test:
+test_script:
+- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.Randomized -Concurrency 16
+- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1
+

appveyordailytests.yml

@@ -1,22 +0,0 @@
-version: 1.0.{build}
-before_build:
-- md %APPVEYOR_BUILD_FOLDER%\build
-- cd %APPVEYOR_BUILD_FOLDER%\build
-- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
-- cd ..
-build:
-  project: build\rocksdb.sln
-  parallel: true
-  verbosity: minimal
-test:
-test_script:
-- ps: build_tools\run_ci_db_test.ps1
-notifications:
-  - provider: Email
-    to:
-      - svmtrocksdb@microsoft.com
-    subject: "Build {{status}}"
-    message: "{{message}}, {{commitId}}, ..."
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true

build_tools/run_ci_db_test.ps1

@@ -5,13 +5,15 @@
 # Run the script from the enlistment
 Param(
   [switch]$EnableJE = $false,  # Use je executable
+  [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
   [string]$WorkFolder = "",  # Direct tests to use that folder
   [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
   [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|tests
+  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
-  [int]$Concurrency = 62
+   # a value of 1 will run everything serially
+  [int]$Concurrency = 16
 )
 
 # Folders and commands must be fullpath to run assuming
@@ -52,7 +54,7 @@ if($EnableJE) {
 }
 
 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "Binaries: $BinariesFolder exe: $db_test"
+Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 
 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -60,7 +62,7 @@ $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
 
 if($Exclude -ne "") {
     Write-Host "Exclude: $Exclude"
-    $l = $Exclude -split ','
+    $l = $Exclude -split ' '
     ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
 }
 
@@ -119,41 +121,60 @@ function Normalize-DbTests($HashTable) {
     }
 }
 
+# The function removes trailing .exe siffix if any,
+# creates a name for the log file
+function MakeAndAdd([string]$token, $HashTable) {
+    $test_name = $token -replace '.exe$', ''
+    $log_name =  -join ($test_name, ".log")
+    if(!$ExcludeTests.Contains($test)) {
+        $HashTable.Add($test_name, $log_name)
+    } else {
+        Write-Warning "Test $test_name is excluded"
+    }
+}
+
 # The function scans build\Debug folder to discover
 # Test executables. It then populates a table with
 # Test executable name -> Log file
-function Discover-TestBinaries($HashTable) {
+function Discover-TestBinaries([string]$Pattern, $HashTable) {
 
     $Exclusions = @("db_test*", "db_sanity_test*")
-    if($EnableJE) {
-        $p = -join ($BinariesFolder, "*_test_je.exe")
-    } else {
-        $p = -join ($BinariesFolder, "*_test.exe")
-    }
+
+    $p = -join ($BinariesFolder, $pattern)
+
+    Write-Host "Path: $p"
 
     dir -Path $p -Exclude $Exclusions | ForEach-Object {
-       $t = ($_.Name) -replace '.exe$', ''
-       if($ExcludeTests.Contains($t)) {
-            continue
-       }
-       $test_log = -join ($t, ".log")
-       $HashTable.Add($t, $test_log)
+       MakeAndAdd -token ($_.Name) -HashTable $HashTable
     }
 }
 
-$TestToLog = [ordered]@{}
+$TestsToRun = [ordered]@{}
 
 if($Run -ceq "db_test") {
-    Normalize-DbTests -HashTable $TestToLog
+    Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
-    Discover-TestBinaries -HashTable $TestToLog
+    if($EnableJE) {
+        $pattern = "*_test_je.exe"
     } else {
-    Write-Warning "Invalid -Run option value"
-    exit 2
+        $pattern = "*_test.exe"
+    }
+    Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
+} else {
+
+    $test_list = $Run -split ' '
+
+    ForEach($t in $test_list) {
+       MakeAndAdd -token $t -HashTable $TestsToRun
+    }
 }
 
+$NumTestsToStart = $TestsToRun.Count
+if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
+    $NumTestsToStart = $Limit
+}
 
-Write-Host "Attempting to start: " ($TestToLog.Count) " tests"
+Write-Host "Attempting to start: $NumTestsToStart tests"
 
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
@@ -167,20 +188,28 @@ $InvokeTestAsync = {
     &$exe > $log 2>&1
 }
 
-$jobs = @()
-$JobToLog = @{}
+# Hash that contains tests to rerun if any failed
+# Those tests will be rerun sequentially
+$Rerun = [ordered]@{}
 # Test limiting factor here
 $count = 0
-
+# Overall status
 [bool]$success = $true;
 
+function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
+{
+    # Array to wait for any of the running jobs
+    $jobs = @()
+    # Hash JobToLog
+    $JobToLog = @{}
+
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
           ($TestToLog.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
         # and the $Limit either not set or allows to proceed
-    while(($JobToLog.Count -lt $Concurrency) -and
+        while(($JobToLog.Count -lt $ConcurrencyVal) -and
               (($TestToLog.Count -gt 0) -and
               (($Limit -lt 0) -or ($count -lt $Limit)))) {
 
@@ -240,7 +269,7 @@ while(($JobToLog.Count -gt 0) -or
 
                 if(($l -match "^\[\s+PASSED") -or
                    ($l -match " : PASSED$") -or
-                ($l -match "^PASSED") -or
+                    ($l -match "^PASS$") -or   # Special c_test case
                     ($l -match "Passed all tests!") ) {
                     $pass_found = $true
                 }
@@ -250,6 +279,9 @@ while(($JobToLog.Count -gt 0) -or
                 $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
+                if($AddForRerun) {
+                    MakeAndAdd -token $completed.Name -HashTable $Rerun
+                }
             } else {
                 Write-Host $message
             }
@@ -259,9 +291,20 @@ while(($JobToLog.Count -gt 0) -or
         # Should be no output
         Receive-Job -Job $completed | Out-Null
     }
+}
+
+RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun
+
+if($Rerun.Count -gt 0) {
+    Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
+    $success = $true
+    $count = 0
+    RunJobs -TestToLog $Rerun -ConcurrencyVal $Concurrency -AddForRerun $false
+}
 
 Get-Date
 
+
 if(!$success) {
 # This does not succeed killing off jobs quick
 # So we simply exit