Run tests imporvements

Add sequential rerun for any failed tests. Add env_test case.
  Limit concurrency
  Allow to specify individual tests
  Take $Limit into account when displaying number of tests

appveyor.yml

@@ -10,14 +10,6 @@ build:
   verbosity: minimal
 test:
 test_script:
-- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16
+- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.Randomized -Concurrency 16
-notifications:
+- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1
-  - provider: Email
-    to:
-      - svmtrocksdb@microsoft.com
-    subject: "Build {{status}}"
-    message: "{{message}}, {{commitId}}, ..."
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true
 

appveyordailytests.yml

@@ -1,22 +0,0 @@
-version: 1.0.{build}
-before_build:
-- md %APPVEYOR_BUILD_FOLDER%\build
-- cd %APPVEYOR_BUILD_FOLDER%\build
-- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
-- cd ..
-build:
-  project: build\rocksdb.sln
-  parallel: true
-  verbosity: minimal
-test:
-test_script:
-- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16
-notifications:
-  - provider: Email
-    to:
-      - svmtrocksdb@microsoft.com
-    subject: "Build {{status}}"
-    message: "{{message}}, {{commitId}}, ..."
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true

build_tools/run_ci_db_test.ps1

@@ -5,13 +5,15 @@
 # Run the script from the enlistment
 Param(
   [switch]$EnableJE = $false,  # Use je executable
+  [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
   [string]$WorkFolder = "",  # Direct tests to use that folder
   [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
   [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|tests
+  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
-  [int]$Concurrency = 62
+   # a value of 1 will run everything serially
+  [int]$Concurrency = 16
 )
 
 # Folders and commands must be fullpath to run assuming
@@ -52,7 +54,7 @@ if($EnableJE) {
 }
 
 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "Binaries: $BinariesFolder exe: $db_test"
+Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 
 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -60,7 +62,7 @@ $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
 
 if($Exclude -ne "") {
     Write-Host "Exclude: $Exclude"
-    $l = $Exclude -split ','
+    $l = $Exclude -split ' '
     ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
 }
 
@@ -119,41 +121,60 @@ function Normalize-DbTests($HashTable) {
     }
 }
 
+# The function removes trailing .exe siffix if any,
+# creates a name for the log file
+function MakeAndAdd([string]$token, $HashTable) {
+    $test_name = $token -replace '.exe$', ''
+    $log_name =  -join ($test_name, ".log")
+    if(!$ExcludeTests.Contains($test)) {
+        $HashTable.Add($test_name, $log_name)
+    } else {
+        Write-Warning "Test $test_name is excluded"
+    }
+}
+
 # The function scans build\Debug folder to discover
 # Test executables. It then populates a table with
 # Test executable name -> Log file
-function Discover-TestBinaries($HashTable) {
+function Discover-TestBinaries([string]$Pattern, $HashTable) {
 
     $Exclusions = @("db_test*", "db_sanity_test*")
-    if($EnableJE) {
+
-        $p = -join ($BinariesFolder, "*_test_je.exe")
+    $p = -join ($BinariesFolder, $pattern)
-    } else {
+
-        $p = -join ($BinariesFolder, "*_test.exe")
+    Write-Host "Path: $p"
-    }
 
     dir -Path $p -Exclude $Exclusions | ForEach-Object {
-       $t = ($_.Name) -replace '.exe$', ''
+       MakeAndAdd -token ($_.Name) -HashTable $HashTable
-       if($ExcludeTests.Contains($t)) {
-            continue
-       }
-       $test_log = -join ($t, ".log")
-       $HashTable.Add($t, $test_log)
     }
 }
 
-$TestToLog = [ordered]@{}
+$TestsToRun = [ordered]@{}
 
 if($Run -ceq "db_test") {
-    Normalize-DbTests -HashTable $TestToLog
+    Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
-    Discover-TestBinaries -HashTable $TestToLog
+    if($EnableJE) {
+        $pattern = "*_test_je.exe"
+    } else {
+        $pattern = "*_test.exe"
+    }
+    Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
 } else {
-    Write-Warning "Invalid -Run option value"
+
-    exit 2
+    $test_list = $Run -split ' '
+
+    ForEach($t in $test_list) {
+       MakeAndAdd -token $t -HashTable $TestsToRun
+    }
 }
 
+$NumTestsToStart = $TestsToRun.Count
+if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
+    $NumTestsToStart = $Limit
+}
 
-Write-Host "Attempting to start: " ($TestToLog.Count) " tests"
+Write-Host "Attempting to start: $NumTestsToStart tests"
 
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
@@ -167,101 +188,123 @@ $InvokeTestAsync = {
     &$exe > $log 2>&1
 }
 
-$jobs = @()
+# Hash that contains tests to rerun if any failed
-$JobToLog = @{}
+# Those tests will be rerun sequentially
+$Rerun = [ordered]@{}
 # Test limiting factor here
 $count = 0
-
+# Overall status
 [bool]$success = $true;
 
-# Wait for all to finish and get the results
+function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
-while(($JobToLog.Count -gt 0) -or
+{
-      ($TestToLog.Count -gt 0)) {
+    # Array to wait for any of the running jobs
-
-    # Make sure we have maximum concurrent jobs running if anything
-    # and the $Limit either not set or allows to proceed
-    while(($JobToLog.Count -lt $Concurrency) -and
-          (($TestToLog.Count -gt 0) -and
-          (($Limit -lt 0) -or ($count -lt $Limit)))) {
-
-
-        # We only need the first key
-        foreach($key in $TestToLog.keys) {
-            $k = $key
-            break
-        }
-
-        Write-Host "Starting $k"
-        $log_path = -join ($LogFolder, ($TestToLog.$k))
-
-        if($Run -ceq "db_test") {
-          $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
-        } else {
-          [string]$Exe =  -Join ($BinariesFolder, $k)
-           $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
-        }
-
-        $JobToLog.Add($job, $log_path)
-        $TestToLog.Remove($k)
-
-        ++$count
-    }
-
-    if($JobToLog.Count -lt 1) {
-      break
-    }
-
     $jobs = @()
-    foreach($k in $JobToLog.Keys) { $jobs += $k }
+    # Hash JobToLog
+    $JobToLog = @{}
 
-    $completed = Wait-Job -Job $jobs -Any
+    # Wait for all to finish and get the results
-    $log = $JobToLog[$completed]
+    while(($JobToLog.Count -gt 0) -or
-    $JobToLog.Remove($completed)
+          ($TestToLog.Count -gt 0)) {
 
-    $message = -join @($completed.Name, " State: ", ($completed.State))
+        # Make sure we have maximum concurrent jobs running if anything
+        # and the $Limit either not set or allows to proceed
+        while(($JobToLog.Count -lt $ConcurrencyVal) -and
+              (($TestToLog.Count -gt 0) -and
+              (($Limit -lt 0) -or ($count -lt $Limit)))) {
 
-    $log_content = @(Get-Content $log)
 
-    if($completed.State -ne "Completed") {
+            # We only need the first key
-        $success = $false
+            foreach($key in $TestToLog.keys) {
-        Write-Warning $message
+                $k = $key
-        $log_content | Write-Warning
-    } else {
-        # Scan the log. If we find PASSED and no occurence of FAILED
-        # then it is a success
-        [bool]$pass_found = $false
-        ForEach($l in $log_content) {
-
-            if(($l -match "^\[\s+FAILED") -or
-               ($l -match "Assertion failed:")) {
-                $pass_found = $false
                 break
             }
 
-            if(($l -match "^\[\s+PASSED") -or
+            Write-Host "Starting $k"
-               ($l -match " : PASSED$") -or
+            $log_path = -join ($LogFolder, ($TestToLog.$k))
-                ($l -match "^PASSED") -or
+
-                ($l -match "Passed all tests!") ) {
+            if($Run -ceq "db_test") {
-                $pass_found = $true
+              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
+            } else {
+              [string]$Exe =  -Join ($BinariesFolder, $k)
+               $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
             }
+
+            $JobToLog.Add($job, $log_path)
+            $TestToLog.Remove($k)
+
+            ++$count
         }
 
-        if(!$pass_found) {
+        if($JobToLog.Count -lt 1) {
-            $success = $false;
+          break
+        }
+
+        $jobs = @()
+        foreach($k in $JobToLog.Keys) { $jobs += $k }
+
+        $completed = Wait-Job -Job $jobs -Any
+        $log = $JobToLog[$completed]
+        $JobToLog.Remove($completed)
+
+        $message = -join @($completed.Name, " State: ", ($completed.State))
+
+        $log_content = @(Get-Content $log)
+
+        if($completed.State -ne "Completed") {
+            $success = $false
             Write-Warning $message
             $log_content | Write-Warning
         } else {
-            Write-Host $message
+            # Scan the log. If we find PASSED and no occurence of FAILED
-        }
+            # then it is a success
-    }
+            [bool]$pass_found = $false
+            ForEach($l in $log_content) {
 
-    # Remove cached job info from the system
+                if(($l -match "^\[\s+FAILED") -or
-    # Should be no output
+                   ($l -match "Assertion failed:")) {
-    Receive-Job -Job $completed | Out-Null
+                    $pass_found = $false
+                    break
+                }
+
+                if(($l -match "^\[\s+PASSED") -or
+                   ($l -match " : PASSED$") -or
+                    ($l -match "^PASS$") -or   # Special c_test case
+                    ($l -match "Passed all tests!") ) {
+                    $pass_found = $true
+                }
+            }
+
+            if(!$pass_found) {
+                $success = $false;
+                Write-Warning $message
+                $log_content | Write-Warning
+                if($AddForRerun) {
+                    MakeAndAdd -token $completed.Name -HashTable $Rerun
+                }
+            } else {
+                Write-Host $message
+            }
+        }
+
+        # Remove cached job info from the system
+        # Should be no output
+        Receive-Job -Job $completed | Out-Null
+    }
+}
+
+RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun
+
+if($Rerun.Count -gt 0) {
+    Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
+    $success = $true
+    $count = 0
+    RunJobs -TestToLog $Rerun -ConcurrencyVal $Concurrency -AddForRerun $false
 }
 
 Get-Date
 
+
 if(!$success) {
 # This does not succeed killing off jobs quick
 # So we simply exit