Regression tests (#37)
* added error regression tests * Add output change tracking * adding integration test files * remove * fix githook * reset changes to formatters * add docs/tools * fix inittest-requirement
Chris Kuech
GitHub
parent
2dd0757946
commit
e8bbc5267e
|
|
@ -0,0 +1,12 @@
|
|||
|
||||
$preCommitHook = @"
|
||||
#/bin/bash
|
||||
|
||||
pwsh ./test/Checkpoint-Output.ps1
|
||||
git add ./test/integration
|
||||
"@
|
||||
|
||||
$preCommitHookPath = "$PSScriptRoot/.git/hooks/pre-commit"
|
||||
|
||||
$preCommitHook > $preCommitHookPath
|
||||
chmod u+x $preCommitHookPath
|
||||
103
example.ps1
103
example.ps1
|
|
@ -1,103 +0,0 @@
|
|||
|
||||
$ErrorActionPreference = "Stop"
|
||||
Import-Module "$PSScriptRoot\Requirements.psd1" -Force
|
||||
|
||||
$requirements = @(
|
||||
@{
|
||||
Name = "Resource 1"
|
||||
Describe = "Resource 1 is present in the system"
|
||||
Test = { $mySystem -contains 1 }
|
||||
Set = { $mySystem.Add(1) | Out-Null; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 2"
|
||||
Describe = "Resource 2 is present in the system"
|
||||
Test = { $mySystem -contains 2 }
|
||||
Set = { $mySystem.Add(2) | Out-Null; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Checkpoint"
|
||||
Describe = "Assert before continuing"
|
||||
Test = { $mySystem }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 3"
|
||||
Describe = "Resource 3 is present in the system"
|
||||
Test = { $mySystem -contains 3 }
|
||||
Set = { throw "This should not have been reached!"; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 4"
|
||||
Describe = "Resource 4 is present in the system"
|
||||
Test = { $mySystem -contains 4 }
|
||||
Set = { throw "This should not have been reached!"; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Always set"
|
||||
Describe = "Always run the set command"
|
||||
Set = { Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 5"
|
||||
Describe = "Resource 5 is present in the system"
|
||||
Test = { $mySystem -contains 5 }
|
||||
Set = { $mySystem.Add(5) | Out-Null; Start-Sleep 1 }
|
||||
}
|
||||
)
|
||||
|
||||
# demo using Format-Table
|
||||
$mySystem = [System.Collections.ArrayList]::new()
|
||||
$mySystem.Add(3) | Out-Null
|
||||
$mySystem.Add(4) | Out-Null
|
||||
$requirements | Invoke-Requirement | Format-Table
|
||||
|
||||
# demo using Format-Checklist
|
||||
$mySystem = [System.Collections.ArrayList]::new()
|
||||
$mySystem.Add(3) | Out-Null
|
||||
$mySystem.Add(4) | Out-Null
|
||||
$requirements | Invoke-Requirement | Format-Checklist
|
||||
|
||||
# demo using Format-CallStack
|
||||
$mySystem = [System.Collections.ArrayList]::new()
|
||||
$mySystem.Add(3) | Out-Null
|
||||
$mySystem.Add(4) | Out-Null
|
||||
$requirements | Invoke-Requirement | Format-CallStack
|
||||
|
||||
# demo using Format-Callstack with nested requirements
|
||||
$mySystem = [System.Collections.ArrayList]::new()
|
||||
$complexRequirements = @(
|
||||
@{
|
||||
Name = "Resource 1"
|
||||
Describe = "Resource 1 is present in the system"
|
||||
Test = { $mySystem -contains 1 }
|
||||
Set = { $mySystem.Add(1) | Out-Null; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 2"
|
||||
Describe = "Resource 2 is present in the system"
|
||||
Test = { $mySystem -contains 3 -and $mySystem -contains 4 }
|
||||
Set = {
|
||||
@(
|
||||
@{
|
||||
Name = "Resource 3"
|
||||
Describe = "Resource 3 is present in the system"
|
||||
Test = { $mySystem -contains 3 }
|
||||
Set = { $mySystem.Add(3) | Out-Null; Start-Sleep 1 }
|
||||
},
|
||||
@{
|
||||
Name = "Resource 4"
|
||||
Describe = "Resource 4 is present in the system"
|
||||
Test = { $mySystem -contains 4 }
|
||||
Set = { $mySystem.Add(4) | Out-Null; Start-Sleep 1 }
|
||||
}
|
||||
) | Invoke-Requirement
|
||||
}
|
||||
},
|
||||
@{
|
||||
Name = "Resource 5"
|
||||
Describe = "Resource 5 is present in the system"
|
||||
Test = { $mySystem -contains 5 }
|
||||
Set = { $mySystem.Add(5) | Out-Null; Start-Sleep 1 }
|
||||
}
|
||||
)
|
||||
$complexRequirements | Invoke-Requirement | Format-CallStack
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Save output of formatters so that output of commands are diffed in tests
|
||||
.NOTES
|
||||
Called in a precommit git hook
|
||||
#>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$RepoRoot = "$PSScriptRoot/.."
|
||||
$OutRoot = "$PSScriptRoot/integration"
|
||||
|
||||
."$RepoRoot/src/interface.ps1"
|
||||
|
||||
@{
|
||||
Describe = "Integration test output root '$OutRoot' exists"
|
||||
Test = { Test-Path $OutRoot -PathType Container }
|
||||
Set = { New-Item -ItemType Directory -Path $OutRoot }
|
||||
} | Invoke-Requirement | Out-Null
|
||||
|
||||
$context = @{count = 0}
|
||||
|
||||
$Requirements = @{
|
||||
Test = @{
|
||||
Name = "MyName"
|
||||
Describe = "MyDescribe"
|
||||
Test = { $true }
|
||||
}
|
||||
Set = @{
|
||||
Name = "MyName"
|
||||
Describe = "MyDescribe"
|
||||
Set = { $true }
|
||||
}
|
||||
TestSet = @{
|
||||
Name = "MyName"
|
||||
Describe = "MyDescribe"
|
||||
Test = { $context.count++ % 2 -eq 1 }
|
||||
Set = { $true }
|
||||
}
|
||||
}
|
||||
|
||||
$Requirements.Keys `
|
||||
| % {
|
||||
$events = $Requirements[$_] | Invoke-Requirement
|
||||
$events | Format-CallStack *> "$OutRoot/Format-CallStack.$_.txt"
|
||||
$events | Format-Checklist *> "$OutRoot/Format-Checklist.$_.txt"
|
||||
$events | Format-Table *> "$OutRoot/Format-Table.$_.txt"
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# Testing
|
||||
|
||||
## Unit testing
|
||||
Pester tests are set up in this folder. They are run in CI by `/.github/scripts/Assert-Tests.ps1`.
|
||||
|
||||
## Managing output of formatters
|
||||
`/test/Checkpoint-Output.ps1` saves cases of formatter output to `/test/integration`. These examples are checked into source control so they can be diffed in pull requests.
|
||||
|
||||
Run `/Initialize-Repo.ps1` to configure a pre-commit git hook to automatically run `/test/Checkpoint-Output.ps1` prior to committing changes.
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
12:04:37 [MyName] BEGIN SET MyDescribe
|
||||
12:04:37 [MyName] END SET
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
12:04:37 [MyName] BEGIN TEST MyDescribe
|
||||
12:04:37 [MyName] END TEST => True
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
12:04:37 [MyName] BEGIN TEST MyDescribe
|
||||
12:04:37 [MyName] END TEST => False
|
||||
12:04:37 [MyName] BEGIN SET MyDescribe
|
||||
12:04:37 [MyName] END SET
|
||||
12:04:37 [MyName] BEGIN TEST MyDescribe
|
||||
12:04:37 [MyName] END TEST => True
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
12:04:37 [ ] MyDescribe
|
||||
|
||||
12:04:37 [ √ ] MyDescribe
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
12:04:37 [ ] MyDescribe
|
||||
|
||||
12:04:37 [ √ ] MyDescribe
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
12:04:37 [ ] MyDescribe
|
||||
|
||||
12:04:37 [ √ ] MyDescribe
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
Date Method State Result Requirement
|
||||
---- ------ ----- ------ -----------
|
||||
9/21/19 12:04:37 AM Set Start MyName
|
||||
9/21/19 12:04:37 AM Set Stop True MyName
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
Date Method State Result Requirement
|
||||
---- ------ ----- ------ -----------
|
||||
9/21/19 12:04:37 AM Test Start MyName
|
||||
9/21/19 12:04:37 AM Test Stop True MyName
|
||||
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
|
||||
Date Method State Result Requirement
|
||||
---- ------ ----- ------ -----------
|
||||
9/21/19 12:04:37 AM Test Start MyName
|
||||
9/21/19 12:04:37 AM Test Stop False MyName
|
||||
9/21/19 12:04:37 AM Set Start MyName
|
||||
9/21/19 12:04:37 AM Set Stop True MyName
|
||||
9/21/19 12:04:37 AM Validate Start MyName
|
||||
9/21/19 12:04:37 AM Validate Stop True MyName
|
||||
|
||||
Loading…
Reference in New Issue