Merge pull request #1490 from bcullman/sql-backup-add-retension

Add retention capabilities to SQL Backup Database Step Template

Author: harrisonmeister

step-templates/sql-backup-database.json

@@ -3,9 +3,9 @@
   "Name": "SQL - Backup Database",
   "Description": "Backup a MS SQL Server database to the file system.",
   "ActionType": "Octopus.Script",
-  "Version": 10,
+  "Version": 11,
   "Properties": {
-    "Octopus.Action.Script.ScriptBody": "$ServerName = $OctopusParameters['Server']\r\n$DatabaseName = $OctopusParameters['Database']\r\n$BackupDirectory = $OctopusParameters['BackupDirectory']\r\n$CompressionOption = [int]$OctopusParameters['Compression']\r\n$Devices = [int]$OctopusParameters['Devices']\r\n$Stamp = $OctopusParameters['Stamp']\r\n$UseSqlServerTimeStamp = $OctopusParameters['UseSqlServerTimeStamp']\r\n$SqlLogin = $OctopusParameters['SqlLogin']\r\n$SqlPassword = $OctopusParameters['SqlPassword']\r\n$ConnectionTimeout = $OctopusParameters['ConnectionTimeout']\r\n$Incremental = [boolean]::Parse($OctopusParameters['Incremental'])\r\n$CopyOnly = [boolean]::Parse($OctopusParameters['CopyOnly'])\r\n\r\n$ErrorActionPreference = \"Stop\"\r\n\r\nfunction ConnectToDatabase()\r\n{\r\n    param($server, $SqlLogin, $SqlPassword)\r\n\r\n    $server.ConnectionContext.StatementTimeout = $ConnectionTimeout\r\n\r\n    if ($SqlLogin -ne $null) {\r\n\r\n        if ($SqlPassword -eq $null) {\r\n            throw \"SQL Password must be specified when using SQL authentication.\"\r\n        }\r\n\r\n        $server.ConnectionContext.LoginSecure = $false\r\n        $server.ConnectionContext.Login = $SqlLogin\r\n        $server.ConnectionContext.Password = $SqlPassword\r\n\r\n        Write-Host \"Connecting to server using SQL authentication as $SqlLogin.\"\r\n        $server = New-Object Microsoft.SqlServer.Management.Smo.Server $server.ConnectionContext\r\n    }\r\n    else {\r\n        Write-Host \"Connecting to server using Windows authentication.\"\r\n    }\r\n\r\n    try {\r\n        $server.ConnectionContext.Connect()\r\n    } catch {\r\n        Write-Error \"An error occurred connecting to the database server!`r`n$($_.Exception.ToString())\"\r\n    }\r\n}\r\n\r\nfunction AddPercentHandler {\r\n    param($smoBackupRestore, $action)\r\n\r\n    $percentEventHandler = [Microsoft.SqlServer.Management.Smo.PercentCompleteEventHandler] { Write-Host $dbName $action $_.Percent \"%\" }\r\n    $completedEventHandler = [Microsoft.SqlServer.Management.Common.ServerMessageEventHandler] { Write-Host $_.Error.Message}\r\n\r\n    $smoBackupRestore.add_PercentComplete($percentEventHandler)\r\n    $smoBackupRestore.add_Complete($completedEventHandler)\r\n    $smoBackupRestore.PercentCompleteNotification=10\r\n}\r\n\r\nfunction CreatDevice {\r\n    param($smoBackupRestore, $directory, $name)\r\n\r\n    $devicePath = [System.IO.Path]::Combine($directory, $name)\r\n    $smoBackupRestore.Devices.AddDevice($devicePath, \"File\")\r\n    return $devicePath\r\n}\r\n\r\nfunction CreateDevices {\r\n    param($smoBackupRestore, $devices, $directory, $dbName, $incremental)\r\n\r\n    $targetPaths = New-Object System.Collections.Generic.List[System.String]\r\n\r\n\t$extension = \".bak\"\r\n\r\n\tif ($Incremental -eq $true){\r\n\t\t$extension = \".trn\"\r\n\t}\r\n\r\n    if ($devices -eq 1){\r\n        $deviceName = $dbName + \"_\" + $timestamp + $extension\r\n        $targetPath = CreatDevice $smoBackupRestore $directory $deviceName\r\n        $targetPaths.Add($targetPath)\r\n    } else {\r\n        for ($i=1; $i -le $devices; $i++){\r\n            $deviceName = $dbName + \"_\" + $timestamp + \"_\" + $i + $extension\r\n            $targetPath = CreatDevice $smoBackupRestore $directory $deviceName\r\n            $targetPaths.Add($targetPath)\r\n        }\r\n    }\r\n    return $targetPaths\r\n}\r\n\r\nfunction BackupDatabase {\r\n    param($dbName, $devices, $compressionOption, $incremental, $copyonly)\r\n\r\n    $smoBackup = New-Object Microsoft.SqlServer.Management.Smo.Backup\r\n    $targetPaths = CreateDevices $smoBackup $devices $BackupDirectory $dbName $incremental\r\n\r\n    Write-Host \"Attempting to backup database $ServerName.$dbName to:\"\r\n    $targetPaths\r\n    Write-Host \"\"\r\n\r\n\tif ($incremental -eq $true){\r\n\t\t$smoBackup.Action = \"Log\";\r\n\t\t$smoBackup.BackupSetDescription = \"Log backup of \" + $dbName\r\n\t\t$smoBackup.LogTruncation = \"Truncate\"\r\n\t} else {\r\n\t\t$smoBackup.Action = \"Database\"\r\n\t\t$smoBackup.BackupSetDescription = \"Full Backup of \" + $dbName\r\n\t}\r\n\r\n\t$smoBackup.BackupSetName = $dbName + \" Backup\"\r\n\t$smoBackup.MediaDescription = \"Disk\"\r\n\t$smoBackup.CompressionOption = $compressionOption\r\n\t$smoBackup.CopyOnly = $copyonly\r\n\t$smoBackup.Initialize = $true\r\n\t$smoBackup.Database = $dbName;\r\n\r\n    try {\r\n        AddPercentHandler $smoBackup \"backed up\"\r\n        $smoBackup.SqlBackup($server)\r\n    } catch {\r\n        Write-Error \"An error occurred backing up the database!`r`n$($_.Exception.ToString())\"\r\n    }\r\n\r\n    Write-Host \"Backup completed successfully.\"\r\n}\r\n\r\n[System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SMO\") | Out-Null\r\n[System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SmoExtended\") | Out-Null\r\n[System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.ConnectionInfo\") | Out-Null\r\n[System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SmoEnum\") | Out-Null\r\n\r\n$server = New-Object Microsoft.SqlServer.Management.Smo.Server $ServerName\r\n\r\nConnectToDatabase $server $SqlLogin $SqlPassword\r\n\r\n$database = $server.Databases | Where-Object { $_.Name -eq $DatabaseName }\r\n$timestampFormat = \"yyyy-MM-dd-HHmmss\"\r\nif ($UseSqlServerTimeStamp -eq $true)\r\n{\r\n    $timestampFormat = \"yyyyMMdd_HHmmss\"\r\n}\r\n$timestamp = if(-not [string]::IsNullOrEmpty($Stamp)) { $Stamp } else { Get-Date -format $timestampFormat }\r\n\r\nif ($database -eq $null) {\r\n    Write-Error \"Database $DatabaseName does not exist on $ServerName\"\r\n}\r\n\r\nif ($Incremental -eq $true) {\r\n\r\n\tif ($database.RecoveryModel -eq 3) {\r\n\t\twrite-error \"$DatabaseName has Recovery Model set to Simple. Log backup cannot be run.\"\r\n\t}\r\n\r\n\tif ($database.LastBackupDate -eq \"1/1/0001 12:00 AM\") {\r\n\t\twrite-error \"$DatabaseName has no Full backups. Log backup cannot be run.\"\r\n\t}\r\n}\r\n\r\nBackupDatabase $DatabaseName $Devices $CompressionOption $Incremental $CopyOnly\r\n",
+    "Octopus.Action.Script.ScriptBody": "$ErrorActionPreference = \"Stop\"\n\nfunction ConnectToDatabase() {\n  param($server, $SqlLogin, $SqlPassword, $ConnectionTimeout)\n\n  $server.ConnectionContext.StatementTimeout = $ConnectionTimeout\n\n  if ($null -ne $SqlLogin) {\n\n    if ($null -eq $SqlPassword) {\n      throw \"SQL Password must be specified when using SQL authentication.\"\n    }\n\n    $server.ConnectionContext.LoginSecure = $false\n    $server.ConnectionContext.Login = $SqlLogin\n    $server.ConnectionContext.Password = $SqlPassword\n\n    Write-Host \"Connecting to server using SQL authentication as $SqlLogin.\"\n    $server = New-Object Microsoft.SqlServer.Management.Smo.Server $server.ConnectionContext\n  }\n  else {\n    Write-Host \"Connecting to server using Windows authentication.\"\n  }\n\n  try {\n    $server.ConnectionContext.Connect()\n  }\n  catch {\n    Write-Error \"An error occurred connecting to the database server!`r`n$($_.Exception.ToString())\"\n  }\n}\n\nfunction AddPercentHandler {\n  param($smoBackupRestore, $action)\n\n  $percentEventHandler = [Microsoft.SqlServer.Management.Smo.PercentCompleteEventHandler] { Write-Host $dbName $action $_.Percent \"%\" }\n  $completedEventHandler = [Microsoft.SqlServer.Management.Common.ServerMessageEventHandler] { Write-Host $_.Error.Message }\n\n  $smoBackupRestore.add_PercentComplete($percentEventHandler)\n  $smoBackupRestore.add_Complete($completedEventHandler)\n  $smoBackupRestore.PercentCompleteNotification = 10\n}\n\nfunction CreateDevice {\n  param($smoBackupRestore, $directory, $name)\n\n  $devicePath = [System.IO.Path]::Combine($directory, $name)\n  $smoBackupRestore.Devices.AddDevice($devicePath, \"File\")\n  return $devicePath\n}\n\nfunction CreateDevices {\n  param($smoBackupRestore, $devices, $directory, $dbName, $incremental, $timestamp)\n\n  $targetPaths = New-Object System.Collections.Generic.List[System.String]\n\n  $extension = \".bak\"\n\n  if ($incremental -eq $true) {\n    $extension = \".trn\"\n  }\n\n  if ($devices -eq 1) {\n    $deviceName = $dbName + \"_\" + $timestamp + $extension\n    $targetPath = CreateDevice $smoBackupRestore $directory $deviceName\n    $targetPaths.Add($targetPath)\n  }\n  else {\n    for ($i = 1; $i -le $devices; $i++) {\n      $deviceName = $dbName + \"_\" + $timestamp + \"_\" + $i + $extension\n      $targetPath = CreateDevice $smoBackupRestore $directory $deviceName\n      $targetPaths.Add($targetPath)\n    }\n  }\n  return $targetPaths\n}\n\nfunction BackupDatabase {\n  param (\n    [Microsoft.SqlServer.Management.Smo.Server]$server,\n    [string]$dbName,\n    [string]$BackupDirectory,\n    [int]$devices,\n    [int]$compressionOption,\n    [boolean]$incremental,\n    [boolean]$copyonly,\n    [string]$timestamp,\n    [string]$timestampFormat,\n    [boolean]$RetentionPolicyEnabled,\n    [int]$RetentionPolicyCount\n  )\n\n  $smoBackup = New-Object Microsoft.SqlServer.Management.Smo.Backup\n  $targetPaths = CreateDevices $smoBackup $devices $BackupDirectory $dbName $incremental $timestamp\n\n  Write-Host \"Attempting to backup database $server.Name.$dbName to:\"\n  $targetPaths | ForEach-Object { Write-Host $_ }\n  Write-Host \"\"\n\n  if ($incremental -eq $true) {\n    $smoBackup.Action = \"Log\"\n    $smoBackup.BackupSetDescription = \"Log backup of \" + $dbName\n    $smoBackup.LogTruncation = \"Truncate\"\n  }\n  else {\n    $smoBackup.Action = \"Database\"\n    $smoBackup.BackupSetDescription = \"Full Backup of \" + $dbName\n  }\n\n  $smoBackup.BackupSetName = $dbName + \" Backup\"\n  $smoBackup.MediaDescription = \"Disk\"\n  $smoBackup.CompressionOption = $compressionOption\n  $smoBackup.CopyOnly = $copyonly\n  $smoBackup.Initialize = $true\n  $smoBackup.Database = $dbName\n\n  try {\n    AddPercentHandler $smoBackup \"backed up\"\n    $smoBackup.SqlBackup($server)\n    Write-Host \"Backup completed successfully.\"\n\n    if ($RetentionPolicyEnabled -eq $true) {\n      ApplyRetentionPolicy $BackupDirectory $dbName $RetentionPolicyCount $Incremental $Devices $timestampFormat\n    }\n  }\n  catch {\n    Write-Error \"An error occurred backing up the database!`r`n$($_.Exception.ToString())\"\n  }\n}\n\nfunction ApplyRetentionPolicy {\n  param (\n    [string]$BackupDirectory,\n    [string]$dbName,\n    [int]$RetentionPolicyCount,\n    [boolean]$Incremental,\n    [int]$Devices,\n    [string]$timestampFormat\n  )\n\n  if ($RetentionPolicyCount -le 0) {\n    Write-Host \"RetentionPolicyCount must be greater than 0. Exiting.\"\n    return\n  }\n\n  $extension = if ($Incremental) { '.trn' } else { '.bak' }\n  $dateRegex = $timestampFormat -replace \"yyyy\", \"\\d{4}\" -replace \"MM\", \"\\d{2}\" -replace \"dd\", \"\\d{2}\" -replace \"HH\", \"\\d{2}\" -replace \"mm\", \"\\d{2}\" -replace \"ss\", \"\\d{2}\"\n  $devicePattern = \"(_\\d+)?\"\n  $regexPattern = \"^${dbName}_${dateRegex}${devicePattern}${extension}$\"\n\n  Write-Host \"Applying retention policy, retaining last $RetentionPolicyCount backups for DB $dbName with pattern: $regexPattern\"\n\n  # Fetch all backup files matching the extension and database name\n  $allBackups = Get-ChildItem $BackupDirectory -Filter \"*$extension\" | Where-Object { $_.Name -match $regexPattern }\n\n  # Group backups by their base name (excluding device number)\n  $groupedBackups = $allBackups | Group-Object { $_.Name -replace \"(_\\d+)?${extension}$\" }\n\n  # Sort groups by the latest creation time of their files in descending order\n  $sortedGroups = $groupedBackups | Sort-Object { ($_.Group | Measure-Object -Property CreationTime -Maximum).Maximum } -Descending\n\n  # Select groups to retain based on RetentionPolicyCount\n  $groupsToRetain = $sortedGroups | Select-Object -First $RetentionPolicyCount\n\n  # Flatten the list of files to retain\n  $filesToRetain = $groupsToRetain | ForEach-Object { $_.Group } | Select-Object -ExpandProperty FullName\n\n  # Determine files to remove by excluding the retained files from all backups\n  $filesToRemove = $allBackups.FullName | Where-Object { $filesToRetain -notcontains $_ }\n\n  foreach ($file in $filesToRemove) {\n    Remove-Item $file\n    Write-Host \"Removed old backup file: $file\"\n  }\n\n  Write-Host \"Retention policy applied successfully. Retained the most recent $RetentionPolicyCount backup sets.\"\n}\n\n\nfunction Invoke-SqlBackupProcess {\n  param (\n    [hashtable]$OctopusParameters\n  )\n\n  # Extracting parameters from the hashtable\n  $ServerName = $OctopusParameters['Server']\n  $DatabaseName = $OctopusParameters['Database']\n  $BackupDirectory = $OctopusParameters['BackupDirectory']\n  $CompressionOption = [int]$OctopusParameters['Compression']\n  $Devices = [int]$OctopusParameters['Devices']\n  $Stamp = $OctopusParameters['Stamp']\n  $UseSqlServerTimeStamp = $OctopusParameters['UseSqlServerTimeStamp']\n  $SqlLogin = $OctopusParameters['SqlLogin']\n  $SqlPassword = $OctopusParameters['SqlPassword']\n  $ConnectionTimeout = $OctopusParameters['ConnectionTimeout']\n  $Incremental = [boolean]::Parse($OctopusParameters['Incremental'])\n  $CopyOnly = [boolean]::Parse($OctopusParameters['CopyOnly'])\n  $RetentionPolicyEnabled = [boolean]::Parse($OctopusParameters['RetentionPolicyEnabled'])\n  $RetentionPolicyCount = [int]$OctopusParameters['RetentionPolicyCount']\n\n  [System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SMO\") | Out-Null\n  [System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SmoExtended\") | Out-Null\n  [System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.ConnectionInfo\") | Out-Null\n  [System.Reflection.Assembly]::LoadWithPartialName(\"Microsoft.SqlServer.SmoEnum\") | Out-Null\n\n  $server = New-Object Microsoft.SqlServer.Management.Smo.Server $ServerName\n\n  ConnectToDatabase $server $SqlLogin $SqlPassword $ConnectionTimeout\n\n  $database = $server.Databases | Where-Object { $_.Name -eq $DatabaseName }\n  $timestampFormat = \"yyyy-MM-dd-HHmmss\"\n  if ($UseSqlServerTimeStamp -eq $true) {\n    $timestampFormat = \"yyyyMMdd_HHmmss\"\n  }\n  $timestamp = if (-not [string]::IsNullOrEmpty($Stamp)) { $Stamp } else { Get-Date -format $timestampFormat }\n\n  if ($null -eq $database) {\n    Write-Error \"Database $DatabaseName does not exist on $ServerName\"\n  }\n\n  if ($Incremental -eq $true) {\n    if ($database.RecoveryModel -eq 3) {\n      write-error \"$DatabaseName has Recovery Model set to Simple. Log backup cannot be run.\"\n    }\n\n    if ($database.LastBackupDate -eq \"1/1/0001 12:00 AM\") {\n      write-error \"$DatabaseName has no Full backups. Log backup cannot be run.\"\n    }\n  }\n\n  if ($RetentionPolicyEnabled -eq $true -and $RetentionPolicyCount -gt 0) {\n    if (-not [int]::TryParse($RetentionPolicyCount, [ref]$null) -or $RetentionPolicyCount -le 0) {\n      Write-Error \"RetentionPolicyCount must be an integer greater than zero.\"\n    }\n  }\n\n  BackupDatabase $server $DatabaseName $BackupDirectory $Devices $CompressionOption $Incremental $CopyOnly $timestamp $timestampFormat $RetentionPolicyEnabled $RetentionPolicyCount\n}\n\nif (Test-Path -Path \"Variable:OctopusParameters\") {\n  Invoke-SqlBackupProcess -OctopusParameters $OctopusParameters\n}\n",
     "Octopus.Action.Script.Syntax": "PowerShell"
   },
   "SensitiveProperties": {},
@@ -120,13 +120,30 @@
       "DisplaySettings": {
         "Octopus.ControlType": "Checkbox"
       }
+    },
+    {
+      "Name": "RetentionPolicyEnabled",
+      "Label": "Retention Policy Enabled",
+      "HelpText": "Specify if a limit should be imposed on retaining older backups",
+      "DefaultValue": "false",
+      "DisplaySettings": {
+        "Octopus.ControlType": "Checkbox"
+      }
+    },
+    {
+      "Name": "RetentionPolicyCount",
+      "Label": "Retention Policy Count",
+      "HelpText": "Specify how many old copies of the DB should be retained (only if Retention Policy Enabled is true)",
+      "DisplaySettings": {
+        "Octopus.ControlType": "SingleLineText"
+      }
     }
   ],
-  "LastModifiedOn": "2023-08-16T16:50:00.000+00:00",
-  "LastModifiedBy": "ebalders",
+  "LastModifiedOn": "2024-03-19T20:30:00.0000000-07:00",
+  "LastModifiedBy": "bcullman",
   "$Meta": {
-    "ExportedAt": "2015-08-21T06:05:08.080+00:00",
-    "OctopusVersion": "2.6.0.778",
+    "ExportedAt": "2024-03-19T20:30:00.0000000-07:00",
+    "OctopusVersion": "2022.3.10640",
     "Type": "ActionTemplate"
   },
   "Category": "sql"

step-templates/tests/Invoke-PesterTests.ps1

@@ -4,11 +4,12 @@ Set-StrictMode -Version "Latest";
 $thisScript = $MyInvocation.MyCommand.Path;
 $thisFolder = [System.IO.Path]::GetDirectoryName($thisScript);
 $rootFolder = [System.IO.Path]::GetDirectoryName($thisFolder);
-$testFolder = [System.IO.Path]::Combine($thisFolder, "scripts");
 
 $testableScripts = @(
-    "windows-scheduled-task-create.ScriptBody.ps1"
+    "windows-scheduled-task-create.ScriptBody.ps1",
+    "sql-backup-database.ScriptBody.ps1"
 );
+
 foreach( $script in $testableScripts )
 {
     $filename = [System.IO.Path]::Combine($rootFolder, $script);
@@ -19,11 +20,14 @@ foreach( $script in $testableScripts )
     . $filename;
 }
 
-$packagesFolder = $thisFolder;
-$packagesFolder = [System.IO.Path]::GetDirectoryName($packagesFolder);
-$packagesFolder = [System.IO.Path]::GetDirectoryName($packagesFolder);
-$packagesFolder = [System.IO.Path]::Combine($packagesFolder, "packages");
-
-Import-Module -Name ([System.IO.Path]::Combine($packagesFolder, "Pester.3.4.3\tools\Pester"));
+try {
+  $packagesFolder = $thisFolder;
+  $packagesFolder = [System.IO.Path]::GetDirectoryName($packagesFolder);
+  $packagesFolder = [System.IO.Path]::GetDirectoryName($packagesFolder);
+  $packagesFolder = [System.IO.Path]::Combine($packagesFolder, "packages");
+  Import-Module -Name ([System.IO.Path]::Combine($packagesFolder, "Pester.3.4.3\tools\Pester")) -ErrorAction Stop
+} catch {
+  Import-Module Pester
+}
 
-Invoke-Pester;
+Invoke-Pester;