3.x.1 (#201)

* change failed job severity from 16 to 10 during deployment as this not critical and shuold not stop deployment. We're running the CONFIG job as part of the deployment so the collection can start as soon as the deployment completes

* changed the index collector to run every 24 hours at 0015 instead of every 6 hours. On instances with large number of indexes this job can have a significant impact.

* step order

* removed legacy fields that are no longer used

* change outer applies to joins for performance improvements and because we're changing how these views are consumed

* account for view changes

* Fixed #191

* Updated version

* Docs Reference

* New tests

SqlWatch.Documentation/README.MD

@@ -0,0 +1 @@
+SQLWATCH Documentation is available at https://docs.sqlwatch.io

SqlWatch.Monitor/Project.SqlWatch.Database/SQLWATCH.refactorlog

@@ -2457,4 +2457,11 @@
     <Property Name="ParentElementType" Value="SqlTable" />
     <Property Name="NewName" Value="[date_updated]" />
   </Operation>
+  <Operation Name="Rename Refactor" Key="e4ad08c0-9530-47ed-a718-70b266df0d1b" ChangeDateTime="08/20/2020 16:28:08">
+    <Property Name="ElementName" Value="[dbo].[sqlwatch_meta_table].[date_created]" />
+    <Property Name="ElementType" Value="SqlSimpleColumn" />
+    <Property Name="ParentElementName" Value="[dbo].[sqlwatch_meta_table]" />
+    <Property Name="ParentElementType" Value="SqlTable" />
+    <Property Name="NewName" Value="[date_first_seen]" />
+  </Operation>
 </Operations>

SqlWatch.Monitor/Project.SqlWatch.Database/Scripts/Post-Deployment/Reference-Data/Script.PostDeployment-CreateDefaultChecks.sql

@@ -412,20 +412,34 @@ and mtb.sql_instance = @@SERVERNAME'
 --------------------------------------------------------------------------------------
 exec [dbo].[usp_sqlwatch_config_add_check]
 	 @check_id = -17
-	,@check_name = 'Oldest LOG backup (minutes)'
+	,@check_name = 'Latest LOG backup age (minutes)'
 	,@check_description = 'There is one or more databases that has no recent log backup. Databases that are in either FULL or BULK_LOGGED recovery must have frequent Transaction Log backups. The recovery point will be to the last Transaction Log backup and therefore these must happen often to minimise data loss.
 https://docs.microsoft.com/en-us/sql/relational-databases/backup-restore/recovery-models-sql-server'
-	,@check_query = 'select @output=isnull(max(datediff(minute,last_backup_finish_date,getdate())),999)
-from (	
-	select database_name = d.name,
-	last_backup_finish_date = max(bs.backup_finish_date)
-	from sys.sysdatabases d
-	left join msdb.dbo.backupset bs ON bs.database_name = d.name
-		and bs.type = ''L''
-	where d.name not in (''tempdb'')
-	and d.recovery_model_desc <> ''SIMPLE''
-	group by d.name
-	) t'
+	,@check_query = 'select @output=case when (
+	--if we have no databases in FULL recovery, 
+	--we have to return a success result
+	select count(*) 
+	from sys.databases (nolock)
+	where recovery_model_desc <> ''SIMPLE''
+	) = 0 
+	
+	then 0 
+	
+	else (
+	select min(isnull(last_backup_age_minutes,999))
+	from (	
+		select database_name = d.name,
+		last_backup_age_minutes = datediff(minute,max(bs.backup_finish_date),getdate())
+		from sys.databases d  (nolock)
+		left join msdb.dbo.backupset bs  (nolock) 
+			on bs.database_name = d.name
+			and bs.type = ''L''
+		where d.name not in (''tempdb'')
+		and d.recovery_model_desc <> ''SIMPLE''
+		group by d.name
+		) t
+		)
+	end'
 	,@check_frequency_minutes = 5
 	,@check_threshold_warning = '>10' --warn if log backup over 10 minutes old
 	,@check_threshold_critical = '>60' --critical if log backup over 1 hour old
@@ -441,12 +455,12 @@ from (
 --------------------------------------------------------------------------------------
 exec [dbo].[usp_sqlwatch_config_add_check]
 	 @check_id = -18
-	,@check_name = 'Oldest DATA backup (days)'
+	,@check_name = 'Latest DATA backup age (days)'
 	,@check_description = 'There is one or more databases that has no recent data backup.'
-	,@check_query = 'select @output=isnull(max(datediff(day,last_backup_finish_date,getdate())),999)
+	,@check_query = 'select @output=isnull(max(last_backup_age),999)
 from (	
 	select database_name = d.name,
-	last_backup_finish_date = max(bs.backup_finish_date)
+	last_backup_age = datediff(day,max(bs.backup_finish_date),getdate())
 	from sys.sysdatabases d
 	left join msdb.dbo.backupset bs ON bs.database_name = d.name
 		and bs.type <> ''L''

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_config_create_default_agent_jobs.sql

@@ -86,7 +86,7 @@ insert into ##sqlwatch_jobs
 
 			('SQLWATCH-LOGGER-PERFORMANCE',		4,			1,				4,					1,						0,						1,							20180101,			99991231,			12,					235959,				1),
 			('SQLWATCH-LOGGER-DISK-UTILISATION',4,			1,				8,					1,						0,						1,							20180101,			99991231,			437,				235959,				1),
-			('SQLWATCH-LOGGER-INDEXES',			4,			1,				8,					6,						0,						1,							20180101,			99991231,			420,				235959,				1),
+			('SQLWATCH-LOGGER-INDEXES',			4,			1,				1,					24,						0,						1,							20180101,			99991231,			1500,				235959,				1),
 			('SQLWATCH-LOGGER-AGENT-HISTORY',	4,			1,				4,					10,						0,						1,							20180101,			99991231,			0,					235959,				1),
 
 			('SQLWATCH-INTERNAL-RETENTION',		4,			1,				8,					1,						0,						1,							20180101,			99991231,			20,					235959,				1),
@@ -205,13 +205,13 @@ Get-WMIObject Win32_Volume | ?{$_.DriveType -eq 3 -And $_.Name -notlike "\\?\Vol
 			('dbo.usp_sqlwatch_logger_index_usage_stats',	4,		'SQLWATCH-LOGGER-INDEXES',		'TSQL', 'exec dbo.usp_sqlwatch_logger_index_usage_stats'),
 			('dbo.usp_sqlwatch_logger_index_histogram',		5,		'SQLWATCH-LOGGER-INDEXES',		'TSQL', 'exec dbo.usp_sqlwatch_logger_index_histogram'),
 
-			('dbo.usp_sqlwatch_internal_add_database',		1,			'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_database'),
-			('dbo.usp_sqlwatch_internal_add_job',			2,			'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_job'),
-			('dbo.usp_sqlwatch_internal_add_performance_counter',	3,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_performance_counter'),
-			('dbo.usp_sqlwatch_internal_add_master_file',			4,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_master_file'),
-			('dbo.usp_sqlwatch_internal_add_wait_type',				5,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_wait_type'),
+			('dbo.usp_sqlwatch_internal_add_database',				1,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_database'),
+			('dbo.usp_sqlwatch_internal_add_master_file',			2,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_master_file'),
+			('dbo.usp_sqlwatch_internal_add_table',					3,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_table'),
+			('dbo.usp_sqlwatch_internal_add_job',					4,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_job'),
+			('dbo.usp_sqlwatch_internal_add_performance_counter',	5,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_performance_counter'),
 			('dbo.usp_sqlwatch_internal_add_memory_clerk',			6,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_memory_clerk'),
-			('dbo.usp_sqlwatch_internal_add_table',					7,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_table'),
+			('dbo.usp_sqlwatch_internal_add_wait_type',				7,	'SQLWATCH-INTERNAL-CONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_wait_type'),
 
 			('dbo.usp_sqlwatch_internal_add_system_configuration',	1,	'SQLWATCH-LOGGER-SYSCONFIG','TSQL', 'exec dbo.usp_sqlwatch_internal_add_system_configuration'),
 			('dbo.usp_sqlwatch_logger_system_configuration',	    2,	'SQLWATCH-LOGGER-SYSCONFIG','TSQL', 'exec dbo.usp_sqlwatch_logger_system_configuration')

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_database.sql

@@ -4,16 +4,18 @@ as
 
 	/*	using database_create_data to distinguish databases that have been dropped and re-created 
 		this is particulary useful when doing performance testing and we are re-creating test databases throughout the process and want to compare them later.
+		However, with every SQL Server restart, tempdb will be recreated and will have new date_created. On some dev and uat servers we may end up with dozen or more
+		tempdbs. To account for this, we are going to default the create_date for tempdb to '1970-01-01'
 	*/
 	;merge [dbo].[sqlwatch_meta_database] as target
 	using (
-		select [name], [create_date], [sql_instance]
+		select [name], [create_date] = case when [name] = 'tempdb' then convert(datetime,'1970-01-01') else [create_date] end, [sql_instance]
 			, [is_auto_close_on], [is_auto_shrink_on], [is_auto_update_stats_on]
 			, [user_access], [state], [snapshot_isolation_state] , [is_read_committed_snapshot_on] 
 			, [recovery_model] , [page_verify_option] 
-			, BC = binary_checksum([is_auto_close_on], [is_auto_shrink_on], [is_auto_update_stats_on]
-								 , [user_access], [state], [snapshot_isolation_state] , [is_read_committed_snapshot_on] 
-							 	 , [recovery_model] , [page_verify_option])
+			--, BC = binary_checksum([is_auto_close_on], [is_auto_shrink_on], [is_auto_update_stats_on]
+			--					 , [user_access], [state], [snapshot_isolation_state] , [is_read_committed_snapshot_on] 
+			--				 	 , [recovery_model] , [page_verify_option])
 		from dbo.vw_sqlwatch_sys_databases
 		--union all
 		/* mssqlsystemresource database appears in the performance counters
@@ -28,37 +30,29 @@ as
 		)
 	/* dropped databases are going to be updated to current = 0 */
 	when not matched by source and target.sql_instance = @@SERVERNAME then
-		update set [is_record_deleted] = 1
+		update set [is_current] = 0
 
 	when matched then
-		update set [date_last_seen] = GETUTCDATE()
-			,[is_record_deleted] = 0
-			,[is_auto_close_on] = case when target.[is_auto_close_on] is null or target.[is_auto_close_on] <> source.[is_auto_close_on] then source.[is_auto_close_on] else target.[is_auto_close_on] end
-			,[is_auto_shrink_on] = case when target.[is_auto_shrink_on] is null or target.[is_auto_shrink_on] <> source.[is_auto_shrink_on] then source.[is_auto_shrink_on] else target.[is_auto_shrink_on] end
-			,[is_auto_update_stats_on] = case when target.[is_auto_update_stats_on] is null or target.[is_auto_update_stats_on] <> source.[is_auto_update_stats_on] then source.[is_auto_update_stats_on] else target.[is_auto_update_stats_on] end
-			,[user_access] = case when target.[user_access] is null or target.[user_access] <> source.[user_access] then source.[user_access] else target.[user_access] end
-			,[state] = case when target.[state] is null or target.[state] <> source.[state] then source.[state] else target.[state] end
-			,[snapshot_isolation_state] = case when target.[snapshot_isolation_state] is null or target.[snapshot_isolation_state] <> source.[snapshot_isolation_state] then source.[snapshot_isolation_state] else target.[snapshot_isolation_state] end
-			,[is_read_committed_snapshot_on] = case when target.[is_read_committed_snapshot_on] is null or target.[is_read_committed_snapshot_on] <> source.[is_read_committed_snapshot_on] then source.[is_read_committed_snapshot_on] else target.[is_read_committed_snapshot_on] end
-			,[recovery_model] = case when target.[recovery_model] is null or target.[recovery_model] <> source.[recovery_model] then source.[recovery_model] else target.[recovery_model] end
-			,[page_verify_option] = case when target.[page_verify_option] is null or target.[page_verify_option] <> source.[page_verify_option] then source.[page_verify_option] else target.[page_verify_option] end
-
-			,[date_updated] = case when binary_checksum(target.[is_auto_close_on], target.[is_auto_shrink_on], target.[is_auto_update_stats_on]
-								 , target.[user_access], target.[state], target.[snapshot_isolation_state] , target.[is_read_committed_snapshot_on] 
-							 	 , target.[recovery_model] , target.[page_verify_option]) <> source.BC then getutcdate() else target.[date_updated] end
-	/* new databases are going to be inserted */
+		update set [is_current] = 1,
+				[date_last_seen] = case when datediff(hour,[date_last_seen],getutcdate()) >= 24 then getutcdate() else [date_last_seen] end,
+				[is_auto_close_on] = source.[is_auto_close_on],
+				[is_auto_shrink_on] = source.[is_auto_shrink_on],
+				[is_auto_update_stats_on] = source.[is_auto_update_stats_on],
+				[user_access] = source.[user_access],
+				[snapshot_isolation_state] = source.[snapshot_isolation_state],
+				[is_read_committed_snapshot_on] = source.[is_read_committed_snapshot_on],
+				[recovery_model] = source.[recovery_model],
+				[page_verify_option] = source.[page_verify_option]
+
 	when not matched by target then
 		insert ([database_name], [database_create_date], [sql_instance], [is_auto_close_on], [is_auto_shrink_on], [is_auto_update_stats_on]
 			,[user_access], [state], [snapshot_isolation_state], [is_read_committed_snapshot_on], [recovery_model], [page_verify_option]
+			,[date_last_seen], [is_current]
 		)
 		values (source.[name], source.[create_date], source.[sql_instance]
 			, source.[is_auto_close_on], source.[is_auto_shrink_on], source.[is_auto_update_stats_on]
 			, source.[user_access], source.[state], source.[snapshot_isolation_state]
 			, source.[is_read_committed_snapshot_on], source.[recovery_model]
 			, source.[page_verify_option]
-			);
-
-	/*	the above only accounts for databases that have been removed and re-added
-		if you rename database it will be treated as if it was removed and new
-		database created so you will lose history continuation. Why would you
-		rename a database anyway */
+			, getutcdate(), 1
+			);

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_master_file.sql

@@ -20,25 +20,25 @@ using (
 		on mdb.sql_instance = @@SERVERNAME
 		and mdb.database_name = convert(nvarchar(128),db.name) collate database_default
 		and mdb.database_create_date = db.create_date
-	)as source
+	) as source
  on (
 		source.file_id = target.file_id
 	and source.[file_name] = target.[file_name] collate database_default
 	and source.physical_name = target.file_physical_name collate database_default
 	and	source.sql_instance = target.sql_instance
  )
 
- when not matched by source and target.sql_instance = @@SERVERNAME then
-	update set [is_record_deleted] = 1
+--when not matched by source and target.sql_instance = @@SERVERNAME then
+--	update set [is_record_deleted] = 1
 
-when matched then
+when matched and datediff(hour,[date_last_seen],getutcdate()) >= 24 then
 	update
-		set [date_last_seen] = getutcdate(),
-			[is_record_deleted] = 0
+		set [date_last_seen] = getutcdate()
+			--[is_record_deleted] = 0
 
 when not matched by target then
-	insert ( [sqlwatch_database_id], [file_id], [file_type], [file_physical_name], [sql_instance], [file_name], [logical_disk] )
-	values ( source.[sqlwatch_database_id], source.[file_id], source.[type], source.[physical_name], source.[sql_instance], source.[file_name], source.[logical_disk] );
+	insert ( [sqlwatch_database_id], [file_id], [file_type], [file_physical_name], [sql_instance], [file_name], [logical_disk],[date_last_seen] )
+	values ( source.[sqlwatch_database_id], source.[file_id], source.[type], source.[physical_name], source.[sql_instance], source.[file_name], source.[logical_disk], getutcdate() );
 
 --when not matched by source and target.sql_instance = @@SERVERNAME then 
 --	update set deleted_when = GETUTCDATE();

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_table.sql

@@ -34,7 +34,7 @@ exec [dbo].[usp_sqlwatch_internal_foreachdb] @command = '
 USE [?]
 insert into ##98308FFC2C634BF98B347EECB98E3490 ([TABLE_CATALOG],[table_name],[TABLE_TYPE])
 SELECT [TABLE_CATALOG],[table_name] = [TABLE_SCHEMA] + ''.'' + [TABLE_NAME],[TABLE_TYPE] 
-from INFORMATION_SCHEMA.TABLES
+from INFORMATION_SCHEMA.TABLES with (nolock)
 WHERE''?'' <> ''tempdb''', @databases = @databases, @calling_proc_id = @@PROCID
 
 /* when collecting tables we only consider name as a primary key. 
@@ -62,18 +62,29 @@ using (
  and	target.[sqlwatch_database_id] = source.[sqlwatch_database_id]
 
 
-when not matched by source and target.sql_instance = @@SERVERNAME then
-	update set [is_record_deleted] = 1
+/* we dont need is record deleted field as its not always possible to tell.
+   we're using date last seen to handle this status */
+--when not matched by source and target.sql_instance = @@SERVERNAME then
+--	update set [is_record_deleted] = 1
 
- when matched and target.sql_instance = @@SERVERNAME 
-	then update set [date_last_seen] = GETUTCDATE(),
-		[is_record_deleted] = 0
+ when matched 
+	and target.sql_instance = @@SERVERNAME 
+	-- The SqlWatchImport relies on date_last_seen to speed up imports
+	-- and the field is only used for the retention purposes.
+	-- We will only update it if its passsed 24h.
+	-- New tables will be picked up immediately.
 
-								/* a new database could have been added since last db collection.
-								   in which case we have not got id yet, it will be picked up with the next cycle */
+	-- On instances with large number of databases and tables,
+	-- this procedures should only run once a day.
+	and datediff(hour,[date_last_seen],GETUTCDATE()) >= 24
+	then update set [date_last_seen] = GETUTCDATE()
+		--,[is_record_deleted] = 0
+
+/* a new database and/or table could have been added since last collection.
+	in which case we have not got id yet, it will be picked up with the next cycle */
  when not matched by target and source.[sqlwatch_database_id] is not null then
-	insert ([sql_instance],[sqlwatch_database_id],[table_name],[table_type],[date_created])
-	values (@@SERVERNAME,source.[sqlwatch_database_id],source.[table_name],source.[table_type],GETUTCDATE());
+	insert ([sql_instance],[sqlwatch_database_id],[table_name],[table_type],[date_first_seen],[date_last_seen])
+	values (@@SERVERNAME,source.[sqlwatch_database_id],source.[table_name],source.[table_type],GETUTCDATE(),GETUTCDATE());
 
  --when matched and [date_deleted] is not null and target.sql_instance = @@SERVERNAME then
 	--update set [date_deleted] = null;

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_create_agent_job.sql

@@ -72,8 +72,10 @@ else
 					This was only done so the check job does not return NULL error when perf counters 
 					tables are empty after the deployment. However, often they fail due to a number of
 					reasons on the client server - agent disabled, or broker running behind, accounts not up to date
-					this should not stop the deployment. */ 
-					1=2 
+					this should not stop the deployment. 
+					
+					The only exception is the SQLWATCH-INTERNAL-CONFIG job as if this is not run, the data collection cannot happen */ 
+					job_name = 'SQLWATCH-INTERNAL-CONFIG'
 					and /* job has not run yet */ h.run_status is null 
 					and /* only if its the first deployment */ v.deployment_count = 0 
 					and job_enabled = 1 

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_run_job.sql

@@ -31,7 +31,7 @@ exec master.dbo.xp_sqlagent_enum_jobs 1, @job_owner, @job_id
 if exists (select top 1 * FROM @xp_results where running = 1)
 	begin
 		--job is running, quit
-		raiserror('Job ''%s'' is already running.',16, 1, @job_name)
+		raiserror('Job ''%s'' is already running.',10, 1, @job_name)
         return
 	end
 
@@ -58,6 +58,6 @@ if (select top 1 run_status
 	end
 else
 	begin
-		raiserror('Job ''%s'' has not finished successfuly or the state is not known.',16, 1, @job_name)
+		raiserror('Job ''%s'' has not finished successfuly or the state is not known.',10, 1, @job_name)
 	end
 

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_database.sql

@@ -15,8 +15,9 @@
 	[is_read_committed_snapshot_on] bit,
 	[recovery_model] tinyint,
 	[page_verify_option] tinyint,
-	[date_updated] datetime null,
-	[is_record_deleted] bit
+	--[date_updated] datetime null,
+	--[is_record_deleted] bit,
+	[is_current] bit,
 
 	/*	primary key */
 	constraint PK_database primary key clustered (

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_master_file.sql

@@ -10,8 +10,6 @@
 	[sql_instance] varchar(32) not null constraint df_sqlwatch_meta_master_file_sql_instance default (@@SERVERNAME),
 	[date_last_seen] datetime null constraint df_sqlwatch_meta_master_file_last_seen default (getutcdate()),
 	[logical_disk] varchar(260),
-	[date_created] datetime not null constraint df_sqlwatch_meta_master_file_date_created default (getutcdate()),
-	[is_record_deleted] bit
 	constraint PK_sql_perf_mon_master_files primary key clustered (
 		[sql_instance], [sqlwatch_database_id], [sqlwatch_master_file_id]
 		),

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_table.sql

@@ -5,9 +5,9 @@
 	[sqlwatch_table_id] int identity(1,1) not null,
 	[table_name] nvarchar(512), --table lenght is 128 in SQL2008 and 256 in newer versions but as we also keep schema name here we need to account for that too.
 	[table_type] nvarchar(128),
-	[date_created] datetime not null constraint df_sqlwatch_meta_table_date_created default (getutcdate()),
-	[date_last_seen] datetime null constraint df_sqlwatch_meta_table_last_seen default (getutcdate()),
-	[is_record_deleted] bit
+	[date_first_seen] datetime not null constraint df_sqlwatch_meta_table_date_created default (getutcdate()),
+	[date_last_seen] datetime not null constraint df_sqlwatch_meta_table_last_seen default (getutcdate()),
+	--[is_record_deleted] bit
 	constraint pk_sqlwatch_meta_database_table primary key clustered (
 		[sql_instance], [sqlwatch_database_id], [sqlwatch_table_id]
 		),