3.x SqlWatchImporter tweaks (#199)

* more explicit class naming

* fixes #198

* By default, NULL check values should be raising warnings, not errors. This behavior can be configured by a user.

* sql instance removal improvement

* improvements to SqlWatchImporter and corresponding tables. introduction of date_updated to speed up data import

* ssis validation

* PBI maintenance update

.gitignore

@@ -54,3 +54,4 @@ SQLWATCHSSIS/SQLWATCHSSIS.dtproj.user
 # format (NuGet v4 and .NET Core).
 project.lock.json
 project.assets.json
+/SqlWatch.Monitor/Project.SqlWatch.IntegrationServices/obj/Development

SqlWatch.Monitor/Project.SqlWatch.Database/SQLWATCH.refactorlog

@@ -2436,4 +2436,25 @@
     <Property Name="ParentElementType" Value="SqlSchema" />
     <Property Name="NewName" Value="[sqlwatch_meta_system_configuration_scd]" />
   </Operation>
+  <Operation Name="Rename Refactor" Key="a624394f-9701-43f2-97d2-7d50676bb8aa" ChangeDateTime="08/20/2020 12:04:53">
+    <Property Name="ElementName" Value="[dbo].[sqlwatch_meta_wait_stats].[date_last_seen]" />
+    <Property Name="ElementType" Value="SqlSimpleColumn" />
+    <Property Name="ParentElementName" Value="[dbo].[sqlwatch_meta_wait_stats]" />
+    <Property Name="ParentElementType" Value="SqlTable" />
+    <Property Name="NewName" Value="[date_updated]" />
+  </Operation>
+  <Operation Name="Rename Refactor" Key="f5e711b5-fc56-495b-aea6-633b225305fd" ChangeDateTime="08/20/2020 12:08:02">
+    <Property Name="ElementName" Value="[dbo].[sqlwatch_meta_performance_counter].[date_last_seen]" />
+    <Property Name="ElementType" Value="SqlSimpleColumn" />
+    <Property Name="ParentElementName" Value="[dbo].[sqlwatch_meta_performance_counter]" />
+    <Property Name="ParentElementType" Value="SqlTable" />
+    <Property Name="NewName" Value="[date_updated]" />
+  </Operation>
+  <Operation Name="Rename Refactor" Key="3906bfc2-7e89-472b-8ba6-5d8bc27dba32" ChangeDateTime="08/20/2020 12:10:15">
+    <Property Name="ElementName" Value="[dbo].[sqlwatch_meta_memory_clerk].[date_last_seen]" />
+    <Property Name="ElementType" Value="SqlSimpleColumn" />
+    <Property Name="ParentElementName" Value="[dbo].[sqlwatch_meta_memory_clerk]" />
+    <Property Name="ParentElementType" Value="SqlTable" />
+    <Property Name="NewName" Value="[date_updated]" />
+  </Operation>
 </Operations>

SqlWatch.Monitor/Project.SqlWatch.Database/Scripts/Post-Deployment/Reference-Data/Script.PostDeployment-LoadConfig-Global.sql

@@ -46,7 +46,7 @@ values
 
 	/*	when check returns null we can log error and fail the job, 
 		or log warning and not fail the job.	*/
-	(8	,'Error on NULL check value'						,1), 
+	(8	,'Error on NULL check value'						,0), 
 
 	/*	when SQLWATCH xes are disabled we will try get data from system_health session if enabled.
 		Quite often however system_health session is quite large and it can take couple of minutes ot parse xml

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_memory_clerk.sql

@@ -1,15 +1,20 @@
 CREATE PROCEDURE [dbo].[usp_sqlwatch_internal_add_memory_clerk]
 as
 
-insert into [dbo].[sqlwatch_meta_memory_clerk] ([sql_instance], [clerk_name])
-select [sql_instance]=@@SERVERNAME, [type]
-from (
-	select distinct [type] 
+;merge [dbo].[sqlwatch_meta_memory_clerk] as target
+using (
+	select distinct 
+		[clerk_name] = [type] 
 	from sys.dm_os_memory_clerks s
 	union all
-	select 'OTHER'
-) s
-left join [dbo].[sqlwatch_meta_memory_clerk] d
-	on d.[clerk_name] = s.[type] collate database_default
-	and d.sql_instance = @@SERVERNAME
-where d.[clerk_name] is null
+	select [clerk_name] = 'OTHER'
+	) as source
+on target.[clerk_name] = source.[clerk_name] collate database_default
+and target.[sql_instance] = @@SERVERNAME
+
+--when matched then 
+--	update set date_last_seen = getutcdate()
+
+when not matched then
+	insert ([sql_instance], [clerk_name])
+	values (@@SERVERNAME, source.[clerk_name]);

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_performance_counter.sql

@@ -1,22 +1,29 @@
 CREATE PROCEDURE [dbo].[usp_sqlwatch_internal_add_performance_counter]
 as
 
-  insert into [dbo].[sqlwatch_meta_performance_counter] ([sql_instance],[object_name],[counter_name],[cntr_type])
-  select distinct @@SERVERNAME, rtrim(pc.[object_name]), rtrim(pc.[counter_name]), pc.[cntr_type]
-  from sys.dm_os_performance_counters pc
-  left join [dbo].[sqlwatch_meta_performance_counter] mc
-	on rtrim(pc.[object_name]) = mc.[object_name] collate database_default
-	and rtrim(pc.[counter_name]) = mc.[counter_name] collate database_default
-	and mc.[sql_instance] = @@SERVERNAME
-  where mc.counter_name is null
 
-  insert into [dbo].[sqlwatch_meta_performance_counter] ([sql_instance],[object_name],[counter_name],[cntr_type])
-  select pc.[sql_instance],pc.[object_name],pc.[counter_name],pc.[cntr_type]
-  from (
-	select [sql_instance] = @@SERVERNAME, [object_name] = 'win32_perfformatteddata_perfos_processor', [counter_name] = 'Processor Time %', [cntr_type] = 65792
-	) pc
-  left join [dbo].[sqlwatch_meta_performance_counter] mc
-	on rtrim(pc.[object_name]) = mc.[object_name] collate database_default
-	and rtrim(pc.[counter_name]) = mc.[counter_name] collate database_default
-	and mc.[sql_instance] = @@SERVERNAME
-  where mc.counter_name is null
+;merge [dbo].[sqlwatch_meta_performance_counter] as target
+using (
+		select distinct 
+			[sql_instance] = @@SERVERNAME
+			, [object_name] = rtrim(pc.[object_name])
+			, [counter_name] = rtrim(pc.[counter_name])
+			, [cntr_type] = pc.[cntr_type]
+		from sys.dm_os_performance_counters pc
+		union all
+		select 
+			[sql_instance] = @@SERVERNAME
+			, [object_name] = 'win32_perfformatteddata_perfos_processor'
+			, [counter_name] = 'Processor Time %'
+			, [cntr_type] = 65792
+		) as source
+	on target.sql_instance = source.sql_instance collate database_default
+	and target.object_name = source.object_name collate database_default
+	and target.counter_name = source.counter_name collate database_default
+
+--when matched then
+--	update set date_last_seen = getutcdate()
+
+when not matched then
+	insert ([sql_instance],[object_name],[counter_name],[cntr_type])
+	values (source.[sql_instance],source.[object_name],source.[counter_name],source.[cntr_type]);

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_system_configuration.sql

@@ -14,18 +14,17 @@ using (
 on target.configuration_id = source.configuration_id
 and target.[sql_instance] = source.[sql_instance]
 
-when matched then 
+when matched 
+	and (
+			target.[value] <> source.[value] 
+		or	target.[value_in_use] <> source.[value_in_use]
+		)
+	then 
 	update set [value] = source.[value],
-		[value_in_use] = source.[value_in_use],
-		[date_updated] = case when 		
-									target.[value] <> source.[value]
-								or	target.[value_in_use] <> source.[value_in_use]
-								then GETUTCDATE() else [date_updated] end,
-		[date_last_seen] = GETUTCDATE(),
-		[is_record_deleted] = 0
+		[value_in_use] = source.[value_in_use]
 
 when not matched by target then
-	insert ([sql_instance], [configuration_id], [name], [description], [value], [value_in_use], [date_created], [date_last_seen])
-	values (source.[sql_instance], source.[configuration_id], source.[name], source.[description], source.[value], source.[value_in_use], GETUTCDATE(), GETUTCDATE());
+	insert ([sql_instance], [configuration_id], [name], [description], [value], [value_in_use], [date_created])
+	values (source.[sql_instance], source.[configuration_id], source.[name], source.[description], source.[value], source.[value_in_use], GETUTCDATE());
 
 

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_add_wait_type.sql

@@ -1,11 +1,15 @@
 CREATE PROCEDURE [dbo].[usp_sqlwatch_internal_add_wait_type]
 AS
 
+
+;merge [dbo].[sqlwatch_meta_wait_stats] as target
+using sys.dm_os_wait_stats as source
+	on target.[wait_type] = source.[wait_type] collate database_default
+	and target.[sql_instance] = @@SERVERNAME
 
-insert into [dbo].[sqlwatch_meta_wait_stats] ([sql_instance], [wait_type])
-select distinct @@SERVERNAME, dm.[wait_type]
-from sys.dm_os_wait_stats dm
-left join [dbo].[sqlwatch_meta_wait_stats] ws
-	on ws.[sql_instance] = @@SERVERNAME
-	and ws.[wait_type] = dm.[wait_type] collate database_default
-where ws.[wait_type] is null
+--when matched then 
+--	update set [date_last_seen] = getutcdate()
+
+when not matched then 
+	insert ([sql_instance], [wait_type])
+	values (@@SERVERNAME, source.[wait_type]);

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_internal_create_agent_job.sql

@@ -68,7 +68,16 @@ else
 	begin
 		Print ''Job ''''' + job_name + ''''' not created because it already exists.''
 	end;
-	' + case when /* job has not run yet */ h.run_status is null and /* only if its the first deployment */ v.deployment_count = 0 and job_enabled = 1 then 'exec [dbo].[usp_sqlwatch_internal_run_job] @job_name = ''' + job_name + '''' else '' end + '
+	' + case when /* #198 we will not be executing jobs as part of the deployment anymore. 
+					This was only done so the check job does not return NULL error when perf counters 
+					tables are empty after the deployment. However, often they fail due to a number of
+					reasons on the client server - agent disabled, or broker running behind, accounts not up to date
+					this should not stop the deployment. */ 
+					1=2 
+					and /* job has not run yet */ h.run_status is null 
+					and /* only if its the first deployment */ v.deployment_count = 0 
+					and job_enabled = 1 
+					then 'exec [dbo].[usp_sqlwatch_internal_run_job] @job_name = ''' + job_name + '''' else '' end + '
 	'
 	from ##sqlwatch_jobs
 	outer apply (

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Procedures/usp_sqlwatch_repository_populate_tables_to_import.sql

@@ -86,8 +86,15 @@ where name like 'sqlwatch_meta%' or	name like 'sqlwatch_logger%'
 	from cte_base_tables d
 	group by d.schema_Name, d.name
 )
-insert into dbo.sqlwatch_stage_repository_tables_to_import
-select *
+insert into dbo.sqlwatch_stage_repository_tables_to_import(
+	[table_name],[dependency_level],[has_last_seen],[has_last_updated],
+	[has_identity],[primary_key],[joins],[updatecolumns],[allcolumns] 
+	)
+
+select d.[table_name],d.[dependency_level],
+	c.[has_last_seen],
+	[has_last_updated],
+	[has_identity],[primary_key],[joins],[updatecolumns],[allcolumns] 
 from cte_dependency d
 
 --check if the table contains date_last_seen column
@@ -97,6 +104,13 @@ outer apply (
 	where TABLE_SCHEMA + '.' + TABLE_NAME = d.table_name
 ) c
 
+-- check if the table has date_updated column
+outer apply (
+	select has_last_updated = max(case when COLUMN_NAME = 'date_updated' then 1 else 0 end)
+	from INFORMATION_SCHEMA.COLUMNS
+	where TABLE_SCHEMA + '.' + TABLE_NAME = d.table_name
+) u
+
 -- build concatenated string of primary keys
 outer apply (
 select primary_key = isnull(stuff ((
@@ -179,6 +193,7 @@ select allcolumns = isnull(stuff ((
 		and COLUMN_NAME not in (
 				select name 
 				from sys.computed_columns
+				where object_id = OBJECT_ID(d.TABLE_NAME)
 		)
 		order by ORDINAL_POSITION
 		for xml path('')),1,1,''),'')

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_config_sql_instance.sql

@@ -58,19 +58,34 @@ go
 
 create trigger dbo.trg_sqlwatch_config_sql_instance_remove_meta
 	on [dbo].[sqlwatch_config_sql_instance]
-	for delete
+	instead of delete
 	as
 	begin
+		set nocount on
+
 		declare @deleted_instance table (
 			sql_instance varchar(32)
 			)
 
 		insert into @deleted_instance
 		select sql_instance from deleted
 
-		set nocount on
+		-- first, disable the remote collection:
+		update c
+			set repo_collector_is_active = 0
+		from [dbo].[sqlwatch_config_sql_instance] c
+		inner join @deleted_instance d
+			on c.sql_instance = d.sql_instance
+
+		-- delete each server, leverage cascade deletes:
 		delete from dbo.sqlwatch_meta_server
 		where servername in (
 			select sql_instance from @deleted_instance
 			)
+
+		-- finally, delete from the config table:
+		delete from [dbo].[sqlwatch_config_sql_instance]
+		where sql_instance in (
+			select sql_instance from @deleted_instance
+			)
 	end

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_logger_system_configuration.sql

@@ -7,7 +7,7 @@
 	snapshot_time datetime2(0),
 	snapshot_type_id tinyint default(26),
 	constraint pk_sqlwatch_logger_system_configuration primary key clustered (
-		sqlwatch_configuration_id, snapshot_time, snapshot_type_id
+		sql_instance, sqlwatch_configuration_id, snapshot_time, snapshot_type_id
 		),
 	constraint fk_sqlwatch_logger_system_configuration_keyword foreign key (sql_instance, sqlwatch_configuration_id) 
 		references dbo.sqlwatch_meta_system_configuration (sql_instance, sqlwatch_configuration_id)on delete cascade,

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_check.sql

@@ -15,6 +15,7 @@
 	[last_check_value] real null,
 	[last_check_status] varchar(50) null,
 	[last_status_change_date] datetime null,
+	[date_updated] datetime not null constraint df_sqlwatch_meta_check_updated default (getutcdate()),
 
 	/*	primary key */
 	constraint pk_sqlwatch_meta_alert primary key clustered ([sql_instance], [check_id]),
@@ -34,6 +35,27 @@
 )
 go
 
+create nonclustered index idx_sqlwatch_meta_check_1 on [dbo].[sqlwatch_meta_check] ([date_updated])
+go
+
+create trigger trg_sqlwatch_meta_last_updated
+	on [dbo].[sqlwatch_meta_check]
+	for insert,update
+	as
+	begin
+		set nocount on;
+		set xact_abort on;
+
+		update t
+			set date_updated = getutcdate()
+		from [dbo].[sqlwatch_meta_check] t
+		inner join inserted i
+			on i.[sql_instance] = t.[sql_instance]
+			and i.[check_id] = t.[check_id]
+			and i.sql_instance = @@SERVERNAME
+	end
+go
+
 create trigger trg_sqlwatch_meta_check_delete
 	on [dbo].[sqlwatch_meta_check]
 	for delete

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_errorlog_attribute.sql

@@ -4,9 +4,32 @@
 	attribute_id smallint identity(1,1),
 	attribute_name varchar(255), 
 	attribute_value varchar(255),
+	[date_updated] datetime not null constraint df_sqlwatch_meta_errorlog_attribute_updated default (getutcdate()),
 	constraint pk_sqlwatch_meta_errorlog_attributes primary key clustered (
 		sql_instance, attribute_id
 		),
 	constraint fk_sqlwatch_meta_errorlog_attributes_server foreign key (sql_instance)
 		references dbo.sqlwatch_meta_server (servername) on delete cascade
 )
+go
+
+create nonclustered index idx_sqlwatch_meta_errorlog_attribute_1 on [dbo].[sqlwatch_meta_errorlog_attribute] ([date_updated])
+go
+
+create trigger trg_sqlwatch_meta_errorlog_attribute_last_updated
+	on [dbo].[sqlwatch_meta_errorlog_attribute]
+	for insert,update
+	as
+	begin
+		set nocount on;
+		set xact_abort on;
+
+		update t
+			set date_updated = getutcdate()
+		from [dbo].[sqlwatch_meta_errorlog_attribute] t
+		inner join inserted i
+			on i.sql_instance = t.sql_instance
+			and i.attribute_id = t.attribute_id
+			and i.sql_instance = @@SERVERNAME
+	end
+go

SqlWatch.Monitor/Project.SqlWatch.Database/dbo/Tables/sqlwatch_meta_errorlog_keyword.sql

@@ -4,6 +4,7 @@
 	keyword_id smallint identity(1,1) not null,
 	log_type_id int not null,
 	keyword nvarchar(255),
+	[date_updated] datetime not null constraint df_sqlwatch_meta_errorlog_keyword_updated default (getutcdate()),
 	constraint pk_sqlwatch_meta_errorlog_keyword primary key clustered (
 		sql_instance, keyword_id, log_type_id
 		),
@@ -13,4 +14,27 @@
 go
 
 create nonclustered index idx_sqlwatch_meta_errorlog_keyword_1 on [dbo].[sqlwatch_meta_errorlog_keyword] (keyword)
-	include (sql_instance, keyword_id, log_type_id)
+	include (sql_instance, keyword_id, log_type_id)
+go
+
+create nonclustered index idx_sqlwatch_meta_errorlog_keyword_2 on [dbo].[sqlwatch_meta_errorlog_keyword] ([date_updated])
+go
+
+create trigger trg_sqlwatch_meta_errorlog_keyword_last_updated
+	on [dbo].[sqlwatch_meta_errorlog_keyword]
+	for insert,update
+	as
+	begin
+		set nocount on;
+		set xact_abort on;
+
+		update t
+			set date_updated = getutcdate()
+		from [dbo].[sqlwatch_meta_errorlog_keyword] t
+		inner join inserted i
+			on i.sql_instance = t.sql_instance
+			and i.keyword_id = t.keyword_id
+			and i.log_type_id = t.log_type_id
+			and i.sql_instance = @@SERVERNAME
+	end
+go