Thursday, December 6, 2018

Fun with SQL Server Plan Cache, Trace Flag 8666, and Trace Flag 2388

Ok, last one for a while.  This time, pulling the stats relevant to a plan from the plan XML in the plan cache (thanks to trace flag 8666), then getting a little bit of stats update history via trace flag 2388.

I'm not particularly happy with performance; it took between 10 and 12 seconds to return a 12 row resultset 😥

But its good enough for the immediate troubleshooting needs.

Some sample output is below; note that the output is sorted by database, schema, table, and stats name.  But the updates of any one stat are NOT sorted based on the date in [Updated].  I don't have the brainpower right now to figure out how to convert the [Updated] values to DATETIME to get them to sort properly.  Maybe sometime soon 😏

CREATE OR ALTER PROCEDURE sasquatch__hist_one_plan_stats_T8666 @plan_handle VARBINARY(64)
AS
/* stored procedure based on work explained in the following blog post
https://sql-sasquatch.blogspot.com/2018/06/harvesting-sql-server-trace-flag-8666.html

supply a plan_handle and if trace flags 8666 was enabled at system level or session when plan was compiled
and trace flag 2388 is enabled at session level when stored procedure is executed, recent history of stats used to compile the plan will dsiplayed
*/

DECLARE @startdb NVARCHAR(256),
        @db      NVARCHAR(256),
        @schema  NVARCHAR(256),
        @table   NVARCHAR(256),
        @stats   NVARCHAR(256),
        @cmd     NVARCHAR(MAX);
        
SET @startdb = DB_NAME();
DROP TABLE IF EXISTS #plan;
CREATE TABLE #plan(planXML XML);

INSERT INTO #plan
SELECT CONVERT(XML, query_plan) planXML
FROM sys.dm_exec_query_plan(@plan_handle);

;WITH XMLNAMESPACES(default 'http://schemas.microsoft.com/sqlserver/2004/07/showplan')
SELECT DbName.DbName_Node.value('@FieldValue','NVarChar(128)')              dbName,
       SchemaName.SchemaName_Node.value('@FieldValue','NVarChar(128)')      schemaName,
       TableName.TableName_Node.value('@FieldValue','NVarChar(128)')        tableName,
       StatsName_Node.value('@FieldValue','NVarChar(128)')                  statName,
       CONVERT(INT, NULL) done
INTO #QO_stats
FROM #plan qsp
CROSS APPLY qsp.planXML.nodes(N'//InternalInfo/EnvColl/Recompile')       AS Recomp(Recomp_node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszDb"]')       AS DbName(DbName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszSchema"]')   AS SchemaName(SchemaName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszTable"]')    AS TableName(TableName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'ModTrackingInfo')                 AS [Table](Table_Node)
CROSS APPLY Table_Node.nodes(N'Field[@FieldName="wszStatName"]')         AS [Stats](StatsName_Node)
OPTION (MAXDOP 1);

DROP TABLE IF EXISTS #one_stat_hist;
CREATE TABLE #one_stat_hist
(Updated NVARCHAR(50), [Table Cardinality] BIGINT, [Snapshot Ctr] BIGINT, 
 Steps INT, Density FLOAT, [Rows Above] BIGINT, [Rows Below] BIGINT, 
 [Squared Variance Error] NUMERIC(16,16), [Inserts Since Last Update] BIGINT, 
 [Deletes Since Last Update] BIGINT, [Leading Column Type] NVARCHAR(50));

DROP TABLE IF EXISTS #stats_hist;

 CREATE TABLE #stats_hist
(dbName NVARCHAR(50), schemaName NVARCHAR(50), tableName NVARCHAR(50), statName NVARCHAR(50),
 Updated NVARCHAR(50), [Table Cardinality] BIGINT, [Snapshot Ctr] BIGINT, 
 Steps INT, Density FLOAT, [Rows Above] BIGINT, [Rows Below] BIGINT, 
 [Squared Variance Error] NUMERIC(16,16), [Inserts Since Last Update] BIGINT, 
 [Deletes Since Last Update] BIGINT, [Leading Column Type] NVARCHAR(50));

WHILE EXISTS (SELECT TOP 1 1 FROM #QO_stats WHERE done IS NULL)
BEGIN
     SELECT TOP 1 @db = dbName, @schema = schemaName, @table = tableName, @stats = statName
     FROM #QO_stats
     WHERE done IS NULL
     ORDER BY dbName, schemaName, tableName, statName;

     SET @cmd = N'USE ' + @db + N' DBCC SHOW_STATISTICS(''' + @schema + N'.' + @table + N''', ' + @stats 
              + N') WITH NO_INFOMSGS'

     INSERT INTO #one_stat_hist
     EXEC (@cmd);

  INSERT INTO #stats_hist
  SELECT @db, @schema, @table, @stats, *
  FROM #one_stat_hist;

     UPDATE #QO_stats
     SET done = 1 
     WHERE @db =   #QO_stats.dbName
     AND @schema = #QO_stats.schemaName
     AND @table =  #QO_stats.tableName 
     AND @stats =  #QO_stats.statName;

     TRUNCATE TABLE #one_stat_hist;

END

SELECT * FROM #stats_hist ORDER BY dbName, schemaName, tableName, statName

EXEC (N'USE ' + @startDB);

/* 20181206 */





Fun with SQL Server Plan Cache, Trace Flag 8666, and Stats_stream

More fun.  This time with trace flag 8666, so that it can be used prior to SQL Server 2016 SP2 or with Legacy Cardinality estimator.  If you want to transfer stats between two systems with the same schema for query plan-related testing and research.

Note: this version of the stored procedure takes a plan_handle as a parameter rather than a plan_id for the query store.




CREATE OR ALTER PROCEDURE sasquatch__xfer_one_plan_stats_T8666 @plan_handle VARBINARY(64)
AS
/* stored procedure based on work explained in the following blog post
https://sql-sasquatch.blogspot.com/2018/06/harvesting-sql-server-trace-flag-8666.html

supply a plan_handle and if trace flag 8666 was enabled at system level or session when plan was compiled
UPDATE STATISTICS commands for each of the stats used to compile the plan will be generated
*/

DECLARE @startdb NVARCHAR(256),
        @db      NVARCHAR(256),
        @schema  NVARCHAR(256),
        @table   NVARCHAR(256),
        @stats   NVARCHAR(256),
        @cmd     NVARCHAR(MAX);
        
SET @startdb = DB_NAME();
DROP TABLE IF EXISTS #plan;
CREATE TABLE #plan(planXML XML);

INSERT INTO #plan
SELECT CONVERT(XML, query_plan) planXML
FROM sys.dm_exec_query_plan(@plan_handle);

;WITH XMLNAMESPACES(default 'http://schemas.microsoft.com/sqlserver/2004/07/showplan')
SELECT DbName.DbName_Node.value('@FieldValue','NVarChar(128)')              dbName,
       SchemaName.SchemaName_Node.value('@FieldValue','NVarChar(128)')      schemaName,
       TableName.TableName_Node.value('@FieldValue','NVarChar(128)')        tableName,
       StatsName_Node.value('@FieldValue','NVarChar(128)')                  statName,
       CONVERT(VARBINARY(MAX), NULL) [stats_stream],
       CONVERT(BIGINT, NULL) [rowcount], CONVERT(BIGINT, NULL) [pagecount]
INTO #QO_stats
FROM #plan qsp
CROSS APPLY qsp.planXML.nodes(N'//InternalInfo/EnvColl/Recompile')       AS Recomp(Recomp_node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszDb"]')       AS DbName(DbName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszSchema"]')   AS SchemaName(SchemaName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'Field[@FieldName="wszTable"]')    AS TableName(TableName_Node)
CROSS APPLY Recomp.Recomp_Node.nodes(N'ModTrackingInfo')                 AS [Table](Table_Node)
CROSS APPLY Table_Node.nodes(N'Field[@FieldName="wszStatName"]')         AS [Stats](StatsName_Node)
OPTION (MAXDOP 1);

CREATE TABLE #one_stat ([stats_stream] VARBINARY(MAX), [rowcount] BIGINT, [pagecount] BIGINT);

WHILE EXISTS (SELECT TOP 1 1 FROM #QO_stats WHERE [stats_stream] IS NULL)
BEGIN
     SELECT TOP 1 @db = dbName, @schema = schemaName, @table = tableName, @stats = statName
     FROM #QO_stats
     WHERE [stats_stream] IS NULL
     ORDER BY dbName, schemaName, tableName, statName;

     SET @cmd = N'USE ' + @db + N' DBCC SHOW_STATISTICS(''' + @schema + N'.' + @table + N''', ' + @stats 
              + N') WITH STATS_STREAM, NO_INFOMSGS'

     INSERT INTO #one_stat
     EXEC (@cmd);
     UPDATE #QO_stats
     SET [stats_stream] = os.[stats_stream], [rowcount] = os.[rowcount], [pagecount] = os.[pagecount] 
     FROM #one_stat os
     WHERE @db =   #QO_stats.dbName
     AND @schema = #QO_stats.schemaName
     AND @table =  #QO_stats.tableName 
     AND @stats =  #QO_stats.statName;

     TRUNCATE TABLE #one_stat;

END

SELECT N'USE ' + qo.dbName + N' UPDATE STATISTICS ' + qo.schemaName + N'.' + qo.tableName + N'(' + qo.statName + N') WITH ' +
CASE WHEN qo.[pagecount] IS NULL THEN '' ELSE N'PAGECOUNT = ' + CONVERT(NVARCHAR(20), qo.[pagecount]) + N', ' END +
CASE WHEN qo.[rowcount] IS NULL THEN '' ELSE N'ROWCOUNT = ' + CONVERT(NVARCHAR(20), qo.[rowcount]) + N', ' END + 
N'STATS_STREAM = ' + CONVERT(NVARCHAR(MAX), qo.[stats_stream], 1) 
FROM #QO_stats qo;

EXEC (N'USE ' + @startDB);

/* 20181206 */

Wednesday, December 5, 2018

Fun with SQL Server Query Store, Query Plan 'StatisticsInfo' XML nodes, and STATS_STREAM

Note: The stored procedure in this post works with SQL Server 2016 SP2++ and on planXML for plans NOT using the Legacy CE.  There will be a future post with a version using trace flag 8666 style planXML, which can be used prior to SQL Server 2016 SP2 or with the Legacy CE.

~~~~~

DBCC CLONEDATABASE is a good thing.

But sometimes its too broadly scoped to be useful, especially between organizations.

What if I need to troubleshoot a single query from (a) known, shared schema(s) when there are 20,000 + statistics and some may contain sensitive data in the range_hi key values?

Below is a tool that can help, provided the system is SQL Server 2016 SP2 or higher, the Query Store is enabled and captured the relevant plan, and the Legacy CE was NOT used to compile the plan.  This stored procedure in that case takes the plan_id and retrieves from the Query Store and SHOW_STATISTICS commands what is needed to generate UPDATE STATISTICS statements including stats_stream, rowcount, and pagecount.

The basic mining of optimizer stats from Query Store for 2016 SP2 ++ can be seen here...

Harvesting SQL Server optimizer stats detail from Query Plan XML: Part II SQL Server 2016 SP2++
https://sql-sasquatch.blogspot.com/2018/11/harvesting-sql-server-optimizer-stats.html


CREATE OR ALTER PROCEDURE sasquatch__xfer_one_plan_stats @plan_id INT
AS
/* stored procedure based on work explained in the following blog post
https://sql-sasquatch.blogspot.com/2018/11/harvesting-sql-server-optimizer-stats.html

supply a plan_id from the Query Store in @plan_id, and if SQL Server version is 2016 SP2 or higher
AND if the Legacy CE was NOT used UPDATE STATISTICS commands for each of the stats used 
to compile the plan will be generated
*/

DECLARE @startdb NVARCHAR(256),
        @db      NVARCHAR(256),
        @schema  NVARCHAR(256),
        @table   NVARCHAR(256),
        @stats   NVARCHAR(256),
        @cmd     NVARCHAR(MAX);
        
SET @startdb = DB_NAME();
DROP TABLE IF EXISTS #plan;
CREATE TABLE #plan(planXML XML);

INSERT INTO #plan
SELECT CONVERT(XML, query_plan) planXML
FROM sys.query_store_plan
WHERE plan_id = @plan_id;

;WITH XMLNAMESPACES(default 'http://schemas.microsoft.com/sqlserver/2004/07/showplan')
 SELECT dbName, schemaName, tableName, statName, CONVERT(VARBINARY(MAX), NULL) [stats_stream],
        CONVERT(BIGINT, NULL) [rowcount], CONVERT(BIGINT, NULL) [pagecount]
 INTO #QO_stats
 FROM #plan
 CROSS APPLY planXML.nodes(N'/ShowPlanXML/BatchSequence/Batch/Statements/StmtSimple/QueryPlan/OptimizerStatsUsage') qp(qpnode)  
 CROSS APPLY qpnode.nodes(N'StatisticsInfo') qp2(statsinfo)
 CROSS APPLY (SELECT qp2.statsinfo.value(N'@Statistics', N'varchar(max)')) statName(statName)
 CROSS APPLY (SELECT qp2.statsinfo.value(N'@Table',      N'varchar(max)')) tableName(tableName)
 CROSS APPLY (SELECT qp2.statsinfo.value(N'@Schema',     N'varchar(max)')) schemaName(schemaName)
 CROSS APPLY (SELECT qp2.statsinfo.value(N'@Database',   N'varchar(max)')) dbName(dbName)
 OPTION (MAXDOP 1);

 CREATE TABLE #one_stat ([stats_stream] VARBINARY(MAX), [rowcount] BIGINT, [pagecount] BIGINT);

 WHILE EXISTS (SELECT TOP 1 1 FROM #QO_stats WHERE [stats_stream] IS NULL)
 BEGIN
      SELECT TOP 1 @db = dbName, @schema = schemaName, @table = tableName, @stats = statName
      FROM #QO_stats
      WHERE [stats_stream] IS NULL
      ORDER BY dbName, schemaName, tableName, statName;

      SET @cmd = N'USE ' + @db + N' DBCC SHOW_STATISTICS(''' + @schema + N'.' + @table + N''', ' + @stats 
            + N') WITH STATS_STREAM, NO_INFOMSGS'

      INSERT INTO #one_stat
      EXEC (@cmd);

      UPDATE #QO_stats
      SET [stats_stream] = os.[stats_stream], [rowcount] = os.[rowcount], [pagecount] = os.[pagecount] 
      FROM #one_stat os
      WHERE @db =   #QO_stats.dbName
      AND @schema = #QO_stats.schemaName
      AND @table =  #QO_stats.tableName 
      AND @stats =  #QO_stats.statName;

      TRUNCATE TABLE #one_stat;

 END

SELECT N'USE ' + qo.dbName + N' UPDATE STATISTICS ' + qo.schemaName + N'.' + qo.tableName + N'(' + qo.statName + N') WITH ' +
CASE WHEN qo.[pagecount] IS NULL THEN '' ELSE N'PAGECOUNT = ' + CONVERT(NVARCHAR(20), qo.[pagecount]) + N', ' END +
CASE WHEN qo.[rowcount] IS NULL THEN '' ELSE N'ROWCOUNT = ' + CONVERT(NVARCHAR(20), qo.[rowcount]) + N', ' END + 
N'STATS_STREAM = ' + CONVERT(NVARCHAR(MAX), qo.[stats_stream], 1) 
FROM #QO_stats qo;

EXEC (N'USE ' + @startDB);

/* 20181205 */