-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- need superuser to adjust statistics in load script
\c :TEST_DBNAME :ROLE_SUPERUSER
\set TEST_BASE_NAME skip_scan
SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME",
    format('include/%s_dagg_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME",
    format('%s/results/%s_dagg_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_UNOPTIMIZED",
    format('%s/results/%s_dagg_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_OPTIMIZED" \gset
SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') AS "DIFF_CMD" \gset
\ir :TEST_LOAD_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
CREATE TABLE skip_scan(time int, dev int, dev_name text, val int);
INSERT INTO skip_scan SELECT t, d, 'device_' || d::text, t*d FROM generate_series(1, 1000) t, generate_series(1, 10) d;
INSERT INTO skip_scan VALUES (NULL, 0, -1, NULL), (0, NULL, -1, NULL);
INSERT INTO skip_scan(time,dev,dev_name,val) SELECT t, NULL, NULL, NULL FROM generate_series(0, 999, 50) t;
ANALYZE skip_scan;
CREATE TABLE skip_scan_nulls(time int);
CREATE INDEX ON skip_scan_nulls(time);
INSERT INTO skip_scan_nulls SELECT NULL FROM generate_series(1,100);
ANALYZE skip_scan_nulls;
-- create hypertable with different physical layouts in the chunks
CREATE TABLE skip_scan_ht(f1 int, f2 int, f3 int, time int NOT NULL, dev int, dev_name text, val int);
SELECT create_hypertable('skip_scan_ht', 'time', chunk_time_interval => 250, create_default_indexes => false);
     create_hypertable     
---------------------------
 (1,public,skip_scan_ht,t)

INSERT INTO skip_scan_ht(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(0, 249) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_ht DROP COLUMN f1;
INSERT INTO skip_scan_ht(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(250, 499) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_ht DROP COLUMN f2;
INSERT INTO skip_scan_ht(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(500, 749) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_ht DROP COLUMN f3;
INSERT INTO skip_scan_ht(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(750, 999) t, generate_series(1, 10) d;
INSERT INTO skip_scan_ht(time,dev,dev_name,val) SELECT t, NULL, NULL, NULL FROM generate_series(0, 999, 50) t;
ANALYZE skip_scan_ht;
ALTER TABLE skip_scan_ht SET (timescaledb.compress,timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
-- create compressed hypertable with different physical layouts in the chunks
CREATE TABLE skip_scan_htc(f1 int, f2 int, f3 int, time int NOT NULL, dev int, dev_name text, val int);
SELECT create_hypertable('skip_scan_htc', 'time', chunk_time_interval => 250, create_default_indexes => false);
     create_hypertable      
----------------------------
 (3,public,skip_scan_htc,t)

ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
INSERT INTO skip_scan_htc(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(0, 249) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_htc DROP COLUMN f1;
INSERT INTO skip_scan_htc(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(250, 499) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_htc DROP COLUMN f2;
INSERT INTO skip_scan_htc(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(500, 749) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_htc DROP COLUMN f3;
INSERT INTO skip_scan_htc(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(750, 999) t, generate_series(1, 10) d;
INSERT INTO skip_scan_htc(time,dev,dev_name,val) SELECT t, NULL, NULL, NULL FROM generate_series(0, 999, 50) t;
ANALYZE skip_scan_htc;
CREATE TABLE skip_scan_insert(time int, dev int, dev_name text, val int, query text);
CREATE OR REPLACE FUNCTION int_func_immutable() RETURNS int LANGUAGE SQL IMMUTABLE SECURITY DEFINER AS $$SELECT 1; $$;
CREATE OR REPLACE FUNCTION int_func_stable() RETURNS int LANGUAGE SQL STABLE SECURITY DEFINER AS $$ SELECT 2; $$;
CREATE OR REPLACE FUNCTION int_func_volatile() RETURNS int LANGUAGE SQL VOLATILE SECURITY DEFINER AS $$ SELECT 3; $$;
CREATE OR REPLACE FUNCTION inta_func_immutable() RETURNS int[] LANGUAGE SQL IMMUTABLE SECURITY DEFINER AS $$ SELECT ARRAY[1,2,3]; $$;
CREATE OR REPLACE FUNCTION inta_func_stable() RETURNS int[] LANGUAGE SQL STABLE SECURITY DEFINER AS $$ SELECT ARRAY[2,3,4]; $$;
CREATE OR REPLACE FUNCTION inta_func_volatile() RETURNS int[] LANGUAGE SQL VOLATILE SECURITY DEFINER AS $$ SELECT ARRAY[3,4,5]; $$;
-- adjust statistics so we get skipscan plans
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid='skip_scan'::regclass;
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid='skip_scan_nulls'::regclass;
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid='skip_scan_ht'::regclass;
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid IN (select inhrelid from pg_inherits where inhparent='skip_scan_ht'::regclass);
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid='skip_scan_htc'::regclass;
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid IN (select inhrelid from pg_inherits where inhparent='skip_scan_htc'::regclass);
-- Turn off autovacuum to not trigger new vacuums that restores the
-- adjusted statistics
alter table skip_scan set (autovacuum_enabled = off);
alter table skip_scan_nulls set (autovacuum_enabled = off);
alter table skip_scan_ht set (autovacuum_enabled = off);
alter table skip_scan_htc set (autovacuum_enabled = off);
-- create compressed hypertable with different physical layouts in the compressed chunks
CREATE TABLE skip_scan_htcl(f1 int, f2 int, f3 int, time int NOT NULL, dev int, dev_name text, val int);
SELECT create_hypertable('skip_scan_htcl', 'time', chunk_time_interval => 250, create_default_indexes => false);
      create_hypertable      
-----------------------------
 (5,public,skip_scan_htcl,t)

ALTER TABLE skip_scan_htcl SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
INSERT INTO skip_scan_htcl(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(0, 249) t, generate_series(1, 10) d;
-- Make sure 1st compressed chunk has columns which will be dropped later, it also doesn't have NULLs
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htcl') ch;
             compress_chunk             
----------------------------------------
 _timescaledb_internal._hyper_5_9_chunk

ALTER TABLE skip_scan_htcl DROP COLUMN f1;
INSERT INTO skip_scan_htcl(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(250, 499) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_htcl DROP COLUMN f2;
INSERT INTO skip_scan_htcl(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(500, 749) t, generate_series(1, 10) d;
ALTER TABLE skip_scan_htcl DROP COLUMN f3;
INSERT INTO skip_scan_htcl(time,dev,dev_name,val) SELECT t, d, 'device_' || d::text, random() FROM generate_series(750, 999) t, generate_series(1, 10) d;
INSERT INTO skip_scan_htcl(time,dev,dev_name,val) SELECT t, NULL, NULL, NULL FROM generate_series(0, 999, 50) t;
-- The rest of the compressed chunks do not have dropped columns
-- compressed chunks #2 and #3 have attnos out of sync with uncompressed chunks
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htcl') ch order by 1 desc limit 3;
             compress_chunk              
-----------------------------------------
 _timescaledb_internal._hyper_5_13_chunk
 _timescaledb_internal._hyper_5_12_chunk
 _timescaledb_internal._hyper_5_11_chunk

ANALYZE skip_scan_htcl;
-- adjust statistics so we get skipscan plans
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid='skip_scan_htcl'::regclass;
UPDATE pg_statistic SET stadistinct=1, stanullfrac=0.5 WHERE starelid IN (select inhrelid from pg_inherits where inhparent='skip_scan_htcl'::regclass);
-- Turn off autovacuum to not trigger new vacuums that restores the
-- adjusted statistics
alter table skip_scan_htcl set (autovacuum_enabled = off);
-- run tests on normal table and diff results
\set TABLE skip_scan
\set PREFIX ''
\o :TEST_RESULTS_OPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_skipscan_for_distinct_aggregates') AS enable_dagg_skipscan;
-- test different index configurations
-- no index so we cant do SkipScan
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
-- NULLS LAST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_last ON :TABLE(dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_nulls_last;
-- NULLS FIRST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_first ON :TABLE(dev NULLS FIRST);
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev  NULLS FIRST;
DROP INDEX skip_scan_idx_dev_nulls_first;
-- multicolumn index with dev as leading column
CREATE INDEX skip_scan_idx_dev_time_idx ON :TABLE(dev, time);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_time_idx;
-- multicolumn index with dev as non-leading column
CREATE INDEX skip_scan_idx_time_dev_idx ON :TABLE(time, dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE time = 100;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 GROUP BY dev;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 AND dev IS NOT NULL GROUP BY dev;
DROP INDEX skip_scan_idx_time_dev_idx;
-- hash index is not ordered so can't use skipscan
CREATE INDEX skip_scan_idx_hash ON :TABLE USING hash(dev_name);
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IN ('device_1','device_2');
DROP INDEX skip_scan_idx_hash;
-- expression indexes
-- currently not supported by skipscan
CREATE INDEX skip_scan_expr_idx ON :TABLE((dev % 3));
:PREFIX SELECT count(DISTINCT dev%3) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev%3), dev%3 FROM :TABLE GROUP BY dev%3 ORDER BY dev%3;
DROP INDEX skip_scan_expr_idx;
CREATE INDEX ON :TABLE(dev_name);
CREATE INDEX ON :TABLE(dev);
CREATE INDEX ON :TABLE(dev, time);
CREATE INDEX ON :TABLE(time,dev);
CREATE INDEX ON :TABLE(time,dev,val);
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT time_bucket(10,time)), 'q1_12' FROM :TABLE;
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
:PREFIX SELECT count(DISTINCT 3*time), 'q1_14' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
\qecho ordered append on :TABLE
:PREFIX SELECT count(DISTINCT time), time FROM :TABLE WHERE time BETWEEN 0 AND 5000  GROUP BY time;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev_name) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT time) FROM :TABLE;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT time) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls WHERE time IS NOT NULL;
\o
SET timescaledb.enable_skipscan_for_distinct_aggregates TO false;
\o :TEST_RESULTS_UNOPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_skipscan_for_distinct_aggregates') AS enable_dagg_skipscan;
-- test different index configurations
-- no index so we cant do SkipScan
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
-- NULLS LAST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_last ON :TABLE(dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_nulls_last;
-- NULLS FIRST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_first ON :TABLE(dev NULLS FIRST);
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev  NULLS FIRST;
DROP INDEX skip_scan_idx_dev_nulls_first;
-- multicolumn index with dev as leading column
CREATE INDEX skip_scan_idx_dev_time_idx ON :TABLE(dev, time);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_time_idx;
-- multicolumn index with dev as non-leading column
CREATE INDEX skip_scan_idx_time_dev_idx ON :TABLE(time, dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE time = 100;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 GROUP BY dev;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 AND dev IS NOT NULL GROUP BY dev;
DROP INDEX skip_scan_idx_time_dev_idx;
-- hash index is not ordered so can't use skipscan
CREATE INDEX skip_scan_idx_hash ON :TABLE USING hash(dev_name);
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IN ('device_1','device_2');
DROP INDEX skip_scan_idx_hash;
-- expression indexes
-- currently not supported by skipscan
CREATE INDEX skip_scan_expr_idx ON :TABLE((dev % 3));
:PREFIX SELECT count(DISTINCT dev%3) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev%3), dev%3 FROM :TABLE GROUP BY dev%3 ORDER BY dev%3;
DROP INDEX skip_scan_expr_idx;
CREATE INDEX ON :TABLE(dev_name);
CREATE INDEX ON :TABLE(dev);
CREATE INDEX ON :TABLE(dev, time);
CREATE INDEX ON :TABLE(time,dev);
CREATE INDEX ON :TABLE(time,dev,val);
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT time_bucket(10,time)), 'q1_12' FROM :TABLE;
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
:PREFIX SELECT count(DISTINCT 3*time), 'q1_14' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
\qecho ordered append on :TABLE
:PREFIX SELECT count(DISTINCT time), time FROM :TABLE WHERE time BETWEEN 0 AND 5000  GROUP BY time;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev_name) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT time) FROM :TABLE;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT time) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls WHERE time IS NOT NULL;
\o
RESET timescaledb.enable_skipscan_for_distinct_aggregates;
-- compare SkipScan results on normal table
:DIFF_CMD
--- Unoptimized results
+++ Optimized results
@@ -1,6 +1,6 @@
  enable_dagg_skipscan 
 ----------------------
- off
+ on
 
  count 
-- run tests on hypertable and diff results
\set TABLE skip_scan_ht
\set PREFIX ''
\o :TEST_RESULTS_OPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_skipscan_for_distinct_aggregates') AS enable_dagg_skipscan;
-- test different index configurations
-- no index so we cant do SkipScan
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
-- NULLS LAST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_last ON :TABLE(dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_nulls_last;
-- NULLS FIRST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_first ON :TABLE(dev NULLS FIRST);
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev  NULLS FIRST;
DROP INDEX skip_scan_idx_dev_nulls_first;
-- multicolumn index with dev as leading column
CREATE INDEX skip_scan_idx_dev_time_idx ON :TABLE(dev, time);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_time_idx;
-- multicolumn index with dev as non-leading column
CREATE INDEX skip_scan_idx_time_dev_idx ON :TABLE(time, dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE time = 100;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 GROUP BY dev;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 AND dev IS NOT NULL GROUP BY dev;
DROP INDEX skip_scan_idx_time_dev_idx;
-- hash index is not ordered so can't use skipscan
CREATE INDEX skip_scan_idx_hash ON :TABLE USING hash(dev_name);
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IN ('device_1','device_2');
DROP INDEX skip_scan_idx_hash;
-- expression indexes
-- currently not supported by skipscan
CREATE INDEX skip_scan_expr_idx ON :TABLE((dev % 3));
:PREFIX SELECT count(DISTINCT dev%3) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev%3), dev%3 FROM :TABLE GROUP BY dev%3 ORDER BY dev%3;
DROP INDEX skip_scan_expr_idx;
CREATE INDEX ON :TABLE(dev_name);
CREATE INDEX ON :TABLE(dev);
CREATE INDEX ON :TABLE(dev, time);
CREATE INDEX ON :TABLE(time,dev);
CREATE INDEX ON :TABLE(time,dev,val);
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT time_bucket(10,time)), 'q1_12' FROM :TABLE;
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
:PREFIX SELECT count(DISTINCT 3*time), 'q1_14' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
\qecho ordered append on :TABLE
:PREFIX SELECT count(DISTINCT time), time FROM :TABLE WHERE time BETWEEN 0 AND 5000  GROUP BY time;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev_name) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT time) FROM :TABLE;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT time) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls WHERE time IS NOT NULL;
\o
SET timescaledb.enable_skipscan_for_distinct_aggregates TO false;
\o :TEST_RESULTS_UNOPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_skipscan_for_distinct_aggregates') AS enable_dagg_skipscan;
-- test different index configurations
-- no index so we cant do SkipScan
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
-- NULLS LAST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_last ON :TABLE(dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_nulls_last;
-- NULLS FIRST index on dev
CREATE INDEX skip_scan_idx_dev_nulls_first ON :TABLE(dev NULLS FIRST);
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev  NULLS FIRST;
DROP INDEX skip_scan_idx_dev_nulls_first;
-- multicolumn index with dev as leading column
CREATE INDEX skip_scan_idx_dev_time_idx ON :TABLE(dev, time);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
DROP INDEX skip_scan_idx_dev_time_idx;
-- multicolumn index with dev as non-leading column
CREATE INDEX skip_scan_idx_time_dev_idx ON :TABLE(time, dev);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE time = 100;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 GROUP BY dev;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time = 100 AND dev IS NOT NULL GROUP BY dev;
DROP INDEX skip_scan_idx_time_dev_idx;
-- hash index is not ordered so can't use skipscan
CREATE INDEX skip_scan_idx_hash ON :TABLE USING hash(dev_name);
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IN ('device_1','device_2');
DROP INDEX skip_scan_idx_hash;
-- expression indexes
-- currently not supported by skipscan
CREATE INDEX skip_scan_expr_idx ON :TABLE((dev % 3));
:PREFIX SELECT count(DISTINCT dev%3) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev%3), dev%3 FROM :TABLE GROUP BY dev%3 ORDER BY dev%3;
DROP INDEX skip_scan_expr_idx;
CREATE INDEX ON :TABLE(dev_name);
CREATE INDEX ON :TABLE(dev);
CREATE INDEX ON :TABLE(dev, time);
CREATE INDEX ON :TABLE(time,dev);
CREATE INDEX ON :TABLE(time,dev,val);
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT time_bucket(10,time)), 'q1_12' FROM :TABLE;
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
:PREFIX SELECT count(DISTINCT 3*time), 'q1_14' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
\qecho ordered append on :TABLE
:PREFIX SELECT count(DISTINCT time), time FROM :TABLE WHERE time BETWEEN 0 AND 5000  GROUP BY time;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev_name) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT time) FROM :TABLE;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT time) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT time) FROM skip_scan_nulls WHERE time IS NOT NULL;
\o
RESET timescaledb.enable_skipscan_for_distinct_aggregates;
-- compare SkipScan results on hypertable
:DIFF_CMD
--- Unoptimized results
+++ Optimized results
@@ -1,6 +1,6 @@
  enable_dagg_skipscan 
 ----------------------
- off
+ on
 
  count 
-- run tests on compressed hypertable and diff results
SELECT format('include/%s_dagg_comp_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME" \gset
\set TABLE skip_scan_htc
\set PREFIX ''
\o :TEST_RESULTS_OPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_compressed_skipscan') AS enable_compressed_skipscan;
-- To avoid ambiguity in test EXPLAIN outputs due to mixing of chunk plans with no SkipScan
SET max_parallel_workers_per_gather = 0;
-- test different compression configurations
-- compressed index on "segmentby='dev'" has default "dev ASC, NULLS LAST" sort order
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- SkipScan used when sort order on distinct column "dev" matches "segmentby" sort order
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC NULLS FIRST;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev;
-- NULLS FIRST doesn't match segmentby NULL direction
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- multicolumn "segmentby = 'dev, dev_name'"
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev,dev_name');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- multicolumn compressed index with dev as leading column
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
-- multicolumn compressed index with dev as leading column and with extra distinct column pinned
-- TODO: should be able to apply SkipScan here, issue created: #7998
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT dev_name) FROM :TABLE WHERE dev_name = 'device_1';
-- multicolumn compressed index with dev as non-leading column
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev_name), dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev_name;
-- multicolumn compressed index with dev as non-leading column and with leading column pinned
-- TODO: should be able to apply SkipScan here, issue created: #7998
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT dev_name) FROM :TABLE WHERE dev = 1;
-- Basic tests for "segmentby = 'dev'"
-----------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT dev + 1), 'q1_13' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
-- Basic tests for text index "segmentby = 'dev_name'"
------------------------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev_name');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
-- Various tests for "segmentby = 'dev'"
---------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
-- "segmentby = 'dev, val'"
---------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time', timescaledb.compress_segmentby='dev, val');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT val) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT val) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
RESET max_parallel_workers_per_gather;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE where val IS NULL;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE val IS NULL AND dev > 100;
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\o
SET timescaledb.enable_compressed_skipscan TO false;
\o :TEST_RESULTS_UNOPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_compressed_skipscan') AS enable_compressed_skipscan;
-- To avoid ambiguity in test EXPLAIN outputs due to mixing of chunk plans with no SkipScan
SET max_parallel_workers_per_gather = 0;
-- test different compression configurations
-- compressed index on "segmentby='dev'" has default "dev ASC, NULLS LAST" sort order
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- SkipScan used when sort order on distinct column "dev" matches "segmentby" sort order
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC NULLS FIRST;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC NULLS FIRST;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev;
-- NULLS FIRST doesn't match segmentby NULL direction
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev NULLS FIRST;
-- multicolumn "segmentby = 'dev, dev_name'"
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev,dev_name');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- multicolumn compressed index with dev as leading column
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev IS NOT NULL GROUP BY dev ORDER BY dev DESC;
-- multicolumn compressed index with dev as leading column and with extra distinct column pinned
-- TODO: should be able to apply SkipScan here, issue created: #7998
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT dev_name) FROM :TABLE WHERE dev_name = 'device_1';
-- multicolumn compressed index with dev as non-leading column
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev_name), dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev_name;
-- multicolumn compressed index with dev as non-leading column and with leading column pinned
-- TODO: should be able to apply SkipScan here, issue created: #7998
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT dev_name) FROM :TABLE WHERE dev = 1;
-- Basic tests for "segmentby = 'dev'"
-----------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
-- Mix of aggregates on different columns and distinct/not distinct
-- currently not supported by skipscan
:PREFIX SELECT count(DISTINCT dev), max(DISTINCT dev_name), 'q1_9' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), sum(dev), 'q1_10' FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev_name, 'q1_11' FROM :TABLE GROUP BY dev_name ORDER BY dev_name;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT dev + 1), 'q1_13' FROM :TABLE;
-- But expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT time, count(DISTINCT dev), 'q2_10' FROM :TABLE GROUP BY time;
-- Cannot do SkipScan if we group on 2+ columns
:PREFIX SELECT count(DISTINCT dev), dev, tableoid::regclass, 'q2_11' FROM :TABLE GROUP BY dev, tableoid ORDER BY dev, tableoid;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
-- Basic tests for text index "segmentby = 'dev_name'"
------------------------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time desc', timescaledb.compress_segmentby='dev_name');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev_name), max(DISTINCT dev_name), 'q1_2' FROM :TABLE;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev_name), 'q1_6', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev_name), 'q1_8', length(md5(random()::text)) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev_name) FROM :TABLE WHERE dev_name IS NULL;
-- distinct on expressions not supported
:PREFIX SELECT count(DISTINCT length(dev_name)), 'q1_13' FROM :TABLE;
-- DISTINCT aggs grouped on their TEXT args
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_1' FROM :TABLE GROUP BY dev_name;
-- Test not-NULL mode
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_11' FROM :TABLE WHERE dev_name IS NOT NULL GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_2', NULL FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, 'q3_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name::varchar), dev_name::varchar, 'q3_5' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_immutable(), 'q3_6' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_stable(), 'q3_7' FROM :TABLE GROUP BY dev_name;
:PREFIX SELECT count(DISTINCT dev_name), dev_name, int_func_volatile(), 'q3_8' FROM :TABLE GROUP BY dev_name;
-- Cannot do SkipScan as we group on a column which is not the distinct agg argument
:PREFIX SELECT count(DISTINCT dev_name), dev_name, tableoid::regclass, 'q3_9' FROM :TABLE GROUP BY dev_name, tableoid ORDER BY dev_name, tableoid;
-- Can do SkipScan if extra group column is eliminated by pinning to a Const
-- and when it changes group by ordering
:PREFIX SELECT count(DISTINCT dev_name), dev, dev_name FROM :TABLE WHERE dev = 1 GROUP BY dev, dev_name ORDER BY dev, dev_name;
-- Various tests for "segmentby = 'dev'"
---------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time', timescaledb.compress_segmentby='dev');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
-- Distinct aggregate path with no pathkeys because of Const predicate.
-- PG is smart to add LIMIT 1 to SELECT DISTINCT in this case,
-- but not smart enough to add LIMIT 1 to distinct aggregate input.
-- TODO: create an issue for this task, i.e. add LIMIT 1 to distinct aggregate input when input equals to a Const
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE dev = 1 GROUP BY dev ORDER BY dev DESC;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
--  DISTINCT aggs on different columns in different subqueries
-- "segmentby = 'dev, val'"
---------------------------------------
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
ALTER TABLE skip_scan_htc SET (timescaledb.compress, timescaledb.compress_orderby='time', timescaledb.compress_segmentby='dev, val');
SELECT compress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE UNION ALL SELECT count(DISTINCT val) FROM :TABLE WHERE dev = 1;
:PREFIX SELECT *, 'q9_2' FROM (SELECT count(DISTINCT dev) cd, dev FROM :TABLE GROUP BY dev) a, LATERAL (SELECT count(DISTINCT val) ct FROM :TABLE WHERE dev = a.dev) b;
-- SkipScan into INSERT
:PREFIX INSERT INTO skip_scan_insert(dev, val, query) SELECT dev, sd, 'q10_1' FROM (SELECT sum(DISTINCT dev) sd, dev FROM :TABLE GROUP BY dev) a;
-- parallel query
RESET max_parallel_workers_per_gather;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false);
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false);
TRUNCATE skip_scan_insert;
-- table with only nulls
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE where val IS NULL;
-- no tuples in resultset
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE val IS NULL AND dev > 100;
SELECT decompress_chunk(ch) FROM show_chunks('skip_scan_htc') ch;
\o
RESET timescaledb.enable_compressed_skipscan;
-- compare SkipScan results on hypertable
:DIFF_CMD
--- Unoptimized results
+++ Optimized results
@@ -1,6 +1,6 @@
  enable_compressed_skipscan 
 ----------------------------
- off
+ on
 
              compress_chunk             
-- run tests on compressed hypertable with different layouts of compressed chunks
SELECT format('include/%s_dagg_load_comp_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME" \gset
\set TABLE skip_scan_htcl
\set PREFIX ''
\o :TEST_RESULTS_OPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_compressed_skipscan') AS enable_compressed_skipscan;
-- To avoid ambiguity in test EXPLAIN outputs due to mixing of chunk plans with no SkipScan
SET max_parallel_workers_per_gather = 0;
-- Run SkipScan queries on the provided compression layout
-- compressed index on "segmentby='dev'" has default "dev ASC, NULLS LAST" sort order
-- SkipScan used when sort order on distinct column "dev" matches "segmentby" sort order
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC NULLS FIRST;
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
-- expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
set enable_seqscan to off;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
reset enable_seqscan;
-- Various tests for "segmentby = 'dev'"
---------------------------------------
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
-- Force not choosing SeqScan under DecompressChunks before SkipScan can evaluate it,
-- as cost of scanning and sorting a small dataset can be less than scanning an index, including startup cost.
-- After the fix in #8056 we take startup costs into account,
-- but even startup costs for scanning and sorting can be smaller for seqscan if we also have a filter on the index columns.
-- So for below queries with filters on "dev" we still need to disable SeqScan to choose IndexScan+SkipScan.
SET enable_seqscan = false;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
RESET max_parallel_workers_per_gather;
RESET enable_seqscan;
\o
SET timescaledb.enable_compressed_skipscan TO false;
\o :TEST_RESULTS_UNOPTIMIZED
\ir :TEST_QUERY_NAME
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- canary for result diff
SELECT current_setting('timescaledb.enable_compressed_skipscan') AS enable_compressed_skipscan;
-- To avoid ambiguity in test EXPLAIN outputs due to mixing of chunk plans with no SkipScan
SET max_parallel_workers_per_gather = 0;
-- Run SkipScan queries on the provided compression layout
-- compressed index on "segmentby='dev'" has default "dev ASC, NULLS LAST" sort order
-- SkipScan used when sort order on distinct column "dev" matches "segmentby" sort order
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC NULLS FIRST;
\qecho basic DISTINCT queries on :TABLE
-- Various distint aggs over same column is OK
:PREFIX SELECT count(DISTINCT dev), sum(DISTINCT dev), 'q1_1' FROM :TABLE;
-- Distinct agg over Const is OK
:PREFIX SELECT count(DISTINCT dev), count(DISTINCT 2), 'q1_3', NULL FROM :TABLE;
-- DISTINCT over distinct agg is OK
:PREFIX SELECT DISTINCT count(DISTINCT dev), dev, 'q1_4' FROM :TABLE GROUP BY dev ORDER BY dev;
\qecho stable expression in targetlist on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q1_5', length(md5(now()::text)) FROM :TABLE;
-- volatile expression in targetlist
:PREFIX SELECT count(DISTINCT dev), 'q1_7', length(md5(random()::text)) FROM :TABLE;
-- expressions over distinct aggregates are supported
:PREFIX SELECT count(DISTINCT dev) + 1, sum(DISTINCT dev)/count(DISTINCT dev), 'q1_15' FROM :TABLE;
-- DISTINCT aggs grouped on their args
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_1' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_2', NULL FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_3', length(md5(now()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, 'q2_4', length(md5(random()::text)) FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_immutable(), 'q2_5' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_stable(), 'q2_6' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev, int_func_volatile(), 'q2_7' FROM :TABLE GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev+1, 'q2_8' FROM :TABLE GROUP BY dev ORDER BY 2;
:PREFIX SELECT count(DISTINCT dev), dev+1, dev+2, 'q2_9' FROM :TABLE GROUP BY dev, dev;
\qecho LIMIT queries on :TABLE
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev LIMIT 3;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE GROUP BY dev ORDER BY dev DESC LIMIT 3;
\qecho range queries on :TABLE
set enable_seqscan to off;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time BETWEEN 100 AND 300 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time < 200 GROUP BY dev;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 800 GROUP BY dev;
reset enable_seqscan;
-- Various tests for "segmentby = 'dev'"
---------------------------------------
\qecho SUBSELECTS on :TABLE
:PREFIX SELECT c1, c2, 'q4_1' FROM (SELECT count(DISTINCT dev) as c1, sum(DISTINCT dev) as c2 FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, 'q4_2' FROM (SELECT count(DISTINCT dev) as dev FROM :TABLE) a;
:PREFIX SELECT NULL, dev, NULL, c1, 2, c3, 'q4_3' FROM (SELECT count(DISTINCT dev) as c1, dev, 1 as c3 FROM :TABLE GROUP BY dev) a;
\qecho ORDER BY
:PREFIX SELECT c, dev, 'q5_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev) a;
:PREFIX SELECT c, dev, 'q5_2' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev ORDER BY dev DESC) a;
-- Force not choosing SeqScan under DecompressChunks before SkipScan can evaluate it,
-- as cost of scanning and sorting a small dataset can be less than scanning an index, including startup cost.
-- After the fix in #8056 we take startup costs into account,
-- but even startup costs for scanning and sorting can be smaller for seqscan if we also have a filter on the index columns.
-- So for below queries with filters on "dev" we still need to disable SeqScan to choose IndexScan+SkipScan.
SET enable_seqscan = false;
\qecho WHERE CLAUSES
:PREFIX SELECT c, dev, 'q6_1' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE WHERE dev > 5 GROUP BY dev) a;
:PREFIX SELECT c, dev, 'q6_2' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev HAVING sum(DISTINCT dev)>2) a;
:PREFIX SELECT c, dev, 'q6_3' FROM (SELECT count(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE dev > 5;
:PREFIX SELECT c, dev, 'q6_4' FROM (SELECT sum(DISTINCT dev) as c, dev FROM :TABLE GROUP BY dev) a WHERE c > 2;
--\qecho immutable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_5' FROM :TABLE WHERE dev > int_func_immutable();
--\qecho stable func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_6' FROM :TABLE WHERE dev > int_func_stable();
--\qecho volatile func in WHERE clause on :TABLE
:PREFIX SELECT count(DISTINCT dev), 'q6_7' FROM :TABLE WHERE dev > int_func_volatile();
:PREFIX SELECT count(DISTINCT dev), 'q6_8' FROM :TABLE WHERE dev = ANY(inta_func_immutable());
:PREFIX SELECT count(DISTINCT dev), 'q6_9' FROM :TABLE WHERE dev = ANY(inta_func_stable());
:PREFIX SELECT count(DISTINCT dev), 'q6_10' FROM :TABLE WHERE dev = ANY(inta_func_volatile());
-- always false expr similar to our initial skip qual
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > NULL;
-- no tuples matching
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 20;
-- multiple constraints in WHERE clause
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time = 100;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev > 5 AND time > 200;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev >= 5 AND dev < 7 AND dev >= 2;
:PREFIX SELECT count(DISTINCT dev), dev FROM :TABLE WHERE time > 100 AND time < 200 AND val > 10 AND val < 10000 AND dev > 2 AND dev < 7 GROUP BY dev ORDER BY dev;
:PREFIX SELECT count(DISTINCT dev) FROM :TABLE WHERE dev IS NULL;
-- CTE
:PREFIX WITH devices AS (
	SELECT count(DISTINCT dev) FROM :TABLE
)
SELECT * FROM devices;
:PREFIX WITH devices AS (
	SELECT dev, count(DISTINCT dev) FROM :TABLE GROUP BY dev
)
SELECT * FROM devices ORDER BY dev;
-- prepared statements
PREPARE prep AS SELECT count(DISTINCT dev) FROM :TABLE;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
:PREFIX EXECUTE prep;
DEALLOCATE prep;
-- ReScan tests
:PREFIX SELECT c, 'q7_1' FROM (SELECT count(DISTINCT dev) c FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT * FROM :TABLE WHERE time != a.v) b) a;
:PREFIX SELECT c, 'q7_2' FROM (SELECT * FROM (
    VALUES (1), (2)) a(v),
    LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev != a.v) b) a;
-- RuntimeKeys
:PREFIX SELECT c, 'q8_1' FROM (SELECT * FROM (VALUES (1), (2)) a(v), LATERAL (SELECT count(DISTINCT dev) c FROM :TABLE WHERE dev >= a.v) b) c;
RESET max_parallel_workers_per_gather;
RESET enable_seqscan;
\o
RESET timescaledb.enable_compressed_skipscan;
-- compare SkipScan results on hypertable
:DIFF_CMD
--- Unoptimized results
+++ Optimized results
@@ -1,6 +1,6 @@
  enable_compressed_skipscan 
 ----------------------------
- off
+ on
 
  count 
