-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- need superuser for copy from program
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE TABLE metrics (time TIMESTAMPTZ NOT NULL, device TEXT, value float) WITH (tsdb.hypertable, tsdb.partition_column = 'time', tsdb.orderby='time');
-- first try without the GUCs
BEGIN;
COPY metrics FROM PROGRAM 'seq 100 | xargs -II date -d "2025-01-01 + I hour" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Append (actual rows=100.00 loops=1)
   ->  Seq Scan on _hyper_1_1_chunk (actual rows=15.00 loops=1)
   ->  Seq Scan on _hyper_1_2_chunk (actual rows=85.00 loops=1)

ROLLBACK;
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_sort_batches = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
-- simple test with compressed copy enabled
BEGIN;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 + I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Append (actual rows=3000.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_3_chunk (actual rows=959.00 loops=1)
         ->  Seq Scan on compress_hyper_2_4_chunk (actual rows=1.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_5_chunk (actual rows=2041.00 loops=1)
         ->  Seq Scan on compress_hyper_2_6_chunk (actual rows=3.00 loops=1)

SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub;
            first             |             last             
------------------------------+------------------------------
 Wed Jan 01 00:01:00 2025 PST | Fri Jan 03 02:00:00 2025 PST

ROLLBACK;
-- simple test with compressed copy enabled and reversed order
BEGIN;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 - I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_7_chunk (actual rows=3000.00 loops=1)
   ->  Seq Scan on compress_hyper_2_8_chunk (actual rows=3.00 loops=1)

SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub;
            first             |             last             
------------------------------+------------------------------
 Sun Dec 29 22:00:00 2024 PST | Tue Dec 31 23:59:00 2024 PST

ROLLBACK;
SET timescaledb.enable_direct_compress_copy_sort_batches = false;
-- simple test with compressed copy enabled and without batch sorting
BEGIN;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 + I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Append (actual rows=3000.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_9_chunk (actual rows=959.00 loops=1)
         ->  Seq Scan on compress_hyper_2_10_chunk (actual rows=1.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_11_chunk (actual rows=2041.00 loops=1)
         ->  Seq Scan on compress_hyper_2_12_chunk (actual rows=3.00 loops=1)

SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub;
            first             |             last             
------------------------------+------------------------------
 Wed Jan 01 00:01:00 2025 PST | Fri Jan 03 02:00:00 2025 PST

ROLLBACK;
-- simple test with compressed copy enabled and reversed order and no batch sorting
BEGIN;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 - I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_13_chunk (actual rows=3000.00 loops=1)
   ->  Seq Scan on compress_hyper_2_14_chunk (actual rows=3.00 loops=1)

SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub;
            first             |             last             
------------------------------+------------------------------
 Tue Dec 31 23:59:00 2024 PST | Sun Dec 29 22:00:00 2024 PST

ROLLBACK;
-- test compressing into uncompressed chunk
RESET timescaledb.enable_direct_compress_copy;
RESET timescaledb.enable_direct_compress_copy_sort_batches;
RESET timescaledb.enable_direct_compress_copy_client_sorted;
BEGIN;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 + I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 + I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Append (actual rows=6000.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_15_chunk (actual rows=959.00 loops=1)
         ->  Seq Scan on compress_hyper_2_17_chunk (actual rows=1.00 loops=1)
   ->  Seq Scan on _hyper_1_15_chunk (actual rows=959.00 loops=1)
   ->  Custom Scan (ColumnarScan) on _hyper_1_16_chunk (actual rows=2041.00 loops=1)
         ->  Seq Scan on compress_hyper_2_18_chunk (actual rows=3.00 loops=1)
   ->  Seq Scan on _hyper_1_16_chunk (actual rows=2041.00 loops=1)

SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------
      9

ROLLBACK;
-- simple test with compressed copy enabled and reversed order
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 - I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_19_chunk (actual rows=3000.00 loops=1)
   ->  Seq Scan on compress_hyper_2_20_chunk (actual rows=3.00 loops=1)

-- status should be 9
SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------
      1

ROLLBACK;
-- simple test with compressed copy enabled and no presorted
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-01 - I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_21_chunk (actual rows=3000.00 loops=1)
   ->  Seq Scan on compress_hyper_2_22_chunk (actual rows=3.00 loops=1)

-- status should be 11
SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------
      3

ROLLBACK;
-- test with segmentby
BEGIN;
ALTER TABLE metrics SET (tsdb.segmentby = 'device');
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics FROM PROGRAM 'seq 0 0.2 9.8 | sed -e ''s!.[0-9]$!!'' | xargs -II date -d "2025-01-01 - I minute" +"%Y-%m-%d %H:%M:%S,dI,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_23_chunk (actual rows=50.00 loops=1)
   ->  Seq Scan on compress_hyper_2_24_chunk (actual rows=10.00 loops=1)

SELECT format('%I.%I',schema_name,table_name) AS "COMPRESSED_CHUNK" FROM _timescaledb_catalog.chunk where compressed_chunk_id IS NULL \gset
-- should have 10 batches
SELECT count(*) FROM :COMPRESSED_CHUNK;
 count 
-------
    10

ROLLBACK;
-- test unique constraints prevent direct compress
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
ALTER TABLE metrics ADD CONSTRAINT unique_time_device UNIQUE (time, device);
COPY metrics FROM STDIN WITH (FORMAT CSV);
WARNING:  disabling direct compress because the destination table has unique constraints
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Seq Scan on _hyper_1_25_chunk (actual rows=1.00 loops=1)

SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------

ROLLBACK;
-- test triggers prevent direct compress
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql;
CREATE TRIGGER metrics_trigger BEFORE INSERT OR UPDATE ON metrics FOR EACH ROW EXECUTE FUNCTION test_trigger();
COPY metrics FROM STDIN WITH (FORMAT CSV);
WARNING:  disabling direct compress copy due to presence of triggers on the destination table
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Seq Scan on _hyper_1_26_chunk (actual rows=1.00 loops=1)

SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------

ROLLBACK;
-- test caggs with direct compress
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
CREATE MATERIALIZED VIEW metrics_cagg WITH (tsdb.continuous) AS SELECT time_bucket('1 hour', time) AS bucket, device, avg(value) AS avg_value FROM metrics GROUP BY bucket, device WITH NO DATA;
COPY metrics FROM STDIN WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_27_chunk (actual rows=1.00 loops=1)
   ->  Seq Scan on compress_hyper_2_28_chunk (actual rows=1.00 loops=1)

SELECT DISTINCT status FROM _timescaledb_catalog.chunk WHERE compressed_chunk_id IS NOT NULL;
 status 
--------
      3

ROLLBACK;
-- test chunk status handling
CREATE TABLE metrics_status(time timestamptz) WITH (tsdb.hypertable,tsdb.partition_column='time');
-- normal insert should result in chunk status 0
INSERT INTO metrics_status SELECT '2025-01-01';
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
 chunk_status_text 
-------------------
 {}

BEGIN;
-- compressed copy into uncompressed chunk should result in chunk status 11 (compressed,partial,unordered)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
       chunk_status_text        
--------------------------------
 {COMPRESSED,UNORDERED,PARTIAL}

ROLLBACK;
BEGIN;
-- compressed sorted copy into uncompressed chunk should result in chunk status 9 (compressed,partial)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
  chunk_status_text   
----------------------
 {COMPRESSED,PARTIAL}

ROLLBACK;
TRUNCATE metrics_status;
BEGIN;
-- compressed copy into new chunk should result in chunk status 3 (compressed,unordered)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
   chunk_status_text    
------------------------
 {COMPRESSED,UNORDERED}

ROLLBACK;
BEGIN;
-- compressed sorted copy into new chunk should result in chunk status 1 (compressed)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
 chunk_status_text 
-------------------
 {COMPRESSED}

ROLLBACK;
SET timescaledb.enable_direct_compress_copy = false;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
INSERT INTO metrics_status SELECT '2025-01-01';
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
 chunk_status_text 
-------------------
 {}

SELECT compress_chunk(show_chunks('metrics_status'));
             compress_chunk              
-----------------------------------------
 _timescaledb_internal._hyper_4_36_chunk

SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
 chunk_status_text 
-------------------
 {COMPRESSED}

BEGIN;
-- compressed copy into fully compressed chunk should result in chunk status 3 (compressed,unordered)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
   chunk_status_text    
------------------------
 {COMPRESSED,UNORDERED}

ROLLBACK;
BEGIN;
-- compressed copy new chunk should result in chunk status 1 (compressed)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY metrics_status FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('metrics_status') chunk;
 chunk_status_text 
-------------------
 {COMPRESSED}

ROLLBACK;
SELECT tableoid::regclass AS "CHUNK" FROM metrics_status WHERE time = '2025-01-01' LIMIT 1 \gset
-- repeat tests with direct reference to chunk
BEGIN;
-- compressed copy into fully compressed chunk should result in chunk status 3 (compressed,unordered)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = false;
COPY :CHUNK FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(:'CHUNK'::regclass);
   chunk_status_text    
------------------------
 {COMPRESSED,UNORDERED}

ROLLBACK;
BEGIN;
-- compressed copy new chunk should result in chunk status 1 (compressed)
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.enable_direct_compress_copy_client_sorted = true;
COPY :CHUNK FROM STDIN;
SELECT _timescaledb_functions.chunk_status_text(:'CHUNK'::regclass);
 chunk_status_text 
-------------------
 {COMPRESSED}

ROLLBACK;
-- simple test with compressed copy enabled and sorting limited to 500
-- batches should be limited to that amount so we have more compressed batches
-- dataset is tweaked to fall into a single chunk with exactly 3 batches
BEGIN;
SET timescaledb.enable_direct_compress_copy = true;
SET timescaledb.direct_compress_copy_tuple_sort_limit = 1000;
COPY metrics FROM PROGRAM 'seq 3000 | xargs -II date -d "2025-01-02 + I minute" +"%Y-%m-%d %H:%M:%S,d1,0.I"' WITH (FORMAT CSV);
EXPLAIN (ANALYZE, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM metrics;
--- QUERY PLAN ---
 Custom Scan (ColumnarScan) on _hyper_1_38_chunk (actual rows=3000.00 loops=1)
   ->  Seq Scan on compress_hyper_2_39_chunk (actual rows=3.00 loops=1)

SELECT first(time,rn), last(time,rn) FROM (SELECT ROW_NUMBER() OVER () as rn, time FROM metrics) sub;
            first             |             last             
------------------------------+------------------------------
 Thu Jan 02 00:01:00 2025 PST | Sat Jan 04 02:00:00 2025 PST

ROLLBACK;
