-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE ROLE NOLOGIN_ROLE WITH nologin noinherit;
-- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes
GRANT CREATE ON SCHEMA public TO NOLOGIN_ROLE;
GRANT NOLOGIN_ROLE TO :ROLE_DEFAULT_PERM_USER WITH ADMIN OPTION;
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
SET timezone TO 'America/Los_Angeles';
CREATE TABLE conditions (
      time        TIMESTAMPTZ       NOT NULL,
      location    TEXT              NOT NULL,
      location2    char(10)              NOT NULL,
      temperature DOUBLE PRECISION  NULL,
      humidity    DOUBLE PRECISION  NULL
    );
select create_hypertable( 'conditions', 'time', chunk_time_interval=> '31days'::interval);
    create_hypertable    
-------------------------
 (1,public,conditions,t)

--TEST 1--
--cannot set policy without enabling compression --
\set ON_ERROR_STOP 0
select add_compression_policy('conditions', '60d'::interval);
ERROR:  columnstore not enabled on hypertable "conditions"
\set ON_ERROR_STOP 1
-- TEST2 --
--add a policy to compress chunks --
alter table conditions set (timescaledb.compress, timescaledb.compress_segmentby = 'location', timescaledb.compress_orderby = 'time');
insert into conditions
select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 'klick', 55, 75;
select add_compression_policy('conditions', '60d'::interval) AS compressjob_id
\gset
select * from _timescaledb_catalog.bgw_job where id = :compressjob_id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                       config                        |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
 1000 | Columnstore Policy [1000] | @ 12 hours        | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |             1 | {"hypertable_id": 1, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check | 

select * from alter_job(:compressjob_id, schedule_interval=>'1s');
 job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled |                       config                        | next_start |                  check_config                   | fixed_schedule | initial_start | timezone |     application_name      
--------+-------------------+-------------+-------------+--------------+-----------+-----------------------------------------------------+------------+-------------------------------------------------+----------------+---------------+----------+---------------------------
   1000 | @ 1 sec           | @ 0         |          -1 | @ 1 hour     | t         | {"hypertable_id": 1, "compress_after": "@ 60 days"} | -infinity  | _timescaledb_functions.policy_compression_check | f              |               |          | Columnstore Policy [1000]

--enable maxchunks to 1 so that only 1 chunk is compressed by the job
SELECT alter_job(id,config:=jsonb_set(config,'{maxchunks_to_compress}', '1'))
 FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id;
                                                                                                         alter_job                                                                                                          
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1000,"@ 1 sec","@ 0",-1,"@ 1 hour",t,"{""hypertable_id"": 1, ""compress_after"": ""@ 60 days"", ""maxchunks_to_compress"": 1}",-infinity,_timescaledb_functions.policy_compression_check,f,,,"Columnstore Policy [1000]")

select * from _timescaledb_catalog.bgw_job where id >= 1000 ORDER BY id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                                     config                                      |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+---------------------------------------------------------------------------------+------------------------+--------------------------+----------
 1000 | Columnstore Policy [1000] | @ 1 sec           | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |             1 | {"hypertable_id": 1, "compress_after": "@ 60 days", "maxchunks_to_compress": 1} | _timescaledb_functions | policy_compression_check | 

insert into conditions
select now()::timestamp, 'TOK', 'sony', 55, 75;
-- TEST3 --
--only the old chunks will get compressed when policy is executed--
CALL run_job(:compressjob_id);
select chunk_name, pg_size_pretty(before_compression_total_bytes) before_total,
pg_size_pretty( after_compression_total_bytes)  after_total
from chunk_compression_stats('conditions') where compression_status like 'Compressed' order by chunk_name;
    chunk_name    | before_total | after_total 
------------------+--------------+-------------
 _hyper_1_1_chunk | 32 kB        | 40 kB

SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk ORDER BY id;
 id | hypertable_id |      schema_name      |        table_name        | compressed_chunk_id | dropped | status | osm_chunk 
----+---------------+-----------------------+--------------------------+---------------------+---------+--------+-----------
  1 |             1 | _timescaledb_internal | _hyper_1_1_chunk         |                   4 | f       |      1 | f
  2 |             1 | _timescaledb_internal | _hyper_1_2_chunk         |                     | f       |      0 | f
  3 |             1 | _timescaledb_internal | _hyper_1_3_chunk         |                     | f       |      0 | f
  4 |             2 | _timescaledb_internal | compress_hyper_2_4_chunk |                     | f       |      0 | f

-- TEST 4 --
--cannot set another policy
\set ON_ERROR_STOP 0
select add_compression_policy('conditions', '60d'::interval, if_not_exists=>true);
NOTICE:  columnstore policy already exists for hypertable "conditions", skipping
 add_compression_policy 
------------------------
                     -1

select add_compression_policy('conditions', '60d'::interval);
ERROR:  columnstore policy already exists for hypertable or continuous aggregate "conditions"
select add_compression_policy('conditions', '30d'::interval, if_not_exists=>true);
WARNING:  columnstore policy already exists for hypertable "conditions"
 add_compression_policy 
------------------------
                     -1

\set ON_ERROR_STOP 1
--TEST 5 --
-- drop the policy --
select remove_compression_policy('conditions');
 remove_compression_policy 
---------------------------
 t

select count(*) from _timescaledb_catalog.bgw_job WHERE id>=1000;
 count 
-------
     0

--TEST 6 --
-- try to execute the policy after it has been dropped --
\set ON_ERROR_STOP 0
CALL run_job(:compressjob_id);
ERROR:  job 1000 not found
--errors with bad input for add/remove compression policy
create view dummyv1 as select * from conditions limit 1;
select add_compression_policy( 100 , compress_after=> '1 day'::interval);
ERROR:  object with id "100" not found
select add_compression_policy( 'dummyv1', compress_after=> '1 day'::interval );
ERROR:  "dummyv1" is not a hypertable or a continuous aggregate
select remove_compression_policy( 100 );
ERROR:  relation is not a hypertable or continuous aggregate
\set ON_ERROR_STOP 1
-- We're done with the table, so drop it.
DROP TABLE IF EXISTS conditions CASCADE;
NOTICE:  drop cascades to view dummyv1
--TEST 7
--compression policy for smallint, integer or bigint based partition hypertable
--smallint test
CREATE TABLE test_table_smallint(time SMALLINT, val SMALLINT);
SELECT create_hypertable('test_table_smallint', 'time', chunk_time_interval => 1);
        create_hypertable         
----------------------------------
 (3,public,test_table_smallint,t)

CREATE OR REPLACE FUNCTION dummy_now_smallint() RETURNS SMALLINT LANGUAGE SQL IMMUTABLE AS 'SELECT 5::SMALLINT';
SELECT set_integer_now_func('test_table_smallint', 'dummy_now_smallint');
 set_integer_now_func 
----------------------
 

INSERT INTO test_table_smallint SELECT generate_series(1,5), 10;
ALTER TABLE test_table_smallint SET (timescaledb.compress);
\set ON_ERROR_STOP 0
select add_compression_policy( 'test_table_smallint', compress_after=> '1 day'::interval );
ERROR:  invalid value for parameter compress_after
\set ON_ERROR_STOP 1
SELECT add_compression_policy('test_table_smallint', 2::SMALLINT) AS compressjob_id \gset
SELECT * FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                  config                   |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+-------------------------------------------+------------------------+--------------------------+----------
 1001 | Columnstore Policy [1001] | @ 1 day           | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |             3 | {"hypertable_id": 3, "compress_after": 2} | _timescaledb_functions | policy_compression_check | 

--will compress all chunks that need compression
CALL run_job(:compressjob_id);
SELECT chunk_name, before_compression_total_bytes, after_compression_total_bytes
FROM chunk_compression_stats('test_table_smallint')
WHERE compression_status LIKE 'Compressed'
ORDER BY chunk_name;
    chunk_name    | before_compression_total_bytes | after_compression_total_bytes 
------------------+--------------------------------+-------------------------------
 _hyper_3_5_chunk |                          24576 |                         40960
 _hyper_3_6_chunk |                          24576 |                         40960

--integer tests
CREATE TABLE test_table_integer(time INTEGER, val INTEGER);
SELECT create_hypertable('test_table_integer', 'time', chunk_time_interval => 1);
        create_hypertable        
---------------------------------
 (5,public,test_table_integer,t)

CREATE OR REPLACE FUNCTION dummy_now_integer() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS 'SELECT 5::INTEGER';
SELECT set_integer_now_func('test_table_integer', 'dummy_now_integer');
 set_integer_now_func 
----------------------
 

INSERT INTO test_table_integer SELECT generate_series(1,5), 10;
ALTER TABLE test_table_integer SET (timescaledb.compress);
SELECT add_compression_policy('test_table_integer', 2::INTEGER) AS compressjob_id \gset
SELECT * FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                  config                   |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+-------------------------------------------+------------------------+--------------------------+----------
 1002 | Columnstore Policy [1002] | @ 1 day           | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |             5 | {"hypertable_id": 5, "compress_after": 2} | _timescaledb_functions | policy_compression_check | 

--will compress all chunks that need compression
CALL run_job(:compressjob_id);
SELECT chunk_name, before_compression_total_bytes, after_compression_total_bytes
FROM chunk_compression_stats('test_table_integer')
WHERE compression_status LIKE 'Compressed'
ORDER BY chunk_name;
    chunk_name     | before_compression_total_bytes | after_compression_total_bytes 
-------------------+--------------------------------+-------------------------------
 _hyper_5_12_chunk |                          24576 |                         40960
 _hyper_5_13_chunk |                          24576 |                         40960

--bigint test
CREATE TABLE test_table_bigint(time BIGINT, val BIGINT);
SELECT create_hypertable('test_table_bigint', 'time', chunk_time_interval => 1);
       create_hypertable        
--------------------------------
 (7,public,test_table_bigint,t)

CREATE OR REPLACE FUNCTION dummy_now_bigint() RETURNS BIGINT LANGUAGE SQL IMMUTABLE AS 'SELECT 5::BIGINT';
SELECT set_integer_now_func('test_table_bigint', 'dummy_now_bigint');
 set_integer_now_func 
----------------------
 

INSERT INTO test_table_bigint SELECT generate_series(1,5), 10;
ALTER TABLE test_table_bigint SET (timescaledb.compress);
SELECT add_compression_policy('test_table_bigint', 2::BIGINT) AS compressjob_id \gset
SELECT * FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                  config                   |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+-------------------------------------------+------------------------+--------------------------+----------
 1003 | Columnstore Policy [1003] | @ 1 day           | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |             7 | {"hypertable_id": 7, "compress_after": 2} | _timescaledb_functions | policy_compression_check | 

--will compress all chunks that need compression
CALL run_job(:compressjob_id);
SELECT chunk_name, before_compression_total_bytes, after_compression_total_bytes
FROM chunk_compression_stats('test_table_bigint')
WHERE compression_status LIKE 'Compressed'
ORDER BY chunk_name;
    chunk_name     | before_compression_total_bytes | after_compression_total_bytes 
-------------------+--------------------------------+-------------------------------
 _hyper_7_19_chunk |                          24576 |                         40960
 _hyper_7_20_chunk |                          24576 |                         40960

--TEST 8
--hypertable owner lacks permission to start background worker
SET ROLE NOLOGIN_ROLE;
CREATE TABLE test_table_nologin(time bigint, val int);
SELECT create_hypertable('test_table_nologin', 'time', chunk_time_interval => 1);
        create_hypertable        
---------------------------------
 (9,public,test_table_nologin,t)

SELECT set_integer_now_func('test_table_nologin', 'dummy_now_bigint');
 set_integer_now_func 
----------------------
 

ALTER TABLE test_table_nologin set (timescaledb.compress);
\set ON_ERROR_STOP 0
SELECT add_compression_policy('test_table_nologin', 2::int);
ERROR:  permission denied to start background process as role "nologin_role"
\set ON_ERROR_STOP 1
DROP TABLE test_table_nologin;
RESET ROLE;
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
SET timezone TO 'America/Los_Angeles';
CREATE TABLE conditions(
    time TIMESTAMPTZ NOT NULL,
    device INTEGER,
    temperature FLOAT
);
SELECT * FROM create_hypertable('conditions', 'time',
                                chunk_time_interval => '1 day'::interval);
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            11 | public      | conditions | t

INSERT INTO conditions
SELECT time, (random()*30)::int, random()*80 - 40
FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
CREATE MATERIALIZED VIEW conditions_summary
WITH (timescaledb.continuous) AS
SELECT device,
       time_bucket(INTERVAL '1 hour', "time") AS day,
       AVG(temperature) AS avg_temperature,
       MAX(temperature) AS max_temperature,
       MIN(temperature) AS min_temperature
FROM conditions
GROUP BY device, time_bucket(INTERVAL '1 hour', "time") WITH NO DATA;
CALL refresh_continuous_aggregate('conditions_summary', NULL, NULL);
ALTER TABLE conditions SET (timescaledb.compress);
SELECT COUNT(*) AS dropped_chunks_count
  FROM drop_chunks('conditions', TIMESTAMPTZ '2018-12-15 00:00');
 dropped_chunks_count 
----------------------
                   14

SELECT count(*) FROM timescaledb_information.chunks
WHERE hypertable_name = 'conditions' and is_compressed = true;
 count 
-------
     0

SELECT add_compression_policy AS job_id
  FROM add_compression_policy('conditions', INTERVAL '1 day') \gset
-- job compresses only 1 chunk at a time --
SELECT alter_job(id,config:=jsonb_set(config,'{maxchunks_to_compress}', '1'))
 FROM _timescaledb_catalog.bgw_job WHERE id = :job_id;
                                                                                                          alter_job                                                                                                           
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1004,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""hypertable_id"": 11, ""compress_after"": ""@ 1 day"", ""maxchunks_to_compress"": 1}",-infinity,_timescaledb_functions.policy_compression_check,f,,,"Columnstore Policy [1004]")

SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
 FROM _timescaledb_catalog.bgw_job WHERE id = :job_id;
                                                                                                                      alter_job                                                                                                                      
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1004,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""verbose_log"": true, ""hypertable_id"": 11, ""compress_after"": ""@ 1 day"", ""maxchunks_to_compress"": 1}",-infinity,_timescaledb_functions.policy_compression_check,f,,,"Columnstore Policy [1004]")

set client_min_messages TO LOG;
CALL run_job(:job_id);
LOG:  statement: CALL run_job(1004);
LOG:  job 1004 completed processing chunk _timescaledb_internal._hyper_11_40_chunk
set client_min_messages TO NOTICE;
LOG:  statement: set client_min_messages TO NOTICE;
SELECT count(*) FROM timescaledb_information.chunks
WHERE hypertable_name = 'conditions' and is_compressed = true;
 count 
-------
     1

\i include/recompress_basic.sql
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
CREATE OR REPLACE VIEW compressed_chunk_info_view AS
SELECT
   h.schema_name AS hypertable_schema,
   h.table_name AS hypertable_name,
   c.schema_name as chunk_schema,
   c.table_name as chunk_name,
   c.status as chunk_status,
   comp.schema_name as compressed_chunk_schema,
   comp.table_name as compressed_chunk_name
FROM
   _timescaledb_catalog.hypertable h JOIN
  _timescaledb_catalog.chunk c ON h.id = c.hypertable_id
   LEFT JOIN _timescaledb_catalog.chunk comp
ON comp.id = c.compressed_chunk_id
;
CREATE TABLE test2 (timec timestamptz NOT NULL, i integer ,
      b bigint, t text);
SELECT table_name from create_hypertable('test2', 'timec', chunk_time_interval=> INTERVAL '7 days');
 table_name 
------------
 test2

INSERT INTO test2 SELECT q, 10, 11, 'hello' FROM generate_series( '2020-01-03 10:00:00+00', '2020-01-03 12:00:00+00' , '5 min'::interval) q;
ALTER TABLE test2 set (timescaledb.compress,
timescaledb.compress_segmentby = 'b',
timescaledb.compress_orderby = 'timec DESC');
SELECT compress_chunk(c)
FROM show_chunks('test2') c;
              compress_chunk              
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

---insert into the middle of the range ---
INSERT INTO test2 values ( '2020-01-03 10:01:00+00', 20, 11, '2row');
INSERT INTO test2 values ( '2020-01-03 11:01:00+00', 20, 11, '3row');
INSERT INTO test2 values ( '2020-01-03 12:01:00+00', 20, 11, '4row');
--- insert a new segment  by ---
INSERT INTO test2 values ( '2020-01-03 11:01:00+00', 20, 12, '12row');
SELECT time_bucket(INTERVAL '2 hour', timec), b, count(*)
FROM test2
GROUP BY time_bucket(INTERVAL '2 hour', timec), b
ORDER BY 1, 2;
         time_bucket          | b  | count 
------------------------------+----+-------
 Fri Jan 03 02:00:00 2020 PST | 11 |    26
 Fri Jan 03 02:00:00 2020 PST | 12 |     1
 Fri Jan 03 04:00:00 2020 PST | 11 |     2

--check status for chunk --
SELECT chunk_status,
       chunk_name as "CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
 chunk_status |     CHUNK_NAME     
--------------+--------------------
            9 | _hyper_14_62_chunk

SELECT compressed_chunk_schema || '.' || compressed_chunk_name as "COMP_CHUNK_NAME",
        chunk_schema || '.' || chunk_name as "CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' \gset
SELECT count(*) from test2;
 count 
-------
    29

SELECT compress_chunk(:'CHUNK_NAME'::regclass);
              compress_chunk              
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

-- Demonstrate that no locks are held on the hypertable, chunk, or the
-- compressed chunk after recompress_chunk has executed.
SELECT pid, locktype, relation, relation::regclass, mode, granted
FROM pg_locks
WHERE relation::regclass::text IN (:'CHUNK_NAME', :'COMP_CHUNK_NAME', 'test2')
ORDER BY pid;
 pid | locktype | relation | relation | mode | granted 
-----+----------+----------+----------+------+---------

SELECT chunk_status,
       chunk_name as "CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
 chunk_status |     CHUNK_NAME     
--------------+--------------------
            1 | _hyper_14_62_chunk

--- insert into a compressed chunk again + a new chunk--
INSERT INTO test2 values ( '2020-01-03 11:01:03+00', 20, 11, '33row'),
                         ( '2020-01-03 11:01:06+00', 20, 11, '36row'),
                         ( '2020-01-03 11:02:00+00', 20, 12, '12row'),
                         ( '2020-04-03 00:02:00+00', 30, 13, '3013row');
SELECT time_bucket(INTERVAL '2 hour', timec), b, count(*)
FROM test2
GROUP BY time_bucket(INTERVAL '2 hour', timec), b
ORDER BY 1, 2;
         time_bucket          | b  | count 
------------------------------+----+-------
 Fri Jan 03 02:00:00 2020 PST | 11 |    28
 Fri Jan 03 02:00:00 2020 PST | 12 |     2
 Fri Jan 03 04:00:00 2020 PST | 11 |     2
 Thu Apr 02 17:00:00 2020 PDT | 13 |     1

--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
       chunk_name as "CHUNK_NAME",
       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
 chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
--------------+--------------------+----------------------------
            9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
            0 | _hyper_14_64_chunk | 

SELECT add_compression_policy AS job_id
  FROM add_compression_policy('test2', '30d'::interval) \gset
CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
       chunk_name as "CHUNK_NAME",
       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
 chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
--------------+--------------------+----------------------------
            1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
            1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk

\set ON_ERROR_STOP 0
-- call compress_chunk when status is not unordered
SELECT compress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:107: NOTICE:  chunk "_hyper_14_62_chunk" is already converted to columnstore
              compress_chunk              
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

-- This will succeed and compress the chunk for the test below.
SELECT compress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:110: NOTICE:  chunk "_hyper_14_62_chunk" is already converted to columnstore
              compress_chunk              
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

--now decompress it , then try and recompress
SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
             decompress_chunk             
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

SELECT compress_chunk(:'CHUNK_NAME'::regclass);
              compress_chunk              
------------------------------------------
 _timescaledb_internal._hyper_14_62_chunk

\set ON_ERROR_STOP 1
-- test recompress policy
CREATE TABLE metrics(time timestamptz NOT NULL);
SELECT hypertable_id AS "HYPERTABLE_ID", schema_name, table_name, created FROM create_hypertable('metrics','time') \gset
ALTER TABLE metrics SET (timescaledb.compress);
-- create chunk with some data and compress
INSERT INTO metrics SELECT '2000-01-01' FROM generate_series(1,10);
-- create custom compression job without recompress boolean
SELECT add_job('_timescaledb_functions.policy_compression','1w',('{"hypertable_id": '||:'HYPERTABLE_ID'||', "compress_after": "@ 7 days"}')::jsonb, initial_start => '2000-01-01 00:00:00+00'::timestamptz) AS "JOB_COMPRESS" \gset
-- first call should compress
CALL run_job(:JOB_COMPRESS);
-- 2nd call should do nothing
CALL run_job(:JOB_COMPRESS);
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

-- do an INSERT so recompress has something to do
INSERT INTO metrics SELECT '2000-01-01';
---- status should be 3
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            9

-- should recompress
CALL run_job(:JOB_COMPRESS);
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

-- disable recompress in compress job
SELECT alter_job(id,config:=jsonb_set(config,'{recompress}','false'), next_start => '2000-01-01 00:00:00+00'::timestamptz) FROM _timescaledb_catalog.bgw_job WHERE id = :JOB_COMPRESS;
                                                                                                         alter_job                                                                                                         
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1006,"@ 7 days","@ 0",-1,"@ 5 mins",t,"{""recompress"": false, ""hypertable_id"": 16, ""compress_after"": ""@ 7 days""}","Fri Dec 31 16:00:00 1999 PST",,t,"Fri Dec 31 16:00:00 1999 PST",,"User-Defined Action [1006]")

-- nothing to do
CALL run_job(:JOB_COMPRESS);
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

-- do an INSERT so recompress has something to do
INSERT INTO metrics SELECT '2000-01-01';
---- status should be 3
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            9

-- still nothing to do since we disabled recompress
CALL run_job(:JOB_COMPRESS);
---- status should be 3
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            9

-- reenable recompress in compress job
SELECT alter_job(id,config:=jsonb_set(config,'{recompress}','true'), next_start => '2000-01-01 00:00:00+00'::timestamptz) FROM _timescaledb_catalog.bgw_job WHERE id = :JOB_COMPRESS;
                                                                                                        alter_job                                                                                                         
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1006,"@ 7 days","@ 0",-1,"@ 5 mins",t,"{""recompress"": true, ""hypertable_id"": 16, ""compress_after"": ""@ 7 days""}","Fri Dec 31 16:00:00 1999 PST",,t,"Fri Dec 31 16:00:00 1999 PST",,"User-Defined Action [1006]")

-- should recompress now
CALL run_job(:JOB_COMPRESS);
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

SELECT delete_job(:JOB_COMPRESS);
 delete_job 
------------
 

SELECT add_job('_timescaledb_functions.policy_recompression','1w',('{"hypertable_id": '||:'HYPERTABLE_ID'||', "recompress_after": "@ 7 days", "maxchunks_to_compress": 1}')::jsonb) AS "JOB_RECOMPRESS" \gset
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

---- nothing to do yet
CALL run_job(:JOB_RECOMPRESS);
psql:include/recompress_basic.sql:186: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

-- create some work for recompress
INSERT INTO metrics SELECT '2000-01-01';
-- status should be 3
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            9

CALL run_job(:JOB_RECOMPRESS);
-- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
 chunk_status 
--------------
            1

SELECT delete_job(:JOB_RECOMPRESS);
 delete_job 
------------
 

--TEST 7
--compression policy should ignore frozen partially compressed chunks
CREATE TABLE test_table_frozen(time TIMESTAMPTZ, val SMALLINT);
SELECT create_hypertable('test_table_frozen', 'time', chunk_time_interval => '1 day'::interval);
        create_hypertable        
---------------------------------
 (18,public,test_table_frozen,t)

INSERT INTO test_table_frozen SELECT time, (random()*10)::smallint
FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
ALTER TABLE test_table_frozen SET (timescaledb.compress);
select add_compression_policy( 'test_table_frozen', compress_after=> '1 day'::interval ) as compressjob_id \gset
SELECT * FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id;
  id  |     application_name      | schedule_interval | max_runtime | max_retries | retry_period |      proc_schema       |     proc_name      |       owner       | scheduled | fixed_schedule | initial_start | hypertable_id |                       config                       |      check_schema      |        check_name        | timezone 
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+-------------------+-----------+----------------+---------------+---------------+----------------------------------------------------+------------------------+--------------------------+----------
 1008 | Columnstore Policy [1008] | @ 12 hours        | @ 0         |          -1 | @ 1 hour     | _timescaledb_functions | policy_compression | default_perm_user | t         | f              |               |            18 | {"hypertable_id": 18, "compress_after": "@ 1 day"} | _timescaledb_functions | policy_compression_check | 

SELECT show_chunks('test_table_frozen') as first_chunk LIMIT 1 \gset
--will compress all chunks that need compression
CALL run_job(:compressjob_id);
-- make the chunks partial
INSERT INTO test_table_frozen SELECT time, (random()*10)::smallint
FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
SELECT c.id, c.status
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_table_frozen'
ORDER BY c.id
LIMIT 1;
 id | status 
----+--------
 69 |      9

-- freeze first chunk
SELECT _timescaledb_functions.freeze_chunk(:'first_chunk');
 freeze_chunk 
--------------
 t

-- first chunk status is  1 (Compressed) + 8 (Partially compressed) + 4 (Frozen) = 13
SELECT c.id, c.status
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_table_frozen'
ORDER BY c.id
LIMIT 1;
 id | status 
----+--------
 69 |     13

--should recompress all chunks except first since its frozen
CALL run_job(:compressjob_id);
-- first chunk status is unchanged
SELECT c.id, c.status
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_table_frozen'
ORDER BY c.id
LIMIT 1;
 id | status 
----+--------
 69 |     13

-- unfreeze first chunk
SELECT _timescaledb_functions.unfreeze_chunk(:'first_chunk');
 unfreeze_chunk 
----------------
 t

-- should be able to recompress the chunk since its unfrozen
CALL run_job(:compressjob_id);
-- first chunk status is Compressed (1)
SELECT c.id, c.status
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_table_frozen'
ORDER BY c.id
LIMIT 1;
 id | status 
----+--------
 69 |      1

--TEST 8
--reindexing in recompression policy
CREATE TABLE metrics2(time DATE NOT NULL);
CREATE INDEX metrics2_index ON metrics2(time DESC);
SELECT hypertable_id AS "HYPERTABLE_ID", schema_name, table_name, created FROM create_hypertable('metrics2','time') \gset
ALTER TABLE metrics2 SET (timescaledb.compress);
INSERT INTO metrics2 SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval);
SELECT add_job('_timescaledb_functions.policy_compression','1w',('{"hypertable_id": '||:'HYPERTABLE_ID'||', "compress_after": "@ 7 days"}')::jsonb, initial_start => '2000-01-01 00:00:00+00'::timestamptz) AS "JOB_COMPRESS" \gset
-- first call should compress
CALL run_job(:JOB_COMPRESS);
-- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics2';
 chunk_status 
--------------
            1
            1
            1
            1
            1

-- disable reindex in compress job
SELECT alter_job(id,config:=jsonb_set(config,'{reindex}','false'), next_start => '2000-01-01 00:00:00+00'::timestamptz) FROM _timescaledb_catalog.bgw_job WHERE id = :JOB_COMPRESS;
                                                                                                       alter_job                                                                                                        
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1009,"@ 7 days","@ 0",-1,"@ 5 mins",t,"{""reindex"": false, ""hypertable_id"": 20, ""compress_after"": ""@ 7 days""}","Fri Dec 31 16:00:00 1999 PST",,t,"Fri Dec 31 16:00:00 1999 PST",,"User-Defined Action [1009]")

-- do an INSERT so recompress has something to do
INSERT INTO metrics2 SELECT '2000-01-01' FROM generate_series(1,3000);
SELECT chunk_schema, chunk_name FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics2' AND chunk_status = 9 LIMIT 1; \gset
     chunk_schema      |     chunk_name      
-----------------------+---------------------
 _timescaledb_internal | _hyper_20_131_chunk

SELECT format('%I.%I', :'chunk_schema', :'chunk_name') AS "RECOMPRESS_CHUNK_NAME"; \gset
           RECOMPRESS_CHUNK_NAME           
-------------------------------------------
 _timescaledb_internal._hyper_20_131_chunk

-- get size of the chunk that needs recompression
VACUUM ANALYZE :RECOMPRESS_CHUNK_NAME;
SELECT pg_indexes_size(:'RECOMPRESS_CHUNK_NAME') AS "SIZE_BEFORE_REINDEX"; \gset
 SIZE_BEFORE_REINDEX 
---------------------
               40960

CALL run_job(:JOB_COMPRESS);
-- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE chunk_schema = :'chunk_schema' AND chunk_name = :'chunk_name';
 chunk_status 
--------------
            1

-- index size should not have decreased
VACUUM ANALYZE :RECOMPRESS_CHUNK_NAME;
-- index size can vary, vacuuming can even increase the size of the index
-- just check that the index size hasn't decreased, this can only happen
-- when running VACUUM FULL or REINDEX TABLE
SELECT pg_indexes_size(:'RECOMPRESS_CHUNK_NAME') >= :SIZE_BEFORE_REINDEX as size_unchanged;
 size_unchanged 
----------------
 t

-- enable reindex in compress job
SELECT alter_job(id,config:=jsonb_set(config,'{reindex}','true'), next_start => '2000-01-01 00:00:00+00'::timestamptz) FROM _timescaledb_catalog.bgw_job WHERE id = :JOB_COMPRESS;
                                                                                                       alter_job                                                                                                       
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 (1009,"@ 7 days","@ 0",-1,"@ 5 mins",t,"{""reindex"": true, ""hypertable_id"": 20, ""compress_after"": ""@ 7 days""}","Fri Dec 31 16:00:00 1999 PST",,t,"Fri Dec 31 16:00:00 1999 PST",,"User-Defined Action [1009]")

-- do an INSERT so recompress has something to do
INSERT INTO metrics2 SELECT '2000-01-01';
---- status should be 3
SELECT chunk_status FROM compressed_chunk_info_view WHERE chunk_schema = :'chunk_schema' AND chunk_name = :'chunk_name';
 chunk_status 
--------------
            9

-- should recompress
CALL run_job(:JOB_COMPRESS);
-- index size should decrease due to reindexing (8kB or 16kB)
VACUUM ANALYZE metrics2;
SELECT pg_indexes_size(:'RECOMPRESS_CHUNK_NAME') <= 16384 as size_empty;
 size_empty 
------------
 t

DROP TABLE metrics2;
--TEST 8
--compression policy errors
CREATE TABLE test_compression_policy_errors(time TIMESTAMPTZ, val SMALLINT);
SELECT create_hypertable('test_compression_policy_errors', 'time', chunk_time_interval => '1 day'::interval);
              create_hypertable               
----------------------------------------------
 (22,public,test_compression_policy_errors,t)

ALTER TABLE test_compression_policy_errors SET (timescaledb.compress, timescaledb.compress_segmentby = 'val', timescaledb.compress_orderby = 'time');
INSERT INTO test_compression_policy_errors SELECT time, (random()*10)::smallint
FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
SELECT
  add_compression_policy(
    'test_compression_policy_errors',
    compress_after=> '1 day'::interval,
    initial_start => now() - interval '1 day'
  ) as compressjob_id \gset
SELECT config AS compressjob_config FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id \gset
SELECT FROM alter_job(:compressjob_id, config => jsonb_set(:'compressjob_config'::jsonb, '{recompress}', 'true'));
--

-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
SELECT c.status, count(*)
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_compression_policy_errors'
GROUP BY c.status
ORDER BY 2 DESC;
 status | count 
--------+-------
      0 |    31

\c :TEST_DBNAME :ROLE_SUPERUSER
-- Let's mess with the chunk status to for an error when executing the job
WITH chunks AS (
  SELECT c.id, c.status
  FROM _timescaledb_catalog.chunk c
  INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
  WHERE h.table_name = 'test_compression_policy_errors'
  ORDER BY c.id LIMIT 20
)
UPDATE _timescaledb_catalog.chunk
SET status = 3
FROM chunks
WHERE chunk.id = chunks.id
  AND chunk.status = 0;
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
-- After the mess 20 = status 3 and 11 = status 0
SELECT c.status, count(*)
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_compression_policy_errors'
GROUP BY c.status
ORDER BY 2 DESC;
 status | count 
--------+-------
      3 |    20
      0 |    11

\set ON_ERROR_STOP 0
SET client_min_messages TO ERROR;
\set VERBOSITY default
-- This should fail with
-- 20 chunks failed to compress and 11 chunks compressed successfully
CALL run_job(:compressjob_id);
ERROR:  columnstore policy failure
DETAIL:  Failed to convert '20' chunks to columnstore. Successfully converted '11' chunks.
CONTEXT:  PL/pgSQL function _timescaledb_functions.policy_compression_execute(integer,integer,anyelement,integer,boolean,boolean,boolean,boolean) line 118 at RAISE
SQL statement "CALL _timescaledb_functions.policy_compression_execute(job_id, htid, lag_value::INTERVAL, maxchunks, verbose_log, recompress_enabled, reindex_enabled, use_creation_time)"
PL/pgSQL function _timescaledb_functions.policy_compression(integer,jsonb) line 62 at CALL
\set VERBOSITY terse
\set ON_ERROR_STOP 1
-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
SELECT c.status, count(*)
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
WHERE h.table_name = 'test_compression_policy_errors'
GROUP BY c.status
ORDER BY 2 DESC;
 status | count 
--------+-------
      3 |    20
      1 |    11

-- Teardown test
\c :TEST_DBNAME :ROLE_SUPERUSER
REVOKE CREATE ON SCHEMA public FROM NOLOGIN_ROLE;
DROP ROLE NOLOGIN_ROLE;
