/*
 * This file and its contents are licensed under the Timescale License.
 * Please see the included NOTICE for copyright information and
 * LICENSE-TIMESCALE for a copy of the license.
 */
#include <postgres.h>

#include <access/htup.h>
#include <access/htup_details.h>
#include <access/xact.h>
#include <extension.h>
#include <fmgr.h>
#include <funcapi.h>
#include <hypertable_cache.h>
#include <miscadmin.h>
#include <nodes/makefuncs.h>
#include <nodes/memnodes.h>
#include <parser/parse_func.h>
#include <scan_iterator.h>
#include <scanner.h>
#include <storage/lockdefs.h>
#include <time_bucket.h>
#include <time_utils.h>
#include <utils.h>
#include <utils/builtins.h>
#include <utils/elog.h>
#include <utils/memutils.h>
#include <utils/palloc.h>
#include <utils/snapmgr.h>
#include <utils/tuplestore.h>

#include "cache.h"
#include "continuous_aggs/invalidation_threshold.h"
#include "continuous_aggs/materialize.h"
#include "guc.h"
#include "invalidation.h"
#include "refresh.h"
#include "ts_catalog/catalog.h"
#include "ts_catalog/continuous_agg.h"

/*
 * Invalidation processing for continuous aggregates.
 *
 * Invalidations track the regions/intervals [start, end] of a continuous
 * aggregate that are out-of-date relative to the source hypertable on top of
 * which the aggregate is defined. When a continuous aggregate is out-of-date
 * across one or more regions, it can be refreshed with a window covering
 * those regions in order to bring it up-to-date with the source data again.
 *
 * Invalidations are generated by mutations on the source hypertable (INSERT,
 * DELETE, UPDATE, TRUNCATE, drop_chunks, etc.) and are initially written to a
 * hypertable invalidation log [hypertable_id, start, end].
 *
 * When a continuous aggregate is refreshed, invalidations are moved from the
 * hypertable invalidation log to a continuous aggregate invalidation log,
 * where each original entry creates one new entry per continuous aggregate
 * [cagg_id, start, end]. Thus, if one continuous aggregate is refreshed but
 * not others, then only the invalidations for the refreshed aggregate are
 * processed.
 *
 * Simplified, invalidations move through the following stages:
 *
 * insert_trigger => hypertable_inval_log => cagg_inval_log => refreshing
 *
 * Thus, invalidations are generated by mutations and are processed and used
 * as input for refreshing the a continuous aggregate.
 *
 * Invalidations can overlap or be duplicates. Therefore, invalidations are
 * merged during processing to reduce the number of entries in the logs. This
 * typically happens during a refresh of a continuous aggregate, which also
 * cuts invalidations along the refresh window. The cutting will leave some
 * parts of entries in the invalidation log while the entries that fall
 * within the refresh window are stored in an invalidation store and used for
 * refreshing:
 *
 *       |-------------|    refresh window
 *
 *   |-----|  |---| |----|  invalidations
 *
 *             =>
 *
 *   |---|             |--| invalidations that remain in the log
 *
 *       |-|  |---| |--|    invalidations that are used for refreshing
 *
 * The invalidation store will spill to disk in case of many invalidations so
 * it won't blow up memory usage. If there are no invalidations in the store
 * after processing, then the continuous aggregate is up-to-date in the region
 * defined by the refresh window.
 */

/*
 * Processing state used while processing the materialization invalidation log
 * and refreshing the continuous aggregate.
 */
typedef struct ContinuousAggInvalidationState
{
	const ContinuousAgg *cagg;
	MemoryContext per_tuple_mctx;
	Relation cagg_log_rel;
	Relation cagg_queue_rel;
	Snapshot snapshot;
	Tuplestorestate *invalidations;
} ContinuousAggInvalidationState;

/*
 * Processing state used while moving invalidations from hypertable
 * invalidation log to materialization invalidation log.
 */
typedef struct HypertableInvalidationState
{
	int32 hypertable_id;
	Oid dimtype; /* Type of the underlying hypertable's bucketed attribute */
	const ContinuousAggInfo *all_caggs;
	MemoryContext per_tuple_mctx;
	Relation cagg_log_rel;
	Snapshot snapshot;
} HypertableInvalidationState;

typedef enum ContinuousAggTableType
{
	HYPER_INVALIDATION_LOG,
	CAGG_INVALIDATION_LOG,
	CAGG_MATERIALIZATION_RANGES,
} ContinuousAggTableType;

static Relation open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode);
static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id,
											  LOCKMODE lockmode);
static HeapTuple create_materialization_ranges_tup(TupleDesc tupdesc, int32 cagg_hyper_id,
												   int64 start, int64 end);
static void insert_new_cagg_materialization_ranges(const ContinuousAggInvalidationState *state,
												   const InternalTimeRange refresh_window,
												   int32 cagg_hyper_id);
static bool save_invalidation_for_refresh(const ContinuousAggInvalidationState *state,
										  const Invalidation *invalidation);
static void set_remainder_after_cut(Invalidation *remainder, int32 hyper_id,
									int64 lowest_modified_value, int64 greatest_modified_value);
static void invalidation_entry_reset(Invalidation *entry);
static void
invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti,
											   int32 hyper_id, Oid dimtype,
											   const ContinuousAggBucketFunction *bucket_function);
static void
invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype,
											  const ContinuousAggBucketFunction *bucket_function);
static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b);
static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry);
static void insert_new_cagg_invalidation(const HypertableInvalidationState *state,
										 const Invalidation *entry, int32 cagg_hyper_id);
static void move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state);
static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id,
													   LOCKMODE lockmode);
static Invalidation cut_cagg_invalidation(const ContinuousAggInvalidationState *state,
										  const InternalTimeRange *refresh_window,
										  const Invalidation *entry);
static Invalidation cut_cagg_invalidation_and_compute_remainder(
	const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window,
	const Invalidation *mergedentry, const Invalidation *current_remainder);
static void clear_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state,
												 const InternalTimeRange *refresh_window,
												 bool force);
static void cagg_invalidation_state_init(ContinuousAggInvalidationState *state,
										 const ContinuousAgg *cagg);
static void cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state);

static Relation
open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode)
{
	static const CatalogTable logmappings[] = {
		[HYPER_INVALIDATION_LOG] = CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
		[CAGG_INVALIDATION_LOG] = CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
		[CAGG_MATERIALIZATION_RANGES] = CONTINUOUS_AGGS_MATERIALIZATION_RANGES,
	};
	Catalog *catalog = ts_catalog_get();
	Oid relid = catalog_get_table_id(catalog, logmappings[type]);

	return table_open(relid, lockmode);
}

static void
hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id, LOCKMODE lockmode)
{
	*iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
										lockmode,
										CurrentMemoryContext);
	iterator->ctx.index = catalog_get_index(ts_catalog_get(),
											CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG,
											CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX);
	ts_scan_iterator_scan_key_init(
		iterator,
		Anum_continuous_aggs_hypertable_invalidation_log_idx_hypertable_id,
		BTEqualStrategyNumber,
		F_INT4EQ,
		Int32GetDatum(hyper_id));
}

HeapTuple
create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, int64 end)
{
	Datum values[Natts_continuous_aggs_materialization_invalidation_log] = { 0 };
	bool isnull[Natts_continuous_aggs_materialization_invalidation_log] = { false };

	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_materialization_id)] =
		Int32GetDatum(cagg_hyper_id);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_lowest_modified_value)] =
		Int64GetDatum(start);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value)] =
		Int64GetDatum(end);

	return heap_form_tuple(tupdesc, values, isnull);
}

/*
 * Add an entry to the continuous aggregate invalidation log.
 */
void
invalidation_cagg_log_add_entry(int32 cagg_hyper_id, int64 start, int64 end)
{
	Relation rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	CatalogSecurityContext sec_ctx;
	HeapTuple tuple;

	Assert(start <= end);
	tuple = create_invalidation_tup(RelationGetDescr(rel), cagg_hyper_id, start, end);
	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_only(rel, tuple);
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tuple);
	table_close(rel, NoLock);
}

void
invalidation_hyper_log_add_entry(int32 hyper_id, int64 start, int64 end)
{
	Relation rel = open_cagg_table(HYPER_INVALIDATION_LOG, RowExclusiveLock);
	CatalogSecurityContext sec_ctx;
	Datum values[Natts_continuous_aggs_hypertable_invalidation_log];
	bool nulls[Natts_continuous_aggs_hypertable_invalidation_log] = { false };

	Assert(start <= end);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_hypertable_id)] = Int32GetDatum(hyper_id);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_lowest_modified_value)] =
		Int64GetDatum(start);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_hypertable_invalidation_log_greatest_modified_value)] =
		Int64GetDatum(end);

	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_values(rel, RelationGetDescr(rel), values, nulls);
	ts_catalog_restore_user(&sec_ctx);
	table_close(rel, NoLock);
	elog(DEBUG1,
		 "hypertable log for hypertable %d added entry [" INT64_FORMAT ", " INT64_FORMAT "]",
		 hyper_id,
		 start,
		 end);
}

/*
 * Invalidate one or more continuous aggregates.
 *
 * Add an invalidation in the given range. The invalidation is added either to
 * the hypertable invalidation log or the continuous aggregate invalidation
 * log depending on the type of the given hypertable. If the hypertable is a
 * "raw" hypertable (i.e., one that has one or more continuous aggregates), the
 * entry is added to the hypertable invalidation log and will invalidate all
 * the associated continuous aggregates. If the hypertable is instead an
 * materialized hypertable, the entry is added to the cagg invalidation log
 * and only invalidates the continuous aggregate owning that materialized
 * hypertable.
 */
void
continuous_agg_invalidate_raw_ht(const Hypertable *raw_ht, int64 start, int64 end)
{
	Assert(raw_ht != NULL);

	invalidation_hyper_log_add_entry(raw_ht->fd.id, start, end);
}

void
continuous_agg_invalidate_mat_ht(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start,
								 int64 end)
{
	Assert((raw_ht != NULL) && (mat_ht != NULL));

	invalidation_cagg_log_add_entry(mat_ht->fd.id, start, end);
}

static HeapTuple
create_materialization_ranges_tup(TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, int64 end)
{
	Datum values[Natts_continuous_aggs_materialization_ranges] = { 0 };
	bool isnull[Natts_continuous_aggs_materialization_ranges] = { false };

	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_ranges_materialization_id)] =
		Int32GetDatum(cagg_hyper_id);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_ranges_lowest_modified_value)] = Int64GetDatum(start);
	values[AttrNumberGetAttrOffset(
		Anum_continuous_aggs_materialization_ranges_greatest_modified_value)] = Int64GetDatum(end);

	return heap_form_tuple(tupdesc, values, isnull);
}

static void
insert_new_cagg_materialization_ranges(const ContinuousAggInvalidationState *state,
									   const InternalTimeRange refresh_window, int32 cagg_hyper_id)
{
	CatalogSecurityContext sec_ctx;
	TupleDesc tupdesc = RelationGetDescr(state->cagg_queue_rel);
	HeapTuple tuple = create_materialization_ranges_tup(tupdesc,
														cagg_hyper_id,
														refresh_window.start,
														refresh_window.end);

	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_only(state->cagg_queue_rel, tuple);
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tuple);
}

typedef enum InvalidationResult
{
	INVAL_NOMATCH,
	INVAL_DELETE,
	INVAL_CUT,
} InvalidationResult;

static inline bool
IsValidInvalidation(const Invalidation *invalidation)
{
	Assert(invalidation->lowest_modified_value <= invalidation->greatest_modified_value);
	return invalidation->hyper_id != INVALID_HYPERTABLE_ID &&
		   invalidation->lowest_modified_value <= invalidation->greatest_modified_value;
}

static bool
save_invalidation_for_refresh(const ContinuousAggInvalidationState *state,
							  const Invalidation *invalidation)
{
	if (!IsValidInvalidation(invalidation))
		return false;

	int32 cagg_hyper_id = state->cagg->data.mat_hypertable_id;
	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
	HeapTuple refresh_tup = create_invalidation_tup(tupdesc,
													cagg_hyper_id,
													invalidation->lowest_modified_value,
													invalidation->greatest_modified_value);
	tuplestore_puttuple(state->invalidations, refresh_tup);
	heap_freetuple(refresh_tup);

	InternalTimeRange refresh_window = {
		.type = state->cagg->partition_type,
		.start = invalidation->lowest_modified_value,
		/* Invalidations are inclusive at the end, while refresh windows aren't, so add one to the
		   end of the invalidated region */
		.end = ts_time_saturating_add(invalidation->greatest_modified_value,
									  1,
									  state->cagg->partition_type),
	};

	InternalTimeRange bucketed_refresh_window =
		compute_circumscribed_bucketed_refresh_window(state->cagg,
													  &refresh_window,
													  state->cagg->bucket_function);

	insert_new_cagg_materialization_ranges(state, bucketed_refresh_window, cagg_hyper_id);

	return true;
}

static void
set_remainder_after_cut(Invalidation *remainder, int32 hyper_id, int64 lowest_modified_value,
						int64 greatest_modified_value)
{
	MemSet(remainder, 0, sizeof(*remainder));
	remainder->hyper_id = hyper_id;
	remainder->lowest_modified_value = lowest_modified_value;
	remainder->greatest_modified_value = greatest_modified_value;
}

/*
 * Try to cut an invalidation against the refresh window.
 *
 * If an invalidation entry overlaps with the refresh window, it needs
 * additional processing: it is either cut, deleted, or left unmodified.
 *
 * The part(s) of the invalidation that are outside the refresh window after
 * the cut will remain in the log. The part of the invalidation that fits
 * within the window is returned as the "remainder".
 *
 * Note that the refresh window is exclusive in the end while invalidations
 * are inclusive.
 */
static InvalidationResult
cut_invalidation_along_refresh_window(const ContinuousAggInvalidationState *state,
									  const Invalidation *invalidation,
									  const InternalTimeRange *refresh_window,
									  Invalidation *remainder)
{
	int32 cagg_hyper_id = state->cagg->data.mat_hypertable_id;
	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
	InvalidationResult result = INVAL_NOMATCH;
	HeapTuple lower = NULL;
	HeapTuple upper = NULL;

	Assert(remainder != NULL);

	/* Entry is completely enclosed by the refresh window */
	if (invalidation->lowest_modified_value >= refresh_window->start &&
		invalidation->greatest_modified_value < refresh_window->end)
	{
		/*
		 * Entry completely enclosed so can be deleted:
		 *
		 * [---------------)
		 *     [+++++]
		 */

		result = INVAL_DELETE;
		set_remainder_after_cut(remainder,
								cagg_hyper_id,
								invalidation->lowest_modified_value,
								invalidation->greatest_modified_value);
	}
	else
	{
		if (invalidation->lowest_modified_value < refresh_window->start &&
			invalidation->greatest_modified_value >= refresh_window->start)
		{
			/*
			 * Need to cut in right end:
			 *
			 *     [------)
			 * [++++++]
			 *
			 * [++]
			 */
			lower = create_invalidation_tup(tupdesc,
											cagg_hyper_id,
											invalidation->lowest_modified_value,
											refresh_window->start - 1);
			set_remainder_after_cut(remainder,
									cagg_hyper_id,
									refresh_window->start,
									/* Refresh window not exclusive at end */
									MIN(refresh_window->end - 1,
										invalidation->greatest_modified_value));
			result = INVAL_CUT;
		}

		if (invalidation->lowest_modified_value < refresh_window->end &&
			invalidation->greatest_modified_value >= refresh_window->end)
		{
			/*
			 * If the invalidation is already cut on the left above, the reminder is set and
			 * will be reset here. The assert prevents from losing information from the reminder.
			 */
			Assert((result == INVAL_CUT &&
					remainder->lowest_modified_value == refresh_window->start) ||
				   result == INVAL_NOMATCH);

			/*
			 * Need to cut in left end:
			 *
			 * [------)
			 *    [++++++++]
			 *
			 *        [++++]
			 */
			upper = create_invalidation_tup(tupdesc,
											cagg_hyper_id,
											refresh_window->end,
											invalidation->greatest_modified_value);
			set_remainder_after_cut(remainder,
									cagg_hyper_id,
									MAX(invalidation->lowest_modified_value, refresh_window->start),
									/* Refresh window exclusive at end */
									refresh_window->end - 1);
			result = INVAL_CUT;
		}
	}

	/* Insert any modifications into the cagg invalidation log */
	if (result == INVAL_CUT)
	{
		CatalogSecurityContext sec_ctx;
		HeapTuple other_range = NULL;

		ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);

		/* We'd like to do one update (unless the TID is not set), and
		 * optionally one insert. We pick one of the tuples for an update, and
		 * the other one will be an insert. */
		if (lower || upper)
		{
			HeapTuple tup = lower ? lower : upper;
			other_range = lower ? upper : lower;

			/* If the TID is set, we are updating an existing tuple, i.e., we
			 * are processing and entry in the cagg log itself. Otherwise, we
			 * are processing the hypertable invalidation log and need to
			 * insert a new entry. */
			if (ItemPointerIsValid(&invalidation->tid))
			{
				ItemPointerData tid = invalidation->tid;
				ts_catalog_update_tid_only(state->cagg_log_rel, &tid, tup);
			}
			else
				ts_catalog_insert_only(state->cagg_log_rel, tup);

			heap_freetuple(tup);
		}

		if (other_range)
		{
			ts_catalog_insert_only(state->cagg_log_rel, other_range);
			heap_freetuple(other_range);
		}

		ts_catalog_restore_user(&sec_ctx);
	}

	return result;
}

static void
invalidation_entry_reset(Invalidation *entry)
{
	MemSet(entry, 0, sizeof(Invalidation));
}

/*
 * Expand an invalidation to bucket boundaries.
 *
 * Since a refresh always materializes full buckets, we can safely expand an
 * invalidation to bucket boundaries and in the process merge a lot more
 * invalidations.
 */
void
invalidation_expand_to_bucket_boundaries(Invalidation *inv, Oid time_type_oid,
										 const ContinuousAggBucketFunction *bucket_function)
{
	const int64 time_dimension_min = ts_time_get_min(time_type_oid);
	const int64 time_dimension_max = ts_time_get_max(time_type_oid);
	int64 min_bucket_start;
	int64 max_bucket_end;

	if (bucket_function->bucket_fixed_interval == false)
	{
		ts_compute_circumscribed_bucketed_refresh_window_variable(&inv->lowest_modified_value,
																  &inv->greatest_modified_value,
																  bucket_function);
		/* ts_compute_circumscribed_bucketed_refresh_window_variable returns the start of the
		 * next bucket as the end (exclusive). Since invalidations are inclusive at both ends,
		 * subtract 1 to get the last value of the current bucket (inclusive).
		 * Don't adjust infinity values. */
		if (inv->greatest_modified_value != INVAL_POS_INFINITY &&
			inv->greatest_modified_value != INVAL_NEG_INFINITY)
		{
			inv->greatest_modified_value = int64_saturating_sub(inv->greatest_modified_value, 1);
		}
		return;
	}

	int64 bucket_width = ts_continuous_agg_fixed_bucket_width(bucket_function);
	Assert(bucket_width > 0);

	/* Compute the start of the "first" bucket for the type. The min value
	 * must be at the start of the "first" bucket or somewhere in the
	 * bucket. If the min value falls on the exact start of the bucket we are
	 * good. Otherwise, we need to move to the next full bucket. */
	min_bucket_start = ts_time_saturating_add(time_dimension_min, bucket_width - 1, time_type_oid);
	min_bucket_start = ts_time_bucket_by_type(bucket_width, min_bucket_start, time_type_oid);

	/* Compute the end of the "last" bucket for the time type. Remember that
	 * invalidations are inclusive, so the "greatest" value should be the last
	 * value of the last full bucket. Either the max value is already the last
	 * value of the last bucket, or we need to return the last value of the
	 * previous full bucket.  */
	max_bucket_end = ts_time_bucket_by_type(bucket_width, time_dimension_max, time_type_oid);

	/* Check if the max value was already the last value of the last bucket */
	if (ts_time_saturating_add(max_bucket_end, bucket_width - 1, time_type_oid) ==
		time_dimension_max)
	{
		max_bucket_end = time_dimension_max;
	}
	else
	{
		/* The last bucket was partial. To get the end of previous bucket, we
		 * need to move one step down from the partial last bucket. */
		max_bucket_end = ts_time_saturating_sub(max_bucket_end, 1, time_type_oid);
	}

	if (inv->lowest_modified_value < min_bucket_start)
		/* Below the min bucket, so treat as invalid to -infinity. */
		inv->lowest_modified_value = INVAL_NEG_INFINITY;
	else if (inv->lowest_modified_value > max_bucket_end)
		/* Above the max bucket, so treat as invalid to +infinity. */
		inv->lowest_modified_value = INVAL_POS_INFINITY;
	else
		inv->lowest_modified_value =
			ts_time_bucket_by_type(bucket_width, inv->lowest_modified_value, time_type_oid);

	if (inv->greatest_modified_value < min_bucket_start)
		/* Below the min bucket, so treat as invalid to -infinity. */
		inv->greatest_modified_value = INVAL_NEG_INFINITY;
	else if (inv->greatest_modified_value > max_bucket_end)
		/* Above the max bucket, so treat as invalid to +infinity. */
		inv->greatest_modified_value = INVAL_POS_INFINITY;
	else
	{
		inv->greatest_modified_value =
			ts_time_bucket_by_type(bucket_width, inv->greatest_modified_value, time_type_oid);
		inv->greatest_modified_value =
			ts_time_saturating_add(inv->greatest_modified_value, bucket_width - 1, time_type_oid);
	}
}

/*
 * Macro to set an Invalidation from a tuple. The tuple can either have the
 * format of the hypertable invalidation log or the continuous aggregate
 * invalidation log (as determined by the type parameter).
 */
#define INVALIDATION_ENTRY_SET(entry, ti, hypertable_id, type)                                     \
	do                                                                                             \
	{                                                                                              \
		bool should_free;                                                                          \
		HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free);                    \
		type form;                                                                                 \
		form = (type) GETSTRUCT(tuple);                                                            \
		(entry)->hyper_id = form->hypertable_id;                                                   \
		(entry)->lowest_modified_value = form->lowest_modified_value;                              \
		(entry)->greatest_modified_value = form->greatest_modified_value;                          \
		(entry)->is_modified = false;                                                              \
		ItemPointerCopy(&tuple->t_self, &(entry)->tid);                                            \
                                                                                                   \
		if (should_free)                                                                           \
			heap_freetuple(tuple);                                                                 \
	} while (0);

static void
invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti,
											   int32 hyper_id, Oid dimtype,
											   const ContinuousAggBucketFunction *bucket_function)
{
	INVALIDATION_ENTRY_SET(entry,
						   ti,
						   hypertable_id,
						   Form_continuous_aggs_hypertable_invalidation_log);
	/* Since hypertable invalidations are moved to the continuous aggregate
	 * invalidation log, a different hypertable ID must be set (the ID of the
	 * materialized hypertable). */
	entry->hyper_id = hyper_id;
	invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function);
}

static void
invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype,
											  const ContinuousAggBucketFunction *bucket_function)
{
	INVALIDATION_ENTRY_SET(entry,
						   ti,
						   materialization_id,
						   Form_continuous_aggs_materialization_invalidation_log);

	/* It isn't strictly necessary to expand the invalidation to bucket
	 * boundaries here since all invalidations were already expanded when
	 * copied from the hypertable invalidation log. However, since
	 * invalidation expansion wasn't implemented in early 2.0.x versions of
	 * the extension, there might be unexpanded entries in the cagg
	 * invalidation log for some users. Therefore we try to expand
	 * invalidation entries also here, although in most cases it would do
	 * nothing. */
	invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function);
}

/*
 * Check if two invalidations can be merged into one.
 *
 * Since invalidations are inclusive in both ends, two adjacent invalidations
 * can be merged.
 */
static bool
invalidations_can_be_merged(const Invalidation *a, const Invalidation *b)
{
	/* To account for adjacency, expand one window 1 step in each
	 * direction. This makes adjacent invalidations overlapping. */
	int64 a_start = int64_saturating_sub(a->lowest_modified_value, 1);
	int64 a_end = int64_saturating_add(a->greatest_modified_value, 1);

	return a_end >= b->lowest_modified_value && a_start <= b->greatest_modified_value;
}

/*
 * Try to merge two invalidations into one.
 *
 * Returns true if the invalidations were merged, otherwise false.
 *
 * Given that we scan ordered on lowest_modified_value, the previous and
 * current invalidation can overlap in two ways (generalized):
 *
 * |------|
 *    |++++++++|
 *
 * |-------------|
 *    |++++++++|
 *
 * The closest non-overlapping case is (note that adjacent invalidations can
 * be merged since they are inclusive in both ends):
 *
 * |--|
 *     |++++++++|
 *
 */
static bool
invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
{
	if (!IsValidInvalidation(newentry))
		return false;

	/* Quick exit if no overlap */
	if (!invalidations_can_be_merged(entry, newentry))
		return false;

	/* Check if the new entry expands beyond the old one (first case above) */
	if (entry->greatest_modified_value < newentry->greatest_modified_value)
	{
		entry->greatest_modified_value = newentry->greatest_modified_value;
		entry->is_modified = true;
	}

	return true;
}

static void
insert_new_cagg_invalidation(const HypertableInvalidationState *state, const Invalidation *entry,
							 int32 cagg_hyper_id)
{
	CatalogSecurityContext sec_ctx;
	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
	HeapTuple tuple = create_invalidation_tup(tupdesc,
											  cagg_hyper_id,
											  entry->lowest_modified_value,
											  entry->greatest_modified_value);

	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
	ts_catalog_insert_only(state->cagg_log_rel, tuple);
	ts_catalog_restore_user(&sec_ctx);
	heap_freetuple(tuple);
}

/*
 * Process invalidations in the hypertable invalidation log.
 *
 * Copy and delete all entries from the hypertable invalidation log.
 *
 * Note that each entry gets one copy per continuous aggregate in the cagg
 * invalidation log (unless it was merged or matched the refresh
 * window). These copied entries are later used to track invalidations across
 * refreshes on a per-cagg basis.
 *
 * After this function has run, there are no entries left in the hypertable
 * invalidation log.
 */
static void
move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state)
{
	const ContinuousAggInfo *all_caggs = state->all_caggs;
	int32 hyper_id = state->hypertable_id;
	int32 last_cagg_hyper_id;
	ListCell *lc1, *lc2;

	last_cagg_hyper_id = llast_int(all_caggs->mat_hypertable_ids);

	/* We use a per-tuple memory context in the scan loop since we could be
	 * processing a lot of invalidations (basically an unbounded
	 * amount). Initialize it here by resetting it. */
	MemoryContextReset(state->per_tuple_mctx);

	/*
	 * Looping over all continuous aggregates in the outer loop ensures all
	 * tuples for a specific continuous aggregate is inserted consecutively in
	 * the cagg invalidation log. This creates better locality for scanning
	 * the invalidations later.
	 */
	forboth (lc1, all_caggs->mat_hypertable_ids, lc2, all_caggs->bucket_functions)
	{
		int32 cagg_hyper_id = lfirst_int(lc1);
		const ContinuousAggBucketFunction *bucket_function = lfirst(lc2);

		Invalidation mergedentry;
		ScanIterator iterator;

		invalidation_entry_reset(&mergedentry);
		hypertable_invalidation_scan_init(&iterator, hyper_id, RowExclusiveLock);
		iterator.ctx.snapshot = state->snapshot;

		/* Scan all invalidations */
		ts_scanner_foreach(&iterator)
		{
			TupleInfo *ti;
			MemoryContext oldmctx;
			Invalidation logentry;

			oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx);
			ti = ts_scan_iterator_tuple_info(&iterator);

			invalidation_entry_set_from_hyper_invalidation(&logentry,
														   ti,
														   cagg_hyper_id,
														   state->dimtype,
														   bucket_function);

			if (!IsValidInvalidation(&mergedentry))
			{
				mergedentry = logentry;
				mergedentry.hyper_id = cagg_hyper_id;
			}
			else if (!invalidation_entry_try_merge(&mergedentry, &logentry))
			{
				insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id);
				mergedentry = logentry;
			}

			if (cagg_hyper_id == last_cagg_hyper_id)
			{
				CatalogSecurityContext sec_ctx;

				/* The invalidation has been processed for all caggs, so the
				 * only thing left is to delete it from the source hypertable
				 * invalidation log. */
				ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
				ts_catalog_delete_tid_only(ti->scanrel, &logentry.tid);
				ts_catalog_restore_user(&sec_ctx);
			}

			MemoryContextSwitchTo(oldmctx);
			MemoryContextReset(state->per_tuple_mctx);
		}

		ts_scan_iterator_close(&iterator);

		/* Handle the last merged invalidation */
		if (IsValidInvalidation(&mergedentry))
			insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id);
	}
}

static void
cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id,
										   LOCKMODE lockmode)
{
	*iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
										lockmode,
										CurrentMemoryContext);
	iterator->ctx.index = catalog_get_index(ts_catalog_get(),
											CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
											CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX);
	ts_scan_iterator_scan_key_init(
		iterator,
		Anum_continuous_aggs_materialization_invalidation_log_idx_materialization_id,
		BTEqualStrategyNumber,
		F_INT4EQ,
		Int32GetDatum(cagg_hyper_id));
}

/*
 * Cut an invalidation and return the part, if any, that remains within the
 * refresh window.
 */
static Invalidation
cut_cagg_invalidation(const ContinuousAggInvalidationState *state,
					  const InternalTimeRange *refresh_window, const Invalidation *entry)
{
	InvalidationResult result;
	Invalidation remainder;
	ItemPointerData tid = entry->tid;

	invalidation_entry_reset(&remainder);

	result = cut_invalidation_along_refresh_window(state, entry, refresh_window, &remainder);

	switch (result)
	{
		case INVAL_NOMATCH:
			/* If no cutting was done (i.e., the invalidation was outside the
			 * refresh window), but the invalidation was previously merged
			 * (expanded) with another invalidation, then we still need to
			 * update it. */
			if (entry->is_modified)
			{
				HeapTuple tuple = create_invalidation_tup(RelationGetDescr(state->cagg_log_rel),
														  entry->hyper_id,
														  entry->lowest_modified_value,
														  entry->greatest_modified_value);
				ts_catalog_update_tid_only(state->cagg_log_rel, &tid, tuple);
				heap_freetuple(tuple);
			}
			break;
		case INVAL_DELETE:
			ts_catalog_delete_tid_only(state->cagg_log_rel, &tid);
			break;
		case INVAL_CUT:
			/* Nothing to do */
			break;
	}

	return remainder;
}

static Invalidation
cut_cagg_invalidation_and_compute_remainder(const ContinuousAggInvalidationState *state,
											const InternalTimeRange *refresh_window,
											const Invalidation *mergedentry,
											const Invalidation *current_remainder)
{
	Invalidation new_remainder;
	Invalidation remainder = *current_remainder;

	/* The previous and current invalidation could not be merged. We
	 * need to cut the prev invalidation against the refresh window */
	new_remainder = cut_cagg_invalidation(state, refresh_window, mergedentry);

	if (!IsValidInvalidation(&remainder))
		remainder = new_remainder;
	else if (!invalidation_entry_try_merge(&remainder, &new_remainder))
	{
		save_invalidation_for_refresh(state, &remainder);
		remainder = new_remainder;
	}

	return remainder;
}

/*
 * Clear all cagg invalidations that match a refresh window.
 *
 * This function clears all invalidations in the cagg invalidation log that
 * matches a window, and adds the invalidation segments covered by the window
 * to the invalidation store (tuple store) in the state argument. The
 * remaining segments that are added to the invalidation store are regions
 * that require materialization.
 *
 * An invalidation entry that gets processed is either completely enclosed
 * (covered) by the refresh window, or it partially overlaps. In the former
 * case, the invalidation entry is removed and for the latter case it is
 * cut. Thus, an entry can either disappear, reduce in size, or be cut in two.
 *
 * Note that the refresh window is inclusive at the start and exclusive at the
 * end. This function also assumes that invalidations are scanned in order of
 * lowest_modified_value.
 */
static void
clear_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state,
									 const InternalTimeRange *refresh_window, bool force)
{
	ScanIterator iterator;
	Invalidation mergedentry;
	Invalidation remainder;

	invalidation_entry_reset(&mergedentry);
	invalidation_entry_reset(&remainder);
	cagg_invalidations_scan_by_hypertable_init(&iterator,
											   state->cagg->data.mat_hypertable_id,
											   RowExclusiveLock);
	iterator.ctx.data = (void *) &state;
	iterator.ctx.snapshot = state->snapshot;
	ScanTupLock scantuplock = {
		.waitpolicy = LockWaitBlock,
		.lockmode = LockTupleExclusive,
		.lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION,
	};
	iterator.ctx.tuplock = &scantuplock;
	iterator.ctx.flags = SCANNER_F_KEEPLOCK;

	MemoryContextReset(state->per_tuple_mctx);

	/* Force refresh within the entire window */
	if (force)
	{
		mergedentry.hyper_id = state->cagg->data.mat_hypertable_id;
		mergedentry.lowest_modified_value = refresh_window->start;
		mergedentry.greatest_modified_value = refresh_window->end;
		mergedentry.is_modified = false;
		ItemPointerSet(&mergedentry.tid, InvalidBlockNumber, 0);

		/* Jump to process remainder to properly cut the invalidation */
		goto process_remainder;
	}

	/* Process all invalidations for the continuous aggregate */
	ts_scanner_foreach(&iterator)
	{
		TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator);
		MemoryContext oldmctx;
		Invalidation logentry;

		oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx);
		invalidation_entry_set_from_cagg_invalidation(&logentry,
													  ti,
													  state->cagg->partition_type,
													  state->cagg->bucket_function);

		if (!IsValidInvalidation(&mergedentry))
			mergedentry = logentry;
		else if (invalidation_entry_try_merge(&mergedentry, &logentry))
		{
			/*
			 * The previous and current invalidation were merged into
			 * one entry (i.e., they overlapped or were adjacent).
			 */
			ts_catalog_delete_tid_only(state->cagg_log_rel, &logentry.tid);
		}
		else
		{
			remainder = cut_cagg_invalidation_and_compute_remainder(state,
																	refresh_window,
																	&mergedentry,
																	&remainder);
			mergedentry = logentry;
		}

		MemoryContextSwitchTo(oldmctx);
		MemoryContextReset(state->per_tuple_mctx);
	}

	ts_scan_iterator_close(&iterator);

process_remainder:
	/* Handle the last (merged) invalidation */
	if (IsValidInvalidation(&mergedentry))
		remainder = cut_cagg_invalidation_and_compute_remainder(state,
																refresh_window,
																&mergedentry,
																&remainder);

	/* Handle the last (merged) remainder */
	save_invalidation_for_refresh(state, &remainder);
}

static void
cagg_invalidation_state_init(ContinuousAggInvalidationState *state, const ContinuousAgg *cagg)
{
	state->cagg = cagg;
	state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	state->cagg_queue_rel = open_cagg_table(CAGG_MATERIALIZATION_RANGES, RowExclusiveLock);
	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
												  "Materialization invalidations",
												  ALLOCSET_DEFAULT_SIZES);
	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
}

static void
cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state)
{
	table_close(state->cagg_log_rel, NoLock);
	table_close(state->cagg_queue_rel, NoLock);
	UnregisterSnapshot(state->snapshot);
	MemoryContextDelete(state->per_tuple_mctx);
}

static void
hypertable_invalidation_state_init(HypertableInvalidationState *state, int32 hypertable_id,
								   Oid dimtype, const ContinuousAggInfo *all_caggs)
{
	state->hypertable_id = hypertable_id;
	state->dimtype = dimtype;
	state->all_caggs = all_caggs;
	state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock);
	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
												  "Hypertable invalidations",
												  ALLOCSET_DEFAULT_SIZES);
	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
}

static void
hypertable_invalidation_state_cleanup(const HypertableInvalidationState *state)
{
	table_close(state->cagg_log_rel, NoLock);
	UnregisterSnapshot(state->snapshot);
	MemoryContextDelete(state->per_tuple_mctx);
}

/*
 * Move invalidations for a single hypertable from hypertable invalidation log
 * to materialization invalidation log. This will move *all* hypertable
 * invalidations for the hypertable to the associated continuous aggregates.
 */
void
invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype)
{
	HypertableInvalidationState state;
	const ContinuousAggInfo all_caggs = ts_continuous_agg_get_all_caggs_info(hypertable_id);

	hypertable_invalidation_state_init(&state, hypertable_id, dimtype, &all_caggs);
	move_invalidations_from_hyper_to_cagg_log(&state);
	hypertable_invalidation_state_cleanup(&state);
}

InvalidationStore *
invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window,
							  long max_materializations, ContinuousAggRefreshContext context,
							  bool force)
{
	ContinuousAggInvalidationState state;
	InvalidationStore *store = NULL;
	long count;

	cagg_invalidation_state_init(&state, cagg);
	state.invalidations = tuplestore_begin_heap(false, false, work_mem);
	clear_cagg_invalidations_for_refresh(&state, refresh_window, force);
	count = tuplestore_tuple_count(state.invalidations);

	if (count == 0)
	{
		tuplestore_end(state.invalidations);
	}
	else
	{
		store = palloc(sizeof(InvalidationStore));
		store->tupstore = state.invalidations;
		store->tupdesc = CreateTupleDescCopy(RelationGetDescr(state.cagg_log_rel));
	}

	cagg_invalidation_state_cleanup(&state);

	return store;
}

void
invalidation_store_free(InvalidationStore *store)
{
	FreeTupleDesc(store->tupdesc);
	tuplestore_end(store->tupstore);
	pfree(store);
}
