Adding upstream version 16.2.upstream/16.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 602 insertions, 0 deletions

diff --git a/src/include/jit/jit.h b/src/include/jit/jit.h
new file mode 100644
index 0000000..14f2e36
--- /dev/null
+++ b/src/include/jit/jit.h
@@ -0,0 +1,105 @@
+/*-------------------------------------------------------------------------
+ * jit.h
+ *	  Provider independent JIT infrastructure.
+ *
+ * Copyright (c) 2016-2023, PostgreSQL Global Development Group
+ *
+ * src/include/jit/jit.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef JIT_H
+#define JIT_H
+
+#include "executor/instrument.h"
+#include "utils/resowner.h"
+
+
+/* Flags determining what kind of JIT operations to perform */
+#define PGJIT_NONE     0
+#define PGJIT_PERFORM  (1 << 0)
+#define PGJIT_OPT3     (1 << 1)
+#define PGJIT_INLINE   (1 << 2)
+#define PGJIT_EXPR	   (1 << 3)
+#define PGJIT_DEFORM   (1 << 4)
+
+
+typedef struct JitInstrumentation
+{
+	/* number of emitted functions */
+	size_t		created_functions;
+
+	/* accumulated time to generate code */
+	instr_time	generation_counter;
+
+	/* accumulated time for inlining */
+	instr_time	inlining_counter;
+
+	/* accumulated time for optimization */
+	instr_time	optimization_counter;
+
+	/* accumulated time for code emission */
+	instr_time	emission_counter;
+} JitInstrumentation;
+
+/*
+ * DSM structure for accumulating jit instrumentation of all workers.
+ */
+typedef struct SharedJitInstrumentation
+{
+	int			num_workers;
+	JitInstrumentation jit_instr[FLEXIBLE_ARRAY_MEMBER];
+} SharedJitInstrumentation;
+
+typedef struct JitContext
+{
+	/* see PGJIT_* above */
+	int			flags;
+
+	ResourceOwner resowner;
+
+	JitInstrumentation instr;
+} JitContext;
+
+typedef struct JitProviderCallbacks JitProviderCallbacks;
+
+extern PGDLLEXPORT void _PG_jit_provider_init(JitProviderCallbacks *cb);
+typedef void (*JitProviderInit) (JitProviderCallbacks *cb);
+typedef void (*JitProviderResetAfterErrorCB) (void);
+typedef void (*JitProviderReleaseContextCB) (JitContext *context);
+struct ExprState;
+typedef bool (*JitProviderCompileExprCB) (struct ExprState *state);
+
+struct JitProviderCallbacks
+{
+	JitProviderResetAfterErrorCB reset_after_error;
+	JitProviderReleaseContextCB release_context;
+	JitProviderCompileExprCB compile_expr;
+};
+
+
+/* GUCs */
+extern PGDLLIMPORT bool jit_enabled;
+extern PGDLLIMPORT char *jit_provider;
+extern PGDLLIMPORT bool jit_debugging_support;
+extern PGDLLIMPORT bool jit_dump_bitcode;
+extern PGDLLIMPORT bool jit_expressions;
+extern PGDLLIMPORT bool jit_profiling_support;
+extern PGDLLIMPORT bool jit_tuple_deforming;
+extern PGDLLIMPORT double jit_above_cost;
+extern PGDLLIMPORT double jit_inline_above_cost;
+extern PGDLLIMPORT double jit_optimize_above_cost;
+
+
+extern void jit_reset_after_error(void);
+extern void jit_release_context(JitContext *context);
+
+/*
+ * Functions for attempting to JIT code. Callers must accept that these might
+ * not be able to perform JIT (i.e. return false).
+ */
+extern bool jit_compile_expr(struct ExprState *state);
+extern void InstrJitAgg(JitInstrumentation *dst, JitInstrumentation *add);
+
+
+#endif							/* JIT_H */
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
new file mode 100644
index 0000000..2c02b78
--- /dev/null
+++ b/src/include/jit/llvmjit.h
@@ -0,0 +1,161 @@
+/*-------------------------------------------------------------------------
+ * llvmjit.h
+ *	  LLVM JIT provider.
+ *
+ * Copyright (c) 2016-2023, PostgreSQL Global Development Group
+ *
+ * src/include/jit/llvmjit.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef LLVMJIT_H
+#define LLVMJIT_H
+
+/*
+ * To avoid breaking cpluspluscheck, allow including the file even when LLVM
+ * is not available.
+ */
+#ifdef USE_LLVM
+
+#include <llvm-c/Types.h>
+
+
+/*
+ * File needs to be includable by both C and C++ code, and include other
+ * headers doing the same. Therefore wrap C portion in our own extern "C" if
+ * in C++ mode.
+ */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "access/tupdesc.h"
+#include "fmgr.h"
+#include "jit/jit.h"
+#include "nodes/pg_list.h"
+
+typedef struct LLVMJitContext
+{
+	JitContext	base;
+
+	/* number of modules created */
+	size_t		module_generation;
+
+	/*
+	 * The LLVM Context used by this JIT context. An LLVM context is reused
+	 * across many compilations, but occasionally reset to prevent it using
+	 * too much memory due to more and more types accumulating.
+	 */
+	LLVMContextRef llvm_context;
+
+	/* current, "open for write", module */
+	LLVMModuleRef module;
+
+	/* is there any pending code that needs to be emitted */
+	bool		compiled;
+
+	/* # of objects emitted, used to generate non-conflicting names */
+	int			counter;
+
+	/* list of handles for code emitted via Orc */
+	List	   *handles;
+} LLVMJitContext;
+
+/* llvm module containing information about types */
+extern PGDLLIMPORT LLVMModuleRef llvm_types_module;
+
+/* type and struct definitions */
+extern PGDLLIMPORT LLVMTypeRef TypeParamBool;
+extern PGDLLIMPORT LLVMTypeRef TypePGFunction;
+extern PGDLLIMPORT LLVMTypeRef TypeSizeT;
+extern PGDLLIMPORT LLVMTypeRef TypeStorageBool;
+
+extern PGDLLIMPORT LLVMTypeRef StructNullableDatum;
+extern PGDLLIMPORT LLVMTypeRef StructTupleDescData;
+extern PGDLLIMPORT LLVMTypeRef StructHeapTupleData;
+extern PGDLLIMPORT LLVMTypeRef StructHeapTupleHeaderData;
+extern PGDLLIMPORT LLVMTypeRef StructMinimalTupleData;
+extern PGDLLIMPORT LLVMTypeRef StructTupleTableSlot;
+extern PGDLLIMPORT LLVMTypeRef StructHeapTupleTableSlot;
+extern PGDLLIMPORT LLVMTypeRef StructMinimalTupleTableSlot;
+extern PGDLLIMPORT LLVMTypeRef StructMemoryContextData;
+extern PGDLLIMPORT LLVMTypeRef StructFunctionCallInfoData;
+extern PGDLLIMPORT LLVMTypeRef StructExprContext;
+extern PGDLLIMPORT LLVMTypeRef StructExprEvalStep;
+extern PGDLLIMPORT LLVMTypeRef StructExprState;
+extern PGDLLIMPORT LLVMTypeRef StructAggState;
+extern PGDLLIMPORT LLVMTypeRef StructAggStatePerTransData;
+extern PGDLLIMPORT LLVMTypeRef StructAggStatePerGroupData;
+extern PGDLLIMPORT LLVMTypeRef StructPlanState;
+
+extern PGDLLIMPORT LLVMValueRef AttributeTemplate;
+extern PGDLLIMPORT LLVMValueRef ExecEvalBoolSubroutineTemplate;
+extern PGDLLIMPORT LLVMValueRef ExecEvalSubroutineTemplate;
+
+
+extern void llvm_enter_fatal_on_oom(void);
+extern void llvm_leave_fatal_on_oom(void);
+extern bool llvm_in_fatal_on_oom(void);
+extern void llvm_reset_after_error(void);
+extern void llvm_assert_in_fatal_section(void);
+
+extern LLVMJitContext *llvm_create_context(int jitFlags);
+extern LLVMModuleRef llvm_mutable_module(LLVMJitContext *context);
+extern char *llvm_expand_funcname(LLVMJitContext *context, const char *basename);
+extern void *llvm_get_function(LLVMJitContext *context, const char *funcname);
+extern void llvm_split_symbol_name(const char *name, char **modname, char **funcname);
+extern LLVMTypeRef llvm_pg_var_type(const char *varname);
+extern LLVMTypeRef llvm_pg_var_func_type(const char *varname);
+extern LLVMValueRef llvm_pg_func(LLVMModuleRef mod, const char *funcname);
+extern void llvm_copy_attributes(LLVMValueRef from, LLVMValueRef to);
+extern LLVMValueRef llvm_function_reference(LLVMJitContext *context,
+						LLVMBuilderRef builder,
+						LLVMModuleRef mod,
+						FunctionCallInfo fcinfo);
+
+extern void llvm_inline_reset_caches(void);
+extern void llvm_inline(LLVMModuleRef mod);
+
+/*
+ ****************************************************************************
+ * Code generation functions.
+ ****************************************************************************
+ */
+extern bool llvm_compile_expr(struct ExprState *state);
+struct TupleTableSlotOps;
+extern LLVMValueRef slot_compile_deform(struct LLVMJitContext *context, TupleDesc desc,
+										const struct TupleTableSlotOps *ops, int natts);
+
+/*
+ ****************************************************************************
+ * Extensions / Backward compatibility section of the LLVM C API
+ * Error handling related functions.
+ ****************************************************************************
+ */
+#if defined(HAVE_DECL_LLVMGETHOSTCPUNAME) && !HAVE_DECL_LLVMGETHOSTCPUNAME
+/** Get the host CPU as a string. The result needs to be disposed with
+  LLVMDisposeMessage. */
+extern char *LLVMGetHostCPUName(void);
+#endif
+
+#if defined(HAVE_DECL_LLVMGETHOSTCPUFEATURES) && !HAVE_DECL_LLVMGETHOSTCPUFEATURES
+/** Get the host CPU features as a string. The result needs to be disposed
+  with LLVMDisposeMessage. */
+extern char *LLVMGetHostCPUFeatures(void);
+#endif
+
+extern unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx);
+extern LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r);
+extern LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r);
+
+#if LLVM_MAJOR_VERSION < 8
+extern LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef g);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif							/* USE_LLVM */
+#endif							/* LLVMJIT_H */
diff --git a/src/include/jit/llvmjit_emit.h b/src/include/jit/llvmjit_emit.h
new file mode 100644
index 0000000..b1f0ea5
--- /dev/null
+++ b/src/include/jit/llvmjit_emit.h
@@ -0,0 +1,336 @@
+/*
+ * llvmjit_emit.h
+ *	  Helpers to make emitting LLVM IR a bit more concise and pgindent proof.
+ *
+ * Copyright (c) 2018-2023, PostgreSQL Global Development Group
+ *
+ * src/include/jit/llvmjit_emit.h
+ */
+#ifndef LLVMJIT_EMIT_H
+#define LLVMJIT_EMIT_H
+
+/*
+ * To avoid breaking cpluspluscheck, allow including the file even when LLVM
+ * is not available.
+ */
+#ifdef USE_LLVM
+
+#include <llvm-c/Core.h>
+#include <llvm-c/Target.h>
+
+#include "jit/llvmjit.h"
+
+
+/*
+ * Emit a non-LLVM pointer as an LLVM constant.
+ */
+static inline LLVMValueRef
+l_ptr_const(void *ptr, LLVMTypeRef type)
+{
+	LLVMValueRef c = LLVMConstInt(TypeSizeT, (uintptr_t) ptr, false);
+
+	return LLVMConstIntToPtr(c, type);
+}
+
+/*
+ * Emit pointer.
+ */
+static inline LLVMTypeRef
+l_ptr(LLVMTypeRef t)
+{
+	return LLVMPointerType(t, 0);
+}
+
+/*
+ * Emit constant integer.
+ */
+static inline LLVMValueRef
+l_int8_const(LLVMContextRef lc, int8 i)
+{
+	return LLVMConstInt(LLVMInt8TypeInContext(lc), i, false);
+}
+
+/*
+ * Emit constant integer.
+ */
+static inline LLVMValueRef
+l_int16_const(LLVMContextRef lc, int16 i)
+{
+	return LLVMConstInt(LLVMInt16TypeInContext(lc), i, false);
+}
+
+/*
+ * Emit constant integer.
+ */
+static inline LLVMValueRef
+l_int32_const(LLVMContextRef lc, int32 i)
+{
+	return LLVMConstInt(LLVMInt32TypeInContext(lc), i, false);
+}
+
+/*
+ * Emit constant integer.
+ */
+static inline LLVMValueRef
+l_int64_const(LLVMContextRef lc, int64 i)
+{
+	return LLVMConstInt(LLVMInt64TypeInContext(lc), i, false);
+}
+
+/*
+ * Emit constant integer.
+ */
+static inline LLVMValueRef
+l_sizet_const(size_t i)
+{
+	return LLVMConstInt(TypeSizeT, i, false);
+}
+
+/*
+ * Emit constant boolean, as used for storage (e.g. global vars, structs).
+ */
+static inline LLVMValueRef
+l_sbool_const(bool i)
+{
+	return LLVMConstInt(TypeStorageBool, (int) i, false);
+}
+
+/*
+ * Emit constant boolean, as used for parameters (e.g. function parameters).
+ */
+static inline LLVMValueRef
+l_pbool_const(bool i)
+{
+	return LLVMConstInt(TypeParamBool, (int) i, false);
+}
+
+static inline LLVMValueRef
+l_struct_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, int32 idx, const char *name)
+{
+#if LLVM_VERSION_MAJOR < 16
+	return LLVMBuildStructGEP(b, v, idx, "");
+#else
+	return LLVMBuildStructGEP2(b, t, v, idx, "");
+#endif
+}
+
+static inline LLVMValueRef
+l_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, LLVMValueRef *indices, int32 nindices, const char *name)
+{
+#if LLVM_VERSION_MAJOR < 16
+	return LLVMBuildGEP(b, v, indices, nindices, name);
+#else
+	return LLVMBuildGEP2(b, t, v, indices, nindices, name);
+#endif
+}
+
+static inline LLVMValueRef
+l_load(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, const char *name)
+{
+#if LLVM_VERSION_MAJOR < 16
+	return LLVMBuildLoad(b, v, name);
+#else
+	return LLVMBuildLoad2(b, t, v, name);
+#endif
+}
+
+static inline LLVMValueRef
+l_call(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef fn, LLVMValueRef *args, int32 nargs, const char *name)
+{
+#if LLVM_VERSION_MAJOR < 16
+	return LLVMBuildCall(b, fn, args, nargs, name);
+#else
+	return LLVMBuildCall2(b, t, fn, args, nargs, name);
+#endif
+}
+
+/*
+ * Load a pointer member idx from a struct.
+ */
+static inline LLVMValueRef
+l_load_struct_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, int32 idx, const char *name)
+{
+	return l_load(b,
+				  LLVMStructGetTypeAtIndex(t, idx),
+				  l_struct_gep(b, t, v, idx, ""),
+				  name);
+}
+
+/*
+ * Load value of a pointer, after applying one index operation.
+ */
+static inline LLVMValueRef
+l_load_gep1(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, LLVMValueRef idx, const char *name)
+{
+	return l_load(b, t, l_gep(b, t, v, &idx, 1, ""), name);
+}
+
+/* separate, because pg_attribute_printf(2, 3) can't appear in definition */
+static inline LLVMBasicBlockRef l_bb_before_v(LLVMBasicBlockRef r, const char *fmt,...) pg_attribute_printf(2, 3);
+
+/*
+ * Insert a new basic block, just before r, the name being determined by fmt
+ * and arguments.
+ */
+static inline LLVMBasicBlockRef
+l_bb_before_v(LLVMBasicBlockRef r, const char *fmt,...)
+{
+	char		buf[512];
+	va_list		args;
+	LLVMContextRef lc;
+
+	va_start(args, fmt);
+	vsnprintf(buf, sizeof(buf), fmt, args);
+	va_end(args);
+
+	lc = LLVMGetTypeContext(LLVMTypeOf(LLVMGetBasicBlockParent(r)));
+
+	return LLVMInsertBasicBlockInContext(lc, r, buf);
+}
+
+/* separate, because pg_attribute_printf(2, 3) can't appear in definition */
+static inline LLVMBasicBlockRef l_bb_append_v(LLVMValueRef f, const char *fmt,...) pg_attribute_printf(2, 3);
+
+/*
+ * Insert a new basic block after previous basic blocks, the name being
+ * determined by fmt and arguments.
+ */
+static inline LLVMBasicBlockRef
+l_bb_append_v(LLVMValueRef f, const char *fmt,...)
+{
+	char		buf[512];
+	va_list		args;
+	LLVMContextRef lc;
+
+	va_start(args, fmt);
+	vsnprintf(buf, sizeof(buf), fmt, args);
+	va_end(args);
+
+	lc = LLVMGetTypeContext(LLVMTypeOf(f));
+
+	return LLVMAppendBasicBlockInContext(lc, f, buf);
+}
+
+/*
+ * Mark a callsite as readonly.
+ */
+static inline void
+l_callsite_ro(LLVMValueRef f)
+{
+	const char	argname[] = "readonly";
+	LLVMAttributeRef ref;
+
+	ref = LLVMCreateStringAttribute(LLVMGetTypeContext(LLVMTypeOf(f)),
+									argname,
+									sizeof(argname) - 1,
+									NULL, 0);
+
+	LLVMAddCallSiteAttribute(f, LLVMAttributeFunctionIndex, ref);
+}
+
+/*
+ * Mark a callsite as alwaysinline.
+ */
+static inline void
+l_callsite_alwaysinline(LLVMValueRef f)
+{
+	const char	argname[] = "alwaysinline";
+	int			id;
+	LLVMAttributeRef attr;
+
+	id = LLVMGetEnumAttributeKindForName(argname,
+										 sizeof(argname) - 1);
+	attr = LLVMCreateEnumAttribute(LLVMGetTypeContext(LLVMTypeOf(f)), id, 0);
+	LLVMAddCallSiteAttribute(f, LLVMAttributeFunctionIndex, attr);
+}
+
+/*
+ * Emit code to switch memory context.
+ */
+static inline LLVMValueRef
+l_mcxt_switch(LLVMModuleRef mod, LLVMBuilderRef b, LLVMValueRef nc)
+{
+	const char *cmc = "CurrentMemoryContext";
+	LLVMValueRef cur;
+	LLVMValueRef ret;
+
+	if (!(cur = LLVMGetNamedGlobal(mod, cmc)))
+		cur = LLVMAddGlobal(mod, l_ptr(StructMemoryContextData), cmc);
+	ret = l_load(b, l_ptr(StructMemoryContextData), cur, cmc);
+	LLVMBuildStore(b, nc, cur);
+
+	return ret;
+}
+
+/*
+ * Return pointer to the argno'th argument nullness.
+ */
+static inline LLVMValueRef
+l_funcnullp(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno)
+{
+	LLVMValueRef v_args;
+	LLVMValueRef v_argn;
+
+	v_args = l_struct_gep(b,
+						  StructFunctionCallInfoData,
+						  v_fcinfo,
+						  FIELDNO_FUNCTIONCALLINFODATA_ARGS,
+						  "");
+	v_argn = l_struct_gep(b,
+						  LLVMArrayType(StructNullableDatum, 0),
+						  v_args,
+						  argno,
+						  "");
+	return l_struct_gep(b,
+						StructNullableDatum,
+						v_argn,
+						FIELDNO_NULLABLE_DATUM_ISNULL,
+						"");
+}
+
+/*
+ * Return pointer to the argno'th argument datum.
+ */
+static inline LLVMValueRef
+l_funcvaluep(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno)
+{
+	LLVMValueRef v_args;
+	LLVMValueRef v_argn;
+
+	v_args = l_struct_gep(b,
+						  StructFunctionCallInfoData,
+						  v_fcinfo,
+						  FIELDNO_FUNCTIONCALLINFODATA_ARGS,
+						  "");
+	v_argn = l_struct_gep(b,
+						  LLVMArrayType(StructNullableDatum, 0),
+						  v_args,
+						  argno,
+						  "");
+	return l_struct_gep(b,
+						StructNullableDatum,
+						v_argn,
+						FIELDNO_NULLABLE_DATUM_DATUM,
+						"");
+}
+
+/*
+ * Return argno'th argument nullness.
+ */
+static inline LLVMValueRef
+l_funcnull(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno)
+{
+	return l_load(b, TypeStorageBool, l_funcnullp(b, v_fcinfo, argno), "");
+}
+
+/*
+ * Return argno'th argument datum.
+ */
+static inline LLVMValueRef
+l_funcvalue(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno)
+{
+	return l_load(b, TypeSizeT, l_funcvaluep(b, v_fcinfo, argno), "");
+}
+
+#endif							/* USE_LLVM */
+#endif