Merge tag '1.3.1' into debian

CHANGELOG.md

@@ -1,6 +1,41 @@
 Changelog
 =========
 
+2021-06-21 version 1.3.1:
+-------------------------
+
+  **Miscellaneous**:
+
+  - Fix compatibility with PostgreSQL 14 beta 2
+
+2021-06-04 version 1.3.0:
+-------------------------
+
+  **New features**:
+
+  - Add support for hypothetical hash indexes (pg10+)
+
+2021-02-26 version 1.2.0:
+-------------------------
+
+  **New features**:
+
+  - Make hypopg work on standby servers using a new "fake" oid generator, that
+    borrows Oids in the FirstBootstrapObjectId / FirstNormalObjectId range
+    rather than real oids.  If necessary, the old behavior can still be used
+    with the new hypopg.use_real_oids configuration option.
+
+  **Bug fixes**
+
+  - Check if access methods support an INCLUDE clause to avoid creating invalid
+    hypothetical indexes.
+  - Display hypothetical indexes on dropped table in hypopg_list_indexes.
+
+  **Miscellaneous**
+
+  - Change hypopg_list_indexes() to view hypopg_list_indexes.
+  - Various documentation improvements.
+
 2020-06-24 version 1.1.4:
 -------------------------
 

CONTRIBUTORS.md

@@ -15,3 +15,5 @@ People who contributed to hypopg:
   * Jan Koßmann
   * Extortioner01
   * nagaraju11
+  * ibrahim edib kokdemir
+  * github user nikhil-postgres

LICENSE

@@ -1,4 +1,4 @@
-Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group
+Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 

Makefile

@@ -44,7 +44,11 @@ ifeq ($(MAJORVERSION),10)
 endif
 
 ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6 10))
-	REGRESS += hypo_index_part
+	REGRESS += hypo_index_part hypo_include
+endif
+
+ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6))
+	REGRESS += hypo_hash
 endif
 
 DEBUILD_ROOT = /tmp/$(EXTENSION)

README.md

@@ -24,6 +24,14 @@ Installation
 - `sudo make install`
 - In every needed database: `CREATE EXTENSION hypopg;`
 
+
+Updating the extension
+----------------------
+
+Note that hypopg doesn't provide extension upgrade scripts, as there's no
+data saved in any of the objects created.  Therefore, you need to first drop
+the extension then create it again to get the new version.
+
 Usage
 -----
 

debian/changelog

@@ -1,3 +1,9 @@
+hypopg (1.2.0-1) unstable; urgency=medium
+
+  * New upstream version.
+
+ -- Julien Rouhaud <rjuju123@gmail.com>  Fri, 26 Feb 2021 14:51:06 +0800
+
 hypopg (1.1.4-2) unstable; urgency=medium
 
   * Team upload for PostgreSQL 13.

debian/copyright

@@ -1,4 +1,4 @@
-Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group
+Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 

docs/conf.py

@@ -50,7 +50,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'HypoPG'
-copyright = '2015-2018, Julien Rouhaud'
+copyright = '2015-2021, Julien Rouhaud'
 author = 'Julien Rouhaud'
 
 # The version info for the project you're documenting, acts as replacement for

docs/index.rst

@@ -17,7 +17,7 @@ It's compatible with **PostgreSQL 9.2 and above**.
 
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
    :caption: Contents:
 
    hypothetical_indexes

docs/usage.rst

@@ -48,6 +48,49 @@ As you can see, hypopg version 1.1.0 is installed.  If you need to check using
 plain SQL, please refer to the `pg_extension table documentation
 <https://www.postgresql.org/docs/current/static/catalog-pg-extension.html>`_.
 
+Configuration
+-------------
+
+The following configuration parameters (GUCs) are available, and can be changed
+interactively:
+
+hypopg.enabled:
+  Default to ``on``.
+  Use this parameter to globally enable or disable HypoPG.  When HypoPG is
+  disabled, no hypothetical index will be used, but the defined hypothetical
+  indexes won't be removed.
+
+hypopg.use_real_oids:
+  Default to ``off``.
+  By default, HypoPG won't use "real" object identifiers, but instead borrow
+  ones from the ~ 14000 / 16384 (respectively the lowest unused oid less then
+  FirstNormalObjectId and FirstNormalObjectId) range, which are reserved by
+  PostgreSQL for future usage in future releases.  This doesn't cause any
+  problem, as the free range is dynamically computed the first time a
+  connection uses HypoPG, and has the advantage to work on a standby server.
+  But the drawback is that you can't have more than approximately 2500
+  hypothetical indexes at the same time, and creating a new hypothetical index
+  will become very slow once more than the maximum number of objects has been
+  created until ``hypopg_reset()`` is called.
+
+  If those drawbacks are problematic, you can enable this parameter.  HypoPG
+  will then ask for a real object identifier, which will need to obtain more
+  locks and won't work on a standby, but will allow to use the full range of
+  object identifiers.
+
+  Note that switching this parameter doesn't require to reset the entries, both
+  can coexist at the same time.
+
+Supported access methods
+------------------------
+
+The following access methods are supported:
+
+- btree
+- brin
+- hash (requires PostgreSQL 10 or above)
+- bloom (requires the bloom extension to be installed)
+
 Create a hypothetical index
 ---------------------------
 

expected/hypo_hash.out

@@ -0,0 +1,31 @@
+-- hypothetical hash indexes, pg10+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();
+             hypopg_get_indexdef             
+---------------------------------------------
+ CREATE INDEX ON public.hypo USING hash (id)
+(1 row)
+

expected/hypo_include.out

@@ -0,0 +1,57 @@
+-- hypothetical indexes using INCLUDE keyword, pg11+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Make sure stats and visibility map are up to date
+VACUUM ANALYZE hypo;
+-- Should not use hypothetical index
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id)');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Remove all the hypothetical indexes
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Create INCLUDE index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index using an Index Only Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();
+                    hypopg_get_indexdef                     
+------------------------------------------------------------
+ CREATE INDEX ON public.hypo USING btree (id) INCLUDE (val)
+(1 row)
+

expected/hypopg.out

@@ -26,10 +26,10 @@ WARNING:  hypopg: SQL order #3 is not a CREATE INDEX statement
   1
 (1 row)
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes();
- nspname | relname | amname 
----------+---------+--------
- public  | hypo    | btree
+SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes;
+ schema_name | table_name | am_name 
+-------------+------------+---------
+ public      | hypo       | btree
 (1 row)
 
 -- Should use hypothetical index
@@ -162,3 +162,29 @@ SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on
  CREATE INDEX ON public.hypo USING btree (id DESC, id DESC, id DESC NULLS LAST, ((md5(val))::bpchar) bpchar_pattern_ops) WITH (fillfactor = 10) WHERE ((id < 1000) AND ((id + (1 % 2)) = 3))
 (1 row)
 
+-- Make sure the old Oid generator still works.  Test it while keeping existing
+-- entries, as both should be able to coexist.
+SET hypopg.use_real_oids = on;
+-- Should not use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+ count 
+-------
+     0
+(1 row)
+
+SELECT COUNT(*) AS nb
+FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+

hypopg--1.3.1.sql

@@ -1,7 +1,7 @@
 -- This program is open source, licensed under the PostgreSQL License.
 -- For license terms, see the LICENSE file.
 --
--- Copyright (C) 2015-2018: Julien Rouhaud
+-- Copyright (C) 2015-2021: Julien Rouhaud
 
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION hypopg" to load this file. \quit
@@ -40,17 +40,14 @@ CREATE FUNCTION hypopg(OUT indexname text, OUT indexrelid oid,
     LANGUAGE c COST 100
 AS '$libdir/hypopg', 'hypopg';
 
-CREATE FUNCTION hypopg_list_indexes(OUT indexrelid oid, OUT indexname text, OUT nspname name, OUT relname name, OUT amname name)
-    RETURNS SETOF record
+CREATE VIEW hypopg_list_indexes
 AS
-$_$
-    SELECT h.indexrelid, h.indexname, n.nspname, c.relname, am.amname
+    SELECT h.indexrelid, h.indexname AS index_name, n.nspname AS schema_name,
+    coalesce(c.relname, '<dropped>') AS table_name, am.amname AS am_name
     FROM hypopg() h
-    JOIN pg_class c ON c.oid = h.indrelid
-    JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_am am ON am.oid = h.amid
-$_$
-LANGUAGE sql;
+    LEFT JOIN pg_catalog.pg_class c ON c.oid = h.indrelid
+    LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+    LEFT JOIN pg_catalog.pg_am am ON am.oid = h.amid;
 
 CREATE FUNCTION
 hypopg_relation_size(IN indexid oid)

hypopg.c

@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
  */
@@ -17,11 +17,21 @@
 #include "postgres.h"
 #include "fmgr.h"
 
+#if PG_VERSION_NUM < 120000
+#include "access/sysattr.h"
+#endif
+#include "access/transam.h"
+#if PG_VERSION_NUM < 140000
+#include "catalog/indexing.h"
+#endif
 #if PG_VERSION_NUM >= 110000
 #include "catalog/partition.h"
 #include "nodes/pg_list.h"
 #include "utils/lsyscache.h"
 #endif
+#include "executor/spi.h"
+#include "miscadmin.h"
+#include "utils/elog.h"
 
 #include "include/hypopg.h"
 #include "include/hypopg_import.h"
@@ -33,8 +43,15 @@ PG_MODULE_MAGIC;
 
 bool		isExplain;
 bool		hypo_is_enabled;
+bool		hypo_use_real_oids;
 MemoryContext HypoMemoryContext;
 
+/*--- Private variables ---*/
+
+static Oid last_oid = InvalidOid;
+static Oid min_fake_oid = InvalidOid;
+static bool oid_wraparound = false;
+
 /*--- Functions --- */
 
 PGDLLEXPORT void _PG_init(void);
@@ -52,6 +69,9 @@ static void
 							  Node *parsetree,
 #endif
 							  const char *queryString,
+#if PG_VERSION_NUM >= 140000
+							  bool readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 							  ProcessUtilityContext context,
 #endif
@@ -75,6 +95,7 @@ static void hypo_executorEnd_hook(QueryDesc *queryDesc);
 static ExecutorEnd_hook_type prev_ExecutorEnd_hook = NULL;
 
 
+static Oid hypo_get_min_fake_oid(void);
 static void hypo_get_relation_info_hook(PlannerInfo *root,
 										Oid relationObjectId,
 										bool inhparent,
@@ -125,6 +146,18 @@ _PG_init(void)
 							 NULL,
 							 NULL);
 
+	DefineCustomBoolVariable("hypopg.use_real_oids",
+							 "Use real oids rather than the range < 16384",
+							 NULL,
+							 &hypo_use_real_oids,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL);
+
+	EmitWarningsOnPlaceholders("hypopg");
 }
 
 void
@@ -139,39 +172,117 @@ _PG_fini(void)
 }
 
 /*---------------------------------
- * Wrapper around GetNewRelFileNode
  * Return a new OID for an hypothetical index.
+ *
+ * To avoid locking on pg_class (required to safely call GetNewOidWithIndex or
+ * similar) and to be usable on a standby node, use the oids unused in the
+ * FirstBootstrapObjectId / FirstNormalObjectId range rather than real oids.
+ * For performance, always start with the biggest oid lesser than
+ * FirstNormalObjectId.  This way the loop to find an unused oid will only
+ * happens once a single backend has created more than ~2.5k hypothetical
+ * indexes.
+ *
+ * For people needing to have thousands of hypothetical indexes at the same
+ * time, we also allow to use the initial implementation that relies on real
+ * oids, which comes with all the limitations mentioned above.
  */
 Oid
 hypo_getNewOid(Oid relid)
 {
-	Relation	pg_class;
-	Relation	relation;
-	Oid			newoid;
-	Oid			reltablespace;
-	char		relpersistence;
+	Oid			newoid = InvalidOid;
 
-	/* Open the relation on which we want a new OID */
-	relation = table_open(relid, AccessShareLock);
+	if (hypo_use_real_oids)
+	{
+		Relation	pg_class;
+		Relation	relation;
 
-	reltablespace = relation->rd_rel->reltablespace;
-	relpersistence = relation->rd_rel->relpersistence;
+		/* Open the relation on which we want a new OID */
+		relation = table_open(relid, AccessShareLock);
 
-	/* Close the relation and release the lock now */
-	table_close(relation, AccessShareLock);
+		/* Close the relation and release the lock now */
+		table_close(relation, AccessShareLock);
 
-	/* Open pg_class to aks a new OID */
-	pg_class = table_open(RelationRelationId, RowExclusiveLock);
+		/* Open pg_class to aks a new OID */
+		pg_class = table_open(RelationRelationId, RowExclusiveLock);
 
-	/* ask for a new relfilenode */
-	newoid = GetNewRelFileNode(reltablespace, pg_class, relpersistence);
+		/* ask for a new Oid */
+		newoid = GetNewOidWithIndex(pg_class, ClassOidIndexId,
+#if PG_VERSION_NUM < 120000
+									ObjectIdAttributeNumber
+#else
+									Anum_pg_class_oid
+#endif
+									);
 
-	/* Close pg_class and release the lock now */
-	table_close(pg_class, RowExclusiveLock);
+		/* Close pg_class and release the lock now */
+		table_close(pg_class, RowExclusiveLock);
+	}
+	else
+	{
+		/*
+		 * First, make sure we know what is the biggest oid smaller than
+		 * FirstNormalObjectId present in pg_class.  This can never change so
+		 * we cache the value.
+		 */
+		if (!OidIsValid(min_fake_oid))
+			min_fake_oid = hypo_get_min_fake_oid();
 
+		Assert(OidIsValid(min_fake_oid));
+
+		/* Make sure there's enough room to get one more Oid */
+		if (list_length(hypoIndexes) >= (FirstNormalObjectId - min_fake_oid))
+		{
+			ereport(ERROR,
+					(errmsg("hypopg: not more oid available"),
+					errhint("Remove hypothetical indexes "
+						"or enable hypopg.use_real_oids")));
+		}
+
+		while(!OidIsValid(newoid))
+		{
+			CHECK_FOR_INTERRUPTS();
+
+			if (!OidIsValid(last_oid))
+				newoid = last_oid = min_fake_oid;
+			else
+				newoid = ++last_oid;
+
+			/* Check if we just exceeded the fake oids range */
+			if (newoid >= FirstNormalObjectId)
+			{
+				newoid = min_fake_oid;
+				last_oid = InvalidOid;
+				oid_wraparound = true;
+			}
+
+			/*
+			 * If we already used all available fake oids, we have to make sure
+			 * that the oid isn't used anymore.
+			 */
+			if (oid_wraparound)
+			{
+				if (hypo_get_index(newoid) != NULL)
+				{
+					/* We can't use this oid.  Reset newoid and start again */
+					newoid = InvalidOid;
+				}
+			}
+		}
+	}
+
+	Assert(OidIsValid(newoid));
 	return newoid;
 }
 
+/* Reset the state of the fake oid generator. */
+void
+hypo_reset_fake_oids(void)
+{
+	Assert(hypoIndexes == NIL);
+	last_oid = InvalidOid;
+	oid_wraparound = false;
+}
+
 /* This function setup the "isExplain" flag for next hooks.
  * If this flag is setup, we can add hypothetical indexes.
  */
@@ -183,6 +294,9 @@ hypo_utility_hook(
 				  Node *parsetree,
 #endif
 				  const char *queryString,
+#if PG_VERSION_NUM >= 140000
+				  bool readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 				  ProcessUtilityContext context,
 #endif
@@ -218,6 +332,9 @@ hypo_utility_hook(
 						  parsetree,
 #endif
 						  queryString,
+#if PG_VERSION_NUM >= 140000
+						  readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 						  context,
 #endif
@@ -243,6 +360,9 @@ hypo_utility_hook(
 								parsetree,
 #endif
 								queryString,
+#if PG_VERSION_NUM >= 140000
+								readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 								context,
 #endif
@@ -341,6 +461,46 @@ hypo_executorEnd_hook(QueryDesc *queryDesc)
 		standard_ExecutorEnd(queryDesc);
 }
 
+/*
+ * Return the minmum usable oid in the FirstBootstrapObjectId -
+ * FirstNormalObjectId range.
+ */
+static Oid
+hypo_get_min_fake_oid(void)
+{
+	int			ret, nb;
+	Oid			oid = InvalidOid;
+
+	/*
+	 * Connect to SPI manager
+	 */
+	if ((ret = SPI_connect()) < 0)
+		/* internal error */
+		elog(ERROR, "SPI connect failure - returned %d", ret);
+
+	ret = SPI_execute("SELECT max(oid)"
+			" FROM pg_catalog.pg_class"
+			" WHERE oid < " CppAsString2(FirstNormalObjectId),
+			true, 1);
+	nb = SPI_processed;
+
+	if (ret != SPI_OK_SELECT || nb == 0)
+	{
+		SPI_finish();
+		elog(ERROR, "hypopg: could not find the minimum fake oid");
+	}
+
+	oid = atooid(SPI_getvalue(SPI_tuptable->vals[0],
+				 SPI_tuptable->tupdesc,
+				 1)) + 1;
+
+	/* release SPI related resources (and return to caller's context) */
+	SPI_finish();
+
+	Assert(OidIsValid(oid));
+	return oid;
+}
+
 /*
  * This function will execute the "hypo_injectHypotheticalIndex" for every
  * hypothetical index found for each relation if the isExplain flag is setup.

hypopg.control

@@ -1,6 +1,6 @@
 # hypopg extension
 comment = 'Hypothetical indexes for PostgreSQL'
-default_version = '1.1.4'
+default_version = '1.3.1'
 module_pathname = '$libdir/hypopg'
 relocatable = true
 

hypopg_index.c

@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
  */
@@ -57,6 +57,9 @@
 #endif
 #include "parser/parse_utilcmd.h"
 #include "parser/parser.h"
+#if PG_VERSION_NUM >= 120000
+#include "port/pg_bitutils.h"
+#endif
 #include "storage/bufmgr.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
@@ -173,6 +176,7 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns,
 	entry->amcanorder = amroutine->amcanorder;
 #if PG_VERSION_NUM >= 110000
 	entry->amcanparallel = amroutine->amcanparallel;
+	entry->amcaninclude = amroutine->amcaninclude;
 #endif
 #else
 	/* Up to 9.5, all information is available in the pg_am tuple */
@@ -250,6 +254,14 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns,
 #endif
 #if PG_VERSION_NUM >= 90600
 			&& entry->relam != BLOOM_AM_OID
+#endif
+#if PG_VERSION_NUM >= 100000
+			/*
+			 * Only support hash indexes for pg10+.  In previous version they
+			 * weren't crash safe, and changes in pg10+ also significantly
+			 * changed the disk space allocation.
+			 */
+			 && entry->relam != HASH_AM_OID
 #endif
 			)
 		{
@@ -311,6 +323,9 @@ hypo_index_reset(void)
 
 	list_free(hypoIndexes);
 	hypoIndexes = NIL;
+
+	hypo_reset_fake_oids();
+
 	return;
 }
 
@@ -440,6 +455,14 @@ hypo_index_store_parsetree(IndexStmt *node, const char *queryString)
 					 errmsg("hypopg: access method \"%s\" does not support multicolumn indexes",
 							node->accessMethod)));
 
+#if PG_VERSION_NUM >= 110000
+		if (node-> indexIncludingParams != NIL && !entry->amcaninclude)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("hypopg: access method \"%s\" does not support included columns",
+							node->accessMethod)));
+#endif
+
 		entry->unique = node->unique;
 		entry->ncolumns = nkeycolumns + ninccolumns;
 		entry->nkeycolumns = nkeycolumns;
@@ -1116,36 +1139,41 @@ hypo_injectHypotheticalIndex(PlannerInfo *root,
 	rel->indexlist = lcons(index, rel->indexlist);
 }
 
-/* Return the hypothetical index name is indexId is ours, NULL otherwise, as
+/*
+ * Return the stored hypothetical index for a given oid if any, NULL otherwise
+ */
+hypoIndex *
+hypo_get_index(Oid indexId)
+{
+	ListCell   *lc;
+
+	foreach(lc, hypoIndexes)
+	{
+		hypoIndex  *entry = (hypoIndex *) lfirst(lc);
+
+		if (entry->oid == indexId)
+			return entry;
+	}
+
+	return NULL;
+}
+
+/* Return the hypothetical index name ifs indexId is ours, NULL otherwise, as
  * this is what explain_get_index_name expects to continue his job.
  */
 const char *
 hypo_explain_get_index_name_hook(Oid indexId)
 {
-	char	   *ret = NULL;
-
 	if (isExplain)
 	{
-		/*
-		 * we're in an explain-only command. Return the name of the
-		 * hypothetical index name if it's one of ours, otherwise return NULL
-		 */
-		ListCell   *lc;
+		hypoIndex  *index = NULL;
 
-		foreach(lc, hypoIndexes)
-		{
-			hypoIndex  *entry = (hypoIndex *) lfirst(lc);
+		index = hypo_get_index(indexId);
 
-			if (entry->oid == indexId)
-			{
-				ret = entry->indexname;
-			}
-		}
+		if (index)
+			return index->indexname;
 	}
 
-	if (ret)
-		return ret;
-
 	if (prev_explain_get_index_name_hook)
 		return prev_explain_get_index_name_hook(indexId);
 
@@ -1839,6 +1867,110 @@ hypo_estimate_index(hypoIndex * entry, RelOptInfo *rel)
 		entry->pages += (BlockNumber) ceil(
 										   ((double) entry->tuples * line_size) / usable_page_size);
 	}
+#endif
+#if PG_VERSION_NUM >= 100000
+	else if (entry->relam == HASH_AM_OID)
+	{
+		/* ----------------------------
+		 * From hash AM readme (src/backend/access/hash/README):
+		 *
+		 *   There are four kinds of pages in a hash index: the meta page (page
+		 *   zero), which contains statically allocated control information;
+		 *   primary bucket pages; overflow pages; and bitmap pages, which keep
+		 *   track of overflow pages that have been freed and are available for
+		 *   re-use.  For addressing purposes, bitmap pages are regarded as a
+		 *   subset of the overflow pages.
+		 * [...]
+		 *   A hash index consists of two or more "buckets", into which tuples
+		 *   are placed whenever their hash key maps to the bucket number.
+		 *   [...]
+		 *   Each bucket in the hash index comprises one or more index pages.
+		 *   The bucket's first page is permanently assigned to it when the
+		 *   bucket is created.  Additional pages, called "overflow pages", are
+		 *   added if the bucket receives too many tuples to fit in the primary
+		 *   bucket page.
+		 *
+		 * Hash AM also already provides some functions to compute an initial
+		 * number of buckets given the estimated number of tuples the index
+		 * will contains, which is a good enough estimate for hypothetical
+		 * index.
+		 *
+		 * The code below is simply an adaptation of original code to compute
+		 * the initial number of bucket, modified to cope with hypothetical
+		 * index, plus some naive estimates for the overflow and bitmap pages.
+		 *
+		 * For more details, refer to the original code, in:
+		 *   - _hash_init()
+		 *   - _hash_init_metabuffer()
+		 */
+		int32 data_width;
+		int32 item_width;
+		int32 ffactor;
+		double dnumbuckets;
+		uint32 num_buckets;
+		uint32 num_overflow;
+		uint32 num_bitmap;
+		uint32 lshift;
+
+	/*
+	 * Determine the target fill factor (in tuples per bucket) for this index.
+	 * The idea is to make the fill factor correspond to pages about as full
+	 * as the user-settable fillfactor parameter says.  We can compute it
+	 * exactly since the index datatype (i.e. uint32 hash key) is fixed-width.
+	 */
+	data_width = sizeof(uint32);
+	item_width = MAXALIGN(sizeof(IndexTupleData)) + MAXALIGN(data_width) +
+		sizeof(ItemIdData);		/* include the line pointer */
+	ffactor = HypoHashGetTargetPageUsage(fillfactor) / item_width;
+	/* keep to a sane range */
+	if (ffactor < 10)
+		ffactor = 10;
+
+	/*
+	 * Choose the number of initial bucket pages to match the fill factor
+	 * given the estimated number of tuples.  We round up the result to the
+	 * total number of buckets which has to be allocated before using its
+	 * hashm_spares element. However always force at least 2 bucket pages. The
+	 * upper limit is determined by considerations explained in
+	 * _hash_expandtable().
+	 */
+	dnumbuckets = entry->tuples / ffactor;
+	if (dnumbuckets <= 2.0)
+		num_buckets = 2;
+	else if (dnumbuckets >= (double) 0x40000000)
+		num_buckets = 0x40000000;
+	else
+		num_buckets = _hash_get_totalbuckets(_hash_spareindex(dnumbuckets));
+
+	/*
+	 * Naive estimate of overflow pages, knowing that a page can store ffactor
+	 * tuples: we compute the number of tuples that wouldn't fit in the
+	 * previously computed number of buckets, and compute the number of pages
+	 * needed to store them.
+	 */
+	num_overflow = Max(0, ((entry->tuples - (num_buckets * ffactor)) /
+					   ffactor) + 1);
+
+	/* find largest bitmap array size that will fit in page size */
+#if PG_VERSION_NUM >= 120000
+	lshift = pg_leftmost_one_pos32(HypoHashGetMaxBitmapSize());
+#else
+	for (lshift = _hash_log2(HypoHashGetMaxBitmapSize()); lshift > 0; --lshift)
+	{
+		if ((1 << lshift) <= HypoHashGetMaxBitmapSize())
+			break;
+	}
+#endif
+
+	/*
+	 * Naive estimate of bitmap pages, using the previously computed number of
+	 * overflow pages.
+	 */
+	num_bitmap = Max(1, num_overflow / (1 <<lshift));
+
+	/* Simply add all computed pages, plus one extra block for the meta page */
+	entry->pages = num_buckets + num_overflow + num_bitmap + 1;
+	}
 #endif
 	else
 	{

import/hypopg_import.c

@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */

import/hypopg_import_index.c

@@ -6,7 +6,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */

include/hypopg.h

@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
 */
@@ -40,6 +40,11 @@
 #define LNEXT2(list, lc)	((lc)->next)
 #endif
 
+/* Backport of atooid macro */
+#if PG_VERSION_NUM < 100000
+#define atooid(x) ((Oid) strtoul((x), NULL, 10))
+#endif
+
 extern bool isExplain;
 
 /* GUC for enabling / disabling hypopg during EXPLAIN */
@@ -47,5 +52,6 @@ extern bool hypo_is_enabled;
 extern MemoryContext HypoMemoryContext;
 
 Oid			hypo_getNewOid(Oid relid);
+void		hypo_reset_fake_oids(void);
 
 #endif

include/hypopg_import.h

@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */

include/hypopg_import_index.h

@@ -6,7 +6,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,29 @@
 				MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \
 				MAXALIGN(sizeof(BTPageOpaqueData))) / 3)
 
+#if PG_VERSION_NUM >= 100000
+#include "access/hash.h"
+
+/* adapted from src/include/access/hash.h */
+#define HypoHashGetFillFactor(ffactor) \
+	(((fillfactor) == 0) ? HASH_DEFAULT_FILLFACTOR : (ffactor))
+
+#define HypoHashGetTargetPageUsage(ffactor) \
+	(BLCKSZ * HypoHashGetFillFactor(ffactor) / 100)
+
+#define HypoHashGetMaxBitmapSize() \
+	(BLCKSZ - \
+	 (MAXALIGN(SizeOfPageHeaderData) + MAXALIGN(sizeof(HashPageOpaqueData))))
+
+#define HypoHashMaxItemSize() \
+	MAXALIGN_DOWN(BLCKSZ - \
+				  SizeOfPageHeaderData - \
+				  sizeof(ItemIdData) - \
+				  MAXALIGN(sizeof(HashPageOpaqueData)))
+
+#endif
+
+
 extern List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index,
 				  Relation heapRelation);
 #if PG_VERSION_NUM < 100000

include/hypopg_index.h

@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
 */
@@ -95,6 +95,8 @@ typedef struct hypoIndex
 	bool		amhasgetbitmap; /* does AM have amgetbitmap interface? */
 #if PG_VERSION_NUM >= 110000
 	bool		amcanparallel;	/* does AM support parallel scan? */
+	bool		amcaninclude;	/* does AM support columns included with clause
+								   INCLUDE? */
 #endif
 	bool		amcanunique;	/* does AM support UNIQUE indexes? */
 	bool		amcanmulticol;	/* does AM support multi-column indexes? */
@@ -120,6 +122,7 @@ PGDLLEXPORT Datum hypopg_get_indexdef(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum hypopg_reset_index(PG_FUNCTION_ARGS);
 
 extern explain_get_index_name_hook_type prev_explain_get_index_name_hook;
+hypoIndex *hypo_get_index(Oid indexId);
 const char *hypo_explain_get_index_name_hook(Oid indexId);
 
 void		hypo_injectHypotheticalIndex(PlannerInfo *root,

test/sql/hypo_hash.sql

@@ -0,0 +1,15 @@
+-- hypothetical hash indexes, pg10+
+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)');
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*';
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();

test/sql/hypo_include.sql

@@ -0,0 +1,31 @@
+-- hypothetical indexes using INCLUDE keyword, pg11+
+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+
+-- Make sure stats and visibility map are up to date
+VACUUM ANALYZE hypo;
+
+-- Should not use hypothetical index
+
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id)');
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*';
+
+-- Remove all the hypothetical indexes
+SELECT hypopg_reset();
+
+-- Create INCLUDE index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+
+-- Should use hypothetical index using an Index Only Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*';
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();

test/sql/hypopg.sql

@@ -25,7 +25,7 @@ ANALYZE hypo;
 SELECT COUNT(*) AS nb
 FROM public.hypopg_create_index('SELECT 1;CREATE INDEX ON hypo(id); SELECT 2');
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes();
+SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes;
 
 -- Should use hypothetical index
 SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
@@ -100,3 +100,18 @@ WHERE e ~ 'Index.*<\d+>btree_hypo.*';
 
 -- Deparse an index DDL, with almost every possible pathcode
 SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on hypo using btree(id desc, id desc nulls first, id desc nulls last, cast(md5(val) as bpchar)  bpchar_pattern_ops) with (fillfactor = 10) WHERE id < 1000 AND id +1 %2 = 3');
+
+-- Make sure the old Oid generator still works.  Test it while keeping existing
+-- entries, as both should be able to coexist.
+SET hypopg.use_real_oids = on;
+
+-- Should not use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+
+SELECT COUNT(*) AS nb
+FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);');
+
+-- Should use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';