From e1cdd9e0cf1f14b70f4517a3f63720d59a4df232 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sat, 18 Jul 2020 11:15:55 +0200
Subject: [PATCH 01/22] Update contributors.

---
 CONTRIBUTORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index d6f28d9..1da12f9 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -15,3 +15,4 @@ People who contributed to hypopg:
   * Jan Koßmann
   * Extortioner01
   * nagaraju11
+  * ibrahim edib kokdemir

From 82900d1cafc4a466e4b2bc715c1d24b22795a6b9 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Mon, 14 Dec 2020 12:08:58 +0800
Subject: [PATCH 02/22] Add extension update documentation

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 10807b6..d4f153a 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,14 @@ Installation
 - `sudo make install`
 - In every needed database: `CREATE EXTENSION hypopg;`
 
+
+Updating the extension
+----------------------
+
+Note that hypopg doesn't provide extension upgrade scripts, as there's no
+data saved in any of the objects created.  Therefore, you need to first drop
+the extension then create it again to get the new version.
+
 Usage
 -----
 

From a2214b81b910f9611db2464d63549919a997237c Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 18:40:53 +0800
Subject: [PATCH 03/22] Start working on version 1.2.0

---
 hypopg--1.1.4.sql => hypopg--1.2.0.sql | 0
 hypopg.control                         | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hypopg--1.1.4.sql => hypopg--1.2.0.sql (100%)

diff --git a/hypopg--1.1.4.sql b/hypopg--1.2.0.sql
similarity index 100%
rename from hypopg--1.1.4.sql
rename to hypopg--1.2.0.sql
diff --git a/hypopg.control b/hypopg.control
index f20af9b..3a691f5 100644
--- a/hypopg.control
+++ b/hypopg.control
@@ -1,6 +1,6 @@
 # hypopg extension
 comment = 'Hypothetical indexes for PostgreSQL'
-default_version = '1.1.4'
+default_version = '1.2.0'
 module_pathname = '$libdir/hypopg'
 relocatable = true
 

From 5da0519a03138a9acad664c420c314641f764f46 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 18:41:20 +0800
Subject: [PATCH 04/22] Update copyright year

---
 LICENSE                       | 2 +-
 debian/copyright              | 2 +-
 docs/conf.py                  | 2 +-
 hypopg--1.2.0.sql             | 2 +-
 hypopg.c                      | 2 +-
 hypopg_index.c                | 2 +-
 import/hypopg_import.c        | 2 +-
 import/hypopg_import_index.c  | 2 +-
 include/hypopg.h              | 2 +-
 include/hypopg_import.h       | 2 +-
 include/hypopg_import_index.h | 2 +-
 include/hypopg_index.h        | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/LICENSE b/LICENSE
index 442100f..1ed145e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group
+Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 
diff --git a/debian/copyright b/debian/copyright
index dc9cfee..a978fb4 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -1,4 +1,4 @@
-Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group
+Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group
 
 Portions Copyright (c) 1994, The Regents of the University of California
 
diff --git a/docs/conf.py b/docs/conf.py
index a9327e0..524d540 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,7 +50,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'HypoPG'
-copyright = '2015-2018, Julien Rouhaud'
+copyright = '2015-2020, Julien Rouhaud'
 author = 'Julien Rouhaud'
 
 # The version info for the project you're documenting, acts as replacement for
diff --git a/hypopg--1.2.0.sql b/hypopg--1.2.0.sql
index 49f276d..d8243fa 100644
--- a/hypopg--1.2.0.sql
+++ b/hypopg--1.2.0.sql
@@ -1,7 +1,7 @@
 -- This program is open source, licensed under the PostgreSQL License.
 -- For license terms, see the LICENSE file.
 --
--- Copyright (C) 2015-2018: Julien Rouhaud
+-- Copyright (C) 2015-2021: Julien Rouhaud
 
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION hypopg" to load this file. \quit
diff --git a/hypopg.c b/hypopg.c
index 24c2ab6..3f15502 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
  */
diff --git a/hypopg_index.c b/hypopg_index.c
index 6c3fe69..d79648c 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
  */
diff --git a/import/hypopg_import.c b/import/hypopg_import.c
index 4b03663..d21dfa0 100644
--- a/import/hypopg_import.c
+++ b/import/hypopg_import.c
@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */
diff --git a/import/hypopg_import_index.c b/import/hypopg_import_index.c
index e6b6a2c..22e2902 100644
--- a/import/hypopg_import_index.c
+++ b/import/hypopg_import_index.c
@@ -6,7 +6,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */
diff --git a/include/hypopg.h b/include/hypopg.h
index 9043264..bf4dc2e 100644
--- a/include/hypopg.h
+++ b/include/hypopg.h
@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
 */
diff --git a/include/hypopg_import.h b/include/hypopg_import.h
index 80a3d04..0569190 100644
--- a/include/hypopg_import.h
+++ b/include/hypopg_import.h
@@ -5,7 +5,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */
diff --git a/include/hypopg_import_index.h b/include/hypopg_import_index.h
index 29b5ec8..7704d44 100644
--- a/include/hypopg_import_index.h
+++ b/include/hypopg_import_index.h
@@ -6,7 +6,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (c) 2008-2018, PostgreSQL Global Development Group
+ * Copyright (c) 2008-2021, PostgreSQL Global Development Group
  *
  *-------------------------------------------------------------------------
  */
diff --git a/include/hypopg_index.h b/include/hypopg_index.h
index 0c56db0..323cadc 100644
--- a/include/hypopg_index.h
+++ b/include/hypopg_index.h
@@ -8,7 +8,7 @@
  * This program is open source, licensed under the PostgreSQL license.
  * For license terms, see the LICENSE file.
  *
- * Copyright (C) 2015-2018: Julien Rouhaud
+ * Copyright (C) 2015-2021: Julien Rouhaud
  *
  *-------------------------------------------------------------------------
 */

From 3131560b8588afee01d73f39e36a24d53bbe3d2b Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 18:54:20 +0800
Subject: [PATCH 05/22] Change hypopg_list_indexes() to a view.

And while at schema-qualify relation names used in that view.
---
 expected/hypopg.out |  2 +-
 hypopg--1.2.0.sql   | 12 ++++--------
 test/sql/hypopg.sql |  2 +-
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/expected/hypopg.out b/expected/hypopg.out
index e51e948..f5a8b0a 100644
--- a/expected/hypopg.out
+++ b/expected/hypopg.out
@@ -26,7 +26,7 @@ WARNING:  hypopg: SQL order #3 is not a CREATE INDEX statement
   1
 (1 row)
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes();
+SELECT nspname, relname, amname FROM public.hypopg_list_indexes;
  nspname | relname | amname 
 ---------+---------+--------
  public  | hypo    | btree
diff --git a/hypopg--1.2.0.sql b/hypopg--1.2.0.sql
index d8243fa..b763644 100644
--- a/hypopg--1.2.0.sql
+++ b/hypopg--1.2.0.sql
@@ -40,17 +40,13 @@ CREATE FUNCTION hypopg(OUT indexname text, OUT indexrelid oid,
     LANGUAGE c COST 100
 AS '$libdir/hypopg', 'hypopg';
 
-CREATE FUNCTION hypopg_list_indexes(OUT indexrelid oid, OUT indexname text, OUT nspname name, OUT relname name, OUT amname name)
-    RETURNS SETOF record
+CREATE VIEW hypopg_list_indexes
 AS
-$_$
     SELECT h.indexrelid, h.indexname, n.nspname, c.relname, am.amname
     FROM hypopg() h
-    JOIN pg_class c ON c.oid = h.indrelid
-    JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_am am ON am.oid = h.amid
-$_$
-LANGUAGE sql;
+    JOIN pg_catalog.pg_class c ON c.oid = h.indrelid
+    JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+    JOIN pg_catalog.pg_am am ON am.oid = h.amid;
 
 CREATE FUNCTION
 hypopg_relation_size(IN indexid oid)
diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql
index 3aac4af..7712c3f 100644
--- a/test/sql/hypopg.sql
+++ b/test/sql/hypopg.sql
@@ -25,7 +25,7 @@ ANALYZE hypo;
 SELECT COUNT(*) AS nb
 FROM public.hypopg_create_index('SELECT 1;CREATE INDEX ON hypo(id); SELECT 2');
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes();
+SELECT nspname, relname, amname FROM public.hypopg_list_indexes;
 
 -- Should use hypothetical index
 SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e

From 0eca85f2aaab90f7f8f0cd28505633037b0bb3e6 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 19:08:32 +0800
Subject: [PATCH 06/22] Warn users if they used wrong GUC name.

---
 hypopg.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hypopg.c b/hypopg.c
index 3f15502..e0f3575 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -125,6 +125,7 @@ _PG_init(void)
 							 NULL,
 							 NULL);
 
+	EmitWarningsOnPlaceholders("hypopg");
 }
 
 void

From 0864fbe1fbff2000050afaee19bd50f34d2c714f Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 17:49:48 +0800
Subject: [PATCH 07/22] Add a new hypo_get_index() function.

This will be helpful in a following patch.  The immediate effect is to fix a
long standing small performance issue, as hypo_explain_get_index_name_hook
didn't stop its loop once it found a matching entry.
---
 hypopg_index.c         | 43 +++++++++++++++++++++++-------------------
 include/hypopg_index.h |  1 +
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/hypopg_index.c b/hypopg_index.c
index d79648c..12d1dc5 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -1116,36 +1116,41 @@ hypo_injectHypotheticalIndex(PlannerInfo *root,
 	rel->indexlist = lcons(index, rel->indexlist);
 }
 
-/* Return the hypothetical index name is indexId is ours, NULL otherwise, as
+/*
+ * Return the stored hypothetical index for a given oid if any, NULL otherwise
+ */
+hypoIndex *
+hypo_get_index(Oid indexId)
+{
+	ListCell   *lc;
+
+	foreach(lc, hypoIndexes)
+	{
+		hypoIndex  *entry = (hypoIndex *) lfirst(lc);
+
+		if (entry->oid == indexId)
+			return entry;
+	}
+
+	return NULL;
+}
+
+/* Return the hypothetical index name ifs indexId is ours, NULL otherwise, as
  * this is what explain_get_index_name expects to continue his job.
  */
 const char *
 hypo_explain_get_index_name_hook(Oid indexId)
 {
-	char	   *ret = NULL;
-
 	if (isExplain)
 	{
-		/*
-		 * we're in an explain-only command. Return the name of the
-		 * hypothetical index name if it's one of ours, otherwise return NULL
-		 */
-		ListCell   *lc;
+		hypoIndex  *index = NULL;
 
-		foreach(lc, hypoIndexes)
-		{
-			hypoIndex  *entry = (hypoIndex *) lfirst(lc);
+		index = hypo_get_index(indexId);
 
-			if (entry->oid == indexId)
-			{
-				ret = entry->indexname;
-			}
-		}
+		if (index)
+			return index->indexname;
 	}
 
-	if (ret)
-		return ret;
-
 	if (prev_explain_get_index_name_hook)
 		return prev_explain_get_index_name_hook(indexId);
 
diff --git a/include/hypopg_index.h b/include/hypopg_index.h
index 323cadc..3ffb637 100644
--- a/include/hypopg_index.h
+++ b/include/hypopg_index.h
@@ -120,6 +120,7 @@ PGDLLEXPORT Datum hypopg_get_indexdef(PG_FUNCTION_ARGS);
 PGDLLEXPORT Datum hypopg_reset_index(PG_FUNCTION_ARGS);
 
 extern explain_get_index_name_hook_type prev_explain_get_index_name_hook;
+hypoIndex *hypo_get_index(Oid indexId);
 const char *hypo_explain_get_index_name_hook(Oid indexId);
 
 void		hypo_injectHypotheticalIndex(PlannerInfo *root,

From 2bd376d6bdf331d719a6e7e838893f902a41d5f8 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 18:16:02 +0800
Subject: [PATCH 08/22] Change the Oid generator.

To avoid locking on pg_class (required to safely call GetNewOidWithIndex or
similar) and to be usable on a standby node, use the oids unused in the
FirstBootstrapObjectId / FirstNormalObjectId range rather than real oids.  For
performance, always start with the biggest oid lesser than FirstNormalObjectId.
This way the loop to find an unused oid will only happens once a single backend
has created more than ~2.5k hypothetical indexes.

For people needing to have thousands of hypothetical indexes at the same time,
we also allow to use the initial implementation that relies on real oids, which
comes with all the limitations mentioned above, with the new
hypopg.use_real_oids GUC.
---
 docs/usage.rst      |  34 +++++++++
 expected/hypopg.out |  26 +++++++
 hypopg.c            | 173 +++++++++++++++++++++++++++++++++++++++-----
 hypopg_index.c      |   3 +
 include/hypopg.h    |   6 ++
 test/sql/hypopg.sql |  15 ++++
 6 files changed, 239 insertions(+), 18 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index e759300..38cac20 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -48,6 +48,40 @@ As you can see, hypopg version 1.1.0 is installed.  If you need to check using
 plain SQL, please refer to the `pg_extension table documentation
 <https://www.postgresql.org/docs/current/static/catalog-pg-extension.html>`_.
 
+Configuration
+-------------
+
+The following configuration parameters (GUCs) are available, and can be changed
+interactively:
+
+hypopg.enabled:
+  Default to ``on``.
+  Use this parameter to globally enable or disable HypoPG.  When HypoPG is
+  disabled, no hypothetical index will be used, but the defined hypothetical
+  indexes won't be removed.
+
+hypopg.use_real_oids:
+  Default to ``off``.
+  By default, HypoPG won't use "real" object identifiers, but instead borrow
+  ones from the ~ 14000 / 16384 (respectively the lowest unused oid less then
+  FirstNormalObjectId and FirstNormalObjectId) range, which are reserved by
+  PostgreSQL for future usage in future releases.  This doesn't cause any
+  problem, as the free range is dynamically computed the first time a
+  connection uses HypoPG, and has the advantage to work on a standby server.
+  But the drawback is that you can't have more than approximately 2500
+  hypothetical indexes at the same time, and creating a new hypothetical index
+  will become very slow once more than the maximum number of objects has been
+  created until ``hypopg_reset()`` is called.
+
+  If those drawbacks are problematic, you can enable this parameter.  HypoPG
+  will then ask for a real object identifier, which will need to obtain more
+  locks and won't work on a standby, but will allow to use the full range of
+  object identifiers.
+
+  Note that switching this parameter doesn't require to reset the entries, both
+  can coexist at the same time.
+
+
 Create a hypothetical index
 ---------------------------
 
diff --git a/expected/hypopg.out b/expected/hypopg.out
index f5a8b0a..09ce930 100644
--- a/expected/hypopg.out
+++ b/expected/hypopg.out
@@ -162,3 +162,29 @@ SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on
  CREATE INDEX ON public.hypo USING btree (id DESC, id DESC, id DESC NULLS LAST, ((md5(val))::bpchar) bpchar_pattern_ops) WITH (fillfactor = 10) WHERE ((id < 1000) AND ((id + (1 % 2)) = 3))
 (1 row)
 
+-- Make sure the old Oid generator still works.  Test it while keeping existing
+-- entries, as both should be able to coexist.
+SET hypopg.use_real_oids = on;
+-- Should not use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+ count 
+-------
+     0
+(1 row)
+
+SELECT COUNT(*) AS nb
+FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
diff --git a/hypopg.c b/hypopg.c
index e0f3575..2442410 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -17,11 +17,14 @@
 #include "postgres.h"
 #include "fmgr.h"
 
+#include "access/transam.h"
 #if PG_VERSION_NUM >= 110000
 #include "catalog/partition.h"
 #include "nodes/pg_list.h"
 #include "utils/lsyscache.h"
 #endif
+#include "executor/spi.h"
+#include "utils/elog.h"
 
 #include "include/hypopg.h"
 #include "include/hypopg_import.h"
@@ -33,8 +36,15 @@ PG_MODULE_MAGIC;
 
 bool		isExplain;
 bool		hypo_is_enabled;
+bool		hypo_use_real_oids;
 MemoryContext HypoMemoryContext;
 
+/*--- Private variables ---*/
+
+static Oid last_oid = InvalidOid;
+static Oid min_fake_oid = InvalidOid;
+static bool oid_wraparound = false;
+
 /*--- Functions --- */
 
 PGDLLEXPORT void _PG_init(void);
@@ -75,6 +85,7 @@ static void hypo_executorEnd_hook(QueryDesc *queryDesc);
 static ExecutorEnd_hook_type prev_ExecutorEnd_hook = NULL;
 
 
+static Oid hypo_get_min_fake_oid(void);
 static void hypo_get_relation_info_hook(PlannerInfo *root,
 										Oid relationObjectId,
 										bool inhparent,
@@ -125,6 +136,17 @@ _PG_init(void)
 							 NULL,
 							 NULL);
 
+	DefineCustomBoolVariable("hypopg.use_real_oids",
+							 "Use real oids rather than the range < 16384",
+							 NULL,
+							 &hypo_use_real_oids,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL);
+
 	EmitWarningsOnPlaceholders("hypopg");
 }
 
@@ -140,39 +162,114 @@ _PG_fini(void)
 }
 
 /*---------------------------------
- * Wrapper around GetNewRelFileNode
  * Return a new OID for an hypothetical index.
+ *
+ * To avoid locking on pg_class (required to safely call GetNewOidWithIndex or
+ * similar) and to be usable on a standby node, use the oids unused in the
+ * FirstBootstrapObjectId / FirstNormalObjectId range rather than real oids.
+ * For performance, always start with the biggest oid lesser than
+ * FirstNormalObjectId.  This way the loop to find an unused oid will only
+ * happens once a single backend has created more than ~2.5k hypothetical
+ * indexes.
+ *
+ * For people needing to have thousands of hypothetical indexes at the same
+ * time, we also allow to use the initial implementation that relies on real
+ * oids, which comes with all the limitations mentioned above.
  */
 Oid
 hypo_getNewOid(Oid relid)
 {
-	Relation	pg_class;
-	Relation	relation;
-	Oid			newoid;
-	Oid			reltablespace;
-	char		relpersistence;
+	Oid			newoid = InvalidOid;
 
-	/* Open the relation on which we want a new OID */
-	relation = table_open(relid, AccessShareLock);
+	if (hypo_use_real_oids)
+	{
+		Relation	pg_class;
+		Relation	relation;
+		Oid			reltablespace;
+		char		relpersistence;
 
-	reltablespace = relation->rd_rel->reltablespace;
-	relpersistence = relation->rd_rel->relpersistence;
+		/* Open the relation on which we want a new OID */
+		relation = table_open(relid, AccessShareLock);
 
-	/* Close the relation and release the lock now */
-	table_close(relation, AccessShareLock);
+		reltablespace = relation->rd_rel->reltablespace;
+		relpersistence = relation->rd_rel->relpersistence;
 
-	/* Open pg_class to aks a new OID */
-	pg_class = table_open(RelationRelationId, RowExclusiveLock);
+		/* Close the relation and release the lock now */
+		table_close(relation, AccessShareLock);
 
-	/* ask for a new relfilenode */
-	newoid = GetNewRelFileNode(reltablespace, pg_class, relpersistence);
+		/* Open pg_class to aks a new OID */
+		pg_class = table_open(RelationRelationId, RowExclusiveLock);
 
-	/* Close pg_class and release the lock now */
-	table_close(pg_class, RowExclusiveLock);
+		/* ask for a new relfilenode */
+		newoid = GetNewRelFileNode(reltablespace, pg_class, relpersistence);
 
+		/* Close pg_class and release the lock now */
+		table_close(pg_class, RowExclusiveLock);
+	}
+	else
+	{
+		/*
+		 * First, make sure we know what is the biggest oid smaller than
+		 * FirstNormalObjectId present in pg_class.  This can never change so
+		 * we cache the value.
+		 */
+		if (!OidIsValid(min_fake_oid))
+			min_fake_oid = hypo_get_min_fake_oid();
+
+		Assert(OidIsValid(min_fake_oid));
+
+		/* Make sure there's enough room to get one more Oid */
+		if (list_length(hypoIndexes) >= (FirstNormalObjectId - min_fake_oid))
+		{
+			ereport(ERROR,
+					(errmsg("hypopg: not more oid available"),
+					errhint("Remove hypothetical indexes "
+						"or enable hypopg.use_real_oids")));
+		}
+
+		while(!OidIsValid(newoid))
+		{
+			if (!OidIsValid(last_oid))
+				newoid = last_oid = min_fake_oid;
+			else
+				newoid = ++last_oid;
+
+			/* Check if we just exceeded the fake oids range */
+			if (newoid >= FirstNormalObjectId)
+			{
+				newoid = min_fake_oid;
+				last_oid = InvalidOid;
+				oid_wraparound = true;
+			}
+
+			/*
+			 * If we already used all available fake oids, we have to make sure
+			 * that the oid isn't used anymore.
+			 */
+			if (oid_wraparound)
+			{
+				if (hypo_get_index(newoid) != NULL)
+				{
+					/* We can't use this oid.  Reset newoid and start again */
+					newoid = InvalidOid;
+				}
+			}
+		}
+	}
+
+	Assert(OidIsValid(newoid));
 	return newoid;
 }
 
+/* Reset the state of the fake oid generator. */
+void
+hypo_reset_fake_oids(void)
+{
+	Assert(hypoIndexes == NIL);
+	last_oid = InvalidOid;
+	oid_wraparound = false;
+}
+
 /* This function setup the "isExplain" flag for next hooks.
  * If this flag is setup, we can add hypothetical indexes.
  */
@@ -342,6 +439,46 @@ hypo_executorEnd_hook(QueryDesc *queryDesc)
 		standard_ExecutorEnd(queryDesc);
 }
 
+/*
+ * Return the minmum usable oid in the FirstBootstrapObjectId -
+ * FirstNormalObjectId range.
+ */
+static Oid
+hypo_get_min_fake_oid(void)
+{
+	int			ret, nb;
+	Oid			oid = InvalidOid;
+
+	/*
+	 * Connect to SPI manager
+	 */
+	if ((ret = SPI_connect()) < 0)
+		/* internal error */
+		elog(ERROR, "SPI connect failure - returned %d", ret);
+
+	ret = SPI_execute("SELECT max(oid)"
+			" FROM pg_catalog.pg_class"
+			" WHERE oid < " CppAsString2(FirstNormalObjectId),
+			true, 1);
+	nb = SPI_processed;
+
+	if (ret != SPI_OK_SELECT || nb == 0)
+	{
+		SPI_finish();
+		elog(ERROR, "hypopg: could not find the minimum fake oid");
+	}
+
+	oid = atooid(SPI_getvalue(SPI_tuptable->vals[0],
+				 SPI_tuptable->tupdesc,
+				 1)) + 1;
+
+	/* release SPI related resources (and return to caller's context) */
+	SPI_finish();
+
+	Assert(OidIsValid(oid));
+	return oid;
+}
+
 /*
  * This function will execute the "hypo_injectHypotheticalIndex" for every
  * hypothetical index found for each relation if the isExplain flag is setup.
diff --git a/hypopg_index.c b/hypopg_index.c
index 12d1dc5..c56bfe6 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -311,6 +311,9 @@ hypo_index_reset(void)
 
 	list_free(hypoIndexes);
 	hypoIndexes = NIL;
+
+	hypo_reset_fake_oids();
+
 	return;
 }
 
diff --git a/include/hypopg.h b/include/hypopg.h
index bf4dc2e..0d47093 100644
--- a/include/hypopg.h
+++ b/include/hypopg.h
@@ -40,6 +40,11 @@
 #define LNEXT2(list, lc)	((lc)->next)
 #endif
 
+/* Backport of atooid macro */
+#if PG_VERSION_NUM < 100000
+#define atooid(x) ((Oid) strtoul((x), NULL, 10))
+#endif
+
 extern bool isExplain;
 
 /* GUC for enabling / disabling hypopg during EXPLAIN */
@@ -47,5 +52,6 @@ extern bool hypo_is_enabled;
 extern MemoryContext HypoMemoryContext;
 
 Oid			hypo_getNewOid(Oid relid);
+void		hypo_reset_fake_oids(void);
 
 #endif
diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql
index 7712c3f..b19e484 100644
--- a/test/sql/hypopg.sql
+++ b/test/sql/hypopg.sql
@@ -100,3 +100,18 @@ WHERE e ~ 'Index.*<\d+>btree_hypo.*';
 
 -- Deparse an index DDL, with almost every possible pathcode
 SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on hypo using btree(id desc, id desc nulls first, id desc nulls last, cast(md5(val) as bpchar)  bpchar_pattern_ops) with (fillfactor = 10) WHERE id < 1000 AND id +1 %2 = 3');
+
+-- Make sure the old Oid generator still works.  Test it while keeping existing
+-- entries, as both should be able to coexist.
+SET hypopg.use_real_oids = on;
+
+-- Should not use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';
+
+SELECT COUNT(*) AS nb
+FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);');
+
+-- Should use hypothetical index
+SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index.*<\d+>btree_hypo.*';

From 4b3d55df85ec09a7c11c28f2aba331a67b9502b9 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 20:48:32 +0800
Subject: [PATCH 09/22] Check for interrupts in the oid generator loop.

As this loop can take a significant amount of time to find a suitable oid in
some extreme case, it's important to make sure that the query is cancellable.
---
 hypopg.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hypopg.c b/hypopg.c
index 2442410..edbb529 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -24,6 +24,7 @@
 #include "utils/lsyscache.h"
 #endif
 #include "executor/spi.h"
+#include "miscadmin.h"
 #include "utils/elog.h"
 
 #include "include/hypopg.h"
@@ -229,6 +230,8 @@ hypo_getNewOid(Oid relid)
 
 		while(!OidIsValid(newoid))
 		{
+			CHECK_FOR_INTERRUPTS();
+
 			if (!OidIsValid(last_oid))
 				newoid = last_oid = min_fake_oid;
 			else

From 119686936e20ad98e037262809764c55244550c4 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 22 Jan 2021 21:02:38 +0800
Subject: [PATCH 10/22] Document which access methods are supported.

---
 docs/index.rst | 2 +-
 docs/usage.rst | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index e9683e9..2f64e82 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -17,7 +17,7 @@ It's compatible with **PostgreSQL 9.2 and above**.
 
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
    :caption: Contents:
 
    hypothetical_indexes
diff --git a/docs/usage.rst b/docs/usage.rst
index 38cac20..0ab3e6b 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -81,6 +81,14 @@ hypopg.use_real_oids:
   Note that switching this parameter doesn't require to reset the entries, both
   can coexist at the same time.
 
+Supported access methods
+------------------------
+
+The following access methods are supported:
+
+- btree
+- brin
+- bloom (requires the bloom extension to be installed)
 
 Create a hypothetical index
 ---------------------------

From 3e3339b4b8af19517dfd226d5a49ce1b8c3127e5 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Thu, 4 Feb 2021 00:16:30 +0800
Subject: [PATCH 11/22] Check if access methods support an INCLUDE clause.

Trying to create an hypothetical index with an INCLUDE clause on pg10- will
obviously fail with a syntax error.

Also add regression test for the INCLUDE clause on btree indexes.
---
 Makefile                  |  2 +-
 expected/hypo_include.out | 55 +++++++++++++++++++++++++++++++++++++++
 hypopg_index.c            |  9 +++++++
 include/hypopg_index.h    |  2 ++
 test/sql/hypo_include.sql | 29 +++++++++++++++++++++
 5 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 expected/hypo_include.out
 create mode 100644 test/sql/hypo_include.sql

diff --git a/Makefile b/Makefile
index ecae2e1..79b7f0a 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ ifeq ($(MAJORVERSION),10)
 endif
 
 ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6 10))
-	REGRESS += hypo_index_part
+	REGRESS += hypo_index_part hypo_include
 endif
 
 DEBUILD_ROOT = /tmp/$(EXTENSION)
diff --git a/expected/hypo_include.out b/expected/hypo_include.out
new file mode 100644
index 0000000..3febb3a
--- /dev/null
+++ b/expected/hypo_include.out
@@ -0,0 +1,55 @@
+-- hypothetical indexes using INCLUDE keyword, pg11+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Make sure stats and visibility map are up to date
+VACUUM ANALYZE hypo;
+-- Should not use hypothetical index
+-- Create normal index
+SELECT hypopg_create_index('CREATE INDEX ON hypo (id)');
+     hypopg_create_index      
+------------------------------
+ (13719,<13719>btree_hypo_id)
+(1 row)
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Remove all the hypothetical indexes
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Create INCLUDE index
+SELECT hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+       hypopg_create_index        
+----------------------------------
+ (13719,<13719>btree_hypo_id_val)
+(1 row)
+
+-- Should use hypothetical index using an Index Only Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();
+                    hypopg_get_indexdef                     
+------------------------------------------------------------
+ CREATE INDEX ON public.hypo USING btree (id) INCLUDE (val)
+(1 row)
+
diff --git a/hypopg_index.c b/hypopg_index.c
index c56bfe6..f03d4c7 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -173,6 +173,7 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns,
 	entry->amcanorder = amroutine->amcanorder;
 #if PG_VERSION_NUM >= 110000
 	entry->amcanparallel = amroutine->amcanparallel;
+	entry->amcaninclude = amroutine->amcaninclude;
 #endif
 #else
 	/* Up to 9.5, all information is available in the pg_am tuple */
@@ -443,6 +444,14 @@ hypo_index_store_parsetree(IndexStmt *node, const char *queryString)
 					 errmsg("hypopg: access method \"%s\" does not support multicolumn indexes",
 							node->accessMethod)));
 
+#if PG_VERSION_NUM >= 110000
+		if (node-> indexIncludingParams != NIL && !entry->amcaninclude)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("hypopg: access method \"%s\" does not support included columns",
+							node->accessMethod)));
+#endif
+
 		entry->unique = node->unique;
 		entry->ncolumns = nkeycolumns + ninccolumns;
 		entry->nkeycolumns = nkeycolumns;
diff --git a/include/hypopg_index.h b/include/hypopg_index.h
index 3ffb637..a07b03e 100644
--- a/include/hypopg_index.h
+++ b/include/hypopg_index.h
@@ -95,6 +95,8 @@ typedef struct hypoIndex
 	bool		amhasgetbitmap; /* does AM have amgetbitmap interface? */
 #if PG_VERSION_NUM >= 110000
 	bool		amcanparallel;	/* does AM support parallel scan? */
+	bool		amcaninclude;	/* does AM support columns included with clause
+								   INCLUDE? */
 #endif
 	bool		amcanunique;	/* does AM support UNIQUE indexes? */
 	bool		amcanmulticol;	/* does AM support multi-column indexes? */
diff --git a/test/sql/hypo_include.sql b/test/sql/hypo_include.sql
new file mode 100644
index 0000000..9c944b5
--- /dev/null
+++ b/test/sql/hypo_include.sql
@@ -0,0 +1,29 @@
+-- hypothetical indexes using INCLUDE keyword, pg11+
+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+
+-- Make sure stats and visibility map are up to date
+VACUUM ANALYZE hypo;
+
+-- Should not use hypothetical index
+
+-- Create normal index
+SELECT hypopg_create_index('CREATE INDEX ON hypo (id)');
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*';
+
+-- Remove all the hypothetical indexes
+SELECT hypopg_reset();
+
+-- Create INCLUDE index
+SELECT hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+
+-- Should use hypothetical index using an Index Only Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*';
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();

From f49d369bd686861414843521b8f55b567fb02d14 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 7 Feb 2021 21:55:02 +0800
Subject: [PATCH 12/22] Display hypothetical indexes on dropped tables in
 hypopg_list_indexes.

We could try to automatically remove such indexes by registering a relcache
callback, but this seems like an unlikely corner case so it doesn't seems worth
spending effort for that, accurately displaying the list of stored hypothetical
indexes should be enough.
---
 expected/hypopg.out | 8 ++++----
 hypopg--1.2.0.sql   | 9 +++++----
 test/sql/hypopg.sql | 2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/expected/hypopg.out b/expected/hypopg.out
index 09ce930..990cb46 100644
--- a/expected/hypopg.out
+++ b/expected/hypopg.out
@@ -26,10 +26,10 @@ WARNING:  hypopg: SQL order #3 is not a CREATE INDEX statement
   1
 (1 row)
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes;
- nspname | relname | amname 
----------+---------+--------
- public  | hypo    | btree
+SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes;
+ schema_name | table_name | am_name 
+-------------+------------+---------
+ public      | hypo       | btree
 (1 row)
 
 -- Should use hypothetical index
diff --git a/hypopg--1.2.0.sql b/hypopg--1.2.0.sql
index b763644..8462df6 100644
--- a/hypopg--1.2.0.sql
+++ b/hypopg--1.2.0.sql
@@ -42,11 +42,12 @@ AS '$libdir/hypopg', 'hypopg';
 
 CREATE VIEW hypopg_list_indexes
 AS
-    SELECT h.indexrelid, h.indexname, n.nspname, c.relname, am.amname
+    SELECT h.indexrelid, h.indexname AS index_name, n.nspname AS schema_name,
+    coalesce(c.relname, '<dropped>') AS table_name, am.amname AS am_name
     FROM hypopg() h
-    JOIN pg_catalog.pg_class c ON c.oid = h.indrelid
-    JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_catalog.pg_am am ON am.oid = h.amid;
+    LEFT JOIN pg_catalog.pg_class c ON c.oid = h.indrelid
+    LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+    LEFT JOIN pg_catalog.pg_am am ON am.oid = h.amid;
 
 CREATE FUNCTION
 hypopg_relation_size(IN indexid oid)
diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql
index b19e484..28e3fcb 100644
--- a/test/sql/hypopg.sql
+++ b/test/sql/hypopg.sql
@@ -25,7 +25,7 @@ ANALYZE hypo;
 SELECT COUNT(*) AS nb
 FROM public.hypopg_create_index('SELECT 1;CREATE INDEX ON hypo(id); SELECT 2');
 
-SELECT nspname, relname, amname FROM public.hypopg_list_indexes;
+SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes;
 
 -- Should use hypothetical index
 SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e

From b6ddfba44e6c1d53557155e3537afc4300943167 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 7 Feb 2021 22:17:39 +0800
Subject: [PATCH 13/22] Use GetNewOidWithIndex rather that GetNewRelFileNode.

HypoPG usually tries to stick to related upstream code as much as possible, and
index_create() does use GetNewRelFileNode(), but since hypothetical indexes
don't have a storage applying GetNewRelFileNode() heuristics seems like a way
to possibly waste ressources when creating indexes.  Also since hypothetical
indexes aren't stored in the catalogs, if the cluster is close to Oid
exhaustion it wouldn't avoid a possible conflict with a new object created
later anyway.
---
 hypopg.c | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/hypopg.c b/hypopg.c
index edbb529..82c1c6a 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -17,7 +17,13 @@
 #include "postgres.h"
 #include "fmgr.h"
 
+#if PG_VERSION_NUM < 120000
+#include "access/sysattr.h"
+#endif
 #include "access/transam.h"
+#if PG_VERSION_NUM < 140000
+#include "catalog/indexing.h"
+#endif
 #if PG_VERSION_NUM >= 110000
 #include "catalog/partition.h"
 #include "nodes/pg_list.h"
@@ -186,23 +192,24 @@ hypo_getNewOid(Oid relid)
 	{
 		Relation	pg_class;
 		Relation	relation;
-		Oid			reltablespace;
-		char		relpersistence;
 
 		/* Open the relation on which we want a new OID */
 		relation = table_open(relid, AccessShareLock);
 
-		reltablespace = relation->rd_rel->reltablespace;
-		relpersistence = relation->rd_rel->relpersistence;
-
 		/* Close the relation and release the lock now */
 		table_close(relation, AccessShareLock);
 
 		/* Open pg_class to aks a new OID */
 		pg_class = table_open(RelationRelationId, RowExclusiveLock);
 
-		/* ask for a new relfilenode */
-		newoid = GetNewRelFileNode(reltablespace, pg_class, relpersistence);
+		/* ask for a new Oid */
+		newoid = GetNewOidWithIndex(pg_class, ClassOidIndexId,
+#if PG_VERSION_NUM < 120000
+									ObjectIdAttributeNumber
+#else
+									Anum_pg_class_oid
+#endif
+									);
 
 		/* Close pg_class and release the lock now */
 		table_close(pg_class, RowExclusiveLock);

From 7e072c1f94e42a3659783bed21e494708ca05419 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 26 Feb 2021 14:59:55 +0800
Subject: [PATCH 14/22] Release 1.2.0

---
 CHANGELOG.md     | 21 +++++++++++++++++++++
 debian/changelog |  6 ++++++
 docs/conf.py     |  2 +-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64c97a8..55ede94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,27 @@
 Changelog
 =========
 
+2021-02-26 version 1.2.0:
+-------------------------
+
+  **New features**:
+
+  - Make hypopg work on standby servers using a new "fake" oid generator, that
+    borrows Oids in the FirstBootstrapObjectId / FirstNormalObjectId range
+    rather than real oids.  If necessary, the old behavior can still be used
+    with the new hypopg.use_real_oids configuration option.
+
+  **Bug fixes**
+
+  - Check if access methods support an INCLUDE clause to avoid creating invalid
+    hypothetical indexes.
+  - Display hypothetical indexes on dropped table in hypopg_list_indexes.
+
+  **Miscellaneous**
+
+  - Change hypopg_list_indexes() to view hypopg_list_indexes.
+  - Various documentation improvements.
+
 2020-06-24 version 1.1.4:
 -------------------------
 
diff --git a/debian/changelog b/debian/changelog
index 50d847d..58ded8a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+hypopg (1.2.0-1) unstable; urgency=medium
+
+  * New upstream version.
+
+ -- Julien Rouhaud <rjuju123@gmail.com>  Fri, 26 Feb 2021 14:51:06 +0800
+
 hypopg (1.1.4-1) unstable; urgency=medium
 
   * New upstream version with PG13 support.
diff --git a/docs/conf.py b/docs/conf.py
index 524d540..689ac91 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,7 +50,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'HypoPG'
-copyright = '2015-2020, Julien Rouhaud'
+copyright = '2015-2021, Julien Rouhaud'
 author = 'Julien Rouhaud'
 
 # The version info for the project you're documenting, acts as replacement for

From 8eb71b0dda1ec4b8c42f7a15348ab7fcf8529f8a Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 28 Feb 2021 17:02:49 +0800
Subject: [PATCH 15/22] Make new regression tests for INCLUDE stable.

The regression tests included the oid of the hypothetical index.  Since the
hypothetical Oid generator was introduced, the sequence of generated Oids is
somewhat stable for a given PostgreSQL major version, but is defenitely not
across multiple major versions.

(cherry picked from commit 203a0200464e7bbe55bc386eca6cb0c938bb48c1)
---
 expected/hypo_include.out | 18 ++++++++++--------
 test/sql/hypo_include.sql |  6 ++++--
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/expected/hypo_include.out b/expected/hypo_include.out
index 3febb3a..fff33b4 100644
--- a/expected/hypo_include.out
+++ b/expected/hypo_include.out
@@ -10,10 +10,11 @@ SELECT hypopg_reset();
 VACUUM ANALYZE hypo;
 -- Should not use hypothetical index
 -- Create normal index
-SELECT hypopg_create_index('CREATE INDEX ON hypo (id)');
-     hypopg_create_index      
-------------------------------
- (13719,<13719>btree_hypo_id)
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id)');
+ nb 
+----
+  1
 (1 row)
 
 -- Should use hypothetical index using a regular Index Scan
@@ -32,10 +33,11 @@ SELECT hypopg_reset();
 (1 row)
 
 -- Create INCLUDE index
-SELECT hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
-       hypopg_create_index        
-----------------------------------
- (13719,<13719>btree_hypo_id_val)
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+ nb 
+----
+  1
 (1 row)
 
 -- Should use hypothetical index using an Index Only Scan
diff --git a/test/sql/hypo_include.sql b/test/sql/hypo_include.sql
index 9c944b5..4ee483b 100644
--- a/test/sql/hypo_include.sql
+++ b/test/sql/hypo_include.sql
@@ -9,7 +9,8 @@ VACUUM ANALYZE hypo;
 -- Should not use hypothetical index
 
 -- Create normal index
-SELECT hypopg_create_index('CREATE INDEX ON hypo (id)');
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id)');
 
 -- Should use hypothetical index using a regular Index Scan
 SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
@@ -19,7 +20,8 @@ WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*';
 SELECT hypopg_reset();
 
 -- Create INCLUDE index
-SELECT hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)');
 
 -- Should use hypothetical index using an Index Only Scan
 SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e

From 49f5aff612cc43261bc0ade9d6d4b3ffadc08534 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 28 Feb 2021 14:51:00 +0800
Subject: [PATCH 16/22] Start working on 1.3.0.

---
 hypopg--1.2.0.sql => hypopg--1.3.0.sql | 0
 hypopg.control                         | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hypopg--1.2.0.sql => hypopg--1.3.0.sql (100%)

diff --git a/hypopg--1.2.0.sql b/hypopg--1.3.0.sql
similarity index 100%
rename from hypopg--1.2.0.sql
rename to hypopg--1.3.0.sql
diff --git a/hypopg.control b/hypopg.control
index 3a691f5..3d510e3 100644
--- a/hypopg.control
+++ b/hypopg.control
@@ -1,6 +1,6 @@
 # hypopg extension
 comment = 'Hypothetical indexes for PostgreSQL'
-default_version = '1.2.0'
+default_version = '1.3.0'
 module_pathname = '$libdir/hypopg'
 relocatable = true
 

From c93081f35e8949f96af8423d9d2121743c6c944d Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 7 Mar 2021 14:38:42 +0800
Subject: [PATCH 17/22] Add support for hypothetical hash indexes.

Only support such indexes for PostgreSQL 10+, as they were previously not crash
safe.
---
 Makefile                      |   4 ++
 expected/hypo_hash.out        |  31 +++++++++++
 hypopg_index.c                | 101 ++++++++++++++++++++++++++++++++++
 include/hypopg_import_index.h |  23 ++++++++
 test/sql/hypo_hash.sql        |  15 +++++
 5 files changed, 174 insertions(+)
 create mode 100644 expected/hypo_hash.out
 create mode 100644 test/sql/hypo_hash.sql

diff --git a/Makefile b/Makefile
index 79b7f0a..e245936 100644
--- a/Makefile
+++ b/Makefile
@@ -47,6 +47,10 @@ ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6 10))
 	REGRESS += hypo_index_part hypo_include
 endif
 
+ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6))
+	REGRESS += hypo_hash
+endif
+
 DEBUILD_ROOT = /tmp/$(EXTENSION)
 
 deb: release-zip
diff --git a/expected/hypo_hash.out b/expected/hypo_hash.out
new file mode 100644
index 0000000..6605100
--- /dev/null
+++ b/expected/hypo_hash.out
@@ -0,0 +1,31 @@
+-- hypothetical hash indexes, pg10+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+ hypopg_reset 
+--------------
+ 
+(1 row)
+
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)');
+ nb 
+----
+  1
+(1 row)
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*';
+ count 
+-------
+     1
+(1 row)
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();
+             hypopg_get_indexdef             
+---------------------------------------------
+ CREATE INDEX ON public.hypo USING hash (id)
+(1 row)
+
diff --git a/hypopg_index.c b/hypopg_index.c
index f03d4c7..0680585 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -251,6 +251,14 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns,
 #endif
 #if PG_VERSION_NUM >= 90600
 			&& entry->relam != BLOOM_AM_OID
+#endif
+#if PG_VERSION_NUM >= 100000
+			/*
+			 * Only support hash indexes for pg10+.  In previous version they
+			 * weren't crash safe, and changes in pg10+ also significantly
+			 * changed the disk space allocation.
+			 */
+			 && entry->relam != HASH_AM_OID
 #endif
 			)
 		{
@@ -1856,6 +1864,99 @@ hypo_estimate_index(hypoIndex * entry, RelOptInfo *rel)
 		entry->pages += (BlockNumber) ceil(
 										   ((double) entry->tuples * line_size) / usable_page_size);
 	}
+#endif
+#if PG_VERSION_NUM >= 100000
+	else if (entry->relam == HASH_AM_OID)
+	{
+		/* ----------------------------
+		 * From hash AM readme (src/backend/access/hash/README):
+		 *
+		 *   There are four kinds of pages in a hash index: the meta page (page
+		 *   zero), which contains statically allocated control information;
+		 *   primary bucket pages; overflow pages; and bitmap pages, which keep
+		 *   track of overflow pages that have been freed and are available for
+		 *   re-use.  For addressing purposes, bitmap pages are regarded as a
+		 *   subset of the overflow pages.
+		 * [...]
+		 *   A hash index consists of two or more "buckets", into which tuples
+		 *   are placed whenever their hash key maps to the bucket number.
+		 *   [...]
+		 *   Each bucket in the hash index comprises one or more index pages.
+		 *   The bucket's first page is permanently assigned to it when the
+		 *   bucket is created.  Additional pages, called "overflow pages", are
+		 *   added if the bucket receives too many tuples to fit in the primary
+		 *   bucket page.
+		 *
+		 * Hash AM also already provides some functions to compute an initial
+		 * number of buckets given the estimated number of tuples the index
+		 * will contains, which is a good enough estimate for hypothetical
+		 * index.
+		 *
+		 * The code below is simply an adaptation of original code to compute
+		 * the initial number of bucket, modified to cope with hypothetical
+		 * index, plus some naive estimates for the overflow and bitmap pages.
+		 *
+		 * For more details, refer to the original code, in:
+		 *   - _hash_init()
+		 *   - _hash_init_metabuffer()
+		 */
+		int32 data_width;
+		int32 item_width;
+		int32 ffactor;
+		double dnumbuckets;
+		uint32 num_buckets;
+		uint32 num_overflow;
+		uint32 num_bitmap;
+
+	/*
+	 * Determine the target fill factor (in tuples per bucket) for this index.
+	 * The idea is to make the fill factor correspond to pages about as full
+	 * as the user-settable fillfactor parameter says.  We can compute it
+	 * exactly since the index datatype (i.e. uint32 hash key) is fixed-width.
+	 */
+	data_width = sizeof(uint32);
+	item_width = MAXALIGN(sizeof(IndexTupleData)) + MAXALIGN(data_width) +
+		sizeof(ItemIdData);		/* include the line pointer */
+	ffactor = HypoHashGetTargetPageUsage(fillfactor) / item_width;
+	/* keep to a sane range */
+	if (ffactor < 10)
+		ffactor = 10;
+
+	/*
+	 * Choose the number of initial bucket pages to match the fill factor
+	 * given the estimated number of tuples.  We round up the result to the
+	 * total number of buckets which has to be allocated before using its
+	 * hashm_spares element. However always force at least 2 bucket pages. The
+	 * upper limit is determined by considerations explained in
+	 * _hash_expandtable().
+	 */
+	dnumbuckets = entry->tuples / ffactor;
+	if (dnumbuckets <= 2.0)
+		num_buckets = 2;
+	else if (dnumbuckets >= (double) 0x40000000)
+		num_buckets = 0x40000000;
+	else
+		num_buckets = _hash_get_totalbuckets(_hash_spareindex(dnumbuckets));
+
+	/*
+	 * Naive estimate of overflow pages, knowing that a page can store ffactor
+	 * tuples: we compute the number of tuples that wouldn't fit in the
+	 * previously computed number of buckets, and compute the number of pages
+	 * needed to store them.
+	 */
+	num_overflow = Max(0, ((entry->tuples - (num_buckets * ffactor)) /
+					   ffactor) + 1);
+
+	/*
+	 * Naive estimate of bitmap pages, using the previously computed  number of
+	 * overflow pages.
+	 */
+	num_bitmap = Max(1, num_overflow /
+					 pg_leftmost_one_pos32(HypoHashGetMaxBitmapSize()));
+
+	/* Simply add all computed pages, plus one extra block for the meta page */
+	entry->pages = num_buckets + num_overflow + num_bitmap + 1;
+	}
 #endif
 	else
 	{
diff --git a/include/hypopg_import_index.h b/include/hypopg_import_index.h
index 7704d44..9af3d2f 100644
--- a/include/hypopg_import_index.h
+++ b/include/hypopg_import_index.h
@@ -19,6 +19,29 @@
 				MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \
 				MAXALIGN(sizeof(BTPageOpaqueData))) / 3)
 
+#if PG_VERSION_NUM >= 100000
+#include "access/hash.h"
+
+/* adapted from src/include/access/hash.h */
+#define HypoHashGetFillFactor(ffactor) \
+	(((fillfactor) == 0) ? HASH_DEFAULT_FILLFACTOR : (ffactor))
+
+#define HypoHashGetTargetPageUsage(ffactor) \
+	(BLCKSZ * HypoHashGetFillFactor(ffactor) / 100)
+
+#define HypoHashGetMaxBitmapSize() \
+	(BLCKSZ - \
+	 (MAXALIGN(SizeOfPageHeaderData) + MAXALIGN(sizeof(HashPageOpaqueData))))
+
+#define HypoHashMaxItemSize() \
+	MAXALIGN_DOWN(BLCKSZ - \
+				  SizeOfPageHeaderData - \
+				  sizeof(ItemIdData) - \
+				  MAXALIGN(sizeof(HashPageOpaqueData)))
+
+#endif
+
+
 extern List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index,
 				  Relation heapRelation);
 #if PG_VERSION_NUM < 100000
diff --git a/test/sql/hypo_hash.sql b/test/sql/hypo_hash.sql
new file mode 100644
index 0000000..3288bed
--- /dev/null
+++ b/test/sql/hypo_hash.sql
@@ -0,0 +1,15 @@
+-- hypothetical hash indexes, pg10+
+
+-- Remove all the hypothetical indexes if any
+SELECT hypopg_reset();
+
+-- Create normal index
+SELECT COUNT(*) AS NB
+FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)');
+
+-- Should use hypothetical index using a regular Index Scan
+SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e
+WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*';
+
+-- Deparse the index DDL
+SELECT hypopg_get_indexdef(indexrelid) FROM hypopg();

From a4acac04d125cae2eb8f2358dd196dbe029a813e Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Wed, 24 Mar 2021 16:27:40 +0800
Subject: [PATCH 18/22] Fix hash size estimation compatbility for PG11 and
 below.

Thanks to github user nikhil-postgres for the report.

While at it also fix a thinko in the size estimation which lead to greatly
overestimated hypothetical hash index sizes.
---
 CONTRIBUTORS.md |  1 +
 hypopg_index.c  | 20 +++++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 1da12f9..7c96ed9 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -16,3 +16,4 @@ People who contributed to hypopg:
   * Extortioner01
   * nagaraju11
   * ibrahim edib kokdemir
+  * github user nikhil-postgres
diff --git a/hypopg_index.c b/hypopg_index.c
index 0680585..dbf445d 100644
--- a/hypopg_index.c
+++ b/hypopg_index.c
@@ -57,6 +57,9 @@
 #endif
 #include "parser/parse_utilcmd.h"
 #include "parser/parser.h"
+#if PG_VERSION_NUM >= 120000
+#include "port/pg_bitutils.h"
+#endif
 #include "storage/bufmgr.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
@@ -1907,6 +1910,7 @@ hypo_estimate_index(hypoIndex * entry, RelOptInfo *rel)
 		uint32 num_buckets;
 		uint32 num_overflow;
 		uint32 num_bitmap;
+		uint32 lshift;
 
 	/*
 	 * Determine the target fill factor (in tuples per bucket) for this index.
@@ -1947,12 +1951,22 @@ hypo_estimate_index(hypoIndex * entry, RelOptInfo *rel)
 	num_overflow = Max(0, ((entry->tuples - (num_buckets * ffactor)) /
 					   ffactor) + 1);
 
+	/* find largest bitmap array size that will fit in page size */
+#if PG_VERSION_NUM >= 120000
+	lshift = pg_leftmost_one_pos32(HypoHashGetMaxBitmapSize());
+#else
+	for (lshift = _hash_log2(HypoHashGetMaxBitmapSize()); lshift > 0; --lshift)
+	{
+		if ((1 << lshift) <= HypoHashGetMaxBitmapSize())
+			break;
+	}
+#endif
+
 	/*
-	 * Naive estimate of bitmap pages, using the previously computed  number of
+	 * Naive estimate of bitmap pages, using the previously computed number of
 	 * overflow pages.
 	 */
-	num_bitmap = Max(1, num_overflow /
-					 pg_leftmost_one_pos32(HypoHashGetMaxBitmapSize()));
+	num_bitmap = Max(1, num_overflow / (1 <<lshift));
 
 	/* Simply add all computed pages, plus one extra block for the meta page */
 	entry->pages = num_buckets + num_overflow + num_bitmap + 1;

From 29a669d8648b8948629f08fe49a1c1acd3431e93 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Fri, 4 Jun 2021 15:39:12 +0800
Subject: [PATCH 19/22] Release 1.3.0

---
 CHANGELOG.md   | 7 +++++++
 docs/usage.rst | 1 +
 2 files changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55ede94..7cc39bd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 Changelog
 =========
 
+2021-06-04 version 1.3.0:
+-------------------------
+
+  **New features**:
+
+  - Add support for hypothetical hash indexes (pg10+)
+
 2021-02-26 version 1.2.0:
 -------------------------
 
diff --git a/docs/usage.rst b/docs/usage.rst
index 0ab3e6b..d3fcbef 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -88,6 +88,7 @@ The following access methods are supported:
 
 - btree
 - brin
+- hash (requires PostgreSQL 10 or above)
 - bloom (requires the bloom extension to be installed)
 
 Create a hypothetical index

From 2b54e44407ac8017be098b5626249b498b5d73d4 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sat, 19 Jun 2021 11:56:53 +0800
Subject: [PATCH 20/22] Start working on version 1.3.1

---
 hypopg--1.3.0.sql => hypopg--1.3.1.sql | 0
 hypopg.control                         | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hypopg--1.3.0.sql => hypopg--1.3.1.sql (100%)

diff --git a/hypopg--1.3.0.sql b/hypopg--1.3.1.sql
similarity index 100%
rename from hypopg--1.3.0.sql
rename to hypopg--1.3.1.sql
diff --git a/hypopg.control b/hypopg.control
index 3d510e3..7de312a 100644
--- a/hypopg.control
+++ b/hypopg.control
@@ -1,6 +1,6 @@
 # hypopg extension
 comment = 'Hypothetical indexes for PostgreSQL'
-default_version = '1.3.0'
+default_version = '1.3.1'
 module_pathname = '$libdir/hypopg'
 relocatable = true
 

From 9ad419e1406c5d939410804349d01b3b9d58113e Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sat, 19 Jun 2021 11:57:17 +0800
Subject: [PATCH 21/22] Fix compatibility with postgres 14b2

processUtility has a new readOnlyTree parameter since 7c337b6b527.
---
 hypopg.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/hypopg.c b/hypopg.c
index 82c1c6a..872d24c 100644
--- a/hypopg.c
+++ b/hypopg.c
@@ -69,6 +69,9 @@ static void
 							  Node *parsetree,
 #endif
 							  const char *queryString,
+#if PG_VERSION_NUM >= 140000
+							  bool readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 							  ProcessUtilityContext context,
 #endif
@@ -291,6 +294,9 @@ hypo_utility_hook(
 				  Node *parsetree,
 #endif
 				  const char *queryString,
+#if PG_VERSION_NUM >= 140000
+				  bool readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 				  ProcessUtilityContext context,
 #endif
@@ -326,6 +332,9 @@ hypo_utility_hook(
 						  parsetree,
 #endif
 						  queryString,
+#if PG_VERSION_NUM >= 140000
+						  readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 						  context,
 #endif
@@ -351,6 +360,9 @@ hypo_utility_hook(
 								parsetree,
 #endif
 								queryString,
+#if PG_VERSION_NUM >= 140000
+								readOnlyTree,
+#endif
 #if PG_VERSION_NUM >= 90300
 								context,
 #endif

From 57d711bc4e37164c8edac81580a5f477e2a33d86 Mon Sep 17 00:00:00 2001
From: Julien Rouhaud <julien.rouhaud@free.fr>
Date: Sun, 20 Jun 2021 21:25:07 +0800
Subject: [PATCH 22/22] Release 1.3.1

---
 CHANGELOG.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7cc39bd..34b7fd3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 Changelog
 =========
 
+2021-06-21 version 1.3.1:
+-------------------------
+
+  **Miscellaneous**:
+
+  - Fix compatibility with PostgreSQL 14 beta 2
+
 2021-06-04 version 1.3.0:
 -------------------------