diff --git a/CHANGELOG.md b/CHANGELOG.md index 64c97a8..34b7fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,41 @@ Changelog ========= +2021-06-21 version 1.3.1: +------------------------- + + **Miscellaneous**: + + - Fix compatibility with PostgreSQL 14 beta 2 + +2021-06-04 version 1.3.0: +------------------------- + + **New features**: + + - Add support for hypothetical hash indexes (pg10+) + +2021-02-26 version 1.2.0: +------------------------- + + **New features**: + + - Make hypopg work on standby servers using a new "fake" oid generator, that + borrows Oids in the FirstBootstrapObjectId / FirstNormalObjectId range + rather than real oids. If necessary, the old behavior can still be used + with the new hypopg.use_real_oids configuration option. + + **Bug fixes** + + - Check if access methods support an INCLUDE clause to avoid creating invalid + hypothetical indexes. + - Display hypothetical indexes on dropped table in hypopg_list_indexes. + + **Miscellaneous** + + - Change hypopg_list_indexes() to view hypopg_list_indexes. + - Various documentation improvements. + 2020-06-24 version 1.1.4: ------------------------- diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d6f28d9..7c96ed9 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -15,3 +15,5 @@ People who contributed to hypopg: * Jan Koßmann * Extortioner01 * nagaraju11 + * ibrahim edib kokdemir + * github user nikhil-postgres diff --git a/LICENSE b/LICENSE index 442100f..1ed145e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group +Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/Makefile b/Makefile index ecae2e1..e245936 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,11 @@ ifeq ($(MAJORVERSION),10) endif ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6 10)) - REGRESS += hypo_index_part + REGRESS += hypo_index_part hypo_include +endif + +ifneq ($(MAJORVERSION),$(filter $(MAJORVERSION), 9.2 9.3 9.4 9.5 9.6)) + REGRESS += hypo_hash endif DEBUILD_ROOT = /tmp/$(EXTENSION) diff --git a/README.md b/README.md index 10807b6..d4f153a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,14 @@ Installation - `sudo make install` - In every needed database: `CREATE EXTENSION hypopg;` + +Updating the extension +---------------------- + +Note that hypopg doesn't provide extension upgrade scripts, as there's no +data saved in any of the objects created. Therefore, you need to first drop +the extension then create it again to get the new version. + Usage ----- diff --git a/debian/changelog b/debian/changelog index 16d03c7..40938de 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +hypopg (1.2.0-1) unstable; urgency=medium + + * New upstream version. + + -- Julien Rouhaud Fri, 26 Feb 2021 14:51:06 +0800 + hypopg (1.1.4-2) unstable; urgency=medium * Team upload for PostgreSQL 13. diff --git a/debian/copyright b/debian/copyright index dc9cfee..a978fb4 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -Portions Copyright (c) 2015-2018, PostgreSQL GLobal Development Group +Portions Copyright (c) 2015-2021, PostgreSQL GLobal Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/docs/conf.py b/docs/conf.py index a9327e0..689ac91 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ master_doc = 'index' # General information about the project. project = 'HypoPG' -copyright = '2015-2018, Julien Rouhaud' +copyright = '2015-2021, Julien Rouhaud' author = 'Julien Rouhaud' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/index.rst b/docs/index.rst index e9683e9..2f64e82 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ It's compatible with **PostgreSQL 9.2 and above**. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Contents: hypothetical_indexes diff --git a/docs/usage.rst b/docs/usage.rst index e759300..d3fcbef 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -48,6 +48,49 @@ As you can see, hypopg version 1.1.0 is installed. If you need to check using plain SQL, please refer to the `pg_extension table documentation `_. +Configuration +------------- + +The following configuration parameters (GUCs) are available, and can be changed +interactively: + +hypopg.enabled: + Default to ``on``. + Use this parameter to globally enable or disable HypoPG. When HypoPG is + disabled, no hypothetical index will be used, but the defined hypothetical + indexes won't be removed. + +hypopg.use_real_oids: + Default to ``off``. + By default, HypoPG won't use "real" object identifiers, but instead borrow + ones from the ~ 14000 / 16384 (respectively the lowest unused oid less then + FirstNormalObjectId and FirstNormalObjectId) range, which are reserved by + PostgreSQL for future usage in future releases. This doesn't cause any + problem, as the free range is dynamically computed the first time a + connection uses HypoPG, and has the advantage to work on a standby server. + But the drawback is that you can't have more than approximately 2500 + hypothetical indexes at the same time, and creating a new hypothetical index + will become very slow once more than the maximum number of objects has been + created until ``hypopg_reset()`` is called. + + If those drawbacks are problematic, you can enable this parameter. HypoPG + will then ask for a real object identifier, which will need to obtain more + locks and won't work on a standby, but will allow to use the full range of + object identifiers. + + Note that switching this parameter doesn't require to reset the entries, both + can coexist at the same time. + +Supported access methods +------------------------ + +The following access methods are supported: + +- btree +- brin +- hash (requires PostgreSQL 10 or above) +- bloom (requires the bloom extension to be installed) + Create a hypothetical index --------------------------- diff --git a/expected/hypo_hash.out b/expected/hypo_hash.out new file mode 100644 index 0000000..6605100 --- /dev/null +++ b/expected/hypo_hash.out @@ -0,0 +1,31 @@ +-- hypothetical hash indexes, pg10+ +-- Remove all the hypothetical indexes if any +SELECT hypopg_reset(); + hypopg_reset +-------------- + +(1 row) + +-- Create normal index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)'); + nb +---- + 1 +(1 row) + +-- Should use hypothetical index using a regular Index Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*'; + count +------- + 1 +(1 row) + +-- Deparse the index DDL +SELECT hypopg_get_indexdef(indexrelid) FROM hypopg(); + hypopg_get_indexdef +--------------------------------------------- + CREATE INDEX ON public.hypo USING hash (id) +(1 row) + diff --git a/expected/hypo_include.out b/expected/hypo_include.out new file mode 100644 index 0000000..fff33b4 --- /dev/null +++ b/expected/hypo_include.out @@ -0,0 +1,57 @@ +-- hypothetical indexes using INCLUDE keyword, pg11+ +-- Remove all the hypothetical indexes if any +SELECT hypopg_reset(); + hypopg_reset +-------------- + +(1 row) + +-- Make sure stats and visibility map are up to date +VACUUM ANALYZE hypo; +-- Should not use hypothetical index +-- Create normal index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo (id)'); + nb +---- + 1 +(1 row) + +-- Should use hypothetical index using a regular Index Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*'; + count +------- + 1 +(1 row) + +-- Remove all the hypothetical indexes +SELECT hypopg_reset(); + hypopg_reset +-------------- + +(1 row) + +-- Create INCLUDE index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)'); + nb +---- + 1 +(1 row) + +-- Should use hypothetical index using an Index Only Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*'; + count +------- + 1 +(1 row) + +-- Deparse the index DDL +SELECT hypopg_get_indexdef(indexrelid) FROM hypopg(); + hypopg_get_indexdef +------------------------------------------------------------ + CREATE INDEX ON public.hypo USING btree (id) INCLUDE (val) +(1 row) + diff --git a/expected/hypopg.out b/expected/hypopg.out index e51e948..990cb46 100644 --- a/expected/hypopg.out +++ b/expected/hypopg.out @@ -26,10 +26,10 @@ WARNING: hypopg: SQL order #3 is not a CREATE INDEX statement 1 (1 row) -SELECT nspname, relname, amname FROM public.hypopg_list_indexes(); - nspname | relname | amname ----------+---------+-------- - public | hypo | btree +SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes; + schema_name | table_name | am_name +-------------+------------+--------- + public | hypo | btree (1 row) -- Should use hypothetical index @@ -162,3 +162,29 @@ SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on CREATE INDEX ON public.hypo USING btree (id DESC, id DESC, id DESC NULLS LAST, ((md5(val))::bpchar) bpchar_pattern_ops) WITH (fillfactor = 10) WHERE ((id < 1000) AND ((id + (1 % 2)) = 3)) (1 row) +-- Make sure the old Oid generator still works. Test it while keeping existing +-- entries, as both should be able to coexist. +SET hypopg.use_real_oids = on; +-- Should not use hypothetical index +SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e +WHERE e ~ 'Index.*<\d+>btree_hypo.*'; + count +------- + 0 +(1 row) + +SELECT COUNT(*) AS nb +FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);'); + nb +---- + 1 +(1 row) + +-- Should use hypothetical index +SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e +WHERE e ~ 'Index.*<\d+>btree_hypo.*'; + count +------- + 1 +(1 row) + diff --git a/hypopg--1.1.4.sql b/hypopg--1.3.1.sql similarity index 80% rename from hypopg--1.1.4.sql rename to hypopg--1.3.1.sql index 49f276d..8462df6 100644 --- a/hypopg--1.1.4.sql +++ b/hypopg--1.3.1.sql @@ -1,7 +1,7 @@ -- This program is open source, licensed under the PostgreSQL License. -- For license terms, see the LICENSE file. -- --- Copyright (C) 2015-2018: Julien Rouhaud +-- Copyright (C) 2015-2021: Julien Rouhaud -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION hypopg" to load this file. \quit @@ -40,17 +40,14 @@ CREATE FUNCTION hypopg(OUT indexname text, OUT indexrelid oid, LANGUAGE c COST 100 AS '$libdir/hypopg', 'hypopg'; -CREATE FUNCTION hypopg_list_indexes(OUT indexrelid oid, OUT indexname text, OUT nspname name, OUT relname name, OUT amname name) - RETURNS SETOF record +CREATE VIEW hypopg_list_indexes AS -$_$ - SELECT h.indexrelid, h.indexname, n.nspname, c.relname, am.amname + SELECT h.indexrelid, h.indexname AS index_name, n.nspname AS schema_name, + coalesce(c.relname, '') AS table_name, am.amname AS am_name FROM hypopg() h - JOIN pg_class c ON c.oid = h.indrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN pg_am am ON am.oid = h.amid -$_$ -LANGUAGE sql; + LEFT JOIN pg_catalog.pg_class c ON c.oid = h.indrelid + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_am am ON am.oid = h.amid; CREATE FUNCTION hypopg_relation_size(IN indexid oid) diff --git a/hypopg.c b/hypopg.c index 24c2ab6..872d24c 100644 --- a/hypopg.c +++ b/hypopg.c @@ -8,7 +8,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (C) 2015-2018: Julien Rouhaud + * Copyright (C) 2015-2021: Julien Rouhaud * *------------------------------------------------------------------------- */ @@ -17,11 +17,21 @@ #include "postgres.h" #include "fmgr.h" +#if PG_VERSION_NUM < 120000 +#include "access/sysattr.h" +#endif +#include "access/transam.h" +#if PG_VERSION_NUM < 140000 +#include "catalog/indexing.h" +#endif #if PG_VERSION_NUM >= 110000 #include "catalog/partition.h" #include "nodes/pg_list.h" #include "utils/lsyscache.h" #endif +#include "executor/spi.h" +#include "miscadmin.h" +#include "utils/elog.h" #include "include/hypopg.h" #include "include/hypopg_import.h" @@ -33,8 +43,15 @@ PG_MODULE_MAGIC; bool isExplain; bool hypo_is_enabled; +bool hypo_use_real_oids; MemoryContext HypoMemoryContext; +/*--- Private variables ---*/ + +static Oid last_oid = InvalidOid; +static Oid min_fake_oid = InvalidOid; +static bool oid_wraparound = false; + /*--- Functions --- */ PGDLLEXPORT void _PG_init(void); @@ -52,6 +69,9 @@ static void Node *parsetree, #endif const char *queryString, +#if PG_VERSION_NUM >= 140000 + bool readOnlyTree, +#endif #if PG_VERSION_NUM >= 90300 ProcessUtilityContext context, #endif @@ -75,6 +95,7 @@ static void hypo_executorEnd_hook(QueryDesc *queryDesc); static ExecutorEnd_hook_type prev_ExecutorEnd_hook = NULL; +static Oid hypo_get_min_fake_oid(void); static void hypo_get_relation_info_hook(PlannerInfo *root, Oid relationObjectId, bool inhparent, @@ -125,6 +146,18 @@ _PG_init(void) NULL, NULL); + DefineCustomBoolVariable("hypopg.use_real_oids", + "Use real oids rather than the range < 16384", + NULL, + &hypo_use_real_oids, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + EmitWarningsOnPlaceholders("hypopg"); } void @@ -139,39 +172,117 @@ _PG_fini(void) } /*--------------------------------- - * Wrapper around GetNewRelFileNode * Return a new OID for an hypothetical index. + * + * To avoid locking on pg_class (required to safely call GetNewOidWithIndex or + * similar) and to be usable on a standby node, use the oids unused in the + * FirstBootstrapObjectId / FirstNormalObjectId range rather than real oids. + * For performance, always start with the biggest oid lesser than + * FirstNormalObjectId. This way the loop to find an unused oid will only + * happens once a single backend has created more than ~2.5k hypothetical + * indexes. + * + * For people needing to have thousands of hypothetical indexes at the same + * time, we also allow to use the initial implementation that relies on real + * oids, which comes with all the limitations mentioned above. */ Oid hypo_getNewOid(Oid relid) { - Relation pg_class; - Relation relation; - Oid newoid; - Oid reltablespace; - char relpersistence; + Oid newoid = InvalidOid; - /* Open the relation on which we want a new OID */ - relation = table_open(relid, AccessShareLock); + if (hypo_use_real_oids) + { + Relation pg_class; + Relation relation; - reltablespace = relation->rd_rel->reltablespace; - relpersistence = relation->rd_rel->relpersistence; + /* Open the relation on which we want a new OID */ + relation = table_open(relid, AccessShareLock); - /* Close the relation and release the lock now */ - table_close(relation, AccessShareLock); + /* Close the relation and release the lock now */ + table_close(relation, AccessShareLock); - /* Open pg_class to aks a new OID */ - pg_class = table_open(RelationRelationId, RowExclusiveLock); + /* Open pg_class to aks a new OID */ + pg_class = table_open(RelationRelationId, RowExclusiveLock); - /* ask for a new relfilenode */ - newoid = GetNewRelFileNode(reltablespace, pg_class, relpersistence); + /* ask for a new Oid */ + newoid = GetNewOidWithIndex(pg_class, ClassOidIndexId, +#if PG_VERSION_NUM < 120000 + ObjectIdAttributeNumber +#else + Anum_pg_class_oid +#endif + ); - /* Close pg_class and release the lock now */ - table_close(pg_class, RowExclusiveLock); + /* Close pg_class and release the lock now */ + table_close(pg_class, RowExclusiveLock); + } + else + { + /* + * First, make sure we know what is the biggest oid smaller than + * FirstNormalObjectId present in pg_class. This can never change so + * we cache the value. + */ + if (!OidIsValid(min_fake_oid)) + min_fake_oid = hypo_get_min_fake_oid(); + Assert(OidIsValid(min_fake_oid)); + + /* Make sure there's enough room to get one more Oid */ + if (list_length(hypoIndexes) >= (FirstNormalObjectId - min_fake_oid)) + { + ereport(ERROR, + (errmsg("hypopg: not more oid available"), + errhint("Remove hypothetical indexes " + "or enable hypopg.use_real_oids"))); + } + + while(!OidIsValid(newoid)) + { + CHECK_FOR_INTERRUPTS(); + + if (!OidIsValid(last_oid)) + newoid = last_oid = min_fake_oid; + else + newoid = ++last_oid; + + /* Check if we just exceeded the fake oids range */ + if (newoid >= FirstNormalObjectId) + { + newoid = min_fake_oid; + last_oid = InvalidOid; + oid_wraparound = true; + } + + /* + * If we already used all available fake oids, we have to make sure + * that the oid isn't used anymore. + */ + if (oid_wraparound) + { + if (hypo_get_index(newoid) != NULL) + { + /* We can't use this oid. Reset newoid and start again */ + newoid = InvalidOid; + } + } + } + } + + Assert(OidIsValid(newoid)); return newoid; } +/* Reset the state of the fake oid generator. */ +void +hypo_reset_fake_oids(void) +{ + Assert(hypoIndexes == NIL); + last_oid = InvalidOid; + oid_wraparound = false; +} + /* This function setup the "isExplain" flag for next hooks. * If this flag is setup, we can add hypothetical indexes. */ @@ -183,6 +294,9 @@ hypo_utility_hook( Node *parsetree, #endif const char *queryString, +#if PG_VERSION_NUM >= 140000 + bool readOnlyTree, +#endif #if PG_VERSION_NUM >= 90300 ProcessUtilityContext context, #endif @@ -218,6 +332,9 @@ hypo_utility_hook( parsetree, #endif queryString, +#if PG_VERSION_NUM >= 140000 + readOnlyTree, +#endif #if PG_VERSION_NUM >= 90300 context, #endif @@ -243,6 +360,9 @@ hypo_utility_hook( parsetree, #endif queryString, +#if PG_VERSION_NUM >= 140000 + readOnlyTree, +#endif #if PG_VERSION_NUM >= 90300 context, #endif @@ -341,6 +461,46 @@ hypo_executorEnd_hook(QueryDesc *queryDesc) standard_ExecutorEnd(queryDesc); } +/* + * Return the minmum usable oid in the FirstBootstrapObjectId - + * FirstNormalObjectId range. + */ +static Oid +hypo_get_min_fake_oid(void) +{ + int ret, nb; + Oid oid = InvalidOid; + + /* + * Connect to SPI manager + */ + if ((ret = SPI_connect()) < 0) + /* internal error */ + elog(ERROR, "SPI connect failure - returned %d", ret); + + ret = SPI_execute("SELECT max(oid)" + " FROM pg_catalog.pg_class" + " WHERE oid < " CppAsString2(FirstNormalObjectId), + true, 1); + nb = SPI_processed; + + if (ret != SPI_OK_SELECT || nb == 0) + { + SPI_finish(); + elog(ERROR, "hypopg: could not find the minimum fake oid"); + } + + oid = atooid(SPI_getvalue(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, + 1)) + 1; + + /* release SPI related resources (and return to caller's context) */ + SPI_finish(); + + Assert(OidIsValid(oid)); + return oid; +} + /* * This function will execute the "hypo_injectHypotheticalIndex" for every * hypothetical index found for each relation if the isExplain flag is setup. diff --git a/hypopg.control b/hypopg.control index f20af9b..7de312a 100644 --- a/hypopg.control +++ b/hypopg.control @@ -1,6 +1,6 @@ # hypopg extension comment = 'Hypothetical indexes for PostgreSQL' -default_version = '1.1.4' +default_version = '1.3.1' module_pathname = '$libdir/hypopg' relocatable = true diff --git a/hypopg_index.c b/hypopg_index.c index 6c3fe69..dbf445d 100644 --- a/hypopg_index.c +++ b/hypopg_index.c @@ -8,7 +8,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (C) 2015-2018: Julien Rouhaud + * Copyright (C) 2015-2021: Julien Rouhaud * *------------------------------------------------------------------------- */ @@ -57,6 +57,9 @@ #endif #include "parser/parse_utilcmd.h" #include "parser/parser.h" +#if PG_VERSION_NUM >= 120000 +#include "port/pg_bitutils.h" +#endif #include "storage/bufmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -173,6 +176,7 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns, entry->amcanorder = amroutine->amcanorder; #if PG_VERSION_NUM >= 110000 entry->amcanparallel = amroutine->amcanparallel; + entry->amcaninclude = amroutine->amcaninclude; #endif #else /* Up to 9.5, all information is available in the pg_am tuple */ @@ -250,6 +254,14 @@ hypo_newIndex(Oid relid, char *accessMethod, int nkeycolumns, int ninccolumns, #endif #if PG_VERSION_NUM >= 90600 && entry->relam != BLOOM_AM_OID +#endif +#if PG_VERSION_NUM >= 100000 + /* + * Only support hash indexes for pg10+. In previous version they + * weren't crash safe, and changes in pg10+ also significantly + * changed the disk space allocation. + */ + && entry->relam != HASH_AM_OID #endif ) { @@ -311,6 +323,9 @@ hypo_index_reset(void) list_free(hypoIndexes); hypoIndexes = NIL; + + hypo_reset_fake_oids(); + return; } @@ -440,6 +455,14 @@ hypo_index_store_parsetree(IndexStmt *node, const char *queryString) errmsg("hypopg: access method \"%s\" does not support multicolumn indexes", node->accessMethod))); +#if PG_VERSION_NUM >= 110000 + if (node-> indexIncludingParams != NIL && !entry->amcaninclude) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("hypopg: access method \"%s\" does not support included columns", + node->accessMethod))); +#endif + entry->unique = node->unique; entry->ncolumns = nkeycolumns + ninccolumns; entry->nkeycolumns = nkeycolumns; @@ -1116,36 +1139,41 @@ hypo_injectHypotheticalIndex(PlannerInfo *root, rel->indexlist = lcons(index, rel->indexlist); } -/* Return the hypothetical index name is indexId is ours, NULL otherwise, as +/* + * Return the stored hypothetical index for a given oid if any, NULL otherwise + */ +hypoIndex * +hypo_get_index(Oid indexId) +{ + ListCell *lc; + + foreach(lc, hypoIndexes) + { + hypoIndex *entry = (hypoIndex *) lfirst(lc); + + if (entry->oid == indexId) + return entry; + } + + return NULL; +} + +/* Return the hypothetical index name ifs indexId is ours, NULL otherwise, as * this is what explain_get_index_name expects to continue his job. */ const char * hypo_explain_get_index_name_hook(Oid indexId) { - char *ret = NULL; - if (isExplain) { - /* - * we're in an explain-only command. Return the name of the - * hypothetical index name if it's one of ours, otherwise return NULL - */ - ListCell *lc; + hypoIndex *index = NULL; - foreach(lc, hypoIndexes) - { - hypoIndex *entry = (hypoIndex *) lfirst(lc); + index = hypo_get_index(indexId); - if (entry->oid == indexId) - { - ret = entry->indexname; - } - } + if (index) + return index->indexname; } - if (ret) - return ret; - if (prev_explain_get_index_name_hook) return prev_explain_get_index_name_hook(indexId); @@ -1839,6 +1867,110 @@ hypo_estimate_index(hypoIndex * entry, RelOptInfo *rel) entry->pages += (BlockNumber) ceil( ((double) entry->tuples * line_size) / usable_page_size); } +#endif +#if PG_VERSION_NUM >= 100000 + else if (entry->relam == HASH_AM_OID) + { + /* ---------------------------- + * From hash AM readme (src/backend/access/hash/README): + * + * There are four kinds of pages in a hash index: the meta page (page + * zero), which contains statically allocated control information; + * primary bucket pages; overflow pages; and bitmap pages, which keep + * track of overflow pages that have been freed and are available for + * re-use. For addressing purposes, bitmap pages are regarded as a + * subset of the overflow pages. + * [...] + * A hash index consists of two or more "buckets", into which tuples + * are placed whenever their hash key maps to the bucket number. + * [...] + * Each bucket in the hash index comprises one or more index pages. + * The bucket's first page is permanently assigned to it when the + * bucket is created. Additional pages, called "overflow pages", are + * added if the bucket receives too many tuples to fit in the primary + * bucket page. + * + * Hash AM also already provides some functions to compute an initial + * number of buckets given the estimated number of tuples the index + * will contains, which is a good enough estimate for hypothetical + * index. + * + * The code below is simply an adaptation of original code to compute + * the initial number of bucket, modified to cope with hypothetical + * index, plus some naive estimates for the overflow and bitmap pages. + * + * For more details, refer to the original code, in: + * - _hash_init() + * - _hash_init_metabuffer() + */ + int32 data_width; + int32 item_width; + int32 ffactor; + double dnumbuckets; + uint32 num_buckets; + uint32 num_overflow; + uint32 num_bitmap; + uint32 lshift; + + /* + * Determine the target fill factor (in tuples per bucket) for this index. + * The idea is to make the fill factor correspond to pages about as full + * as the user-settable fillfactor parameter says. We can compute it + * exactly since the index datatype (i.e. uint32 hash key) is fixed-width. + */ + data_width = sizeof(uint32); + item_width = MAXALIGN(sizeof(IndexTupleData)) + MAXALIGN(data_width) + + sizeof(ItemIdData); /* include the line pointer */ + ffactor = HypoHashGetTargetPageUsage(fillfactor) / item_width; + /* keep to a sane range */ + if (ffactor < 10) + ffactor = 10; + + /* + * Choose the number of initial bucket pages to match the fill factor + * given the estimated number of tuples. We round up the result to the + * total number of buckets which has to be allocated before using its + * hashm_spares element. However always force at least 2 bucket pages. The + * upper limit is determined by considerations explained in + * _hash_expandtable(). + */ + dnumbuckets = entry->tuples / ffactor; + if (dnumbuckets <= 2.0) + num_buckets = 2; + else if (dnumbuckets >= (double) 0x40000000) + num_buckets = 0x40000000; + else + num_buckets = _hash_get_totalbuckets(_hash_spareindex(dnumbuckets)); + + /* + * Naive estimate of overflow pages, knowing that a page can store ffactor + * tuples: we compute the number of tuples that wouldn't fit in the + * previously computed number of buckets, and compute the number of pages + * needed to store them. + */ + num_overflow = Max(0, ((entry->tuples - (num_buckets * ffactor)) / + ffactor) + 1); + + /* find largest bitmap array size that will fit in page size */ +#if PG_VERSION_NUM >= 120000 + lshift = pg_leftmost_one_pos32(HypoHashGetMaxBitmapSize()); +#else + for (lshift = _hash_log2(HypoHashGetMaxBitmapSize()); lshift > 0; --lshift) + { + if ((1 << lshift) <= HypoHashGetMaxBitmapSize()) + break; + } +#endif + + /* + * Naive estimate of bitmap pages, using the previously computed number of + * overflow pages. + */ + num_bitmap = Max(1, num_overflow / (1 <pages = num_buckets + num_overflow + num_bitmap + 1; + } #endif else { diff --git a/import/hypopg_import.c b/import/hypopg_import.c index 4b03663..d21dfa0 100644 --- a/import/hypopg_import.c +++ b/import/hypopg_import.c @@ -5,7 +5,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (c) 2008-2018, PostgreSQL Global Development Group + * Copyright (c) 2008-2021, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ diff --git a/import/hypopg_import_index.c b/import/hypopg_import_index.c index e6b6a2c..22e2902 100644 --- a/import/hypopg_import_index.c +++ b/import/hypopg_import_index.c @@ -6,7 +6,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (c) 2008-2018, PostgreSQL Global Development Group + * Copyright (c) 2008-2021, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ diff --git a/include/hypopg.h b/include/hypopg.h index 9043264..0d47093 100644 --- a/include/hypopg.h +++ b/include/hypopg.h @@ -5,7 +5,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (C) 2015-2018: Julien Rouhaud + * Copyright (C) 2015-2021: Julien Rouhaud * *------------------------------------------------------------------------- */ @@ -40,6 +40,11 @@ #define LNEXT2(list, lc) ((lc)->next) #endif +/* Backport of atooid macro */ +#if PG_VERSION_NUM < 100000 +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) +#endif + extern bool isExplain; /* GUC for enabling / disabling hypopg during EXPLAIN */ @@ -47,5 +52,6 @@ extern bool hypo_is_enabled; extern MemoryContext HypoMemoryContext; Oid hypo_getNewOid(Oid relid); +void hypo_reset_fake_oids(void); #endif diff --git a/include/hypopg_import.h b/include/hypopg_import.h index 80a3d04..0569190 100644 --- a/include/hypopg_import.h +++ b/include/hypopg_import.h @@ -5,7 +5,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (c) 2008-2018, PostgreSQL Global Development Group + * Copyright (c) 2008-2021, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ diff --git a/include/hypopg_import_index.h b/include/hypopg_import_index.h index 29b5ec8..9af3d2f 100644 --- a/include/hypopg_import_index.h +++ b/include/hypopg_import_index.h @@ -6,7 +6,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (c) 2008-2018, PostgreSQL Global Development Group + * Copyright (c) 2008-2021, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ @@ -19,6 +19,29 @@ MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \ MAXALIGN(sizeof(BTPageOpaqueData))) / 3) +#if PG_VERSION_NUM >= 100000 +#include "access/hash.h" + +/* adapted from src/include/access/hash.h */ +#define HypoHashGetFillFactor(ffactor) \ + (((fillfactor) == 0) ? HASH_DEFAULT_FILLFACTOR : (ffactor)) + +#define HypoHashGetTargetPageUsage(ffactor) \ + (BLCKSZ * HypoHashGetFillFactor(ffactor) / 100) + +#define HypoHashGetMaxBitmapSize() \ + (BLCKSZ - \ + (MAXALIGN(SizeOfPageHeaderData) + MAXALIGN(sizeof(HashPageOpaqueData)))) + +#define HypoHashMaxItemSize() \ + MAXALIGN_DOWN(BLCKSZ - \ + SizeOfPageHeaderData - \ + sizeof(ItemIdData) - \ + MAXALIGN(sizeof(HashPageOpaqueData))) + +#endif + + extern List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index, Relation heapRelation); #if PG_VERSION_NUM < 100000 diff --git a/include/hypopg_index.h b/include/hypopg_index.h index 0c56db0..a07b03e 100644 --- a/include/hypopg_index.h +++ b/include/hypopg_index.h @@ -8,7 +8,7 @@ * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * - * Copyright (C) 2015-2018: Julien Rouhaud + * Copyright (C) 2015-2021: Julien Rouhaud * *------------------------------------------------------------------------- */ @@ -95,6 +95,8 @@ typedef struct hypoIndex bool amhasgetbitmap; /* does AM have amgetbitmap interface? */ #if PG_VERSION_NUM >= 110000 bool amcanparallel; /* does AM support parallel scan? */ + bool amcaninclude; /* does AM support columns included with clause + INCLUDE? */ #endif bool amcanunique; /* does AM support UNIQUE indexes? */ bool amcanmulticol; /* does AM support multi-column indexes? */ @@ -120,6 +122,7 @@ PGDLLEXPORT Datum hypopg_get_indexdef(PG_FUNCTION_ARGS); PGDLLEXPORT Datum hypopg_reset_index(PG_FUNCTION_ARGS); extern explain_get_index_name_hook_type prev_explain_get_index_name_hook; +hypoIndex *hypo_get_index(Oid indexId); const char *hypo_explain_get_index_name_hook(Oid indexId); void hypo_injectHypotheticalIndex(PlannerInfo *root, diff --git a/test/sql/hypo_hash.sql b/test/sql/hypo_hash.sql new file mode 100644 index 0000000..3288bed --- /dev/null +++ b/test/sql/hypo_hash.sql @@ -0,0 +1,15 @@ +-- hypothetical hash indexes, pg10+ + +-- Remove all the hypothetical indexes if any +SELECT hypopg_reset(); + +-- Create normal index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo USING hash (id)'); + +-- Should use hypothetical index using a regular Index Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Scan.*<\d+>hash_hypo.*'; + +-- Deparse the index DDL +SELECT hypopg_get_indexdef(indexrelid) FROM hypopg(); diff --git a/test/sql/hypo_include.sql b/test/sql/hypo_include.sql new file mode 100644 index 0000000..4ee483b --- /dev/null +++ b/test/sql/hypo_include.sql @@ -0,0 +1,31 @@ +-- hypothetical indexes using INCLUDE keyword, pg11+ + +-- Remove all the hypothetical indexes if any +SELECT hypopg_reset(); + +-- Make sure stats and visibility map are up to date +VACUUM ANALYZE hypo; + +-- Should not use hypothetical index + +-- Create normal index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo (id)'); + +-- Should use hypothetical index using a regular Index Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Scan.*<\d+>btree_hypo.*'; + +-- Remove all the hypothetical indexes +SELECT hypopg_reset(); + +-- Create INCLUDE index +SELECT COUNT(*) AS NB +FROM hypopg_create_index('CREATE INDEX ON hypo (id) INCLUDE (val)'); + +-- Should use hypothetical index using an Index Only Scan +SELECT COUNT(*) FROM do_explain('SELECT val FROM hypo WHERE id = 1') e +WHERE e ~ 'Index Only Scan.*<\d+>btree_hypo.*'; + +-- Deparse the index DDL +SELECT hypopg_get_indexdef(indexrelid) FROM hypopg(); diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql index 3aac4af..28e3fcb 100644 --- a/test/sql/hypopg.sql +++ b/test/sql/hypopg.sql @@ -25,7 +25,7 @@ ANALYZE hypo; SELECT COUNT(*) AS nb FROM public.hypopg_create_index('SELECT 1;CREATE INDEX ON hypo(id); SELECT 2'); -SELECT nspname, relname, amname FROM public.hypopg_list_indexes(); +SELECT schema_name, table_name, am_name FROM public.hypopg_list_indexes; -- Should use hypothetical index SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e @@ -100,3 +100,18 @@ WHERE e ~ 'Index.*<\d+>btree_hypo.*'; -- Deparse an index DDL, with almost every possible pathcode SELECT hypopg_get_indexdef(indexrelid) FROM hypopg_create_index('create index on hypo using btree(id desc, id desc nulls first, id desc nulls last, cast(md5(val) as bpchar) bpchar_pattern_ops) with (fillfactor = 10) WHERE id < 1000 AND id +1 %2 = 3'); + +-- Make sure the old Oid generator still works. Test it while keeping existing +-- entries, as both should be able to coexist. +SET hypopg.use_real_oids = on; + +-- Should not use hypothetical index +SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e +WHERE e ~ 'Index.*<\d+>btree_hypo.*'; + +SELECT COUNT(*) AS nb +FROM public.hypopg_create_index('CREATE INDEX ON hypo(id);'); + +-- Should use hypothetical index +SELECT COUNT(*) FROM do_explain('SELECT * FROM hypo WHERE id = 1') e +WHERE e ~ 'Index.*<\d+>btree_hypo.*';