diff --git a/3rdparty/libmdbx/.clang-format b/3rdparty/libmdbx/.clang-format
deleted file mode 100644
index 6c59ef3a3913b651d60f7ec3f911058a7cf2cbbb..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/.clang-format
+++ /dev/null
@@ -1,3 +0,0 @@
-BasedOnStyle: LLVM
-Standard: Cpp11
-ReflowComments: true
diff --git a/3rdparty/libmdbx/.le.ini b/3rdparty/libmdbx/.le.ini
deleted file mode 100644
index 93c8e4bb3990efd3a1056f106289081da1dc7bf0..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/.le.ini
+++ /dev/null
@@ -1,40 +0,0 @@
-tabsize=8
-indentsize=8
-autoindent=1
-bsunindents=1
-insert=1
-inputmode=0
-editmode=1
-makebak=0
-bakpath=
-make=exec make
-shell=exec $SHELL
-run=exec make run
-compile=exec make "$FNAME.o"
-scroll=1
-hscroll=32
-rblock=0
-savepos=1
-savehst=1
-noreg=1
-match_case=1
-linelen=72
-leftmrg=0
-flnmarg=0
-leftadj=1
-rightadj=0
-helpcmd=exec /usr/share/le/help
-usecolor=1
-usetabs=1
-scrollbar=0
-statusline=0
-backupext=.~%d~
-backupnum=9
-preferpagetop=1
-wordwrap=0
-syntaxhl=1
-undo_enable=1
-undo_min_levels=4
-undo_max_memory=128
-undo_glue=1
-usemouse=0
diff --git a/3rdparty/libmdbx/AUTHORS b/3rdparty/libmdbx/AUTHORS
deleted file mode 100644
index beb01868163b4d41c52b4820f6088d8e9f090a9d..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/AUTHORS
+++ /dev/null
@@ -1,34 +0,0 @@
-Contributors
-============
-
-- Alexey Naumov <alexey.naumov@gmail.com>
-- Andrew Ashikhmin <andrey.ashikhmin@gmail.com>
-- Chris Mikkelson <cmikk@qwest.net>
-- Claude Brisson <claude.brisson@gmail.com>
-- David Barbour <dmbarbour@gmail.com>
-- David Wilson <dw@botanicus.net>
-- dreamsxin <dreamsxin@126.com>
-- Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
-- Heiko Becker <heirecka@exherbo.org>
-- Howard Chu <hyc@openldap.org>, <hyc@symas.com>
-- Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
-- James Rouzier <rouzier@gmail.com>
-- Jean-Christophe DUBOIS <jcd@tribudubois.net>
-- John Hewson <john@jahewson.com>
-- Klaus Malorny <klaus.malorny@knipp.de>
-- Kurt Zeilenga <kurt.zeilenga@isode.com>
-- Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.ru>
-- Lorenz Bauer <lmb@cloudflare.com>
-- Luke Yeager <lyeager@nvidia.com>
-- Martin Hedenfalk <martin@bzero.se>
-- Ondrej Kuznik <ondrej.kuznik@acision.com>
-- Orivej Desh <orivej@gmx.fr>
-- Oskari Timperi <oskari.timperi@iki.fi>
-- Pavel Medvedev <pmedvedev@gmail.com>
-- Philipp Storz <philipp.storz@bareos.com>
-- Quanah Gibson-Mount <quanah@openldap.org>
-- Salvador Ortiz <sog@msg.com.mx>
-- Sebastien Launay <sebastien@slaunay.fr>
-- Vladimir Romanov <vromanov@gmail.com>
-- Zano Foundation <crypto.sowle@gmail.com>
-- 장세연 <sasgas@castis.com>
diff --git a/3rdparty/libmdbx/CMakeSettings.json b/3rdparty/libmdbx/CMakeSettings.json
deleted file mode 100644
index 00cd4632e20cdb0f5734abefb1221d19be385db5..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/CMakeSettings.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "configurations": [
- {
- "name": "x64-Debug",
- "generator": "Ninja",
- "configurationType": "Debug",
- "inheritEnvironments": [ "msvc_x64_x64" ],
- "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\libmdbx\\build\\${name}",
- "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\libmdbx\\install\\${name}",
- "cmakeCommandArgs": "",
- "buildCommandArgs": "",
- "ctestCommandArgs": ""
- },
- {
- "name": "x64-Release",
- "generator": "Ninja",
- "configurationType": "Release",
- "inheritEnvironments": [ "msvc_x64_x64" ],
- "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\libmdbx\\build\\${name}",
- "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\libmdbx\\install\\${name}",
- "cmakeCommandArgs": "",
- "buildCommandArgs": "",
- "ctestCommandArgs": ""
- }
- ]
-}
\ No newline at end of file
diff --git a/3rdparty/libmdbx/COPYRIGHT b/3rdparty/libmdbx/COPYRIGHT
deleted file mode 100644
index bd3acace3a9523ce74724d8aacacb9771a0263d1..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/COPYRIGHT
+++ /dev/null
@@ -1,22 +0,0 @@
-Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
-Copyright 2011-2015 Howard Chu, Symas Corp.
-Copyright 2015,2016 Peter-Service R&D LLC.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted only as authorized by the OpenLDAP
-Public License.
-
-A copy of this license is available in the file LICENSE in the
-top-level directory of the distribution or, alternatively, at
-<http://www.OpenLDAP.org/license.html>.
-
-OpenLDAP is a registered trademark of the OpenLDAP Foundation.
-
-Individual files and/or contributed packages may be copyright by
-other parties and/or subject to additional restrictions.
-
-This work also contains materials derived from public sources.
-
-Additional information about OpenLDAP can be obtained at
-<http://www.openldap.org/>.
diff --git a/3rdparty/libmdbx/ChangeLog.md b/3rdparty/libmdbx/ChangeLog.md
index 623709f8832bcf58a121d458ae0d2ae1bd80ba09..f677fed8d9822d262f63df2d8f2fe49b40df40c3 100644
--- a/3rdparty/libmdbx/ChangeLog.md
+++ b/3rdparty/libmdbx/ChangeLog.md
@@ -1,27 +1,21 @@
ChangeLog
---------
-## Underway
+### TODO
+
+ - [Engage an "overlapped I/O" on Windows](todo4recovery://erased_by_github/libmdbx/issues/224).
+ - [Simple careful mode for working with corrupted DB](todo4recovery://erased_by_github/libmdbx/issues/223).
+ - [Move most of `mdbx_chk` functional to the library API](todo4recovery://erased_by_github/libmdbx/issues/204).
+ - [Replace SRW-lock on Windows to allow shrink DB with `MDBX_NOTLS` option](todo4recovery://erased_by_github/libmdbx/issues/210).
+ - [More flexible support of asynchronous runtime/framework(s)](todo4recovery://erased_by_github/libmdbx/issues/200).
+ - [Migration guide from LMDB to MDBX](todo4recovery://erased_by_github/libmdbx/issues/199).
+ - [Get rid of dirty-pages list in MDBX_WRITEMAP mode](todo4recovery://erased_by_github/libmdbx/issues/193).
+ - [Large/Overflow pages accounting for dirty-room](todo4recovery://erased_by_github/libmdbx/issues/192).
+ - [Support for RAW devices](todo4recovery://erased_by_github/libmdbx/issues/124).
+ - [Support MessagePack for Keys & Values](todo4recovery://erased_by_github/libmdbx/issues/115).
+ - [Engage new terminology](todo4recovery://erased_by_github/libmdbx/issues/137).
+ - Packages for [Astra Linux](https://astralinux.ru/), [ALT Linux](https://www.altlinux.org/), [ROSA Linux](https://www.rosalinux.ru/), etc.
-Acknowledgements:
-
- - [Masatoshi Fukunaga](https://github.com/mah0x211) for [Lua bindings](https://github.com/mah0x211/lua-libmdbx).
-
-New:
-
- - Added most of transactions flags to the public API.
-
-Fixes:
-
- - A series of fixes and improvements for automatically generated documentation (Doxygen).
-
-Minors:
-
- - Refined `release-assets` GNU Make target.
- - Added logging to `mdbx_fetch_sdb()` to help debugging complex DBI-handels use cases.
-
-
--------------------------------------------------------------------------------
## v0.11.7 at 2022-04-22
@@ -43,8 +37,8 @@ New:
- Support for Microsoft Visual Studio 2022.
- Support build by MinGW' make from command line without CMake.
- Added `mdbx::filesystem` C++ API namespace that corresponds to `std::filesystem` or `std::experimental::filesystem`.
- - Created [website](https://libmdbx.dqdkfa.ru/) for online auto-generated documentation.
- - Used `https://web.archive.org/web/20220414235959/https://github.com/erthink/` for dead (or temporarily lost) resources deleted by ~~Github~~.
+ - Created [website](https://libmdbx.website.yandexcloud.net/) for online auto-generated documentation.
+ - Used `todo4recovery://erased_by_github/` for dead (or temporarily lost) resources deleted by ~~Github~~.
- Added `--loglevel=` command-line option to the `mdbx_test` tool.
- Added few fast smoke-like tests into CMake builds.
@@ -65,25 +59,22 @@ Fixes:
Minors:
- Resolve minor MSVC warnings: avoid `/INCREMENTAL[:YES]` with `/LTCG`, `/W4` with `/W3`, the `C5105` warning.
- - Switched to using `MDBX_EPERM` instead of `MDBX_RESULT_TRUE` to indicate that the geometry cannot be updated.
+ - Switched to using `MDBX_EPERM` instead of `MDBX_RESULT_TRUE' to indicate that the geometry cannot be updated.
- Added `NULL` checking during memory allocation inside `mdbx_chk`.
- Resolved all warnings from MinGW while used without CMake.
- Added inheretable `target_include_directories()` to `CMakeLists.txt` for easy integration.
- Added build-time checks and paranoid runtime assertions for the `off_t` arguments of `fcntl()` which are used for locking.
- Added `-Wno-lto-type-mismatch` to avoid false-positive warnings from old GCC during LTO-enabled builds.
- - Added checking for TID (system thread id) to avoid hang on 32-bit Bionic/Android within `pthread_mutex_lock()`.
+ - Added checking for TID (system thread id) to avoid hang on 32-bit Bionic/Android within `pthread_mutex_lock()`.
- Reworked `MDBX_BUILD_TARGET` of CMake builds.
- Added `CMAKE_HOST_ARCH` and `CMAKE_HOST_CAN_RUN_EXECUTABLES_BUILT_FOR_TARGET`.
--------------------------------------------------------------------------------
-
-
## v0.11.6 at 2022-03-24
The stable release with the complete workaround for an incoherence flaw of Linux unified page/buffer cache.
Nonetheless the cause for this trouble may be an issue of Intel CPU cache/MESI.
-See [issue#269](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/269) for more information.
+See [issue#269](todo4recovery://erased_by_github/libmdbx/issues/269) for more information.
Acknowledgements:
@@ -92,8 +83,8 @@ Acknowledgements:
Fixes:
- - [Added complete workaround](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/269) for an incoherence flaw of Linux unified page/buffer cache.
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/272) cursor reusing for read-only transactions.
+ - [Added complete workaround](todo4recovery://erased_by_github/libmdbx/issues/269) for an incoherence flaw of Linux unified page/buffer cache.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/272) cursor reusing for read-only transactions.
- Fixed copy&paste typo inside `mdbx::cursor::find_multivalue()`.
Minors:
@@ -108,7 +99,7 @@ Minors:
## v0.11.5 at 2022-02-23
The release with the temporary hotfix for a flaw of Linux unified page/buffer cache.
-See [issue#269](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/269) for more information.
+See [issue#269](todo4recovery://erased_by_github/libmdbx/issues/269) for more information.
Acknowledgements:
@@ -118,10 +109,10 @@ Acknowledgements:
Fixes:
- - [Added hotfix](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/269) for a flaw of Linux unified page/buffer cache.
- - [Fixed/Reworked](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/270) move-assignment operators for "managed" classes of C++ API.
+ - [Added hotfix](todo4recovery://erased_by_github/libmdbx/issues/269) for a flaw of Linux unified page/buffer cache.
+ - [Fixed/Reworked](todo4recovery://erased_by_github/libmdbx/pull/270) move-assignment operators for "managed" classes of C++ API.
- Fixed potential `SIGSEGV` while open DB with overrided non-default page size.
- - [Made](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/267) `mdbx_env_open()` idempotence in failure cases.
+ - [Made](todo4recovery://erased_by_github/libmdbx/issues/267) `mdbx_env_open()` idempotence in failure cases.
- Refined/Fixed pages reservation inside `mdbx_update_gc()` to avoid non-reclamation in a rare cases.
- Fixed typo in a retained space calculation for the hsr-callback.
@@ -154,15 +145,15 @@ New features, extensions and improvements:
Fixes:
- Fixed handling `MDBX_opt_rp_augment_limit` for GC's records from huge transactions (Erigon/Akula/Ethereum).
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/258) build on Android (avoid including `sys/sem.h`).
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/261) missing copy assignment operator for `mdbx::move_result`.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/258) build on Android (avoid including `sys/sem.h`).
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/pull/261) missing copy assignment operator for `mdbx::move_result`.
- Fixed missing `&` for `std::ostream &operator<<()` overloads.
- Fixed unexpected `EXDEV` (Cross-device link) error from `mdbx_env_copy()`.
- Fixed base64 encoding/decoding bugs in auxillary C++ API.
- Fixed overflow of `pgno_t` during checking PNL on 64-bit platforms.
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/260) excessive PNL checking after sort for spilling.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/260) excessive PNL checking after sort for spilling.
- Reworked checking `MAX_PAGENO` and DB upper-size geometry limit.
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/265) build for some combinations of versions of MSVC and Windows SDK.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/265) build for some combinations of versions of MSVC and Windows SDK.
Minors:
@@ -189,10 +180,10 @@ Acknowledgements:
New features, extensions and improvements:
- - [Added](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/236) `mdbx_cursor_get_batch()`.
- - [Added](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/250) `MDBX_SET_UPPERBOUND`.
+ - [Added](todo4recovery://erased_by_github/libmdbx/issues/236) `mdbx_cursor_get_batch()`.
+ - [Added](todo4recovery://erased_by_github/libmdbx/issues/250) `MDBX_SET_UPPERBOUND`.
- C++ API is finalized now.
- - The GC update stage has been [significantly speeded](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/254) when fixing huge Erigon's transactions (Ethereum ecosystem).
+ - The GC update stage has been [significantly speeded](todo4recovery://erased_by_github/libmdbx/issues/254) when fixing huge Erigon's transactions (Ethereum ecosystem).
Fixes:
@@ -203,12 +194,12 @@ Minors:
- Fixed returning `MDBX_RESULT_TRUE` (unexpected -1) from `mdbx_env_set_option()`.
- Added `mdbx_env_get_syncbytes()` and `mdbx_env_get_syncperiod()`.
- - [Clarified](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/249) description of `MDBX_INTEGERKEY`.
+ - [Clarified](todo4recovery://erased_by_github/libmdbx/pull/249) description of `MDBX_INTEGERKEY`.
- Reworked/simplified `mdbx_env_sync_internal()`.
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/248) extra assertion inside `mdbx_cursor_put()` for `MDBX_DUPFIXED` cases.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/248) extra assertion inside `mdbx_cursor_put()` for `MDBX_DUPFIXED` cases.
- Avoiding extra looping inside `mdbx_env_info_ex()`.
- Explicitly enabled core dumps from stochastic tests scripts on Linux.
- - [Fixed](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/253) `mdbx_override_meta()` to avoid false-positive assertions.
+ - [Fixed](todo4recovery://erased_by_github/libmdbx/issues/253) `mdbx_override_meta()` to avoid false-positive assertions.
- For compatibility reverted returning `MDBX_ENODATA`for some cases.
@@ -224,10 +215,10 @@ Acknowledgements:
Fixes:
- - [Fixed compilation](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/239) with `devtoolset-9` on CentOS/RHEL 7.
- - [Fixed unexpected `MDBX_PROBLEM` error](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/242) because of update an obsolete meta-page.
- - [Fixed returning `MDBX_NOTFOUND` error](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/243) in case an inexact value found for `MDBX_GET_BOTH` operation.
- - [Fixed compilation](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/245) without kernel/libc-devel headers.
+ - [Fixed compilation](todo4recovery://erased_by_github/libmdbx/pull/239) with `devtoolset-9` on CentOS/RHEL 7.
+ - [Fixed unexpected `MDBX_PROBLEM` error](todo4recovery://erased_by_github/libmdbx/issues/242) because of update an obsolete meta-page.
+ - [Fixed returning `MDBX_NOTFOUND` error](todo4recovery://erased_by_github/libmdbx/issues/243) in case an inexact value found for `MDBX_GET_BOTH` operation.
+ - [Fixed compilation](todo4recovery://erased_by_github/libmdbx/issues/245) without kernel/libc-devel headers.
Minors:
@@ -244,7 +235,7 @@ Minors:
The database format signature has been changed to prevent
forward-interoperability with an previous releases, which may lead to a
-[false positive diagnosis of database corruption](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/238)
+[false positive diagnosis of database corruption](todo4recovery://erased_by_github/libmdbx/issues/238)
due to flaws of an old library versions.
This change is mostly invisible:
@@ -257,9 +248,6 @@ Acknowledgements:
- [Alex Sharov](https://github.com/AskAlexSharov) for reporting and testing.
--------------------------------------------------------------------------------
-
-
## v0.10.5 at 2021-10-13 (obsolete, please use v0.11.1)
Unfortunately, the `v0.10.5` accidentally comes not full-compatible with previous releases:
@@ -296,7 +284,7 @@ Acknowledgements:
Fixes:
- Fixed possibility of looping update GC during transaction commit (no public issue since the problem was discovered inside [Positive Technologies](https://www.ptsecurity.ru)).
- - Fixed `#pragma pack` to avoid provoking some compilers to generate code with [unaligned access](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/235).
+ - Fixed `#pragma pack` to avoid provoking some compilers to generate code with [unaligned access](todo4recovery://erased_by_github/libmdbx/issues/235).
- Fixed `noexcept` for potentially throwing `txn::put()` of C++ API.
Minors:
@@ -322,7 +310,7 @@ Extensions and improvements:
Fixes:
- - Always setup `madvise` while opening DB (fixes https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/231).
+ - Always setup `madvise` while opening DB (fixes todo4recovery://erased_by_github/libmdbx/issues/231).
- Fixed checking legacy `P_DIRTY` flag (`0x10`) for nested/sub-pages.
Minors:
@@ -343,11 +331,11 @@ Acknowledgements:
- [Lionel Debroux](https://github.com/debrouxl) for fuzzing tests and reporting bugs.
- [Sergey Fedotov](https://github.com/SergeyFromHell/) for [`node-mdbx` NodeJS bindings](https://www.npmjs.com/package/node-mdbx).
- [Kris Zyp](https://github.com/kriszyp) for [`lmdbx-store` NodeJS bindings](https://github.com/kriszyp/lmdbx-store).
- - [Noel Kuntze](https://github.com/Thermi) for [draft Python bindings](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/commits/python-bindings).
+ - [Noel Kuntze](https://github.com/Thermi) for [draft Python bindings](todo4recovery://erased_by_github/libmdbx/commits/python-bindings).
New features, extensions and improvements:
- - [Allow to predefine/override `MDBX_BUILD_TIMESTAMP` for builds reproducibility](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/201).
+ - [Allow to predefine/override `MDBX_BUILD_TIMESTAMP` for builds reproducibility](todo4recovery://erased_by_github/libmdbx/issues/201).
- Added options support for `long-stochastic` script.
- Avoided `MDBX_TXN_FULL` error for large transactions when possible.
- The `MDBX_READERS_LIMIT` increased to `32767`.
@@ -355,7 +343,7 @@ New features, extensions and improvements:
- Minimized the size of poisoned/unpoisoned regions to avoid Valgrind/ASAN stuck.
- Added more workarounds for QEMU for testing builds for 32-bit platforms, Alpha and Sparc architectures.
- `mdbx_chk` now skips iteration & checking of DB' records if corresponding page-tree is corrupted (to avoid `SIGSEGV`, ASAN failures, etc).
- - Added more checks for [rare/fuzzing corruption cases](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/217).
+ - Added more checks for [rare/fuzzing corruption cases](todo4recovery://erased_by_github/libmdbx/issues/217).
Backward compatibility break:
@@ -367,18 +355,18 @@ Backward compatibility break:
Fixes:
- Fixed excess meta-pages checks in case `mdbx_chk` is called to check the DB for a specific meta page and thus could prevent switching to the selected meta page, even if the check passed without errors.
- - Fixed [recursive use of SRW-lock on Windows cause by `MDBX_NOTLS` option](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/203).
- - Fixed [log a warning during a new DB creation](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/205).
- - Fixed [false-negative `mdbx_cursor_eof()` result](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/207).
- - Fixed [`make install` with non-GNU `install` utility (OSX, BSD)](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/208).
- - Fixed [installation by `CMake` in special cases by complete use `GNUInstallDirs`'s variables](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/209).
- - Fixed [C++ Buffer issue with `std::string` and alignment](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/191).
+ - Fixed [recursive use of SRW-lock on Windows cause by `MDBX_NOTLS` option](todo4recovery://erased_by_github/libmdbx/issues/203).
+ - Fixed [log a warning during a new DB creation](todo4recovery://erased_by_github/libmdbx/issues/205).
+ - Fixed [false-negative `mdbx_cursor_eof()` result](todo4recovery://erased_by_github/libmdbx/issues/207).
+ - Fixed [`make install` with non-GNU `install` utility (OSX, BSD)](todo4recovery://erased_by_github/libmdbx/issues/208).
+ - Fixed [installation by `CMake` in special cases by complete use `GNUInstallDirs`'s variables](todo4recovery://erased_by_github/libmdbx/issues/209).
+ - Fixed [C++ Buffer issue with `std::string` and alignment](todo4recovery://erased_by_github/libmdbx/issues/191).
- Fixed `safe64_reset()` for platforms without atomic 64-bit compare-and-swap.
- Fixed hang/shutdown on big-endian platforms without `__cxa_thread_atexit()`.
- - Fixed [using bad meta-pages if DB was partially/recoverable corrupted](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/217).
+ - Fixed [using bad meta-pages if DB was partially/recoverable corrupted](todo4recovery://erased_by_github/libmdbx/issues/217).
- Fixed extra `noexcept` for `buffer::&assign_reference()`.
- Fixed `bootid` generation on Windows for case of change system' time.
- - Fixed [test framework keygen-related issue](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/127).
+ - Fixed [test framework keygen-related issue](todo4recovery://erased_by_github/libmdbx/issues/127).
## v0.10.1 at 2021-06-01
@@ -399,10 +387,10 @@ New features:
Fixes:
- Fixed minor "foo not used" warnings from modern C++ compilers when building the C++ part of the library.
- - Fixed confusing/messy errors when build library from unfit github's archives (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/197).
+ - Fixed confusing/messy errors when build library from unfit github's archives (todo4recovery://erased_by_github/libmdbx/issues/197).
- Fixed `#​e​l​s​i​f` typo.
- - Fixed rare unexpected `MDBX_PROBLEM` error during altering data in huge transactions due to wrong spilling/oust of dirty pages (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/195).
- - Re-Fixed WSL1/WSL2 detection with distinguishing (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/97).
+ - Fixed rare unexpected `MDBX_PROBLEM` error during altering data in huge transactions due to wrong spilling/oust of dirty pages (todo4recovery://erased_by_github/libmdbx/issues/195).
+ - Re-Fixed WSL1/WSL2 detection with distinguishing (todo4recovery://erased_by_github/libmdbx/issues/97).
## v0.10.0 at 2021-05-09
@@ -425,7 +413,7 @@ New features:
and conjointly with the `MDBX_ENV_CHECKPID=0` and `MDBX_TXN_CHECKOWNER=0` options can yield
up to 30% more performance compared to LMDB.
- Using float point (exponential quantized) representation for internal 16-bit values
- of grow step and shrink threshold when huge ones (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/166).
+ of grow step and shrink threshold when huge ones (todo4recovery://erased_by_github/libmdbx/issues/166).
To minimize the impact on compatibility, only the odd values inside the upper half
of the range (i.e. 32769..65533) are used for the new representation.
- Added the `mdbx_drop` similar to LMDB command-line tool to purge or delete (sub)database(s).
@@ -434,7 +422,7 @@ New features:
- The internal node sizes were refined, resulting in a reduction in large/overflow pages in some use cases
and a slight increase in limits for a keys size to ≈½ of page size.
- Added to `mdbx_chk` output number of keys/items on pages.
- - Added explicit `install-strip` and `install-no-strip` targets to the `Makefile` (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/180).
+ - Added explicit `install-strip` and `install-no-strip` targets to the `Makefile` (todo4recovery://erased_by_github/libmdbx/pull/180).
- Major rework page splitting (af9b7b560505684249b76730997f9e00614b8113) for
- An "auto-appending" feature upon insertion for both ascending and
descending key sequences. As a result, the optimality of page filling
@@ -442,7 +430,7 @@ New features:
inserting ordered sequences of keys,
- A "splitting at middle" to make page tree more balanced on average.
- Added `mdbx_get_sysraminfo()` to the API.
- - Added guessing a reasonable maximum DB size for the default upper limit of geometry (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/183).
+ - Added guessing a reasonable maximum DB size for the default upper limit of geometry (todo4recovery://erased_by_github/libmdbx/issues/183).
- Major rework internal labeling of a dirty pages (958fd5b9479f52f2124ab7e83c6b18b04b0e7dda) for
a "transparent spilling" feature with the gist to make a dirty pages
be ready to spilling (writing to a disk) without further altering ones.
@@ -458,7 +446,7 @@ New features:
- Support `make help` to list available make targets.
- Silently `make`'s build by default.
- Preliminary [Python bindings](https://github.com/Thermi/libmdbx/tree/python-bindings) is available now
- by [Noel Kuntze](https://github.com/Thermi) (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/147).
+ by [Noel Kuntze](https://github.com/Thermi) (todo4recovery://erased_by_github/libmdbx/issues/147).
Backward compatibility break:
@@ -473,25 +461,22 @@ Backward compatibility break:
Fixes:
- - Fixed performance regression due non-optimal C11 atomics usage (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/160).
- - Fixed "reincarnation" of subDB after it deletion (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/168).
+ - Fixed performance regression due non-optimal C11 atomics usage (todo4recovery://erased_by_github/libmdbx/issues/160).
+ - Fixed "reincarnation" of subDB after it deletion (todo4recovery://erased_by_github/libmdbx/issues/168).
- Fixed (disallowing) implicit subDB deletion via operations on `@MAIN`'s DBI-handle.
- - Fixed a crash of `mdbx_env_info_ex()` in case of a call for a non-open environment (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/171).
- - Fixed the selecting/adjustment values inside `mdbx_env_set_geometry()` for implicit out-of-range cases (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/170).
- - Fixed `mdbx_env_set_option()` for set initial and limit size of dirty page list ((https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/179).
- - Fixed an unreasonably huge default upper limit for DB geometry (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/183).
+ - Fixed a crash of `mdbx_env_info_ex()` in case of a call for a non-open environment (todo4recovery://erased_by_github/libmdbx/issues/171).
+ - Fixed the selecting/adjustment values inside `mdbx_env_set_geometry()` for implicit out-of-range cases (todo4recovery://erased_by_github/libmdbx/issues/170).
+ - Fixed `mdbx_env_set_option()` for set initial and limit size of dirty page list ((todo4recovery://erased_by_github/libmdbx/issues/179).
+ - Fixed an unreasonably huge default upper limit for DB geometry (todo4recovery://erased_by_github/libmdbx/issues/183).
- Fixed `constexpr` specifier for the `slice::invalid()`.
- - Fixed (no)readahead auto-handling (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/164).
+ - Fixed (no)readahead auto-handling (todo4recovery://erased_by_github/libmdbx/issues/164).
- Fixed non-alloy build for Windows.
- Switched to using Heap-functions instead of LocalAlloc/LocalFree on Windows.
- - Fixed `mdbx_env_stat_ex()` to returning statistics of the whole environment instead of MainDB only (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/190).
+ - Fixed `mdbx_env_stat_ex()` to returning statistics of the whole environment instead of MainDB only (todo4recovery://erased_by_github/libmdbx/issues/190).
- Fixed building by GCC 4.8.5 (added workaround for a preprocessor's bug).
- Fixed building C++ part for iOS <= 13.0 (unavailability of `std::filesystem::path`).
- Fixed building for Windows target versions prior to Windows Vista (`WIN32_WINNT < 0x0600`).
- - Fixed building by MinGW for Windows (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/155).
-
-
--------------------------------------------------------------------------------
+ - Fixed building by MinGW for Windows (todo4recovery://erased_by_github/libmdbx/issues/155).
## v0.9.3 at 2021-02-02
@@ -511,7 +496,7 @@ Removed options and features:
New features:
- Package for FreeBSD is available now by Mahlon E. Smith.
- - New API functions to get/set various options (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/128):
+ - New API functions to get/set various options (todo4recovery://erased_by_github/libmdbx/issues/128):
- the maximum number of named databases for the environment;
- the maximum number of threads/reader slots;
- threshold (since the last unsteady commit) to force flush the data buffers to disk;
@@ -524,7 +509,7 @@ New features:
- maximal part of the dirty pages may be spilled when necessary;
- minimal part of the dirty pages should be spilled when necessary;
- how much of the parent transaction dirty pages will be spilled while start each child transaction;
- - Unlimited/Dynamic size of retired and dirty page lists (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/123).
+ - Unlimited/Dynamic size of retired and dirty page lists (todo4recovery://erased_by_github/libmdbx/issues/123).
- Added `-p` option (purge subDB before loading) to `mdbx_load` tool.
- Reworked spilling of large transaction and committing of nested transactions:
- page spilling code reworked to avoid the flaws and bugs inherited from LMDB;
@@ -534,22 +519,22 @@ New features:
- Added `MDBX_ENABLE_REFUND` and `MDBX_PNL_ASCENDING` internal/advanced build options.
- Added `mdbx_default_pagesize()` function.
- Better support architectures with a weak/relaxed memory consistency model (ARM, AARCH64, PPC, MIPS, RISC-V, etc) by means [C11 atomics](https://en.cppreference.com/w/c/atomic).
- - Speed up page number lists and dirty page lists (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/132).
+ - Speed up page number lists and dirty page lists (todo4recovery://erased_by_github/libmdbx/issues/132).
- Added `LIBMDBX_NO_EXPORTS_LEGACY_API` build option.
Fixes:
- - Fixed missing cleanup (null assigned) in the C++ commit/abort (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/pull/143).
+ - Fixed missing cleanup (null assigned) in the C++ commit/abort (todo4recovery://erased_by_github/libmdbx/pull/143).
- Fixed `mdbx_realloc()` for case of nullptr and `MDBX_WITHOUT_MSVC_CRT=ON` for Windows.
- - Fixed the possibility to use invalid and renewed (closed & re-opened, dropped & re-created) DBI-handles (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/146).
- - Fixed 4-byte aligned access to 64-bit integers, including access to the `bootid` meta-page's field (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/153).
+ - Fixed the possibility to use invalid and renewed (closed & re-opened, dropped & re-created) DBI-handles (todo4recovery://erased_by_github/libmdbx/issues/146).
+ - Fixed 4-byte aligned access to 64-bit integers, including access to the `bootid` meta-page's field (todo4recovery://erased_by_github/libmdbx/issues/153).
- Fixed minor/potential memory leak during page flushing and unspilling.
- Fixed handling states of cursors's and subDBs's for nested transactions.
- Fixed page leak in extra rare case the list of retired pages changed during update GC on transaction commit.
- - Fixed assertions to avoid false-positive UB detection by CLANG/LLVM (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/153).
- - Fixed `MDBX_TXN_FULL` and regressive `MDBX_KEYEXIST` during large transaction commit with `MDBX_LIFORECLAIM` (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/123).
+ - Fixed assertions to avoid false-positive UB detection by CLANG/LLVM (todo4recovery://erased_by_github/libmdbx/issues/153).
+ - Fixed `MDBX_TXN_FULL` and regressive `MDBX_KEYEXIST` during large transaction commit with `MDBX_LIFORECLAIM` (todo4recovery://erased_by_github/libmdbx/issues/123).
- Fixed auto-recovery (`weak->steady` with the same boot-id) when Database size at last weak checkpoint is large than at last steady checkpoint.
- - Fixed operation on systems with unusual small/large page size, including PowerPC (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/157).
+ - Fixed operation on systems with unusual small/large page size, including PowerPC (todo4recovery://erased_by_github/libmdbx/issues/157).
## v0.9.2 at 2020-11-27
@@ -587,11 +572,11 @@ Fixes:
- Fixed copy&paste typos.
- Fixed minor false-positive GCC warning.
- Added workaround for broken `DEFINE_ENUM_FLAG_OPERATORS` from Windows SDK.
- - Fixed cursor state after multimap/dupsort repeated deletes (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/121).
+ - Fixed cursor state after multimap/dupsort repeated deletes (todo4recovery://erased_by_github/libmdbx/issues/121).
- Added `SIGPIPE` suppression for internal thread during `mdbx_env_copy()`.
- - Fixed extra-rare `MDBX_KEY_EXIST` error during `mdbx_commit()` (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/131).
- - Fixed spilled pages checking (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/126).
- - Fixed `mdbx_load` for 'plain text' and without `-s name` cases (https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/136).
+ - Fixed extra-rare `MDBX_KEY_EXIST` error during `mdbx_commit()` (todo4recovery://erased_by_github/libmdbx/issues/131).
+ - Fixed spilled pages checking (todo4recovery://erased_by_github/libmdbx/issues/126).
+ - Fixed `mdbx_load` for 'plain text' and without `-s name` cases (todo4recovery://erased_by_github/libmdbx/issues/136).
- Fixed save/restore/commit of cursors for nested transactions.
- Fixed cursors state in rare/special cases (move next beyond end-of-data, after deletion and so on).
- Added workaround for MSVC 19.28 (Visual Studio 16.8) (but may still hang during compilation).
@@ -607,7 +592,7 @@ Fixes:
Added features:
- Preliminary C++ API with support for C++17 polymorphic allocators.
- - [Online C++ API reference](https://libmdbx.dqdkfa.ru/) by Doxygen.
+ - [Online C++ API reference](https://erased_by_github_and_to_be_restored.todo/libmdbx/) by Doxygen.
- Quick reference for Insert/Update/Delete operations.
- Explicit `MDBX_SYNC_DURABLE` to sync modes for API clarity.
- Explicit `MDBX_ALLDUPS` and `MDBX_UPSERT` for API clarity.
@@ -652,7 +637,7 @@ Fixes:
Added features:
- - [Online C API reference](https://libmdbx.dqdkfa.ru/) by Doxygen.
+ - [Online C API reference](https://erased_by_github_and_to_be_restored.todo/libmdbx/) by Doxygen.
- Separated enums for environment, sub-databases, transactions, copying and data-update flags.
Deprecated functions and flags:
@@ -661,146 +646,132 @@ Deprecated functions and flags:
Please use the value-to-key functions to provide keys that are compatible with the built-in libmdbx comparators.
--------------------------------------------------------------------------------
-
-
-## 2020-07-06
-
- - Added support multi-opening the same DB in a process with SysV locking (BSD).
- - Fixed warnings & minors for LCC compiler (E2K).
- - Enabled to simultaneously open the same database from processes with and without the `MDBX_WRITEMAP` option.
- - Added key-to-value, `mdbx_get_keycmp()` and `mdbx_get_datacmp()` functions (helpful to avoid using custom comparators).
- - Added `ENABLE_UBSAN` CMake option to enabling the UndefinedBehaviorSanitizer from GCC/CLANG.
- - Workaround for [CLANG bug](https://bugs.llvm.org/show_bug.cgi?id=43275).
- - Returning `MDBX_CORRUPTED` in case all meta-pages are weak and no other error.
- - Refined mode bits while auto-creating LCK-file.
- - Avoids unnecessary database file re-mapping in case geometry changed by another process(es).
- From the user's point of view, the `MDBX_UNABLE_EXTEND_MAPSIZE` error will now be returned less frequently and only when using the DB in the current process really requires it to be reopened.
- - Remapping on-the-fly and of the database file was implemented.
- Now remapping with a change of address is performed automatically if there are no dependent readers in the current process.
-
-
-## 2020-06-12
-
- - Minor change versioning. The last number in the version now means the number of commits since last release/tag.
- - Provide ChangeLog file.
- - Fix for using libmdbx as a C-only sub-project with CMake.
- - Fix `mdbx_env_set_geometry()` for case it is called from an opened environment outside of a write transaction.
- - Add support for huge transactions and `MDBX_HUGE_TRANSACTIONS` build-option (default `OFF`).
- - Refine LTO (link time optimization) for clang.
- - Force enabling exceptions handling for MSVC (`/EHsc` option).
-
-
-## 2020-06-05
-
- - Support for Android/Bionic.
- - Support for iOS.
- - Auto-handling `MDBX_NOSUBDIR` while opening for any existing database.
- - Engage github-actions to make release-assets.
- - Clarify API description.
- - Extended keygen-cases in stochastic test.
- - Fix fetching of first/lower key from LEAF2-page during page merge.
- - Fix missing comma in array of error messages.
- - Fix div-by-zero while copy-with-compaction for non-resizable environments.
- - Fixes & enhancements for custom-comparators.
- - Fix `MDBX_WITHOUT_MSVC_CRT` option and missing `ntdll.def`.
- - Fix `mdbx_env_close()` to work correctly called concurrently from several threads.
- - Fix null-deref in an ASAN-enabled builds while opening the environment with error and/or read-only.
- - Fix AddressSanitizer errors after closing the environment.
- - Fix/workaround to avoid GCC 10.x pedantic warnings.
- - Fix using `ENODATA` for FreeBSD.
- - Avoid invalidation of DBI-handle(s) when it just closes.
- - Avoid using `pwritev()` for single-writes (up to 10% speedup for some kernels & scenarios).
- - Avoiding `MDBX_UTTERLY_NOSYNC` as result of flags merge.
- - Add `mdbx_dbi_dupsort_depthmask()` function.
- - Add `MDBX_CP_FORCE_RESIZEABLE` option.
- - Add deprecated `MDBX_MAP_RESIZED` for compatibility.
- - Add `MDBX_BUILD_TOOLS` option (default `ON`).
- - Refine `mdbx_dbi_open_ex()` to safe concurrently opening the same handle from different threads.
- - Truncate clk-file during environment closing. So a zero-length lck-file indicates that the environment was closed properly.
- - Refine `mdbx_update_gc()` for huge transactions with small sizes of database page.
- - Extends dump/load to support all MDBX attributes.
- - Avoid upsertion the same key-value data, fix related assertions.
- - Rework min/max length checking for keys & values.
- - Checking the order of keys on all pages during checking.
- - Support `CFLAGS_EXTRA` make-option for convenience.
- - Preserve the last txnid while copying with compactification.
- - Auto-reset running transaction in mdbx_txn_renew().
- - Automatically abort errored transaction in mdbx_txn_commit().
- - Auto-choose page size for large databases.
- - Rearrange source files, rework build, options-support by CMake.
- - Crutch for WSL1 (Windows subsystem for Linux).
- - Refine install/uninstall targets.
- - Support for Valgrind 3.14 and later.
- - Add check-analyzer check-ubsan check-asan check-leak targets to Makefile.
- - Minor fix/workaround to avoid UBSAN traps for `memcpy(ptr, NULL, 0)`.
- - Avoid some GCC-analyzer false-positive warnings.
-
-
-## 2020-03-18
-
- - Workarounds for Wine (Windows compatibility layer for Linux).
- - `MDBX_MAP_RESIZED` renamed to `MDBX_UNABLE_EXTEND_MAPSIZE`.
- - Clarify API description, fix typos.
- - Speedup runtime checks in debug/checked builds.
- - Added checking for read/write transactions overlapping for the same thread, added `MDBX_TXN_OVERLAPPING` error and `MDBX_DBG_LEGACY_OVERLAP` option.
- - Added `mdbx_key_from_jsonInteger()`, `mdbx_key_from_double()`, `mdbx_key_from_float()`, `mdbx_key_from_int64()` and `mdbx_key_from_int32()` functions. See `mdbx.h` for description.
- - Fix compatibility (use zero for invalid DBI).
- - Refine/clarify error messages.
- - Avoids extra error messages "bad txn" from mdbx_chk when DB is corrupted.
-
-
-## 2020-01-21
-
- - Fix `mdbx_load` utility for custom comparators.
- - Fix checks related to `MDBX_APPEND` flag inside `mdbx_cursor_put()`.
- - Refine/fix dbi_bind() internals.
- - Refine/fix handling `STATUS_CONFLICTING_ADDRESSES`.
- - Rework `MDBX_DBG_DUMP` option to avoid disk I/O performance degradation.
- - Add built-in help to test tool.
- - Fix `mdbx_env_set_geometry()` for large page size.
- - Fix env_set_geometry() for large pagesize.
- - Clarify API description & comments, fix typos.
-
-
-## 2019-12-31
-
- - Fix returning MDBX_RESULT_TRUE from page_alloc().
- - Fix false-positive ASAN issue.
- - Fix assertion for `MDBX_NOTLS` option.
- - Rework `MADV_DONTNEED` threshold.
- - Fix `mdbx_chk` utility for don't checking some numbers if walking on the B-tree was disabled.
- - Use page's mp_txnid for basic integrity checking.
- - Add `MDBX_FORCE_ASSERTIONS` built-time option.
- - Rework `MDBX_DBG_DUMP` to avoid performance degradation.
- - Rename `MDBX_NOSYNC` to `MDBX_SAFE_NOSYNC` for clarity.
- - Interpret `ERROR_ACCESS_DENIED` from `OpenProcess()` as 'process exists'.
- - Avoid using `FILE_FLAG_NO_BUFFERING` for compatibility with small database pages.
- - Added install section for CMake.
-
-
-## 2019-12-02
-
- - Support for Mac OSX, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, OpenSolaris, OpenIndiana (AIX and HP-UX pending).
- - Use bootid for decisions of rollback.
- - Counting retired pages and extended transaction info.
- - Add `MDBX_ACCEDE` flag for database opening.
- - Using OFD-locks and tracking for in-process multi-opening.
- - Hot backup into pipe.
- - Support for cmake & amalgamated sources.
- - Fastest internal sort implementation.
- - New internal dirty-list implementation with lazy sorting.
- - Support for lazy-sync-to-disk with polling.
- - Extended key length.
- - Last update transaction number for each sub-database.
- - Automatic read ahead enabling/disabling.
- - More auto-compactification.
- - Using -fsanitize=undefined and -Wpedantic options.
- - Rework page merging.
- - Nested transactions.
- - API description.
- - Checking for non-local filesystems to avoid DB corruption.
-
--------------------------------------------------------------------------------
-
-For early changes see the git commit history.
+## v0.8.2 2020-07-06
+- Added support multi-opening the same DB in a process with SysV locking (BSD).
+- Fixed warnings & minors for LCC compiler (E2K).
+- Enabled to simultaneously open the same database from processes with and without the `MDBX_WRITEMAP` option.
+- Added key-to-value, `mdbx_get_keycmp()` and `mdbx_get_datacmp()` functions (helpful to avoid using custom comparators).
+- Added `ENABLE_UBSAN` CMake option to enabling the UndefinedBehaviorSanitizer from GCC/CLANG.
+- Workaround for [CLANG bug](https://bugs.llvm.org/show_bug.cgi?id=43275).
+- Returning `MDBX_CORRUPTED` in case all meta-pages are weak and no other error.
+- Refined mode bits while auto-creating LCK-file.
+- Avoids unnecessary database file re-mapping in case geometry changed by another process(es).
+ From the user's point of view, the `MDBX_UNABLE_EXTEND_MAPSIZE` error will now be returned less frequently and only when using the DB in the current process really requires it to be reopened.
+- Remapping on-the-fly and of the database file was implemented.
+ Now remapping with a change of address is performed automatically if there are no dependent readers in the current process.
+
+
+## v0.8.1 2020-06-12
+- Minor change versioning. The last number in the version now means the number of commits since last release/tag.
+- Provide ChangeLog file.
+- Fix for using libmdbx as a C-only sub-project with CMake.
+- Fix `mdbx_env_set_geometry()` for case it is called from an opened environment outside of a write transaction.
+- Add support for huge transactions and `MDBX_HUGE_TRANSACTIONS` build-option (default `OFF`).
+- Refine LTO (link time optimization) for clang.
+- Force enabling exceptions handling for MSVC (`/EHsc` option).
+
+
+## v0.8.0 2020-06-05
+- Support for Android/Bionic.
+- Support for iOS.
+- Auto-handling `MDBX_NOSUBDIR` while opening for any existing database.
+- Engage github-actions to make release-assets.
+- Clarify API description.
+- Extended keygen-cases in stochastic test.
+- Fix fetching of first/lower key from LEAF2-page during page merge.
+- Fix missing comma in array of error messages.
+- Fix div-by-zero while copy-with-compaction for non-resizable environments.
+- Fixes & enhancements for custom-comparators.
+- Fix `MDBX_WITHOUT_MSVC_CRT` option and missing `ntdll.def`.
+- Fix `mdbx_env_close()` to work correctly called concurrently from several threads.
+- Fix null-deref in an ASAN-enabled builds while opening the environment with error and/or read-only.
+- Fix AddressSanitizer errors after closing the environment.
+- Fix/workaround to avoid GCC 10.x pedantic warnings.
+- Fix using `ENODATA` for FreeBSD.
+- Avoid invalidation of DBI-handle(s) when it just closes.
+- Avoid using `pwritev()` for single-writes (up to 10% speedup for some kernels & scenarios).
+- Avoiding `MDBX_UTTERLY_NOSYNC` as result of flags merge.
+- Add `mdbx_dbi_dupsort_depthmask()` function.
+- Add `MDBX_CP_FORCE_RESIZEABLE` option.
+- Add deprecated `MDBX_MAP_RESIZED` for compatibility.
+- Add `MDBX_BUILD_TOOLS` option (default `ON`).
+- Refine `mdbx_dbi_open_ex()` to safe concurrently opening the same handle from different threads.
+- Truncate clk-file during environment closing. So a zero-length lck-file indicates that the environment was closed properly.
+- Refine `mdbx_update_gc()` for huge transactions with small sizes of database page.
+- Extends dump/load to support all MDBX attributes.
+- Avoid upsertion the same key-value data, fix related assertions.
+- Rework min/max length checking for keys & values.
+- Checking the order of keys on all pages during checking.
+- Support `CFLAGS_EXTRA` make-option for convenience.
+- Preserve the last txnid while copying with compactification.
+- Auto-reset running transaction in mdbx_txn_renew().
+- Automatically abort errored transaction in mdbx_txn_commit().
+- Auto-choose page size for large databases.
+- Rearrange source files, rework build, options-support by CMake.
+- Crutch for WSL1 (Windows subsystem for Linux).
+- Refine install/uninstall targets.
+- Support for Valgrind 3.14 and later.
+- Add check-analyzer check-ubsan check-asan check-leak targets to Makefile.
+- Minor fix/workaround to avoid UBSAN traps for `memcpy(ptr, NULL, 0)`.
+- Avoid some GCC-analyzer false-positive warnings.
+
+
+## v0.7.0 2020-03-18
+- Workarounds for Wine (Windows compatibility layer for Linux).
+- `MDBX_MAP_RESIZED` renamed to `MDBX_UNABLE_EXTEND_MAPSIZE`.
+- Clarify API description, fix typos.
+- Speedup runtime checks in debug/checked builds.
+- Added checking for read/write transactions overlapping for the same thread, added `MDBX_TXN_OVERLAPPING` error and `MDBX_DBG_LEGACY_OVERLAP` option.
+- Added `mdbx_key_from_jsonInteger()`, `mdbx_key_from_double()`, `mdbx_key_from_float()`, `mdbx_key_from_int64()` and `mdbx_key_from_int32()` functions. See `mdbx.h` for description.
+- Fix compatibility (use zero for invalid DBI).
+- Refine/clarify error messages.
+- Avoids extra error messages "bad txn" from mdbx_chk when DB is corrupted.
+
+
+## v0.6.0 2020-01-21
+- Fix `mdbx_load` utility for custom comparators.
+- Fix checks related to `MDBX_APPEND` flag inside `mdbx_cursor_put()`.
+- Refine/fix dbi_bind() internals.
+- Refine/fix handling `STATUS_CONFLICTING_ADDRESSES`.
+- Rework `MDBX_DBG_DUMP` option to avoid disk I/O performance degradation.
+- Add built-in help to test tool.
+- Fix `mdbx_env_set_geometry()` for large page size.
+- Fix env_set_geometry() for large pagesize.
+- Clarify API description & comments, fix typos.
+
+
+## v0.5.0 2019-12-31
+- Fix returning MDBX_RESULT_TRUE from page_alloc().
+- Fix false-positive ASAN issue.
+- Fix assertion for `MDBX_NOTLS` option.
+- Rework `MADV_DONTNEED` threshold.
+- Fix `mdbx_chk` utility for don't checking some numbers if walking on the B-tree was disabled.
+- Use page's mp_txnid for basic integrity checking.
+- Add `MDBX_FORCE_ASSERTIONS` built-time option.
+- Rework `MDBX_DBG_DUMP` to avoid performance degradation.
+- Rename `MDBX_NOSYNC` to `MDBX_SAFE_NOSYNC` for clarity.
+- Interpret `ERROR_ACCESS_DENIED` from `OpenProcess()` as 'process exists'.
+- Avoid using `FILE_FLAG_NO_BUFFERING` for compatibility with small database pages.
+- Added install section for CMake.
+
+
+## v0.4.0 2019-12-02
+- Support for Mac OSX, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, OpenSolaris, OpenIndiana (AIX and HP-UX pending).
+- Use bootid for decisions of rollback.
+- Counting retired pages and extended transaction info.
+- Add `MDBX_ACCEDE` flag for database opening.
+- Using OFD-locks and tracking for in-process multi-opening.
+- Hot backup into pipe.
+- Support for cmake & amalgamated sources.
+- Fastest internal sort implementation.
+- New internal dirty-list implementation with lazy sorting.
+- Support for lazy-sync-to-disk with polling.
+- Extended key length.
+- Last update transaction number for each sub-database.
+- Automatic read ahead enabling/disabling.
+- More auto-compactification.
+- Using -fsanitize=undefined and -Wpedantic options.
+- Rework page merging.
+- Nested transactions.
+- API description.
+- Checking for non-local filesystems to avoid DB corruption.
diff --git a/3rdparty/libmdbx/GNUmakefile b/3rdparty/libmdbx/GNUmakefile
index 16a02f864495cacd3f9719715e6ebfe1e7d9045f..d6b9fc7a7cf14a26ce72e8c1f8f54d404c603173 100644
--- a/3rdparty/libmdbx/GNUmakefile
+++ b/3rdparty/libmdbx/GNUmakefile
@@ -166,30 +166,6 @@ help:
@echo " make bench-triplet - run ioarena-benchmark for mdbx, lmdb, sqlite3"
@echo " make bench-quartet - run ioarena-benchmark for mdbx, lmdb, rocksdb, wiredtiger"
@echo " make bench-clean - remove temp database(s) after benchmark"
-#> dist-cutoff-begin
- @echo ""
- @echo " make smoke - fast smoke test"
- @echo " make test - basic test"
- @echo " make check - smoke test with amalgamation and installation checking"
- @echo " make long-test - execute long test which runs for several weeks, or until you interrupt it"
- @echo " make memcheck - build with Valgrind's and smoke test with memcheck tool"
- @echo " make test-valgrind - build with Valgrind's and basic test with memcheck tool"
- @echo " make test-asan - build with AddressSanitizer and basic test"
- @echo " make test-leak - build with LeakSanitizer and basic test"
- @echo " make test-ubsan - build with UndefinedBehaviourSanitizer and basic test"
- @echo " make cross-gcc - check cross-compilation without test execution"
- @echo " make cross-qemu - run cross-compilation and execution basic test with QEMU"
- @echo " make gcc-analyzer - run gcc-analyzer (mostly useless for now)"
- @echo " make build-test - build test executable(s)"
- @echo " make smoke-fault - execute transaction owner failure smoke testcase"
- @echo " make smoke-singleprocess - execute single-process smoke test"
- @echo " make test-singleprocess - execute single-process basic test (also used by make cross-qemu)"
- @echo ""
- @echo " make dist - build amalgamated source code"
- @echo " make doxygen - build HTML documentation"
- @echo " make release-assets - build release assets"
- @echo " make reformat - reformat source code with clang-format"
-#< dist-cutoff-end
show-options:
@echo " MDBX_BUILD_OPTIONS = $(MDBX_BUILD_OPTIONS)"
@@ -226,17 +202,8 @@ options:
@echo "## Assortment items for MDBX_BUILD_OPTIONS:"
@echo "## Note that the defaults should already be correct for most platforms;"
@echo "## you should not need to change any of these. Read their descriptions"
-#> dist-cutoff-begin
-ifeq ($(wildcard mdbx.c),mdbx.c)
-#< dist-cutoff-end
@echo "## in README and source code (see mdbx.c) if you do."
@grep -h '#ifndef MDBX_' mdbx.c | grep -v BUILD | uniq | sed 's/#ifndef / /'
-#> dist-cutoff-begin
-else
- @echo "## in README and source code (see src/options.h) if you do."
- @grep -h '#ifndef MDBX_' src/internals.h src/options.h | grep -v BUILD | uniq | sed 's/#ifndef / /'
-endif
-#< dist-cutoff-end
lib libs libmdbx mdbx: libmdbx.a libmdbx.$(SO_SUFFIX)
@@ -272,9 +239,6 @@ lib-shared libmdbx.$(SO_SUFFIX): mdbx-dylib.o mdbx++-dylib.o
@echo ' LD $@'
$(QUIET)$(CXX) $(CXXFLAGS) $^ -pthread -shared $(LDFLAGS) $(LIB_STDCXXFS) $(LIBS) -o $@
-#> dist-cutoff-begin
-ifeq ($(wildcard mdbx.c),mdbx.c)
-#< dist-cutoff-end
################################################################################
# Amalgamated source code, i.e. distributed after `make dist`
@@ -317,430 +281,6 @@ mdbx_%.static-lto: mdbx_%.c config.h mdbx.c mdbx.h
$(QUIET)$(CC) $(CFLAGS) -Os -flto $(MDBX_BUILD_OPTIONS) '-DLIBMDBX_API=' '-DMDBX_CONFIG_H="config.h"' \
$< mdbx.c $(EXE_LDFLAGS) $(LIBS) -static -Wl,--strip-all -o $@
-#> dist-cutoff-begin
-else
-################################################################################
-# Plain (non-amalgamated) sources with test
-
-.PHONY: build-test build-test-with-valgrind check cross-gcc cross-qemu dist doxygen gcc-analyzer long-test
-.PHONY: reformat release-assets tags smoke test test-asan smoke-fault test-leak
-.PHONY: smoke-singleprocess test-singleprocess test-ubsan test-valgrind memcheck
-.PHONY: smoke-assertion test-assertion long-test-assertion
-
-define uname2osal
- case "$(UNAME)" in
- CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
- *) echo unix;;
- esac
-endef
-
-define uname2titer
- case "$(UNAME)" in
- Darwin*|Mach*) echo 2;;
- *) echo 12;;
- esac
-endef
-
-DIST_EXTRA := LICENSE README.md CMakeLists.txt GNUmakefile Makefile ChangeLog.md VERSION.txt config.h.in ntdll.def \
- $(addprefix man1/, $(MANPAGES)) cmake/compiler.cmake cmake/profile.cmake cmake/utils.cmake
-DIST_SRC := mdbx.h mdbx.h++ mdbx.c mdbx.c++ $(addsuffix .c, $(TOOLS))
-
-TEST_DB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db
-TEST_LOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log
-TEST_OSAL := $(shell $(uname2osal))
-TEST_ITER := $(shell $(uname2titer))
-TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc))
-TEST_INC := $(wildcard test/*.h)
-TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
-TAR ?= $(shell which gnu-tar || echo tar)
-ZIP ?= $(shell which zip || echo "echo 'Please install zip'")
-CLANG_FORMAT ?= $(shell (which clang-format-14 || which clang-format-13 || which clang-format) 2>/dev/null)
-
-reformat:
- @echo ' RUNNING clang-format...'
- $(QUIET)if [ -n "$(CLANG_FORMAT)" ]; then \
- git ls-files | grep -E '\.(c|cxx|cc|cpp|h|hxx|hpp)(\.in)?$$' | xargs -r $(CLANG_FORMAT) -i --style=file; \
- else \
- echo "clang-format version 13..14 not found for 'reformat'"; \
- fi
-
-MAN_SRCDIR := src/man1/
-ALLOY_DEPS := $(shell git ls-files src/)
-git_DIR := $(shell if [ -d .git ]; then echo .git; elif [ -s .git -a -f .git ]; then grep '^gitdir: ' .git | cut -d ':' -f 2; else echo git_directory_is_absent; fi)
-MDBX_GIT_VERSION = $(shell set -o pipefail; git describe --tags '--match=v[0-9]*' 2>&- | sed -n 's|^v*\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\)\(.*\)|\1|p' || echo 'Please fetch tags and/or use non-obsolete git version')
-MDBX_GIT_REVISION = $(shell set -o pipefail; git rev-list `git describe --tags --abbrev=0`..HEAD --count 2>&- || echo 'Please fetch tags and/or use non-obsolete git version')
-MDBX_GIT_TIMESTAMP = $(shell git show --no-patch --format=%cI HEAD 2>&- || echo 'Please install latest get version')
-MDBX_GIT_DESCRIBE = $(shell git describe --tags --long --dirty=-dirty '--match=v[0-9]*' 2>&- || echo 'Please fetch tags and/or install non-obsolete git version')
-MDBX_VERSION_IDENT = $(shell set -o pipefail; echo -n '$(MDBX_GIT_DESCRIBE)' | tr -c -s '[a-zA-Z0-9.]' _)
-MDBX_VERSION_NODOT = $(subst .,_,$(MDBX_VERSION_IDENT))
-MDBX_BUILD_SOURCERY = $(shell set -o pipefail; $(MAKE) IOARENA=false CXXSTD= -s src/version.c >/dev/null && (openssl dgst -r -sha256 src/version.c || sha256sum src/version.c || shasum -a 256 src/version.c) 2>/dev/null | cut -d ' ' -f 1 || (echo 'Please install openssl or sha256sum or shasum' >&2 && echo sha256sum_is_no_available))_$(MDBX_VERSION_NODOT)
-MDBX_DIST_DIR = libmdbx-$(MDBX_VERSION_NODOT)
-
-# Extra options mdbx_test utility
-MDBX_SMOKE_EXTRA ?=
-
-check: DESTDIR = $(shell pwd)/@check-install
-check: test dist install
-
-smoke-assertion: MDBX_BUILD_OPTIONS=-DMDBX_FORCE_ASSERTIONS=1
-smoke-assertion: smoke
-test-assertion: MDBX_BUILD_OPTIONS=-DMDBX_FORCE_ASSERTIONS=1
-test-assertion: smoke
-long-test-assertion: MDBX_BUILD_OPTIONS=-DMDBX_FORCE_ASSERTIONS=1
-long-test-assertion: smoke
-
-smoke: build-test
- @echo ' SMOKE `mdbx_test basic`...'
- $(QUIET)rm -f $(TEST_DB) $(TEST_LOG).gz && (set -o pipefail; \
- (./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) basic && \
- ./mdbx_test --mode=-writemap,-nosync-safe,-lifo --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) basic) \
- | tee >(gzip --stdout >$(TEST_LOG).gz) | tail -n 42) \
- && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
-
-smoke-singleprocess: build-test
- @echo ' SMOKE `mdbx_test --nested`...'
- $(QUIET)rm -f $(TEST_DB) $(TEST_LOG).gz && (set -o pipefail; \
- (./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) --hill && \
- ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
- ./mdbx_test --mode=-writemap,-nosync-safe,-lifo --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) --nested) \
- | tee >(gzip --stdout >$(TEST_LOG).gz) | tail -n 42) \
- && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
-
-smoke-fault: build-test
- @echo ' SMOKE `mdbx_test --inject-writefault=42 basic`...'
- $(QUIET)rm -f $(TEST_DB) $(TEST_LOG).gz && (set -o pipefail; ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --inject-writefault=42 --dump-config --dont-cleanup-after $(MDBX_SMOKE_EXTRA) basic \
- | tee >(gzip --stdout >$(TEST_LOG).gz) | tail -n 42) \
- ; ./mdbx_chk -vvnw $(TEST_DB) && ([ ! -e $(TEST_DB)-copy ] || ./mdbx_chk -vvn $(TEST_DB)-copy)
-
-test: build-test
- @echo ' RUNNING `test/long_stochastic.sh --loops 2`...'
- $(QUIET)test/long_stochastic.sh --loops 2 --db-upto-mb 256 --skip-make >$(TEST_LOG)
-
-long-test: build-test
- @echo ' RUNNING `test/long_stochastic.sh --loops 42`...'
- $(QUIET)test/long_stochastic.sh --loops 42 --db-upto-mb 1024 --skip-make
-
-test-singleprocess: build-test
- @echo ' RUNNING `test/long_stochastic.sh --single --loops 2`...'
- $(QUIET)test/long_stochastic.sh --single --loops 2 --db-upto-mb 256 --skip-make >$(TEST_LOG)
-
-test-valgrind: CFLAGS_EXTRA=-Ofast -DMDBX_USE_VALGRIND
-test-valgrind: build-test
- @echo ' RUNNING `test/long_stochastic.sh --with-valgrind --loops 2`...'
- $(QUIET)test/long_stochastic.sh --with-valgrind --loops 2 --db-upto-mb 256 --skip-make >$(TEST_LOG)
-
-memcheck: VALGRIND=valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt
-memcheck: CFLAGS_EXTRA=-Ofast -DMDBX_USE_VALGRIND
-memcheck: build-test
- @echo " SMOKE \`mdbx_test basic\` under Valgrind's memcheck..."
- $(QUIET)rm -f valgrind-*.log $(TEST_DB) $(TEST_LOG).gz && (set -o pipefail; ( \
- $(VALGRIND) ./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) basic && \
- $(VALGRIND) ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
- $(VALGRIND) ./mdbx_test --mode=-writemap,-nosync-safe,-lifo --progress --console=no --repeat=4 --pathname=$(TEST_DB) --dont-cleanup-after $(MDBX_SMOKE_EXTRA) basic && \
- $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB) && \
- $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB)-copy \
- ) | tee >(gzip --stdout >$(TEST_LOG).gz) | tail -n 42)
-
-gcc-analyzer:
- @echo ' RE-BUILD with `-fanalyzer` option...'
- @echo "NOTE: There a lot of false-positive warnings at 2020-05-01 by pre-release GCC-10 (20200328, Red Hat 10.0.1-0.11)"
- $(QUIET)$(MAKE) IOARENA=false CXXSTD=$(CXXSTD) CFLAGS_EXTRA="-Og -fanalyzer -Wno-error" build-test
-
-test-ubsan:
- @echo ' RE-TEST with `-fsanitize=undefined` option...'
- $(QUIET)$(MAKE) IOARENA=false CXXSTD=$(CXXSTD) CFLAGS_EXTRA="-Ofast -fsanitize=undefined -fsanitize-undefined-trap-on-error" test
-
-test-asan:
- @echo ' RE-TEST with `-fsanitize=address` option...'
- $(QUIET)$(MAKE) IOARENA=false CXXSTD=$(CXXSTD) CFLAGS_EXTRA="-Os -fsanitize=address" test
-
-test-leak:
- @echo ' RE-TEST with `-fsanitize=leak` option...'
- $(QUIET)$(MAKE) IOARENA=false CXXSTD=$(CXXSTD) CFLAGS_EXTRA="-fsanitize=leak" test
-
-mdbx_example: mdbx.h example/example-mdbx.c libmdbx.$(SO_SUFFIX)
- @echo ' CC+LD $@'
- $(QUIET)$(CC) $(CFLAGS) -I. example/example-mdbx.c ./libmdbx.$(SO_SUFFIX) -o $@
-
-build-test: all mdbx_example mdbx_test
-
-define test-rule
-$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) $(HEADERS) $(lastword $(MAKEFILE_LIST))
- @echo ' CC $$@'
- $(QUIET)$$(CXX) $$(CXXFLAGS) $$(MDBX_BUILD_OPTIONS) -c $(1) -o $$@
-
-endef
-$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file))))
-
-mdbx_%: src/mdbx_%.c libmdbx.a
- @echo ' CC+LD $@'
- $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) $(LIBS) -o $@
-
-mdbx_%.static: src/mdbx_%.c mdbx-static.o
- @echo ' CC+LD $@'
- $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) $(LIBS) -static -Wl,--strip-all -o $@
-
-mdbx_%.static-lto: src/mdbx_%.c src/config.h src/version.c src/alloy.c $(ALLOY_DEPS)
- @echo ' CC+LD $@'
- $(QUIET)$(CC) $(CFLAGS) -Os -flto $(MDBX_BUILD_OPTIONS) '-DLIBMDBX_API=' '-DMDBX_CONFIG_H="config.h"' \
- $< src/alloy.c $(EXE_LDFLAGS) $(LIBS) -static -Wl,--strip-all -o $@
-
-mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX)
- @echo ' LD $@'
- $(QUIET)$(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) $(LIBS) -o $@
-
-$(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags:
- @echo '*** ' >&2
- @echo '*** Please don''t use tarballs nor zips which are automatically provided by Github !' >&2
- @echo '*** These archives do not contain version information and thus are unfit to build libmdbx.' >&2
- @echo '*** You can vote for ability of disabling auto-creation such unsuitable archives at https://github.community/t/disable-tarball' >&2
- @echo '*** ' >&2
- @echo '*** Instead of above, just clone the git repository, either download a tarball or zip with the properly amalgamated source core.' >&2
- @echo '*** For embedding libmdbx use a git-submodule or the amalgamated source code.' >&2
- @echo '*** ' >&2
- @echo '*** Please, avoid using any other techniques.' >&2
- @echo '*** ' >&2
- @false
-
-src/version.c: src/version.c.in $(lastword $(MAKEFILE_LIST)) $(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags
- @echo ' MAKE $@'
- $(QUIET)sed \
- -e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \
- -e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \
- -e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \
- -e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \
- -e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 1)|" \
- -e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 2)|" \
- -e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 3)|" \
- -e "s|\$${MDBX_VERSION_REVISION}|$(MDBX_GIT_REVISION)|" \
- src/version.c.in >$@
-
-src/config.h: buildflags.tag src/version.c $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $@'
- $(QUIET)(echo '#define MDBX_BUILD_TIMESTAMP "$(MDBX_BUILD_TIMESTAMP)"' \
- && echo "#define MDBX_BUILD_FLAGS \"$$(cat buildflags.tag)\"" \
- && echo '#define MDBX_BUILD_COMPILER "$(shell (LC_ALL=C $(CC) --version || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \
- && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; (LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || (LC_ALL=C $(CC) --version | grep -qi e2k && echo E2K) || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \
- && echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' \
- ) >$@
-
-mdbx-dylib.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
- @echo ' CC $@'
- $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/alloy.c -o $@
-
-mdbx-static.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
- @echo ' CC $@'
- $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@
-
-docs/Doxyfile: docs/Doxyfile.in src/version.c $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $@'
- $(QUIET)sed \
- -e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \
- -e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \
- -e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \
- -e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \
- -e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 1)|" \
- -e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 2)|" \
- -e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 3)|" \
- -e "s|\$${MDBX_VERSION_REVISION}|$(MDBX_GIT_REVISION)|" \
- docs/Doxyfile.in >$@
-
-define md-extract-section
-docs/__$(1).md: $(2) $(lastword $(MAKEFILE_LIST))
- @echo ' EXTRACT $1'
- $(QUIET)sed -n '/<!-- section-begin $(1) -->/,/<!-- section-end -->/p' $(2) >$$@ && test -s $$@
-
-endef
-$(foreach section,overview mithril characteristics improvements history usage performance bindings,$(eval $(call md-extract-section,$(section),README.md)))
-
-docs/overall.md: docs/__overview.md docs/_toc.md docs/__mithril.md docs/__history.md AUTHORS LICENSE $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $@'
- $(QUIET)echo -e "\\mainpage Overall\n\\section brief Brief" | cat - $(filter %.md, $^) >$@ && echo -e "\n\n\nLicense\n=======\n" | cat AUTHORS - LICENSE >>$@
-
-docs/intro.md: docs/_preface.md docs/__characteristics.md docs/__improvements.md docs/_restrictions.md docs/__performance.md
- @echo ' MAKE $@'
- $(QUIET)cat $^ | sed 's/^Performance comparison$$/Performance comparison {#performance}/;s/^Improvements beyond LMDB$$/Improvements beyond LMDB {#improvements}/' >$@
-
-docs/usage.md: docs/__usage.md docs/_starting.md docs/__bindings.md
- @echo ' MAKE $@'
- $(QUIET)echo -e "\\page usage Usage\n\\section getting Building & Embedding" | cat - $^ | sed 's/^Bindings$$/Bindings {#bindings}/' >$@
-
-doxygen: docs/Doxyfile docs/overall.md docs/intro.md docs/usage.md mdbx.h mdbx.h++ src/options.h ChangeLog.md AUTHORS LICENSE $(lastword $(MAKEFILE_LIST))
- @echo ' RUNNING doxygen...'
- $(QUIET)rm -rf docs/html && \
- cat mdbx.h | tr '\n' '\r' | sed -e 's/LIBMDBX_INLINE_API\s*(\s*\([^,]\+\),\s*\([^,]\+\),\s*(\s*\([^)]\+\)\s*)\s*)\s*{/inline \1 \2(\3) {/g' | tr '\r' '\n' >docs/mdbx.h && \
- cp mdbx.h++ src/options.h ChangeLog.md docs/ && (cd docs && doxygen Doxyfile $(HUSH)) && cp AUTHORS LICENSE docs/html/
-
-mdbx++-dylib.o: src/config.h src/mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST))
- @echo ' CC $@'
- $(QUIET)$(CXX) $(CXXFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/mdbx.c++ -o $@
-
-mdbx++-static.o: src/config.h src/mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST))
- @echo ' CC $@'
- $(QUIET)$(CXX) $(CXXFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/mdbx.c++ -o $@
-
-dist: tags dist-checked.tag libmdbx-sources-$(MDBX_VERSION_IDENT).tar.gz $(lastword $(MAKEFILE_LIST))
- @echo ' AMALGAMATION is done'
-
-tags:
- @echo ' FETCH git tags...'
- $(QUIET)git fetch --tags --force
-
-release-assets: libmdbx-amalgamated-$(MDBX_GIT_VERSION).zpaq \
- libmdbx-amalgamated-$(MDBX_GIT_VERSION).tar.xz \
- libmdbx-amalgamated-$(MDBX_GIT_VERSION).tar.bz2 \
- libmdbx-amalgamated-$(MDBX_GIT_VERSION).tar.gz \
- libmdbx-amalgamated-$(subst .,_,$(MDBX_GIT_VERSION)).zip
- $(QUIET)([ \
- "$$(set -o pipefail; git describe | sed -n '/^v[0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}$$/p' || echo fail-left)" \
- == \
- "$$(git describe --tags --dirty=-dirty || echo fail-right)" ] \
- || (echo 'ERROR: Is not a valid release because not in the clean state with a suitable annotated tag!!!' >&2 && false)) \
- && echo ' RELEASE ASSETS are done'
-
-dist-checked.tag: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA))
- @echo -n ' VERIFY amalgamated sources...'
- $(QUIET)rm -rf $@ dist/@tmp-shared_internals.inc \
- && if grep -R "define xMDBX_ALLOY" dist | grep -q MDBX_BUILD_SOURCERY; then echo "sed output is WRONG!" >&2; exit 2; fi \
- && rm -rf dist-check && cp -r -p dist dist-check && ($(MAKE) IOARENA=false CXXSTD=$(CXXSTD) -C dist-check >dist-check.log 2>dist-check.err || (cat dist-check.err && exit 1)) \
- && touch $@ || (echo " FAILED! See dist-check.log and dist-check.err" >&2; exit 2) && echo " Ok"
-
-%.tar.gz: dist-checked.tag
- @echo ' CREATE $@'
- $(QUIET)$(TAR) -c $(shell LC_ALL=C $(TAR) --help | grep -q -- '--owner' && echo '--owner=0 --group=0') -f - -C dist $(DIST_SRC) $(DIST_EXTRA) | gzip -c -9 >$@
-
-%.tar.xz: dist-checked.tag
- @echo ' CREATE $@'
- $(QUIET)$(TAR) -c $(shell LC_ALL=C $(TAR) --help | grep -q -- '--owner' && echo '--owner=0 --group=0') -f - -C dist $(DIST_SRC) $(DIST_EXTRA) | xz -9 -z >$@
-
-%.tar.bz2: dist-checked.tag
- @echo ' CREATE $@'
- $(QUIET)$(TAR) -c $(shell LC_ALL=C $(TAR) --help | grep -q -- '--owner' && echo '--owner=0 --group=0') -f - -C dist $(DIST_SRC) $(DIST_EXTRA) | bzip2 -9 -z >$@
-
-
-%.zip: dist-checked.tag
- @echo ' CREATE $@'
- $(QUIET)rm -rf $@ && (cd dist && $(ZIP) -9 ../$@ $(DIST_SRC) $(DIST_EXTRA)) &>zip.log
-
-%.zpaq: dist-checked.tag
- @echo ' CREATE $@'
- $(QUIET)rm -rf $@ && (cd dist && zpaq a ../$@ $(DIST_SRC) $(DIST_EXTRA) -m59) &>zpaq.log
-
-dist/mdbx.h: mdbx.h src/version.c $(lastword $(MAKEFILE_LIST))
- @echo ' COPY $@'
- $(QUIET)mkdir -p dist && cp $< $@
-
-dist/mdbx.h++: mdbx.h++ src/version.c $(lastword $(MAKEFILE_LIST))
- @echo ' COPY $@'
- $(QUIET)mkdir -p dist && cp $< $@
-
-dist/@tmp-shared_internals.inc: src/version.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
- @echo ' ALLOYING...'
- $(QUIET)mkdir -p dist \
- && echo '#define xMDBX_ALLOY 1' >dist/@tmp-sed.inc && echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' >>dist/@tmp-sed.inc \
- && sed \
- -e '/#pragma once/r dist/@tmp-sed.inc' \
- -e 's|#include "../mdbx.h"|@INCLUDE "mdbx.h"|' \
- -e '/#include "defs.h"/r src/defs.h' \
- -e '/#include "osal.h"/r src/osal.h' \
- -e '/#include "options.h"/r src/options.h' \
- src/internals.h >$@ \
- && rm -rf dist/@tmp-sed.inc
-
-dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $@'
- $(QUIET)mkdir -p dist && (cat dist/@tmp-shared_internals.inc \
- && cat src/core.c src/osal.c src/version.c src/lck-windows.c src/lck-posix.c | sed \
- -e '/#include "debug_begin.h"/r src/debug_begin.h' \
- -e '/#include "debug_end.h"/r src/debug_end.h' \
- ) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' >$@
-
-dist/mdbx.c++: dist/@tmp-shared_internals.inc src/mdbx.c++ $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $@'
- $(QUIET)mkdir -p dist && (cat dist/@tmp-shared_internals.inc && cat src/mdbx.c++) \
- | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|;s|"mdbx.h"|"mdbx.h++"|' >$@
-
-define dist-tool-rule
-dist/$(1).c: src/$(1).c src/wingetopt.h src/wingetopt.c \
- dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
- @echo ' MAKE $$@'
- $(QUIET)mkdir -p dist && sed \
- -e '/#include "internals.h"/r dist/@tmp-shared_internals.inc' \
- -e '/#include "wingetopt.h"/r src/wingetopt.c' \
- src/$(1).c \
- | grep -v -e '#include "' -e '#pragma once' -e '#define xMDBX_ALLOY' \
- | sed 's|@INCLUDE|#include|' >$$@
-
-endef
-$(foreach file,$(TOOLS),$(eval $(call dist-tool-rule,$(file))))
-
-define dist-extra-rule
-dist/$(1): $(1)
- @echo ' REFINE $$@'
- $(QUIET)mkdir -p $$(dir $$@) && sed -e '/^#> dist-cutoff-begin/,/^#< dist-cutoff-end/d' $$< >$$@
-
-endef
-$(foreach file,$(filter-out man1/% VERSION.txt %.in ntdll.def,$(DIST_EXTRA)),$(eval $(call dist-extra-rule,$(file))))
-
-dist/VERSION.txt: src/version.c
- @echo ' MAKE $@'
- $(QUIET)mkdir -p dist/ && echo "$(MDBX_GIT_VERSION).$(MDBX_GIT_REVISION)" >$@
-
-dist/ntdll.def: src/ntdll.def
- @echo ' COPY $@'
- $(QUIET)mkdir -p dist/cmake/ && cp $< $@
-
-dist/config.h.in: src/config.h.in
- @echo ' COPY $@'
- $(QUIET)mkdir -p dist/cmake/ && cp $< $@
-
-dist/man1/mdbx_%.1: src/man1/mdbx_%.1
- @echo ' COPY $@'
- $(QUIET)mkdir -p dist/man1/ && cp $< $@
-
-endif
-
-################################################################################
-# Cross-compilation simple test
-
-CROSS_LIST = mips-linux-gnu-gcc \
- powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \
- arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc \
- sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc \
- hppa-linux-gnu-gcc s390x-linux-gnu-gcc
-
-## On Ubuntu Focal (20.04) with QEMU 4.2 (1:4.2-3ubuntu6.6) & GCC 9.3 (9.3.0-17ubuntu1~20.04)
-# hppa-linux-gnu-gcc - works (previously: don't supported by qemu)
-# s390x-linux-gnu-gcc - works (previously: qemu hang/abort)
-# sparc64-linux-gnu-gcc - coredump (qemu mmap-troubles, previously: qemu fails fcntl for F_SETLK/F_GETLK)
-# alpha-linux-gnu-gcc - coredump (qemu mmap-troubles)
-CROSS_LIST_NOQEMU = sparc64-linux-gnu-gcc alpha-linux-gnu-gcc
-
-cross-gcc:
- @echo ' Re-building by cross-compiler for: $(CROSS_LIST_NOQEMU) $(CROSS_LIST)'
- @echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED."
- @echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
- $(QUIET)for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \
- echo "===================== $$CC"; \
- $(MAKE) IOARENA=false CXXSTD= clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) IOARENA=false all || exit $$?; \
- done
-
-# Unfortunately qemu don't provide robust support for futexes.
-# Therefore it is impossible to run full multi-process tests.
-cross-qemu:
- @echo ' Re-building by cross-compiler and re-check by QEMU for: $(CROSS_LIST)'
- @echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED."
- @echo "FOR INSTANCE: "
- @echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
- @echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc"
- $(QUIET)for CC in $(CROSS_LIST); do \
- echo "===================== $$CC + qemu"; \
- $(MAKE) IOARENA=false CXXSTD= clean && \
- CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static MDBX_BUILD_OPTIONS="-DMDBX_SAFE4QEMU $(MDBX_BUILD_OPTIONS)" \
- $(MAKE) IOARENA=false smoke-singleprocess test-singleprocess || exit $$?; \
- done
-
-#< dist-cutoff-end
install: $(LIBRARIES) $(TOOLS) $(HEADERS)
@echo ' INSTALLING...'
diff --git a/3rdparty/libmdbx/README.md b/3rdparty/libmdbx/README.md
index 1ac865233d740007707bcd0ff326652f6b6ca140..b27fadb211a231b7f5f9106aac68afa34cafe184 100644
--- a/3rdparty/libmdbx/README.md
+++ b/3rdparty/libmdbx/README.md
@@ -19,14 +19,13 @@ including English å’Œ ä¸æ–‡, in the near future.
*The Future will (be) [Positive](https://www.ptsecurity.com). Ð’ÑÑ‘ будет хорошо.*
-> Please refer to the online [documentation](https://libmdbx.dqdkfa.ru)
-> with [`C` API description](https://libmdbx.dqdkfa.ru/group__c__api.html)
+> Please refer to the online [documentation](https://libmdbx.website.yandexcloud.net)
+> with [`C` API description](https://libmdbx.website.yandexcloud.net/group__c__api.html)
> and pay attention to the [`C++` API](https://gitflic.ru/project/erthink/libmdbx/blob?file=mdbx.h%2B%2B#line-num-1).
> Questions, feedback and suggestions are welcome to the [Telegram' group](https://t.me/libmdbx).
-> For NEWS take a look to the [ChangeLog](https://gitflic.ru/project/erthink/libmdbx/blob?file=ChangeLog.md)
-> or the [TODO](https://gitflic.ru/project/erthink/libmdbx/blob?file=TODO.md).
+> For NEWS take a look to the [ChangeLog](https://gitflic.ru/project/erthink/libmdbx/blob?file=ChangeLog.md).
libmdbx
@@ -220,45 +219,52 @@ the user's point of view.
## Added Features
1. Keys could be more than 2 times longer than _LMDB_.
- > For DB with default page size _libmdbx_ support keys up to 2022 bytes
- > and up to 32742 bytes for 64K page size. _LMDB_ allows key size up to
- > 511 bytes and may silently loses data with large values.
+
+ > For DB with default page size _libmdbx_ support keys up to 2022 bytes
+ > and up to 32742 bytes for 64K page size. _LMDB_ allows key size up to
+ > 511 bytes and may silently loses data with large values.
2. Up to 30% faster than _LMDB_ in [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) benchmarks.
- > Benchmarks of the in-[tmpfs](https://en.wikipedia.org/wiki/Tmpfs) scenarios,
- > that tests the speed of the engine itself, showned that _libmdbx_ 10-20% faster than _LMDB_,
- > and up to 30% faster when _libmdbx_ compiled with specific build options
- > which downgrades several runtime checks to be match with LMDB behaviour.
- >
- > These and other results could be easily reproduced with [ioArena](https://github.com/pmwkaa/ioarena) just by `make bench-quartet` command,
- > including comparisons with [RockDB](https://en.wikipedia.org/wiki/RocksDB)
- > and [WiredTiger](https://en.wikipedia.org/wiki/WiredTiger).
+
+ > Benchmarks of the in-[tmpfs](https://en.wikipedia.org/wiki/Tmpfs) scenarios,
+ > that tests the speed of the engine itself, showned that _libmdbx_ 10-20% faster than _LMDB_,
+ > and up to 30% faster when _libmdbx_ compiled with specific build options
+ > which downgrades several runtime checks to be match with LMDB behaviour.
+ >
+ > These and other results could be easily reproduced with [ioArena](https://github.com/pmwkaa/ioarena) just by `make bench-quartet` command,
+ > including comparisons with [RockDB](https://en.wikipedia.org/wiki/RocksDB)
+ > and [WiredTiger](https://en.wikipedia.org/wiki/WiredTiger).
3. Automatic on-the-fly database size adjustment, both increment and reduction.
- > _libmdbx_ manages the database size according to parameters specified
- > by `mdbx_env_set_geometry()` function,
- > ones include the growth step and the truncation threshold.
- >
- > Unfortunately, on-the-fly database size adjustment doesn't work under [Wine](https://en.wikipedia.org/wiki/Wine_(software))
- > due to its internal limitations and unimplemented functions, i.e. the `MDBX_UNABLE_EXTEND_MAPSIZE` error will be returned.
+
+ > _libmdbx_ manages the database size according to parameters specified
+ > by `mdbx_env_set_geometry()` function,
+ > ones include the growth step and the truncation threshold.
+ >
+ > Unfortunately, on-the-fly database size adjustment doesn't work under [Wine](https://en.wikipedia.org/wiki/Wine_(software))
+ > due to its internal limitations and unimplemented functions, i.e. the `MDBX_UNABLE_EXTEND_MAPSIZE` error will be returned.
4. Automatic continuous zero-overhead database compactification.
- > During each commit _libmdbx_ merges a freeing pages which adjacent with the unallocated area
- > at the end of file, and then truncates unused space when a lot enough of.
+
+ > During each commit _libmdbx_ merges a freeing pages which adjacent with the unallocated area
+ > at the end of file, and then truncates unused space when a lot enough of.
5. The same database format for 32- and 64-bit builds.
- > _libmdbx_ database format depends only on the [endianness](https://en.wikipedia.org/wiki/Endianness) but not on the [bitness](https://en.wiktionary.org/wiki/bitness).
+
+ > _libmdbx_ database format depends only on the [endianness](https://en.wikipedia.org/wiki/Endianness) but not on the [bitness](https://en.wiktionary.org/wiki/bitness).
6. LIFO policy for Garbage Collection recycling. This can significantly increase write performance due write-back disk cache up to several times in a best case scenario.
- > LIFO means that for reuse will be taken the latest becomes unused pages.
- > Therefore the loop of database pages circulation becomes as short as possible.
- > In other words, the set of pages, that are (over)written in memory and on disk during a series of write transactions, will be as small as possible.
- > Thus creates ideal conditions for the battery-backed or flash-backed disk cache efficiency.
+
+ > LIFO means that for reuse will be taken the latest becomes unused pages.
+ > Therefore the loop of database pages circulation becomes as short as possible.
+ > In other words, the set of pages, that are (over)written in memory and on disk during a series of write transactions, will be as small as possible.
+ > Thus creates ideal conditions for the battery-backed or flash-backed disk cache efficiency.
7. Fast estimation of range query result volume, i.e. how many items can
be found between a `KEY1` and a `KEY2`. This is a prerequisite for build
and/or optimize query execution plans.
- > _libmdbx_ performs a rough estimate based on common B-tree pages of the paths from root to corresponding keys.
+
+ > _libmdbx_ performs a rough estimate based on common B-tree pages of the paths from root to corresponding keys.
8. `mdbx_chk` utility for database integrity check.
Since version 0.9.1, the utility supports checking the database using any of the three meta pages and the ability to switch to it.
@@ -271,12 +277,14 @@ Since version 0.9.1, the utility supports checking the database using any of the
or not, that allows to avoid copy-out before updates.
12. Extended information of whole-database, sub-databases, transactions, readers enumeration.
- > _libmdbx_ provides a lot of information, including dirty and leftover pages
- > for a write transaction, reading lag and holdover space for read transactions.
+
+ > _libmdbx_ provides a lot of information, including dirty and leftover pages
+ > for a write transaction, reading lag and holdover space for read transactions.
13. Extended update and delete operations.
- > _libmdbx_ allows one _at once_ with getting previous value
- > and addressing the particular item from multi-value with the same key.
+
+ > _libmdbx_ allows one _at once_ with getting previous value
+ > and addressing the particular item from multi-value with the same key.
14. Useful runtime options for tuning engine to application's requirements and use cases specific.
@@ -306,9 +314,10 @@ pre-opening is not needed.
4. Returning `MDBX_EMULTIVAL` error in case of ambiguous update or delete.
5. Guarantee of database integrity even in asynchronous unordered write-to-disk mode.
- > _libmdbx_ propose additional trade-off by `MDBX_SAFE_NOSYNC` with append-like manner for updates,
- > that avoids database corruption after a system crash contrary to LMDB.
- > Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode is available to match LMDB's behaviour for `MDB_NOSYNC`.
+
+ > _libmdbx_ propose additional trade-off by `MDBX_SAFE_NOSYNC` with append-like manner for updates,
+ > that avoids database corruption after a system crash contrary to LMDB.
+ > Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode is available to match LMDB's behaviour for `MDB_NOSYNC`.
6. On **MacOS & iOS** the `fcntl(F_FULLFSYNC)` syscall is used _by
default_ to synchronize data with the disk, as this is [the only way to
@@ -335,7 +344,7 @@ named mutexes are used.
Historically, _libmdbx_ is a deeply revised and extended descendant of the
[Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
At first the development was carried out within the
-[ReOpenLDAP](https://web.archive.org/web/20220414235959/https://github.com/erthink/ReOpenLDAP) project. About a
+[ReOpenLDAP](todo4recovery://erased_by_github/ReOpenLDAP) project. About a
year later _libmdbx_ was separated into a standalone project, which was
[presented at Highload++ 2015
conference](http://www.highload.ru/2015/abstracts/1831.html).
@@ -348,7 +357,7 @@ explanation, deleted _libmdbx_ along with a lot of other projects,
simultaneously blocking access for many developers. Therefore on
2022-04-21 we have migrated to a reliable trusted infrastructure.
The origin for now is at [GitFlic](https://gitflic.ru/project/erthink/libmdbx)
-with backup at [ABF by ROSA Лаб](https://abf.rosalinux.ru/erthink/libmdbx).
+with backup at [ABF by ROSA Лаб](https://abf.io/erthink/libmdbx).
For the same reason ~~Github~~ is blacklisted forever.
## Acknowledgments
@@ -377,12 +386,14 @@ since release the version 1.0.
_libmdbx_ provides two official ways for integration in source code form:
1. Using the amalgamated source code.
- > The amalgamated source code includes all files required to build and
- > use _libmdbx_, but not for testing _libmdbx_ itself.
+
+ > The amalgamated source code includes all files required to build and
+ > use _libmdbx_, but not for testing _libmdbx_ itself.
2. Adding the complete original source code as a `git submodule`.
- > This allows you to build as _libmdbx_ and testing tool.
- > On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool.
+
+ > This allows you to build as _libmdbx_ and testing tool.
+ > On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool.
_**Please, avoid using any other techniques.**_ Otherwise, at least
don't ask for support and don't name such chimeras `libmdbx`.
@@ -417,7 +428,7 @@ unexpected or broken down.
### Testing
The amalgamated source code does not contain any tests for or several reasons.
-Please read [the explanation](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/214#issuecomment-870717981) and don't ask to alter this.
+Please read [the explanation](todo4recovery://erased_by_github/libmdbx/issues/214#issuecomment-870717981) and don't ask to alter this.
So for testing _libmdbx_ itself you need a full source code, i.e. the clone of a git repository, there is no option.
The full source code of _libmdbx_ has a [`test` subdirectory](https://gitflic.ru/project/erthink/libmdbx/tree/master/test) with minimalistic test "framework".
@@ -579,7 +590,7 @@ from the [ios-cmake](https://github.com/leetal/ios-cmake) project.
## API description
-Please refer to the online [_libmdbx_ API reference](https://libmdbx.dqdkfa.ru/docs)
+Please refer to the online [_libmdbx_ API reference](https://libmdbx.website.yandexcloud.net/docs)
and/or see the [mdbx.h++](mdbx.h%2B%2B) and [mdbx.h](mdbx.h) headers.
<!-- section-begin bindings -->
@@ -596,11 +607,10 @@ Bindings
| Ruby | [ruby-mdbx](https://rubygems.org/gems/mdbx/) | [Mahlon E. Smith](https://github.com/mahlonsmith) |
| Go | [mdbx-go](https://github.com/torquem-ch/mdbx-go) | [Alex Sharov](https://github.com/AskAlexSharov) |
| [Nim](https://en.wikipedia.org/wiki/Nim_(programming_language)) | [NimDBX](https://github.com/snej/nimdbx) | [Jens Alfke](https://github.com/snej)
-| Lua | [lua-libmdbx](https://github.com/mah0x211/lua-libmdbx) | [Masatoshi Fukunaga](https://github.com/mah0x211) |
| Rust | [libmdbx-rs](https://github.com/vorot93/libmdbx-rs) | [Artem Vorotnikov](https://github.com/vorot93) |
| Rust | [mdbx](https://crates.io/crates/mdbx) | [gcxfd](https://github.com/gcxfd) |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
-| Python (draft) | [python-bindings](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/commits/python-bindings) branch | [Noel Kuntze](https://github.com/Thermi)
+| Python (draft) | [python-bindings](todo4recovery://erased_by_github/libmdbx/commits/python-bindings) branch | [Noel Kuntze](https://github.com/Thermi)
| .NET (obsolete) | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
<!-- section-end -->
@@ -639,7 +649,7 @@ Here showed sum of performance metrics in 3 benchmarks:
2. Performance gap is too high to compare in any meaningful way.
-
+
--------------------------------------------------------------------------------
@@ -648,7 +658,7 @@ Here showed sum of performance metrics in 3 benchmarks:
Summary performance with concurrent read/search queries in 1-2-4-8
threads on the machine with 4 logical CPUs in HyperThreading mode (i.e. actually 2 physical CPU cores).
-
+
--------------------------------------------------------------------------------
@@ -670,7 +680,7 @@ In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on an empty database
and after full run the database contains 10,000 small key-value records.
-
+
--------------------------------------------------------------------------------
@@ -697,7 +707,7 @@ and after full run the database contains 100,000 small key-value
records.
-
+
--------------------------------------------------------------------------------
@@ -721,7 +731,7 @@ In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on an empty database
and after full run the database contains 10,000 small key-value records.
-
+
--------------------------------------------------------------------------------
@@ -745,6 +755,6 @@ All benchmark data is gathered by
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html)
syscall and by scanning the data directory.
-
+
<!-- section-end -->
diff --git a/3rdparty/libmdbx/TODO.md b/3rdparty/libmdbx/TODO.md
deleted file mode 100644
index 79c4e54d761bbfbd5188ee96eaf9b805a9072fc2..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/TODO.md
+++ /dev/null
@@ -1,25 +0,0 @@
-TODO
-----
-
-Unfortunately, on 2022-04-15 the Github administration, without any
-warning nor explanation, deleted _libmdbx_ along with a lot of other
-projects, simultaneously blocking access for many developers. Therefore
-on 2022-04-21 we have migrated to a reliable trusted infrastructure.
-The origin for now is at[GitFlic](https://gitflic.ru/project/erthink/libmdbx)
-with backup at [ABF by ROSA Лаб](https://abf.rosalinux.ru/erthink/libmdbx).
-For the same reason ~~Github~~ is blacklisted forever.
-
-So currently most of the links are broken due to noted malicious ~~Github~~ sabotage.
-
- - [Engage an "overlapped I/O" on Windows](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/224).
- - [Simple careful mode for working with corrupted DB](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/223).
- - [Move most of `mdbx_chk` functional to the library API](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/204).
- - [Replace SRW-lock on Windows to allow shrink DB with `MDBX_NOTLS` option](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/210).
- - [More flexible support of asynchronous runtime/framework(s)](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/200).
- - [Migration guide from LMDB to MDBX](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/199).
- - [Get rid of dirty-pages list in MDBX_WRITEMAP mode](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/193).
- - [Large/Overflow pages accounting for dirty-room](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/192).
- - [Support for RAW devices](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/124).
- - [Support MessagePack for Keys & Values](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/115).
- - [Engage new terminology](https://web.archive.org/web/20220414235959/https://github.com/erthink/libmdbx/issues/137).
- - Packages for [Astra Linux](https://astralinux.ru/), [ALT Linux](https://www.altlinux.org/), [ROSA Linux](https://www.rosalinux.ru/), etc.
diff --git a/3rdparty/libmdbx/VERSION.txt b/3rdparty/libmdbx/VERSION.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a9da9632c3d3a984f84670e0e4d1458e18896c49
--- /dev/null
+++ b/3rdparty/libmdbx/VERSION.txt
@@ -0,0 +1 @@
+0.11.7.0
diff --git a/3rdparty/libmdbx/src/config.h.in b/3rdparty/libmdbx/config.h.in
similarity index 100%
rename from 3rdparty/libmdbx/src/config.h.in
rename to 3rdparty/libmdbx/config.h.in
diff --git a/3rdparty/libmdbx/docs/Doxyfile.in b/3rdparty/libmdbx/docs/Doxyfile.in
deleted file mode 100644
index f8d1cf2790d9cf86dfb9cede6bfc1d925ace4730..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/docs/Doxyfile.in
+++ /dev/null
@@ -1,2635 +0,0 @@
-# Doxyfile 1.9.1
-
-# This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project.
-#
-# All text after a double hash (##) is considered a comment and is placed in
-# front of the TAG it is preceding.
-#
-# All text after a single hash (#) is considered a comment and will be ignored.
-# The format is:
-# TAG = value [value, ...]
-# For lists, items can also be appended using:
-# TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (\" \").
-
-#---------------------------------------------------------------------------
-# Project related configuration options
-#---------------------------------------------------------------------------
-
-# This tag specifies the encoding used for all characters in the configuration
-# file that follow. The default is UTF-8 which is also the encoding used for all
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the
-# iconv built into libc) for the transcoding. See
-# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
-# The default value is: UTF-8.
-
-DOXYFILE_ENCODING = UTF-8
-
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
-# double-quotes, unless you are using Doxywizard) that should identify the
-# project for which the documentation is generated. This name is used in the
-# title of most generated pages and in a few other places.
-# The default value is: My Project.
-
-PROJECT_NAME = libmdbx
-
-# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
-# could be handy for archiving the generated documentation or if some version
-# control system is used.
-
-PROJECT_NUMBER = "${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE}.${MDBX_VERSION_REVISION} (@MDBX_GIT_TIMESTAMP@)"
-
-# Using the PROJECT_BRIEF tag one can provide an optional one line description
-# for a project that appears at the top of each page and should give viewer a
-# quick idea about the purpose of the project. Keep the description short.
-
-PROJECT_BRIEF = "One of the fastest compact embeddable key-value ACID database without WAL."
-
-# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
-# in the documentation. The maximum height of the logo should not exceed 55
-# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
-# the logo to the output directory.
-
-PROJECT_LOGO =
-
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
-# into which the generated documentation will be written. If a relative path is
-# entered, it will be relative to the location where doxygen was started. If
-# left blank the current directory will be used.
-
-OUTPUT_DIRECTORY = .
-
-# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
-# directories (in 2 levels) under the output directory of each output format and
-# will distribute the generated files over these directories. Enabling this
-# option can be useful when feeding doxygen a huge amount of source files, where
-# putting all generated files in the same directory would otherwise causes
-# performance problems for the file system.
-# The default value is: NO.
-
-CREATE_SUBDIRS = NO
-
-# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
-# characters to appear in the names of generated files. If set to NO, non-ASCII
-# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
-# U+3044.
-# The default value is: NO.
-
-ALLOW_UNICODE_NAMES = NO
-
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all constant output in the proper language.
-# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
-# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
-# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
-# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
-# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
-# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
-# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
-# Ukrainian and Vietnamese.
-# The default value is: English.
-
-OUTPUT_LANGUAGE = English
-
-# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all generated output in the proper direction.
-# Possible values are: None, LTR, RTL and Context.
-# The default value is: None.
-
-OUTPUT_TEXT_DIRECTION = None
-
-# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
-# descriptions after the members that are listed in the file and class
-# documentation (similar to Javadoc). Set to NO to disable this.
-# The default value is: YES.
-
-BRIEF_MEMBER_DESC = YES
-
-# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
-# description of a member or function before the detailed description
-#
-# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
-# brief descriptions will be completely suppressed.
-# The default value is: YES.
-
-REPEAT_BRIEF = YES
-
-# This tag implements a quasi-intelligent brief description abbreviator that is
-# used to form the text in various listings. Each string in this list, if found
-# as the leading text of the brief description, will be stripped from the text
-# and the result, after processing the whole list, is used as the annotated
-# text. Otherwise, the brief description is used as-is. If left blank, the
-# following values are used ($name is automatically replaced with the name of
-# the entity):The $name class, The $name widget, The $name file, is, provides,
-# specifies, contains, represents, a, an and the.
-
-ABBREVIATE_BRIEF = "The $name class" \
- "The $name widget" \
- "The $name file" \
- is \
- provides \
- specifies \
- contains \
- represents \
- a \
- an \
- the
-
-# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
-# doxygen will generate a detailed section even if there is only a brief
-# description.
-# The default value is: NO.
-
-ALWAYS_DETAILED_SEC = NO
-
-# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
-# inherited members of a class in the documentation of that class as if those
-# members were ordinary class members. Constructors, destructors and assignment
-# operators of the base classes will not be shown.
-# The default value is: NO.
-
-INLINE_INHERITED_MEMB = NO
-
-# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
-# before files name in the file list and in the header files. If set to NO the
-# shortest path that makes the file name unique will be used
-# The default value is: YES.
-
-FULL_PATH_NAMES = YES
-
-# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
-# Stripping is only done if one of the specified strings matches the left-hand
-# part of the path. The tag can be used to show relative paths in the file list.
-# If left blank the directory from which doxygen is run is used as the path to
-# strip.
-#
-# Note that you can specify absolute paths here, but also relative paths, which
-# will be relative from the directory where doxygen is started.
-# This tag requires that the tag FULL_PATH_NAMES is set to YES.
-
-STRIP_FROM_PATH =
-
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
-# path mentioned in the documentation of a class, which tells the reader which
-# header file to include in order to use a class. If left blank only the name of
-# the header file containing the class definition is used. Otherwise one should
-# specify the list of include paths that are normally passed to the compiler
-# using the -I flag.
-
-STRIP_FROM_INC_PATH =
-
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
-# less readable) file names. This can be useful is your file systems doesn't
-# support long names like on DOS, Mac, or CD-ROM.
-# The default value is: NO.
-
-SHORT_NAMES = NO
-
-# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
-# first line (until the first dot) of a Javadoc-style comment as the brief
-# description. If set to NO, the Javadoc-style will behave just like regular Qt-
-# style comments (thus requiring an explicit @brief command for a brief
-# description.)
-# The default value is: NO.
-
-JAVADOC_AUTOBRIEF = NO
-
-# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
-# such as
-# /***************
-# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
-# Javadoc-style will behave just like regular comments and it will not be
-# interpreted by doxygen.
-# The default value is: NO.
-
-JAVADOC_BANNER = NO
-
-# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
-# line (until the first dot) of a Qt-style comment as the brief description. If
-# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
-# requiring an explicit \brief command for a brief description.)
-# The default value is: NO.
-
-QT_AUTOBRIEF = NO
-
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
-# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
-# a brief description. This used to be the default behavior. The new default is
-# to treat a multi-line C++ comment block as a detailed description. Set this
-# tag to YES if you prefer the old behavior instead.
-#
-# Note that setting this tag to YES also means that rational rose comments are
-# not recognized any more.
-# The default value is: NO.
-
-MULTILINE_CPP_IS_BRIEF = NO
-
-# By default Python docstrings are displayed as preformatted text and doxygen's
-# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the
-# doxygen's special commands can be used and the contents of the docstring
-# documentation blocks is shown as doxygen documentation.
-# The default value is: YES.
-
-PYTHON_DOCSTRING = YES
-
-# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
-# documentation from any documented member that it re-implements.
-# The default value is: YES.
-
-INHERIT_DOCS = YES
-
-# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
-# page for each member. If set to NO, the documentation of a member will be part
-# of the file/class/namespace that contains it.
-# The default value is: NO.
-
-SEPARATE_MEMBER_PAGES = NO
-
-# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
-# uses this value to replace tabs by spaces in code fragments.
-# Minimum value: 1, maximum value: 16, default value: 4.
-
-TAB_SIZE = 4
-
-# This tag can be used to specify a number of aliases that act as commands in
-# the documentation. An alias has the form:
-# name=value
-# For example adding
-# "sideeffect=@par Side Effects:\n"
-# will allow you to put the command \sideeffect (or @sideeffect) in the
-# documentation, which will result in a user-defined paragraph with heading
-# "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines (in the resulting output). You can put ^^ in the value part of an
-# alias to insert a newline as if a physical newline was in the original file.
-# When you need a literal { or } or , in the value part of an alias you have to
-# escape them by means of a backslash (\), this can lead to conflicts with the
-# commands \{ and \} for these it is advised to use the version @{ and @} or use
-# a double escape (\\{ and \\})
-
-ALIASES =
-
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
-# only. Doxygen will then generate output that is more tailored for C. For
-# instance, some of the names that are used will be different. The list of all
-# members will be omitted, etc.
-# The default value is: NO.
-
-OPTIMIZE_OUTPUT_FOR_C = NO
-
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
-# Python sources only. Doxygen will then generate output that is more tailored
-# for that language. For instance, namespaces will be presented as packages,
-# qualified scopes will look different, etc.
-# The default value is: NO.
-
-OPTIMIZE_OUTPUT_JAVA = NO
-
-# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
-# sources. Doxygen will then generate output that is tailored for Fortran.
-# The default value is: NO.
-
-OPTIMIZE_FOR_FORTRAN = NO
-
-# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
-# sources. Doxygen will then generate output that is tailored for VHDL.
-# The default value is: NO.
-
-OPTIMIZE_OUTPUT_VHDL = NO
-
-# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
-# sources only. Doxygen will then generate output that is more tailored for that
-# language. For instance, namespaces will be presented as modules, types will be
-# separated into more groups, etc.
-# The default value is: NO.
-
-OPTIMIZE_OUTPUT_SLICE = NO
-
-# Doxygen selects the parser to use depending on the extension of the files it
-# parses. With this tag you can assign which parser to use for a given
-# extension. Doxygen has a built-in mapping, but you can override or extend it
-# using this tag. The format is ext=language, where ext is a file extension, and
-# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
-# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL,
-# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
-# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
-# tries to guess whether the code is fixed or free formatted code, this is the
-# default for Fortran type files). For instance to make doxygen treat .inc files
-# as Fortran files (default is PHP), and .f files as C (default is Fortran),
-# use: inc=Fortran f=C.
-#
-# Note: For files without extension you can use no_extension as a placeholder.
-#
-# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
-# the files are not read by doxygen. When specifying no_extension you should add
-# * to the FILE_PATTERNS.
-#
-# Note see also the list of default file extension mappings.
-
-EXTENSION_MAPPING =
-
-# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
-# according to the Markdown format, which allows for more readable
-# documentation. See https://daringfireball.net/projects/markdown/ for details.
-# The output of markdown processing is further processed by doxygen, so you can
-# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
-# case of backward compatibilities issues.
-# The default value is: YES.
-
-MARKDOWN_SUPPORT = YES
-
-# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
-# to that level are automatically included in the table of contents, even if
-# they do not have an id attribute.
-# Note: This feature currently applies only to Markdown headings.
-# Minimum value: 0, maximum value: 99, default value: 5.
-# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
-
-TOC_INCLUDE_HEADINGS = 5
-
-# When enabled doxygen tries to link words that correspond to documented
-# classes, or namespaces to their corresponding documentation. Such a link can
-# be prevented in individual cases by putting a % sign in front of the word or
-# globally by setting AUTOLINK_SUPPORT to NO.
-# The default value is: YES.
-
-AUTOLINK_SUPPORT = YES
-
-# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
-# to include (a tag file for) the STL sources as input, then you should set this
-# tag to YES in order to let doxygen match functions declarations and
-# definitions whose arguments contain STL classes (e.g. func(std::string);
-# versus func(std::string) {}). This also make the inheritance and collaboration
-# diagrams that involve STL classes more complete and accurate.
-# The default value is: NO.
-
-BUILTIN_STL_SUPPORT = YES
-
-# If you use Microsoft's C++/CLI language, you should set this option to YES to
-# enable parsing support.
-# The default value is: NO.
-
-CPP_CLI_SUPPORT = NO
-
-# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
-# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
-# will parse them like normal C++ but will assume all classes use public instead
-# of private inheritance when no explicit protection keyword is present.
-# The default value is: NO.
-
-SIP_SUPPORT = NO
-
-# For Microsoft's IDL there are propget and propput attributes to indicate
-# getter and setter methods for a property. Setting this option to YES will make
-# doxygen to replace the get and set methods by a property in the documentation.
-# This will only work if the methods are indeed getting or setting a simple
-# type. If this is not the case, or you want to show the methods anyway, you
-# should set this option to NO.
-# The default value is: YES.
-
-IDL_PROPERTY_SUPPORT = YES
-
-# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
-# tag is set to YES then doxygen will reuse the documentation of the first
-# member in the group (if any) for the other members of the group. By default
-# all members of a group must be documented explicitly.
-# The default value is: NO.
-
-DISTRIBUTE_GROUP_DOC = NO
-
-# If one adds a struct or class to a group and this option is enabled, then also
-# any nested class or struct is added to the same group. By default this option
-# is disabled and one has to add nested compounds explicitly via \ingroup.
-# The default value is: NO.
-
-GROUP_NESTED_COMPOUNDS = NO
-
-# Set the SUBGROUPING tag to YES to allow class member groups of the same type
-# (for instance a group of public functions) to be put as a subgroup of that
-# type (e.g. under the Public Functions section). Set it to NO to prevent
-# subgrouping. Alternatively, this can be done per class using the
-# \nosubgrouping command.
-# The default value is: YES.
-
-SUBGROUPING = YES
-
-# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
-# are shown inside the group in which they are included (e.g. using \ingroup)
-# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
-# and RTF).
-#
-# Note that this feature does not work in combination with
-# SEPARATE_MEMBER_PAGES.
-# The default value is: NO.
-
-INLINE_GROUPED_CLASSES = YES
-
-# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
-# with only public data fields or simple typedef fields will be shown inline in
-# the documentation of the scope in which they are defined (i.e. file,
-# namespace, or group documentation), provided this scope is documented. If set
-# to NO, structs, classes, and unions are shown on a separate page (for HTML and
-# Man pages) or section (for LaTeX and RTF).
-# The default value is: NO.
-
-INLINE_SIMPLE_STRUCTS = YES
-
-# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
-# enum is documented as struct, union, or enum with the name of the typedef. So
-# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
-# with name TypeT. When disabled the typedef will appear as a member of a file,
-# namespace, or class. And the struct will be named TypeS. This can typically be
-# useful for C code in case the coding convention dictates that all compound
-# types are typedef'ed and only the typedef is referenced, never the tag name.
-# The default value is: NO.
-
-TYPEDEF_HIDES_STRUCT = YES
-
-# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
-# cache is used to resolve symbols given their name and scope. Since this can be
-# an expensive process and often the same symbol appears multiple times in the
-# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
-# doxygen will become slower. If the cache is too large, memory is wasted. The
-# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
-# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
-# symbols. At the end of a run doxygen will report the cache usage and suggest
-# the optimal cache size from a speed point of view.
-# Minimum value: 0, maximum value: 9, default value: 0.
-
-LOOKUP_CACHE_SIZE = 0
-
-# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use
-# during processing. When set to 0 doxygen will based this on the number of
-# cores available in the system. You can set it explicitly to a value larger
-# than 0 to get more control over the balance between CPU load and processing
-# speed. At this moment only the input processing can be done using multiple
-# threads. Since this is still an experimental feature the default is set to 1,
-# which efficively disables parallel processing. Please report any issues you
-# encounter. Generating dot graphs in parallel is controlled by the
-# DOT_NUM_THREADS setting.
-# Minimum value: 0, maximum value: 32, default value: 1.
-
-NUM_PROC_THREADS = 1
-
-#---------------------------------------------------------------------------
-# Build related configuration options
-#---------------------------------------------------------------------------
-
-# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
-# documentation are documented, even if no documentation was available. Private
-# class members and static file members will be hidden unless the
-# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
-# Note: This will also disable the warnings about undocumented members that are
-# normally produced when WARNINGS is set to YES.
-# The default value is: NO.
-
-EXTRACT_ALL = YES
-
-# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
-# be included in the documentation.
-# The default value is: NO.
-
-EXTRACT_PRIVATE = NO
-
-# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
-# methods of a class will be included in the documentation.
-# The default value is: NO.
-
-EXTRACT_PRIV_VIRTUAL = NO
-
-# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
-# scope will be included in the documentation.
-# The default value is: NO.
-
-EXTRACT_PACKAGE = NO
-
-# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
-# included in the documentation.
-# The default value is: NO.
-
-EXTRACT_STATIC = NO
-
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
-# locally in source files will be included in the documentation. If set to NO,
-# only classes defined in header files are included. Does not have any effect
-# for Java sources.
-# The default value is: YES.
-
-EXTRACT_LOCAL_CLASSES = YES
-
-# This flag is only useful for Objective-C code. If set to YES, local methods,
-# which are defined in the implementation section but not in the interface are
-# included in the documentation. If set to NO, only methods in the interface are
-# included.
-# The default value is: NO.
-
-EXTRACT_LOCAL_METHODS = NO
-
-# If this flag is set to YES, the members of anonymous namespaces will be
-# extracted and appear in the documentation as a namespace called
-# 'anonymous_namespace{file}', where file will be replaced with the base name of
-# the file that contains the anonymous namespace. By default anonymous namespace
-# are hidden.
-# The default value is: NO.
-
-EXTRACT_ANON_NSPACES = NO
-
-# If this flag is set to YES, the name of an unnamed parameter in a declaration
-# will be determined by the corresponding definition. By default unnamed
-# parameters remain unnamed in the output.
-# The default value is: YES.
-
-RESOLVE_UNNAMED_PARAMS = YES
-
-# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
-# undocumented members inside documented classes or files. If set to NO these
-# members will be included in the various overviews, but no documentation
-# section is generated. This option has no effect if EXTRACT_ALL is enabled.
-# The default value is: NO.
-
-HIDE_UNDOC_MEMBERS = NO
-
-# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
-# undocumented classes that are normally visible in the class hierarchy. If set
-# to NO, these classes will be included in the various overviews. This option
-# has no effect if EXTRACT_ALL is enabled.
-# The default value is: NO.
-
-HIDE_UNDOC_CLASSES = NO
-
-# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
-# declarations. If set to NO, these declarations will be included in the
-# documentation.
-# The default value is: NO.
-
-HIDE_FRIEND_COMPOUNDS = NO
-
-# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
-# documentation blocks found inside the body of a function. If set to NO, these
-# blocks will be appended to the function's detailed documentation block.
-# The default value is: NO.
-
-HIDE_IN_BODY_DOCS = NO
-
-# The INTERNAL_DOCS tag determines if documentation that is typed after a
-# \internal command is included. If the tag is set to NO then the documentation
-# will be excluded. Set it to YES to include the internal documentation.
-# The default value is: NO.
-
-INTERNAL_DOCS = NO
-
-# With the correct setting of option CASE_SENSE_NAMES doxygen will better be
-# able to match the capabilities of the underlying filesystem. In case the
-# filesystem is case sensitive (i.e. it supports files in the same directory
-# whose names only differ in casing), the option must be set to YES to properly
-# deal with such files in case they appear in the input. For filesystems that
-# are not case sensitive the option should be be set to NO to properly deal with
-# output files written for symbols that only differ in casing, such as for two
-# classes, one named CLASS and the other named Class, and to also support
-# references to files without having to specify the exact matching casing. On
-# Windows (including Cygwin) and MacOS, users should typically set this option
-# to NO, whereas on Linux or other Unix flavors it should typically be set to
-# YES.
-# The default value is: system dependent.
-
-CASE_SENSE_NAMES = NO
-
-# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
-# their full class and namespace scopes in the documentation. If set to YES, the
-# scope will be hidden.
-# The default value is: NO.
-
-HIDE_SCOPE_NAMES = NO
-
-# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
-# append additional text to a page's title, such as Class Reference. If set to
-# YES the compound reference will be hidden.
-# The default value is: NO.
-
-HIDE_COMPOUND_REFERENCE= NO
-
-# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
-# the files that are included by a file in the documentation of that file.
-# The default value is: YES.
-
-SHOW_INCLUDE_FILES = YES
-
-# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
-# grouped member an include statement to the documentation, telling the reader
-# which file to include in order to use the member.
-# The default value is: NO.
-
-SHOW_GROUPED_MEMB_INC = NO
-
-# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
-# files with double quotes in the documentation rather than with sharp brackets.
-# The default value is: NO.
-
-FORCE_LOCAL_INCLUDES = NO
-
-# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
-# documentation for inline members.
-# The default value is: YES.
-
-INLINE_INFO = YES
-
-# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
-# (detailed) documentation of file and class members alphabetically by member
-# name. If set to NO, the members will appear in declaration order.
-# The default value is: YES.
-
-SORT_MEMBER_DOCS = YES
-
-# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
-# descriptions of file, namespace and class members alphabetically by member
-# name. If set to NO, the members will appear in declaration order. Note that
-# this will also influence the order of the classes in the class list.
-# The default value is: NO.
-
-SORT_BRIEF_DOCS = NO
-
-# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
-# (brief and detailed) documentation of class members so that constructors and
-# destructors are listed first. If set to NO the constructors will appear in the
-# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
-# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
-# member documentation.
-# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
-# detailed member documentation.
-# The default value is: NO.
-
-SORT_MEMBERS_CTORS_1ST = YES
-
-# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
-# of group names into alphabetical order. If set to NO the group names will
-# appear in their defined order.
-# The default value is: NO.
-
-SORT_GROUP_NAMES = NO
-
-# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
-# fully-qualified names, including namespaces. If set to NO, the class list will
-# be sorted only by class name, not including the namespace part.
-# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
-# Note: This option applies only to the class list, not to the alphabetical
-# list.
-# The default value is: NO.
-
-SORT_BY_SCOPE_NAME = NO
-
-# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
-# type resolution of all parameters of a function it will reject a match between
-# the prototype and the implementation of a member function even if there is
-# only one candidate or it is obvious which candidate to choose by doing a
-# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
-# accept a match between prototype and implementation in such cases.
-# The default value is: NO.
-
-STRICT_PROTO_MATCHING = NO
-
-# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
-# list. This list is created by putting \todo commands in the documentation.
-# The default value is: YES.
-
-GENERATE_TODOLIST = NO
-
-# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
-# list. This list is created by putting \test commands in the documentation.
-# The default value is: YES.
-
-GENERATE_TESTLIST = NO
-
-# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
-# list. This list is created by putting \bug commands in the documentation.
-# The default value is: YES.
-
-GENERATE_BUGLIST = NO
-
-# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
-# the deprecated list. This list is created by putting \deprecated commands in
-# the documentation.
-# The default value is: YES.
-
-GENERATE_DEPRECATEDLIST= YES
-
-# The ENABLED_SECTIONS tag can be used to enable conditional documentation
-# sections, marked by \if <section_label> ... \endif and \cond <section_label>
-# ... \endcond blocks.
-
-ENABLED_SECTIONS = doxygen
-
-# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
-# initial value of a variable or macro / define can have for it to appear in the
-# documentation. If the initializer consists of more lines than specified here
-# it will be hidden. Use a value of 0 to hide initializers completely. The
-# appearance of the value of individual variables and macros / defines can be
-# controlled using \showinitializer or \hideinitializer command in the
-# documentation regardless of this setting.
-# Minimum value: 0, maximum value: 10000, default value: 30.
-
-MAX_INITIALIZER_LINES = 30
-
-# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
-# the bottom of the documentation of classes and structs. If set to YES, the
-# list will mention the files that were used to generate the documentation.
-# The default value is: YES.
-
-SHOW_USED_FILES = YES
-
-# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
-# will remove the Files entry from the Quick Index and from the Folder Tree View
-# (if specified).
-# The default value is: YES.
-
-SHOW_FILES = YES
-
-# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
-# page. This will remove the Namespaces entry from the Quick Index and from the
-# Folder Tree View (if specified).
-# The default value is: YES.
-
-SHOW_NAMESPACES = YES
-
-# The FILE_VERSION_FILTER tag can be used to specify a program or script that
-# doxygen should invoke to get the current version for each file (typically from
-# the version control system). Doxygen will invoke the program by executing (via
-# popen()) the command command input-file, where command is the value of the
-# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
-# by doxygen. Whatever the program writes to standard output is used as the file
-# version. For an example see the documentation.
-
-FILE_VERSION_FILTER =
-
-# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
-# by doxygen. The layout file controls the global structure of the generated
-# output files in an output format independent way. To create the layout file
-# that represents doxygen's defaults, run doxygen with the -l option. You can
-# optionally specify a file name after the option, if omitted DoxygenLayout.xml
-# will be used as the name of the layout file.
-#
-# Note that if you run doxygen from a directory containing a file called
-# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
-# tag is left empty.
-
-LAYOUT_FILE =
-
-# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
-# the reference definitions. This must be a list of .bib files. The .bib
-# extension is automatically appended if omitted. This requires the bibtex tool
-# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
-# For LaTeX the style of the bibliography can be controlled using
-# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
-# search path. See also \cite for info how to create references.
-
-CITE_BIB_FILES =
-
-#---------------------------------------------------------------------------
-# Configuration options related to warning and progress messages
-#---------------------------------------------------------------------------
-
-# The QUIET tag can be used to turn on/off the messages that are generated to
-# standard output by doxygen. If QUIET is set to YES this implies that the
-# messages are off.
-# The default value is: NO.
-
-QUIET = NO
-
-# The WARNINGS tag can be used to turn on/off the warning messages that are
-# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
-# this implies that the warnings are on.
-#
-# Tip: Turn warnings on while writing the documentation.
-# The default value is: YES.
-
-WARNINGS = YES
-
-# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
-# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
-# will automatically be disabled.
-# The default value is: YES.
-
-WARN_IF_UNDOCUMENTED = YES
-
-# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
-# potential errors in the documentation, such as not documenting some parameters
-# in a documented function, or documenting parameters that don't exist or using
-# markup commands wrongly.
-# The default value is: YES.
-
-WARN_IF_DOC_ERROR = YES
-
-# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
-# are documented, but have no documentation for their parameters or return
-# value. If set to NO, doxygen will only warn about wrong or incomplete
-# parameter documentation, but not about the absence of documentation. If
-# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
-# The default value is: NO.
-
-WARN_NO_PARAMDOC = NO
-
-# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
-# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS
-# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but
-# at the end of the doxygen process doxygen will return with a non-zero status.
-# Possible values are: NO, YES and FAIL_ON_WARNINGS.
-# The default value is: NO.
-
-WARN_AS_ERROR = NO
-
-# The WARN_FORMAT tag determines the format of the warning messages that doxygen
-# can produce. The string should contain the $file, $line, and $text tags, which
-# will be replaced by the file and line number from which the warning originated
-# and the warning text. Optionally the format may contain $version, which will
-# be replaced by the version of the file (if it could be obtained via
-# FILE_VERSION_FILTER)
-# The default value is: $file:$line: $text.
-
-WARN_FORMAT = "$file:$line: $text"
-
-# The WARN_LOGFILE tag can be used to specify a file to which warning and error
-# messages should be written. If left blank the output is written to standard
-# error (stderr).
-
-WARN_LOGFILE =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the input files
-#---------------------------------------------------------------------------
-
-# The INPUT tag is used to specify the files and/or directories that contain
-# documented source files. You may enter file names like myfile.cpp or
-# directories like /usr/src/myproject. Separate the files or directories with
-# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
-# Note: If this tag is empty the current directory is searched.
-
-INPUT = overall.md \
- intro.md \
- usage.md \
- mdbx.h \
- mdbx.h++ \
- options.h \
- ChangeLog.md
-
-# This tag can be used to specify the character encoding of the source files
-# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
-# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see:
-# https://www.gnu.org/software/libiconv/) for the list of possible encodings.
-# The default value is: UTF-8.
-
-INPUT_ENCODING = UTF-8
-
-# If the value of the INPUT tag contains directories, you can use the
-# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
-# *.h) to filter out the source-files in the directories.
-#
-# Note that for custom extensions or not directly supported extensions you also
-# need to set EXTENSION_MAPPING for the extension otherwise the files are not
-# read by doxygen.
-#
-# Note the list of default checked file patterns might differ from the list of
-# default file extension mappings.
-#
-# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
-# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
-# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
-# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
-# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl,
-# *.ucf, *.qsf and *.ice.
-
-FILE_PATTERNS = *.h
-
-# The RECURSIVE tag can be used to specify whether or not subdirectories should
-# be searched for input files as well.
-# The default value is: NO.
-
-RECURSIVE = NO
-
-# The EXCLUDE tag can be used to specify files and/or directories that should be
-# excluded from the INPUT source files. This way you can easily exclude a
-# subdirectory from a directory tree whose root is specified with the INPUT tag.
-#
-# Note that relative paths are relative to the directory from which doxygen is
-# run.
-
-EXCLUDE =
-
-# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
-# directories that are symbolic links (a Unix file system feature) are excluded
-# from the input.
-# The default value is: NO.
-
-EXCLUDE_SYMLINKS = NO
-
-# If the value of the INPUT tag contains directories, you can use the
-# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
-# certain files from those directories.
-#
-# Note that the wildcards are matched against the file with absolute path, so to
-# exclude all test directories for example use the pattern */test/*
-
-EXCLUDE_PATTERNS =
-
-# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
-# (namespaces, classes, functions, etc.) that should be excluded from the
-# output. The symbol name can be a fully qualified name, a word, or if the
-# wildcard * is used, a substring. Examples: ANamespace, AClass,
-# AClass::ANamespace, ANamespace::*Test
-#
-# Note that the wildcards are matched against the file with absolute path, so to
-# exclude all test directories use the pattern */test/*
-
-EXCLUDE_SYMBOLS = NOMINMAX __ORDER_BIG_ENDIAN__ __ORDER_LITTLE_ENDIAN__ \
- __has_include __has_attribute __has_builtin __has_cpp_attribute __has_extension __has_feature \
- HAVE_STRUCT_IOVEC MDBX_STRINGIFY_HELPER MDBX_STRINGIFY \
- MDBX_NOSANITIZE_ENUM MDBX_PRINTF_ARGS \
- MDBX_HAVE_CXX20_CONCEPTS \
- CONSTEXPR_ENUM_FLAGS_OPERATIONS DEFINE_ENUM_FLAG_OPERATORS \
- bool false true __dll_export __dll_import \
- MDBX_64BIT_ATOMIC_CONFIG MDBX_64BIT_CAS_CONFIG MDBX_ENV_CHECKPID_CONFIG MDBX_LOCKING_CONFIG \
- MDBX_TRUST_RTC_CONFIG MDBX_TXN_CHECKOWNER_CONFIG MDBX_USE_OFDLOCKS_CONFIG
-
-# The EXAMPLE_PATH tag can be used to specify one or more files or directories
-# that contain example code fragments that are included (see the \include
-# command).
-
-EXAMPLE_PATH = ../
-
-# If the value of the EXAMPLE_PATH tag contains directories, you can use the
-# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
-# *.h) to filter out the source-files in the directories. If left blank all
-# files are included.
-
-EXAMPLE_PATTERNS = *.c
-
-# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
-# searched for input files to be used with the \include or \dontinclude commands
-# irrespective of the value of the RECURSIVE tag.
-# The default value is: NO.
-
-EXAMPLE_RECURSIVE = NO
-
-# The IMAGE_PATH tag can be used to specify one or more files or directories
-# that contain images that are to be included in the documentation (see the
-# \image command).
-
-IMAGE_PATH =
-
-# The INPUT_FILTER tag can be used to specify a program that doxygen should
-# invoke to filter for each input file. Doxygen will invoke the filter program
-# by executing (via popen()) the command:
-#
-# <filter> <input-file>
-#
-# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
-# name of an input file. Doxygen will then use the output that the filter
-# program writes to standard output. If FILTER_PATTERNS is specified, this tag
-# will be ignored.
-#
-# Note that the filter must not add or remove lines; it is applied before the
-# code is scanned, but not when the output code is generated. If lines are added
-# or removed, the anchors will not be placed correctly.
-#
-# Note that for custom extensions or not directly supported extensions you also
-# need to set EXTENSION_MAPPING for the extension otherwise the files are not
-# properly processed by doxygen.
-
-INPUT_FILTER =
-
-# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
-# basis. Doxygen will compare the file name with each pattern and apply the
-# filter if there is a match. The filters are a list of the form: pattern=filter
-# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
-# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
-# patterns match the file name, INPUT_FILTER is applied.
-#
-# Note that for custom extensions or not directly supported extensions you also
-# need to set EXTENSION_MAPPING for the extension otherwise the files are not
-# properly processed by doxygen.
-
-FILTER_PATTERNS =
-
-# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
-# INPUT_FILTER) will also be used to filter the input files that are used for
-# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
-# The default value is: NO.
-
-FILTER_SOURCE_FILES = NO
-
-# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
-# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
-# it is also possible to disable source filtering for a specific pattern using
-# *.ext= (so without naming a filter).
-# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
-
-FILTER_SOURCE_PATTERNS =
-
-# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
-# is part of the input, its contents will be placed on the main page
-# (index.html). This can be useful if you have a project on for instance GitHub
-# and want to reuse the introduction page also for the doxygen output.
-
-USE_MDFILE_AS_MAINPAGE =
-
-#---------------------------------------------------------------------------
-# Configuration options related to source browsing
-#---------------------------------------------------------------------------
-
-# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
-# generated. Documented entities will be cross-referenced with these sources.
-#
-# Note: To get rid of all source code in the generated output, make sure that
-# also VERBATIM_HEADERS is set to NO.
-# The default value is: NO.
-
-SOURCE_BROWSER = NO
-
-# Setting the INLINE_SOURCES tag to YES will include the body of functions,
-# classes and enums directly into the documentation.
-# The default value is: NO.
-
-INLINE_SOURCES = NO
-
-# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
-# special comment blocks from generated source code fragments. Normal C, C++ and
-# Fortran comments will always remain visible.
-# The default value is: YES.
-
-STRIP_CODE_COMMENTS = YES
-
-# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
-# entity all documented functions referencing it will be listed.
-# The default value is: NO.
-
-REFERENCED_BY_RELATION = NO
-
-# If the REFERENCES_RELATION tag is set to YES then for each documented function
-# all documented entities called/used by that function will be listed.
-# The default value is: NO.
-
-REFERENCES_RELATION = NO
-
-# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
-# to YES then the hyperlinks from functions in REFERENCES_RELATION and
-# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
-# link to the documentation.
-# The default value is: YES.
-
-REFERENCES_LINK_SOURCE = YES
-
-# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
-# source code will show a tooltip with additional information such as prototype,
-# brief description and links to the definition and documentation. Since this
-# will make the HTML file larger and loading of large files a bit slower, you
-# can opt to disable this feature.
-# The default value is: YES.
-# This tag requires that the tag SOURCE_BROWSER is set to YES.
-
-SOURCE_TOOLTIPS = YES
-
-# If the USE_HTAGS tag is set to YES then the references to source code will
-# point to the HTML generated by the htags(1) tool instead of doxygen built-in
-# source browser. The htags tool is part of GNU's global source tagging system
-# (see https://www.gnu.org/software/global/global.html). You will need version
-# 4.8.6 or higher.
-#
-# To use it do the following:
-# - Install the latest version of global
-# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
-# - Make sure the INPUT points to the root of the source tree
-# - Run doxygen as normal
-#
-# Doxygen will invoke htags (and that will in turn invoke gtags), so these
-# tools must be available from the command line (i.e. in the search path).
-#
-# The result: instead of the source browser generated by doxygen, the links to
-# source code will now point to the output of htags.
-# The default value is: NO.
-# This tag requires that the tag SOURCE_BROWSER is set to YES.
-
-USE_HTAGS = NO
-
-# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
-# verbatim copy of the header file for each class for which an include is
-# specified. Set to NO to disable this.
-# See also: Section \class.
-# The default value is: YES.
-
-VERBATIM_HEADERS = YES
-
-# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
-# clang parser (see:
-# http://clang.llvm.org/) for more accurate parsing at the cost of reduced
-# performance. This can be particularly helpful with template rich C++ code for
-# which doxygen's built-in parser lacks the necessary type information.
-# Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse_libclang=ON option for CMake.
-# The default value is: NO.
-
-CLANG_ASSISTED_PARSING = NO
-
-# If clang assisted parsing is enabled and the CLANG_ADD_INC_PATHS tag is set to
-# YES then doxygen will add the directory of each input to the include path.
-# The default value is: YES.
-
-CLANG_ADD_INC_PATHS = YES
-
-# If clang assisted parsing is enabled you can provide the compiler with command
-# line options that you would normally use when invoking the compiler. Note that
-# the include paths will already be set by doxygen for the files and directories
-# specified with INPUT and INCLUDE_PATH.
-# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
-
-CLANG_OPTIONS =
-
-# If clang assisted parsing is enabled you can provide the clang parser with the
-# path to the directory containing a file called compile_commands.json. This
-# file is the compilation database (see:
-# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) containing the
-# options used when the source files were built. This is equivalent to
-# specifying the -p option to a clang tool, such as clang-check. These options
-# will then be passed to the parser. Any options specified with CLANG_OPTIONS
-# will be added as well.
-# Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse_libclang=ON option for CMake.
-
-CLANG_DATABASE_PATH =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the alphabetical class index
-#---------------------------------------------------------------------------
-
-# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
-# compounds will be generated. Enable this if the project contains a lot of
-# classes, structs, unions or interfaces.
-# The default value is: YES.
-
-ALPHABETICAL_INDEX = YES
-
-# In case all classes in a project start with a common prefix, all classes will
-# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
-# can be used to specify a prefix (or a list of prefixes) that should be ignored
-# while generating the index headers.
-# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
-
-IGNORE_PREFIX =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the HTML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
-# The default value is: YES.
-
-GENERATE_HTML = YES
-
-# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: html.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_OUTPUT = html
-
-# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
-# generated HTML page (for example: .htm, .php, .asp).
-# The default value is: .html.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_FILE_EXTENSION = .html
-
-# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
-# each generated HTML page. If the tag is left blank doxygen will generate a
-# standard header.
-#
-# To get valid HTML the header file that includes any scripts and style sheets
-# that doxygen needs, which is dependent on the configuration options used (e.g.
-# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
-# default header using
-# doxygen -w html new_header.html new_footer.html new_stylesheet.css
-# YourConfigFile
-# and then modify the file new_header.html. See also section "Doxygen usage"
-# for information on how to generate the default header that doxygen normally
-# uses.
-# Note: The header is subject to change so you typically have to regenerate the
-# default header when upgrading to a newer version of doxygen. For a description
-# of the possible markers and block names see the documentation.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_HEADER =
-
-# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
-# generated HTML page. If the tag is left blank doxygen will generate a standard
-# footer. See HTML_HEADER for more information on how to generate a default
-# footer and what special commands can be used inside the footer. See also
-# section "Doxygen usage" for information on how to generate the default footer
-# that doxygen normally uses.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_FOOTER =
-
-# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
-# sheet that is used by each HTML page. It can be used to fine-tune the look of
-# the HTML output. If left blank doxygen will generate a default style sheet.
-# See also section "Doxygen usage" for information on how to generate the style
-# sheet that doxygen normally uses.
-# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
-# it is more robust and this tag (HTML_STYLESHEET) will in the future become
-# obsolete.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_STYLESHEET =
-
-# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
-# cascading style sheets that are included after the standard style sheets
-# created by doxygen. Using this option one can overrule certain style aspects.
-# This is preferred over using HTML_STYLESHEET since it does not replace the
-# standard style sheet and is therefore more robust against future updates.
-# Doxygen will copy the style sheet files to the output directory.
-# Note: The order of the extra style sheet files is of importance (e.g. the last
-# style sheet in the list overrules the setting of the previous ones in the
-# list). For an example see the documentation.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_EXTRA_STYLESHEET =
-
-# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
-# other source files which should be copied to the HTML output directory. Note
-# that these files will be copied to the base HTML output directory. Use the
-# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
-# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
-# files will be copied as-is; there are no commands or markers available.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_EXTRA_FILES =
-
-# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
-# will adjust the colors in the style sheet and background images according to
-# this color. Hue is specified as an angle on a colorwheel, see
-# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
-# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
-# purple, and 360 is red again.
-# Minimum value: 0, maximum value: 359, default value: 220.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_COLORSTYLE_HUE = 220
-
-# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
-# in the HTML output. For a value of 0 the output will use grayscales only. A
-# value of 255 will produce the most vivid colors.
-# Minimum value: 0, maximum value: 255, default value: 100.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_COLORSTYLE_SAT = 100
-
-# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
-# luminance component of the colors in the HTML output. Values below 100
-# gradually make the output lighter, whereas values above 100 make the output
-# darker. The value divided by 100 is the actual gamma applied, so 80 represents
-# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
-# change the gamma.
-# Minimum value: 40, maximum value: 240, default value: 80.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_COLORSTYLE_GAMMA = 80
-
-# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
-# page will contain the date and time when the page was generated. Setting this
-# to YES can help to show when doxygen was last run and thus if the
-# documentation is up to date.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_TIMESTAMP = NO
-
-# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
-# documentation will contain a main index with vertical navigation menus that
-# are dynamically created via JavaScript. If disabled, the navigation index will
-# consists of multiple levels of tabs that are statically embedded in every HTML
-# page. Disable this option to support browsers that do not have JavaScript,
-# like the Qt help browser.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_DYNAMIC_MENUS = YES
-
-# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
-# documentation will contain sections that can be hidden and shown after the
-# page has loaded.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_DYNAMIC_SECTIONS = NO
-
-# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
-# shown in the various tree structured indices initially; the user can expand
-# and collapse entries dynamically later on. Doxygen will expand the tree to
-# such a level that at most the specified number of entries are visible (unless
-# a fully collapsed tree already exceeds this amount). So setting the number of
-# entries 1 will produce a full collapsed tree by default. 0 is a special value
-# representing an infinite number of entries and will result in a full expanded
-# tree by default.
-# Minimum value: 0, maximum value: 9999, default value: 100.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_INDEX_NUM_ENTRIES = 100
-
-# If the GENERATE_DOCSET tag is set to YES, additional index files will be
-# generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see:
-# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To
-# create a documentation set, doxygen will generate a Makefile in the HTML
-# output directory. Running make will produce the docset in that directory and
-# running make install will install the docset in
-# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
-# genXcode/_index.html for more information.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-GENERATE_DOCSET = NO
-
-# This tag determines the name of the docset feed. A documentation feed provides
-# an umbrella under which multiple documentation sets from a single provider
-# (such as a company or product suite) can be grouped.
-# The default value is: Doxygen generated docs.
-# This tag requires that the tag GENERATE_DOCSET is set to YES.
-
-DOCSET_FEEDNAME = "Doxygen generated docs"
-
-# This tag specifies a string that should uniquely identify the documentation
-# set bundle. This should be a reverse domain-name style string, e.g.
-# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
-# The default value is: org.doxygen.Project.
-# This tag requires that the tag GENERATE_DOCSET is set to YES.
-
-DOCSET_BUNDLE_ID = org.doxygen.Project
-
-# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
-# the documentation publisher. This should be a reverse domain-name style
-# string, e.g. com.mycompany.MyDocSet.documentation.
-# The default value is: org.doxygen.Publisher.
-# This tag requires that the tag GENERATE_DOCSET is set to YES.
-
-DOCSET_PUBLISHER_ID = org.doxygen.Publisher
-
-# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
-# The default value is: Publisher.
-# This tag requires that the tag GENERATE_DOCSET is set to YES.
-
-DOCSET_PUBLISHER_NAME = Publisher
-
-# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
-# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
-# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
-# (see:
-# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows.
-#
-# The HTML Help Workshop contains a compiler that can convert all HTML output
-# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
-# files are now used as the Windows 98 help format, and will replace the old
-# Windows help format (.hlp) on all Windows platforms in the future. Compressed
-# HTML files also contain an index, a table of contents, and you can search for
-# words in the documentation. The HTML workshop also contains a viewer for
-# compressed HTML files.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-GENERATE_HTMLHELP = NO
-
-# The CHM_FILE tag can be used to specify the file name of the resulting .chm
-# file. You can add a path in front of the file if the result should not be
-# written to the html output directory.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-CHM_FILE =
-
-# The HHC_LOCATION tag can be used to specify the location (absolute path
-# including file name) of the HTML help compiler (hhc.exe). If non-empty,
-# doxygen will try to run the HTML help compiler on the generated index.hhp.
-# The file has to be specified with full path.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-HHC_LOCATION =
-
-# The GENERATE_CHI flag controls if a separate .chi index file is generated
-# (YES) or that it should be included in the main .chm file (NO).
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-GENERATE_CHI = NO
-
-# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
-# and project file content.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-CHM_INDEX_ENCODING =
-
-# The BINARY_TOC flag controls whether a binary table of contents is generated
-# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
-# enables the Previous and Next buttons.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-BINARY_TOC = NO
-
-# The TOC_EXPAND flag can be set to YES to add extra items for group members to
-# the table of contents of the HTML help documentation and to the tree view.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
-
-TOC_EXPAND = NO
-
-# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
-# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
-# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
-# (.qch) of the generated HTML documentation.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-GENERATE_QHP = NO
-
-# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
-# the file name of the resulting .qch file. The path specified is relative to
-# the HTML output folder.
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QCH_FILE =
-
-# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
-# Project output. For more information please see Qt Help Project / Namespace
-# (see:
-# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
-# The default value is: org.doxygen.Project.
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHP_NAMESPACE = org.doxygen.Project
-
-# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
-# Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see:
-# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders).
-# The default value is: doc.
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHP_VIRTUAL_FOLDER = doc
-
-# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
-# filter to add. For more information please see Qt Help Project / Custom
-# Filters (see:
-# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters).
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHP_CUST_FILTER_NAME =
-
-# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
-# custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see:
-# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters).
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHP_CUST_FILTER_ATTRS =
-
-# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
-# project's filter section matches. Qt Help Project / Filter Attributes (see:
-# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHP_SECT_FILTER_ATTRS =
-
-# The QHG_LOCATION tag can be used to specify the location (absolute path
-# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to
-# run qhelpgenerator on the generated .qhp file.
-# This tag requires that the tag GENERATE_QHP is set to YES.
-
-QHG_LOCATION =
-
-# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
-# generated, together with the HTML files, they form an Eclipse help plugin. To
-# install this plugin and make it available under the help contents menu in
-# Eclipse, the contents of the directory containing the HTML and XML files needs
-# to be copied into the plugins directory of eclipse. The name of the directory
-# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
-# After copying Eclipse needs to be restarted before the help appears.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-GENERATE_ECLIPSEHELP = NO
-
-# A unique identifier for the Eclipse help plugin. When installing the plugin
-# the directory name containing the HTML and XML files should also have this
-# name. Each documentation set should have its own identifier.
-# The default value is: org.doxygen.Project.
-# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
-
-ECLIPSE_DOC_ID = org.doxygen.Project
-
-# If you want full control over the layout of the generated HTML pages it might
-# be necessary to disable the index and replace it with your own. The
-# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
-# of each HTML page. A value of NO enables the index and the value YES disables
-# it. Since the tabs in the index contain the same information as the navigation
-# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-DISABLE_INDEX = YES
-
-# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
-# structure should be generated to display hierarchical information. If the tag
-# value is set to YES, a side panel will be generated containing a tree-like
-# index structure (just like the one that is generated for HTML Help). For this
-# to work a browser that supports JavaScript, DHTML, CSS and frames is required
-# (i.e. any modern browser). Windows users are probably better off using the
-# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
-# further fine-tune the look of the index. As an example, the default style
-# sheet generated by doxygen has an example that shows how to put an image at
-# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
-# the same information as the tab index, you could consider setting
-# DISABLE_INDEX to YES when enabling this option.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-GENERATE_TREEVIEW = YES
-
-# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
-# doxygen will group on one line in the generated HTML documentation.
-#
-# Note that a value of 0 will completely suppress the enum values from appearing
-# in the overview section.
-# Minimum value: 0, maximum value: 20, default value: 4.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-ENUM_VALUES_PER_LINE = 4
-
-# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
-# to set the initial width (in pixels) of the frame in which the tree is shown.
-# Minimum value: 0, maximum value: 1500, default value: 250.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-TREEVIEW_WIDTH = 250
-
-# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
-# external symbols imported via tag files in a separate window.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-EXT_LINKS_IN_WINDOW = NO
-
-# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg
-# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see
-# https://inkscape.org) to generate formulas as SVG images instead of PNGs for
-# the HTML output. These images will generally look nicer at scaled resolutions.
-# Possible values are: png (the default) and svg (looks nicer but requires the
-# pdf2svg or inkscape tool).
-# The default value is: png.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_FORMULA_FORMAT = png
-
-# Use this tag to change the font size of LaTeX formulas included as images in
-# the HTML documentation. When you change the font size after a successful
-# doxygen run you need to manually remove any form_*.png images from the HTML
-# output directory to force them to be regenerated.
-# Minimum value: 8, maximum value: 50, default value: 10.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-FORMULA_FONTSIZE = 10
-
-# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
-# generated for formulas are transparent PNGs. Transparent PNGs are not
-# supported properly for IE 6.0, but are supported on all modern browsers.
-#
-# Note that when changing this option you need to delete any form_*.png files in
-# the HTML output directory before the changes have effect.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-FORMULA_TRANSPARENT = YES
-
-# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
-# to create new LaTeX commands to be used in formulas as building blocks. See
-# the section "Including formulas" for details.
-
-FORMULA_MACROFILE =
-
-# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# https://www.mathjax.org) which uses client side JavaScript for the rendering
-# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
-# installed or if you want to formulas look prettier in the HTML output. When
-# enabled you may also need to install MathJax separately and configure the path
-# to it using the MATHJAX_RELPATH option.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-USE_MATHJAX = YES
-
-# When MathJax is enabled you can set the default output format to be used for
-# the MathJax output. See the MathJax site (see:
-# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details.
-# Possible values are: HTML-CSS (which is slower, but has the best
-# compatibility), NativeMML (i.e. MathML) and SVG.
-# The default value is: HTML-CSS.
-# This tag requires that the tag USE_MATHJAX is set to YES.
-
-MATHJAX_FORMAT = HTML-CSS
-
-# When MathJax is enabled you need to specify the location relative to the HTML
-# output directory using the MATHJAX_RELPATH option. The destination directory
-# should contain the MathJax.js script. For instance, if the mathjax directory
-# is located at the same level as the HTML output directory, then
-# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
-# Content Delivery Network so you can quickly see the result without installing
-# MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from https://www.mathjax.org before deployment.
-# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2.
-# This tag requires that the tag USE_MATHJAX is set to YES.
-
-MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/
-
-# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
-# extension names that should be enabled during MathJax rendering. For example
-# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
-# This tag requires that the tag USE_MATHJAX is set to YES.
-
-MATHJAX_EXTENSIONS =
-
-# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
-# of code that will be used on startup of the MathJax code. See the MathJax site
-# (see:
-# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an
-# example see the documentation.
-# This tag requires that the tag USE_MATHJAX is set to YES.
-
-MATHJAX_CODEFILE =
-
-# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
-# the HTML output. The underlying search engine uses javascript and DHTML and
-# should work on any modern browser. Note that when using HTML help
-# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
-# there is already a search function so this one should typically be disabled.
-# For large projects the javascript based search engine can be slow, then
-# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
-# search using the keyboard; to jump to the search box use <access key> + S
-# (what the <access key> is depends on the OS and browser, but it is typically
-# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
-# key> to jump into the search results window, the results can be navigated
-# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
-# the search. The filter options can be selected when the cursor is inside the
-# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
-# to select a filter and <Enter> or <escape> to activate or cancel the filter
-# option.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-SEARCHENGINE = YES
-
-# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a web server instead of a web client using JavaScript. There
-# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
-# setting. When disabled, doxygen will generate a PHP script for searching and
-# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
-# and searching needs to be provided by external tools. See the section
-# "External Indexing and Searching" for details.
-# The default value is: NO.
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-SERVER_BASED_SEARCH = NO
-
-# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
-# script for searching. Instead the search results are written to an XML file
-# which needs to be processed by an external indexer. Doxygen will invoke an
-# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
-# search results.
-#
-# Doxygen ships with an example indexer (doxyindexer) and search engine
-# (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see:
-# https://xapian.org/).
-#
-# See the section "External Indexing and Searching" for details.
-# The default value is: NO.
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-EXTERNAL_SEARCH = NO
-
-# The SEARCHENGINE_URL should point to a search engine hosted by a web server
-# which will return the search results when EXTERNAL_SEARCH is enabled.
-#
-# Doxygen ships with an example indexer (doxyindexer) and search engine
-# (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see:
-# https://xapian.org/). See the section "External Indexing and Searching" for
-# details.
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-SEARCHENGINE_URL =
-
-# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
-# search data is written to a file for indexing by an external tool. With the
-# SEARCHDATA_FILE tag the name of this file can be specified.
-# The default file is: searchdata.xml.
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-SEARCHDATA_FILE = searchdata.xml
-
-# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
-# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
-# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
-# projects and redirect the results back to the right project.
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-EXTERNAL_SEARCH_ID =
-
-# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
-# projects other than the one defined by this configuration file, but that are
-# all added to the same external search index. Each project needs to have a
-# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
-# to a relative location where the documentation can be found. The format is:
-# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
-# This tag requires that the tag SEARCHENGINE is set to YES.
-
-EXTRA_SEARCH_MAPPINGS =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the LaTeX output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
-# The default value is: YES.
-
-GENERATE_LATEX = NO
-
-# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: latex.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_OUTPUT = latex
-
-# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
-# invoked.
-#
-# Note that when not enabling USE_PDFLATEX the default is latex when enabling
-# USE_PDFLATEX the default is pdflatex and when in the later case latex is
-# chosen this is overwritten by pdflatex. For specific output languages the
-# default can have been set differently, this depends on the implementation of
-# the output language.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_CMD_NAME =
-
-# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
-# index for LaTeX.
-# Note: This tag is used in the Makefile / make.bat.
-# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
-# (.tex).
-# The default file is: makeindex.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-MAKEINDEX_CMD_NAME = makeindex
-
-# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
-# generate index for LaTeX. In case there is no backslash (\) as first character
-# it will be automatically added in the LaTeX code.
-# Note: This tag is used in the generated output file (.tex).
-# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
-# The default value is: makeindex.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_MAKEINDEX_CMD = makeindex
-
-# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
-# documents. This may be useful for small projects and may help to save some
-# trees in general.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-COMPACT_LATEX = NO
-
-# The PAPER_TYPE tag can be used to set the paper type that is used by the
-# printer.
-# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
-# 14 inches) and executive (7.25 x 10.5 inches).
-# The default value is: a4.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-PAPER_TYPE = a4
-
-# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
-# that should be included in the LaTeX output. The package can be specified just
-# by its name or with the correct syntax as to be used with the LaTeX
-# \usepackage command. To get the times font for instance you can specify :
-# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
-# To use the option intlimits with the amsmath package you can specify:
-# EXTRA_PACKAGES=[intlimits]{amsmath}
-# If left blank no extra packages will be included.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-EXTRA_PACKAGES =
-
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
-# generated LaTeX document. The header should contain everything until the first
-# chapter. If it is left blank doxygen will generate a standard header. See
-# section "Doxygen usage" for information on how to let doxygen write the
-# default header to a separate file.
-#
-# Note: Only use a user-defined header if you know what you are doing! The
-# following commands have a special meaning inside the header: $title,
-# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
-# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
-# string, for the replacement values of the other commands the user is referred
-# to HTML_HEADER.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_HEADER =
-
-# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
-# generated LaTeX document. The footer should contain everything after the last
-# chapter. If it is left blank doxygen will generate a standard footer. See
-# LATEX_HEADER for more information on how to generate a default footer and what
-# special commands can be used inside the footer.
-#
-# Note: Only use a user-defined footer if you know what you are doing!
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_FOOTER =
-
-# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
-# LaTeX style sheets that are included after the standard style sheets created
-# by doxygen. Using this option one can overrule certain style aspects. Doxygen
-# will copy the style sheet files to the output directory.
-# Note: The order of the extra style sheet files is of importance (e.g. the last
-# style sheet in the list overrules the setting of the previous ones in the
-# list).
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_EXTRA_STYLESHEET =
-
-# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
-# other source files which should be copied to the LATEX_OUTPUT output
-# directory. Note that the files will be copied as-is; there are no commands or
-# markers available.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_EXTRA_FILES =
-
-# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
-# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
-# contain links (just like the HTML output) instead of page references. This
-# makes the output suitable for online browsing using a PDF viewer.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-PDF_HYPERLINKS = YES
-
-# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as
-# specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX
-# files. Set this option to YES, to get a higher quality PDF documentation.
-#
-# See also section LATEX_CMD_NAME for selecting the engine.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-USE_PDFLATEX = YES
-
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
-# command to the generated LaTeX files. This will instruct LaTeX to keep running
-# if errors occur, instead of asking the user for help. This option is also used
-# when generating formulas in HTML.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_BATCHMODE = NO
-
-# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
-# index chapters (such as File Index, Compound Index, etc.) in the output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_HIDE_INDICES = NO
-
-# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
-# code with syntax highlighting in the LaTeX output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_SOURCE_CODE = NO
-
-# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
-# bibliography, e.g. plainnat, or ieeetr. See
-# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
-# The default value is: plain.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_BIB_STYLE = plain
-
-# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
-# page will contain the date and time when the page was generated. Setting this
-# to NO can help when comparing the output of multiple runs.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_TIMESTAMP = NO
-
-# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
-# path from which the emoji images will be read. If a relative path is entered,
-# it will be relative to the LATEX_OUTPUT directory. If left blank the
-# LATEX_OUTPUT directory will be used.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_EMOJI_DIRECTORY =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the RTF output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
-# RTF output is optimized for Word 97 and may not look too pretty with other RTF
-# readers/editors.
-# The default value is: NO.
-
-GENERATE_RTF = NO
-
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: rtf.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_OUTPUT = rtf
-
-# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
-# documents. This may be useful for small projects and may help to save some
-# trees in general.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-COMPACT_RTF = NO
-
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
-# contain hyperlink fields. The RTF file will contain links (just like the HTML
-# output) instead of page references. This makes the output suitable for online
-# browsing using Word or some other Word compatible readers that support those
-# fields.
-#
-# Note: WordPad (write) and others do not support links.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_HYPERLINKS = NO
-
-# Load stylesheet definitions from file. Syntax is similar to doxygen's
-# configuration file, i.e. a series of assignments. You only have to provide
-# replacements, missing definitions are set to their default value.
-#
-# See also section "Doxygen usage" for information on how to generate the
-# default style sheet that doxygen normally uses.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_STYLESHEET_FILE =
-
-# Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's configuration file. A template extensions file can be
-# generated using doxygen -e rtf extensionFile.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_EXTENSIONS_FILE =
-
-# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
-# with syntax highlighting in the RTF output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_SOURCE_CODE = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the man page output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
-# classes and files.
-# The default value is: NO.
-
-GENERATE_MAN = NO
-
-# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it. A directory man3 will be created inside the directory specified by
-# MAN_OUTPUT.
-# The default directory is: man.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_OUTPUT = man
-
-# The MAN_EXTENSION tag determines the extension that is added to the generated
-# man pages. In case the manual section does not start with a number, the number
-# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
-# optional.
-# The default value is: .3.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_EXTENSION = .3
-
-# The MAN_SUBDIR tag determines the name of the directory created within
-# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
-# MAN_EXTENSION with the initial . removed.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_SUBDIR =
-
-# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
-# will generate one additional man file for each entity documented in the real
-# man page(s). These additional files only source the real man page, but without
-# them the man command would be unable to find the correct page.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_LINKS = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the XML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
-# captures the structure of the code including all documentation.
-# The default value is: NO.
-
-GENERATE_XML = NO
-
-# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: xml.
-# This tag requires that the tag GENERATE_XML is set to YES.
-
-XML_OUTPUT = xml
-
-# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
-# listings (including syntax highlighting and cross-referencing information) to
-# the XML output. Note that enabling this will significantly increase the size
-# of the XML output.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_XML is set to YES.
-
-XML_PROGRAMLISTING = YES
-
-# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
-# namespace members in file scope as well, matching the HTML output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_XML is set to YES.
-
-XML_NS_MEMB_FILE_SCOPE = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the DOCBOOK output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
-# that can be used to generate PDF.
-# The default value is: NO.
-
-GENERATE_DOCBOOK = NO
-
-# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
-# front of it.
-# The default directory is: docbook.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_OUTPUT = docbook
-
-# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
-# program listings (including syntax highlighting and cross-referencing
-# information) to the DOCBOOK output. Note that enabling this will significantly
-# increase the size of the DOCBOOK output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_PROGRAMLISTING = NO
-
-#---------------------------------------------------------------------------
-# Configuration options for the AutoGen Definitions output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
-# the structure of the code including all documentation. Note that this feature
-# is still experimental and incomplete at the moment.
-# The default value is: NO.
-
-GENERATE_AUTOGEN_DEF = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the Perl module output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
-# file that captures the structure of the code including all documentation.
-#
-# Note that this feature is still experimental and incomplete at the moment.
-# The default value is: NO.
-
-GENERATE_PERLMOD = NO
-
-# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
-# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
-# output from the Perl module output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_LATEX = NO
-
-# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
-# formatted so it can be parsed by a human reader. This is useful if you want to
-# understand what is going on. On the other hand, if this tag is set to NO, the
-# size of the Perl module output will be much smaller and Perl will parse it
-# just the same.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_PRETTY = YES
-
-# The names of the make variables in the generated doxyrules.make file are
-# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
-# so different doxyrules.make files included by the same Makefile don't
-# overwrite each other's variables.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_MAKEVAR_PREFIX =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the preprocessor
-#---------------------------------------------------------------------------
-
-# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
-# C-preprocessor directives found in the sources and include files.
-# The default value is: YES.
-
-ENABLE_PREPROCESSING = YES
-
-# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
-# in the source code. If set to NO, only conditional compilation will be
-# performed. Macro expansion can be done in a controlled way by setting
-# EXPAND_ONLY_PREDEF to YES.
-# The default value is: NO.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-MACRO_EXPANSION = YES
-
-# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
-# the macro expansion is limited to the macros specified with the PREDEFINED and
-# EXPAND_AS_DEFINED tags.
-# The default value is: NO.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-EXPAND_ONLY_PREDEF = YES
-
-# If the SEARCH_INCLUDES tag is set to YES, the include files in the
-# INCLUDE_PATH will be searched if a #include is found.
-# The default value is: YES.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-SEARCH_INCLUDES = YES
-
-# The INCLUDE_PATH tag can be used to specify one or more directories that
-# contain include files that are not input files but should be processed by the
-# preprocessor.
-# This tag requires that the tag SEARCH_INCLUDES is set to YES.
-
-INCLUDE_PATH =
-
-# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
-# patterns (like *.h and *.hpp) to filter out the header-files in the
-# directories. If left blank, the patterns specified with FILE_PATTERNS will be
-# used.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-INCLUDE_FILE_PATTERNS =
-
-# The PREDEFINED tag can be used to specify one or more macro names that are
-# defined before the preprocessor is started (similar to the -D option of e.g.
-# gcc). The argument of the tag is a list of macros of the form: name or
-# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
-# is assumed. To prevent a macro definition from being undefined via #undef or
-# recursively expanded use the := operator instead of the = operator.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-PREDEFINED = DOXYGEN MDBX_DECLARE_EXCEPTION \
- MDBX_NOTHROW_PURE_FUNCTION MDBX_PURE_FUNCTION \
- MDBX_NOTHROW_CONST_FUNCTION \
- MDBX_CXX01_CONSTEXPR MDBX_CXX01_CONSTEXPR_VAR \
- MDBX_CXX11_CONSTEXPR MDBX_CXX11_CONSTEXPR_VAR \
- MDBX_CXX14_CONSTEXPR MDBX_CXX14_CONSTEXPR_VAR \
- MDBX_CXX17_CONSTEXPR MDBX_CXX20_CONSTEXPR \
- MDBX_CXX17_NOEXCEPT
-
-# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
-# tag can be used to specify a list of macro names that should be expanded. The
-# macro definition that is found in the sources will be used. Use the PREDEFINED
-# tag if you want to use a different macro definition that overrules the
-# definition found in the source code.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-EXPAND_AS_DEFINED =
-
-# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
-# remove all references to function-like macros that are alone on a line, have
-# an all uppercase name, and do not end with a semicolon. Such function macros
-# are typically used for boiler-plate code, and will confuse the parser if not
-# removed.
-# The default value is: YES.
-# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
-
-SKIP_FUNCTION_MACROS = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to external references
-#---------------------------------------------------------------------------
-
-# The TAGFILES tag can be used to specify one or more tag files. For each tag
-# file the location of the external documentation should be added. The format of
-# a tag file without this location is as follows:
-# TAGFILES = file1 file2 ...
-# Adding location for the tag files is done as follows:
-# TAGFILES = file1=loc1 "file2 = loc2" ...
-# where loc1 and loc2 can be relative or absolute paths or URLs. See the
-# section "Linking to external documentation" for more information about the use
-# of tag files.
-# Note: Each tag file must have a unique name (where the name does NOT include
-# the path). If a tag file is not located in the directory in which doxygen is
-# run, you must also specify the path to the tagfile here.
-
-TAGFILES =
-
-# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
-# tag file that is based on the input files it reads. See section "Linking to
-# external documentation" for more information about the usage of tag files.
-
-GENERATE_TAGFILE =
-
-# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
-# the class index. If set to NO, only the inherited external classes will be
-# listed.
-# The default value is: NO.
-
-ALLEXTERNALS = NO
-
-# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
-# in the modules index. If set to NO, only the current project's groups will be
-# listed.
-# The default value is: YES.
-
-EXTERNAL_GROUPS = YES
-
-# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
-# the related pages index. If set to NO, only the current project's pages will
-# be listed.
-# The default value is: YES.
-
-EXTERNAL_PAGES = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the dot tool
-#---------------------------------------------------------------------------
-
-# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
-# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
-# NO turns the diagrams off. Note that this option also works with HAVE_DOT
-# disabled, but it is recommended to install and use dot, since it yields more
-# powerful graphs.
-# The default value is: YES.
-
-CLASS_DIAGRAMS = NO
-
-# You can include diagrams made with dia in doxygen documentation. Doxygen will
-# then run dia to produce the diagram and insert it in the documentation. The
-# DIA_PATH tag allows you to specify the directory where the dia binary resides.
-# If left empty dia is assumed to be found in the default search path.
-
-DIA_PATH =
-
-# If set to YES the inheritance and collaboration graphs will hide inheritance
-# and usage relations if the target is undocumented or is not a class.
-# The default value is: YES.
-
-HIDE_UNDOC_RELATIONS = YES
-
-# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
-# available from the path. This tool is part of Graphviz (see:
-# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
-# Bell Labs. The other options in this section have no effect if this option is
-# set to NO
-# The default value is: YES.
-
-HAVE_DOT = NO
-
-# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
-# to run in parallel. When set to 0 doxygen will base this on the number of
-# processors available in the system. You can set it explicitly to a value
-# larger than 0 to get control over the balance between CPU load and processing
-# speed.
-# Minimum value: 0, maximum value: 32, default value: 0.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_NUM_THREADS = 0
-
-# When you want a differently looking font in the dot files that doxygen
-# generates you can specify the font name using DOT_FONTNAME. You need to make
-# sure dot is able to find the font, which can be done by putting it in a
-# standard location or by setting the DOTFONTPATH environment variable or by
-# setting DOT_FONTPATH to the directory containing the font.
-# The default value is: Helvetica.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTNAME = Helvetica
-
-# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
-# dot graphs.
-# Minimum value: 4, maximum value: 24, default value: 10.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTSIZE = 10
-
-# By default doxygen will tell dot to use the default font as specified with
-# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
-# the path where dot can find it using this tag.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTPATH =
-
-# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
-# each documented class showing the direct and indirect inheritance relations.
-# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-CLASS_GRAPH = YES
-
-# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
-# graph for each documented class showing the direct and indirect implementation
-# dependencies (inheritance, containment, and class references variables) of the
-# class with other documented classes.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-COLLABORATION_GRAPH = YES
-
-# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
-# groups, showing the direct groups dependencies.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GROUP_GRAPHS = YES
-
-# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
-# collaboration diagrams in a style similar to the OMG's Unified Modeling
-# Language.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-UML_LOOK = NO
-
-# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
-# class node. If there are many fields or methods and many nodes the graph may
-# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
-# number of items for each type to make the size more manageable. Set this to 0
-# for no limit. Note that the threshold may be exceeded by 50% before the limit
-# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
-# but if the number exceeds 15, the total amount of fields shown is limited to
-# 10.
-# Minimum value: 0, maximum value: 100, default value: 10.
-# This tag requires that the tag UML_LOOK is set to YES.
-
-UML_LIMIT_NUM_FIELDS = 10
-
-# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and
-# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS
-# tag is set to YES, doxygen will add type and arguments for attributes and
-# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen
-# will not generate fields with class member information in the UML graphs. The
-# class diagrams will look similar to the default class diagrams but using UML
-# notation for the relationships.
-# Possible values are: NO, YES and NONE.
-# The default value is: NO.
-# This tag requires that the tag UML_LOOK is set to YES.
-
-DOT_UML_DETAILS = NO
-
-# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters
-# to display on a single line. If the actual line length exceeds this threshold
-# significantly it will wrapped across multiple lines. Some heuristics are apply
-# to avoid ugly line breaks.
-# Minimum value: 0, maximum value: 1000, default value: 17.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_WRAP_THRESHOLD = 17
-
-# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
-# collaboration graphs will show the relations between templates and their
-# instances.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-TEMPLATE_RELATIONS = NO
-
-# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
-# YES then doxygen will generate a graph for each documented file showing the
-# direct and indirect include dependencies of the file with other documented
-# files.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INCLUDE_GRAPH = YES
-
-# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
-# set to YES then doxygen will generate a graph for each documented file showing
-# the direct and indirect include dependencies of the file with other documented
-# files.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INCLUDED_BY_GRAPH = YES
-
-# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
-# dependency graph for every global function or class method.
-#
-# Note that enabling this option will significantly increase the time of a run.
-# So in most cases it will be better to enable call graphs for selected
-# functions only using the \callgraph command. Disabling a call graph can be
-# accomplished by means of the command \hidecallgraph.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-CALL_GRAPH = NO
-
-# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
-# dependency graph for every global function or class method.
-#
-# Note that enabling this option will significantly increase the time of a run.
-# So in most cases it will be better to enable caller graphs for selected
-# functions only using the \callergraph command. Disabling a caller graph can be
-# accomplished by means of the command \hidecallergraph.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-CALLER_GRAPH = NO
-
-# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
-# hierarchy of all classes instead of a textual one.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GRAPHICAL_HIERARCHY = YES
-
-# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
-# dependencies a directory has on other directories in a graphical way. The
-# dependency relations are determined by the #include relations between the
-# files in the directories.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DIRECTORY_GRAPH = YES
-
-# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
-# generated by dot. For an explanation of the image formats see the section
-# output formats in the documentation of the dot tool (Graphviz (see:
-# http://www.graphviz.org/)).
-# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
-# to make the SVG files visible in IE 9+ (other browsers do not have this
-# requirement).
-# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
-# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
-# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
-# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
-# png:gdiplus:gdiplus.
-# The default value is: png.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_IMAGE_FORMAT = png
-
-# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
-# enable generation of interactive SVG images that allow zooming and panning.
-#
-# Note that this requires a modern browser other than Internet Explorer. Tested
-# and working are Firefox, Chrome, Safari, and Opera.
-# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
-# the SVG files visible. Older versions of IE do not have SVG support.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INTERACTIVE_SVG = NO
-
-# The DOT_PATH tag can be used to specify the path where the dot tool can be
-# found. If left blank, it is assumed the dot tool can be found in the path.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_PATH =
-
-# The DOTFILE_DIRS tag can be used to specify one or more directories that
-# contain dot files that are included in the documentation (see the \dotfile
-# command).
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOTFILE_DIRS =
-
-# The MSCFILE_DIRS tag can be used to specify one or more directories that
-# contain msc files that are included in the documentation (see the \mscfile
-# command).
-
-MSCFILE_DIRS =
-
-# The DIAFILE_DIRS tag can be used to specify one or more directories that
-# contain dia files that are included in the documentation (see the \diafile
-# command).
-
-DIAFILE_DIRS =
-
-# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
-# path where java can find the plantuml.jar file. If left blank, it is assumed
-# PlantUML is not used or called during a preprocessing step. Doxygen will
-# generate a warning when it encounters a \startuml command in this case and
-# will not generate output for the diagram.
-
-PLANTUML_JAR_PATH =
-
-# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
-# configuration file for plantuml.
-
-PLANTUML_CFG_FILE =
-
-# When using plantuml, the specified paths are searched for files specified by
-# the !include statement in a plantuml block.
-
-PLANTUML_INCLUDE_PATH =
-
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
-# that will be shown in the graph. If the number of nodes in a graph becomes
-# larger than this value, doxygen will truncate the graph, which is visualized
-# by representing a node as a red box. Note that doxygen if the number of direct
-# children of the root node in a graph is already larger than
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
-# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
-# Minimum value: 0, maximum value: 10000, default value: 50.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_GRAPH_MAX_NODES = 50
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
-# generated by dot. A depth value of 3 means that only nodes reachable from the
-# root by following a path via at most 3 edges will be shown. Nodes that lay
-# further from the root node will be omitted. Note that setting this option to 1
-# or 2 may greatly reduce the computation time needed for large code bases. Also
-# note that the size of a graph can be further restricted by
-# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
-# Minimum value: 0, maximum value: 1000, default value: 0.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-MAX_DOT_GRAPH_DEPTH = 0
-
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not seem
-# to support this out of the box.
-#
-# Warning: Depending on the platform used, enabling this option may lead to
-# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
-# read).
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_TRANSPARENT = NO
-
-# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
-# files in one run (i.e. multiple -o and -T options on the command line). This
-# makes dot run faster, but since only newer versions of dot (>1.8.10) support
-# this, this feature is disabled by default.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_MULTI_TARGETS = NO
-
-# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
-# explaining the meaning of the various boxes and arrows in the dot generated
-# graphs.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GENERATE_LEGEND = YES
-
-# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate
-# files that are used to generate the various graphs.
-#
-# Note: This setting is not only used for dot files but also for msc and
-# plantuml temporary files.
-# The default value is: YES.
-
-DOT_CLEANUP = YES
diff --git a/3rdparty/libmdbx/docs/_preface.md b/3rdparty/libmdbx/docs/_preface.md
deleted file mode 100644
index b63615237548491e18c1a5d82d0ff9258b21f488..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/docs/_preface.md
+++ /dev/null
@@ -1,47 +0,0 @@
-\page intro Introduction
-\section characteristics Characteristics
-
-Preface {#preface}
-------------------
-
-> For the most part, this section is a copy of the corresponding text
-> from LMDB description, but with some edits reflecting the improvements
-> and enhancements were made in MDBX.
-
-MDBX is a Btree-based database management library modeled loosely on the
-BerkeleyDB API, but much simplified. The entire database (aka "environment")
-is exposed in a memory map, and all data fetches return data directly from
-the mapped memory, so no malloc's or memcpy's occur during data fetches.
-As such, the library is extremely simple because it requires no page caching
-layer of its own, and it is extremely high performance and memory-efficient.
-It is also fully transactional with full ACID semantics, and when the memory
-map is read-only, the database integrity cannot be corrupted by stray pointer
-writes from application code.
-
-The library is fully thread-aware and supports concurrent read/write access
-from multiple processes and threads. Data pages use a copy-on-write strategy
-so no active data pages are ever overwritten, which also provides resistance
-to corruption and eliminates the need of any special recovery procedures
-after a system crash. Writes are fully serialized; only one write transaction
-may be active at a time, which guarantees that writers can never deadlock.
-The database structure is multi-versioned so readers run with no locks;
-writers cannot block readers, and readers don't block writers.
-
-Unlike other well-known database mechanisms which use either write-ahead
-transaction logs or append-only data writes, MDBX requires no maintenance
-during operation. Both write-ahead loggers and append-only databases require
-periodic checkpointing and/or compaction of their log or database files
-otherwise they grow without bound. MDBX tracks retired/freed pages within the
-database and re-uses them for new write operations, so the database size does
-not grow without bound in normal use. It is worth noting that the "next"
-version libmdbx (\ref MithrilDB) will solve this problem.
-
-The memory map can be used as a read-only or read-write map. It is read-only
-by default as this provides total immunity to corruption. Using read-write
-mode offers much higher write performance, but adds the possibility for stray
-application writes thru pointers to silently corrupt the database.
-Of course if your application code is known to be bug-free (...) then this is
-not an issue.
-
-If this is your first time using a transactional embedded key-value store,
-you may find the \ref starting section below to be helpful.
diff --git a/3rdparty/libmdbx/docs/_restrictions.md b/3rdparty/libmdbx/docs/_restrictions.md
deleted file mode 100644
index bda43571db862311f8887f1af6e04f8d75aa29e5..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/docs/_restrictions.md
+++ /dev/null
@@ -1,202 +0,0 @@
-Restrictions & Caveats {#restrictions}
-======================
-In addition to those listed for some functions.
-
-
-## Long-lived read transactions {#long-lived-read}
-Avoid long-lived read transactions, especially in the scenarios with a
-high rate of write transactions. Long-lived read transactions prevents
-recycling pages retired/freed by newer write transactions, thus the
-database can grow quickly.
-
-Understanding the problem of long-lived read transactions requires some
-explanation, but can be difficult for quick perception. So is is
-reasonable to simplify this as follows:
- 1. Garbage collection problem exists in all databases one way or
- another, e.g. VACUUM in PostgreSQL. But in MDBX it's even more
- discernible because of high transaction rate and intentional
- internals simplification in favor of performance.
-
- 2. MDBX employs [Multiversion concurrency control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
- on the [Copy-on-Write](https://en.wikipedia.org/wiki/Copy-on-write)
- basis, that allows multiple readers runs in parallel with a write
- transaction without blocking. An each write transaction needs free
- pages to put the changed data, that pages will be placed in the new
- b-tree snapshot at commit. MDBX efficiently recycling pages from
- previous created unused snapshots, BUT this is impossible if anyone
- a read transaction use such snapshot.
-
- 3. Thus massive altering of data during a parallel long read operation
- will increase the process's work set and may exhaust entire free
- database space.
-
-A good example of long readers is a hot backup to the slow destination
-or debugging of a client application while retaining an active read
-transaction. LMDB this results in `MDB_MAP_FULL` error and subsequent write
-performance degradation.
-
-MDBX mostly solve "long-lived" readers issue by using the Handle-Slow-Readers
-\ref MDBX_hsr_func callback which allows to abort long-lived read transactions,
-and using the \ref MDBX_LIFORECLAIM mode which addresses subsequent performance degradation.
-The "next" version of libmdbx (\ref MithrilDB) will completely solve this.
-
-- Avoid suspending a process with active transactions. These would then be
- "long-lived" as above.
-
-- Avoid aborting a process with an active read-only transaction in scenarios
- with high rate of write transactions. The transaction becomes "long-lived"
- as above until a check for stale readers is performed or the LCK-file is
- reset, since the process may not remove it from the lockfile. This does
- not apply to write transactions if the system clears stale writers, see
- above.
-
-
-## Large data items and huge transactions
-
-MDBX allows you to store values up to 1 gigabyte in size, but this is
-not the main functionality for a key-value storage, but an additional
-feature that should not be abused. Such long values are stored in
-consecutive/adjacent DB pages, which has both pros and cons. This allows
-you to read long values directly without copying and without any
-overhead from a linear section of memory.
-
-On the other hand, when putting such values in the database, it is
-required to find a sufficient number of free consecutive/adjacent
-database pages, which can be very difficult and expensive, moreover
-sometimes impossible since b-tree tends to fragmentation. So, when
-placing very long values, the engine may need to process the entire GC,
-and in the absence of a sufficient sequence of free pages, increase the
-DB file. Thus, for long values, MDBX provides maximum read performance
-at the expense of write performance.
-
-A similar situation can be with huge transactions, in which a lot of
-database pages are retired. The retired pages should be put into GC as a
-list of page numbers for future reuse. But in huge transactions, such a
-list of retired page numbers can also be huge, i.e. it is a very long
-value and requires a long sequence of free pages to be saved. Thus, if
-you delete large amounts of information from the database in a single
-transaction, MDBX may need to increase the database file to save the
-list of pages to be retired.
-
-Both of these issues will be addressed in MithrilDB.
-
-
-## Space reservation
-An MDBX database configuration will often reserve considerable unused
-memory address space and maybe file size for future growth. This does
-not use actual memory or disk space, but users may need to understand
-the difference so they won't be scared off.
-
-
-## Remote filesystems
-Do not use MDBX databases on remote filesystems, even between processes
-on the same host. This breaks file locks on some platforms, possibly
-memory map sync, and certainly sync between programs on different hosts.
-
-On the other hand, MDBX support the exclusive database operation over
-a network, and cooperative read-only access to the database placed on
-a read-only network shares.
-
-
-## Child processes
-Do not use opened \ref MDBX_env instance(s) in a child processes after `fork()`.
-It would be insane to call fork() and any MDBX-functions simultaneously
-from multiple threads. The best way is to prevent the presence of open
-MDBX-instances during `fork()`.
-
-The \ref MDBX_ENV_CHECKPID build-time option, which is ON by default on
-non-Windows platforms (i.e. where `fork()` is available), enables PID
-checking at a few critical points. But this does not give any guarantees,
-but only allows you to detect such errors a little sooner. Depending on
-the platform, you should expect an application crash and/or database
-corruption in such cases.
-
-On the other hand, MDBX allow calling \ref mdbx_env_close() in such cases to
-release resources, but no more and in general this is a wrong way.
-
-
-## Read-only mode
-There is no pure read-only mode in a normal explicitly way, since
-readers need write access to LCK-file to be ones visible for writer.
-
-So MDBX always tries to open/create LCK-file for read-write, but switches
-to without-LCK mode on appropriate errors (`EROFS`, `EACCESS`, `EPERM`)
-if the read-only mode was requested by the \ref MDBX_RDONLY flag which is
-described below.
-
-The "next" version of libmdbx (\ref MithrilDB) will solve this issue for the "many
-readers without writer" case.
-
-
-## Troubleshooting the LCK-file
-1. A broken LCK-file can cause sync issues, including appearance of
- wrong/inconsistent data for readers. When database opened in the
- cooperative read-write mode the LCK-file requires to be mapped to
- memory in read-write access. In this case it is always possible for
- stray/malfunctioned application could writes thru pointers to
- silently corrupt the LCK-file.
-
- Unfortunately, there is no any portable way to prevent such
- corruption, since the LCK-file is updated concurrently by
- multiple processes in a lock-free manner and any locking is
- unwise due to a large overhead.
-
- The "next" version of libmdbx (\ref MithrilDB) will solve this issue.
-
- \note Workaround: Just make all programs using the database close it;
- the LCK-file is always reset on first open.
-
-2. Stale reader transactions left behind by an aborted program cause
- further writes to grow the database quickly, and stale locks can
- block further operation.
- MDBX checks for stale readers while opening environment and before
- growth the database. But in some cases, this may not be enough.
-
- \note Workaround: Check for stale readers periodically, using the
- \ref mdbx_reader_check() function or the mdbx_stat tool.
-
-3. Stale writers will be cleared automatically by MDBX on supported
- platforms. But this is platform-specific, especially of
- implementation of shared POSIX-mutexes and support for robust
- mutexes. For instance there are no known issues on Linux, OSX,
- Windows and FreeBSD.
-
- \note Workaround: Otherwise just make all programs using the database
- close it; the LCK-file is always reset on first open of the environment.
-
-
-## One thread - One transaction
- A thread can only use one transaction at a time, plus any nested
- read-write transactions in the non-writemap mode. Each transaction
- belongs to one thread. The \ref MDBX_NOTLS flag changes this for read-only
- transactions. See below.
-
- Do not start more than one transaction for a one thread. If you think
- about this, it's really strange to do something with two data snapshots
- at once, which may be different. MDBX checks and preventing this by
- returning corresponding error code (\ref MDBX_TXN_OVERLAPPING, \ref MDBX_BAD_RSLOT,
- \ref MDBX_BUSY) unless you using \ref MDBX_NOTLS option on the environment.
- Nonetheless, with the `MDBX_NOTLS` option, you must know exactly what you
- are doing, otherwise you will get deadlocks or reading an alien data.
-
-
-## Do not open twice
-Do not have open an MDBX database twice in the same process at the same
-time. By default MDBX prevent this in most cases by tracking databases
-opening and return \ref MDBX_BUSY if anyone LCK-file is already open.
-
-The reason for this is that when the "Open file description" locks (aka
-OFD-locks) are not available, MDBX uses POSIX locks on files, and these
-locks have issues if one process opens a file multiple times. If a single
-process opens the same environment multiple times, closing it once will
-remove all the locks held on it, and the other instances will be
-vulnerable to corruption from other processes.
-
-For compatibility with LMDB which allows multi-opening, MDBX can be
-configured at runtime by `mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...)`
-prior to calling other MDBX functions. In this way MDBX will track
-databases opening, detect multi-opening cases and then recover POSIX file
-locks as necessary. However, lock recovery can cause unexpected pauses,
-such as when another process opened the database in exclusive mode before
-the lock was restored - we have to wait until such a process releases the
-database, and so on.
diff --git a/3rdparty/libmdbx/docs/_starting.md b/3rdparty/libmdbx/docs/_starting.md
deleted file mode 100644
index 95783fd017589ceac775da3445014c10367d7839..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/docs/_starting.md
+++ /dev/null
@@ -1,242 +0,0 @@
-Getting started {#starting}
-===============
-
-> This section is based on Bert Hubert's intro "LMDB Semantics", with
-> edits reflecting the improvements and enhancements were made in MDBX.
-> See Bert Hubert's [original](https://github.com/ahupowerdns/ahutils/blob/master/lmdb-semantics.md).
-
-Everything starts with an environment, created by \ref mdbx_env_create().
-Once created, this environment must also be opened with \ref mdbx_env_open(),
-and after use be closed by \ref mdbx_env_close(). At that a non-zero value
-of the last argument "mode" supposes MDBX will create database and directory
-if ones does not exist. In this case the non-zero "mode" argument specifies
-the file mode bits be applied when a new files are created by `open()` function.
-
-Within that directory, a lock file (aka LCK-file) and a storage file (aka
-DXB-file) will be generated. If you don't want to use a directory, you can
-pass the \ref MDBX_NOSUBDIR option, in which case the path you provided is used
-directly as the DXB-file, and another file with a "-lck" suffix added
-will be used for the LCK-file.
-
-Once the environment is open, a transaction can be created within it using
-\ref mdbx_txn_begin(). Transactions may be read-write or read-only, and read-write
-transactions may be nested. A transaction must only be used by one thread at
-a time. Transactions are always required, even for read-only access. The
-transaction provides a consistent view of the data.
-
-Once a transaction has been created, a database (i.e. key-value space inside
-the environment) can be opened within it using \ref mdbx_dbi_open(). If only one
-database will ever be used in the environment, a `NULL` can be passed as the
-database name. For named databases, the \ref MDBX_CREATE flag must be used to
-create the database if it doesn't already exist. Also, \ref mdbx_env_set_maxdbs()
-must be called after \ref mdbx_env_create() and before \ref mdbx_env_open() to set
-the maximum number of named databases you want to support.
-
-\note A single transaction can open multiple databases. Generally databases
-should only be opened once, by the first transaction in the process.
-
-Within a transaction, \ref mdbx_get() and \ref mdbx_put() can store single key-value
-pairs if that is all you need to do (but see \ref Cursors below if you want to do
-more).
-
-A key-value pair is expressed as two \ref MDBX_val structures. This struct that is
-exactly similar to POSIX's `struct iovec` and has two fields, `iov_len` and
-`iov_base`. The data is a `void` pointer to an array of `iov_len` bytes.
-\note The notable difference between MDBX and LMDB is that MDBX support zero
-length keys.
-
-Because MDBX is very efficient (and usually zero-copy), the data returned in
-an \ref MDBX_val structure may be memory-mapped straight from disk. In other words
-look but do not touch (or `free()` for that matter). Once a transaction is
-closed, the values can no longer be used, so make a copy if you need to keep
-them after that.
-
-## Cursors {#Cursors}
-To do more powerful things, we must use a cursor.
-
-Within the transaction, a cursor can be created with \ref mdbx_cursor_open().
-With this cursor we can store/retrieve/delete (multiple) values using
-\ref mdbx_cursor_get(), \ref mdbx_cursor_put() and \ref mdbx_cursor_del().
-
-The \ref mdbx_cursor_get() positions itself depending on the cursor operation
-requested, and for some operations, on the supplied key. For example, to list
-all key-value pairs in a database, use operation \ref MDBX_FIRST for the first
-call to \ref mdbx_cursor_get(), and \ref MDBX_NEXT on subsequent calls, until
-the end is hit.
-
-To retrieve all keys starting from a specified key value, use \ref MDBX_SET. For
-more cursor operations, see the \ref c_api reference.
-
-When using \ref mdbx_cursor_put()\ref , either the function will position the cursor
-for you based on the key, or you can use operation \ref MDBX_CURRENT to use the
-current position of the cursor. \note Note that key must then match the current
-position's key.
-
-
-## Summarizing the opening
-
-So we have a cursor in a transaction which opened a database in an
-environment which is opened from a filesystem after it was separately
-created.
-
-Or, we create an environment, open it from a filesystem, create a transaction
-within it, open a database within that transaction, and create a cursor
-within all of the above.
-
-Got it?
-
-
-## Threads and processes
-
-Do not have open an database twice in the same process at the same time, MDBX
-will track and prevent this. Instead, share the MDBX environment that has
-opened the file across all threads. The reason for this is:
- - When the "Open file description" locks (aka OFD-locks) are not available,
- MDBX uses POSIX locks on files, and these locks have issues if one process
- opens a file multiple times.
- - If a single process opens the same environment multiple times, closing it
- once will remove all the locks held on it, and the other instances will be
- vulnerable to corruption from other processes.
- + For compatibility with LMDB which allows multi-opening, MDBX can be
- configured at runtime by \ref mdbx_setup_debug() with \ref MDBX_DBG_LEGACY_MULTIOPEN` option
- prior to calling other MDBX functions. In this way MDBX will track
- databases opening, detect multi-opening cases and then recover POSIX file
- locks as necessary. However, lock recovery can cause unexpected pauses,
- such as when another process opened the database in exclusive mode before
- the lock was restored - we have to wait until such a process releases the
- database, and so on.
-
-Do not use opened MDBX environment(s) after `fork()` in a child process(es),
-MDBX will check and prevent this at critical points. Instead, ensure there is
-no open MDBX-instance(s) during fork(), or at least close it immediately after
-`fork()` in the child process and reopen if required - for instance by using
-`pthread_atfork()`. The reason for this is:
- - For competitive consistent reading, MDBX assigns a slot in the shared
- table for each process that interacts with the database. This slot is
- populated with process attributes, including the PID.
- - After `fork()`, in order to remain connected to a database, the child
- process must have its own such "slot", which can't be assigned in any
- simple and robust way another than the regular.
- - A write transaction from a parent process cannot continue in a child
- process for obvious reasons.
- - Moreover, in a multithreaded process at the fork() moment any number of
- threads could run in critical and/or intermediate sections of MDBX code
- with interaction and/or racing conditions with threads from other
- process(es). For instance: shrinking a database or copying it to a pipe,
- opening or closing environment, beginning or finishing a transaction,
- and so on.
- = Therefore, any solution other than simply close database (and reopen if
- necessary) in a child process would be both extreme complicated and so
- fragile.
-
-Do not start more than one transaction for a one thread. If you think about
-this, it's really strange to do something with two data snapshots at once,
-which may be different. MDBX checks and preventing this by returning
-corresponding error code (\ref MDBX_TXN_OVERLAPPING, \ref MDBX_BAD_RSLOT, \ref MDBX_BUSY)
-unless you using \ref MDBX_NOTLS option on the environment. Nonetheless, with the
-\ref MDBX_NOTLS option, you must know exactly what you are doing, otherwise you
-will get deadlocks or reading an alien data.
-
-Also note that a transaction is tied to one thread by default using Thread
-Local Storage. If you want to pass read-only transactions across threads,
-you can use the \ref MDBX_NOTLS option on the environment. Nevertheless, a write
-transaction entirely should only be used in one thread from start to finish.
-MDBX checks this in a reasonable manner and return the \ref MDBX_THREAD_MISMATCH
-error in rules violation.
-
-
-## Transactions, rollbacks etc
-
-To actually get anything done, a transaction must be committed using
-\ref mdbx_txn_commit(). Alternatively, all of a transaction's operations
-can be discarded using \ref mdbx_txn_abort().
-
-\attention An important difference between MDBX and LMDB is that MDBX required
-that any opened cursors can be reused and must be freed explicitly, regardless
-ones was opened in a read-only or write transaction. The REASON for this is
-eliminates ambiguity which helps to avoid errors such as: use-after-free,
-double-free, i.e. memory corruption and segfaults.
-
-For read-only transactions, obviously there is nothing to commit to storage.
-\attention An another notable difference between MDBX and LMDB is that MDBX make
-handles opened for existing databases immediately available for other
-transactions, regardless this transaction will be aborted or reset. The
-REASON for this is to avoiding the requirement for multiple opening a same
-handles in concurrent read transactions, and tracking of such open but hidden
-handles until the completion of read transactions which opened them.
-
-In addition, as long as a transaction is open, a consistent view of the
-database is kept alive, which requires storage. A read-only transaction that
-no longer requires this consistent view should be terminated (committed or
-aborted) when the view is no longer needed (but see below for an
-optimization).
-
-There can be multiple simultaneously active read-only transactions but only
-one that can write. Once a single read-write transaction is opened, all
-further attempts to begin one will block until the first one is committed or
-aborted. This has no effect on read-only transactions, however, and they may
-continue to be opened at any time.
-
-
-## Duplicate keys aka Multi-values
-
-\ref mdbx_get() and \ref mdbx_put() respectively have no and only some support or
-multiple key-value pairs with identical keys. If there are multiple values
-for a key, \ref mdbx_get() will only return the first value.
-
-When multiple values for one key are required, pass the \ref MDBX_DUPSORT flag to
-\ref mdbx_dbi_open(). In an \ref MDBX_DUPSORT database, by default \ref mdbx_put() will
-not replace the value for a key if the key existed already. Instead it will add
-the new value to the key. In addition, \ref mdbx_del() will pay attention to the
-value field too, allowing for specific values of a key to be deleted.
-
-Finally, additional cursor operations become available for traversing through
-and retrieving duplicate values.
-
-
-## Some optimization
-
-If you frequently begin and abort read-only transactions, as an optimization,
-it is possible to only reset and renew a transaction.
-
-\ref mdbx_txn_reset() releases any old copies of data kept around for a read-only
-transaction. To reuse this reset transaction, call \ref mdbx_txn_renew() on it.
-Any cursors in this transaction can also be renewed using \ref mdbx_cursor_renew()
-or freed by \ref mdbx_cursor_close().
-
-To permanently free a transaction, reset or not, use \ref mdbx_txn_abort().
-
-
-## Cleaning up
-
-Any created cursors must be closed using \ref mdbx_cursor_close(). It is advisable
-to repeat:
-\note An important difference between MDBX and LMDB is that MDBX required that
-any opened cursors can be reused and must be freed explicitly, regardless
-ones was opened in a read-only or write transaction. The REASON for this is
-eliminates ambiguity which helps to avoid errors such as: use-after-free,
-double-free, i.e. memory corruption and segfaults.
-
-It is very rarely necessary to close a database handle, and in general they
-should just be left open. When you close a handle, it immediately becomes
-unavailable for all transactions in the environment. Therefore, you should
-avoid closing the handle while at least one transaction is using it.
-
-
-## Now read up on the full API!
-
-The full \ref c_api documentation lists further details below, like how to:
-
-- Configure database size and automatic size management: \ref mdbx_env_set_geometry().
-- Drop and clean a database: \ref mdbx_drop().
-- Detect and report errors: \ref c_err.
-- Optimize (bulk) loading speed: \ref MDBX_MULTIPLE, \ref MDBX_APPEND.
-- Reduce (temporarily) robustness to gain even more speed: \ref sync_modes.
-- Gather statistics about the database: \ref c_statinfo.
-- Sstimate size of range query result: \ref c_rqest.
-- Double performance by LIFO reclaiming on storages with write-back: \ref MDBX_LIFORECLAIM.
-- Use sequences and canary markers: \ref mdbx_dbi_sequence(), \ref MDBX_canary.
-- Use Handle-Slow-Readers callback to resolve a database full/overflow issues
- due to long-lived read transactions: \ref mdbx_env_set_hsr().
-- Use exclusive mode: \ref MDBX_EXCLUSIVE.
-- Define custom sort orders (but this is recommended to be avoided).
diff --git a/3rdparty/libmdbx/docs/_toc.md b/3rdparty/libmdbx/docs/_toc.md
deleted file mode 100644
index 434806f08368e5c82651a881e93c5526b94f7662..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/docs/_toc.md
+++ /dev/null
@@ -1,41 +0,0 @@
-### We have migrated to a reliable trusted infrastructure
-
-The origin for now is at
-[GitFlic](https://gitflic.ru/project/erthink/libmdbx) since on
-2022-04-15 the Github administration, without any warning nor
-explanation, deleted _libmdbx_ along with a lot of other projects,
-simultaneously blocking access for many developers. For the same reason
-~~Github~~ is blacklisted forever.
-
-> Questions, feedback and suggestions are welcome to the [Telegram' group](https://t.me/libmdbx).
-
-_The Future will (be) [Positive](https://www.ptsecurity.com). Ð’ÑÑ‘ будет хорошо._
-
-\section toc Table of Contents
-
-This manual is divided into parts,
-each of which is divided into several sections.
-
-1. The \ref intro
- - \ref characteristics
- - \ref improvements
- - \ref restrictions
- - \ref performance
-2. \ref usage
- - \ref getting
- - \ref starting
- - \ref bindings
-
-3. The `C/C++` API manual:
- - The \ref c_api reference
- - \ref c_crud_hints "Quick reference for Insert/Update/Delete operations"
- - The \ref mdbx.h header file reference
- - The \ref cxx_api reference
- - The \ref mdbx.h++ header file reference
-
-Please do not hesitate to point out errors in the documentation,
-including creating [merge-request](https://gitflic.ru/project/erthink/libmdbx/merge-request) with corrections and improvements.
-
----
-
-\section MithrilDB MithrilDB
diff --git a/3rdparty/libmdbx/example/CMakeLists.txt b/3rdparty/libmdbx/example/CMakeLists.txt
deleted file mode 100644
index d3e56e82e9b33a48df7f7c1ddbba26da42c10c7c..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/example/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-set(TARGET mdbx_example)
-project(${TARGET})
-
-add_executable(${TARGET} example-mdbx.c)
-
-target_link_libraries(${TARGET} mdbx)
diff --git a/3rdparty/libmdbx/example/README.md b/3rdparty/libmdbx/example/README.md
deleted file mode 100644
index b819cf4a403a980fa73d90e87cad2e79d1ee2865..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/example/README.md
+++ /dev/null
@@ -1 +0,0 @@
-See [example-mdbx.c](example-mdbx.c) as an example of using _libmdbx_, and do a line-by-line comparison of it with the [sample-bdb.txt](sample-bdb.txt) file.
diff --git a/3rdparty/libmdbx/example/example-mdbx.c b/3rdparty/libmdbx/example/example-mdbx.c
deleted file mode 100644
index 501c430d18fee169669087d97a6ca1677985b71c..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/example/example-mdbx.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/* MDBX usage example
- *
- * Do a line-by-line comparison of this and sample-bdb.txt
- */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
- * Copyright 2017 Ilya Shipitsin <chipitsine@gmail.com>.
- * Copyright 2012-2015 Howard Chu, Symas Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "mdbx.h"
-#include <limits.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-int main(int argc, char *argv[]) {
- (void)argc;
- (void)argv;
-
- int rc;
- MDBX_env *env = NULL;
- MDBX_dbi dbi = 0;
- MDBX_val key, data;
- MDBX_txn *txn = NULL;
- MDBX_cursor *cursor = NULL;
- char sval[32];
-
- printf("MDBX limits:\n");
-#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
- const double scale_factor = 1099511627776.0;
- const char *const scale_unit = "TiB";
-#else
- const double scale_factor = 1073741824.0;
- const char *const scale_unit = "GiB";
-#endif
- const size_t pagesize_min = mdbx_limits_pgsize_min();
- const size_t pagesize_max = mdbx_limits_pgsize_max();
- const size_t pagesize_default = mdbx_default_pagesize();
-
- printf("\tPage size: a power of 2, minimum %zu, maximum %zu bytes,"
- " default %zu bytes.\n",
- pagesize_min, pagesize_max, pagesize_default);
- printf("\tKey size: minimum %zu, maximum ≈¼ pagesize (%zu bytes for default"
- " %zuK pagesize, %zu bytes for %zuK pagesize).\n",
- (size_t)0, mdbx_limits_keysize_max(-1, MDBX_DB_DEFAULTS),
- pagesize_default / 1024,
- mdbx_limits_keysize_max(pagesize_max, MDBX_DB_DEFAULTS),
- pagesize_max / 1024);
- printf("\tValue size: minimum %zu, maximum %zu (0x%08zX) bytes for maps,"
- " ≈¼ pagesize for multimaps (%zu bytes for default %zuK pagesize,"
- " %zu bytes for %zuK pagesize).\n",
- (size_t)0, mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS),
- mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS),
- mdbx_limits_valsize_max(-1, MDBX_DUPSORT), pagesize_default / 1024,
- mdbx_limits_valsize_max(pagesize_max, MDBX_DUPSORT),
- pagesize_max / 1024);
- printf("\tWrite transaction size: up to %zu (0x%zX) pages (%f %s for default "
- "%zuK pagesize, %f %s for %zuK pagesize).\n",
- mdbx_limits_txnsize_max(pagesize_min) / pagesize_min,
- mdbx_limits_txnsize_max(pagesize_min) / pagesize_min,
- mdbx_limits_txnsize_max(-1) / scale_factor, scale_unit,
- pagesize_default / 1024,
- mdbx_limits_txnsize_max(pagesize_max) / scale_factor, scale_unit,
- pagesize_max / 1024);
- printf("\tDatabase size: up to %zu pages (%f %s for default %zuK "
- "pagesize, %f %s for %zuK pagesize).\n",
- mdbx_limits_dbsize_max(pagesize_min) / pagesize_min,
- mdbx_limits_dbsize_max(-1) / scale_factor, scale_unit,
- pagesize_default / 1024,
- mdbx_limits_dbsize_max(pagesize_max) / scale_factor, scale_unit,
- pagesize_max / 1024);
- printf("\tMaximum sub-databases: %u.\n", MDBX_MAX_DBI);
- printf("-----\n");
-
- rc = mdbx_env_create(&env);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- rc = mdbx_env_open(env, "./example-db",
- MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- rc = mdbx_txn_begin(env, NULL, 0, &txn);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- rc = mdbx_dbi_open(txn, NULL, 0, &dbi);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- key.iov_len = sizeof(int);
- key.iov_base = sval;
- data.iov_len = sizeof(sval);
- data.iov_base = sval;
-
- sprintf(sval, "%03x %d foo bar", 32, 3141592);
- rc = mdbx_put(txn, dbi, &key, &data, 0);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- rc = mdbx_txn_commit(txn);
- if (rc) {
- fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- txn = NULL;
-
- rc = mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &txn);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- rc = mdbx_cursor_open(txn, dbi, &cursor);
- if (rc != MDBX_SUCCESS) {
- fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- int found = 0;
- while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
- printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len,
- (char *)key.iov_base, data.iov_base, (int)data.iov_len,
- (char *)data.iov_base);
- found += 1;
- }
- if (rc != MDBX_NOTFOUND || found == 0) {
- fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- } else {
- rc = MDBX_SUCCESS;
- }
-bailout:
- if (cursor)
- mdbx_cursor_close(cursor);
- if (txn)
- mdbx_txn_abort(txn);
- if (dbi)
- mdbx_dbi_close(env, dbi);
- if (env)
- mdbx_env_close(env);
- return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/example/sample-bdb.txt b/3rdparty/libmdbx/example/sample-bdb.txt
deleted file mode 100644
index 503d97cb7ea9c50e0f809b55457f9a147604af13..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/example/sample-bdb.txt
+++ /dev/null
@@ -1,77 +0,0 @@
-/* BerkeleyDB toy/sample
- *
- * Do a line-by-line comparison of this and example-mdbx.c
- */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
- * Copyright 2012-2015 Howard Chu, Symas Corp.
- * Copyright 2015,2016 Peter-Service R&D LLC.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include <stdio.h>
-#include <string.h>
-#include <db.h>
-
-int main(int argc,char * argv[])
-{
- int rc;
- DB_ENV *env;
- DB *dbi;
- DBT key, data;
- DB_TXN *txn;
- DBC *cursor;
- char sval[32], kval[32];
-
- /* Note: Most error checking omitted for simplicity */
-
-#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD)
- rc = db_env_create(&env, 0);
- rc = env->open(env, "./testdb", FLAGS, 0664);
- rc = db_create(&dbi, env, 0);
- rc = env->txn_begin(env, NULL, &txn, 0);
- rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664);
-
- memset(&key, 0, sizeof(DBT));
- memset(&data, 0, sizeof(DBT));
- key.size = sizeof(int);
- key.data = sval;
- data.size = sizeof(sval);
- data.data = sval;
-
- sprintf(sval, "%03x %d foo bar", 32, 3141592);
- rc = dbi->put(dbi, txn, &key, &data, 0);
- rc = txn->commit(txn, 0);
- if (rc) {
- fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc));
- goto leave;
- }
- rc = env->txn_begin(env, NULL, &txn, 0);
- rc = dbi->cursor(dbi, txn, &cursor, 0);
- key.flags = DB_DBT_USERMEM;
- key.data = kval;
- key.ulen = sizeof(kval);
- data.flags = DB_DBT_USERMEM;
- data.data = sval;
- data.ulen = sizeof(sval);
- while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
- printf("key: %p %.*s, data: %p %.*s\n",
- key.data, (int) key.size, (char *) key.data,
- data.data, (int) data.size, (char *) data.data);
- }
- rc = cursor->c_close(cursor);
- rc = txn->abort(txn);
-leave:
- rc = dbi->close(dbi, 0);
- rc = env->close(env, 0);
- return rc;
-}
diff --git a/3rdparty/libmdbx/src/man1/mdbx_chk.1 b/3rdparty/libmdbx/man1/mdbx_chk.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_chk.1
rename to 3rdparty/libmdbx/man1/mdbx_chk.1
diff --git a/3rdparty/libmdbx/src/man1/mdbx_copy.1 b/3rdparty/libmdbx/man1/mdbx_copy.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_copy.1
rename to 3rdparty/libmdbx/man1/mdbx_copy.1
diff --git a/3rdparty/libmdbx/src/man1/mdbx_drop.1 b/3rdparty/libmdbx/man1/mdbx_drop.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_drop.1
rename to 3rdparty/libmdbx/man1/mdbx_drop.1
diff --git a/3rdparty/libmdbx/src/man1/mdbx_dump.1 b/3rdparty/libmdbx/man1/mdbx_dump.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_dump.1
rename to 3rdparty/libmdbx/man1/mdbx_dump.1
diff --git a/3rdparty/libmdbx/src/man1/mdbx_load.1 b/3rdparty/libmdbx/man1/mdbx_load.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_load.1
rename to 3rdparty/libmdbx/man1/mdbx_load.1
diff --git a/3rdparty/libmdbx/src/man1/mdbx_stat.1 b/3rdparty/libmdbx/man1/mdbx_stat.1
similarity index 100%
rename from 3rdparty/libmdbx/src/man1/mdbx_stat.1
rename to 3rdparty/libmdbx/man1/mdbx_stat.1
diff --git a/3rdparty/libmdbx/src/core.c b/3rdparty/libmdbx/mdbx.c
similarity index 76%
rename from 3rdparty/libmdbx/src/core.c
rename to 3rdparty/libmdbx/mdbx.c
index d6711fb9d13374ea7387da00f2d81c8eb6ed26ba..a6db608ee4d17cc4a67e0cc44342f5658c34f42a 100644
--- a/3rdparty/libmdbx/src/core.c
+++ b/3rdparty/libmdbx/mdbx.c
@@ -1,1780 +1,5213 @@
/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
- * This code is derived from "LMDB engine" written by
- * Howard Chu (Symas Corporation), which itself derived from btree.c
- * written by Martin Hedenfalk.
- *
- * ---
- *
- * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
- *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define xMDBX_ALLOY 1
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
*
- * ---
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
*
- * Portions Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
*
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
*
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
-#include "internals.h"
+/* *INDENT-OFF* */
+/* clang-format off */
-/*------------------------------------------------------------------------------
- * Internal inline functions */
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
-MDBX_NOTHROW_CONST_FUNCTION static unsigned branchless_abs(int value) {
- assert(value > INT_MIN);
- const unsigned expanded_sign =
- (unsigned)(value >> (sizeof(value) * CHAR_BIT - 1));
- return ((unsigned)value + expanded_sign) ^ expanded_sign;
-}
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
-/* Pack/Unpack 16-bit values for Grow step & Shrink threshold */
-MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t me2v(unsigned m,
- unsigned e) {
- assert(m < 2048 && e < 8);
- return (pgno_t)(32768 + ((m + 1) << (e + 8)));
-}
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
-MDBX_NOTHROW_CONST_FUNCTION static __inline uint16_t v2me(size_t v,
- unsigned e) {
- assert(v > (e ? me2v(2047, e - 1) : 32768));
- assert(v <= me2v(2047, e));
- size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8);
- m -= m > 0;
- assert(m < 2048 && e < 8);
- // f e d c b a 9 8 7 6 5 4 3 2 1 0
- // 1 e e e m m m m m m m m m m m 1
- const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1));
- assert(pv != 65535);
- return pv;
-}
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
-/* Convert 16-bit packed (exponential quantized) value to number of pages */
-MDBX_NOTHROW_CONST_FUNCTION static pgno_t pv2pages(uint16_t pv) {
- if ((pv & 0x8001) != 0x8001)
- return pv;
- if (pv == 65535)
- return 65536;
- // f e d c b a 9 8 7 6 5 4 3 2 1 0
- // 1 e e e m m m m m m m m m m m 1
- return me2v((pv >> 1) & 2047, (pv >> 12) & 7);
-}
+/*----------------------------------------------------------------------------*/
-/* Convert number of pages to 16-bit packed (exponential quantized) value */
-MDBX_NOTHROW_CONST_FUNCTION static uint16_t pages2pv(size_t pages) {
- if (pages < 32769 || (pages < 65536 && (pages & 1) == 0))
- return (uint16_t)pages;
- if (pages <= me2v(2047, 0))
- return v2me(pages, 0);
- if (pages <= me2v(2047, 1))
- return v2me(pages, 1);
- if (pages <= me2v(2047, 2))
- return v2me(pages, 2);
- if (pages <= me2v(2047, 3))
- return v2me(pages, 3);
- if (pages <= me2v(2047, 4))
- return v2me(pages, 4);
- if (pages <= me2v(2047, 5))
- return v2me(pages, 5);
- if (pages <= me2v(2047, 6))
- return v2me(pages, 6);
- return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533;
-}
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
-/*------------------------------------------------------------------------------
- * Unaligned access */
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline unsigned
-field_alignment(unsigned alignment_baseline, size_t field_offset) {
- unsigned merge = alignment_baseline | (unsigned)field_offset;
- return merge & -(int)merge;
-}
+/*----------------------------------------------------------------------------*/
-/* read-thunk for UB-sanitizer */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t
-peek_u8(const uint8_t *const __restrict ptr) {
- return *ptr;
-}
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
-/* write-thunk for UB-sanitizer */
-static __always_inline void poke_u8(uint8_t *const __restrict ptr,
- const uint8_t v) {
- *ptr = v;
-}
+/*----------------------------------------------------------------------------*/
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint16_t
-unaligned_peek_u16(const unsigned expected_alignment, const void *const ptr) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0)
- return *(const uint16_t *)ptr;
- else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- return *(const __unaligned uint16_t *)ptr;
-#else
- uint16_t v;
- memcpy(&v, ptr, sizeof(v));
- return v;
-#endif /* _MSC_VER || __unaligned */
- }
-}
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
-static __always_inline void
-unaligned_poke_u16(const unsigned expected_alignment,
- void *const __restrict ptr, const uint16_t v) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0)
- *(uint16_t *)ptr = v;
- else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- *((uint16_t __unaligned *)ptr) = v;
-#else
- memcpy(ptr, &v, sizeof(v));
-#endif /* _MSC_VER || __unaligned */
- }
-}
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t unaligned_peek_u32(
- const unsigned expected_alignment, const void *const __restrict ptr) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0)
- return *(const uint32_t *)ptr;
- else if ((expected_alignment % sizeof(uint16_t)) == 0) {
- const uint16_t lo =
- ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
- const uint16_t hi =
- ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
- return lo | (uint32_t)hi << 16;
- } else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- return *(const __unaligned uint32_t *)ptr;
-#else
- uint32_t v;
- memcpy(&v, ptr, sizeof(v));
- return v;
-#endif /* _MSC_VER || __unaligned */
- }
-}
+/*----------------------------------------------------------------------------*/
-static __always_inline void
-unaligned_poke_u32(const unsigned expected_alignment,
- void *const __restrict ptr, const uint32_t v) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0)
- *(uint32_t *)ptr = v;
- else if ((expected_alignment % sizeof(uint16_t)) == 0) {
- ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v;
- ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
- (uint16_t)(v >> 16);
- } else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- *((uint32_t __unaligned *)ptr) = v;
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
#else
- memcpy(ptr, &v, sizeof(v));
-#endif /* _MSC_VER || __unaligned */
- }
-}
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t unaligned_peek_u64(
- const unsigned expected_alignment, const void *const __restrict ptr) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
- return *(const uint64_t *)ptr;
- else if ((expected_alignment % sizeof(uint32_t)) == 0) {
- const uint32_t lo =
- ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
- const uint32_t hi =
- ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
- return lo | (uint64_t)hi << 32;
- } else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- return *(const __unaligned uint64_t *)ptr;
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
#else
- uint64_t v;
- memcpy(&v, ptr, sizeof(v));
- return v;
-#endif /* _MSC_VER || __unaligned */
- }
-}
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
-static __always_inline uint64_t
-unaligned_peek_u64_volatile(const unsigned expected_alignment,
- volatile const void *const __restrict ptr) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- assert(expected_alignment % sizeof(uint32_t) == 0);
- if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
- return *(volatile const uint64_t *)ptr;
- else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- return *(volatile const __unaligned uint64_t *)ptr;
-#else
- const uint32_t lo = ((volatile const uint32_t *)
- ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
- const uint32_t hi = ((volatile const uint32_t *)
- ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
- return lo | (uint64_t)hi << 32;
-#endif /* _MSC_VER || __unaligned */
- }
-}
+#ifdef __cplusplus
+extern "C" {
+#endif
-static __always_inline void
-unaligned_poke_u64(const unsigned expected_alignment,
- void *const __restrict ptr, const uint64_t v) {
- assert((uintptr_t)ptr % expected_alignment == 0);
- if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0)
- *(uint64_t *)ptr = v;
- else if ((expected_alignment % sizeof(uint32_t)) == 0) {
- ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v;
- ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
- (uint32_t)(v >> 32);
- } else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- *((uint64_t __unaligned *)ptr) = v;
-#else
- memcpy(ptr, &v, sizeof(v));
-#endif /* _MSC_VER || __unaligned */
- }
-}
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
-#define UNALIGNED_PEEK_8(ptr, struct, field) \
- peek_u8((const uint8_t *)(ptr) + offsetof(struct, field))
-#define UNALIGNED_POKE_8(ptr, struct, field, value) \
- poke_u8((uint8_t *)(ptr) + offsetof(struct, field), value)
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
-#define UNALIGNED_PEEK_16(ptr, struct, field) \
- unaligned_peek_u16(1, (const char *)(ptr) + offsetof(struct, field))
-#define UNALIGNED_POKE_16(ptr, struct, field, value) \
- unaligned_poke_u16(1, (char *)(ptr) + offsetof(struct, field), value)
-#define UNALIGNED_PEEK_32(ptr, struct, field) \
- unaligned_peek_u32(1, (const char *)(ptr) + offsetof(struct, field))
-#define UNALIGNED_POKE_32(ptr, struct, field, value) \
- unaligned_poke_u32(1, (char *)(ptr) + offsetof(struct, field), value)
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
-#define UNALIGNED_PEEK_64(ptr, struct, field) \
- unaligned_peek_u64(1, (const char *)(ptr) + offsetof(struct, field))
-#define UNALIGNED_POKE_64(ptr, struct, field, value) \
- unaligned_poke_u64(1, (char *)(ptr) + offsetof(struct, field), value)
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
-/* Get the page number pointed to by a branch node */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
-node_pgno(const MDBX_node *const __restrict node) {
- pgno_t pgno = UNALIGNED_PEEK_32(node, MDBX_node, mn_pgno32);
- if (sizeof(pgno) > 4)
- pgno |= ((uint64_t)UNALIGNED_PEEK_8(node, MDBX_node, mn_extra)) << 32;
- return pgno;
-}
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
-/* Set the page number in a branch node */
-static __always_inline void node_set_pgno(MDBX_node *const __restrict node,
- pgno_t pgno) {
- assert(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO);
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
- UNALIGNED_POKE_32(node, MDBX_node, mn_pgno32, (uint32_t)pgno);
- if (sizeof(pgno) > 4)
- UNALIGNED_POKE_8(node, MDBX_node, mn_extra,
- (uint8_t)((uint64_t)pgno >> 32));
-}
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
-/* Get the size of the data in a leaf node */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-node_ds(const MDBX_node *const __restrict node) {
- return UNALIGNED_PEEK_32(node, MDBX_node, mn_dsize);
-}
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
-/* Set the size of the data for a leaf node */
-static __always_inline void node_set_ds(MDBX_node *const __restrict node,
- size_t size) {
- assert(size < INT_MAX);
- UNALIGNED_POKE_32(node, MDBX_node, mn_dsize, (uint32_t)size);
-}
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
-/* The size of a key in a node */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-node_ks(const MDBX_node *const __restrict node) {
- return UNALIGNED_PEEK_16(node, MDBX_node, mn_ksize);
-}
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
-/* Set the size of the key for a leaf node */
-static __always_inline void node_set_ks(MDBX_node *const __restrict node,
- size_t size) {
- assert(size < INT16_MAX);
- UNALIGNED_POKE_16(node, MDBX_node, mn_ksize, (uint16_t)size);
-}
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t
-node_flags(const MDBX_node *const __restrict node) {
- return UNALIGNED_PEEK_8(node, MDBX_node, mn_flags);
-}
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
-static __always_inline void node_set_flags(MDBX_node *const __restrict node,
- uint8_t flags) {
- UNALIGNED_POKE_8(node, MDBX_node, mn_flags, flags);
-}
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
-/* Size of the node header, excluding dynamic data at the end */
-#define NODESIZE offsetof(MDBX_node, mn_data)
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
-/* Address of the key for the node */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
-node_key(const MDBX_node *const __restrict node) {
- return (char *)node + NODESIZE;
-}
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
-/* Address of the data for a node */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
-node_data(const MDBX_node *const __restrict node) {
- return (char *)node_key(node) + node_ks(node);
-}
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
-/* Size of a node in a leaf page with a given key and data.
- * This is node header plus key plus data size. */
-MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
-node_size_len(const size_t key_len, const size_t value_len) {
- return NODESIZE + EVEN(key_len + value_len);
-}
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-node_size(const MDBX_val *key, const MDBX_val *value) {
- return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0);
-}
+#if MDBX_WITHOUT_MSVC_CRT
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
-peek_pgno(const void *const __restrict ptr) {
- if (sizeof(pgno_t) == sizeof(uint32_t))
- return (pgno_t)unaligned_peek_u32(1, ptr);
- else if (sizeof(pgno_t) == sizeof(uint64_t))
- return (pgno_t)unaligned_peek_u64(1, ptr);
- else {
- pgno_t pgno;
- memcpy(&pgno, ptr, sizeof(pgno));
- return pgno;
- }
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
}
+#endif /* mdbx_malloc */
-static __always_inline void poke_pgno(void *const __restrict ptr,
- const pgno_t pgno) {
- if (sizeof(pgno) == sizeof(uint32_t))
- unaligned_poke_u32(1, ptr, pgno);
- else if (sizeof(pgno) == sizeof(uint64_t))
- unaligned_poke_u64(1, ptr, pgno);
- else
- memcpy(ptr, &pgno, sizeof(pgno));
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
}
+#endif /* mdbx_calloc */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
-node_largedata_pgno(const MDBX_node *const __restrict node) {
- assert(node_flags(node) & F_BIGDATA);
- return peek_pgno(node_data(node));
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
}
+#endif /* mdbx_realloc */
-/*------------------------------------------------------------------------------
- * Nodes, Keys & Values length limitation factors:
- *
- * BRANCH_NODE_MAX
- * Branch-page must contain at least two nodes, within each a key and a child
- * page number. But page can't be splitted if it contains less that 4 keys,
- * i.e. a page should not overflow before adding the fourth key. Therefore,
- * at least 3 branch-node should fit in the single branch-page. Further, the
- * first node of a branch-page doesn't contain a key, i.e. the first node
- * is always require space just for itself. Thus:
- * PAGEROOM = pagesize - page_hdr_len;
- * BRANCH_NODE_MAX = even_floor(
- * (PAGEROOM - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t));
- * KEYLEN_MAX = BRANCH_NODE_MAX - node_hdr_len;
- *
- * LEAF_NODE_MAX
- * Leaf-node must fit into single leaf-page, where a value could be placed on
- * a large/overflow page. However, may require to insert a nearly page-sized
- * node between two large nodes are already fill-up a page. In this case the
- * page must be splitted to two if some pair of nodes fits on one page, or
- * otherwise the page should be splitted to the THREE with a single node
- * per each of ones. Such 1-into-3 page splitting is costly and complex since
- * requires TWO insertion into the parent page, that could lead to split it
- * and so on up to the root. Therefore double-splitting is avoided here and
- * the maximum node size is half of a leaf page space:
- * LEAF_NODE_MAX = even_floor(PAGEROOM / 2 - sizeof(indx_t));
- * DATALEN_NO_OVERFLOW = LEAF_NODE_MAX - KEYLEN_MAX;
- *
- * - SubDatabase-node must fit into one leaf-page:
- * SUBDB_NAME_MAX = LEAF_NODE_MAX - node_hdr_len - sizeof(MDBX_db);
- *
- * - Dupsort values itself are a keys in a dupsort-subdb and couldn't be longer
- * than the KEYLEN_MAX. But dupsort node must not great than LEAF_NODE_MAX,
- * since dupsort value couldn't be placed on a large/overflow page:
- * DUPSORT_DATALEN_MAX = min(KEYLEN_MAX,
- * max(DATALEN_NO_OVERFLOW, sizeof(MDBX_db));
- */
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
-#define PAGEROOM(pagesize) ((pagesize)-PAGEHDRSZ)
-#define EVEN_FLOOR(n) ((n) & ~(size_t)1)
-#define BRANCH_NODE_MAX(pagesize) \
- (EVEN_FLOOR((PAGEROOM(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - \
- sizeof(indx_t)))
-#define LEAF_NODE_MAX(pagesize) \
- (EVEN_FLOOR(PAGEROOM(pagesize) / 2) - sizeof(indx_t))
-#define MAX_GC1OVPAGE(pagesize) (PAGEROOM(pagesize) / sizeof(pgno_t) - 1)
+#else /* MDBX_WITHOUT_MSVC_CRT */
-static __inline unsigned keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
- assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE &&
- is_powerof2(pagesize));
- STATIC_ASSERT(BRANCH_NODE_MAX(MIN_PAGESIZE) - NODESIZE >= 8);
- if (flags & MDBX_INTEGERKEY)
- return 8 /* sizeof(uint64_t) */;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
- const intptr_t max_branch_key = BRANCH_NODE_MAX(pagesize) - NODESIZE;
- STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) - NODESIZE -
- /* sizeof(uint64) as a key */ 8 >
- sizeof(MDBX_db));
- if (flags &
- (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
- const intptr_t max_dupsort_leaf_key =
- LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(MDBX_db);
- return (max_branch_key < max_dupsort_leaf_key)
- ? (unsigned)max_branch_key
- : (unsigned)max_dupsort_leaf_key;
- }
- return (unsigned)max_branch_key;
-}
+#endif /* MDBX_WITHOUT_MSVC_CRT */
-static __inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
- assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE &&
- is_powerof2(pagesize));
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
- if (flags & MDBX_INTEGERDUP)
- return 8 /* sizeof(uint64_t) */;
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
- if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP))
- return keysize_max(pagesize, 0);
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
- const unsigned page_ln2 = log2n_powerof2(pagesize);
- const size_t hard = 0x7FF00000ul;
- const size_t hard_pages = hard >> page_ln2;
- STATIC_ASSERT(MDBX_PGL_LIMIT <= MAX_PAGENO);
- const size_t pages_limit = MDBX_PGL_LIMIT / 4;
- const size_t limit =
- (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2);
- return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
-}
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
-__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) {
- return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT);
-}
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
-__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env,
- MDBX_db_flags_t flags) {
- if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE))
- return -1;
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
- return (int)mdbx_limits_keysize_max((intptr_t)env->me_psize, flags);
-}
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
-size_t mdbx_default_pagesize(void) {
- size_t pagesize = mdbx_syspagesize();
- mdbx_ensure(nullptr, is_powerof2(pagesize));
- pagesize = (pagesize >= MIN_PAGESIZE) ? pagesize : MIN_PAGESIZE;
- pagesize = (pagesize <= MAX_PAGESIZE) ? pagesize : MAX_PAGESIZE;
- return pagesize;
-}
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
-__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize,
- MDBX_db_flags_t flags) {
- if (pagesize < 1)
- pagesize = (intptr_t)mdbx_default_pagesize();
- if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
- pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2((size_t)pagesize)))
- return -1;
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
- return keysize_max(pagesize, flags);
-}
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
-__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env,
- MDBX_db_flags_t flags) {
- if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE))
- return -1;
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
- return (int)mdbx_limits_valsize_max((intptr_t)env->me_psize, flags);
-}
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
-__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize,
- MDBX_db_flags_t flags) {
- if (pagesize < 1)
- pagesize = (intptr_t)mdbx_default_pagesize();
- if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
- pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2((size_t)pagesize)))
- return -1;
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
- return valsize_max(pagesize, flags);
-}
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
-/* Calculate the size of a leaf node.
- *
- * The size depends on the environment's page size; if a data item
- * is too large it will be put onto an overflow page and the node
- * size will only include the key and not the data. Sizes are always
- * rounded up to an even number of bytes, to guarantee 2-byte alignment
- * of the MDBX_node headers. */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) {
- size_t node_bytes = node_size(key, data);
- if (node_bytes > env->me_leaf_nodemax) {
- /* put on overflow page */
- node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t);
- }
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
- return node_bytes + sizeof(indx_t);
-}
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
-/* Calculate the size of a branch node.
- *
- * The size should depend on the environment's page size but since
- * we currently don't support spilling large keys onto overflow
- * pages, it's simply the size of the MDBX_node header plus the
- * size of the key. Sizes are always rounded up to an even number
- * of bytes, to guarantee 2-byte alignment of the MDBX_node headers.
- *
- * [in] env The environment handle.
- * [in] key The key for the node.
- *
- * Returns The number of bytes needed to store the node. */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-branch_size(const MDBX_env *env, const MDBX_val *key) {
- /* Size of a node in a branch page with a given key.
- * This is just the node header plus the key, there is no data. */
- size_t node_bytes = node_size(key, nullptr);
- if (unlikely(node_bytes > env->me_leaf_nodemax)) {
- /* put on overflow page */
- /* not implemented */
- mdbx_assert_fail(env, "INDXSIZE(key) <= env->me_nodemax", __func__,
- __LINE__);
- node_bytes = node_size(key, nullptr) + sizeof(pgno_t);
- }
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
- return node_bytes + sizeof(indx_t);
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
}
-MDBX_NOTHROW_CONST_FUNCTION static __always_inline uint16_t
-flags_db2sub(uint16_t db_flags) {
- uint16_t sub_flags = db_flags & MDBX_DUPFIXED;
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
- /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */
-#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2
- STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) ==
- MDBX_INTEGERKEY);
- sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY;
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
- /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */
-#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5
- STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) ==
- MDBX_REVERSEKEY);
- sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY;
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
- return sub_flags;
-}
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
-pgno2bytes(const MDBX_env *env, pgno_t pgno) {
- mdbx_assert(env, (1u << env->me_psize2log) == env->me_psize);
- return ((size_t)pgno) << env->me_psize2log;
-}
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_page *
-pgno2page(const MDBX_env *env, pgno_t pgno) {
- return (MDBX_page *)(env->me_map + pgno2bytes(env, pgno));
-}
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
-bytes2pgno(const MDBX_env *env, size_t bytes) {
- mdbx_assert(env, (env->me_psize >> env->me_psize2log) == 1);
- return (pgno_t)(bytes >> env->me_psize2log);
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
}
-MDBX_NOTHROW_PURE_FUNCTION static size_t
-pgno_align2os_bytes(const MDBX_env *env, pgno_t pgno) {
- return ceil_powerof2(pgno2bytes(env, pgno), env->me_os_psize);
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
}
-MDBX_NOTHROW_PURE_FUNCTION static pgno_t pgno_align2os_pgno(const MDBX_env *env,
- pgno_t pgno) {
- return bytes2pgno(env, pgno_align2os_bytes(env, pgno));
-}
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
-MDBX_NOTHROW_PURE_FUNCTION static size_t
-bytes_align2os_bytes(const MDBX_env *env, size_t bytes) {
- return ceil_powerof2(ceil_powerof2(bytes, env->me_psize), env->me_os_psize);
-}
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
-/* Address of first usable data byte in a page, after the header */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
-page_data(const MDBX_page *mp) {
- return (char *)mp + PAGEHDRSZ;
-}
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline const MDBX_page *
-data_page(const void *data) {
- return container_of(data, MDBX_page, mp_ptrs);
-}
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_meta *
-page_meta(MDBX_page *mp) {
- return (MDBX_meta *)page_data(mp);
-}
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
-/* Number of nodes on a page */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
-page_numkeys(const MDBX_page *mp) {
- return mp->mp_lower >> 1;
-}
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
-/* The amount of space remaining in the page */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
-page_room(const MDBX_page *mp) {
- return mp->mp_upper - mp->mp_lower;
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
}
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
-page_space(const MDBX_env *env) {
- STATIC_ASSERT(PAGEHDRSZ % 2 == 0);
- return env->me_psize - PAGEHDRSZ;
-}
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
-page_used(const MDBX_env *env, const MDBX_page *mp) {
- return page_space(env) - page_room(mp);
-}
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
-/* The percentage of space used in the page, in a percents. */
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __inline double
-page_fill(const MDBX_env *env, const MDBX_page *mp) {
- return page_used(env, mp) * 100.0 / page_space(env);
-}
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
-/* The number of overflow pages needed to store the given size. */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
-number_of_ovpages(const MDBX_env *env, size_t bytes) {
- return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1;
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
}
-__cold static int MDBX_PRINTF_ARGS(2, 3)
- bad_page(const MDBX_page *mp, const char *fmt, ...) {
- if (mdbx_log_enabled(MDBX_LOG_ERROR)) {
- static const MDBX_page *prev;
- if (prev != mp) {
- prev = mp;
- const char *type;
- switch (mp->mp_flags & (P_BRANCH | P_LEAF | P_OVERFLOW | P_META |
- P_LEAF2 | P_BAD | P_SUBP)) {
- case P_BRANCH:
- type = "branch";
- break;
- case P_LEAF:
- type = "leaf";
- break;
- case P_LEAF | P_SUBP:
- type = "subleaf";
- break;
- case P_LEAF | P_LEAF2:
- type = "dupfixed-leaf";
- break;
- case P_LEAF | P_LEAF2 | P_SUBP:
- type = "dupfixed-subleaf";
- break;
- case P_OVERFLOW:
- type = "large";
- break;
- default:
- type = "broken";
- }
- mdbx_debug_log(MDBX_LOG_ERROR, "badpage", 0,
- "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", type,
- mp->mp_pgno, mp->mp_txnid);
- }
-
- va_list args;
- va_start(args, fmt);
- mdbx_debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args);
- va_end(args);
- }
- return MDBX_CORRUPTED;
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
}
-/* Address of node i in page p */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_node *
-page_node(const MDBX_page *mp, unsigned i) {
- assert((mp->mp_flags & (P_LEAF2 | P_OVERFLOW | P_META)) == 0);
- assert(page_numkeys(mp) > (unsigned)(i));
- assert(mp->mp_ptrs[i] % 2 == 0);
- return (MDBX_node *)((char *)mp + mp->mp_ptrs[i] + PAGEHDRSZ);
-}
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
-/* The address of a key in a LEAF2 page.
- * LEAF2 pages are used for MDBX_DUPFIXED sorted-duplicate sub-DBs.
- * There are no node headers, keys are stored contiguously. */
-MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
-page_leaf2key(const MDBX_page *mp, unsigned i, size_t keysize) {
- assert((mp->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_META)) ==
- (P_LEAF | P_LEAF2));
- assert(mp->mp_leaf2_ksize == keysize);
- (void)keysize;
- return (char *)mp + PAGEHDRSZ + (i * mp->mp_leaf2_ksize);
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
}
+#endif /* !Windows */
-/* Set the node's key into keyptr. */
-static __always_inline void get_key(const MDBX_node *node, MDBX_val *keyptr) {
- keyptr->iov_len = node_ks(node);
- keyptr->iov_base = node_key(node);
-}
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
-/* Set the node's key into keyptr, if requested. */
-static __always_inline void
-get_key_optional(const MDBX_node *node, MDBX_val *keyptr /* __may_null */) {
- if (keyptr)
- get_key(node, keyptr);
-}
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
-/*------------------------------------------------------------------------------
- * Workaround for mmaped-lookahead-cross-page-boundary bug
- * in an obsolete versions of Elbrus's libc and kernels. */
-#if defined(__e2k__) && defined(MDBX_E2K_MLHCPB_WORKAROUND) && \
- MDBX_E2K_MLHCPB_WORKAROUND
-int __hot mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2,
- size_t n) {
- if (unlikely(n > 42
- /* LY: align followed access if reasonable possible */
- && (((uintptr_t)s1) & 7) != 0 &&
- (((uintptr_t)s1) & 7) == (((uintptr_t)s2) & 7))) {
- if (((uintptr_t)s1) & 1) {
- const int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
- if (diff)
- return diff;
- s1 = (char *)s1 + 1;
- s2 = (char *)s2 + 1;
- n -= 1;
- }
+#if defined(_WIN32) || defined(_WIN64)
- if (((uintptr_t)s1) & 2) {
- const uint16_t a = *(uint16_t *)s1;
- const uint16_t b = *(uint16_t *)s2;
- if (likely(a != b))
- return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
- s1 = (char *)s1 + 2;
- s2 = (char *)s2 + 2;
- n -= 2;
- }
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
- if (((uintptr_t)s1) & 4) {
- const uint32_t a = *(uint32_t *)s1;
- const uint32_t b = *(uint32_t *)s2;
- if (likely(a != b))
- return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
- s1 = (char *)s1 + 4;
- s2 = (char *)s2 + 4;
- n -= 4;
- }
- }
+#endif /* Windows */
- while (n >= 8) {
- const uint64_t a = *(uint64_t *)s1;
- const uint64_t b = *(uint64_t *)s2;
- if (likely(a != b))
- return (__builtin_bswap64(a) > __builtin_bswap64(b)) ? 1 : -1;
- s1 = (char *)s1 + 8;
- s2 = (char *)s2 + 8;
- n -= 8;
- }
+#endif /* !__cplusplus */
- if (n & 4) {
- const uint32_t a = *(uint32_t *)s1;
- const uint32_t b = *(uint32_t *)s2;
- if (likely(a != b))
- return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
- s1 = (char *)s1 + 4;
- s2 = (char *)s2 + 4;
- }
+/*----------------------------------------------------------------------------*/
- if (n & 2) {
- const uint16_t a = *(uint16_t *)s1;
- const uint16_t b = *(uint16_t *)s2;
- if (likely(a != b))
- return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
- s1 = (char *)s1 + 2;
- s2 = (char *)s2 + 2;
- }
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
- return (n & 1) ? *(uint8_t *)s1 - *(uint8_t *)s2 : 0;
-}
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
-int __hot mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2) {
- while (true) {
- int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
- if (likely(diff != 0) || *s1 == '\0')
- return diff;
- s1 += 1;
- s2 += 1;
- }
-}
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
-int __hot mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2,
- size_t n) {
- while (n > 0) {
- int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
- if (likely(diff != 0) || *s1 == '\0')
- return diff;
- s1 += 1;
- s2 += 1;
- n -= 1;
- }
- return 0;
-}
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
-size_t __hot mdbx_e2k_strlen_bug_workaround(const char *s) {
- size_t n = 0;
- while (*s) {
- s += 1;
- n += 1;
- }
- return n;
-}
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
-size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) {
- size_t n = 0;
- while (maxlen > n && *s) {
- s += 1;
- n += 1;
- }
- return n;
-}
-#endif /* MDBX_E2K_MLHCPB_WORKAROUND */
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
-/*------------------------------------------------------------------------------
- * safe read/write volatile 64-bit fields on 32-bit architectures. */
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
-MDBX_MAYBE_UNUSED static __always_inline uint64_t
-atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value,
- enum MDBX_memory_order order) {
- STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
-#if MDBX_64BIT_ATOMIC
-#ifdef MDBX_HAVE_C11ATOMICS
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
- atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order));
-#else /* MDBX_HAVE_C11ATOMICS */
- if (order != mo_Relaxed)
- mdbx_compiler_barrier();
- p->weak = value;
- mdbx_memory_fence(order, true);
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
#endif /* MDBX_HAVE_C11ATOMICS */
-#else /* !MDBX_64BIT_ATOMIC */
- mdbx_compiler_barrier();
- atomic_store32(&p->low, (uint32_t)value, mo_Relaxed);
- mdbx_jitter4testing(true);
- atomic_store32(&p->high, (uint32_t)(value >> 32), order);
- mdbx_jitter4testing(true);
-#endif /* !MDBX_64BIT_ATOMIC */
- return value;
-}
-MDBX_MAYBE_UNUSED static
-#if MDBX_64BIT_ATOMIC
- __always_inline
-#endif /* MDBX_64BIT_ATOMIC */
- uint64_t
- atomic_load64(const MDBX_atomic_uint64_t *p,
- enum MDBX_memory_order order) {
- STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
-#if MDBX_64BIT_ATOMIC
-#ifdef MDBX_HAVE_C11ATOMICS
- assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p)));
- return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order));
-#else /* MDBX_HAVE_C11ATOMICS */
- mdbx_memory_fence(order, false);
- const uint64_t value = p->weak;
- if (order != mo_Relaxed)
- mdbx_compiler_barrier();
- return value;
-#endif /* MDBX_HAVE_C11ATOMICS */
-#else /* !MDBX_64BIT_ATOMIC */
- mdbx_compiler_barrier();
- uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
- mdbx_jitter4testing(true);
- value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
- : mo_AcquireRelease);
- mdbx_jitter4testing(true);
- for (;;) {
- mdbx_compiler_barrier();
- uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
- mdbx_jitter4testing(true);
- again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
- : mo_AcquireRelease);
- mdbx_jitter4testing(true);
- if (likely(value == again))
- return value;
- value = again;
- }
-#endif /* !MDBX_64BIT_ATOMIC */
-}
+//------------------------------------------------------------------------------
-static __always_inline void atomic_yield(void) {
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
#if defined(_WIN32) || defined(_WIN64)
- YieldProcessor();
-#elif defined(__ia32__) || defined(__e2k__)
- __builtin_ia32_pause();
-#elif defined(__ia64__)
-#if defined(__HP_cc__) || defined(__HP_aCC__)
- _Asm_hint(_HINT_PAUSE);
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
#else
- __asm__ __volatile__("hint @pause");
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
#endif
-#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \
- defined(__ARM_ARCH_6K__)
-#ifdef __CC_ARM
- __yield();
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
#else
- __asm__ __volatile__("yield");
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
#endif
-#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \
- __mips_isa_rev >= 2
- __asm__ __volatile__("pause");
-#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
- __asm__ __volatile__(".word 0x00000140");
-#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
- sched_yield();
-#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
- pthread_yield();
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
#endif
-}
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
-#if MDBX_64BIT_CAS
-static __always_inline bool atomic_cas64(MDBX_atomic_uint64_t *p, uint64_t c,
- uint64_t v) {
-#ifdef MDBX_HAVE_C11ATOMICS
- STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t));
-#ifdef ATOMIC_LLONG_LOCK_FREE
- STATIC_ASSERT(ATOMIC_LLONG_LOCK_FREE > 0);
-#if ATOMIC_LLONG_LOCK_FREE < 2
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
-#endif /* ATOMIC_LLONG_LOCK_FREE < 2 */
-#else /* defined(ATOMIC_LLONG_LOCK_FREE) */
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
-#endif
- return atomic_compare_exchange_strong(MDBX_c11a_rw(uint64_t, p), &c, v);
-#elif defined(__GNUC__) || defined(__clang__)
- return __sync_bool_compare_and_swap(&p->weak, c, v);
-#elif defined(_MSC_VER)
- return c == (uint64_t)_InterlockedCompareExchange64(
- (volatile __int64 *)&p->weak, v, c);
-#elif defined(__APPLE__)
- return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak);
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
#else
-#error FIXME: Unsupported compiler
+#define MDBX_64BIT_ATOMIC 0
#endif
-}
-#endif /* MDBX_64BIT_CAS */
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
-static __always_inline bool atomic_cas32(MDBX_atomic_uint32_t *p, uint32_t c,
- uint32_t v) {
-#ifdef MDBX_HAVE_C11ATOMICS
- STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
-#ifdef ATOMIC_INT_LOCK_FREE
- STATIC_ASSERT(ATOMIC_INT_LOCK_FREE > 0);
-#if ATOMIC_INT_LOCK_FREE < 2
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
#else
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#define MDBX_64BIT_CAS 0
#endif
- return atomic_compare_exchange_strong(MDBX_c11a_rw(uint32_t, p), &c, v);
-#elif defined(__GNUC__) || defined(__clang__)
- return __sync_bool_compare_and_swap(&p->weak, c, v);
-#elif defined(_MSC_VER)
- STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
- return c ==
- (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c);
-#elif defined(__APPLE__)
- return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak);
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
#else
-#error FIXME: Unsupported compiler
+#define MDBX_64BIT_CAS 0
#endif
-}
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
-static __always_inline uint32_t atomic_add32(MDBX_atomic_uint32_t *p,
- uint32_t v) {
-#ifdef MDBX_HAVE_C11ATOMICS
- STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
-#ifdef ATOMIC_INT_LOCK_FREE
- STATIC_ASSERT(ATOMIC_INT_LOCK_FREE > 0);
-#if ATOMIC_INT_LOCK_FREE < 2
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
#else
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
#endif
- return atomic_fetch_add(MDBX_c11a_rw(uint32_t, p), v);
-#elif defined(__GNUC__) || defined(__clang__)
- return __sync_fetch_and_add(&p->weak, v);
-#elif defined(_MSC_VER)
- STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
- return (uint32_t)_InterlockedExchangeAdd((volatile long *)&p->weak, v);
-#elif defined(__APPLE__)
- return OSAtomicAdd32Barrier(v, &p->weak);
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
#else
-#error FIXME: Unsupported compiler
+#define MDBX_CACHELINE_SIZE 64
#endif
-}
+#endif /* MDBX_CACHELINE_SIZE */
-#define atomic_sub32(p, v) atomic_add32(p, 0 - (v))
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
-static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) {
- txnid += xMDBX_TXNID_STEP;
-#if !MDBX_64BIT_CAS
- /* avoid overflow of low-part in safe64_reset() */
- txnid += (UINT32_MAX == (uint32_t)txnid);
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
#endif
- return txnid;
-}
-static __always_inline void safe64_reset(MDBX_atomic_uint64_t *p,
- bool single_writer) {
-#if !MDBX_64BIT_CAS
- if (!single_writer) {
- STATIC_ASSERT(xMDBX_TXNID_STEP > 1);
- /* it is safe to increment low-part to avoid ABA, since xMDBX_TXNID_STEP > 1
- * and overflow was preserved in safe64_txnid_next() */
- atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
- atomic_store32(
- &p->high, UINT32_MAX,
- mo_Relaxed) /* atomically make >= SAFE64_INVALID_THRESHOLD */;
- atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
- } else
-#endif /* !MDBX_64BIT_CAS */
-#if MDBX_64BIT_ATOMIC
- /* atomically make value >= SAFE64_INVALID_THRESHOLD by 64-bit operation */
- atomic_store64(p, UINT64_MAX,
- single_writer ? mo_AcquireRelease
- : mo_SequentialConsistency);
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
#else
- /* atomically make value >= SAFE64_INVALID_THRESHOLD by 32-bit operation */
- atomic_store32(&p->high, UINT32_MAX,
- single_writer ? mo_AcquireRelease : mo_SequentialConsistency);
-#endif /* MDBX_64BIT_ATOMIC */
- assert(p->weak >= SAFE64_INVALID_THRESHOLD);
- mdbx_jitter4testing(true);
-}
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
-static __always_inline bool safe64_reset_compare(MDBX_atomic_uint64_t *p,
- txnid_t compare) {
- /* LY: This function is used to reset `mr_txnid` from hsr-handler in case
- * the asynchronously cancellation of read transaction. Therefore,
- * there may be a collision between the cleanup performed here and
- * asynchronous termination and restarting of the read transaction
- * in another proces/thread. In general we MUST NOT reset the `mr_txnid`
- * if a new transaction was started (i.e. if `mr_txnid` was changed). */
-#if MDBX_64BIT_CAS
- bool rc = atomic_cas64(p, compare, UINT64_MAX);
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
#else
- /* LY: There is no gold ratio here since shared mutex is too costly,
- * in such way we must acquire/release it for every update of mr_txnid,
- * i.e. twice for each read transaction). */
- bool rc = false;
- if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare &&
- atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) {
- if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) !=
- (uint32_t)compare))
- atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32));
- else
- rc = true;
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
}
-#endif /* MDBX_64BIT_CAS */
- mdbx_jitter4testing(true);
- return rc;
}
-static __always_inline void safe64_write(MDBX_atomic_uint64_t *p,
- const uint64_t v) {
- assert(p->weak >= SAFE64_INVALID_THRESHOLD);
-#if MDBX_64BIT_ATOMIC
- atomic_store64(p, v, mo_AcquireRelease);
-#else /* MDBX_64BIT_ATOMIC */
- mdbx_compiler_barrier();
- /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */
- atomic_store32(&p->low, (uint32_t)v, mo_Relaxed);
- assert(p->weak >= SAFE64_INVALID_THRESHOLD);
- mdbx_jitter4testing(true);
- /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */
- atomic_store32(&p->high, (uint32_t)(v >> 32), mo_AcquireRelease);
-#endif /* MDBX_64BIT_ATOMIC */
- assert(p->weak == v);
- mdbx_jitter4testing(true);
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
}
+#endif /* MDBX_HAVE_C11ATOMICS */
-static __always_inline uint64_t safe64_read(const MDBX_atomic_uint64_t *p) {
- mdbx_jitter4testing(true);
- uint64_t v = atomic_load64(p, mo_AcquireRelease);
- mdbx_jitter4testing(true);
- return v;
-}
+#ifndef __cplusplus
-#if 0 /* unused for now */
-MDBX_MAYBE_UNUSED static __always_inline bool safe64_is_valid(uint64_t v) {
-#if MDBX_WORDBITS >= 64
- return v < SAFE64_INVALID_THRESHOLD;
-#else
- return (v >> 32) != UINT32_MAX;
-#endif /* MDBX_WORDBITS */
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
}
-MDBX_MAYBE_UNUSED static __always_inline bool
- safe64_is_valid_ptr(const MDBX_atomic_uint64_t *p) {
-#if MDBX_64BIT_ATOMIC
- return atomic_load64(p, mo_AcquireRelease) < SAFE64_INVALID_THRESHOLD;
-#else
- return atomic_load32(&p->high, mo_AcquireRelease) != UINT32_MAX;
-#endif /* MDBX_64BIT_ATOMIC */
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
}
-#endif /* unused for now */
-/* non-atomic write with safety for reading a half-updated value */
-static __always_inline void safe64_update(MDBX_atomic_uint64_t *p,
- const uint64_t v) {
-#if MDBX_64BIT_ATOMIC
- atomic_store64(p, v, mo_Relaxed);
-#else
- safe64_reset(p, true);
- safe64_write(p, v);
-#endif /* MDBX_64BIT_ATOMIC */
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
}
-/* non-atomic increment with safety for reading a half-updated value */
-MDBX_MAYBE_UNUSED static
-#if MDBX_64BIT_ATOMIC
- __always_inline
-#endif /* MDBX_64BIT_ATOMIC */
- void
- safe64_inc(MDBX_atomic_uint64_t *p, const uint64_t v) {
- assert(v > 0);
- safe64_update(p, atomic_load64(p, mo_Relaxed) + v);
-}
+#endif /* !__cplusplus */
/*----------------------------------------------------------------------------*/
-/* rthc (tls keys and destructors) */
+/* Basic constants and types */
-typedef struct rthc_entry_t {
- MDBX_reader *begin;
- MDBX_reader *end;
- mdbx_thread_key_t thr_tls_key;
- bool key_valid;
-} rthc_entry_t;
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
-#if MDBX_DEBUG
-#define RTHC_INITIAL_LIMIT 1
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
#else
-#define RTHC_INITIAL_LIMIT 16
+#define xMDBX_TXNID_STEP 2u
#endif
+#endif /* xMDBX_TXNID_STEP */
-static bin128_t bootid;
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
-#if defined(_WIN32) || defined(_WIN64)
-static CRITICAL_SECTION rthc_critical_section;
-static CRITICAL_SECTION lcklist_critical_section;
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
#else
-int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj, void *dso_symbol)
- __attribute__((__weak__));
-#ifdef __APPLE__ /* FIXME: Thread-Local Storage destructors & DSO-unloading */
-int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj,
- void *dso_symbol) {
- (void)dtor;
- (void)obj;
- (void)dso_symbol;
- return -1;
-}
-#endif /* __APPLE__ */
+#error "FIXME"
+#endif /* MDBX_LOCKING */
-static pthread_mutex_t lcklist_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER;
-static mdbx_thread_key_t rthc_key;
-static MDBX_atomic_uint32_t rthc_pending;
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
-__cold static void workaround_glibc_bug21031(void) {
- /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031
- *
- * Due race between pthread_key_delete() and __nptl_deallocate_tsd()
- * The destructor(s) of thread-local-storage object(s) may be running
- * in another thread(s) and be blocked or not finished yet.
- * In such case we get a SEGFAULT after unload this library DSO.
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
*
- * So just by yielding a few timeslices we give a chance
- * to such destructor(s) for completion and avoids segfault. */
- sched_yield();
- sched_yield();
- sched_yield();
-}
-#endif
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
-static unsigned rthc_count, rthc_limit;
-static rthc_entry_t *rthc_table;
-static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT];
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
-static __inline void rthc_lock(void) {
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
#if defined(_WIN32) || defined(_WIN64)
- EnterCriticalSection(&rthc_critical_section);
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
#else
- mdbx_ensure(nullptr, mdbx_pthread_mutex_lock(&rthc_mutex) == 0);
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
#endif
-}
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
-static __inline void rthc_unlock(void) {
-#if defined(_WIN32) || defined(_WIN64)
- LeaveCriticalSection(&rthc_critical_section);
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
#else
- mdbx_ensure(nullptr, pthread_mutex_unlock(&rthc_mutex) == 0);
-#endif
-}
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
-static __inline int thread_key_create(mdbx_thread_key_t *key) {
- int rc;
-#if defined(_WIN32) || defined(_WIN64)
- *key = TlsAlloc();
- rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError();
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
#else
- rc = pthread_key_create(key, nullptr);
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
#endif
- mdbx_trace("&key = %p, value %" PRIuPTR ", rc %d",
- __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc);
- return rc;
-}
-static __inline void thread_key_delete(mdbx_thread_key_t key) {
- mdbx_trace("key = %" PRIuPTR, (uintptr_t)key);
-#if defined(_WIN32) || defined(_WIN64)
- mdbx_ensure(nullptr, TlsFree(key));
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
#else
- mdbx_ensure(nullptr, pthread_key_delete(key) == 0);
- workaround_glibc_bug21031();
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
#endif
-}
-static __inline void *thread_rthc_get(mdbx_thread_key_t key) {
-#if defined(_WIN32) || defined(_WIN64)
- return TlsGetValue(key);
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
#else
- return pthread_getspecific(key);
+#define CURSOR_STACK 24
#endif
-}
-static void thread_rthc_set(mdbx_thread_key_t key, const void *value) {
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
#if defined(_WIN32) || defined(_WIN64)
- mdbx_ensure(nullptr, TlsSetValue(key, (void *)value));
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
#else
-#define MDBX_THREAD_RTHC_ZERO 0
-#define MDBX_THREAD_RTHC_REGISTERED 1
-#define MDBX_THREAD_RTHC_COUNTED 2
- static __thread char thread_registration_state;
- if (value && unlikely(thread_registration_state == MDBX_THREAD_RTHC_ZERO)) {
- thread_registration_state = MDBX_THREAD_RTHC_REGISTERED;
- mdbx_trace("thread registered 0x%" PRIxPTR, mdbx_thread_self());
- if (&__cxa_thread_atexit_impl == nullptr ||
- __cxa_thread_atexit_impl(mdbx_rthc_thread_dtor,
- &thread_registration_state,
- (void *)&mdbx_version /* dso_anchor */)) {
- mdbx_ensure(nullptr, pthread_setspecific(
- rthc_key, &thread_registration_state) == 0);
- thread_registration_state = MDBX_THREAD_RTHC_COUNTED;
- const unsigned count_before = atomic_add32(&rthc_pending, 1);
- mdbx_ensure(nullptr, count_before < INT_MAX);
- mdbx_trace("fallback to pthreads' tsd, key %" PRIuPTR ", count %u",
- (uintptr_t)rthc_key, count_before);
- (void)count_before;
- }
- }
- mdbx_ensure(nullptr, pthread_setspecific(key, value) == 0);
+ mdbx_fastmutex_t me_remap_guard;
#endif
-}
-__cold void mdbx_rthc_global_init(void) {
- rthc_limit = RTHC_INITIAL_LIMIT;
- rthc_table = rthc_table_static;
-#if defined(_WIN32) || defined(_WIN64)
- InitializeCriticalSection(&rthc_critical_section);
- InitializeCriticalSection(&lcklist_critical_section);
-#else
- mdbx_ensure(nullptr,
- pthread_key_create(&rthc_key, mdbx_rthc_thread_dtor) == 0);
- mdbx_trace("pid %d, &mdbx_rthc_key = %p, value 0x%x", mdbx_getpid(),
- __Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key);
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
#endif
- /* checking time conversion, this also avoids racing on 32-bit architectures
- * during writing calculated 64-bit ratio(s) into memory. */
- uint32_t proba = UINT32_MAX;
- while (true) {
- unsigned time_conversion_checkup =
- mdbx_osal_monotime_to_16dot16(mdbx_osal_16dot16_to_monotime(proba));
- unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba;
- unsigned one_less = (proba > 0) ? proba - 1 : proba;
- mdbx_ensure(nullptr, time_conversion_checkup >= one_less &&
- time_conversion_checkup <= one_more);
- if (proba == 0)
- break;
- proba >>= 1;
- }
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
- bootid = mdbx_osal_bootid();
-#if 0 /* debug */
- for (unsigned i = 0; i < 65536; ++i) {
- size_t pages = pv2pages(i);
- unsigned x = pages2pv(pages);
- size_t xp = pv2pages(x);
- if (!(x == i || (x % 2 == 0 && x < 65536)) || pages != xp)
- printf("%u => %zu => %u => %zu\n", i, pages, x, xp);
- assert(pages == xp);
- }
- fflush(stdout);
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
#endif
}
-/* dtor called for thread, i.e. for all mdbx's environment objects */
-__cold void mdbx_rthc_thread_dtor(void *ptr) {
- rthc_lock();
- mdbx_trace(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", mdbx_getpid(),
- mdbx_thread_self(), ptr);
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
- const uint32_t self_pid = mdbx_getpid();
- for (unsigned i = 0; i < rthc_count; ++i) {
- if (!rthc_table[i].key_valid)
- continue;
- const mdbx_thread_key_t key = rthc_table[i].thr_tls_key;
- MDBX_reader *const rthc = thread_rthc_get(key);
- if (rthc < rthc_table[i].begin || rthc >= rthc_table[i].end)
- continue;
-#if !defined(_WIN32) && !defined(_WIN64)
- if (pthread_setspecific(key, nullptr) != 0) {
- mdbx_trace("== thread 0x%" PRIxPTR
- ", rthc %p: ignore race with tsd-key deletion",
- mdbx_thread_self(), ptr);
- continue /* ignore race with tsd-key deletion by mdbx_env_close() */;
- }
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
#endif
- mdbx_trace("== thread 0x%" PRIxPTR
- ", rthc %p, [%i], %p ... %p (%+i), rtch-pid %i, "
- "current-pid %i",
- mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), i,
- __Wpedantic_format_voidptr(rthc_table[i].begin),
- __Wpedantic_format_voidptr(rthc_table[i].end),
- (int)(rthc - rthc_table[i].begin), rthc->mr_pid.weak, self_pid);
- if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
- mdbx_trace("==== thread 0x%" PRIxPTR ", rthc %p, cleanup",
- mdbx_thread_self(), __Wpedantic_format_voidptr(rthc));
- atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
- }
- }
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
-#if defined(_WIN32) || defined(_WIN64)
- mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", mdbx_thread_self(), ptr);
- rthc_unlock();
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
#else
- const char self_registration = *(volatile char *)ptr;
- *(volatile char *)ptr = MDBX_THREAD_RTHC_ZERO;
- mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d",
- mdbx_thread_self(), ptr, mdbx_getpid(), self_registration);
- if (self_registration == MDBX_THREAD_RTHC_COUNTED)
- mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
- if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) {
- mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake",
- mdbx_thread_self(), ptr, mdbx_getpid());
- mdbx_ensure(nullptr, pthread_cond_broadcast(&rthc_cond) == 0);
- }
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
- mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", mdbx_thread_self(), ptr);
- /* Allow tail call optimization, i.e. gcc should generate the jmp instruction
- * instead of a call for pthread_mutex_unlock() and therefore CPU could not
- * return to current DSO's code section, which may be unloaded immediately
- * after the mutex got released. */
- pthread_mutex_unlock(&rthc_mutex);
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
#endif
}
-__cold void mdbx_rthc_global_dtor(void) {
- mdbx_trace(">> pid %d", mdbx_getpid());
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
- rthc_lock();
-#if !defined(_WIN32) && !defined(_WIN64)
- char *rthc = pthread_getspecific(rthc_key);
- mdbx_trace(
- "== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d, left %d",
- mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), mdbx_getpid(),
- rthc ? *rthc : -1, atomic_load32(&rthc_pending, mo_Relaxed));
- if (rthc) {
- const char self_registration = *rthc;
- *rthc = MDBX_THREAD_RTHC_ZERO;
- if (self_registration == MDBX_THREAD_RTHC_COUNTED)
- mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
- }
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
- struct timespec abstime;
- mdbx_ensure(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0);
- abstime.tv_nsec += 1000000000l / 10;
- if (abstime.tv_nsec >= 1000000000l) {
- abstime.tv_nsec -= 1000000000l;
- abstime.tv_sec += 1;
- }
-#if MDBX_DEBUG > 0
- abstime.tv_sec += 600;
-#endif
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
- for (unsigned left;
- (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) {
- mdbx_trace("pid %d, pending %u, wait for...", mdbx_getpid(), left);
- const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime);
- if (rc && rc != EINTR)
- break;
- }
- thread_key_delete(rthc_key);
-#endif
+#endif /* !__cplusplus */
- const uint32_t self_pid = mdbx_getpid();
- for (unsigned i = 0; i < rthc_count; ++i) {
- if (!rthc_table[i].key_valid)
- continue;
- const mdbx_thread_key_t key = rthc_table[i].thr_tls_key;
- thread_key_delete(key);
- for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
- ++rthc) {
- mdbx_trace(
- "== [%i] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), "
- "rthc-pid %i, current-pid %i",
- i, (uintptr_t)key, __Wpedantic_format_voidptr(rthc_table[i].begin),
- __Wpedantic_format_voidptr(rthc_table[i].end),
- __Wpedantic_format_voidptr(rthc), (int)(rthc - rthc_table[i].begin),
- rthc->mr_pid.weak, self_pid);
- if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
- atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
- mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc));
- }
- }
- }
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
- rthc_limit = rthc_count = 0;
- if (rthc_table != rthc_table_static)
- mdbx_free(rthc_table);
- rthc_table = nullptr;
- rthc_unlock();
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
-#if defined(_WIN32) || defined(_WIN64)
- DeleteCriticalSection(&lcklist_critical_section);
- DeleteCriticalSection(&rthc_critical_section);
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
#else
- /* LY: yielding a few timeslices to give a more chance
- * to racing destructor(s) for completion. */
- workaround_glibc_bug21031();
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
#endif
- mdbx_trace("<< pid %d\n", mdbx_getpid());
-}
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
-__cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
- MDBX_reader *end) {
- int rc;
- if (key) {
-#ifndef NDEBUG
- *key = (mdbx_thread_key_t)0xBADBADBAD;
-#endif /* NDEBUG */
- rc = thread_key_create(key);
- if (rc != MDBX_SUCCESS)
- return rc;
- }
-
- rthc_lock();
- const mdbx_thread_key_t new_key = key ? *key : 0;
- mdbx_trace(">> key %" PRIuPTR ", rthc_count %u, rthc_limit %u",
- (uintptr_t)new_key, rthc_count, rthc_limit);
- if (rthc_count == rthc_limit) {
- rthc_entry_t *new_table =
- mdbx_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table,
- sizeof(rthc_entry_t) * rthc_limit * 2);
- if (new_table == nullptr) {
- rc = MDBX_ENOMEM;
- goto bailout;
- }
- if (rthc_table == rthc_table_static)
- memcpy(new_table, rthc_table_static, sizeof(rthc_table_static));
- rthc_table = new_table;
- rthc_limit *= 2;
- }
- mdbx_trace("== [%i] = key %" PRIuPTR ", %p ... %p", rthc_count,
- (uintptr_t)new_key, __Wpedantic_format_voidptr(begin),
- __Wpedantic_format_voidptr(end));
- rthc_table[rthc_count].key_valid = key ? true : false;
- rthc_table[rthc_count].thr_tls_key = key ? new_key : 0;
- rthc_table[rthc_count].begin = begin;
- rthc_table[rthc_count].end = end;
- ++rthc_count;
- mdbx_trace("<< key %" PRIuPTR ", rthc_count %u, rthc_limit %u",
- (uintptr_t)new_key, rthc_count, rthc_limit);
- rthc_unlock();
- return MDBX_SUCCESS;
-
-bailout:
- if (key)
- thread_key_delete(*key);
- rthc_unlock();
- return rc;
-}
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
-__cold void mdbx_rthc_remove(const mdbx_thread_key_t key) {
- thread_key_delete(key);
- rthc_lock();
- mdbx_trace(">> key %zu, rthc_count %u, rthc_limit %u", (size_t)key,
- rthc_count, rthc_limit);
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
- for (unsigned i = 0; i < rthc_count; ++i) {
- if (rthc_table[i].key_valid && key == rthc_table[i].thr_tls_key) {
- const uint32_t self_pid = mdbx_getpid();
- mdbx_trace("== [%i], %p ...%p, current-pid %d", i,
- __Wpedantic_format_voidptr(rthc_table[i].begin),
- __Wpedantic_format_voidptr(rthc_table[i].end), self_pid);
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
- for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
- ++rthc) {
- if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
- atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
- mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc));
- }
- }
- if (--rthc_count > 0)
- rthc_table[i] = rthc_table[rthc_count];
- else if (rthc_table != rthc_table_static) {
- mdbx_free(rthc_table);
- rthc_table = rthc_table_static;
- rthc_limit = RTHC_INITIAL_LIMIT;
- }
- break;
- }
- }
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
- mdbx_trace("<< key %zu, rthc_count %u, rthc_limit %u", (size_t)key,
- rthc_count, rthc_limit);
- rthc_unlock();
-}
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
-//------------------------------------------------------------------------------
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
-#define RTHC_ENVLIST_END ((MDBX_env *)((uintptr_t)50459))
-static MDBX_env *inprocess_lcklist_head = RTHC_ENVLIST_END;
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
-static __inline void lcklist_lock(void) {
-#if defined(_WIN32) || defined(_WIN64)
- EnterCriticalSection(&lcklist_critical_section);
-#else
- mdbx_ensure(nullptr, mdbx_pthread_mutex_lock(&lcklist_mutex) == 0);
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
#endif
-}
-static __inline void lcklist_unlock(void) {
-#if defined(_WIN32) || defined(_WIN64)
- LeaveCriticalSection(&lcklist_critical_section);
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
#else
- mdbx_ensure(nullptr, pthread_mutex_unlock(&lcklist_mutex) == 0);
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
#endif
-}
-MDBX_NOTHROW_CONST_FUNCTION static uint64_t rrxmrrxmsx_0(uint64_t v) {
- /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */
- v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50);
- v *= UINT64_C(0xA24BAED4963EE407);
- v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49);
- v *= UINT64_C(0x9FB21C651E98DF25);
- return v ^ v >> 28;
-}
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
-static int uniq_peek(const mdbx_mmap_t *pending, mdbx_mmap_t *scan) {
- int rc;
- uint64_t bait;
- MDBX_lockinfo *const pending_lck = pending->lck;
- MDBX_lockinfo *const scan_lck = scan->lck;
- if (pending_lck) {
- bait = atomic_load64(&pending_lck->mti_bait_uniqueness, mo_AcquireRelease);
- rc = MDBX_SUCCESS;
- } else {
- bait = 0 /* hush MSVC warning */;
- rc = mdbx_msync(scan, 0, sizeof(MDBX_lockinfo), MDBX_SYNC_DATA);
- if (rc == MDBX_SUCCESS)
- rc = mdbx_pread(pending->fd, &bait, sizeof(scan_lck->mti_bait_uniqueness),
- offsetof(MDBX_lockinfo, mti_bait_uniqueness));
- }
- if (likely(rc == MDBX_SUCCESS) &&
- bait == atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease))
- rc = MDBX_RESULT_TRUE;
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
- mdbx_trace("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d",
- pending_lck ? "mem" : "file", bait,
- (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc);
- return rc;
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
}
-static int uniq_poke(const mdbx_mmap_t *pending, mdbx_mmap_t *scan,
- uint64_t *abra) {
- if (*abra == 0) {
- const uintptr_t tid = mdbx_thread_self();
- uintptr_t uit = 0;
- memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit));
- *abra =
- rrxmrrxmsx_0(mdbx_osal_monotime() + UINT64_C(5873865991930747) * uit);
- }
- const uint64_t cadabra =
- rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)mdbx_getpid())
- << 24 |
- *abra >> 40;
- MDBX_lockinfo *const scan_lck = scan->lck;
- atomic_store64(&scan_lck->mti_bait_uniqueness, cadabra,
- mo_SequentialConsistency);
- *abra = *abra * UINT64_C(6364136223846793005) + 1;
- return uniq_peek(pending, scan);
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
}
-__cold static int uniq_check(const mdbx_mmap_t *pending, MDBX_env **found) {
- *found = nullptr;
- uint64_t salt = 0;
- for (MDBX_env *scan = inprocess_lcklist_head; scan != RTHC_ENVLIST_END;
- scan = scan->me_lcklist_next) {
- MDBX_lockinfo *const scan_lck = scan->me_lck_mmap.lck;
- int err = atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease)
- ? uniq_peek(pending, &scan->me_lck_mmap)
- : uniq_poke(pending, &scan->me_lck_mmap, &salt);
- if (err == MDBX_ENODATA) {
- uint64_t length;
- if (likely(mdbx_filesize(pending->fd, &length) == MDBX_SUCCESS &&
- length == 0)) {
- /* LY: skip checking since LCK-file is empty, i.e. just created. */
- mdbx_debug("uniq-probe: %s", "unique (new/empty lck)");
- return MDBX_RESULT_TRUE;
- }
- }
- if (err == MDBX_RESULT_TRUE)
- err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
- if (err == MDBX_RESULT_TRUE) {
- (void)mdbx_msync(&scan->me_lck_mmap, 0, sizeof(MDBX_lockinfo),
- MDBX_SYNC_NONE);
- err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
- }
- if (err == MDBX_RESULT_TRUE) {
- err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
- *found = scan;
- mdbx_debug("uniq-probe: found %p", __Wpedantic_format_voidptr(*found));
- return MDBX_RESULT_FALSE;
- }
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_debug("uniq-probe: failed rc %d", err);
- return err;
- }
- }
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
- mdbx_debug("uniq-probe: %s", "unique");
- return MDBX_RESULT_TRUE;
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
}
-static int lcklist_detach_locked(MDBX_env *env) {
- MDBX_env *inprocess_neighbor = nullptr;
- int rc = MDBX_SUCCESS;
- if (env->me_lcklist_next != nullptr) {
- mdbx_ensure(env, env->me_lcklist_next != nullptr);
- mdbx_ensure(env, inprocess_lcklist_head != RTHC_ENVLIST_END);
- for (MDBX_env **ptr = &inprocess_lcklist_head; *ptr != RTHC_ENVLIST_END;
- ptr = &(*ptr)->me_lcklist_next) {
- if (*ptr == env) {
- *ptr = env->me_lcklist_next;
- env->me_lcklist_next = nullptr;
- break;
- }
- }
- mdbx_ensure(env, env->me_lcklist_next == nullptr);
- }
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
- rc = likely(mdbx_getpid() == env->me_pid)
- ? uniq_check(&env->me_lck_mmap, &inprocess_neighbor)
- : MDBX_PANIC;
- if (!inprocess_neighbor && env->me_live_reader)
- (void)mdbx_rpid_clear(env);
- if (!MDBX_IS_ERROR(rc))
- rc = mdbx_lck_destroy(env, inprocess_neighbor);
- return rc;
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
}
-/*------------------------------------------------------------------------------
- * LY: State of the art quicksort-based sorting, with internal stack
- * and network-sort for small chunks.
- * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
-#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
do { \
- const TYPE swap_tmp = (a); \
- const bool swap_cmp = CMP(swap_tmp, b); \
- (a) = swap_cmp ? swap_tmp : b; \
- (b) = swap_cmp ? b : swap_tmp; \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
} while (0)
-// 3 comparators, 3 parallel operations
-// o-----^--^--o
-// | |
-// o--^--|--v--o
-// | |
-// o--v--v-----o
-//
-// [[1,2]]
-// [[0,2]]
-// [[0,1]]
-#define SORT_NETWORK_3(TYPE, CMP, begin) \
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
do { \
- SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
- SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
- SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
} while (0)
-
-// 5 comparators, 3 parallel operations
-// o--^--^--------o
-// | |
-// o--v--|--^--^--o
-// | | |
-// o--^--v--|--v--o
-// | |
-// o--v-----v-----o
-//
-// [[0,1],[2,3]]
-// [[0,2],[1,3]]
-// [[1,2]]
-#define SORT_NETWORK_4(TYPE, CMP, begin) \
- do { \
- SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * This code is derived from "LMDB engine" written by
+ * Howard Chu (Symas Corporation), which itself derived from btree.c
+ * written by Martin Hedenfalk.
+ *
+ * ---
+ *
+ * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ *
+ * ---
+ *
+ * Portions Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+
+/*------------------------------------------------------------------------------
+ * Internal inline functions */
+
+MDBX_NOTHROW_CONST_FUNCTION static unsigned branchless_abs(int value) {
+ assert(value > INT_MIN);
+ const unsigned expanded_sign =
+ (unsigned)(value >> (sizeof(value) * CHAR_BIT - 1));
+ return ((unsigned)value + expanded_sign) ^ expanded_sign;
+}
+
+/* Pack/Unpack 16-bit values for Grow step & Shrink threshold */
+MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t me2v(unsigned m,
+ unsigned e) {
+ assert(m < 2048 && e < 8);
+ return (pgno_t)(32768 + ((m + 1) << (e + 8)));
+}
+
+MDBX_NOTHROW_CONST_FUNCTION static __inline uint16_t v2me(size_t v,
+ unsigned e) {
+ assert(v > (e ? me2v(2047, e - 1) : 32768));
+ assert(v <= me2v(2047, e));
+ size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8);
+ m -= m > 0;
+ assert(m < 2048 && e < 8);
+ // f e d c b a 9 8 7 6 5 4 3 2 1 0
+ // 1 e e e m m m m m m m m m m m 1
+ const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1));
+ assert(pv != 65535);
+ return pv;
+}
+
+/* Convert 16-bit packed (exponential quantized) value to number of pages */
+MDBX_NOTHROW_CONST_FUNCTION static pgno_t pv2pages(uint16_t pv) {
+ if ((pv & 0x8001) != 0x8001)
+ return pv;
+ if (pv == 65535)
+ return 65536;
+ // f e d c b a 9 8 7 6 5 4 3 2 1 0
+ // 1 e e e m m m m m m m m m m m 1
+ return me2v((pv >> 1) & 2047, (pv >> 12) & 7);
+}
+
+/* Convert number of pages to 16-bit packed (exponential quantized) value */
+MDBX_NOTHROW_CONST_FUNCTION static uint16_t pages2pv(size_t pages) {
+ if (pages < 32769 || (pages < 65536 && (pages & 1) == 0))
+ return (uint16_t)pages;
+ if (pages <= me2v(2047, 0))
+ return v2me(pages, 0);
+ if (pages <= me2v(2047, 1))
+ return v2me(pages, 1);
+ if (pages <= me2v(2047, 2))
+ return v2me(pages, 2);
+ if (pages <= me2v(2047, 3))
+ return v2me(pages, 3);
+ if (pages <= me2v(2047, 4))
+ return v2me(pages, 4);
+ if (pages <= me2v(2047, 5))
+ return v2me(pages, 5);
+ if (pages <= me2v(2047, 6))
+ return v2me(pages, 6);
+ return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533;
+}
+
+/*------------------------------------------------------------------------------
+ * Unaligned access */
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline unsigned
+field_alignment(unsigned alignment_baseline, size_t field_offset) {
+ unsigned merge = alignment_baseline | (unsigned)field_offset;
+ return merge & -(int)merge;
+}
+
+/* read-thunk for UB-sanitizer */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t
+peek_u8(const uint8_t *const __restrict ptr) {
+ return *ptr;
+}
+
+/* write-thunk for UB-sanitizer */
+static __always_inline void poke_u8(uint8_t *const __restrict ptr,
+ const uint8_t v) {
+ *ptr = v;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint16_t
+unaligned_peek_u16(const unsigned expected_alignment, const void *const ptr) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0)
+ return *(const uint16_t *)ptr;
+ else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ return *(const __unaligned uint16_t *)ptr;
+#else
+ uint16_t v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+static __always_inline void
+unaligned_poke_u16(const unsigned expected_alignment,
+ void *const __restrict ptr, const uint16_t v) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0)
+ *(uint16_t *)ptr = v;
+ else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ *((uint16_t __unaligned *)ptr) = v;
+#else
+ memcpy(ptr, &v, sizeof(v));
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t unaligned_peek_u32(
+ const unsigned expected_alignment, const void *const __restrict ptr) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0)
+ return *(const uint32_t *)ptr;
+ else if ((expected_alignment % sizeof(uint16_t)) == 0) {
+ const uint16_t lo =
+ ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
+ const uint16_t hi =
+ ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
+ return lo | (uint32_t)hi << 16;
+ } else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ return *(const __unaligned uint32_t *)ptr;
+#else
+ uint32_t v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+static __always_inline void
+unaligned_poke_u32(const unsigned expected_alignment,
+ void *const __restrict ptr, const uint32_t v) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0)
+ *(uint32_t *)ptr = v;
+ else if ((expected_alignment % sizeof(uint16_t)) == 0) {
+ ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v;
+ ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
+ (uint16_t)(v >> 16);
+ } else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ *((uint32_t __unaligned *)ptr) = v;
+#else
+ memcpy(ptr, &v, sizeof(v));
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t unaligned_peek_u64(
+ const unsigned expected_alignment, const void *const __restrict ptr) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
+ return *(const uint64_t *)ptr;
+ else if ((expected_alignment % sizeof(uint32_t)) == 0) {
+ const uint32_t lo =
+ ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
+ const uint32_t hi =
+ ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
+ return lo | (uint64_t)hi << 32;
+ } else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ return *(const __unaligned uint64_t *)ptr;
+#else
+ uint64_t v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+static __always_inline uint64_t
+unaligned_peek_u64_volatile(const unsigned expected_alignment,
+ volatile const void *const __restrict ptr) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ assert(expected_alignment % sizeof(uint32_t) == 0);
+ if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
+ return *(volatile const uint64_t *)ptr;
+ else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ return *(volatile const __unaligned uint64_t *)ptr;
+#else
+ const uint32_t lo = ((volatile const uint32_t *)
+ ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
+ const uint32_t hi = ((volatile const uint32_t *)
+ ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
+ return lo | (uint64_t)hi << 32;
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+static __always_inline void
+unaligned_poke_u64(const unsigned expected_alignment,
+ void *const __restrict ptr, const uint64_t v) {
+ assert((uintptr_t)ptr % expected_alignment == 0);
+ if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0)
+ *(uint64_t *)ptr = v;
+ else if ((expected_alignment % sizeof(uint32_t)) == 0) {
+ ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v;
+ ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
+ (uint32_t)(v >> 32);
+ } else {
+#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
+ defined(_M_X64) || defined(_M_IA64)
+ *((uint64_t __unaligned *)ptr) = v;
+#else
+ memcpy(ptr, &v, sizeof(v));
+#endif /* _MSC_VER || __unaligned */
+ }
+}
+
+#define UNALIGNED_PEEK_8(ptr, struct, field) \
+ peek_u8((const uint8_t *)(ptr) + offsetof(struct, field))
+#define UNALIGNED_POKE_8(ptr, struct, field, value) \
+ poke_u8((uint8_t *)(ptr) + offsetof(struct, field), value)
+
+#define UNALIGNED_PEEK_16(ptr, struct, field) \
+ unaligned_peek_u16(1, (const char *)(ptr) + offsetof(struct, field))
+#define UNALIGNED_POKE_16(ptr, struct, field, value) \
+ unaligned_poke_u16(1, (char *)(ptr) + offsetof(struct, field), value)
+
+#define UNALIGNED_PEEK_32(ptr, struct, field) \
+ unaligned_peek_u32(1, (const char *)(ptr) + offsetof(struct, field))
+#define UNALIGNED_POKE_32(ptr, struct, field, value) \
+ unaligned_poke_u32(1, (char *)(ptr) + offsetof(struct, field), value)
+
+#define UNALIGNED_PEEK_64(ptr, struct, field) \
+ unaligned_peek_u64(1, (const char *)(ptr) + offsetof(struct, field))
+#define UNALIGNED_POKE_64(ptr, struct, field, value) \
+ unaligned_poke_u64(1, (char *)(ptr) + offsetof(struct, field), value)
+
+/* Get the page number pointed to by a branch node */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
+node_pgno(const MDBX_node *const __restrict node) {
+ pgno_t pgno = UNALIGNED_PEEK_32(node, MDBX_node, mn_pgno32);
+ if (sizeof(pgno) > 4)
+ pgno |= ((uint64_t)UNALIGNED_PEEK_8(node, MDBX_node, mn_extra)) << 32;
+ return pgno;
+}
+
+/* Set the page number in a branch node */
+static __always_inline void node_set_pgno(MDBX_node *const __restrict node,
+ pgno_t pgno) {
+ assert(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO);
+
+ UNALIGNED_POKE_32(node, MDBX_node, mn_pgno32, (uint32_t)pgno);
+ if (sizeof(pgno) > 4)
+ UNALIGNED_POKE_8(node, MDBX_node, mn_extra,
+ (uint8_t)((uint64_t)pgno >> 32));
+}
+
+/* Get the size of the data in a leaf node */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+node_ds(const MDBX_node *const __restrict node) {
+ return UNALIGNED_PEEK_32(node, MDBX_node, mn_dsize);
+}
+
+/* Set the size of the data for a leaf node */
+static __always_inline void node_set_ds(MDBX_node *const __restrict node,
+ size_t size) {
+ assert(size < INT_MAX);
+ UNALIGNED_POKE_32(node, MDBX_node, mn_dsize, (uint32_t)size);
+}
+
+/* The size of a key in a node */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+node_ks(const MDBX_node *const __restrict node) {
+ return UNALIGNED_PEEK_16(node, MDBX_node, mn_ksize);
+}
+
+/* Set the size of the key for a leaf node */
+static __always_inline void node_set_ks(MDBX_node *const __restrict node,
+ size_t size) {
+ assert(size < INT16_MAX);
+ UNALIGNED_POKE_16(node, MDBX_node, mn_ksize, (uint16_t)size);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t
+node_flags(const MDBX_node *const __restrict node) {
+ return UNALIGNED_PEEK_8(node, MDBX_node, mn_flags);
+}
+
+static __always_inline void node_set_flags(MDBX_node *const __restrict node,
+ uint8_t flags) {
+ UNALIGNED_POKE_8(node, MDBX_node, mn_flags, flags);
+}
+
+/* Size of the node header, excluding dynamic data at the end */
+#define NODESIZE offsetof(MDBX_node, mn_data)
+
+/* Address of the key for the node */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
+node_key(const MDBX_node *const __restrict node) {
+ return (char *)node + NODESIZE;
+}
+
+/* Address of the data for a node */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
+node_data(const MDBX_node *const __restrict node) {
+ return (char *)node_key(node) + node_ks(node);
+}
+
+/* Size of a node in a leaf page with a given key and data.
+ * This is node header plus key plus data size. */
+MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+node_size_len(const size_t key_len, const size_t value_len) {
+ return NODESIZE + EVEN(key_len + value_len);
+}
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+node_size(const MDBX_val *key, const MDBX_val *value) {
+ return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
+peek_pgno(const void *const __restrict ptr) {
+ if (sizeof(pgno_t) == sizeof(uint32_t))
+ return (pgno_t)unaligned_peek_u32(1, ptr);
+ else if (sizeof(pgno_t) == sizeof(uint64_t))
+ return (pgno_t)unaligned_peek_u64(1, ptr);
+ else {
+ pgno_t pgno;
+ memcpy(&pgno, ptr, sizeof(pgno));
+ return pgno;
+ }
+}
+
+static __always_inline void poke_pgno(void *const __restrict ptr,
+ const pgno_t pgno) {
+ if (sizeof(pgno) == sizeof(uint32_t))
+ unaligned_poke_u32(1, ptr, pgno);
+ else if (sizeof(pgno) == sizeof(uint64_t))
+ unaligned_poke_u64(1, ptr, pgno);
+ else
+ memcpy(ptr, &pgno, sizeof(pgno));
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
+node_largedata_pgno(const MDBX_node *const __restrict node) {
+ assert(node_flags(node) & F_BIGDATA);
+ return peek_pgno(node_data(node));
+}
+
+/*------------------------------------------------------------------------------
+ * Nodes, Keys & Values length limitation factors:
+ *
+ * BRANCH_NODE_MAX
+ * Branch-page must contain at least two nodes, within each a key and a child
+ * page number. But page can't be splitted if it contains less that 4 keys,
+ * i.e. a page should not overflow before adding the fourth key. Therefore,
+ * at least 3 branch-node should fit in the single branch-page. Further, the
+ * first node of a branch-page doesn't contain a key, i.e. the first node
+ * is always require space just for itself. Thus:
+ * PAGEROOM = pagesize - page_hdr_len;
+ * BRANCH_NODE_MAX = even_floor(
+ * (PAGEROOM - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t));
+ * KEYLEN_MAX = BRANCH_NODE_MAX - node_hdr_len;
+ *
+ * LEAF_NODE_MAX
+ * Leaf-node must fit into single leaf-page, where a value could be placed on
+ * a large/overflow page. However, may require to insert a nearly page-sized
+ * node between two large nodes are already fill-up a page. In this case the
+ * page must be splitted to two if some pair of nodes fits on one page, or
+ * otherwise the page should be splitted to the THREE with a single node
+ * per each of ones. Such 1-into-3 page splitting is costly and complex since
+ * requires TWO insertion into the parent page, that could lead to split it
+ * and so on up to the root. Therefore double-splitting is avoided here and
+ * the maximum node size is half of a leaf page space:
+ * LEAF_NODE_MAX = even_floor(PAGEROOM / 2 - sizeof(indx_t));
+ * DATALEN_NO_OVERFLOW = LEAF_NODE_MAX - KEYLEN_MAX;
+ *
+ * - SubDatabase-node must fit into one leaf-page:
+ * SUBDB_NAME_MAX = LEAF_NODE_MAX - node_hdr_len - sizeof(MDBX_db);
+ *
+ * - Dupsort values itself are a keys in a dupsort-subdb and couldn't be longer
+ * than the KEYLEN_MAX. But dupsort node must not great than LEAF_NODE_MAX,
+ * since dupsort value couldn't be placed on a large/overflow page:
+ * DUPSORT_DATALEN_MAX = min(KEYLEN_MAX,
+ * max(DATALEN_NO_OVERFLOW, sizeof(MDBX_db));
+ */
+
+#define PAGEROOM(pagesize) ((pagesize)-PAGEHDRSZ)
+#define EVEN_FLOOR(n) ((n) & ~(size_t)1)
+#define BRANCH_NODE_MAX(pagesize) \
+ (EVEN_FLOOR((PAGEROOM(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - \
+ sizeof(indx_t)))
+#define LEAF_NODE_MAX(pagesize) \
+ (EVEN_FLOOR(PAGEROOM(pagesize) / 2) - sizeof(indx_t))
+#define MAX_GC1OVPAGE(pagesize) (PAGEROOM(pagesize) / sizeof(pgno_t) - 1)
+
+static __inline unsigned keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
+ assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE &&
+ is_powerof2(pagesize));
+ STATIC_ASSERT(BRANCH_NODE_MAX(MIN_PAGESIZE) - NODESIZE >= 8);
+ if (flags & MDBX_INTEGERKEY)
+ return 8 /* sizeof(uint64_t) */;
+
+ const intptr_t max_branch_key = BRANCH_NODE_MAX(pagesize) - NODESIZE;
+ STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) - NODESIZE -
+ /* sizeof(uint64) as a key */ 8 >
+ sizeof(MDBX_db));
+ if (flags &
+ (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
+ const intptr_t max_dupsort_leaf_key =
+ LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(MDBX_db);
+ return (max_branch_key < max_dupsort_leaf_key)
+ ? (unsigned)max_branch_key
+ : (unsigned)max_dupsort_leaf_key;
+ }
+ return (unsigned)max_branch_key;
+}
+
+static __inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
+ assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE &&
+ is_powerof2(pagesize));
+
+ if (flags & MDBX_INTEGERDUP)
+ return 8 /* sizeof(uint64_t) */;
+
+ if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP))
+ return keysize_max(pagesize, 0);
+
+ const unsigned page_ln2 = log2n_powerof2(pagesize);
+ const size_t hard = 0x7FF00000ul;
+ const size_t hard_pages = hard >> page_ln2;
+ STATIC_ASSERT(MDBX_PGL_LIMIT <= MAX_PAGENO);
+ const size_t pages_limit = MDBX_PGL_LIMIT / 4;
+ const size_t limit =
+ (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2);
+ return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
+}
+
+__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) {
+ return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT);
+}
+
+__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env,
+ MDBX_db_flags_t flags) {
+ if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE))
+ return -1;
+
+ return (int)mdbx_limits_keysize_max((intptr_t)env->me_psize, flags);
+}
+
+size_t mdbx_default_pagesize(void) {
+ size_t pagesize = mdbx_syspagesize();
+ mdbx_ensure(nullptr, is_powerof2(pagesize));
+ pagesize = (pagesize >= MIN_PAGESIZE) ? pagesize : MIN_PAGESIZE;
+ pagesize = (pagesize <= MAX_PAGESIZE) ? pagesize : MAX_PAGESIZE;
+ return pagesize;
+}
+
+__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize,
+ MDBX_db_flags_t flags) {
+ if (pagesize < 1)
+ pagesize = (intptr_t)mdbx_default_pagesize();
+ if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
+ pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2((size_t)pagesize)))
+ return -1;
+
+ return keysize_max(pagesize, flags);
+}
+
+__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env,
+ MDBX_db_flags_t flags) {
+ if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE))
+ return -1;
+
+ return (int)mdbx_limits_valsize_max((intptr_t)env->me_psize, flags);
+}
+
+__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize,
+ MDBX_db_flags_t flags) {
+ if (pagesize < 1)
+ pagesize = (intptr_t)mdbx_default_pagesize();
+ if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
+ pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2((size_t)pagesize)))
+ return -1;
+
+ return valsize_max(pagesize, flags);
+}
+
+/* Calculate the size of a leaf node.
+ *
+ * The size depends on the environment's page size; if a data item
+ * is too large it will be put onto an overflow page and the node
+ * size will only include the key and not the data. Sizes are always
+ * rounded up to an even number of bytes, to guarantee 2-byte alignment
+ * of the MDBX_node headers. */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) {
+ size_t node_bytes = node_size(key, data);
+ if (node_bytes > env->me_leaf_nodemax) {
+ /* put on overflow page */
+ node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t);
+ }
+
+ return node_bytes + sizeof(indx_t);
+}
+
+/* Calculate the size of a branch node.
+ *
+ * The size should depend on the environment's page size but since
+ * we currently don't support spilling large keys onto overflow
+ * pages, it's simply the size of the MDBX_node header plus the
+ * size of the key. Sizes are always rounded up to an even number
+ * of bytes, to guarantee 2-byte alignment of the MDBX_node headers.
+ *
+ * [in] env The environment handle.
+ * [in] key The key for the node.
+ *
+ * Returns The number of bytes needed to store the node. */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+branch_size(const MDBX_env *env, const MDBX_val *key) {
+ /* Size of a node in a branch page with a given key.
+ * This is just the node header plus the key, there is no data. */
+ size_t node_bytes = node_size(key, nullptr);
+ if (unlikely(node_bytes > env->me_leaf_nodemax)) {
+ /* put on overflow page */
+ /* not implemented */
+ mdbx_assert_fail(env, "INDXSIZE(key) <= env->me_nodemax", __func__,
+ __LINE__);
+ node_bytes = node_size(key, nullptr) + sizeof(pgno_t);
+ }
+
+ return node_bytes + sizeof(indx_t);
+}
+
+MDBX_NOTHROW_CONST_FUNCTION static __always_inline uint16_t
+flags_db2sub(uint16_t db_flags) {
+ uint16_t sub_flags = db_flags & MDBX_DUPFIXED;
+
+ /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */
+#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2
+ STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) ==
+ MDBX_INTEGERKEY);
+ sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY;
+
+ /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */
+#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5
+ STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) ==
+ MDBX_REVERSEKEY);
+ sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY;
+
+ return sub_flags;
+}
+
+/*----------------------------------------------------------------------------*/
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t
+pgno2bytes(const MDBX_env *env, pgno_t pgno) {
+ mdbx_assert(env, (1u << env->me_psize2log) == env->me_psize);
+ return ((size_t)pgno) << env->me_psize2log;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_page *
+pgno2page(const MDBX_env *env, pgno_t pgno) {
+ return (MDBX_page *)(env->me_map + pgno2bytes(env, pgno));
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
+bytes2pgno(const MDBX_env *env, size_t bytes) {
+ mdbx_assert(env, (env->me_psize >> env->me_psize2log) == 1);
+ return (pgno_t)(bytes >> env->me_psize2log);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static size_t
+pgno_align2os_bytes(const MDBX_env *env, pgno_t pgno) {
+ return ceil_powerof2(pgno2bytes(env, pgno), env->me_os_psize);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static pgno_t pgno_align2os_pgno(const MDBX_env *env,
+ pgno_t pgno) {
+ return bytes2pgno(env, pgno_align2os_bytes(env, pgno));
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static size_t
+bytes_align2os_bytes(const MDBX_env *env, size_t bytes) {
+ return ceil_powerof2(ceil_powerof2(bytes, env->me_psize), env->me_os_psize);
+}
+
+/* Address of first usable data byte in a page, after the header */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
+page_data(const MDBX_page *mp) {
+ return (char *)mp + PAGEHDRSZ;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline const MDBX_page *
+data_page(const void *data) {
+ return container_of(data, MDBX_page, mp_ptrs);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_meta *
+page_meta(MDBX_page *mp) {
+ return (MDBX_meta *)page_data(mp);
+}
+
+/* Number of nodes on a page */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
+page_numkeys(const MDBX_page *mp) {
+ return mp->mp_lower >> 1;
+}
+
+/* The amount of space remaining in the page */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
+page_room(const MDBX_page *mp) {
+ return mp->mp_upper - mp->mp_lower;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
+page_space(const MDBX_env *env) {
+ STATIC_ASSERT(PAGEHDRSZ % 2 == 0);
+ return env->me_psize - PAGEHDRSZ;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
+page_used(const MDBX_env *env, const MDBX_page *mp) {
+ return page_space(env) - page_room(mp);
+}
+
+/* The percentage of space used in the page, in a percents. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __inline double
+page_fill(const MDBX_env *env, const MDBX_page *mp) {
+ return page_used(env, mp) * 100.0 / page_space(env);
+}
+
+/* The number of overflow pages needed to store the given size. */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t
+number_of_ovpages(const MDBX_env *env, size_t bytes) {
+ return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1;
+}
+
+__cold static int MDBX_PRINTF_ARGS(2, 3)
+ bad_page(const MDBX_page *mp, const char *fmt, ...) {
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) {
+ static const MDBX_page *prev;
+ if (prev != mp) {
+ prev = mp;
+ const char *type;
+ switch (mp->mp_flags & (P_BRANCH | P_LEAF | P_OVERFLOW | P_META |
+ P_LEAF2 | P_BAD | P_SUBP)) {
+ case P_BRANCH:
+ type = "branch";
+ break;
+ case P_LEAF:
+ type = "leaf";
+ break;
+ case P_LEAF | P_SUBP:
+ type = "subleaf";
+ break;
+ case P_LEAF | P_LEAF2:
+ type = "dupfixed-leaf";
+ break;
+ case P_LEAF | P_LEAF2 | P_SUBP:
+ type = "dupfixed-subleaf";
+ break;
+ case P_OVERFLOW:
+ type = "large";
+ break;
+ default:
+ type = "broken";
+ }
+ mdbx_debug_log(MDBX_LOG_ERROR, "badpage", 0,
+ "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", type,
+ mp->mp_pgno, mp->mp_txnid);
+ }
+
+ va_list args;
+ va_start(args, fmt);
+ mdbx_debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args);
+ va_end(args);
+ }
+ return MDBX_CORRUPTED;
+}
+
+/* Address of node i in page p */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_node *
+page_node(const MDBX_page *mp, unsigned i) {
+ assert((mp->mp_flags & (P_LEAF2 | P_OVERFLOW | P_META)) == 0);
+ assert(page_numkeys(mp) > (unsigned)(i));
+ assert(mp->mp_ptrs[i] % 2 == 0);
+ return (MDBX_node *)((char *)mp + mp->mp_ptrs[i] + PAGEHDRSZ);
+}
+
+/* The address of a key in a LEAF2 page.
+ * LEAF2 pages are used for MDBX_DUPFIXED sorted-duplicate sub-DBs.
+ * There are no node headers, keys are stored contiguously. */
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline void *
+page_leaf2key(const MDBX_page *mp, unsigned i, size_t keysize) {
+ assert((mp->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_META)) ==
+ (P_LEAF | P_LEAF2));
+ assert(mp->mp_leaf2_ksize == keysize);
+ (void)keysize;
+ return (char *)mp + PAGEHDRSZ + (i * mp->mp_leaf2_ksize);
+}
+
+/* Set the node's key into keyptr. */
+static __always_inline void get_key(const MDBX_node *node, MDBX_val *keyptr) {
+ keyptr->iov_len = node_ks(node);
+ keyptr->iov_base = node_key(node);
+}
+
+/* Set the node's key into keyptr, if requested. */
+static __always_inline void
+get_key_optional(const MDBX_node *node, MDBX_val *keyptr /* __may_null */) {
+ if (keyptr)
+ get_key(node, keyptr);
+}
+
+/*------------------------------------------------------------------------------
+ * Workaround for mmaped-lookahead-cross-page-boundary bug
+ * in an obsolete versions of Elbrus's libc and kernels. */
+#if defined(__e2k__) && defined(MDBX_E2K_MLHCPB_WORKAROUND) && \
+ MDBX_E2K_MLHCPB_WORKAROUND
+int __hot mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2,
+ size_t n) {
+ if (unlikely(n > 42
+ /* LY: align followed access if reasonable possible */
+ && (((uintptr_t)s1) & 7) != 0 &&
+ (((uintptr_t)s1) & 7) == (((uintptr_t)s2) & 7))) {
+ if (((uintptr_t)s1) & 1) {
+ const int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
+ if (diff)
+ return diff;
+ s1 = (char *)s1 + 1;
+ s2 = (char *)s2 + 1;
+ n -= 1;
+ }
+
+ if (((uintptr_t)s1) & 2) {
+ const uint16_t a = *(uint16_t *)s1;
+ const uint16_t b = *(uint16_t *)s2;
+ if (likely(a != b))
+ return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
+ s1 = (char *)s1 + 2;
+ s2 = (char *)s2 + 2;
+ n -= 2;
+ }
+
+ if (((uintptr_t)s1) & 4) {
+ const uint32_t a = *(uint32_t *)s1;
+ const uint32_t b = *(uint32_t *)s2;
+ if (likely(a != b))
+ return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
+ s1 = (char *)s1 + 4;
+ s2 = (char *)s2 + 4;
+ n -= 4;
+ }
+ }
+
+ while (n >= 8) {
+ const uint64_t a = *(uint64_t *)s1;
+ const uint64_t b = *(uint64_t *)s2;
+ if (likely(a != b))
+ return (__builtin_bswap64(a) > __builtin_bswap64(b)) ? 1 : -1;
+ s1 = (char *)s1 + 8;
+ s2 = (char *)s2 + 8;
+ n -= 8;
+ }
+
+ if (n & 4) {
+ const uint32_t a = *(uint32_t *)s1;
+ const uint32_t b = *(uint32_t *)s2;
+ if (likely(a != b))
+ return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
+ s1 = (char *)s1 + 4;
+ s2 = (char *)s2 + 4;
+ }
+
+ if (n & 2) {
+ const uint16_t a = *(uint16_t *)s1;
+ const uint16_t b = *(uint16_t *)s2;
+ if (likely(a != b))
+ return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
+ s1 = (char *)s1 + 2;
+ s2 = (char *)s2 + 2;
+ }
+
+ return (n & 1) ? *(uint8_t *)s1 - *(uint8_t *)s2 : 0;
+}
+
+int __hot mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2) {
+ while (true) {
+ int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
+ if (likely(diff != 0) || *s1 == '\0')
+ return diff;
+ s1 += 1;
+ s2 += 1;
+ }
+}
+
+int __hot mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2,
+ size_t n) {
+ while (n > 0) {
+ int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
+ if (likely(diff != 0) || *s1 == '\0')
+ return diff;
+ s1 += 1;
+ s2 += 1;
+ n -= 1;
+ }
+ return 0;
+}
+
+size_t __hot mdbx_e2k_strlen_bug_workaround(const char *s) {
+ size_t n = 0;
+ while (*s) {
+ s += 1;
+ n += 1;
+ }
+ return n;
+}
+
+size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) {
+ size_t n = 0;
+ while (maxlen > n && *s) {
+ s += 1;
+ n += 1;
+ }
+ return n;
+}
+#endif /* MDBX_E2K_MLHCPB_WORKAROUND */
+
+/*------------------------------------------------------------------------------
+ * safe read/write volatile 64-bit fields on 32-bit architectures. */
+
+MDBX_MAYBE_UNUSED static __always_inline uint64_t
+atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
+#if MDBX_64BIT_ATOMIC
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+#else /* !MDBX_64BIT_ATOMIC */
+ mdbx_compiler_barrier();
+ atomic_store32(&p->low, (uint32_t)value, mo_Relaxed);
+ mdbx_jitter4testing(true);
+ atomic_store32(&p->high, (uint32_t)(value >> 32), order);
+ mdbx_jitter4testing(true);
+#endif /* !MDBX_64BIT_ATOMIC */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static
+#if MDBX_64BIT_ATOMIC
+ __always_inline
+#endif /* MDBX_64BIT_ATOMIC */
+ uint64_t
+ atomic_load64(const MDBX_atomic_uint64_t *p,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
+#if MDBX_64BIT_ATOMIC
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint64_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+#else /* !MDBX_64BIT_ATOMIC */
+ mdbx_compiler_barrier();
+ uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
+ mdbx_jitter4testing(true);
+ value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
+ : mo_AcquireRelease);
+ mdbx_jitter4testing(true);
+ for (;;) {
+ mdbx_compiler_barrier();
+ uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
+ mdbx_jitter4testing(true);
+ again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
+ : mo_AcquireRelease);
+ mdbx_jitter4testing(true);
+ if (likely(value == again))
+ return value;
+ value = again;
+ }
+#endif /* !MDBX_64BIT_ATOMIC */
+}
+
+static __always_inline void atomic_yield(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ YieldProcessor();
+#elif defined(__ia32__) || defined(__e2k__)
+ __builtin_ia32_pause();
+#elif defined(__ia64__)
+#if defined(__HP_cc__) || defined(__HP_aCC__)
+ _Asm_hint(_HINT_PAUSE);
+#else
+ __asm__ __volatile__("hint @pause");
+#endif
+#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \
+ defined(__ARM_ARCH_6K__)
+#ifdef __CC_ARM
+ __yield();
+#else
+ __asm__ __volatile__("yield");
+#endif
+#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \
+ __mips_isa_rev >= 2
+ __asm__ __volatile__("pause");
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+ __asm__ __volatile__(".word 0x00000140");
+#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
+ sched_yield();
+#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
+ pthread_yield();
+#endif
+}
+
+#if MDBX_64BIT_CAS
+static __always_inline bool atomic_cas64(MDBX_atomic_uint64_t *p, uint64_t c,
+ uint64_t v) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t));
+#ifdef ATOMIC_LLONG_LOCK_FREE
+ STATIC_ASSERT(ATOMIC_LLONG_LOCK_FREE > 0);
+#if ATOMIC_LLONG_LOCK_FREE < 2
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
+#endif /* ATOMIC_LLONG_LOCK_FREE < 2 */
+#else /* defined(ATOMIC_LLONG_LOCK_FREE) */
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
+#endif
+ return atomic_compare_exchange_strong(MDBX_c11a_rw(uint64_t, p), &c, v);
+#elif defined(__GNUC__) || defined(__clang__)
+ return __sync_bool_compare_and_swap(&p->weak, c, v);
+#elif defined(_MSC_VER)
+ return c == (uint64_t)_InterlockedCompareExchange64(
+ (volatile __int64 *)&p->weak, v, c);
+#elif defined(__APPLE__)
+ return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak);
+#else
+#error FIXME: Unsupported compiler
+#endif
+}
+#endif /* MDBX_64BIT_CAS */
+
+static __always_inline bool atomic_cas32(MDBX_atomic_uint32_t *p, uint32_t c,
+ uint32_t v) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
+#ifdef ATOMIC_INT_LOCK_FREE
+ STATIC_ASSERT(ATOMIC_INT_LOCK_FREE > 0);
+#if ATOMIC_INT_LOCK_FREE < 2
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#endif
+#else
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#endif
+ return atomic_compare_exchange_strong(MDBX_c11a_rw(uint32_t, p), &c, v);
+#elif defined(__GNUC__) || defined(__clang__)
+ return __sync_bool_compare_and_swap(&p->weak, c, v);
+#elif defined(_MSC_VER)
+ STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
+ return c ==
+ (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c);
+#elif defined(__APPLE__)
+ return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak);
+#else
+#error FIXME: Unsupported compiler
+#endif
+}
+
+static __always_inline uint32_t atomic_add32(MDBX_atomic_uint32_t *p,
+ uint32_t v) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
+#ifdef ATOMIC_INT_LOCK_FREE
+ STATIC_ASSERT(ATOMIC_INT_LOCK_FREE > 0);
+#if ATOMIC_INT_LOCK_FREE < 2
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#endif
+#else
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+#endif
+ return atomic_fetch_add(MDBX_c11a_rw(uint32_t, p), v);
+#elif defined(__GNUC__) || defined(__clang__)
+ return __sync_fetch_and_add(&p->weak, v);
+#elif defined(_MSC_VER)
+ STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
+ return (uint32_t)_InterlockedExchangeAdd((volatile long *)&p->weak, v);
+#elif defined(__APPLE__)
+ return OSAtomicAdd32Barrier(v, &p->weak);
+#else
+#error FIXME: Unsupported compiler
+#endif
+}
+
+#define atomic_sub32(p, v) atomic_add32(p, 0 - (v))
+
+static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) {
+ txnid += xMDBX_TXNID_STEP;
+#if !MDBX_64BIT_CAS
+ /* avoid overflow of low-part in safe64_reset() */
+ txnid += (UINT32_MAX == (uint32_t)txnid);
+#endif
+ return txnid;
+}
+
+static __always_inline void safe64_reset(MDBX_atomic_uint64_t *p,
+ bool single_writer) {
+#if !MDBX_64BIT_CAS
+ if (!single_writer) {
+ STATIC_ASSERT(xMDBX_TXNID_STEP > 1);
+ /* it is safe to increment low-part to avoid ABA, since xMDBX_TXNID_STEP > 1
+ * and overflow was preserved in safe64_txnid_next() */
+ atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
+ atomic_store32(
+ &p->high, UINT32_MAX,
+ mo_Relaxed) /* atomically make >= SAFE64_INVALID_THRESHOLD */;
+ atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
+ } else
+#endif /* !MDBX_64BIT_CAS */
+#if MDBX_64BIT_ATOMIC
+ /* atomically make value >= SAFE64_INVALID_THRESHOLD by 64-bit operation */
+ atomic_store64(p, UINT64_MAX,
+ single_writer ? mo_AcquireRelease
+ : mo_SequentialConsistency);
+#else
+ /* atomically make value >= SAFE64_INVALID_THRESHOLD by 32-bit operation */
+ atomic_store32(&p->high, UINT32_MAX,
+ single_writer ? mo_AcquireRelease : mo_SequentialConsistency);
+#endif /* MDBX_64BIT_ATOMIC */
+ assert(p->weak >= SAFE64_INVALID_THRESHOLD);
+ mdbx_jitter4testing(true);
+}
+
+static __always_inline bool safe64_reset_compare(MDBX_atomic_uint64_t *p,
+ txnid_t compare) {
+ /* LY: This function is used to reset `mr_txnid` from hsr-handler in case
+ * the asynchronously cancellation of read transaction. Therefore,
+ * there may be a collision between the cleanup performed here and
+ * asynchronous termination and restarting of the read transaction
+ * in another proces/thread. In general we MUST NOT reset the `mr_txnid`
+ * if a new transaction was started (i.e. if `mr_txnid` was changed). */
+#if MDBX_64BIT_CAS
+ bool rc = atomic_cas64(p, compare, UINT64_MAX);
+#else
+ /* LY: There is no gold ratio here since shared mutex is too costly,
+ * in such way we must acquire/release it for every update of mr_txnid,
+ * i.e. twice for each read transaction). */
+ bool rc = false;
+ if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare &&
+ atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) {
+ if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) !=
+ (uint32_t)compare))
+ atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32));
+ else
+ rc = true;
+ }
+#endif /* MDBX_64BIT_CAS */
+ mdbx_jitter4testing(true);
+ return rc;
+}
+
+static __always_inline void safe64_write(MDBX_atomic_uint64_t *p,
+ const uint64_t v) {
+ assert(p->weak >= SAFE64_INVALID_THRESHOLD);
+#if MDBX_64BIT_ATOMIC
+ atomic_store64(p, v, mo_AcquireRelease);
+#else /* MDBX_64BIT_ATOMIC */
+ mdbx_compiler_barrier();
+ /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */
+ atomic_store32(&p->low, (uint32_t)v, mo_Relaxed);
+ assert(p->weak >= SAFE64_INVALID_THRESHOLD);
+ mdbx_jitter4testing(true);
+ /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */
+ atomic_store32(&p->high, (uint32_t)(v >> 32), mo_AcquireRelease);
+#endif /* MDBX_64BIT_ATOMIC */
+ assert(p->weak == v);
+ mdbx_jitter4testing(true);
+}
+
+static __always_inline uint64_t safe64_read(const MDBX_atomic_uint64_t *p) {
+ mdbx_jitter4testing(true);
+ uint64_t v = atomic_load64(p, mo_AcquireRelease);
+ mdbx_jitter4testing(true);
+ return v;
+}
+
+#if 0 /* unused for now */
+MDBX_MAYBE_UNUSED static __always_inline bool safe64_is_valid(uint64_t v) {
+#if MDBX_WORDBITS >= 64
+ return v < SAFE64_INVALID_THRESHOLD;
+#else
+ return (v >> 32) != UINT32_MAX;
+#endif /* MDBX_WORDBITS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline bool
+ safe64_is_valid_ptr(const MDBX_atomic_uint64_t *p) {
+#if MDBX_64BIT_ATOMIC
+ return atomic_load64(p, mo_AcquireRelease) < SAFE64_INVALID_THRESHOLD;
+#else
+ return atomic_load32(&p->high, mo_AcquireRelease) != UINT32_MAX;
+#endif /* MDBX_64BIT_ATOMIC */
+}
+#endif /* unused for now */
+
+/* non-atomic write with safety for reading a half-updated value */
+static __always_inline void safe64_update(MDBX_atomic_uint64_t *p,
+ const uint64_t v) {
+#if MDBX_64BIT_ATOMIC
+ atomic_store64(p, v, mo_Relaxed);
+#else
+ safe64_reset(p, true);
+ safe64_write(p, v);
+#endif /* MDBX_64BIT_ATOMIC */
+}
+
+/* non-atomic increment with safety for reading a half-updated value */
+MDBX_MAYBE_UNUSED static
+#if MDBX_64BIT_ATOMIC
+ __always_inline
+#endif /* MDBX_64BIT_ATOMIC */
+ void
+ safe64_inc(MDBX_atomic_uint64_t *p, const uint64_t v) {
+ assert(v > 0);
+ safe64_update(p, atomic_load64(p, mo_Relaxed) + v);
+}
+
+/*----------------------------------------------------------------------------*/
+/* rthc (tls keys and destructors) */
+
+typedef struct rthc_entry_t {
+ MDBX_reader *begin;
+ MDBX_reader *end;
+ mdbx_thread_key_t thr_tls_key;
+ bool key_valid;
+} rthc_entry_t;
+
+#if MDBX_DEBUG
+#define RTHC_INITIAL_LIMIT 1
+#else
+#define RTHC_INITIAL_LIMIT 16
+#endif
+
+static bin128_t bootid;
+
+#if defined(_WIN32) || defined(_WIN64)
+static CRITICAL_SECTION rthc_critical_section;
+static CRITICAL_SECTION lcklist_critical_section;
+#else
+int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj, void *dso_symbol)
+ __attribute__((__weak__));
+#ifdef __APPLE__ /* FIXME: Thread-Local Storage destructors & DSO-unloading */
+int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj,
+ void *dso_symbol) {
+ (void)dtor;
+ (void)obj;
+ (void)dso_symbol;
+ return -1;
+}
+#endif /* __APPLE__ */
+
+static pthread_mutex_t lcklist_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER;
+static mdbx_thread_key_t rthc_key;
+static MDBX_atomic_uint32_t rthc_pending;
+
+__cold static void workaround_glibc_bug21031(void) {
+ /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031
+ *
+ * Due race between pthread_key_delete() and __nptl_deallocate_tsd()
+ * The destructor(s) of thread-local-storage object(s) may be running
+ * in another thread(s) and be blocked or not finished yet.
+ * In such case we get a SEGFAULT after unload this library DSO.
+ *
+ * So just by yielding a few timeslices we give a chance
+ * to such destructor(s) for completion and avoids segfault. */
+ sched_yield();
+ sched_yield();
+ sched_yield();
+}
+#endif
+
+static unsigned rthc_count, rthc_limit;
+static rthc_entry_t *rthc_table;
+static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT];
+
+static __inline void rthc_lock(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ EnterCriticalSection(&rthc_critical_section);
+#else
+ mdbx_ensure(nullptr, mdbx_pthread_mutex_lock(&rthc_mutex) == 0);
+#endif
+}
+
+static __inline void rthc_unlock(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ LeaveCriticalSection(&rthc_critical_section);
+#else
+ mdbx_ensure(nullptr, pthread_mutex_unlock(&rthc_mutex) == 0);
+#endif
+}
+
+static __inline int thread_key_create(mdbx_thread_key_t *key) {
+ int rc;
+#if defined(_WIN32) || defined(_WIN64)
+ *key = TlsAlloc();
+ rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError();
+#else
+ rc = pthread_key_create(key, nullptr);
+#endif
+ mdbx_trace("&key = %p, value %" PRIuPTR ", rc %d",
+ __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc);
+ return rc;
+}
+
+static __inline void thread_key_delete(mdbx_thread_key_t key) {
+ mdbx_trace("key = %" PRIuPTR, (uintptr_t)key);
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_ensure(nullptr, TlsFree(key));
+#else
+ mdbx_ensure(nullptr, pthread_key_delete(key) == 0);
+ workaround_glibc_bug21031();
+#endif
+}
+
+static __inline void *thread_rthc_get(mdbx_thread_key_t key) {
+#if defined(_WIN32) || defined(_WIN64)
+ return TlsGetValue(key);
+#else
+ return pthread_getspecific(key);
+#endif
+}
+
+static void thread_rthc_set(mdbx_thread_key_t key, const void *value) {
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_ensure(nullptr, TlsSetValue(key, (void *)value));
+#else
+#define MDBX_THREAD_RTHC_ZERO 0
+#define MDBX_THREAD_RTHC_REGISTERED 1
+#define MDBX_THREAD_RTHC_COUNTED 2
+ static __thread char thread_registration_state;
+ if (value && unlikely(thread_registration_state == MDBX_THREAD_RTHC_ZERO)) {
+ thread_registration_state = MDBX_THREAD_RTHC_REGISTERED;
+ mdbx_trace("thread registered 0x%" PRIxPTR, mdbx_thread_self());
+ if (&__cxa_thread_atexit_impl == nullptr ||
+ __cxa_thread_atexit_impl(mdbx_rthc_thread_dtor,
+ &thread_registration_state,
+ (void *)&mdbx_version /* dso_anchor */)) {
+ mdbx_ensure(nullptr, pthread_setspecific(
+ rthc_key, &thread_registration_state) == 0);
+ thread_registration_state = MDBX_THREAD_RTHC_COUNTED;
+ const unsigned count_before = atomic_add32(&rthc_pending, 1);
+ mdbx_ensure(nullptr, count_before < INT_MAX);
+ mdbx_trace("fallback to pthreads' tsd, key %" PRIuPTR ", count %u",
+ (uintptr_t)rthc_key, count_before);
+ (void)count_before;
+ }
+ }
+ mdbx_ensure(nullptr, pthread_setspecific(key, value) == 0);
+#endif
+}
+
+__cold void mdbx_rthc_global_init(void) {
+ rthc_limit = RTHC_INITIAL_LIMIT;
+ rthc_table = rthc_table_static;
+#if defined(_WIN32) || defined(_WIN64)
+ InitializeCriticalSection(&rthc_critical_section);
+ InitializeCriticalSection(&lcklist_critical_section);
+#else
+ mdbx_ensure(nullptr,
+ pthread_key_create(&rthc_key, mdbx_rthc_thread_dtor) == 0);
+ mdbx_trace("pid %d, &mdbx_rthc_key = %p, value 0x%x", mdbx_getpid(),
+ __Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key);
+#endif
+ /* checking time conversion, this also avoids racing on 32-bit architectures
+ * during writing calculated 64-bit ratio(s) into memory. */
+ uint32_t proba = UINT32_MAX;
+ while (true) {
+ unsigned time_conversion_checkup =
+ mdbx_osal_monotime_to_16dot16(mdbx_osal_16dot16_to_monotime(proba));
+ unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba;
+ unsigned one_less = (proba > 0) ? proba - 1 : proba;
+ mdbx_ensure(nullptr, time_conversion_checkup >= one_less &&
+ time_conversion_checkup <= one_more);
+ if (proba == 0)
+ break;
+ proba >>= 1;
+ }
+
+ bootid = mdbx_osal_bootid();
+#if 0 /* debug */
+ for (unsigned i = 0; i < 65536; ++i) {
+ size_t pages = pv2pages(i);
+ unsigned x = pages2pv(pages);
+ size_t xp = pv2pages(x);
+ if (!(x == i || (x % 2 == 0 && x < 65536)) || pages != xp)
+ printf("%u => %zu => %u => %zu\n", i, pages, x, xp);
+ assert(pages == xp);
+ }
+ fflush(stdout);
+#endif
+}
+
+/* dtor called for thread, i.e. for all mdbx's environment objects */
+__cold void mdbx_rthc_thread_dtor(void *ptr) {
+ rthc_lock();
+ mdbx_trace(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", mdbx_getpid(),
+ mdbx_thread_self(), ptr);
+
+ const uint32_t self_pid = mdbx_getpid();
+ for (unsigned i = 0; i < rthc_count; ++i) {
+ if (!rthc_table[i].key_valid)
+ continue;
+ const mdbx_thread_key_t key = rthc_table[i].thr_tls_key;
+ MDBX_reader *const rthc = thread_rthc_get(key);
+ if (rthc < rthc_table[i].begin || rthc >= rthc_table[i].end)
+ continue;
+#if !defined(_WIN32) && !defined(_WIN64)
+ if (pthread_setspecific(key, nullptr) != 0) {
+ mdbx_trace("== thread 0x%" PRIxPTR
+ ", rthc %p: ignore race with tsd-key deletion",
+ mdbx_thread_self(), ptr);
+ continue /* ignore race with tsd-key deletion by mdbx_env_close() */;
+ }
+#endif
+
+ mdbx_trace("== thread 0x%" PRIxPTR
+ ", rthc %p, [%i], %p ... %p (%+i), rtch-pid %i, "
+ "current-pid %i",
+ mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), i,
+ __Wpedantic_format_voidptr(rthc_table[i].begin),
+ __Wpedantic_format_voidptr(rthc_table[i].end),
+ (int)(rthc - rthc_table[i].begin), rthc->mr_pid.weak, self_pid);
+ if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
+ mdbx_trace("==== thread 0x%" PRIxPTR ", rthc %p, cleanup",
+ mdbx_thread_self(), __Wpedantic_format_voidptr(rthc));
+ atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
+ }
+ }
+
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", mdbx_thread_self(), ptr);
+ rthc_unlock();
+#else
+ const char self_registration = *(volatile char *)ptr;
+ *(volatile char *)ptr = MDBX_THREAD_RTHC_ZERO;
+ mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d",
+ mdbx_thread_self(), ptr, mdbx_getpid(), self_registration);
+ if (self_registration == MDBX_THREAD_RTHC_COUNTED)
+ mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
+
+ if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) {
+ mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake",
+ mdbx_thread_self(), ptr, mdbx_getpid());
+ mdbx_ensure(nullptr, pthread_cond_broadcast(&rthc_cond) == 0);
+ }
+
+ mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", mdbx_thread_self(), ptr);
+ /* Allow tail call optimization, i.e. gcc should generate the jmp instruction
+ * instead of a call for pthread_mutex_unlock() and therefore CPU could not
+ * return to current DSO's code section, which may be unloaded immediately
+ * after the mutex got released. */
+ pthread_mutex_unlock(&rthc_mutex);
+#endif
+}
+
+__cold void mdbx_rthc_global_dtor(void) {
+ mdbx_trace(">> pid %d", mdbx_getpid());
+
+ rthc_lock();
+#if !defined(_WIN32) && !defined(_WIN64)
+ char *rthc = pthread_getspecific(rthc_key);
+ mdbx_trace(
+ "== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d, left %d",
+ mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), mdbx_getpid(),
+ rthc ? *rthc : -1, atomic_load32(&rthc_pending, mo_Relaxed));
+ if (rthc) {
+ const char self_registration = *rthc;
+ *rthc = MDBX_THREAD_RTHC_ZERO;
+ if (self_registration == MDBX_THREAD_RTHC_COUNTED)
+ mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
+ }
+
+ struct timespec abstime;
+ mdbx_ensure(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0);
+ abstime.tv_nsec += 1000000000l / 10;
+ if (abstime.tv_nsec >= 1000000000l) {
+ abstime.tv_nsec -= 1000000000l;
+ abstime.tv_sec += 1;
+ }
+#if MDBX_DEBUG > 0
+ abstime.tv_sec += 600;
+#endif
+
+ for (unsigned left;
+ (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) {
+ mdbx_trace("pid %d, pending %u, wait for...", mdbx_getpid(), left);
+ const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime);
+ if (rc && rc != EINTR)
+ break;
+ }
+ thread_key_delete(rthc_key);
+#endif
+
+ const uint32_t self_pid = mdbx_getpid();
+ for (unsigned i = 0; i < rthc_count; ++i) {
+ if (!rthc_table[i].key_valid)
+ continue;
+ const mdbx_thread_key_t key = rthc_table[i].thr_tls_key;
+ thread_key_delete(key);
+ for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
+ ++rthc) {
+ mdbx_trace(
+ "== [%i] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), "
+ "rthc-pid %i, current-pid %i",
+ i, (uintptr_t)key, __Wpedantic_format_voidptr(rthc_table[i].begin),
+ __Wpedantic_format_voidptr(rthc_table[i].end),
+ __Wpedantic_format_voidptr(rthc), (int)(rthc - rthc_table[i].begin),
+ rthc->mr_pid.weak, self_pid);
+ if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
+ atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
+ mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc));
+ }
+ }
+ }
+
+ rthc_limit = rthc_count = 0;
+ if (rthc_table != rthc_table_static)
+ mdbx_free(rthc_table);
+ rthc_table = nullptr;
+ rthc_unlock();
+
+#if defined(_WIN32) || defined(_WIN64)
+ DeleteCriticalSection(&lcklist_critical_section);
+ DeleteCriticalSection(&rthc_critical_section);
+#else
+ /* LY: yielding a few timeslices to give a more chance
+ * to racing destructor(s) for completion. */
+ workaround_glibc_bug21031();
+#endif
+
+ mdbx_trace("<< pid %d\n", mdbx_getpid());
+}
+
+__cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
+ MDBX_reader *end) {
+ int rc;
+ if (key) {
+#ifndef NDEBUG
+ *key = (mdbx_thread_key_t)0xBADBADBAD;
+#endif /* NDEBUG */
+ rc = thread_key_create(key);
+ if (rc != MDBX_SUCCESS)
+ return rc;
+ }
+
+ rthc_lock();
+ const mdbx_thread_key_t new_key = key ? *key : 0;
+ mdbx_trace(">> key %" PRIuPTR ", rthc_count %u, rthc_limit %u",
+ (uintptr_t)new_key, rthc_count, rthc_limit);
+ if (rthc_count == rthc_limit) {
+ rthc_entry_t *new_table =
+ mdbx_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table,
+ sizeof(rthc_entry_t) * rthc_limit * 2);
+ if (new_table == nullptr) {
+ rc = MDBX_ENOMEM;
+ goto bailout;
+ }
+ if (rthc_table == rthc_table_static)
+ memcpy(new_table, rthc_table_static, sizeof(rthc_table_static));
+ rthc_table = new_table;
+ rthc_limit *= 2;
+ }
+ mdbx_trace("== [%i] = key %" PRIuPTR ", %p ... %p", rthc_count,
+ (uintptr_t)new_key, __Wpedantic_format_voidptr(begin),
+ __Wpedantic_format_voidptr(end));
+ rthc_table[rthc_count].key_valid = key ? true : false;
+ rthc_table[rthc_count].thr_tls_key = key ? new_key : 0;
+ rthc_table[rthc_count].begin = begin;
+ rthc_table[rthc_count].end = end;
+ ++rthc_count;
+ mdbx_trace("<< key %" PRIuPTR ", rthc_count %u, rthc_limit %u",
+ (uintptr_t)new_key, rthc_count, rthc_limit);
+ rthc_unlock();
+ return MDBX_SUCCESS;
+
+bailout:
+ if (key)
+ thread_key_delete(*key);
+ rthc_unlock();
+ return rc;
+}
+
+__cold void mdbx_rthc_remove(const mdbx_thread_key_t key) {
+ thread_key_delete(key);
+ rthc_lock();
+ mdbx_trace(">> key %zu, rthc_count %u, rthc_limit %u", (size_t)key,
+ rthc_count, rthc_limit);
+
+ for (unsigned i = 0; i < rthc_count; ++i) {
+ if (rthc_table[i].key_valid && key == rthc_table[i].thr_tls_key) {
+ const uint32_t self_pid = mdbx_getpid();
+ mdbx_trace("== [%i], %p ...%p, current-pid %d", i,
+ __Wpedantic_format_voidptr(rthc_table[i].begin),
+ __Wpedantic_format_voidptr(rthc_table[i].end), self_pid);
+
+ for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
+ ++rthc) {
+ if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) {
+ atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease);
+ mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc));
+ }
+ }
+ if (--rthc_count > 0)
+ rthc_table[i] = rthc_table[rthc_count];
+ else if (rthc_table != rthc_table_static) {
+ mdbx_free(rthc_table);
+ rthc_table = rthc_table_static;
+ rthc_limit = RTHC_INITIAL_LIMIT;
+ }
+ break;
+ }
+ }
+
+ mdbx_trace("<< key %zu, rthc_count %u, rthc_limit %u", (size_t)key,
+ rthc_count, rthc_limit);
+ rthc_unlock();
+}
+
+//------------------------------------------------------------------------------
+
+#define RTHC_ENVLIST_END ((MDBX_env *)((uintptr_t)50459))
+static MDBX_env *inprocess_lcklist_head = RTHC_ENVLIST_END;
+
+static __inline void lcklist_lock(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ EnterCriticalSection(&lcklist_critical_section);
+#else
+ mdbx_ensure(nullptr, mdbx_pthread_mutex_lock(&lcklist_mutex) == 0);
+#endif
+}
+
+static __inline void lcklist_unlock(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ LeaveCriticalSection(&lcklist_critical_section);
+#else
+ mdbx_ensure(nullptr, pthread_mutex_unlock(&lcklist_mutex) == 0);
+#endif
+}
+
+MDBX_NOTHROW_CONST_FUNCTION static uint64_t rrxmrrxmsx_0(uint64_t v) {
+ /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */
+ v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50);
+ v *= UINT64_C(0xA24BAED4963EE407);
+ v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49);
+ v *= UINT64_C(0x9FB21C651E98DF25);
+ return v ^ v >> 28;
+}
+
+static int uniq_peek(const mdbx_mmap_t *pending, mdbx_mmap_t *scan) {
+ int rc;
+ uint64_t bait;
+ MDBX_lockinfo *const pending_lck = pending->lck;
+ MDBX_lockinfo *const scan_lck = scan->lck;
+ if (pending_lck) {
+ bait = atomic_load64(&pending_lck->mti_bait_uniqueness, mo_AcquireRelease);
+ rc = MDBX_SUCCESS;
+ } else {
+ bait = 0 /* hush MSVC warning */;
+ rc = mdbx_msync(scan, 0, sizeof(MDBX_lockinfo), MDBX_SYNC_DATA);
+ if (rc == MDBX_SUCCESS)
+ rc = mdbx_pread(pending->fd, &bait, sizeof(scan_lck->mti_bait_uniqueness),
+ offsetof(MDBX_lockinfo, mti_bait_uniqueness));
+ }
+ if (likely(rc == MDBX_SUCCESS) &&
+ bait == atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease))
+ rc = MDBX_RESULT_TRUE;
+
+ mdbx_trace("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d",
+ pending_lck ? "mem" : "file", bait,
+ (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc);
+ return rc;
+}
+
+static int uniq_poke(const mdbx_mmap_t *pending, mdbx_mmap_t *scan,
+ uint64_t *abra) {
+ if (*abra == 0) {
+ const uintptr_t tid = mdbx_thread_self();
+ uintptr_t uit = 0;
+ memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit));
+ *abra =
+ rrxmrrxmsx_0(mdbx_osal_monotime() + UINT64_C(5873865991930747) * uit);
+ }
+ const uint64_t cadabra =
+ rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)mdbx_getpid())
+ << 24 |
+ *abra >> 40;
+ MDBX_lockinfo *const scan_lck = scan->lck;
+ atomic_store64(&scan_lck->mti_bait_uniqueness, cadabra,
+ mo_SequentialConsistency);
+ *abra = *abra * UINT64_C(6364136223846793005) + 1;
+ return uniq_peek(pending, scan);
+}
+
+__cold static int uniq_check(const mdbx_mmap_t *pending, MDBX_env **found) {
+ *found = nullptr;
+ uint64_t salt = 0;
+ for (MDBX_env *scan = inprocess_lcklist_head; scan != RTHC_ENVLIST_END;
+ scan = scan->me_lcklist_next) {
+ MDBX_lockinfo *const scan_lck = scan->me_lck_mmap.lck;
+ int err = atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease)
+ ? uniq_peek(pending, &scan->me_lck_mmap)
+ : uniq_poke(pending, &scan->me_lck_mmap, &salt);
+ if (err == MDBX_ENODATA) {
+ uint64_t length;
+ if (likely(mdbx_filesize(pending->fd, &length) == MDBX_SUCCESS &&
+ length == 0)) {
+ /* LY: skip checking since LCK-file is empty, i.e. just created. */
+ mdbx_debug("uniq-probe: %s", "unique (new/empty lck)");
+ return MDBX_RESULT_TRUE;
+ }
+ }
+ if (err == MDBX_RESULT_TRUE)
+ err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
+ if (err == MDBX_RESULT_TRUE) {
+ (void)mdbx_msync(&scan->me_lck_mmap, 0, sizeof(MDBX_lockinfo),
+ MDBX_SYNC_NONE);
+ err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
+ }
+ if (err == MDBX_RESULT_TRUE) {
+ err = uniq_poke(pending, &scan->me_lck_mmap, &salt);
+ *found = scan;
+ mdbx_debug("uniq-probe: found %p", __Wpedantic_format_voidptr(*found));
+ return MDBX_RESULT_FALSE;
+ }
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_debug("uniq-probe: failed rc %d", err);
+ return err;
+ }
+ }
+
+ mdbx_debug("uniq-probe: %s", "unique");
+ return MDBX_RESULT_TRUE;
+}
+
+static int lcklist_detach_locked(MDBX_env *env) {
+ MDBX_env *inprocess_neighbor = nullptr;
+ int rc = MDBX_SUCCESS;
+ if (env->me_lcklist_next != nullptr) {
+ mdbx_ensure(env, env->me_lcklist_next != nullptr);
+ mdbx_ensure(env, inprocess_lcklist_head != RTHC_ENVLIST_END);
+ for (MDBX_env **ptr = &inprocess_lcklist_head; *ptr != RTHC_ENVLIST_END;
+ ptr = &(*ptr)->me_lcklist_next) {
+ if (*ptr == env) {
+ *ptr = env->me_lcklist_next;
+ env->me_lcklist_next = nullptr;
+ break;
+ }
+ }
+ mdbx_ensure(env, env->me_lcklist_next == nullptr);
+ }
+
+ rc = likely(mdbx_getpid() == env->me_pid)
+ ? uniq_check(&env->me_lck_mmap, &inprocess_neighbor)
+ : MDBX_PANIC;
+ if (!inprocess_neighbor && env->me_live_reader)
+ (void)mdbx_rpid_clear(env);
+ if (!MDBX_IS_ERROR(rc))
+ rc = mdbx_lck_destroy(env, inprocess_neighbor);
+ return rc;
+}
+
+/*------------------------------------------------------------------------------
+ * LY: State of the art quicksort-based sorting, with internal stack
+ * and network-sort for small chunks.
+ * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */
+
+#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
+ do { \
+ const TYPE swap_tmp = (a); \
+ const bool swap_cmp = CMP(swap_tmp, b); \
+ (a) = swap_cmp ? swap_tmp : b; \
+ (b) = swap_cmp ? b : swap_tmp; \
+ } while (0)
+
+// 3 comparators, 3 parallel operations
+// o-----^--^--o
+// | |
+// o--^--|--v--o
+// | |
+// o--v--v-----o
+//
+// [[1,2]]
+// [[0,2]]
+// [[0,1]]
+#define SORT_NETWORK_3(TYPE, CMP, begin) \
+ do { \
+ SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
+ SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
+ SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
+ } while (0)
+
+// 5 comparators, 3 parallel operations
+// o--^--^--------o
+// | |
+// o--v--|--^--^--o
+// | | |
+// o--^--v--|--v--o
+// | |
+// o--v-----v-----o
+//
+// [[0,1],[2,3]]
+// [[0,2],[1,3]]
+// [[1,2]]
+#define SORT_NETWORK_4(TYPE, CMP, begin) \
+ do { \
+ SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
@@ -4895,18188 +8328,22597 @@ status_done:
return MDBX_SUCCESS;
}
-#if !MDBX_DEBUG && !defined(MDBX_USE_VALGRIND) && !defined(__SANITIZE_ADDRESS__)
- if (unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB))
-#endif
- {
- /* Страница могла быть изменена в одной из родительÑких транзакций,
- * в том чиÑле, позже выгружена и затем Ñнова загружена и изменена.
- * Ð’ обоих ÑлучаÑÑ… её Ð½ÐµÐ»ÑŒÐ·Ñ Ð·Ð°Ñ‚Ð¸Ñ€Ð°Ñ‚ÑŒ на диÑке и помечать недоÑтупной
- * в asan и/или valgrind */
- for (MDBX_txn *parent = txn->mt_parent;
- parent && (parent->mt_flags & MDBX_TXN_SPILLS);
- parent = parent->mt_parent) {
- if (mdbx_intersect_spilled(parent, pgno, npages))
- goto skip_invalidate;
- if (mdbx_dpl_intersect(parent, pgno, npages))
- goto skip_invalidate;
- }
+#if !MDBX_DEBUG && !defined(MDBX_USE_VALGRIND) && !defined(__SANITIZE_ADDRESS__)
+ if (unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB))
+#endif
+ {
+ /* Страница могла быть изменена в одной из родительÑких транзакций,
+ * в том чиÑле, позже выгружена и затем Ñнова загружена и изменена.
+ * Ð’ обоих ÑлучаÑÑ… её Ð½ÐµÐ»ÑŒÐ·Ñ Ð·Ð°Ñ‚Ð¸Ñ€Ð°Ñ‚ÑŒ на диÑке и помечать недоÑтупной
+ * в asan и/или valgrind */
+ for (MDBX_txn *parent = txn->mt_parent;
+ parent && (parent->mt_flags & MDBX_TXN_SPILLS);
+ parent = parent->mt_parent) {
+ if (mdbx_intersect_spilled(parent, pgno, npages))
+ goto skip_invalidate;
+ if (mdbx_dpl_intersect(parent, pgno, npages))
+ goto skip_invalidate;
+ }
+
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ if (MDBX_DEBUG != 0 || unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB))
+#endif
+ mdbx_kill_page(txn, mp, pgno, npages);
+ if (!(txn->mt_flags & MDBX_WRITEMAP)) {
+ VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->mt_env, pgno)),
+ pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ);
+ MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->mt_env, pgno)),
+ pgno2bytes(txn->mt_env, npages) -
+ PAGEHDRSZ);
+ }
+ }
+ skip_invalidate:
+ /* Remove from dirty list */
+ mdbx_page_wash(txn, di, mp, npages);
+
+ reclaim:
+ mdbx_debug("reclaim %u %s page %" PRIaPGNO, npages, "dirty", pgno);
+ rc = mdbx_pnl_insert_range(&txn->tw.reclaimed_pglist, pgno, npages);
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ return rc;
+ }
+
+ if (si) {
+ /* Page ws spilled in this txn */
+ mdbx_spill_remove(txn, si, npages);
+ /* Страница могла быть выделена и затем пролита в Ñтой транзакции,
+ * тогда её необходимо помеÑтить в reclaimed-ÑпиÑок.
+ * Либо она могла быть выделена в одной из родительÑких транзакций и затем
+ * пролита в Ñтой транзакции, тогда её необходимо помеÑтить в
+ * retired-ÑпиÑок Ð´Ð»Ñ Ð¿Ð¾Ñледующей фильтрации при коммите. */
+ for (MDBX_txn *parent = txn->mt_parent; parent;
+ parent = parent->mt_parent) {
+ if (mdbx_dpl_exist(parent, pgno))
+ goto retire;
+ }
+ /* Страница точно была выделена в Ñтой транзакции
+ * и теперь может быть иÑпользована повторно. */
+ goto reclaim;
+ }
+
+ if (is_shadowed) {
+ /* Dirty page MUST BE a clone from (one of) parent transaction(s). */
+ if (mdbx_assert_enabled()) {
+ const MDBX_page *parent_dp = nullptr;
+ /* Check parent(s)'s dirty lists. */
+ for (MDBX_txn *parent = txn->mt_parent; parent && !parent_dp;
+ parent = parent->mt_parent) {
+ mdbx_tassert(txn, !mdbx_search_spilled(parent, pgno));
+ parent_dp = debug_dpl_find(parent, pgno);
+ }
+ mdbx_tassert(txn, parent_dp && (!mp || parent_dp == mp));
+ }
+ /* Страница была выделена в родительÑкой транзакции и теперь может быть
+ * иÑпользована повторно, но только внутри Ñтой транзакции, либо дочерних.
+ */
+ goto reclaim;
+ }
+
+ /* Страница может входить в доÑтупный читателÑм MVCC-Ñнимок, либо же она
+ * могла быть выделена, а затем пролита в одной из родительÑких
+ * транзакций. ПоÑтому пока помещаем её в retired-ÑпиÑок, который будет
+ * фильтроватьÑÑ Ð¾Ñ‚Ð½Ð¾Ñительно dirty- и spilled-ÑпиÑков родительÑких
+ * транзакций при коммите дочерних транзакций, либо же будет запиÑан
+ * в GC в неизменном виде. */
+ goto retire;
+}
+
+static __inline int mdbx_page_retire(MDBX_cursor *mc, MDBX_page *mp) {
+ return mdbx_page_retire_ex(mc, mp->mp_pgno, mp, PAGETYPE(mp));
+}
+
+struct mdbx_iov_ctx {
+ unsigned iov_items;
+ size_t iov_bytes;
+ size_t iov_off;
+ pgno_t flush_begin;
+ pgno_t flush_end;
+ struct iovec iov[MDBX_COMMIT_PAGES];
+};
+
+static __inline void mdbx_iov_init(MDBX_txn *const txn,
+ struct mdbx_iov_ctx *ctx) {
+ ctx->flush_begin = MAX_PAGENO;
+ ctx->flush_end = MIN_PAGENO;
+ ctx->iov_items = 0;
+ ctx->iov_bytes = 0;
+ ctx->iov_off = 0;
+ (void)txn;
+}
+
+static __inline void mdbx_iov_done(MDBX_txn *const txn,
+ struct mdbx_iov_ctx *ctx) {
+ mdbx_tassert(txn, ctx->iov_items == 0);
+#if defined(__linux__) || defined(__gnu_linux__)
+ MDBX_env *const env = txn->mt_env;
+ if (!(txn->mt_flags & MDBX_WRITEMAP) &&
+ mdbx_linux_kernel_version < 0x02060b00)
+ /* Linux kernels older than version 2.6.11 ignore the addr and nbytes
+ * arguments, making this function fairly expensive. Therefore, the
+ * whole cache is always flushed. */
+ mdbx_flush_incoherent_mmap(
+ env->me_map + pgno2bytes(env, ctx->flush_begin),
+ pgno2bytes(env, ctx->flush_end - ctx->flush_begin), env->me_os_psize);
+#endif /* Linux */
+}
+
+static int mdbx_iov_write(MDBX_txn *const txn, struct mdbx_iov_ctx *ctx) {
+ mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
+ mdbx_tassert(txn, ctx->iov_items > 0);
+
+ MDBX_env *const env = txn->mt_env;
+ int rc;
+ if (likely(ctx->iov_items == 1)) {
+ mdbx_assert(env, ctx->iov_bytes == (size_t)ctx->iov[0].iov_len);
+ rc = mdbx_pwrite(env->me_lazy_fd, ctx->iov[0].iov_base, ctx->iov[0].iov_len,
+ ctx->iov_off);
+ } else {
+ rc = mdbx_pwritev(env->me_lazy_fd, ctx->iov, ctx->iov_items, ctx->iov_off,
+ ctx->iov_bytes);
+ }
+
+ if (unlikely(rc != MDBX_SUCCESS))
+ mdbx_error("Write error: %s", mdbx_strerror(rc));
+ else {
+ VALGRIND_MAKE_MEM_DEFINED(txn->mt_env->me_map + ctx->iov_off,
+ ctx->iov_bytes);
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(txn->mt_env->me_map + ctx->iov_off,
+ ctx->iov_bytes);
+ }
+
+ unsigned iov_items = ctx->iov_items;
+#if MDBX_ENABLE_PGOP_STAT
+ txn->mt_env->me_lck->mti_pgop_stat.wops.weak += iov_items;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ ctx->iov_items = 0;
+ ctx->iov_bytes = 0;
+
+ uint64_t timestamp = 0;
+ for (unsigned i = 0; i < iov_items; i++) {
+ MDBX_page *wp = (MDBX_page *)ctx->iov[i].iov_base;
+ const MDBX_page *rp = pgno2page(txn->mt_env, wp->mp_pgno);
+ /* check with timeout as the workaround
+ * for todo4recovery://erased_by_github/libmdbx/issues/269 */
+ while (likely(rc == MDBX_SUCCESS) &&
+ unlikely(memcmp(wp, rp, ctx->iov[i].iov_len) != 0)) {
+ if (!timestamp) {
+ timestamp = mdbx_osal_monotime();
+ mdbx_iov_done(txn, ctx);
+ mdbx_warning(
+ "catch delayed/non-arrived page %" PRIaPGNO " %s", wp->mp_pgno,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ } else if (unlikely(mdbx_osal_monotime() - timestamp > 65536 / 10)) {
+ mdbx_error(
+ "bailout waiting for %" PRIaPGNO " page arrival %s", wp->mp_pgno,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ rc = MDBX_CORRUPTED;
+ }
+#if defined(_WIN32) || defined(_WIN64)
+ SwitchToThread();
+#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
+ sched_yield();
+#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
+ pthread_yield();
+#else
+ usleep(42);
+#endif
+ }
+ mdbx_dpage_free(env, wp, bytes2pgno(env, ctx->iov[i].iov_len));
+ }
+ return rc;
+}
+
+static int iov_page(MDBX_txn *txn, struct mdbx_iov_ctx *ctx, MDBX_page *dp,
+ unsigned npages) {
+ MDBX_env *const env = txn->mt_env;
+ mdbx_tassert(txn,
+ dp->mp_pgno >= MIN_PAGENO && dp->mp_pgno < txn->mt_next_pgno);
+ mdbx_tassert(txn, IS_MODIFIABLE(txn, dp));
+ mdbx_tassert(txn,
+ !(dp->mp_flags & ~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)));
+
+ ctx->flush_begin =
+ (ctx->flush_begin < dp->mp_pgno) ? ctx->flush_begin : dp->mp_pgno;
+ ctx->flush_end = (ctx->flush_end > dp->mp_pgno + npages)
+ ? ctx->flush_end
+ : dp->mp_pgno + npages;
+ env->me_lck->mti_unsynced_pages.weak += npages;
+
+ if (IS_SHADOWED(txn, dp)) {
+ mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
+ dp->mp_txnid = txn->mt_txnid;
+ mdbx_tassert(txn, IS_SPILLED(txn, dp));
+ const size_t size = pgno2bytes(env, npages);
+ if (ctx->iov_off + ctx->iov_bytes != pgno2bytes(env, dp->mp_pgno) ||
+ ctx->iov_items == ARRAY_LENGTH(ctx->iov) ||
+ ctx->iov_bytes + size > MAX_WRITE) {
+ if (ctx->iov_items) {
+ int err = mdbx_iov_write(txn, ctx);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+#if defined(__linux__) || defined(__gnu_linux__)
+ if (mdbx_linux_kernel_version >= 0x02060b00)
+ /* Linux kernels older than version 2.6.11 ignore the addr and nbytes
+ * arguments, making this function fairly expensive. Therefore, the
+ * whole cache is always flushed. */
+#endif /* Linux */
+ mdbx_flush_incoherent_mmap(env->me_map + ctx->iov_off, ctx->iov_bytes,
+ env->me_os_psize);
+ }
+ ctx->iov_off = pgno2bytes(env, dp->mp_pgno);
+ }
+ ctx->iov[ctx->iov_items].iov_base = (void *)dp;
+ ctx->iov[ctx->iov_items].iov_len = size;
+ ctx->iov_items += 1;
+ ctx->iov_bytes += size;
+ } else {
+ mdbx_tassert(txn, txn->mt_flags & MDBX_WRITEMAP);
+ }
+ return MDBX_SUCCESS;
+}
+
+static int spill_page(MDBX_txn *txn, struct mdbx_iov_ctx *ctx, MDBX_page *dp,
+ unsigned npages) {
+ mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
+ pgno_t pgno = dp->mp_pgno;
+ int err = iov_page(txn, ctx, dp, npages);
+ if (likely(err == MDBX_SUCCESS)) {
+ err = mdbx_pnl_append_range(true, &txn->tw.spill_pages, pgno << 1, npages);
+#if MDBX_ENABLE_PGOP_STAT
+ if (likely(err == MDBX_SUCCESS))
+ txn->mt_env->me_lck->mti_pgop_stat.spill.weak += npages;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ }
+ return err;
+}
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- if (MDBX_DEBUG != 0 || unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB))
-#endif
- mdbx_kill_page(txn, mp, pgno, npages);
- if (!(txn->mt_flags & MDBX_WRITEMAP)) {
- VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->mt_env, pgno)),
- pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ);
- MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->mt_env, pgno)),
- pgno2bytes(txn->mt_env, npages) -
- PAGEHDRSZ);
+/* Set unspillable LRU-label for dirty pages watched by txn.
+ * Returns the number of pages marked as unspillable. */
+static unsigned mdbx_cursor_keep(MDBX_txn *txn, MDBX_cursor *mc) {
+ unsigned keep = 0;
+ while (mc->mc_flags & C_INITIALIZED) {
+ for (unsigned i = 0; i < mc->mc_snum; ++i) {
+ const MDBX_page *mp = mc->mc_pg[i];
+ if (IS_MODIFIABLE(txn, mp) && !IS_SUBP(mp)) {
+ unsigned const n = mdbx_dpl_search(txn, mp->mp_pgno);
+ if (txn->tw.dirtylist->items[n].pgno == mp->mp_pgno &&
+ mdbx_dpl_age(txn, n)) {
+ txn->tw.dirtylist->items[n].lru = txn->tw.dirtylru;
+ ++keep;
+ }
}
}
- skip_invalidate:
- /* Remove from dirty list */
- mdbx_page_wash(txn, di, mp, npages);
+ if (!mc->mc_xcursor)
+ break;
+ mc = &mc->mc_xcursor->mx_cursor;
+ }
+ return keep;
+}
- reclaim:
- mdbx_debug("reclaim %u %s page %" PRIaPGNO, npages, "dirty", pgno);
- rc = mdbx_pnl_insert_range(&txn->tw.reclaimed_pglist, pgno, npages);
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- return rc;
+static unsigned mdbx_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
+ unsigned keep = m0 ? mdbx_cursor_keep(txn, m0) : 0;
+ for (unsigned i = FREE_DBI; i < txn->mt_numdbs; ++i)
+ if (F_ISSET(txn->mt_dbistate[i], DBI_DIRTY | DBI_VALID) &&
+ txn->mt_dbs[i].md_root != P_INVALID)
+ for (MDBX_cursor *mc = txn->mt_cursors[i]; mc; mc = mc->mc_next)
+ if (mc != m0)
+ keep += mdbx_cursor_keep(txn, mc);
+ return keep;
+}
+
+/* Returns the spilling priority (0..255) for a dirty page:
+ * 0 = should be spilled;
+ * ...
+ * > 255 = must not be spilled. */
+static unsigned spill_prio(const MDBX_txn *txn, const unsigned i,
+ const uint32_t reciprocal) {
+ MDBX_dpl *const dl = txn->tw.dirtylist;
+ const uint32_t age = mdbx_dpl_age(txn, i);
+ const unsigned npages = dpl_npages(dl, i);
+ const pgno_t pgno = dl->items[i].pgno;
+ if (age == 0) {
+ mdbx_debug("skip %s %u page %" PRIaPGNO, "keep", npages, pgno);
+ return 256;
}
- if (si) {
- /* Page ws spilled in this txn */
- mdbx_spill_remove(txn, si, npages);
- /* Страница могла быть выделена и затем пролита в Ñтой транзакции,
- * тогда её необходимо помеÑтить в reclaimed-ÑпиÑок.
- * Либо она могла быть выделена в одной из родительÑких транзакций и затем
- * пролита в Ñтой транзакции, тогда её необходимо помеÑтить в
- * retired-ÑпиÑок Ð´Ð»Ñ Ð¿Ð¾Ñледующей фильтрации при коммите. */
- for (MDBX_txn *parent = txn->mt_parent; parent;
- parent = parent->mt_parent) {
- if (mdbx_dpl_exist(parent, pgno))
- goto retire;
- }
- /* Страница точно была выделена в Ñтой транзакции
- * и теперь может быть иÑпользована повторно. */
- goto reclaim;
+ MDBX_page *const dp = dl->items[i].ptr;
+ if (dp->mp_flags & (P_LOOSE | P_SPILLED)) {
+ mdbx_debug("skip %s %u page %" PRIaPGNO,
+ (dp->mp_flags & P_LOOSE) ? "loose"
+ : (dp->mp_flags & P_LOOSE) ? "loose"
+ : "parent-spilled",
+ npages, pgno);
+ return 256;
}
- if (is_shadowed) {
- /* Dirty page MUST BE a clone from (one of) parent transaction(s). */
- if (mdbx_assert_enabled()) {
- const MDBX_page *parent_dp = nullptr;
- /* Check parent(s)'s dirty lists. */
- for (MDBX_txn *parent = txn->mt_parent; parent && !parent_dp;
- parent = parent->mt_parent) {
- mdbx_tassert(txn, !mdbx_search_spilled(parent, pgno));
- parent_dp = debug_dpl_find(parent, pgno);
+ /* Can't spill twice,
+ * make sure it's not already in a parent's spill list(s). */
+ MDBX_txn *parent = txn->mt_parent;
+ if (parent && (parent->mt_flags & MDBX_TXN_SPILLS)) {
+ do
+ if (mdbx_intersect_spilled(parent, pgno, npages)) {
+ mdbx_debug("skip-2 parent-spilled %u page %" PRIaPGNO, npages, pgno);
+ dp->mp_flags |= P_SPILLED;
+ return 256;
}
- mdbx_tassert(txn, parent_dp && (!mp || parent_dp == mp));
- }
- /* Страница была выделена в родительÑкой транзакции и теперь может быть
- * иÑпользована повторно, но только внутри Ñтой транзакции, либо дочерних.
- */
- goto reclaim;
+ while ((parent = parent->mt_parent) != nullptr);
}
- /* Страница может входить в доÑтупный читателÑм MVCC-Ñнимок, либо же она
- * могла быть выделена, а затем пролита в одной из родительÑких
- * транзакций. ПоÑтому пока помещаем её в retired-ÑпиÑок, который будет
- * фильтроватьÑÑ Ð¾Ñ‚Ð½Ð¾Ñительно dirty- и spilled-ÑпиÑков родительÑких
- * транзакций при коммите дочерних транзакций, либо же будет запиÑан
- * в GC в неизменном виде. */
- goto retire;
-}
+ mdbx_tassert(txn, age * (uint64_t)reciprocal < UINT32_MAX);
+ unsigned prio = age * reciprocal >> 24;
+ mdbx_tassert(txn, prio < 256);
+ if (likely(npages == 1))
+ return prio = 256 - prio;
-static __inline int mdbx_page_retire(MDBX_cursor *mc, MDBX_page *mp) {
- return mdbx_page_retire_ex(mc, mp->mp_pgno, mp, PAGETYPE(mp));
+ /* make a large/overflow pages be likely to spill */
+ uint32_t factor = npages | npages >> 1;
+ factor |= factor >> 2;
+ factor |= factor >> 4;
+ factor |= factor >> 8;
+ factor |= factor >> 16;
+ factor = prio * log2n_powerof2(factor + 1) + /* golden ratio */ 157;
+ factor = (factor < 256) ? 255 - factor : 0;
+ mdbx_tassert(txn, factor < 256 && factor < (256 - prio));
+ return prio = factor;
}
-struct mdbx_iov_ctx {
- unsigned iov_items;
- size_t iov_bytes;
- size_t iov_off;
- pgno_t flush_begin;
- pgno_t flush_end;
- struct iovec iov[MDBX_COMMIT_PAGES];
-};
+/* Spill pages from the dirty list back to disk.
+ * This is intended to prevent running into MDBX_TXN_FULL situations,
+ * but note that they may still occur in a few cases:
+ *
+ * 1) our estimate of the txn size could be too small. Currently this
+ * seems unlikely, except with a large number of MDBX_MULTIPLE items.
+ *
+ * 2) child txns may run out of space if their parents dirtied a
+ * lot of pages and never spilled them. TODO: we probably should do
+ * a preemptive spill during mdbx_txn_begin() of a child txn, if
+ * the parent's dirtyroom is below a given threshold.
+ *
+ * Otherwise, if not using nested txns, it is expected that apps will
+ * not run into MDBX_TXN_FULL any more. The pages are flushed to disk
+ * the same way as for a txn commit, e.g. their dirty status is cleared.
+ * If the txn never references them again, they can be left alone.
+ * If the txn only reads them, they can be used without any fuss.
+ * If the txn writes them again, they can be dirtied immediately without
+ * going thru all of the work of mdbx_page_touch(). Such references are
+ * handled by mdbx_page_unspill().
+ *
+ * Also note, we never spill DB root pages, nor pages of active cursors,
+ * because we'll need these back again soon anyway. And in nested txns,
+ * we can't spill a page in a child txn if it was already spilled in a
+ * parent txn. That would alter the parent txns' data even though
+ * the child hasn't committed yet, and we'd have no way to undo it if
+ * the child aborted. */
+static int mdbx_txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0,
+ const unsigned need) {
+#if xMDBX_DEBUG_SPILLING != 1
+ /* production mode */
+ if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need))
+ return MDBX_SUCCESS;
+ unsigned wanna_spill = need - txn->tw.dirtyroom;
+#else
+ /* debug mode: spill at least one page if xMDBX_DEBUG_SPILLING == 1 */
+ unsigned wanna_spill =
+ (need > txn->tw.dirtyroom) ? need - txn->tw.dirtyroom : 1;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ const unsigned dirty = txn->tw.dirtylist->length;
+ const unsigned spill_min =
+ txn->mt_env->me_options.spill_min_denominator
+ ? dirty / txn->mt_env->me_options.spill_min_denominator
+ : 0;
+ const unsigned spill_max =
+ dirty - (txn->mt_env->me_options.spill_max_denominator
+ ? dirty / txn->mt_env->me_options.spill_max_denominator
+ : 0);
+ wanna_spill = (wanna_spill > spill_min) ? wanna_spill : spill_min;
+ wanna_spill = (wanna_spill < spill_max) ? wanna_spill : spill_max;
+ if (!wanna_spill)
+ return MDBX_SUCCESS;
+
+ mdbx_notice("spilling %u dirty-entries (have %u dirty-room, need %u)",
+ wanna_spill, txn->tw.dirtyroom, need);
+ mdbx_tassert(txn, txn->tw.dirtylist->length >= wanna_spill);
+
+ struct mdbx_iov_ctx ctx;
+ mdbx_iov_init(txn, &ctx);
+ int rc = MDBX_SUCCESS;
+ if (txn->mt_flags & MDBX_WRITEMAP) {
+ MDBX_dpl *const dl = txn->tw.dirtylist;
+ const unsigned span = dl->length - txn->tw.loose_count;
+ txn->tw.dirtyroom += span;
+ unsigned r, w;
+ for (w = 0, r = 1; r <= dl->length; ++r) {
+ MDBX_page *dp = dl->items[r].ptr;
+ if (dp->mp_flags & P_LOOSE)
+ dl->items[++w] = dl->items[r];
+ else if (!MDBX_FAKE_SPILL_WRITEMAP) {
+ rc = iov_page(txn, &ctx, dp, dpl_npages(dl, r));
+ mdbx_tassert(txn, rc == MDBX_SUCCESS);
+ }
+ }
-static __inline void mdbx_iov_init(MDBX_txn *const txn,
- struct mdbx_iov_ctx *ctx) {
- ctx->flush_begin = MAX_PAGENO;
- ctx->flush_end = MIN_PAGENO;
- ctx->iov_items = 0;
- ctx->iov_bytes = 0;
- ctx->iov_off = 0;
- (void)txn;
-}
+ mdbx_tassert(txn, span == r - 1 - w && w == txn->tw.loose_count);
+ dl->sorted = (dl->sorted == dl->length) ? w : 0;
+ dpl_setlen(dl, w);
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
-static __inline void mdbx_iov_done(MDBX_txn *const txn,
- struct mdbx_iov_ctx *ctx) {
- mdbx_tassert(txn, ctx->iov_items == 0);
-#if defined(__linux__) || defined(__gnu_linux__)
- MDBX_env *const env = txn->mt_env;
- if (!(txn->mt_flags & MDBX_WRITEMAP) &&
- mdbx_linux_kernel_version < 0x02060b00)
- /* Linux kernels older than version 2.6.11 ignore the addr and nbytes
- * arguments, making this function fairly expensive. Therefore, the
- * whole cache is always flushed. */
- mdbx_flush_incoherent_mmap(
- env->me_map + pgno2bytes(env, ctx->flush_begin),
- pgno2bytes(env, ctx->flush_end - ctx->flush_begin), env->me_os_psize);
-#endif /* Linux */
-}
+ if (!MDBX_FAKE_SPILL_WRITEMAP && ctx.flush_end > ctx.flush_begin) {
+ MDBX_env *const env = txn->mt_env;
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ rc = mdbx_msync(&env->me_dxb_mmap,
+ pgno_align2os_bytes(env, ctx.flush_begin),
+ pgno_align2os_bytes(env, ctx.flush_end - ctx.flush_begin),
+ MDBX_SYNC_NONE);
+ }
+ return rc;
+ }
-static int mdbx_iov_write(MDBX_txn *const txn, struct mdbx_iov_ctx *ctx) {
mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
- mdbx_tassert(txn, ctx->iov_items > 0);
-
- MDBX_env *const env = txn->mt_env;
- int rc;
- if (likely(ctx->iov_items == 1)) {
- mdbx_assert(env, ctx->iov_bytes == (size_t)ctx->iov[0].iov_len);
- rc = mdbx_pwrite(env->me_lazy_fd, ctx->iov[0].iov_base, ctx->iov[0].iov_len,
- ctx->iov_off);
+ if (!txn->tw.spill_pages) {
+ txn->tw.spill_least_removed = INT_MAX;
+ txn->tw.spill_pages = mdbx_pnl_alloc(wanna_spill);
+ if (unlikely(!txn->tw.spill_pages)) {
+ rc = MDBX_ENOMEM;
+ bailout:
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ return rc;
+ }
} else {
- rc = mdbx_pwritev(env->me_lazy_fd, ctx->iov, ctx->iov_items, ctx->iov_off,
- ctx->iov_bytes);
+ /* purge deleted slots */
+ mdbx_spill_purge(txn);
+ rc = mdbx_pnl_reserve(&txn->tw.spill_pages, wanna_spill);
+ (void)rc /* ignore since the resulting list may be shorter
+ and mdbx_pnl_append() will increase pnl on demand */
+ ;
}
- if (unlikely(rc != MDBX_SUCCESS))
- mdbx_error("Write error: %s", mdbx_strerror(rc));
- else {
- VALGRIND_MAKE_MEM_DEFINED(txn->mt_env->me_map + ctx->iov_off,
- ctx->iov_bytes);
- MDBX_ASAN_UNPOISON_MEMORY_REGION(txn->mt_env->me_map + ctx->iov_off,
- ctx->iov_bytes);
+ /* Сортируем чтобы запиÑÑŒ на диÑк была полее поÑледовательна */
+ MDBX_dpl *const dl = mdbx_dpl_sort(txn);
+
+ /* Preserve pages which may soon be dirtied again */
+ const unsigned unspillable = mdbx_txn_keep(txn, m0);
+ if (unspillable + txn->tw.loose_count >= dl->length) {
+#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */
+ if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need))
+ return MDBX_SUCCESS;
+#endif /* xMDBX_DEBUG_SPILLING */
+ mdbx_error("all %u dirty pages are unspillable since referenced "
+ "by a cursor(s), use fewer cursors or increase "
+ "MDBX_opt_txn_dp_limit",
+ unspillable);
+ goto done;
}
- unsigned iov_items = ctx->iov_items;
-#if MDBX_ENABLE_PGOP_STAT
- txn->mt_env->me_lck->mti_pgop_stat.wops.weak += iov_items;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- ctx->iov_items = 0;
- ctx->iov_bytes = 0;
+ /* Подзадача: Вытолкнуть чаÑÑ‚ÑŒ Ñтраниц на диÑк в ÑоответÑтвии Ñ LRU,
+ * но при Ñтом учеÑÑ‚ÑŒ важные поправки:
+ * - лучше выталкивать Ñтарые large/overflow Ñтраницы, так будет оÑвобождено
+ * больше памÑти, а также так как они (в текущем понимании) гораздо реже
+ * повторно изменÑÑŽÑ‚ÑÑ;
+ * - при прочих равных лучше выталкивать Ñмежные Ñтраницы, так будет
+ * меньше I/O операций;
+ * - желательно потратить на Ñто меньше времени чем std::partial_sort_copy;
+ *
+ * Решение:
+ * - Квантуем веÑÑŒ диапазон lru-меток до 256 значений и задейÑтвуем один
+ * проход 8-битного radix-sort. В результате получаем 256 уровней
+ * "ÑвежеÑти", в том чиÑле значение lru-метки, Ñтарее которой Ñтраницы
+ * должны быть выгружены;
+ * - ДвигаемÑÑ Ð¿Ð¾Ñледовательно в Ñторону ÑƒÐ²ÐµÐ»Ð¸Ñ‡ÐµÐ½Ð¸Ñ Ð½Ð¾Ð¼ÐµÑ€Ð¾Ð² Ñтраниц
+ * и выталкиваем Ñтраницы Ñ lru-меткой Ñтарее отÑекающего значениÑ,
+ * пока не вытолкнем доÑтаточно;
+ * - Ð’ÑÑ‚Ñ€ÐµÑ‡Ð°Ñ Ñтраницы Ñмежные Ñ Ð²Ñ‹Ñ‚Ð°Ð»ÐºÐ¸Ð²Ð°ÐµÐ¼Ñ‹Ð¼Ð¸ Ð´Ð»Ñ ÑƒÐ¼ÐµÐ½ÑŒÑˆÐµÐ½Ð¸Ñ ÐºÐ¾Ð»-ва
+ * I/O операций выталкиваем и их, еÑли они попадают в первую половину
+ * между выталкиваемыми и Ñамыми Ñвежими lru-метками;
+ * - дополнительно при Ñортировке умышленно Ñтарим large/overflow Ñтраницы,
+ * тем Ñамым Ð¿Ð¾Ð²Ñ‹ÑˆÐ°Ñ Ð¸Ñ… шанÑÑ‹ на выталкивание. */
- uint64_t timestamp = 0;
- for (unsigned i = 0; i < iov_items; i++) {
- MDBX_page *wp = (MDBX_page *)ctx->iov[i].iov_base;
- const MDBX_page *rp = pgno2page(txn->mt_env, wp->mp_pgno);
- /* check with timeout as the workaround
- * for todo4recovery://erased_by_github/libmdbx/issues/269 */
- while (likely(rc == MDBX_SUCCESS) &&
- unlikely(memcmp(wp, rp, ctx->iov[i].iov_len) != 0)) {
- if (!timestamp) {
- timestamp = mdbx_osal_monotime();
- mdbx_iov_done(txn, ctx);
- mdbx_warning(
- "catch delayed/non-arrived page %" PRIaPGNO " %s", wp->mp_pgno,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- } else if (unlikely(mdbx_osal_monotime() - timestamp > 65536 / 10)) {
- mdbx_error(
- "bailout waiting for %" PRIaPGNO " page arrival %s", wp->mp_pgno,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- rc = MDBX_CORRUPTED;
- }
-#if defined(_WIN32) || defined(_WIN64)
- SwitchToThread();
-#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
- sched_yield();
-#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
- pthread_yield();
-#else
- usleep(42);
-#endif
+ /* get min/max of LRU-labels */
+ uint32_t age_max = 0;
+ for (unsigned i = 1; i <= dl->length; ++i) {
+ const uint32_t age = mdbx_dpl_age(txn, i);
+ age_max = (age_max >= age) ? age_max : age;
+ }
+
+ mdbx_verbose("lru-head %u, age-max %u", txn->tw.dirtylru, age_max);
+
+ /* half of 8-bit radix-sort */
+ unsigned radix_counters[256], spillable = 0, spilled = 0;
+ memset(&radix_counters, 0, sizeof(radix_counters));
+ const uint32_t reciprocal = (UINT32_C(255) << 24) / (age_max + 1);
+ for (unsigned i = 1; i <= dl->length; ++i) {
+ unsigned prio = spill_prio(txn, i, reciprocal);
+ if (prio < 256) {
+ radix_counters[prio] += 1;
+ spillable += 1;
}
- mdbx_dpage_free(env, wp, bytes2pgno(env, ctx->iov[i].iov_len));
}
- return rc;
-}
-static int iov_page(MDBX_txn *txn, struct mdbx_iov_ctx *ctx, MDBX_page *dp,
- unsigned npages) {
- MDBX_env *const env = txn->mt_env;
- mdbx_tassert(txn,
- dp->mp_pgno >= MIN_PAGENO && dp->mp_pgno < txn->mt_next_pgno);
- mdbx_tassert(txn, IS_MODIFIABLE(txn, dp));
- mdbx_tassert(txn,
- !(dp->mp_flags & ~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)));
+ if (likely(spillable > 0)) {
+ unsigned prio2spill = 0, prio2adjacent = 128, amount = radix_counters[0];
+ for (unsigned i = 1; i < 256; i++) {
+ if (amount < wanna_spill) {
+ prio2spill = i;
+ prio2adjacent = i + (257 - i) / 2;
+ amount += radix_counters[i];
+ } else if (amount + amount < spillable + wanna_spill
+ /* Ð ÐÐ’ÐОЗÐÐЧÐО: amount - wanna_spill < spillable - amount */) {
+ prio2adjacent = i;
+ amount += radix_counters[i];
+ } else
+ break;
+ }
- ctx->flush_begin =
- (ctx->flush_begin < dp->mp_pgno) ? ctx->flush_begin : dp->mp_pgno;
- ctx->flush_end = (ctx->flush_end > dp->mp_pgno + npages)
- ? ctx->flush_end
- : dp->mp_pgno + npages;
- env->me_lck->mti_unsynced_pages.weak += npages;
+ mdbx_verbose("prio2spill %u, prio2adjacent %u, amount %u, spillable %u, "
+ "wanna_spill %u",
+ prio2spill, prio2adjacent, amount, spillable, wanna_spill);
+ mdbx_tassert(txn, prio2spill < prio2adjacent && prio2adjacent <= 256);
- if (IS_SHADOWED(txn, dp)) {
- mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
- dp->mp_txnid = txn->mt_txnid;
- mdbx_tassert(txn, IS_SPILLED(txn, dp));
- const size_t size = pgno2bytes(env, npages);
- if (ctx->iov_off + ctx->iov_bytes != pgno2bytes(env, dp->mp_pgno) ||
- ctx->iov_items == ARRAY_LENGTH(ctx->iov) ||
- ctx->iov_bytes + size > MAX_WRITE) {
- if (ctx->iov_items) {
- int err = mdbx_iov_write(txn, ctx);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-#if defined(__linux__) || defined(__gnu_linux__)
- if (mdbx_linux_kernel_version >= 0x02060b00)
- /* Linux kernels older than version 2.6.11 ignore the addr and nbytes
- * arguments, making this function fairly expensive. Therefore, the
- * whole cache is always flushed. */
-#endif /* Linux */
- mdbx_flush_incoherent_mmap(env->me_map + ctx->iov_off, ctx->iov_bytes,
- env->me_os_psize);
+ unsigned prev_prio = 256;
+ unsigned r, w, prio;
+ for (w = 0, r = 1; r <= dl->length && spilled < wanna_spill;
+ prev_prio = prio, ++r) {
+ prio = spill_prio(txn, r, reciprocal);
+ MDBX_page *const dp = dl->items[r].ptr;
+ if (prio < prio2adjacent) {
+ const pgno_t pgno = dl->items[r].pgno;
+ const unsigned npages = dpl_npages(dl, r);
+ if (prio <= prio2spill) {
+ if (prev_prio < prio2adjacent && prev_prio > prio2spill &&
+ dpl_endpgno(dl, r - 1) == pgno) {
+ mdbx_debug("co-spill %u prev-adjacent page %" PRIaPGNO
+ " (age %d, prio %u)",
+ dpl_npages(dl, w), dl->items[r - 1].pgno,
+ mdbx_dpl_age(txn, r - 1), prev_prio);
+ --w;
+ rc = spill_page(txn, &ctx, dl->items[r - 1].ptr,
+ dpl_npages(dl, r - 1));
+ if (unlikely(rc != MDBX_SUCCESS))
+ break;
+ ++spilled;
+ }
+
+ mdbx_debug("spill %u page %" PRIaPGNO " (age %d, prio %u)", npages,
+ dp->mp_pgno, mdbx_dpl_age(txn, r), prio);
+ rc = spill_page(txn, &ctx, dp, npages);
+ if (unlikely(rc != MDBX_SUCCESS))
+ break;
+ ++spilled;
+ continue;
+ }
+
+ if (prev_prio <= prio2spill && dpl_endpgno(dl, r - 1) == pgno) {
+ mdbx_debug("co-spill %u next-adjacent page %" PRIaPGNO
+ " (age %d, prio %u)",
+ npages, dp->mp_pgno, mdbx_dpl_age(txn, r), prio);
+ rc = spill_page(txn, &ctx, dp, npages);
+ if (unlikely(rc != MDBX_SUCCESS))
+ break;
+ prio = prev_prio /* to continue co-spilling next adjacent pages */;
+ ++spilled;
+ continue;
+ }
}
- ctx->iov_off = pgno2bytes(env, dp->mp_pgno);
+ dl->items[++w] = dl->items[r];
}
- ctx->iov[ctx->iov_items].iov_base = (void *)dp;
- ctx->iov[ctx->iov_items].iov_len = size;
- ctx->iov_items += 1;
- ctx->iov_bytes += size;
+
+ mdbx_tassert(txn, spillable == 0 || spilled > 0);
+
+ while (r <= dl->length)
+ dl->items[++w] = dl->items[r++];
+ mdbx_tassert(txn, r - 1 - w == spilled);
+
+ dl->sorted = dpl_setlen(dl, w);
+ txn->tw.dirtyroom += spilled;
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+
+ if (ctx.iov_items)
+ rc = mdbx_iov_write(txn, &ctx);
+
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+
+ mdbx_pnl_sort(txn->tw.spill_pages, (size_t)txn->mt_next_pgno << 1);
+ txn->mt_flags |= MDBX_TXN_SPILLS;
+ mdbx_notice("spilled %u dirty-entries, now have %u dirty-room", spilled,
+ txn->tw.dirtyroom);
+ mdbx_iov_done(txn, &ctx);
} else {
- mdbx_tassert(txn, txn->mt_flags & MDBX_WRITEMAP);
+ mdbx_tassert(txn, ctx.iov_items == 0 && rc == MDBX_SUCCESS);
+ for (unsigned i = 1; i <= dl->length; ++i) {
+ MDBX_page *dp = dl->items[i].ptr;
+ mdbx_notice(
+ "dirtylist[%u]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i,
+ dp->mp_pgno, dpl_npages(dl, i), dp->mp_flags, mdbx_dpl_age(txn, i),
+ spill_prio(txn, i, reciprocal));
+ }
}
- return MDBX_SUCCESS;
+
+#if xMDBX_DEBUG_SPILLING == 2
+ if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1)
+ mdbx_error("dirty-list length: before %u, after %u, parent %i, loose %u; "
+ "needed %u, spillable %u; "
+ "spilled %u dirty-entries, now have %u dirty-room",
+ dl->length + spilled, dl->length,
+ (txn->mt_parent && txn->mt_parent->tw.dirtylist)
+ ? (int)txn->mt_parent->tw.dirtylist->length
+ : -1,
+ txn->tw.loose_count, need, spillable, spilled,
+ txn->tw.dirtyroom);
+ mdbx_ensure(txn->mt_env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2);
+#endif /* xMDBX_DEBUG_SPILLING */
+
+done:
+ return likely(txn->tw.dirtyroom + txn->tw.loose_count >
+ ((need > CURSOR_STACK) ? CURSOR_STACK : need))
+ ? MDBX_SUCCESS
+ : MDBX_TXN_FULL;
}
-static int spill_page(MDBX_txn *txn, struct mdbx_iov_ctx *ctx, MDBX_page *dp,
- unsigned npages) {
- mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
- pgno_t pgno = dp->mp_pgno;
- int err = iov_page(txn, ctx, dp, npages);
- if (likely(err == MDBX_SUCCESS)) {
- err = mdbx_pnl_append_range(true, &txn->tw.spill_pages, pgno << 1, npages);
-#if MDBX_ENABLE_PGOP_STAT
- if (likely(err == MDBX_SUCCESS))
- txn->mt_env->me_lck->mti_pgop_stat.spill.weak += npages;
-#endif /* MDBX_ENABLE_PGOP_STAT */
+static int mdbx_cursor_spill(MDBX_cursor *mc, const MDBX_val *key,
+ const MDBX_val *data) {
+ MDBX_txn *txn = mc->mc_txn;
+ /* Estimate how much space this operation will take: */
+ /* 1) Max b-tree height, reasonable enough with including dups' sub-tree */
+ unsigned need = CURSOR_STACK + 3;
+ /* 2) GC/FreeDB for any payload */
+ if (mc->mc_dbi > FREE_DBI) {
+ need += txn->mt_dbs[FREE_DBI].md_depth + 3;
+ /* 3) Named DBs also dirty the main DB */
+ if (mc->mc_dbi > MAIN_DBI)
+ need += txn->mt_dbs[MAIN_DBI].md_depth + 3;
}
- return err;
+#if xMDBX_DEBUG_SPILLING != 2
+ /* production mode */
+ /* 4) Double the page chain estimation
+ * for extensively splitting, rebalance and merging */
+ need += need;
+ /* 5) Factor the key+data which to be put in */
+ need += bytes2pgno(txn->mt_env, node_size(key, data)) + 1;
+#else
+ /* debug mode */
+ (void)key;
+ (void)data;
+ mc->mc_txn->mt_env->debug_dirtied_est = ++need;
+ mc->mc_txn->mt_env->debug_dirtied_act = 0;
+#endif /* xMDBX_DEBUG_SPILLING == 2 */
+
+ return mdbx_txn_spill(txn, mc, need);
}
-/* Set unspillable LRU-label for dirty pages watched by txn.
- * Returns the number of pages marked as unspillable. */
-static unsigned mdbx_cursor_keep(MDBX_txn *txn, MDBX_cursor *mc) {
- unsigned keep = 0;
- while (mc->mc_flags & C_INITIALIZED) {
- for (unsigned i = 0; i < mc->mc_snum; ++i) {
- const MDBX_page *mp = mc->mc_pg[i];
- if (IS_MODIFIABLE(txn, mp) && !IS_SUBP(mp)) {
- unsigned const n = mdbx_dpl_search(txn, mp->mp_pgno);
- if (txn->tw.dirtylist->items[n].pgno == mp->mp_pgno &&
- mdbx_dpl_age(txn, n)) {
- txn->tw.dirtylist->items[n].lru = txn->tw.dirtylru;
- ++keep;
- }
- }
- }
- if (!mc->mc_xcursor)
- break;
- mc = &mc->mc_xcursor->mx_cursor;
- }
- return keep;
+/*----------------------------------------------------------------------------*/
+
+static bool meta_bootid_match(const MDBX_meta *meta) {
+ return memcmp(&meta->mm_bootid, &bootid, 16) == 0 &&
+ (bootid.x | bootid.y) != 0;
}
-static unsigned mdbx_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
- unsigned keep = m0 ? mdbx_cursor_keep(txn, m0) : 0;
- for (unsigned i = FREE_DBI; i < txn->mt_numdbs; ++i)
- if (F_ISSET(txn->mt_dbistate[i], DBI_DIRTY | DBI_VALID) &&
- txn->mt_dbs[i].md_root != P_INVALID)
- for (MDBX_cursor *mc = txn->mt_cursors[i]; mc; mc = mc->mc_next)
- if (mc != m0)
- keep += mdbx_cursor_keep(txn, mc);
- return keep;
+static bool meta_weak_acceptable(const MDBX_env *env, const MDBX_meta *meta,
+ const int lck_exclusive) {
+ return lck_exclusive
+ ? /* exclusive lock */ meta_bootid_match(meta)
+ : /* db already opened */ env->me_lck_mmap.lck &&
+ (env->me_lck_mmap.lck->mti_envmode.weak & MDBX_RDONLY) == 0;
}
-/* Returns the spilling priority (0..255) for a dirty page:
- * 0 = should be spilled;
- * ...
- * > 255 = must not be spilled. */
-static unsigned spill_prio(const MDBX_txn *txn, const unsigned i,
- const uint32_t reciprocal) {
- MDBX_dpl *const dl = txn->tw.dirtylist;
- const uint32_t age = mdbx_dpl_age(txn, i);
- const unsigned npages = dpl_npages(dl, i);
- const pgno_t pgno = dl->items[i].pgno;
- if (age == 0) {
- mdbx_debug("skip %s %u page %" PRIaPGNO, "keep", npages, pgno);
- return 256;
- }
+#define METAPAGE(env, n) page_meta(pgno2page(env, n))
+#define METAPAGE_END(env) METAPAGE(env, NUM_METAS)
- MDBX_page *const dp = dl->items[i].ptr;
- if (dp->mp_flags & (P_LOOSE | P_SPILLED)) {
- mdbx_debug("skip %s %u page %" PRIaPGNO,
- (dp->mp_flags & P_LOOSE) ? "loose"
- : (dp->mp_flags & P_LOOSE) ? "loose"
- : "parent-spilled",
- npages, pgno);
- return 256;
- }
+MDBX_NOTHROW_PURE_FUNCTION static __inline txnid_t
+constmeta_txnid(const MDBX_env *env, const MDBX_meta *meta) {
+ mdbx_memory_fence(mo_AcquireRelease, false);
+ txnid_t a = unaligned_peek_u64(4, &meta->mm_txnid_a);
+ txnid_t b = unaligned_peek_u64(4, &meta->mm_txnid_b);
+ mdbx_assert(env, a == b);
+ (void)env;
+ return (a == b) ? a : 0;
+}
- /* Can't spill twice,
- * make sure it's not already in a parent's spill list(s). */
- MDBX_txn *parent = txn->mt_parent;
- if (parent && (parent->mt_flags & MDBX_TXN_SPILLS)) {
- do
- if (mdbx_intersect_spilled(parent, pgno, npages)) {
- mdbx_debug("skip-2 parent-spilled %u page %" PRIaPGNO, npages, pgno);
- dp->mp_flags |= P_SPILLED;
- return 256;
- }
- while ((parent = parent->mt_parent) != nullptr);
- }
+static __inline txnid_t meta_txnid(const MDBX_env *env,
+ volatile const MDBX_meta *meta) {
+ (void)env;
+ mdbx_memory_fence(mo_AcquireRelease, false);
+ txnid_t a = unaligned_peek_u64_volatile(4, &meta->mm_txnid_a);
+ txnid_t b = unaligned_peek_u64_volatile(4, &meta->mm_txnid_b);
+ return (a == b) ? a : 0;
+}
- mdbx_tassert(txn, age * (uint64_t)reciprocal < UINT32_MAX);
- unsigned prio = age * reciprocal >> 24;
- mdbx_tassert(txn, prio < 256);
- if (likely(npages == 1))
- return prio = 256 - prio;
+static __inline void meta_update_begin(const MDBX_env *env, MDBX_meta *meta,
+ txnid_t txnid) {
+ mdbx_assert(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
+ mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_a) < txnid &&
+ unaligned_peek_u64(4, meta->mm_txnid_b) < txnid);
+ (void)env;
+ unaligned_poke_u64(4, meta->mm_txnid_b, 0);
+ mdbx_memory_fence(mo_AcquireRelease, true);
+ unaligned_poke_u64(4, meta->mm_txnid_a, txnid);
+}
- /* make a large/overflow pages be likely to spill */
- uint32_t factor = npages | npages >> 1;
- factor |= factor >> 2;
- factor |= factor >> 4;
- factor |= factor >> 8;
- factor |= factor >> 16;
- factor = prio * log2n_powerof2(factor + 1) + /* golden ratio */ 157;
- factor = (factor < 256) ? 255 - factor : 0;
- mdbx_tassert(txn, factor < 256 && factor < (256 - prio));
- return prio = factor;
+static __inline void meta_update_end(const MDBX_env *env, MDBX_meta *meta,
+ txnid_t txnid) {
+ mdbx_assert(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
+ mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_a) == txnid);
+ mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_b) < txnid);
+ (void)env;
+ mdbx_jitter4testing(true);
+ memcpy(&meta->mm_bootid, &bootid, 16);
+ unaligned_poke_u64(4, meta->mm_txnid_b, txnid);
+ mdbx_memory_fence(mo_AcquireRelease, true);
+}
+
+static __inline void meta_set_txnid(const MDBX_env *env, MDBX_meta *meta,
+ const txnid_t txnid) {
+ mdbx_assert(env, !env->me_map || meta < METAPAGE(env, 0) ||
+ meta >= METAPAGE_END(env));
+ (void)env;
+ /* update inconsistently since this function used ONLY for filling meta-image
+ * for writing, but not the actual meta-page */
+ memcpy(&meta->mm_bootid, &bootid, 16);
+ unaligned_poke_u64(4, meta->mm_txnid_a, txnid);
+ unaligned_poke_u64(4, meta->mm_txnid_b, txnid);
}
-/* Spill pages from the dirty list back to disk.
- * This is intended to prevent running into MDBX_TXN_FULL situations,
- * but note that they may still occur in a few cases:
- *
- * 1) our estimate of the txn size could be too small. Currently this
- * seems unlikely, except with a large number of MDBX_MULTIPLE items.
- *
- * 2) child txns may run out of space if their parents dirtied a
- * lot of pages and never spilled them. TODO: we probably should do
- * a preemptive spill during mdbx_txn_begin() of a child txn, if
- * the parent's dirtyroom is below a given threshold.
- *
- * Otherwise, if not using nested txns, it is expected that apps will
- * not run into MDBX_TXN_FULL any more. The pages are flushed to disk
- * the same way as for a txn commit, e.g. their dirty status is cleared.
- * If the txn never references them again, they can be left alone.
- * If the txn only reads them, they can be used without any fuss.
- * If the txn writes them again, they can be dirtied immediately without
- * going thru all of the work of mdbx_page_touch(). Such references are
- * handled by mdbx_page_unspill().
- *
- * Also note, we never spill DB root pages, nor pages of active cursors,
- * because we'll need these back again soon anyway. And in nested txns,
- * we can't spill a page in a child txn if it was already spilled in a
- * parent txn. That would alter the parent txns' data even though
- * the child hasn't committed yet, and we'd have no way to undo it if
- * the child aborted. */
-static int mdbx_txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0,
- const unsigned need) {
-#if xMDBX_DEBUG_SPILLING != 1
- /* production mode */
- if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need))
- return MDBX_SUCCESS;
- unsigned wanna_spill = need - txn->tw.dirtyroom;
+static __inline uint64_t meta_sign(const MDBX_meta *meta) {
+ uint64_t sign = MDBX_DATASIGN_NONE;
+#if 0 /* TODO */
+ sign = hippeus_hash64(...);
#else
- /* debug mode: spill at least one page if xMDBX_DEBUG_SPILLING == 1 */
- unsigned wanna_spill =
- (need > txn->tw.dirtyroom) ? need - txn->tw.dirtyroom : 1;
-#endif /* xMDBX_DEBUG_SPILLING */
+ (void)meta;
+#endif
+ /* LY: newer returns MDBX_DATASIGN_NONE or MDBX_DATASIGN_WEAK */
+ return (sign > MDBX_DATASIGN_WEAK) ? sign : ~sign;
+}
- const unsigned dirty = txn->tw.dirtylist->length;
- const unsigned spill_min =
- txn->mt_env->me_options.spill_min_denominator
- ? dirty / txn->mt_env->me_options.spill_min_denominator
- : 0;
- const unsigned spill_max =
- dirty - (txn->mt_env->me_options.spill_max_denominator
- ? dirty / txn->mt_env->me_options.spill_max_denominator
- : 0);
- wanna_spill = (wanna_spill > spill_min) ? wanna_spill : spill_min;
- wanna_spill = (wanna_spill < spill_max) ? wanna_spill : spill_max;
- if (!wanna_spill)
- return MDBX_SUCCESS;
+enum meta_choise_mode { prefer_last, prefer_steady };
- mdbx_notice("spilling %u dirty-entries (have %u dirty-room, need %u)",
- wanna_spill, txn->tw.dirtyroom, need);
- mdbx_tassert(txn, txn->tw.dirtylist->length >= wanna_spill);
+static __inline bool meta_ot(const enum meta_choise_mode mode,
+ const MDBX_env *env, volatile const MDBX_meta *a,
+ volatile const MDBX_meta *b) {
+ mdbx_jitter4testing(true);
+ const txnid_t txnid_a = meta_txnid(env, a);
+ mdbx_jitter4testing(true);
+ const txnid_t txnid_b = meta_txnid(env, b);
+ mdbx_jitter4testing(true);
+ const bool is_stead_b = META_IS_STEADY(b);
- struct mdbx_iov_ctx ctx;
- mdbx_iov_init(txn, &ctx);
- int rc = MDBX_SUCCESS;
- if (txn->mt_flags & MDBX_WRITEMAP) {
- MDBX_dpl *const dl = txn->tw.dirtylist;
- const unsigned span = dl->length - txn->tw.loose_count;
- txn->tw.dirtyroom += span;
- unsigned r, w;
- for (w = 0, r = 1; r <= dl->length; ++r) {
- MDBX_page *dp = dl->items[r].ptr;
- if (dp->mp_flags & P_LOOSE)
- dl->items[++w] = dl->items[r];
- else if (!MDBX_FAKE_SPILL_WRITEMAP) {
- rc = iov_page(txn, &ctx, dp, dpl_npages(dl, r));
- mdbx_tassert(txn, rc == MDBX_SUCCESS);
- }
- }
+ if (mode == prefer_steady) {
+ mdbx_jitter4testing(true);
+ const bool is_stead_a = META_IS_STEADY(a);
+ if (is_stead_a != is_stead_b)
+ return is_stead_b;
+ } else {
+ mdbx_assert(env, mode == prefer_last);
+ }
+ if (txnid_a == txnid_b)
+ return is_stead_b;
+ return txnid_a < txnid_b;
+}
- mdbx_tassert(txn, span == r - 1 - w && w == txn->tw.loose_count);
- dl->sorted = (dl->sorted == dl->length) ? w : 0;
- dpl_setlen(dl, w);
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+static bool meta_eq(const MDBX_env *env, volatile const MDBX_meta *a,
+ volatile const MDBX_meta *b) {
+ mdbx_jitter4testing(true);
+ const txnid_t txnid = meta_txnid(env, a);
+ if (!txnid || txnid != meta_txnid(env, b))
+ return false;
- if (!MDBX_FAKE_SPILL_WRITEMAP && ctx.flush_end > ctx.flush_begin) {
- MDBX_env *const env = txn->mt_env;
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- rc = mdbx_msync(&env->me_dxb_mmap,
- pgno_align2os_bytes(env, ctx.flush_begin),
- pgno_align2os_bytes(env, ctx.flush_end - ctx.flush_begin),
- MDBX_SYNC_NONE);
- }
- return rc;
- }
+ mdbx_jitter4testing(true);
+ if (META_IS_STEADY(a) != META_IS_STEADY(b))
+ return false;
- mdbx_tassert(txn, !(txn->mt_flags & MDBX_WRITEMAP));
- if (!txn->tw.spill_pages) {
- txn->tw.spill_least_removed = INT_MAX;
- txn->tw.spill_pages = mdbx_pnl_alloc(wanna_spill);
- if (unlikely(!txn->tw.spill_pages)) {
- rc = MDBX_ENOMEM;
- bailout:
- txn->mt_flags |= MDBX_TXN_ERROR;
- return rc;
- }
- } else {
- /* purge deleted slots */
- mdbx_spill_purge(txn);
- rc = mdbx_pnl_reserve(&txn->tw.spill_pages, wanna_spill);
- (void)rc /* ignore since the resulting list may be shorter
- and mdbx_pnl_append() will increase pnl on demand */
- ;
- }
+ mdbx_jitter4testing(true);
+ return true;
+}
- /* Сортируем чтобы запиÑÑŒ на диÑк была полее поÑледовательна */
- MDBX_dpl *const dl = mdbx_dpl_sort(txn);
+static int meta_eq_mask(const MDBX_env *env) {
+ volatile const MDBX_meta *m0 = METAPAGE(env, 0);
+ volatile const MDBX_meta *m1 = METAPAGE(env, 1);
+ volatile const MDBX_meta *m2 = METAPAGE(env, 2);
- /* Preserve pages which may soon be dirtied again */
- const unsigned unspillable = mdbx_txn_keep(txn, m0);
- if (unspillable + txn->tw.loose_count >= dl->length) {
-#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */
- if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need))
- return MDBX_SUCCESS;
-#endif /* xMDBX_DEBUG_SPILLING */
- mdbx_error("all %u dirty pages are unspillable since referenced "
- "by a cursor(s), use fewer cursors or increase "
- "MDBX_opt_txn_dp_limit",
- unspillable);
- goto done;
- }
+ int rc = meta_eq(env, m0, m1) ? 1 : 0;
+ if (meta_eq(env, m1, m2))
+ rc += 2;
+ if (meta_eq(env, m2, m0))
+ rc += 4;
+ return rc;
+}
- /* Подзадача: Вытолкнуть чаÑÑ‚ÑŒ Ñтраниц на диÑк в ÑоответÑтвии Ñ LRU,
- * но при Ñтом учеÑÑ‚ÑŒ важные поправки:
- * - лучше выталкивать Ñтарые large/overflow Ñтраницы, так будет оÑвобождено
- * больше памÑти, а также так как они (в текущем понимании) гораздо реже
- * повторно изменÑÑŽÑ‚ÑÑ;
- * - при прочих равных лучше выталкивать Ñмежные Ñтраницы, так будет
- * меньше I/O операций;
- * - желательно потратить на Ñто меньше времени чем std::partial_sort_copy;
- *
- * Решение:
- * - Квантуем веÑÑŒ диапазон lru-меток до 256 значений и задейÑтвуем один
- * проход 8-битного radix-sort. В результате получаем 256 уровней
- * "ÑвежеÑти", в том чиÑле значение lru-метки, Ñтарее которой Ñтраницы
- * должны быть выгружены;
- * - ДвигаемÑÑ Ð¿Ð¾Ñледовательно в Ñторону ÑƒÐ²ÐµÐ»Ð¸Ñ‡ÐµÐ½Ð¸Ñ Ð½Ð¾Ð¼ÐµÑ€Ð¾Ð² Ñтраниц
- * и выталкиваем Ñтраницы Ñ lru-меткой Ñтарее отÑекающего значениÑ,
- * пока не вытолкнем доÑтаточно;
- * - Ð’ÑÑ‚Ñ€ÐµÑ‡Ð°Ñ Ñтраницы Ñмежные Ñ Ð²Ñ‹Ñ‚Ð°Ð»ÐºÐ¸Ð²Ð°ÐµÐ¼Ñ‹Ð¼Ð¸ Ð´Ð»Ñ ÑƒÐ¼ÐµÐ½ÑŒÑˆÐµÐ½Ð¸Ñ ÐºÐ¾Ð»-ва
- * I/O операций выталкиваем и их, еÑли они попадают в первую половину
- * между выталкиваемыми и Ñамыми Ñвежими lru-метками;
- * - дополнительно при Ñортировке умышленно Ñтарим large/overflow Ñтраницы,
- * тем Ñамым Ð¿Ð¾Ð²Ñ‹ÑˆÐ°Ñ Ð¸Ñ… шанÑÑ‹ на выталкивание. */
+static __inline volatile const MDBX_meta *
+meta_recent(const enum meta_choise_mode mode, const MDBX_env *env,
+ volatile const MDBX_meta *a, volatile const MDBX_meta *b) {
+ const bool a_older_that_b = meta_ot(mode, env, a, b);
+ mdbx_assert(env, !meta_eq(env, a, b));
+ return a_older_that_b ? b : a;
+}
- /* get min/max of LRU-labels */
- uint32_t age_max = 0;
- for (unsigned i = 1; i <= dl->length; ++i) {
- const uint32_t age = mdbx_dpl_age(txn, i);
- age_max = (age_max >= age) ? age_max : age;
- }
+static const MDBX_meta *meta_ancient_prefer_weak(const MDBX_env *env,
+ const MDBX_meta *a,
+ const MDBX_meta *b) {
+ const bool a_older_that_b = meta_ot(prefer_steady, env, a, b);
+ mdbx_assert(env, !meta_eq(env, a, b));
+ return a_older_that_b ? a : b;
+}
- mdbx_verbose("lru-head %u, age-max %u", txn->tw.dirtylru, age_max);
+static __inline volatile const MDBX_meta *
+meta_mostrecent(const enum meta_choise_mode mode, const MDBX_env *env) {
+ volatile const MDBX_meta *m0 = METAPAGE(env, 0);
+ volatile const MDBX_meta *m1 = METAPAGE(env, 1);
+ volatile const MDBX_meta *m2 = METAPAGE(env, 2);
- /* half of 8-bit radix-sort */
- unsigned radix_counters[256], spillable = 0, spilled = 0;
- memset(&radix_counters, 0, sizeof(radix_counters));
- const uint32_t reciprocal = (UINT32_C(255) << 24) / (age_max + 1);
- for (unsigned i = 1; i <= dl->length; ++i) {
- unsigned prio = spill_prio(txn, i, reciprocal);
- if (prio < 256) {
- radix_counters[prio] += 1;
- spillable += 1;
- }
- }
+ volatile const MDBX_meta *head = meta_recent(mode, env, m0, m1);
+ head = meta_recent(mode, env, head, m2);
+ return head;
+}
- if (likely(spillable > 0)) {
- unsigned prio2spill = 0, prio2adjacent = 128, amount = radix_counters[0];
- for (unsigned i = 1; i < 256; i++) {
- if (amount < wanna_spill) {
- prio2spill = i;
- prio2adjacent = i + (257 - i) / 2;
- amount += radix_counters[i];
- } else if (amount + amount < spillable + wanna_spill
- /* Ð ÐÐ’ÐОЗÐÐЧÐО: amount - wanna_spill < spillable - amount */) {
- prio2adjacent = i;
- amount += radix_counters[i];
- } else
- break;
- }
+static volatile const MDBX_meta *meta_prefer_steady(const MDBX_env *env) {
+ return meta_mostrecent(prefer_steady, env);
+}
- mdbx_verbose("prio2spill %u, prio2adjacent %u, amount %u, spillable %u, "
- "wanna_spill %u",
- prio2spill, prio2adjacent, amount, spillable, wanna_spill);
- mdbx_tassert(txn, prio2spill < prio2adjacent && prio2adjacent <= 256);
+MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta *
+constmeta_prefer_steady(const MDBX_env *env) {
+ return (const MDBX_meta *)meta_mostrecent(prefer_steady, env);
+}
- unsigned prev_prio = 256;
- unsigned r, w, prio;
- for (w = 0, r = 1; r <= dl->length && spilled < wanna_spill;
- prev_prio = prio, ++r) {
- prio = spill_prio(txn, r, reciprocal);
- MDBX_page *const dp = dl->items[r].ptr;
- if (prio < prio2adjacent) {
- const pgno_t pgno = dl->items[r].pgno;
- const unsigned npages = dpl_npages(dl, r);
- if (prio <= prio2spill) {
- if (prev_prio < prio2adjacent && prev_prio > prio2spill &&
- dpl_endpgno(dl, r - 1) == pgno) {
- mdbx_debug("co-spill %u prev-adjacent page %" PRIaPGNO
- " (age %d, prio %u)",
- dpl_npages(dl, w), dl->items[r - 1].pgno,
- mdbx_dpl_age(txn, r - 1), prev_prio);
- --w;
- rc = spill_page(txn, &ctx, dl->items[r - 1].ptr,
- dpl_npages(dl, r - 1));
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- ++spilled;
- }
+static volatile const MDBX_meta *meta_prefer_last(const MDBX_env *env) {
+ return meta_mostrecent(prefer_last, env);
+}
- mdbx_debug("spill %u page %" PRIaPGNO " (age %d, prio %u)", npages,
- dp->mp_pgno, mdbx_dpl_age(txn, r), prio);
- rc = spill_page(txn, &ctx, dp, npages);
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- ++spilled;
- continue;
- }
+MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta *
+constmeta_prefer_last(const MDBX_env *env) {
+ return (const MDBX_meta *)meta_mostrecent(prefer_last, env);
+}
- if (prev_prio <= prio2spill && dpl_endpgno(dl, r - 1) == pgno) {
- mdbx_debug("co-spill %u next-adjacent page %" PRIaPGNO
- " (age %d, prio %u)",
- npages, dp->mp_pgno, mdbx_dpl_age(txn, r), prio);
- rc = spill_page(txn, &ctx, dp, npages);
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- prio = prev_prio /* to continue co-spilling next adjacent pages */;
- ++spilled;
- continue;
- }
- }
- dl->items[++w] = dl->items[r];
- }
+static txnid_t mdbx_recent_committed_txnid(const MDBX_env *env) {
+ while (true) {
+ volatile const MDBX_meta *head = meta_prefer_last(env);
+ const txnid_t recent = meta_txnid(env, head);
+ mdbx_compiler_barrier();
+ if (likely(head == meta_prefer_last(env) &&
+ recent == meta_txnid(env, head)))
+ return recent;
+ }
+}
- mdbx_tassert(txn, spillable == 0 || spilled > 0);
+static txnid_t mdbx_recent_steady_txnid(const MDBX_env *env) {
+ while (true) {
+ volatile const MDBX_meta *head = meta_prefer_steady(env);
+ const txnid_t recent = meta_txnid(env, head);
+ mdbx_compiler_barrier();
+ if (likely(head == meta_prefer_steady(env) &&
+ recent == meta_txnid(env, head)))
+ return recent;
+ }
+}
- while (r <= dl->length)
- dl->items[++w] = dl->items[r++];
- mdbx_tassert(txn, r - 1 - w == spilled);
+static const char *mdbx_durable_str(volatile const MDBX_meta *const meta) {
+ if (META_IS_STEADY(meta))
+ return (unaligned_peek_u64_volatile(4, meta->mm_datasync_sign) ==
+ meta_sign((const MDBX_meta *)meta))
+ ? "Steady"
+ : "Tainted";
+ return "Weak";
+}
- dl->sorted = dpl_setlen(dl, w);
- txn->tw.dirtyroom += spilled;
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+/*----------------------------------------------------------------------------*/
- if (ctx.iov_items)
- rc = mdbx_iov_write(txn, &ctx);
+/* Find oldest txnid still referenced. */
+static txnid_t mdbx_find_oldest(const MDBX_txn *txn) {
+ mdbx_tassert(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0);
+ MDBX_env *env = txn->mt_env;
+ const txnid_t edge = mdbx_recent_steady_txnid(env);
+ mdbx_tassert(txn, edge <= txn->mt_txnid);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (unlikely(lck == NULL /* exclusive mode */))
+ return atomic_store64(&lck->mti_oldest_reader, edge, mo_Relaxed);
- mdbx_pnl_sort(txn->tw.spill_pages, (size_t)txn->mt_next_pgno << 1);
- txn->mt_flags |= MDBX_TXN_SPILLS;
- mdbx_notice("spilled %u dirty-entries, now have %u dirty-room", spilled,
- txn->tw.dirtyroom);
- mdbx_iov_done(txn, &ctx);
- } else {
- mdbx_tassert(txn, ctx.iov_items == 0 && rc == MDBX_SUCCESS);
- for (unsigned i = 1; i <= dl->length; ++i) {
- MDBX_page *dp = dl->items[i].ptr;
- mdbx_notice(
- "dirtylist[%u]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i,
- dp->mp_pgno, dpl_npages(dl, i), dp->mp_flags, mdbx_dpl_age(txn, i),
- spill_prio(txn, i, reciprocal));
+ const txnid_t last_oldest =
+ atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease);
+ mdbx_tassert(txn, edge >= last_oldest);
+ if (likely(last_oldest == edge))
+ return edge;
+
+ const uint32_t nothing_changed = MDBX_STRING_TETRAD("None");
+ const uint32_t snap_readers_refresh_flag =
+ atomic_load32(&lck->mti_readers_refresh_flag, mo_AcquireRelease);
+ mdbx_jitter4testing(false);
+ if (snap_readers_refresh_flag == nothing_changed)
+ return last_oldest;
+
+ txnid_t oldest = edge;
+ atomic_store32(&lck->mti_readers_refresh_flag, nothing_changed, mo_Relaxed);
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
+ /* mdbx_jitter4testing(true); */
+ const txnid_t snap = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (oldest > snap && last_oldest <= /* ignore pending updates */ snap) {
+ oldest = snap;
+ if (oldest == last_oldest)
+ return oldest;
+ }
}
}
-#if xMDBX_DEBUG_SPILLING == 2
- if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1)
- mdbx_error("dirty-list length: before %u, after %u, parent %i, loose %u; "
- "needed %u, spillable %u; "
- "spilled %u dirty-entries, now have %u dirty-room",
- dl->length + spilled, dl->length,
- (txn->mt_parent && txn->mt_parent->tw.dirtylist)
- ? (int)txn->mt_parent->tw.dirtylist->length
- : -1,
- txn->tw.loose_count, need, spillable, spilled,
- txn->tw.dirtyroom);
- mdbx_ensure(txn->mt_env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2);
-#endif /* xMDBX_DEBUG_SPILLING */
-
-done:
- return likely(txn->tw.dirtyroom + txn->tw.loose_count >
- ((need > CURSOR_STACK) ? CURSOR_STACK : need))
- ? MDBX_SUCCESS
- : MDBX_TXN_FULL;
+ if (oldest != last_oldest) {
+ mdbx_verbose("update oldest %" PRIaTXN " -> %" PRIaTXN, last_oldest,
+ oldest);
+ mdbx_tassert(txn, oldest >= lck->mti_oldest_reader.weak);
+ atomic_store64(&lck->mti_oldest_reader, oldest, mo_Relaxed);
+ }
+ return oldest;
}
-static int mdbx_cursor_spill(MDBX_cursor *mc, const MDBX_val *key,
- const MDBX_val *data) {
- MDBX_txn *txn = mc->mc_txn;
- /* Estimate how much space this operation will take: */
- /* 1) Max b-tree height, reasonable enough with including dups' sub-tree */
- unsigned need = CURSOR_STACK + 3;
- /* 2) GC/FreeDB for any payload */
- if (mc->mc_dbi > FREE_DBI) {
- need += txn->mt_dbs[FREE_DBI].md_depth + 3;
- /* 3) Named DBs also dirty the main DB */
- if (mc->mc_dbi > MAIN_DBI)
- need += txn->mt_dbs[MAIN_DBI].md_depth + 3;
+/* Find largest mvcc-snapshot still referenced. */
+__cold static pgno_t mdbx_find_largest(MDBX_env *env, pgno_t largest) {
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (likely(lck != NULL /* exclusive mode */)) {
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ retry:
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
+ /* mdbx_jitter4testing(true); */
+ const pgno_t snap_pages = atomic_load32(
+ &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed);
+ const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (unlikely(
+ snap_pages !=
+ atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used,
+ mo_AcquireRelease) ||
+ snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
+ goto retry;
+ if (largest < snap_pages &&
+ atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease) <=
+ /* ignore pending updates */ snap_txnid &&
+ snap_txnid <= env->me_txn0->mt_txnid)
+ largest = snap_pages;
+ }
+ }
}
-#if xMDBX_DEBUG_SPILLING != 2
- /* production mode */
- /* 4) Double the page chain estimation
- * for extensively splitting, rebalance and merging */
- need += need;
- /* 5) Factor the key+data which to be put in */
- need += bytes2pgno(txn->mt_env, node_size(key, data)) + 1;
-#else
- /* debug mode */
- (void)key;
- (void)data;
- mc->mc_txn->mt_env->debug_dirtied_est = ++need;
- mc->mc_txn->mt_env->debug_dirtied_act = 0;
-#endif /* xMDBX_DEBUG_SPILLING == 2 */
- return mdbx_txn_spill(txn, mc, need);
+ return largest;
}
-/*----------------------------------------------------------------------------*/
+/* Add a page to the txn's dirty list */
+static int __must_check_result mdbx_page_dirty(MDBX_txn *txn, MDBX_page *mp,
+ unsigned npages) {
+#if xMDBX_DEBUG_SPILLING == 2
+ txn->mt_env->debug_dirtied_act += 1;
+ mdbx_ensure(txn->mt_env,
+ txn->mt_env->debug_dirtied_act < txn->mt_env->debug_dirtied_est);
+ mdbx_ensure(txn->mt_env, txn->tw.dirtyroom + txn->tw.loose_count > 0);
+#endif /* xMDBX_DEBUG_SPILLING == 2 */
+
+ int rc;
+ mp->mp_txnid = txn->mt_front;
+ if (unlikely(txn->tw.dirtyroom == 0)) {
+ if (txn->tw.loose_count) {
+ MDBX_page *loose = txn->tw.loose_pages;
+ mdbx_debug("purge-and-reclaim loose page %" PRIaPGNO, loose->mp_pgno);
+ rc = mdbx_pnl_insert_range(&txn->tw.reclaimed_pglist, loose->mp_pgno, 1);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ unsigned di = mdbx_dpl_search(txn, loose->mp_pgno);
+ mdbx_tassert(txn, txn->tw.dirtylist->items[di].ptr == loose);
+ mdbx_dpl_remove(txn, di);
+ txn->tw.loose_pages = loose->mp_next;
+ txn->tw.loose_count--;
+ txn->tw.dirtyroom++;
+ if (!(txn->mt_flags & MDBX_WRITEMAP))
+ mdbx_dpage_free(txn->mt_env, loose, 1);
+ } else {
+ mdbx_error("Dirtyroom is depleted, DPL length %u",
+ txn->tw.dirtylist->length);
+ if (!(txn->mt_flags & MDBX_WRITEMAP))
+ mdbx_dpage_free(txn->mt_env, mp, npages);
+ return MDBX_TXN_FULL;
+ }
+ }
-static bool meta_bootid_match(const MDBX_meta *meta) {
- return memcmp(&meta->mm_bootid, &bootid, 16) == 0 &&
- (bootid.x | bootid.y) != 0;
+ rc = mdbx_dpl_append(txn, mp->mp_pgno, mp, npages);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ bailout:
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ return rc;
+ }
+ txn->tw.dirtyroom--;
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ return MDBX_SUCCESS;
}
-static bool meta_weak_acceptable(const MDBX_env *env, const MDBX_meta *meta,
- const int lck_exclusive) {
- return lck_exclusive
- ? /* exclusive lock */ meta_bootid_match(meta)
- : /* db already opened */ env->me_lck_mmap.lck &&
- (env->me_lck_mmap.lck->mti_envmode.weak & MDBX_RDONLY) == 0;
+#if !(defined(_WIN32) || defined(_WIN64))
+MDBX_MAYBE_UNUSED static __always_inline int ignore_enosys(int err) {
+#ifdef ENOSYS
+ if (err == ENOSYS)
+ return MDBX_RESULT_TRUE;
+#endif /* ENOSYS */
+#ifdef ENOIMPL
+ if (err == ENOIMPL)
+ return MDBX_RESULT_TRUE;
+#endif /* ENOIMPL */
+#ifdef ENOTSUP
+ if (err == ENOTSUP)
+ return MDBX_RESULT_TRUE;
+#endif /* ENOTSUP */
+#ifdef ENOSUPP
+ if (err == ENOSUPP)
+ return MDBX_RESULT_TRUE;
+#endif /* ENOSUPP */
+#ifdef EOPNOTSUPP
+ if (err == EOPNOTSUPP)
+ return MDBX_RESULT_TRUE;
+#endif /* EOPNOTSUPP */
+ if (err == EAGAIN)
+ return MDBX_RESULT_TRUE;
+ return err;
}
+#endif /* defined(_WIN32) || defined(_WIN64) */
-#define METAPAGE(env, n) page_meta(pgno2page(env, n))
-#define METAPAGE_END(env) METAPAGE(env, NUM_METAS)
-
-MDBX_NOTHROW_PURE_FUNCTION static __inline txnid_t
-constmeta_txnid(const MDBX_env *env, const MDBX_meta *meta) {
- mdbx_memory_fence(mo_AcquireRelease, false);
- txnid_t a = unaligned_peek_u64(4, &meta->mm_txnid_a);
- txnid_t b = unaligned_peek_u64(4, &meta->mm_txnid_b);
- mdbx_assert(env, a == b);
- (void)env;
- return (a == b) ? a : 0;
-}
+#if MDBX_ENABLE_MADVISE
+/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */
+__cold static int mdbx_set_readahead(MDBX_env *env, const pgno_t edge,
+ const bool enable,
+ const bool force_whole) {
+ mdbx_assert(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1);
+ mdbx_assert(env, (enable & 1) == (enable != 0));
+ const bool toggle = force_whole ||
+ ((enable ^ env->me_lck->mti_readahead_anchor) & 1) ||
+ !env->me_lck->mti_readahead_anchor;
+ const pgno_t prev_edge = env->me_lck->mti_readahead_anchor >> 1;
+ const size_t limit = env->me_dxb_mmap.limit;
+ size_t offset =
+ toggle ? 0
+ : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge);
+ offset = (offset < limit) ? offset : limit;
-static __inline txnid_t meta_txnid(const MDBX_env *env,
- volatile const MDBX_meta *meta) {
- (void)env;
- mdbx_memory_fence(mo_AcquireRelease, false);
- txnid_t a = unaligned_peek_u64_volatile(4, &meta->mm_txnid_a);
- txnid_t b = unaligned_peek_u64_volatile(4, &meta->mm_txnid_b);
- return (a == b) ? a : 0;
-}
+ size_t length =
+ pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge);
+ length = (length < limit) ? length : limit;
+ length -= offset;
-static __inline void meta_update_begin(const MDBX_env *env, MDBX_meta *meta,
- txnid_t txnid) {
- mdbx_assert(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
- mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_a) < txnid &&
- unaligned_peek_u64(4, meta->mm_txnid_b) < txnid);
- (void)env;
- unaligned_poke_u64(4, meta->mm_txnid_b, 0);
- mdbx_memory_fence(mo_AcquireRelease, true);
- unaligned_poke_u64(4, meta->mm_txnid_a, txnid);
-}
+ mdbx_assert(env, 0 <= (intptr_t)length);
+ if (length == 0)
+ return MDBX_SUCCESS;
-static __inline void meta_update_end(const MDBX_env *env, MDBX_meta *meta,
- txnid_t txnid) {
- mdbx_assert(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
- mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_a) == txnid);
- mdbx_assert(env, unaligned_peek_u64(4, meta->mm_txnid_b) < txnid);
- (void)env;
- mdbx_jitter4testing(true);
- memcpy(&meta->mm_bootid, &bootid, 16);
- unaligned_poke_u64(4, meta->mm_txnid_b, txnid);
- mdbx_memory_fence(mo_AcquireRelease, true);
-}
+ mdbx_notice("readahead %s %u..%u", enable ? "ON" : "OFF",
+ bytes2pgno(env, offset), bytes2pgno(env, offset + length));
-static __inline void meta_set_txnid(const MDBX_env *env, MDBX_meta *meta,
- const txnid_t txnid) {
- mdbx_assert(env, !env->me_map || meta < METAPAGE(env, 0) ||
- meta >= METAPAGE_END(env));
- (void)env;
- /* update inconsistently since this function used ONLY for filling meta-image
- * for writing, but not the actual meta-page */
- memcpy(&meta->mm_bootid, &bootid, 16);
- unaligned_poke_u64(4, meta->mm_txnid_a, txnid);
- unaligned_poke_u64(4, meta->mm_txnid_b, txnid);
-}
+#if defined(F_RDAHEAD)
+ if (toggle && unlikely(fcntl(env->me_lazy_fd, F_RDAHEAD, enable) == -1))
+ return errno;
+#endif /* F_RDAHEAD */
-static __inline uint64_t meta_sign(const MDBX_meta *meta) {
- uint64_t sign = MDBX_DATASIGN_NONE;
-#if 0 /* TODO */
- sign = hippeus_hash64(...);
+ int err;
+ if (enable) {
+#if defined(MADV_NORMAL)
+ err = madvise(env->me_map + offset, length, MADV_NORMAL)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_MADV_NORMAL)
+ err = ignore_enosys(
+ posix_madvise(env->me_map + offset, length, POSIX_MADV_NORMAL));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED)
+ err = ignore_enosys(
+ posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_NORMAL));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(_WIN32) || defined(_WIN64)
+ /* no madvise on Windows */
#else
- (void)meta;
+#warning "FIXME"
#endif
- /* LY: newer returns MDBX_DATASIGN_NONE or MDBX_DATASIGN_WEAK */
- return (sign > MDBX_DATASIGN_WEAK) ? sign : ~sign;
+ if (toggle) {
+ /* NOTE: Seems there is a bug in the Mach/Darwin/OSX kernel,
+ * because MADV_WILLNEED with offset != 0 may cause SIGBUS
+ * on following access to the hinted region.
+ * 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021;
+ * root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64 */
+#if defined(F_RDADVISE)
+ struct radvisory hint;
+ hint.ra_offset = offset;
+ hint.ra_count = length;
+ (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl(
+ env->me_lazy_fd, F_RDADVISE, &hint);
+#elif defined(MADV_WILLNEED)
+ err = madvise(env->me_map + offset, length, MADV_WILLNEED)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_MADV_WILLNEED)
+ err = ignore_enosys(
+ posix_madvise(env->me_map + offset, length, POSIX_MADV_WILLNEED));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(_WIN32) || defined(_WIN64)
+ if (mdbx_PrefetchVirtualMemory) {
+ WIN32_MEMORY_RANGE_ENTRY hint;
+ hint.VirtualAddress = env->me_map + offset;
+ hint.NumberOfBytes = length;
+ (void)mdbx_PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0);
+ }
+#elif defined(POSIX_FADV_WILLNEED)
+ err = ignore_enosys(
+ posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_WILLNEED));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#else
+#warning "FIXME"
+#endif
+ }
+ } else {
+#if defined(MADV_RANDOM)
+ err = madvise(env->me_map + offset, length, MADV_RANDOM)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_MADV_RANDOM)
+ err = ignore_enosys(
+ posix_madvise(env->me_map + offset, length, POSIX_MADV_RANDOM));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_FADV_RANDOM)
+ err = ignore_enosys(
+ posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_RANDOM));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(_WIN32) || defined(_WIN64)
+ /* no madvise on Windows */
+#else
+#warning "FIXME"
+#endif /* MADV_RANDOM */
+ }
+
+ env->me_lck->mti_readahead_anchor = (enable & 1) + (edge << 1);
+ err = MDBX_SUCCESS;
+ return err;
}
+#endif /* MDBX_ENABLE_MADVISE */
+
+__cold static int mdbx_mapresize(MDBX_env *env, const pgno_t used_pgno,
+ const pgno_t size_pgno,
+ const pgno_t limit_pgno, const bool implicit) {
+ const size_t limit_bytes = pgno_align2os_bytes(env, limit_pgno);
+ const size_t size_bytes = pgno_align2os_bytes(env, size_pgno);
+ const size_t prev_size = env->me_dxb_mmap.current;
+ const size_t prev_limit = env->me_dxb_mmap.limit;
+#if MDBX_ENABLE_MADVISE || defined(MDBX_USE_VALGRIND)
+ const void *const prev_addr = env->me_map;
+#endif /* MDBX_ENABLE_MADVISE || MDBX_USE_VALGRIND */
+
+ mdbx_verbose("resize datafile/mapping: "
+ "present %" PRIuPTR " -> %" PRIuPTR ", "
+ "limit %" PRIuPTR " -> %" PRIuPTR,
+ prev_size, size_bytes, prev_limit, limit_bytes);
+
+ mdbx_assert(env, limit_bytes >= size_bytes);
+ mdbx_assert(env, bytes2pgno(env, size_bytes) >= size_pgno);
+ mdbx_assert(env, bytes2pgno(env, limit_bytes) >= limit_pgno);
+
+ unsigned mresize_flags =
+ env->me_flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC);
+#if defined(_WIN32) || defined(_WIN64)
+ /* Acquire guard in exclusive mode for:
+ * - to avoid collision between read and write txns around env->me_dbgeo;
+ * - to avoid attachment of new reading threads (see mdbx_rdt_lock); */
+ mdbx_srwlock_AcquireExclusive(&env->me_remap_guard);
+ mdbx_handle_array_t *suspended = NULL;
+ mdbx_handle_array_t array_onstack;
+ int rc = MDBX_SUCCESS;
+ if (limit_bytes == env->me_dxb_mmap.limit &&
+ size_bytes == env->me_dxb_mmap.current &&
+ size_bytes == env->me_dxb_mmap.filesize)
+ goto bailout;
-enum meta_choise_mode { prefer_last, prefer_steady };
+ if ((env->me_flags & MDBX_NOTLS) == 0) {
+ /* 1) Windows allows only extending a read-write section, but not a
+ * corresponding mapped view. Therefore in other cases we must suspend
+ * the local threads for safe remap.
+ * 2) At least on Windows 10 1803 the entire mapped section is unavailable
+ * for short time during NtExtendSection() or VirtualAlloc() execution.
+ * 3) Under Wine runtime environment on Linux a section extending is not
+ * supported.
+ *
+ * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */
+ array_onstack.limit = ARRAY_LENGTH(array_onstack.handles);
+ array_onstack.count = 0;
+ suspended = &array_onstack;
+ rc = mdbx_suspend_threads_before_remap(env, &suspended);
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("failed suspend-for-remap: errcode %d", rc);
+ goto bailout;
+ }
+ mresize_flags |= implicit ? MDBX_MRESIZE_MAY_UNMAP
+ : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
+ }
+#else /* Windows */
+ /* Acquire guard to avoid collision between read and write txns
+ * around env->me_dbgeo */
+ int rc = mdbx_fastmutex_acquire(&env->me_remap_guard);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ if (limit_bytes == env->me_dxb_mmap.limit &&
+ size_bytes == env->me_dxb_mmap.current)
+ goto bailout;
-static __inline bool meta_ot(const enum meta_choise_mode mode,
- const MDBX_env *env, volatile const MDBX_meta *a,
- volatile const MDBX_meta *b) {
- mdbx_jitter4testing(true);
- const txnid_t txnid_a = meta_txnid(env, a);
- mdbx_jitter4testing(true);
- const txnid_t txnid_b = meta_txnid(env, b);
- mdbx_jitter4testing(true);
- const bool is_stead_b = META_IS_STEADY(b);
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (limit_bytes != env->me_dxb_mmap.limit && !(env->me_flags & MDBX_NOTLS) &&
+ lck && !implicit) {
+ int err = mdbx_rdt_lock(env) /* lock readers table until remap done */;
+ if (unlikely(MDBX_IS_ERROR(err))) {
+ rc = err;
+ goto bailout;
+ }
- if (mode == prefer_steady) {
- mdbx_jitter4testing(true);
- const bool is_stead_a = META_IS_STEADY(a);
- if (is_stead_a != is_stead_b)
- return is_stead_b;
- } else {
- mdbx_assert(env, mode == prefer_last);
+ /* looking for readers from this process */
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ mdbx_assert(env, !implicit);
+ mresize_flags |= MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ if (lck->mti_readers[i].mr_pid.weak == env->me_pid &&
+ lck->mti_readers[i].mr_tid.weak != mdbx_thread_self()) {
+ /* the base address of the mapping can't be changed since
+ * the other reader thread from this process exists. */
+ mdbx_rdt_unlock(env);
+ mresize_flags &= ~(MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE);
+ break;
+ }
+ }
}
- if (txnid_a == txnid_b)
- return is_stead_b;
- return txnid_a < txnid_b;
-}
+#endif /* ! Windows */
-static bool meta_eq(const MDBX_env *env, volatile const MDBX_meta *a,
- volatile const MDBX_meta *b) {
- mdbx_jitter4testing(true);
- const txnid_t txnid = meta_txnid(env, a);
- if (!txnid || txnid != meta_txnid(env, b))
- return false;
+ if ((env->me_flags & MDBX_WRITEMAP) && env->me_lck->mti_unsynced_pages.weak) {
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno),
+ MDBX_SYNC_NONE);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
- mdbx_jitter4testing(true);
- if (META_IS_STEADY(a) != META_IS_STEADY(b))
- return false;
+#if MDBX_ENABLE_MADVISE
+ if (size_bytes < prev_size) {
+ mdbx_notice("resize-MADV_%s %u..%u",
+ (env->me_flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED",
+ size_pgno, bytes2pgno(env, prev_size));
+ rc = MDBX_RESULT_TRUE;
+#if defined(MADV_REMOVE)
+ if (env->me_flags & MDBX_WRITEMAP)
+ rc =
+ madvise(env->me_map + size_bytes, prev_size - size_bytes, MADV_REMOVE)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+#endif /* MADV_REMOVE */
+#if defined(MADV_DONTNEED)
+ if (rc == MDBX_RESULT_TRUE)
+ rc = madvise(env->me_map + size_bytes, prev_size - size_bytes,
+ MADV_DONTNEED)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+#elif defined(POSIX_MADV_DONTNEED)
+ if (rc == MDBX_RESULT_TRUE)
+ rc = ignore_enosys(posix_madvise(env->me_map + size_bytes,
+ prev_size - size_bytes,
+ POSIX_MADV_DONTNEED));
+#elif defined(POSIX_FADV_DONTNEED)
+ if (rc == MDBX_RESULT_TRUE)
+ rc = ignore_enosys(posix_fadvise(env->me_lazy_fd, size_bytes,
+ prev_size - size_bytes,
+ POSIX_FADV_DONTNEED));
+#endif /* MADV_DONTNEED */
+ if (unlikely(MDBX_IS_ERROR(rc)))
+ goto bailout;
+ if (env->me_lck->mti_discarded_tail.weak > size_pgno)
+ env->me_lck->mti_discarded_tail.weak = size_pgno;
+ }
+#endif /* MDBX_ENABLE_MADVISE */
- mdbx_jitter4testing(true);
- return true;
-}
+ rc = mdbx_mresize(mresize_flags, &env->me_dxb_mmap, size_bytes, limit_bytes);
-static int meta_eq_mask(const MDBX_env *env) {
- volatile const MDBX_meta *m0 = METAPAGE(env, 0);
- volatile const MDBX_meta *m1 = METAPAGE(env, 1);
- volatile const MDBX_meta *m2 = METAPAGE(env, 2);
+#if MDBX_ENABLE_MADVISE
+ if (rc == MDBX_SUCCESS) {
+ env->me_lck->mti_discarded_tail.weak = size_pgno;
+ const bool readahead =
+ !(env->me_flags & MDBX_NORDAHEAD) &&
+ mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size);
+ const bool force = limit_bytes != prev_limit ||
+ env->me_dxb_mmap.address != prev_addr
+#if defined(_WIN32) || defined(_WIN64)
+ || prev_size > size_bytes
+#endif /* Windows */
+ ;
+ rc = mdbx_set_readahead(env, size_pgno, readahead, force);
+ }
+#endif /* MDBX_ENABLE_MADVISE */
- int rc = meta_eq(env, m0, m1) ? 1 : 0;
- if (meta_eq(env, m1, m2))
- rc += 2;
- if (meta_eq(env, m2, m0))
- rc += 4;
+bailout:
+ if (rc == MDBX_SUCCESS) {
+ mdbx_assert(env, size_bytes == env->me_dxb_mmap.current);
+ mdbx_assert(env, size_bytes <= env->me_dxb_mmap.filesize);
+ mdbx_assert(env, limit_bytes == env->me_dxb_mmap.limit);
+#ifdef MDBX_USE_VALGRIND
+ if (prev_limit != env->me_dxb_mmap.limit || prev_addr != env->me_map) {
+ VALGRIND_DISCARD(env->me_valgrind_handle);
+ env->me_valgrind_handle = 0;
+ if (env->me_dxb_mmap.limit)
+ env->me_valgrind_handle =
+ VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx");
+ }
+#endif /* MDBX_USE_VALGRIND */
+ } else {
+ if (rc != MDBX_UNABLE_EXTEND_MAPSIZE && rc != MDBX_EPERM) {
+ mdbx_error("failed resize datafile/mapping: "
+ "present %" PRIuPTR " -> %" PRIuPTR ", "
+ "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
+ prev_size, size_bytes, prev_limit, limit_bytes, rc);
+ } else {
+ mdbx_warning("unable resize datafile/mapping: "
+ "present %" PRIuPTR " -> %" PRIuPTR ", "
+ "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
+ prev_size, size_bytes, prev_limit, limit_bytes, rc);
+ }
+ if (!env->me_dxb_mmap.address) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ if (env->me_txn)
+ env->me_txn->mt_flags |= MDBX_TXN_ERROR;
+ rc = MDBX_PANIC;
+ }
+ }
+
+#if defined(_WIN32) || defined(_WIN64)
+ int err = MDBX_SUCCESS;
+ mdbx_srwlock_ReleaseExclusive(&env->me_remap_guard);
+ if (suspended) {
+ err = mdbx_resume_threads_after_remap(suspended);
+ if (suspended != &array_onstack)
+ mdbx_free(suspended);
+ }
+#else
+ if (env->me_lck_mmap.lck &&
+ (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0)
+ mdbx_rdt_unlock(env);
+ int err = mdbx_fastmutex_release(&env->me_remap_guard);
+#endif /* Windows */
+ if (err != MDBX_SUCCESS) {
+ mdbx_fatal("failed resume-after-remap: errcode %d", err);
+ return MDBX_PANIC;
+ }
return rc;
}
-static __inline volatile const MDBX_meta *
-meta_recent(const enum meta_choise_mode mode, const MDBX_env *env,
- volatile const MDBX_meta *a, volatile const MDBX_meta *b) {
- const bool a_older_that_b = meta_ot(mode, env, a, b);
- mdbx_assert(env, !meta_eq(env, a, b));
- return a_older_that_b ? b : a;
+__cold static int mdbx_mapresize_implicit(MDBX_env *env, const pgno_t used_pgno,
+ const pgno_t size_pgno,
+ const pgno_t limit_pgno) {
+ const pgno_t mapped_pgno = bytes2pgno(env, env->me_dxb_mmap.limit);
+ mdbx_assert(env, mapped_pgno >= used_pgno);
+ return mdbx_mapresize(
+ env, used_pgno, size_pgno,
+ (size_pgno > mapped_pgno)
+ ? limit_pgno
+ : /* The actual mapsize may be less since the geo.upper may be changed
+ by other process. So, avoids remapping until it necessary. */
+ mapped_pgno,
+ true);
}
-static const MDBX_meta *meta_ancient_prefer_weak(const MDBX_env *env,
- const MDBX_meta *a,
- const MDBX_meta *b) {
- const bool a_older_that_b = meta_ot(prefer_steady, env, a, b);
- mdbx_assert(env, !meta_eq(env, a, b));
- return a_older_that_b ? a : b;
+static int mdbx_meta_unsteady(MDBX_env *env, const txnid_t last_steady,
+ MDBX_meta *const meta, mdbx_filehandle_t fd) {
+ const uint64_t wipe = MDBX_DATASIGN_NONE;
+ if (unlikely(META_IS_STEADY(meta)) &&
+ constmeta_txnid(env, meta) <= last_steady) {
+ mdbx_warning("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, last_steady,
+ data_page(meta)->mp_pgno);
+ if (env->me_flags & MDBX_WRITEMAP)
+ unaligned_poke_u64(4, meta->mm_datasync_sign, wipe);
+ else
+ return mdbx_pwrite(fd, &wipe, sizeof(meta->mm_datasync_sign),
+ (uint8_t *)&meta->mm_datasync_sign - env->me_map);
+ if (constmeta_txnid(env, meta) == last_steady)
+ mdbx_assert(env, meta_checktxnid(env, meta, true));
+ }
+ return MDBX_SUCCESS;
}
-static __inline volatile const MDBX_meta *
-meta_mostrecent(const enum meta_choise_mode mode, const MDBX_env *env) {
- volatile const MDBX_meta *m0 = METAPAGE(env, 0);
- volatile const MDBX_meta *m1 = METAPAGE(env, 1);
- volatile const MDBX_meta *m2 = METAPAGE(env, 2);
+__cold static int mdbx_wipe_steady(MDBX_env *env, const txnid_t last_steady) {
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
+ ? env->me_dsync_fd
+ : env->me_lazy_fd;
+ int err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 0), fd);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 1), fd);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 2), fd);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- volatile const MDBX_meta *head = meta_recent(mode, env, m0, m1);
- head = meta_recent(mode, env, head, m2);
- return head;
-}
+ if (env->me_flags & MDBX_WRITEMAP) {
+ mdbx_flush_incoherent_cpu_writeback();
+ err = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
+ MDBX_SYNC_DATA);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ } else {
+ if (fd == env->me_lazy_fd) {
+#if MDBX_USE_SYNCFILERANGE
+ static bool syncfilerange_unavailable;
+ if (!syncfilerange_unavailable &&
+ sync_file_range(env->me_lazy_fd, 0, pgno2bytes(env, NUM_METAS),
+ SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER)) {
+ err = errno;
+ if (ignore_enosys(err) == MDBX_RESULT_TRUE)
+ syncfilerange_unavailable = true;
+ }
+ if (syncfilerange_unavailable)
+#endif /* MDBX_USE_SYNCFILERANGE */
+ err = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS),
+ env->me_os_psize);
+ }
-static volatile const MDBX_meta *meta_prefer_steady(const MDBX_env *env) {
- return meta_mostrecent(prefer_steady, env);
+ /* force oldest refresh */
+ atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, mo_Relaxed);
+ return MDBX_SUCCESS;
}
-MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta *
-constmeta_prefer_steady(const MDBX_env *env) {
- return (const MDBX_meta *)meta_mostrecent(prefer_steady, env);
-}
+/* Allocate page numbers and memory for writing. Maintain mt_last_reclaimed,
+ * mt_reclaimed_pglist and mt_next_pgno. Set MDBX_TXN_ERROR on failure.
+ *
+ * If there are free pages available from older transactions, they
+ * are re-used first. Otherwise allocate a new page at mt_next_pgno.
+ * Do not modify the GC, just merge GC records into mt_reclaimed_pglist
+ * and move mt_last_reclaimed to say which records were consumed. Only this
+ * function can create mt_reclaimed_pglist and move
+ * mt_last_reclaimed/mt_next_pgno.
+ *
+ * [in] mc cursor A cursor handle identifying the transaction and
+ * database for which we are allocating.
+ * [in] num the number of pages to allocate.
+ *
+ * Returns 0 on success, non-zero on failure.*/
-static volatile const MDBX_meta *meta_prefer_last(const MDBX_env *env) {
- return meta_mostrecent(prefer_last, env);
-}
+#define MDBX_ALLOC_CACHE 1
+#define MDBX_ALLOC_GC 2
+#define MDBX_ALLOC_NEW 4
+#define MDBX_ALLOC_SLOT 8
+#define MDBX_ALLOC_FAKE 16
+#define MDBX_ALLOC_ALL (MDBX_ALLOC_CACHE | MDBX_ALLOC_GC | MDBX_ALLOC_NEW)
-MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta *
-constmeta_prefer_last(const MDBX_env *env) {
- return (const MDBX_meta *)meta_mostrecent(prefer_last, env);
-}
+__hot static struct page_result mdbx_page_alloc(MDBX_cursor *mc,
+ const pgno_t num, int flags) {
+ struct page_result ret;
+ MDBX_txn *const txn = mc->mc_txn;
+ MDBX_env *const env = txn->mt_env;
+ mdbx_assert(env, num == 0 || !(flags & MDBX_ALLOC_SLOT));
+ mdbx_assert(env, num > 0 || !(flags & MDBX_ALLOC_NEW));
-static txnid_t mdbx_recent_committed_txnid(const MDBX_env *env) {
- while (true) {
- volatile const MDBX_meta *head = meta_prefer_last(env);
- const txnid_t recent = meta_txnid(env, head);
- mdbx_compiler_barrier();
- if (likely(head == meta_prefer_last(env) &&
- recent == meta_txnid(env, head)))
- return recent;
+ const unsigned coalesce_threshold =
+ env->me_maxgc_ov1page - env->me_maxgc_ov1page / 4;
+ if (likely(flags & MDBX_ALLOC_GC)) {
+ flags |= env->me_flags & (MDBX_COALESCE | MDBX_LIFORECLAIM);
+ if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) > coalesce_threshold)
+ flags &= ~MDBX_COALESCE;
+ if (unlikely(
+ /* If mc is updating the GC, then the retired-list cannot play
+ catch-up with itself by growing while trying to save it. */
+ (mc->mc_flags & C_RECLAIMING) ||
+ /* avoid (recursive) search inside empty tree and while tree is
+ updating, todo4recovery://erased_by_github/libmdbx/issues/31 */
+ txn->mt_dbs[FREE_DBI].md_entries == 0 ||
+ /* If our dirty list is already full, we can't touch GC */
+ (txn->tw.dirtyroom < txn->mt_dbs[FREE_DBI].md_depth &&
+ !(txn->mt_dbistate[FREE_DBI] & DBI_DIRTY))))
+ flags &= ~(MDBX_ALLOC_GC | MDBX_COALESCE);
}
-}
-static txnid_t mdbx_recent_steady_txnid(const MDBX_env *env) {
- while (true) {
- volatile const MDBX_meta *head = meta_prefer_steady(env);
- const txnid_t recent = meta_txnid(env, head);
- mdbx_compiler_barrier();
- if (likely(head == meta_prefer_steady(env) &&
- recent == meta_txnid(env, head)))
- return recent;
+ if (likely(num == 1 && (flags & MDBX_ALLOC_CACHE) != 0)) {
+ /* If there are any loose pages, just use them */
+ mdbx_assert(env, (flags & MDBX_ALLOC_SLOT) == 0);
+ if (likely(txn->tw.loose_pages)) {
+#if MDBX_ENABLE_REFUND
+ if (txn->tw.loose_refund_wl > txn->mt_next_pgno) {
+ mdbx_refund(txn);
+ if (unlikely(!txn->tw.loose_pages))
+ goto no_loose;
+ }
+#endif /* MDBX_ENABLE_REFUND */
+
+ ret.page = txn->tw.loose_pages;
+ txn->tw.loose_pages = ret.page->mp_next;
+ txn->tw.loose_count--;
+ mdbx_debug_extra("db %d use loose page %" PRIaPGNO, DDBI(mc),
+ ret.page->mp_pgno);
+ mdbx_tassert(txn, ret.page->mp_pgno < txn->mt_next_pgno);
+ mdbx_ensure(env, ret.page->mp_pgno >= NUM_METAS);
+ VALGRIND_MAKE_MEM_UNDEFINED(page_data(ret.page), page_space(txn->mt_env));
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(page_data(ret.page),
+ page_space(txn->mt_env));
+ ret.page->mp_txnid = txn->mt_front;
+ ret.err = MDBX_SUCCESS;
+ return ret;
+ }
}
-}
+#if MDBX_ENABLE_REFUND
+no_loose:
+#endif /* MDBX_ENABLE_REFUND */
-static const char *mdbx_durable_str(volatile const MDBX_meta *const meta) {
- if (META_IS_STEADY(meta))
- return (unaligned_peek_u64_volatile(4, meta->mm_datasync_sign) ==
- meta_sign((const MDBX_meta *)meta))
- ? "Steady"
- : "Tainted";
- return "Weak";
-}
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ pgno_t pgno, *re_list = txn->tw.reclaimed_pglist;
+ unsigned range_begin = 0, re_len = MDBX_PNL_SIZE(re_list);
+ txnid_t oldest = 0, last = 0;
-/*----------------------------------------------------------------------------*/
+ while (true) { /* hsr-kick retry loop */
+ MDBX_cursor_couple recur;
+ for (MDBX_cursor_op op = MDBX_FIRST;;
+ op = (flags & MDBX_LIFORECLAIM) ? MDBX_PREV : MDBX_NEXT) {
+ MDBX_val key, data;
-/* Find oldest txnid still referenced. */
-static txnid_t mdbx_find_oldest(const MDBX_txn *txn) {
- mdbx_tassert(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0);
- MDBX_env *env = txn->mt_env;
- const txnid_t edge = mdbx_recent_steady_txnid(env);
- mdbx_tassert(txn, edge <= txn->mt_txnid);
+ /* Seek a big enough contiguous page range.
+ * Prefer pages with lower pgno. */
+ mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno));
+ if (!(flags & (MDBX_COALESCE | MDBX_ALLOC_SLOT)) && re_len >= num) {
+ mdbx_tassert(txn, MDBX_PNL_LAST(re_list) < txn->mt_next_pgno &&
+ MDBX_PNL_FIRST(re_list) < txn->mt_next_pgno);
+ range_begin = MDBX_PNL_ASCENDING ? 1 : re_len;
+ pgno = MDBX_PNL_LEAST(re_list);
+ if (likely(num == 1))
+ goto done;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (unlikely(lck == NULL /* exclusive mode */))
- return atomic_store64(&lck->mti_oldest_reader, edge, mo_Relaxed);
+ const unsigned wanna_range = num - 1;
+#if MDBX_PNL_ASCENDING
+ mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1);
+ while (true) {
+ unsigned range_end = range_begin + wanna_range;
+ if (re_list[range_end] - pgno == wanna_range)
+ goto done;
+ if (range_end == re_len)
+ break;
+ pgno = re_list[++range_begin];
+ }
+#else
+ mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len);
+ while (true) {
+ if (re_list[range_begin - wanna_range] - pgno == wanna_range)
+ goto done;
+ if (range_begin == wanna_range)
+ break;
+ pgno = re_list[--range_begin];
+ }
+#endif /* MDBX_PNL sort-order */
+ }
- const txnid_t last_oldest =
- atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease);
- mdbx_tassert(txn, edge >= last_oldest);
- if (likely(last_oldest == edge))
- return edge;
+ if (op == MDBX_FIRST) { /* 1st iteration, setup cursor, etc */
+ if (unlikely(!(flags & MDBX_ALLOC_GC)))
+ break /* reclaiming is prohibited for now */;
- const uint32_t nothing_changed = MDBX_STRING_TETRAD("None");
- const uint32_t snap_readers_refresh_flag =
- atomic_load32(&lck->mti_readers_refresh_flag, mo_AcquireRelease);
- mdbx_jitter4testing(false);
- if (snap_readers_refresh_flag == nothing_changed)
- return last_oldest;
+ /* Prepare to fetch more and coalesce */
+ oldest = (flags & MDBX_LIFORECLAIM)
+ ? mdbx_find_oldest(txn)
+ : atomic_load64(&env->me_lck->mti_oldest_reader,
+ mo_AcquireRelease);
+ ret.err = mdbx_cursor_init(&recur.outer, txn, FREE_DBI);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ goto fail;
+ if (flags & MDBX_LIFORECLAIM) {
+ /* Begin from oldest reader if any */
+ if (oldest > MIN_TXNID) {
+ last = oldest - 1;
+ op = MDBX_SET_RANGE;
+ }
+ } else if (txn->tw.last_reclaimed) {
+ /* Continue lookup from txn->tw.last_reclaimed to oldest reader */
+ last = txn->tw.last_reclaimed;
+ op = MDBX_SET_RANGE;
+ }
- txnid_t oldest = edge;
- atomic_store32(&lck->mti_readers_refresh_flag, nothing_changed, mo_Relaxed);
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
- /* mdbx_jitter4testing(true); */
- const txnid_t snap = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (oldest > snap && last_oldest <= /* ignore pending updates */ snap) {
- oldest = snap;
- if (oldest == last_oldest)
- return oldest;
+ key.iov_base = &last;
+ key.iov_len = sizeof(last);
}
- }
- }
- if (oldest != last_oldest) {
- mdbx_verbose("update oldest %" PRIaTXN " -> %" PRIaTXN, last_oldest,
- oldest);
- mdbx_tassert(txn, oldest >= lck->mti_oldest_reader.weak);
- atomic_store64(&lck->mti_oldest_reader, oldest, mo_Relaxed);
- }
- return oldest;
-}
-
-/* Find largest mvcc-snapshot still referenced. */
-__cold static pgno_t mdbx_find_largest(MDBX_env *env, pgno_t largest) {
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (likely(lck != NULL /* exclusive mode */)) {
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- retry:
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
- /* mdbx_jitter4testing(true); */
- const pgno_t snap_pages = atomic_load32(
- &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed);
- const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (unlikely(
- snap_pages !=
- atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used,
- mo_AcquireRelease) ||
- snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
- goto retry;
- if (largest < snap_pages &&
- atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease) <=
- /* ignore pending updates */ snap_txnid &&
- snap_txnid <= env->me_txn0->mt_txnid)
- largest = snap_pages;
+ if (!(flags & MDBX_LIFORECLAIM)) {
+ /* Do not try fetch more if the record will be too recent */
+ if (op != MDBX_FIRST && ++last >= oldest) {
+ oldest = mdbx_find_oldest(txn);
+ if (oldest <= last)
+ break;
+ }
}
- }
- }
-
- return largest;
-}
-/* Add a page to the txn's dirty list */
-static int __must_check_result mdbx_page_dirty(MDBX_txn *txn, MDBX_page *mp,
- unsigned npages) {
-#if xMDBX_DEBUG_SPILLING == 2
- txn->mt_env->debug_dirtied_act += 1;
- mdbx_ensure(txn->mt_env,
- txn->mt_env->debug_dirtied_act < txn->mt_env->debug_dirtied_est);
- mdbx_ensure(txn->mt_env, txn->tw.dirtyroom + txn->tw.loose_count > 0);
-#endif /* xMDBX_DEBUG_SPILLING == 2 */
+ ret.err = mdbx_cursor_get(&recur.outer, &key, NULL, op);
+ if (ret.err == MDBX_NOTFOUND && (flags & MDBX_LIFORECLAIM)) {
+ if (op == MDBX_SET_RANGE)
+ continue;
+ txnid_t snap = mdbx_find_oldest(txn);
+ if (oldest < snap) {
+ oldest = snap;
+ last = oldest - 1;
+ key.iov_base = &last;
+ key.iov_len = sizeof(last);
+ op = MDBX_SET_RANGE;
+ ret.err = mdbx_cursor_get(&recur.outer, &key, NULL, op);
+ }
+ }
+ if (unlikely(ret.err)) {
+ if (ret.err == MDBX_NOTFOUND)
+ break;
+ goto fail;
+ }
- int rc;
- mp->mp_txnid = txn->mt_front;
- if (unlikely(txn->tw.dirtyroom == 0)) {
- if (txn->tw.loose_count) {
- MDBX_page *loose = txn->tw.loose_pages;
- mdbx_debug("purge-and-reclaim loose page %" PRIaPGNO, loose->mp_pgno);
- rc = mdbx_pnl_insert_range(&txn->tw.reclaimed_pglist, loose->mp_pgno, 1);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- unsigned di = mdbx_dpl_search(txn, loose->mp_pgno);
- mdbx_tassert(txn, txn->tw.dirtylist->items[di].ptr == loose);
- mdbx_dpl_remove(txn, di);
- txn->tw.loose_pages = loose->mp_next;
- txn->tw.loose_count--;
- txn->tw.dirtyroom++;
- if (!(txn->mt_flags & MDBX_WRITEMAP))
- mdbx_dpage_free(txn->mt_env, loose, 1);
- } else {
- mdbx_error("Dirtyroom is depleted, DPL length %u",
- txn->tw.dirtylist->length);
- if (!(txn->mt_flags & MDBX_WRITEMAP))
- mdbx_dpage_free(txn->mt_env, mp, npages);
- return MDBX_TXN_FULL;
- }
- }
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(key.iov_len != sizeof(txnid_t))) {
+ ret.err = MDBX_CORRUPTED;
+ goto fail;
+ }
+ last = unaligned_peek_u64(4, key.iov_base);
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(last < MIN_TXNID || last > MAX_TXNID)) {
+ ret.err = MDBX_CORRUPTED;
+ goto fail;
+ }
+ if (oldest <= last) {
+ oldest = mdbx_find_oldest(txn);
+ if (oldest <= last) {
+ if (flags & MDBX_LIFORECLAIM)
+ continue;
+ break;
+ }
+ }
- rc = mdbx_dpl_append(txn, mp->mp_pgno, mp, npages);
- if (unlikely(rc != MDBX_SUCCESS)) {
- bailout:
- txn->mt_flags |= MDBX_TXN_ERROR;
- return rc;
- }
- txn->tw.dirtyroom--;
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- return MDBX_SUCCESS;
-}
+ if (flags & MDBX_LIFORECLAIM) {
+ /* skip IDs of records that already reclaimed */
+ if (txn->tw.lifo_reclaimed) {
+ size_t i;
+ for (i = (size_t)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); i > 0; --i)
+ if (txn->tw.lifo_reclaimed[i] == last)
+ break;
+ if (i)
+ continue;
+ }
+ }
-#if !(defined(_WIN32) || defined(_WIN64))
-MDBX_MAYBE_UNUSED static __always_inline int ignore_enosys(int err) {
-#ifdef ENOSYS
- if (err == ENOSYS)
- return MDBX_RESULT_TRUE;
-#endif /* ENOSYS */
-#ifdef ENOIMPL
- if (err == ENOIMPL)
- return MDBX_RESULT_TRUE;
-#endif /* ENOIMPL */
-#ifdef ENOTSUP
- if (err == ENOTSUP)
- return MDBX_RESULT_TRUE;
-#endif /* ENOTSUP */
-#ifdef ENOSUPP
- if (err == ENOSUPP)
- return MDBX_RESULT_TRUE;
-#endif /* ENOSUPP */
-#ifdef EOPNOTSUPP
- if (err == EOPNOTSUPP)
- return MDBX_RESULT_TRUE;
-#endif /* EOPNOTSUPP */
- if (err == EAGAIN)
- return MDBX_RESULT_TRUE;
- return err;
-}
-#endif /* defined(_WIN32) || defined(_WIN64) */
+ /* Reading next GC record */
+ MDBX_page *const mp = recur.outer.mc_pg[recur.outer.mc_top];
+ if (unlikely((ret.err = mdbx_node_read(
+ &recur.outer,
+ page_node(mp, recur.outer.mc_ki[recur.outer.mc_top]),
+ &data, pp_txnid4chk(mp, txn))) != MDBX_SUCCESS))
+ goto fail;
-#if MDBX_ENABLE_MADVISE
-/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */
-__cold static int mdbx_set_readahead(MDBX_env *env, const pgno_t edge,
- const bool enable,
- const bool force_whole) {
- mdbx_assert(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1);
- mdbx_assert(env, (enable & 1) == (enable != 0));
- const bool toggle = force_whole ||
- ((enable ^ env->me_lck->mti_readahead_anchor) & 1) ||
- !env->me_lck->mti_readahead_anchor;
- const pgno_t prev_edge = env->me_lck->mti_readahead_anchor >> 1;
- const size_t limit = env->me_dxb_mmap.limit;
- size_t offset =
- toggle ? 0
- : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge);
- offset = (offset < limit) ? offset : limit;
+ if ((flags & MDBX_LIFORECLAIM) && !txn->tw.lifo_reclaimed) {
+ txn->tw.lifo_reclaimed = mdbx_txl_alloc();
+ if (unlikely(!txn->tw.lifo_reclaimed)) {
+ ret.err = MDBX_ENOMEM;
+ goto fail;
+ }
+ }
- size_t length =
- pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge);
- length = (length < limit) ? length : limit;
- length -= offset;
+ /* Append PNL from GC record to tw.reclaimed_pglist */
+ mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
+ pgno_t *gc_pnl = (pgno_t *)data.iov_base;
+ mdbx_tassert(txn, data.iov_len >= MDBX_PNL_SIZEOF(gc_pnl));
+ if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) ||
+ !mdbx_pnl_check(gc_pnl, txn->mt_next_pgno))) {
+ ret.err = MDBX_CORRUPTED;
+ goto fail;
+ }
+ const unsigned gc_len = MDBX_PNL_SIZE(gc_pnl);
+ if (unlikely(/* list is too long already */ MDBX_PNL_SIZE(
+ txn->tw.reclaimed_pglist) >=
+ env->me_options.rp_augment_limit) &&
+ ((/* not a slot-request from gc-update */
+ (flags & MDBX_ALLOC_SLOT) == 0 &&
+ /* have enough unallocated space */ txn->mt_geo.upper >=
+ txn->mt_next_pgno + (size_t)num) ||
+ gc_len + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) >=
+ MDBX_PGL_LIMIT)) {
+ /* Stop reclaiming to avoid overflow the page list.
+ * This is a rare case while search for a continuously multi-page region
+ * in a large database.
+ * todo4recovery://erased_by_github/libmdbx/issues/123 */
+ mdbx_notice("stop reclaiming to avoid PNL overflow: %u (current) + %u "
+ "(chunk) -> %u",
+ MDBX_PNL_SIZE(txn->tw.reclaimed_pglist), gc_len,
+ gc_len + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
+ flags &= ~(MDBX_ALLOC_GC | MDBX_COALESCE);
+ break;
+ }
+ ret.err = mdbx_pnl_need(&txn->tw.reclaimed_pglist, gc_len);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ goto fail;
+ re_list = txn->tw.reclaimed_pglist;
- mdbx_assert(env, 0 <= (intptr_t)length);
- if (length == 0)
- return MDBX_SUCCESS;
+ /* Remember ID of GC record */
+ if (flags & MDBX_LIFORECLAIM) {
+ ret.err = mdbx_txl_append(&txn->tw.lifo_reclaimed, last);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ goto fail;
+ }
+ txn->tw.last_reclaimed = last;
- mdbx_notice("readahead %s %u..%u", enable ? "ON" : "OFF",
- bytes2pgno(env, offset), bytes2pgno(env, offset + length));
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) {
+ mdbx_debug_extra("PNL read txn %" PRIaTXN " root %" PRIaPGNO
+ " num %u, PNL",
+ last, txn->mt_dbs[FREE_DBI].md_root, gc_len);
+ for (unsigned i = gc_len; i; i--)
+ mdbx_debug_extra_print(" %" PRIaPGNO, gc_pnl[i]);
+ mdbx_debug_extra_print("%s\n", ".");
+ }
-#if defined(F_RDAHEAD)
- if (toggle && unlikely(fcntl(env->me_lazy_fd, F_RDAHEAD, enable) == -1))
- return errno;
-#endif /* F_RDAHEAD */
+ /* Merge in descending sorted order */
+ const unsigned prev_re_len = MDBX_PNL_SIZE(re_list);
+ mdbx_pnl_xmerge(re_list, gc_pnl);
+ /* re-check to avoid duplicates */
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(!mdbx_pnl_check(re_list, txn->mt_next_pgno))) {
+ ret.err = MDBX_CORRUPTED;
+ goto fail;
+ }
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- int err;
- if (enable) {
-#if defined(MADV_NORMAL)
- err = madvise(env->me_map + offset, length, MADV_NORMAL)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_MADV_NORMAL)
- err = ignore_enosys(
- posix_madvise(env->me_map + offset, length, POSIX_MADV_NORMAL));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED)
- err = ignore_enosys(
- posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_NORMAL));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(_WIN32) || defined(_WIN64)
- /* no madvise on Windows */
-#else
-#warning "FIXME"
-#endif
- if (toggle) {
- /* NOTE: Seems there is a bug in the Mach/Darwin/OSX kernel,
- * because MADV_WILLNEED with offset != 0 may cause SIGBUS
- * on following access to the hinted region.
- * 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021;
- * root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64 */
-#if defined(F_RDADVISE)
- struct radvisory hint;
- hint.ra_offset = offset;
- hint.ra_count = length;
- (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl(
- env->me_lazy_fd, F_RDADVISE, &hint);
-#elif defined(MADV_WILLNEED)
- err = madvise(env->me_map + offset, length, MADV_WILLNEED)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_MADV_WILLNEED)
- err = ignore_enosys(
- posix_madvise(env->me_map + offset, length, POSIX_MADV_WILLNEED));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(_WIN32) || defined(_WIN64)
- if (mdbx_PrefetchVirtualMemory) {
- WIN32_MEMORY_RANGE_ENTRY hint;
- hint.VirtualAddress = env->me_map + offset;
- hint.NumberOfBytes = length;
- (void)mdbx_PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0);
+ re_len = MDBX_PNL_SIZE(re_list);
+ mdbx_tassert(txn, re_len == 0 || re_list[re_len] < txn->mt_next_pgno);
+ if (MDBX_ENABLE_REFUND && re_len &&
+ unlikely(MDBX_PNL_MOST(re_list) == txn->mt_next_pgno - 1)) {
+ /* Refund suitable pages into "unallocated" space */
+ mdbx_refund(txn);
+ re_list = txn->tw.reclaimed_pglist;
+ re_len = MDBX_PNL_SIZE(re_list);
}
-#elif defined(POSIX_FADV_WILLNEED)
- err = ignore_enosys(
- posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_WILLNEED));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#else
-#warning "FIXME"
-#endif
- }
- } else {
-#if defined(MADV_RANDOM)
- err = madvise(env->me_map + offset, length, MADV_RANDOM)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_MADV_RANDOM)
- err = ignore_enosys(
- posix_madvise(env->me_map + offset, length, POSIX_MADV_RANDOM));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_FADV_RANDOM)
- err = ignore_enosys(
- posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_RANDOM));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(_WIN32) || defined(_WIN64)
- /* no madvise on Windows */
-#else
-#warning "FIXME"
-#endif /* MADV_RANDOM */
- }
- env->me_lck->mti_readahead_anchor = (enable & 1) + (edge << 1);
- err = MDBX_SUCCESS;
- return err;
-}
-#endif /* MDBX_ENABLE_MADVISE */
+ /* Done for a kick-reclaim mode, actually no page needed */
+ if (unlikely(flags & MDBX_ALLOC_SLOT)) {
+ mdbx_debug("early-return NULL-page for %s mode", "MDBX_ALLOC_SLOT");
+ ret.err = MDBX_SUCCESS;
+ ret.page = NULL;
+ return ret;
+ }
-__cold static int mdbx_mapresize(MDBX_env *env, const pgno_t used_pgno,
- const pgno_t size_pgno,
- const pgno_t limit_pgno, const bool implicit) {
- const size_t limit_bytes = pgno_align2os_bytes(env, limit_pgno);
- const size_t size_bytes = pgno_align2os_bytes(env, size_pgno);
- const size_t prev_size = env->me_dxb_mmap.current;
- const size_t prev_limit = env->me_dxb_mmap.limit;
-#if MDBX_ENABLE_MADVISE || defined(MDBX_USE_VALGRIND)
- const void *const prev_addr = env->me_map;
-#endif /* MDBX_ENABLE_MADVISE || MDBX_USE_VALGRIND */
+ /* Don't try to coalesce too much. */
+ if (flags & MDBX_COALESCE) {
+ if (re_len /* current size */ > coalesce_threshold ||
+ (re_len > prev_re_len &&
+ re_len - prev_re_len /* delta from prev */ >=
+ coalesce_threshold / 2)) {
+ mdbx_trace("clear %s %s", "MDBX_COALESCE", "since got threshold");
+ flags &= ~MDBX_COALESCE;
+ }
+ }
+ }
- mdbx_verbose("resize datafile/mapping: "
- "present %" PRIuPTR " -> %" PRIuPTR ", "
- "limit %" PRIuPTR " -> %" PRIuPTR,
- prev_size, size_bytes, prev_limit, limit_bytes);
+ if (F_ISSET(flags, MDBX_COALESCE | MDBX_ALLOC_GC)) {
+ mdbx_debug_extra("clear %s and continue", "MDBX_COALESCE");
+ flags &= ~MDBX_COALESCE;
+ continue;
+ }
- mdbx_assert(env, limit_bytes >= size_bytes);
- mdbx_assert(env, bytes2pgno(env, size_bytes) >= size_pgno);
- mdbx_assert(env, bytes2pgno(env, limit_bytes) >= limit_pgno);
+ /* There is no suitable pages in the GC and to be able to allocate
+ * we should CHOICE one of:
+ * - make a new steady checkpoint if reclaiming was stopped by
+ * the last steady-sync, or wipe it in the MDBX_UTTERLY_NOSYNC mode;
+ * - kick lagging reader(s) if reclaiming was stopped by ones of it.
+ * - extend the database file. */
- unsigned mresize_flags =
- env->me_flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC);
-#if defined(_WIN32) || defined(_WIN64)
- /* Acquire guard in exclusive mode for:
- * - to avoid collision between read and write txns around env->me_dbgeo;
- * - to avoid attachment of new reading threads (see mdbx_rdt_lock); */
- mdbx_srwlock_AcquireExclusive(&env->me_remap_guard);
- mdbx_handle_array_t *suspended = NULL;
- mdbx_handle_array_t array_onstack;
- int rc = MDBX_SUCCESS;
- if (limit_bytes == env->me_dxb_mmap.limit &&
- size_bytes == env->me_dxb_mmap.current &&
- size_bytes == env->me_dxb_mmap.filesize)
- goto bailout;
+ /* Will use new pages from the map if nothing is suitable in the GC. */
+ range_begin = 0;
+ pgno = txn->mt_next_pgno;
+ const size_t next = (size_t)pgno + num;
- if ((env->me_flags & MDBX_NOTLS) == 0) {
- /* 1) Windows allows only extending a read-write section, but not a
- * corresponding mapped view. Therefore in other cases we must suspend
- * the local threads for safe remap.
- * 2) At least on Windows 10 1803 the entire mapped section is unavailable
- * for short time during NtExtendSection() or VirtualAlloc() execution.
- * 3) Under Wine runtime environment on Linux a section extending is not
- * supported.
- *
- * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */
- array_onstack.limit = ARRAY_LENGTH(array_onstack.handles);
- array_onstack.count = 0;
- suspended = &array_onstack;
- rc = mdbx_suspend_threads_before_remap(env, &suspended);
- if (rc != MDBX_SUCCESS) {
- mdbx_error("failed suspend-for-remap: errcode %d", rc);
- goto bailout;
+ if (flags & MDBX_ALLOC_GC) {
+ const MDBX_meta *const head = constmeta_prefer_last(env);
+ const MDBX_meta *const steady = constmeta_prefer_steady(env);
+ /* does reclaiming stopped at the last steady point? */
+ if (head != steady && META_IS_STEADY(steady) &&
+ oldest == constmeta_txnid(env, steady)) {
+ mdbx_debug("gc-kick-steady: head %" PRIaTXN "-%s, tail %" PRIaTXN
+ "-%s, oldest %" PRIaTXN,
+ constmeta_txnid(env, head), mdbx_durable_str(head),
+ constmeta_txnid(env, steady), mdbx_durable_str(steady),
+ oldest);
+ ret.err = MDBX_RESULT_TRUE;
+ const pgno_t autosync_threshold =
+ atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
+ const uint64_t autosync_period =
+ atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
+ /* wipe the last steady-point if one of:
+ * - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified
+ * - UTTERLY_NOSYNC mode AND free space at steady-point is exhausted
+ * otherwise, make a new steady-point if one of:
+ * - auto-sync threshold is specified and reached;
+ * - upper limit of database size is reached;
+ * - database is full (with the current file size)
+ * AND auto-sync threshold it NOT specified */
+ if (F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC) &&
+ ((autosync_threshold | autosync_period) == 0 ||
+ next >= steady->mm_geo.now)) {
+ /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode
+ * without any auto-sync threshold(s). */
+ ret.err = mdbx_wipe_steady(env, oldest);
+ mdbx_debug("gc-wipe-steady, rc %d", ret.err);
+ mdbx_assert(env, steady != meta_prefer_steady(env));
+ } else if ((flags & MDBX_ALLOC_NEW) == 0 ||
+ (autosync_threshold &&
+ atomic_load32(&env->me_lck->mti_unsynced_pages,
+ mo_Relaxed) >= autosync_threshold) ||
+ (autosync_period &&
+ mdbx_osal_monotime() -
+ atomic_load64(&env->me_lck->mti_sync_timestamp,
+ mo_Relaxed) >=
+ autosync_period) ||
+ next >= txn->mt_geo.upper ||
+ (next >= txn->mt_end_pgno &&
+ (autosync_threshold | autosync_period) == 0)) {
+ /* make steady checkpoint. */
+ MDBX_meta meta = *head;
+ ret.err = mdbx_sync_locked(env, env->me_flags & MDBX_WRITEMAP, &meta);
+ mdbx_debug("gc-make-steady, rc %d", ret.err);
+ mdbx_assert(env, steady != meta_prefer_steady(env));
+ }
+ if (ret.err == MDBX_SUCCESS) {
+ if (mdbx_find_oldest(txn) > oldest)
+ continue;
+ /* it is reasonable check/kick lagging reader(s) here,
+ * since we made a new steady point or wipe the last. */
+ if (oldest < txn->mt_txnid - xMDBX_TXNID_STEP &&
+ mdbx_kick_longlived_readers(env, oldest) > oldest)
+ continue;
+ } else if (unlikely(ret.err != MDBX_RESULT_TRUE))
+ goto fail;
+ }
}
- mresize_flags |= implicit ? MDBX_MRESIZE_MAY_UNMAP
- : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
- }
-#else /* Windows */
- /* Acquire guard to avoid collision between read and write txns
- * around env->me_dbgeo */
- int rc = mdbx_fastmutex_acquire(&env->me_remap_guard);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- if (limit_bytes == env->me_dxb_mmap.limit &&
- size_bytes == env->me_dxb_mmap.current)
- goto bailout;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (limit_bytes != env->me_dxb_mmap.limit && !(env->me_flags & MDBX_NOTLS) &&
- lck && !implicit) {
- int err = mdbx_rdt_lock(env) /* lock readers table until remap done */;
- if (unlikely(MDBX_IS_ERROR(err))) {
- rc = err;
- goto bailout;
- }
+ /* don't kick lagging reader(s) if is enough unallocated space
+ * at the end of database file. */
+ if ((flags & MDBX_ALLOC_NEW) && next <= txn->mt_end_pgno)
+ goto done;
+ if ((flags & MDBX_ALLOC_GC) && oldest < txn->mt_txnid - xMDBX_TXNID_STEP &&
+ mdbx_kick_longlived_readers(env, oldest) > oldest)
+ continue;
- /* looking for readers from this process */
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- mdbx_assert(env, !implicit);
- mresize_flags |= MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- if (lck->mti_readers[i].mr_pid.weak == env->me_pid &&
- lck->mti_readers[i].mr_tid.weak != mdbx_thread_self()) {
- /* the base address of the mapping can't be changed since
- * the other reader thread from this process exists. */
- mdbx_rdt_unlock(env);
- mresize_flags &= ~(MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE);
- break;
+ ret.err = MDBX_NOTFOUND;
+ if (flags & MDBX_ALLOC_NEW) {
+ ret.err = MDBX_MAP_FULL;
+ if (next < txn->mt_geo.upper && txn->mt_geo.grow_pv) {
+ mdbx_assert(env, next > txn->mt_end_pgno);
+ const pgno_t grow_step = pv2pages(txn->mt_geo.grow_pv);
+ size_t aligned = pgno_align2os_pgno(
+ env, (pgno_t)(next + grow_step - next % grow_step));
+
+ if (aligned > txn->mt_geo.upper)
+ aligned = txn->mt_geo.upper;
+ mdbx_assert(env, aligned > txn->mt_end_pgno);
+
+ mdbx_verbose("try growth datafile to %zu pages (+%zu)", aligned,
+ aligned - txn->mt_end_pgno);
+ ret.err = mdbx_mapresize_implicit(env, txn->mt_next_pgno,
+ (pgno_t)aligned, txn->mt_geo.upper);
+ if (ret.err == MDBX_SUCCESS) {
+ env->me_txn->mt_end_pgno = (pgno_t)aligned;
+ goto done;
+ }
+
+ mdbx_error("unable growth datafile to %zu pages (+%zu), errcode %d",
+ aligned, aligned - txn->mt_end_pgno, ret.err);
+ } else {
+ mdbx_notice("gc-alloc: next %zu > upper %" PRIaPGNO, next,
+ txn->mt_geo.upper);
}
}
- }
-#endif /* ! Windows */
-
- if ((env->me_flags & MDBX_WRITEMAP) && env->me_lck->mti_unsynced_pages.weak) {
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno),
- MDBX_SYNC_NONE);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
-#if MDBX_ENABLE_MADVISE
- if (size_bytes < prev_size) {
- mdbx_notice("resize-MADV_%s %u..%u",
- (env->me_flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED",
- size_pgno, bytes2pgno(env, prev_size));
- rc = MDBX_RESULT_TRUE;
-#if defined(MADV_REMOVE)
- if (env->me_flags & MDBX_WRITEMAP)
- rc =
- madvise(env->me_map + size_bytes, prev_size - size_bytes, MADV_REMOVE)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
-#endif /* MADV_REMOVE */
-#if defined(MADV_DONTNEED)
- if (rc == MDBX_RESULT_TRUE)
- rc = madvise(env->me_map + size_bytes, prev_size - size_bytes,
- MADV_DONTNEED)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
-#elif defined(POSIX_MADV_DONTNEED)
- if (rc == MDBX_RESULT_TRUE)
- rc = ignore_enosys(posix_madvise(env->me_map + size_bytes,
- prev_size - size_bytes,
- POSIX_MADV_DONTNEED));
-#elif defined(POSIX_FADV_DONTNEED)
- if (rc == MDBX_RESULT_TRUE)
- rc = ignore_enosys(posix_fadvise(env->me_lazy_fd, size_bytes,
- prev_size - size_bytes,
- POSIX_FADV_DONTNEED));
-#endif /* MADV_DONTNEED */
- if (unlikely(MDBX_IS_ERROR(rc)))
- goto bailout;
- if (env->me_lck->mti_discarded_tail.weak > size_pgno)
- env->me_lck->mti_discarded_tail.weak = size_pgno;
+ fail:
+ mdbx_assert(env,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ if (likely(!(flags & MDBX_ALLOC_FAKE)))
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ if (num != 1 || ret.err != MDBX_NOTFOUND)
+ mdbx_notice("alloc %u pages failed, flags 0x%x, errcode %d", num, flags,
+ ret.err);
+ else
+ mdbx_trace("alloc %u pages failed, flags 0x%x, errcode %d", num, flags,
+ ret.err);
+ mdbx_assert(env, ret.err != MDBX_SUCCESS);
+ ret.page = NULL;
+ return ret;
}
-#endif /* MDBX_ENABLE_MADVISE */
-
- rc = mdbx_mresize(mresize_flags, &env->me_dxb_mmap, size_bytes, limit_bytes);
-#if MDBX_ENABLE_MADVISE
- if (rc == MDBX_SUCCESS) {
- env->me_lck->mti_discarded_tail.weak = size_pgno;
- const bool readahead =
- !(env->me_flags & MDBX_NORDAHEAD) &&
- mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size);
- const bool force = limit_bytes != prev_limit ||
- env->me_dxb_mmap.address != prev_addr
-#if defined(_WIN32) || defined(_WIN64)
- || prev_size > size_bytes
-#endif /* Windows */
- ;
- rc = mdbx_set_readahead(env, size_pgno, readahead, force);
+done:
+ mdbx_assert(env, !(flags & MDBX_ALLOC_SLOT));
+ mdbx_ensure(env, pgno >= NUM_METAS);
+ if (unlikely(flags & MDBX_ALLOC_FAKE)) {
+ mdbx_debug("return NULL-page for %u pages of %s mode", num,
+ "MDBX_ALLOC_FAKE");
+ ret.page = NULL;
+ ret.err = MDBX_SUCCESS;
+ return ret;
}
-#endif /* MDBX_ENABLE_MADVISE */
-bailout:
- if (rc == MDBX_SUCCESS) {
- mdbx_assert(env, size_bytes == env->me_dxb_mmap.current);
- mdbx_assert(env, size_bytes <= env->me_dxb_mmap.filesize);
- mdbx_assert(env, limit_bytes == env->me_dxb_mmap.limit);
-#ifdef MDBX_USE_VALGRIND
- if (prev_limit != env->me_dxb_mmap.limit || prev_addr != env->me_map) {
- VALGRIND_DISCARD(env->me_valgrind_handle);
- env->me_valgrind_handle = 0;
- if (env->me_dxb_mmap.limit)
- env->me_valgrind_handle =
- VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx");
- }
-#endif /* MDBX_USE_VALGRIND */
+ if (env->me_flags & MDBX_WRITEMAP) {
+ ret.page = pgno2page(env, pgno);
+ /* LY: reset no-access flag from mdbx_page_loose() */
+ VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(ret.page, pgno2bytes(env, num));
} else {
- if (rc != MDBX_UNABLE_EXTEND_MAPSIZE && rc != MDBX_EPERM) {
- mdbx_error("failed resize datafile/mapping: "
- "present %" PRIuPTR " -> %" PRIuPTR ", "
- "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
- prev_size, size_bytes, prev_limit, limit_bytes, rc);
- } else {
- mdbx_warning("unable resize datafile/mapping: "
- "present %" PRIuPTR " -> %" PRIuPTR ", "
- "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
- prev_size, size_bytes, prev_limit, limit_bytes, rc);
- }
- if (!env->me_dxb_mmap.address) {
- env->me_flags |= MDBX_FATAL_ERROR;
- if (env->me_txn)
- env->me_txn->mt_flags |= MDBX_TXN_ERROR;
- rc = MDBX_PANIC;
+ ret.page = mdbx_page_malloc(txn, num);
+ if (unlikely(!ret.page)) {
+ ret.err = MDBX_ENOMEM;
+ goto fail;
}
}
-#if defined(_WIN32) || defined(_WIN64)
- int err = MDBX_SUCCESS;
- mdbx_srwlock_ReleaseExclusive(&env->me_remap_guard);
- if (suspended) {
- err = mdbx_resume_threads_after_remap(suspended);
- if (suspended != &array_onstack)
- mdbx_free(suspended);
- }
+ if (range_begin) {
+ mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
+ mdbx_tassert(txn, pgno < txn->mt_next_pgno);
+ mdbx_tassert(txn, pgno == re_list[range_begin]);
+ /* Cutoff allocated pages from tw.reclaimed_pglist */
+#if MDBX_PNL_ASCENDING
+ for (unsigned i = range_begin + num; i <= re_len;)
+ re_list[range_begin++] = re_list[i++];
+ MDBX_PNL_SIZE(re_list) = re_len = range_begin - 1;
#else
- if (env->me_lck_mmap.lck &&
- (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0)
- mdbx_rdt_unlock(env);
- int err = mdbx_fastmutex_release(&env->me_remap_guard);
-#endif /* Windows */
- if (err != MDBX_SUCCESS) {
- mdbx_fatal("failed resume-after-remap: errcode %d", err);
- return MDBX_PANIC;
- }
- return rc;
-}
-
-__cold static int mdbx_mapresize_implicit(MDBX_env *env, const pgno_t used_pgno,
- const pgno_t size_pgno,
- const pgno_t limit_pgno) {
- const pgno_t mapped_pgno = bytes2pgno(env, env->me_dxb_mmap.limit);
- mdbx_assert(env, mapped_pgno >= used_pgno);
- return mdbx_mapresize(
- env, used_pgno, size_pgno,
- (size_pgno > mapped_pgno)
- ? limit_pgno
- : /* The actual mapsize may be less since the geo.upper may be changed
- by other process. So, avoids remapping until it necessary. */
- mapped_pgno,
- true);
-}
+ MDBX_PNL_SIZE(re_list) = re_len -= num;
+ for (unsigned i = range_begin - num; i < re_len;)
+ re_list[++i] = re_list[++range_begin];
+#endif
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ } else {
+ txn->mt_next_pgno = pgno + num;
+ mdbx_assert(env, txn->mt_next_pgno <= txn->mt_end_pgno);
+ }
-static int mdbx_meta_unsteady(MDBX_env *env, const txnid_t last_steady,
- MDBX_meta *const meta, mdbx_filehandle_t fd) {
- const uint64_t wipe = MDBX_DATASIGN_NONE;
- if (unlikely(META_IS_STEADY(meta)) &&
- constmeta_txnid(env, meta) <= last_steady) {
- mdbx_warning("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, last_steady,
- data_page(meta)->mp_pgno);
- if (env->me_flags & MDBX_WRITEMAP)
- unaligned_poke_u64(4, meta->mm_datasync_sign, wipe);
- else
- return mdbx_pwrite(fd, &wipe, sizeof(meta->mm_datasync_sign),
- (uint8_t *)&meta->mm_datasync_sign - env->me_map);
- if (constmeta_txnid(env, meta) == last_steady)
- mdbx_assert(env, meta_checktxnid(env, meta, true));
+ if (unlikely(env->me_flags & MDBX_PAGEPERTURB))
+ memset(ret.page, -1, pgno2bytes(env, num));
+ VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));
+
+ ret.page->mp_pgno = pgno;
+ ret.page->mp_leaf2_ksize = 0;
+ ret.page->mp_flags = 0;
+ if ((mdbx_assert_enabled() || mdbx_audit_enabled()) && num > 1) {
+ ret.page->mp_pages = num;
+ ret.page->mp_flags = P_OVERFLOW;
}
- return MDBX_SUCCESS;
+ ret.err = mdbx_page_dirty(txn, ret.page, num);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ goto fail;
+
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ return ret;
}
-__cold static int mdbx_wipe_steady(MDBX_env *env, const txnid_t last_steady) {
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
- ? env->me_dsync_fd
- : env->me_lazy_fd;
- int err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 0), fd);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 1), fd);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- err = mdbx_meta_unsteady(env, last_steady, METAPAGE(env, 2), fd);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+/* Copy the used portions of a non-overflow page. */
+__hot static void mdbx_page_copy(MDBX_page *dst, const MDBX_page *src,
+ size_t psize) {
+ STATIC_ASSERT(UINT16_MAX > MAX_PAGESIZE - PAGEHDRSZ);
+ STATIC_ASSERT(MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4);
+ if ((src->mp_flags & (P_LEAF2 | P_OVERFLOW)) == 0) {
+ size_t upper = src->mp_upper, lower = src->mp_lower, unused = upper - lower;
- if (env->me_flags & MDBX_WRITEMAP) {
- mdbx_flush_incoherent_cpu_writeback();
- err = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
- MDBX_SYNC_DATA);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- } else {
- if (fd == env->me_lazy_fd) {
-#if MDBX_USE_SYNCFILERANGE
- static bool syncfilerange_unavailable;
- if (!syncfilerange_unavailable &&
- sync_file_range(env->me_lazy_fd, 0, pgno2bytes(env, NUM_METAS),
- SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER)) {
- err = errno;
- if (ignore_enosys(err) == MDBX_RESULT_TRUE)
- syncfilerange_unavailable = true;
- }
- if (syncfilerange_unavailable)
-#endif /* MDBX_USE_SYNCFILERANGE */
- err = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ /* If page isn't full, just copy the used portion. Adjust
+ * alignment so memcpy may copy words instead of bytes. */
+ if (unused >= MDBX_CACHELINE_SIZE * 2) {
+ lower = ceil_powerof2(lower + PAGEHDRSZ, sizeof(void *));
+ upper = floor_powerof2(upper + PAGEHDRSZ, sizeof(void *));
+ memcpy(dst, src, lower);
+ dst = (void *)((char *)dst + upper);
+ src = (void *)((char *)src + upper);
+ psize -= upper;
}
- mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS),
- env->me_os_psize);
}
-
- /* force oldest refresh */
- atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, mo_Relaxed);
- return MDBX_SUCCESS;
+ memcpy(dst, src, psize);
}
-/* Allocate page numbers and memory for writing. Maintain mt_last_reclaimed,
- * mt_reclaimed_pglist and mt_next_pgno. Set MDBX_TXN_ERROR on failure.
- *
- * If there are free pages available from older transactions, they
- * are re-used first. Otherwise allocate a new page at mt_next_pgno.
- * Do not modify the GC, just merge GC records into mt_reclaimed_pglist
- * and move mt_last_reclaimed to say which records were consumed. Only this
- * function can create mt_reclaimed_pglist and move
- * mt_last_reclaimed/mt_next_pgno.
- *
- * [in] mc cursor A cursor handle identifying the transaction and
- * database for which we are allocating.
- * [in] num the number of pages to allocate.
+/* Pull a page off the txn's spill list, if present.
*
- * Returns 0 on success, non-zero on failure.*/
+ * If a page being referenced was spilled to disk in this txn, bring
+ * it back and make it dirty/writable again. */
+static struct page_result __must_check_result
+mdbx_page_unspill(MDBX_txn *const txn, const MDBX_page *const mp) {
+ mdbx_verbose("unspill page %" PRIaPGNO, mp->mp_pgno);
+ mdbx_tassert(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0);
+ mdbx_tassert(txn, IS_SPILLED(txn, mp));
+ const MDBX_txn *scan = txn;
+ struct page_result ret;
+ do {
+ mdbx_tassert(txn, (scan->mt_flags & MDBX_TXN_SPILLS) != 0);
+ const unsigned si = mdbx_search_spilled(scan, mp->mp_pgno);
+ if (!si)
+ continue;
+ const unsigned npages = IS_OVERFLOW(mp) ? mp->mp_pages : 1;
+ ret.page = mdbx_page_malloc(txn, npages);
+ if (unlikely(!ret.page)) {
+ ret.err = MDBX_ENOMEM;
+ return ret;
+ }
+ mdbx_page_copy(ret.page, mp, pgno2bytes(txn->mt_env, npages));
+ if (scan == txn) {
+ /* If in current txn, this page is no longer spilled.
+ * If it happens to be the last page, truncate the spill list.
+ * Otherwise mark it as deleted by setting the LSB. */
+ mdbx_spill_remove(txn, si, npages);
+ } /* otherwise, if belonging to a parent txn, the
+ * page remains spilled until child commits */
-#define MDBX_ALLOC_CACHE 1
-#define MDBX_ALLOC_GC 2
-#define MDBX_ALLOC_NEW 4
-#define MDBX_ALLOC_SLOT 8
-#define MDBX_ALLOC_FAKE 16
-#define MDBX_ALLOC_ALL (MDBX_ALLOC_CACHE | MDBX_ALLOC_GC | MDBX_ALLOC_NEW)
+ ret.err = mdbx_page_dirty(txn, ret.page, npages);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
+#if MDBX_ENABLE_PGOP_STAT
+ txn->mt_env->me_lck->mti_pgop_stat.unspill.weak += npages;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ ret.page->mp_flags |= (scan == txn) ? 0 : P_SPILLED;
+ ret.err = MDBX_SUCCESS;
+ return ret;
+ } while (likely((scan = scan->mt_parent) != nullptr &&
+ (scan->mt_flags & MDBX_TXN_SPILLS) != 0));
+ mdbx_error("Page %" PRIaPGNO " mod-txnid %" PRIaTXN
+ " not found in the spill-list(s), current txn %" PRIaTXN
+ " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN,
+ mp->mp_pgno, mp->mp_txnid, txn->mt_txnid, txn->mt_front,
+ txn->mt_env->me_txn0->mt_txnid, txn->mt_env->me_txn0->mt_front);
+ ret.err = MDBX_PROBLEM;
+ ret.page = NULL;
+ return ret;
+}
-__hot static struct page_result mdbx_page_alloc(MDBX_cursor *mc,
- const pgno_t num, int flags) {
- struct page_result ret;
- MDBX_txn *const txn = mc->mc_txn;
- MDBX_env *const env = txn->mt_env;
- mdbx_assert(env, num == 0 || !(flags & MDBX_ALLOC_SLOT));
- mdbx_assert(env, num > 0 || !(flags & MDBX_ALLOC_NEW));
+/* Touch a page: make it dirty and re-insert into tree with updated pgno.
+ * Set MDBX_TXN_ERROR on failure.
+ *
+ * [in] mc cursor pointing to the page to be touched
+ *
+ * Returns 0 on success, non-zero on failure. */
+__hot static int mdbx_page_touch(MDBX_cursor *mc) {
+ const MDBX_page *const mp = mc->mc_pg[mc->mc_top];
+ MDBX_page *np;
+ MDBX_txn *txn = mc->mc_txn;
+ int rc;
- const unsigned coalesce_threshold =
- env->me_maxgc_ov1page - env->me_maxgc_ov1page / 4;
- if (likely(flags & MDBX_ALLOC_GC)) {
- flags |= env->me_flags & (MDBX_COALESCE | MDBX_LIFORECLAIM);
- if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) > coalesce_threshold)
- flags &= ~MDBX_COALESCE;
- if (unlikely(
- /* If mc is updating the GC, then the retired-list cannot play
- catch-up with itself by growing while trying to save it. */
- (mc->mc_flags & C_RECLAIMING) ||
- /* avoid (recursive) search inside empty tree and while tree is
- updating, todo4recovery://erased_by_github/libmdbx/issues/31 */
- txn->mt_dbs[FREE_DBI].md_entries == 0 ||
- /* If our dirty list is already full, we can't touch GC */
- (txn->tw.dirtyroom < txn->mt_dbs[FREE_DBI].md_depth &&
- !(txn->mt_dbistate[FREE_DBI] & DBI_DIRTY))))
- flags &= ~(MDBX_ALLOC_GC | MDBX_COALESCE);
+ if (mdbx_assert_enabled()) {
+ if (mc->mc_flags & C_SUB) {
+ MDBX_xcursor *mx = container_of(mc->mc_db, MDBX_xcursor, mx_db);
+ MDBX_cursor_couple *couple = container_of(mx, MDBX_cursor_couple, inner);
+ mdbx_tassert(txn, mc->mc_db == &couple->outer.mc_xcursor->mx_db);
+ mdbx_tassert(txn, mc->mc_dbx == &couple->outer.mc_xcursor->mx_dbx);
+ mdbx_tassert(txn, *couple->outer.mc_dbistate & DBI_DIRTY);
+ } else {
+ mdbx_tassert(txn, *mc->mc_dbistate & DBI_DIRTY);
+ }
+ mdbx_tassert(txn, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY);
+ mdbx_tassert(txn, !IS_OVERFLOW(mp));
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
}
- if (likely(num == 1 && (flags & MDBX_ALLOC_CACHE) != 0)) {
- /* If there are any loose pages, just use them */
- mdbx_assert(env, (flags & MDBX_ALLOC_SLOT) == 0);
- if (likely(txn->tw.loose_pages)) {
-#if MDBX_ENABLE_REFUND
- if (txn->tw.loose_refund_wl > txn->mt_next_pgno) {
- mdbx_refund(txn);
- if (unlikely(!txn->tw.loose_pages))
- goto no_loose;
- }
-#endif /* MDBX_ENABLE_REFUND */
+ if (IS_MODIFIABLE(txn, mp) || IS_SUBP(mp))
+ return MDBX_SUCCESS;
- ret.page = txn->tw.loose_pages;
- txn->tw.loose_pages = ret.page->mp_next;
- txn->tw.loose_count--;
- mdbx_debug_extra("db %d use loose page %" PRIaPGNO, DDBI(mc),
- ret.page->mp_pgno);
- mdbx_tassert(txn, ret.page->mp_pgno < txn->mt_next_pgno);
- mdbx_ensure(env, ret.page->mp_pgno >= NUM_METAS);
- VALGRIND_MAKE_MEM_UNDEFINED(page_data(ret.page), page_space(txn->mt_env));
- MDBX_ASAN_UNPOISON_MEMORY_REGION(page_data(ret.page),
- page_space(txn->mt_env));
- ret.page->mp_txnid = txn->mt_front;
- ret.err = MDBX_SUCCESS;
- return ret;
+ if (IS_FROZEN(txn, mp)) {
+ /* CoW the page */
+ rc = mdbx_pnl_need(&txn->tw.retired_pages, 1);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ const struct page_result par = mdbx_page_alloc(mc, 1, MDBX_ALLOC_ALL);
+ rc = par.err;
+ np = par.page;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+
+ const pgno_t pgno = np->mp_pgno;
+ mdbx_debug("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, DDBI(mc),
+ mp->mp_pgno, pgno);
+ mdbx_tassert(txn, mp->mp_pgno != pgno);
+ mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno);
+ /* Update the parent page, if any, to point to the new page */
+ if (mc->mc_top) {
+ MDBX_page *parent = mc->mc_pg[mc->mc_top - 1];
+ MDBX_node *node = page_node(parent, mc->mc_ki[mc->mc_top - 1]);
+ node_set_pgno(node, pgno);
+ } else {
+ mc->mc_db->md_root = pgno;
}
- }
-#if MDBX_ENABLE_REFUND
-no_loose:
-#endif /* MDBX_ENABLE_REFUND */
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- pgno_t pgno, *re_list = txn->tw.reclaimed_pglist;
- unsigned range_begin = 0, re_len = MDBX_PNL_SIZE(re_list);
- txnid_t oldest = 0, last = 0;
+#if MDBX_ENABLE_PGOP_STAT
+ txn->mt_env->me_lck->mti_pgop_stat.cow.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ mdbx_page_copy(np, mp, txn->mt_env->me_psize);
+ np->mp_pgno = pgno;
+ np->mp_txnid = txn->mt_front;
+ } else if (IS_SPILLED(txn, mp)) {
+ struct page_result pur = mdbx_page_unspill(txn, mp);
+ np = pur.page;
+ rc = pur.err;
+ if (likely(rc == MDBX_SUCCESS)) {
+ mdbx_tassert(txn, np != nullptr);
+ goto done;
+ }
+ goto fail;
+ } else {
+ if (unlikely(!txn->mt_parent)) {
+ mdbx_error("Unexpected not frozen/modifiable/spilled but shadowed %s "
+ "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
+ " without parent transaction, current txn %" PRIaTXN
+ " front %" PRIaTXN,
+ IS_BRANCH(mp) ? "branch" : "leaf", mp->mp_pgno, mp->mp_txnid,
+ mc->mc_txn->mt_txnid, mc->mc_txn->mt_front);
+ rc = MDBX_PROBLEM;
+ goto fail;
+ }
- while (true) { /* hsr-kick retry loop */
- MDBX_cursor_couple recur;
- for (MDBX_cursor_op op = MDBX_FIRST;;
- op = (flags & MDBX_LIFORECLAIM) ? MDBX_PREV : MDBX_NEXT) {
- MDBX_val key, data;
+ mdbx_debug("clone db %d page %" PRIaPGNO, DDBI(mc), mp->mp_pgno);
+ mdbx_tassert(txn, txn->tw.dirtylist->length <=
+ MDBX_PGL_LIMIT + MDBX_PNL_GRANULATE);
+ /* No - copy it */
+ np = mdbx_page_malloc(txn, 1);
+ if (unlikely(!np)) {
+ rc = MDBX_ENOMEM;
+ goto fail;
+ }
+ mdbx_page_copy(np, mp, txn->mt_env->me_psize);
- /* Seek a big enough contiguous page range.
- * Prefer pages with lower pgno. */
- mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno));
- if (!(flags & (MDBX_COALESCE | MDBX_ALLOC_SLOT)) && re_len >= num) {
- mdbx_tassert(txn, MDBX_PNL_LAST(re_list) < txn->mt_next_pgno &&
- MDBX_PNL_FIRST(re_list) < txn->mt_next_pgno);
- range_begin = MDBX_PNL_ASCENDING ? 1 : re_len;
- pgno = MDBX_PNL_LEAST(re_list);
- if (likely(num == 1))
- goto done;
+ /* insert a clone of parent's dirty page, so don't touch dirtyroom */
+ rc = mdbx_page_dirty(txn, np, 1);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
- const unsigned wanna_range = num - 1;
-#if MDBX_PNL_ASCENDING
- mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1);
- while (true) {
- unsigned range_end = range_begin + wanna_range;
- if (re_list[range_end] - pgno == wanna_range)
- goto done;
- if (range_end == re_len)
- break;
- pgno = re_list[++range_begin];
- }
-#else
- mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len);
- while (true) {
- if (re_list[range_begin - wanna_range] - pgno == wanna_range)
- goto done;
- if (range_begin == wanna_range)
- break;
- pgno = re_list[--range_begin];
- }
-#endif /* MDBX_PNL sort-order */
+#if MDBX_ENABLE_PGOP_STAT
+ txn->mt_env->me_lck->mti_pgop_stat.clone.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ }
+
+done:
+ /* Adjust cursors pointing to mp */
+ mc->mc_pg[mc->mc_top] = np;
+ MDBX_cursor *m2 = txn->mt_cursors[mc->mc_dbi];
+ if (mc->mc_flags & C_SUB) {
+ for (; m2; m2 = m2->mc_next) {
+ MDBX_cursor *m3 = &m2->mc_xcursor->mx_cursor;
+ if (m3->mc_snum < mc->mc_snum)
+ continue;
+ if (m3->mc_pg[mc->mc_top] == mp)
+ m3->mc_pg[mc->mc_top] = np;
+ }
+ } else {
+ for (; m2; m2 = m2->mc_next) {
+ if (m2->mc_snum < mc->mc_snum)
+ continue;
+ if (m2 == mc)
+ continue;
+ if (m2->mc_pg[mc->mc_top] == mp) {
+ m2->mc_pg[mc->mc_top] = np;
+ if (XCURSOR_INITED(m2) && IS_LEAF(np))
+ XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]);
}
+ }
+ }
+ return MDBX_SUCCESS;
- if (op == MDBX_FIRST) { /* 1st iteration, setup cursor, etc */
- if (unlikely(!(flags & MDBX_ALLOC_GC)))
- break /* reclaiming is prohibited for now */;
+fail:
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ return rc;
+}
- /* Prepare to fetch more and coalesce */
- oldest = (flags & MDBX_LIFORECLAIM)
- ? mdbx_find_oldest(txn)
- : atomic_load64(&env->me_lck->mti_oldest_reader,
- mo_AcquireRelease);
- ret.err = mdbx_cursor_init(&recur.outer, txn, FREE_DBI);
- if (unlikely(ret.err != MDBX_SUCCESS))
- goto fail;
- if (flags & MDBX_LIFORECLAIM) {
- /* Begin from oldest reader if any */
- if (oldest > MIN_TXNID) {
- last = oldest - 1;
- op = MDBX_SET_RANGE;
- }
- } else if (txn->tw.last_reclaimed) {
- /* Continue lookup from txn->tw.last_reclaimed to oldest reader */
- last = txn->tw.last_reclaimed;
- op = MDBX_SET_RANGE;
- }
+__cold static int mdbx_env_sync_internal(MDBX_env *env, bool force,
+ bool nonblock) {
+ bool locked = false;
+ int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */;
- key.iov_base = &last;
- key.iov_len = sizeof(last);
- }
+retry:;
+ unsigned flags = env->me_flags & ~(MDBX_NOMETASYNC | MDBX_SHRINK_ALLOWED);
+ if (unlikely((flags & (MDBX_RDONLY | MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE)) !=
+ MDBX_ENV_ACTIVE)) {
+ rc = MDBX_EACCESS;
+ if (!(flags & MDBX_ENV_ACTIVE))
+ rc = MDBX_EPERM;
+ if (flags & MDBX_FATAL_ERROR)
+ rc = MDBX_PANIC;
+ goto bailout;
+ }
- if (!(flags & MDBX_LIFORECLAIM)) {
- /* Do not try fetch more if the record will be too recent */
- if (op != MDBX_FIRST && ++last >= oldest) {
- oldest = mdbx_find_oldest(txn);
- if (oldest <= last)
- break;
- }
- }
+ const pgno_t unsynced_pages =
+ atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed);
+ volatile const MDBX_meta *head = meta_prefer_last(env);
+ const txnid_t head_txnid = meta_txnid(env, head);
+ const uint32_t synched_meta_txnid_u32 =
+ atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed);
+ if (unsynced_pages == 0 && synched_meta_txnid_u32 == (uint32_t)head_txnid &&
+ META_IS_STEADY(head))
+ goto bailout;
- ret.err = mdbx_cursor_get(&recur.outer, &key, NULL, op);
- if (ret.err == MDBX_NOTFOUND && (flags & MDBX_LIFORECLAIM)) {
- if (op == MDBX_SET_RANGE)
- continue;
- txnid_t snap = mdbx_find_oldest(txn);
- if (oldest < snap) {
- oldest = snap;
- last = oldest - 1;
- key.iov_base = &last;
- key.iov_len = sizeof(last);
- op = MDBX_SET_RANGE;
- ret.err = mdbx_cursor_get(&recur.outer, &key, NULL, op);
- }
- }
- if (unlikely(ret.err)) {
- if (ret.err == MDBX_NOTFOUND)
- break;
- goto fail;
- }
+ const pgno_t autosync_threshold =
+ atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
+ const uint64_t autosync_period =
+ atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
+ if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) ||
+ (autosync_period &&
+ mdbx_osal_monotime() -
+ atomic_load64(&env->me_lck->mti_sync_timestamp, mo_Relaxed) >=
+ autosync_period))
+ flags &= MDBX_WRITEMAP /* clear flags for full steady sync */;
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(key.iov_len != sizeof(txnid_t))) {
- ret.err = MDBX_CORRUPTED;
- goto fail;
- }
- last = unaligned_peek_u64(4, key.iov_base);
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(last < MIN_TXNID || last > MAX_TXNID)) {
- ret.err = MDBX_CORRUPTED;
- goto fail;
- }
- if (oldest <= last) {
- oldest = mdbx_find_oldest(txn);
- if (oldest <= last) {
- if (flags & MDBX_LIFORECLAIM)
- continue;
- break;
- }
- }
+ const bool inside_txn = (env->me_txn0->mt_owner == mdbx_thread_self());
+ if (!inside_txn) {
+ if (!locked) {
+ int err;
+ unsigned wops = 0;
+ /* pre-sync to avoid latency for writer */
+ if (unsynced_pages > /* FIXME: define threshold */ 16 &&
+ (flags & MDBX_SAFE_NOSYNC) == 0) {
+ mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
+ if (flags & MDBX_WRITEMAP) {
+ /* Acquire guard to avoid collision with remap */
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_srwlock_AcquireShared(&env->me_remap_guard);
+#else
+ err = mdbx_fastmutex_acquire(&env->me_remap_guard);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+#endif
+ const size_t usedbytes = pgno_align2os_bytes(env, head->mm_geo.next);
+ err = mdbx_msync(&env->me_dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA);
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
+#else
+ int unlock_err = mdbx_fastmutex_release(&env->me_remap_guard);
+ if (unlikely(unlock_err != MDBX_SUCCESS) && err == MDBX_SUCCESS)
+ err = unlock_err;
+#endif
+ } else
+ err = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA);
- if (flags & MDBX_LIFORECLAIM) {
- /* skip IDs of records that already reclaimed */
- if (txn->tw.lifo_reclaimed) {
- size_t i;
- for (i = (size_t)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); i > 0; --i)
- if (txn->tw.lifo_reclaimed[i] == last)
- break;
- if (i)
- continue;
- }
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+
+ /* pre-sync done */
+ wops = 1;
+ rc = MDBX_SUCCESS /* means "some data was synced" */;
}
- /* Reading next GC record */
- MDBX_page *const mp = recur.outer.mc_pg[recur.outer.mc_top];
- if (unlikely((ret.err = mdbx_node_read(
- &recur.outer,
- page_node(mp, recur.outer.mc_ki[recur.outer.mc_top]),
- &data, pp_txnid4chk(mp, txn))) != MDBX_SUCCESS))
- goto fail;
+ err = mdbx_txn_lock(env, nonblock);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- if ((flags & MDBX_LIFORECLAIM) && !txn->tw.lifo_reclaimed) {
- txn->tw.lifo_reclaimed = mdbx_txl_alloc();
- if (unlikely(!txn->tw.lifo_reclaimed)) {
- ret.err = MDBX_ENOMEM;
- goto fail;
- }
- }
+ locked = true;
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += wops;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ goto retry;
+ }
+ env->me_txn0->mt_txnid = head_txnid;
+ mdbx_assert(env, head_txnid == meta_txnid(env, head));
+ mdbx_assert(env, head_txnid == mdbx_recent_committed_txnid(env));
+ mdbx_find_oldest(env->me_txn0);
+ flags |= MDBX_SHRINK_ALLOWED;
+ }
- /* Append PNL from GC record to tw.reclaimed_pglist */
- mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
- pgno_t *gc_pnl = (pgno_t *)data.iov_base;
- mdbx_tassert(txn, data.iov_len >= MDBX_PNL_SIZEOF(gc_pnl));
- if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) ||
- !mdbx_pnl_check(gc_pnl, txn->mt_next_pgno))) {
- ret.err = MDBX_CORRUPTED;
- goto fail;
- }
- const unsigned gc_len = MDBX_PNL_SIZE(gc_pnl);
- if (unlikely(/* list is too long already */ MDBX_PNL_SIZE(
- txn->tw.reclaimed_pglist) >=
- env->me_options.rp_augment_limit) &&
- ((/* not a slot-request from gc-update */
- (flags & MDBX_ALLOC_SLOT) == 0 &&
- /* have enough unallocated space */ txn->mt_geo.upper >=
- txn->mt_next_pgno + (size_t)num) ||
- gc_len + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) >=
- MDBX_PGL_LIMIT)) {
- /* Stop reclaiming to avoid overflow the page list.
- * This is a rare case while search for a continuously multi-page region
- * in a large database.
- * todo4recovery://erased_by_github/libmdbx/issues/123 */
- mdbx_notice("stop reclaiming to avoid PNL overflow: %u (current) + %u "
- "(chunk) -> %u",
- MDBX_PNL_SIZE(txn->tw.reclaimed_pglist), gc_len,
- gc_len + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
- flags &= ~(MDBX_ALLOC_GC | MDBX_COALESCE);
- break;
- }
- ret.err = mdbx_pnl_need(&txn->tw.reclaimed_pglist, gc_len);
- if (unlikely(ret.err != MDBX_SUCCESS))
- goto fail;
- re_list = txn->tw.reclaimed_pglist;
+ mdbx_assert(env, inside_txn || locked);
+ mdbx_assert(env, !inside_txn || (flags & MDBX_SHRINK_ALLOWED) == 0);
- /* Remember ID of GC record */
- if (flags & MDBX_LIFORECLAIM) {
- ret.err = mdbx_txl_append(&txn->tw.lifo_reclaimed, last);
- if (unlikely(ret.err != MDBX_SUCCESS))
- goto fail;
- }
- txn->tw.last_reclaimed = last;
+ if (!META_IS_STEADY(head) ||
+ ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) {
+ mdbx_debug("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIaPGNO,
+ data_page((const void *)head)->mp_pgno, mdbx_durable_str(head),
+ unsynced_pages);
+ MDBX_meta meta = *head;
+ rc = mdbx_sync_locked(env, flags, &meta);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
- if (mdbx_log_enabled(MDBX_LOG_EXTRA)) {
- mdbx_debug_extra("PNL read txn %" PRIaTXN " root %" PRIaPGNO
- " num %u, PNL",
- last, txn->mt_dbs[FREE_DBI].md_root, gc_len);
- for (unsigned i = gc_len; i; i--)
- mdbx_debug_extra_print(" %" PRIaPGNO, gc_pnl[i]);
- mdbx_debug_extra_print("%s\n", ".");
- }
+ /* LY: sync meta-pages if MDBX_NOMETASYNC enabled
+ * and someone was not synced above. */
+ if (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) !=
+ (uint32_t)head_txnid) {
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ rc = (flags & MDBX_WRITEMAP)
+ ? mdbx_msync(&env->me_dxb_mmap, 0,
+ pgno_align2os_bytes(env, NUM_METAS),
+ MDBX_SYNC_DATA | MDBX_SYNC_IODQ)
+ : mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ if (likely(rc == MDBX_SUCCESS))
+ atomic_store32(&env->me_lck->mti_meta_sync_txnid, (uint32_t)head_txnid,
+ mo_Relaxed);
+ }
- /* Merge in descending sorted order */
- const unsigned prev_re_len = MDBX_PNL_SIZE(re_list);
- mdbx_pnl_xmerge(re_list, gc_pnl);
- /* re-check to avoid duplicates */
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(!mdbx_pnl_check(re_list, txn->mt_next_pgno))) {
- ret.err = MDBX_CORRUPTED;
- goto fail;
- }
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+bailout:
+ if (locked)
+ mdbx_txn_unlock(env);
+ return rc;
+}
- re_len = MDBX_PNL_SIZE(re_list);
- mdbx_tassert(txn, re_len == 0 || re_list[re_len] < txn->mt_next_pgno);
- if (MDBX_ENABLE_REFUND && re_len &&
- unlikely(MDBX_PNL_MOST(re_list) == txn->mt_next_pgno - 1)) {
- /* Refund suitable pages into "unallocated" space */
- mdbx_refund(txn);
- re_list = txn->tw.reclaimed_pglist;
- re_len = MDBX_PNL_SIZE(re_list);
- }
+static __inline int check_env(const MDBX_env *env, const bool wanna_active) {
+ if (unlikely(!env))
+ return MDBX_EINVAL;
- /* Done for a kick-reclaim mode, actually no page needed */
- if (unlikely(flags & MDBX_ALLOC_SLOT)) {
- mdbx_debug("early-return NULL-page for %s mode", "MDBX_ALLOC_SLOT");
- ret.err = MDBX_SUCCESS;
- ret.page = NULL;
- return ret;
- }
+ if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE))
+ return MDBX_EBADSIGN;
- /* Don't try to coalesce too much. */
- if (flags & MDBX_COALESCE) {
- if (re_len /* current size */ > coalesce_threshold ||
- (re_len > prev_re_len &&
- re_len - prev_re_len /* delta from prev */ >=
- coalesce_threshold / 2)) {
- mdbx_trace("clear %s %s", "MDBX_COALESCE", "since got threshold");
- flags &= ~MDBX_COALESCE;
- }
- }
- }
+#if MDBX_ENV_CHECKPID
+ if (unlikely(env->me_pid != mdbx_getpid())) {
+ ((MDBX_env *)env)->me_flags |= MDBX_FATAL_ERROR;
+ return MDBX_PANIC;
+ }
+#endif /* MDBX_ENV_CHECKPID */
- if (F_ISSET(flags, MDBX_COALESCE | MDBX_ALLOC_GC)) {
- mdbx_debug_extra("clear %s and continue", "MDBX_COALESCE");
- flags &= ~MDBX_COALESCE;
- continue;
- }
+ if (unlikely(env->me_flags & MDBX_FATAL_ERROR))
+ return MDBX_PANIC;
- /* There is no suitable pages in the GC and to be able to allocate
- * we should CHOICE one of:
- * - make a new steady checkpoint if reclaiming was stopped by
- * the last steady-sync, or wipe it in the MDBX_UTTERLY_NOSYNC mode;
- * - kick lagging reader(s) if reclaiming was stopped by ones of it.
- * - extend the database file. */
+ if (wanna_active) {
+ if (unlikely((env->me_flags & MDBX_ENV_ACTIVE) == 0))
+ return MDBX_EPERM;
+ mdbx_assert(env, env->me_map != nullptr);
+ }
- /* Will use new pages from the map if nothing is suitable in the GC. */
- range_begin = 0;
- pgno = txn->mt_next_pgno;
- const size_t next = (size_t)pgno + num;
+ return MDBX_SUCCESS;
+}
- if (flags & MDBX_ALLOC_GC) {
- const MDBX_meta *const head = constmeta_prefer_last(env);
- const MDBX_meta *const steady = constmeta_prefer_steady(env);
- /* does reclaiming stopped at the last steady point? */
- if (head != steady && META_IS_STEADY(steady) &&
- oldest == constmeta_txnid(env, steady)) {
- mdbx_debug("gc-kick-steady: head %" PRIaTXN "-%s, tail %" PRIaTXN
- "-%s, oldest %" PRIaTXN,
- constmeta_txnid(env, head), mdbx_durable_str(head),
- constmeta_txnid(env, steady), mdbx_durable_str(steady),
- oldest);
- ret.err = MDBX_RESULT_TRUE;
- const pgno_t autosync_threshold =
- atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
- const uint64_t autosync_period =
- atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
- /* wipe the last steady-point if one of:
- * - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified
- * - UTTERLY_NOSYNC mode AND free space at steady-point is exhausted
- * otherwise, make a new steady-point if one of:
- * - auto-sync threshold is specified and reached;
- * - upper limit of database size is reached;
- * - database is full (with the current file size)
- * AND auto-sync threshold it NOT specified */
- if (F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC) &&
- ((autosync_threshold | autosync_period) == 0 ||
- next >= steady->mm_geo.now)) {
- /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode
- * without any auto-sync threshold(s). */
- ret.err = mdbx_wipe_steady(env, oldest);
- mdbx_debug("gc-wipe-steady, rc %d", ret.err);
- mdbx_assert(env, steady != meta_prefer_steady(env));
- } else if ((flags & MDBX_ALLOC_NEW) == 0 ||
- (autosync_threshold &&
- atomic_load32(&env->me_lck->mti_unsynced_pages,
- mo_Relaxed) >= autosync_threshold) ||
- (autosync_period &&
- mdbx_osal_monotime() -
- atomic_load64(&env->me_lck->mti_sync_timestamp,
- mo_Relaxed) >=
- autosync_period) ||
- next >= txn->mt_geo.upper ||
- (next >= txn->mt_end_pgno &&
- (autosync_threshold | autosync_period) == 0)) {
- /* make steady checkpoint. */
- MDBX_meta meta = *head;
- ret.err = mdbx_sync_locked(env, env->me_flags & MDBX_WRITEMAP, &meta);
- mdbx_debug("gc-make-steady, rc %d", ret.err);
- mdbx_assert(env, steady != meta_prefer_steady(env));
+__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ return mdbx_env_sync_internal(env, force, nonblock);
+}
+
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); }
+
+__cold int mdbx_env_sync_poll(MDBX_env *env) {
+ return __inline_mdbx_env_sync_poll(env);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+
+/* Back up parent txn's cursors, then grab the originals for tracking */
+static int mdbx_cursor_shadow(MDBX_txn *parent, MDBX_txn *nested) {
+ for (int i = parent->mt_numdbs; --i >= 0;) {
+ nested->mt_cursors[i] = NULL;
+ MDBX_cursor *mc = parent->mt_cursors[i];
+ if (mc != NULL) {
+ size_t size = mc->mc_xcursor ? sizeof(MDBX_cursor) + sizeof(MDBX_xcursor)
+ : sizeof(MDBX_cursor);
+ for (MDBX_cursor *bk; mc; mc = bk->mc_next) {
+ bk = mc;
+ if (mc->mc_signature != MDBX_MC_LIVE)
+ continue;
+ bk = mdbx_malloc(size);
+ if (unlikely(!bk))
+ return MDBX_ENOMEM;
+#if MDBX_DEBUG
+ memset(bk, 0xCD, size);
+ VALGRIND_MAKE_MEM_UNDEFINED(bk, size);
+#endif /* MDBX_DEBUG */
+ *bk = *mc;
+ mc->mc_backup = bk;
+ /* Kill pointers into src to reduce abuse: The
+ * user may not use mc until dst ends. But we need a valid
+ * txn pointer here for cursor fixups to keep working. */
+ mc->mc_txn = nested;
+ mc->mc_db = &nested->mt_dbs[i];
+ mc->mc_dbistate = &nested->mt_dbistate[i];
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ if (mx != NULL) {
+ *(MDBX_xcursor *)(bk + 1) = *mx;
+ mx->mx_cursor.mc_txn = nested;
}
- if (ret.err == MDBX_SUCCESS) {
- if (mdbx_find_oldest(txn) > oldest)
- continue;
- /* it is reasonable check/kick lagging reader(s) here,
- * since we made a new steady point or wipe the last. */
- if (oldest < txn->mt_txnid - xMDBX_TXNID_STEP &&
- mdbx_kick_longlived_readers(env, oldest) > oldest)
- continue;
- } else if (unlikely(ret.err != MDBX_RESULT_TRUE))
- goto fail;
+ mc->mc_next = nested->mt_cursors[i];
+ nested->mt_cursors[i] = mc;
}
}
+ }
+ return MDBX_SUCCESS;
+}
- /* don't kick lagging reader(s) if is enough unallocated space
- * at the end of database file. */
- if ((flags & MDBX_ALLOC_NEW) && next <= txn->mt_end_pgno)
- goto done;
- if ((flags & MDBX_ALLOC_GC) && oldest < txn->mt_txnid - xMDBX_TXNID_STEP &&
- mdbx_kick_longlived_readers(env, oldest) > oldest)
+/* Close this txn's cursors, give parent txn's cursors back to parent.
+ *
+ * [in] txn the transaction handle.
+ * [in] merge true to keep changes to parent cursors, false to revert.
+ *
+ * Returns 0 on success, non-zero on failure. */
+static void mdbx_cursors_eot(MDBX_txn *txn, const bool merge) {
+ for (int i = txn->mt_numdbs; --i >= 0;) {
+ MDBX_cursor *next, *mc = txn->mt_cursors[i];
+ if (!mc)
continue;
+ txn->mt_cursors[i] = NULL;
+ do {
+ const unsigned stage = mc->mc_signature;
+ MDBX_cursor *bk = mc->mc_backup;
+ next = mc->mc_next;
+ mdbx_ensure(txn->mt_env,
+ stage == MDBX_MC_LIVE || (stage == MDBX_MC_WAIT4EOT && bk));
+ mdbx_cassert(mc, mc->mc_dbi == (unsigned)i);
+ if (bk) {
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ mdbx_cassert(mc, mx == bk->mc_xcursor);
+ mdbx_tassert(txn, txn->mt_parent != NULL);
+ mdbx_ensure(txn->mt_env, bk->mc_signature == MDBX_MC_LIVE);
+ if (stage == MDBX_MC_WAIT4EOT /* Cursor was closed by user */)
+ mc->mc_signature = stage /* Promote closed state to parent txn */;
+ else if (merge) {
+ /* Restore pointers to parent txn */
+ mc->mc_next = bk->mc_next;
+ mc->mc_backup = bk->mc_backup;
+ mc->mc_txn = bk->mc_txn;
+ mc->mc_db = bk->mc_db;
+ mc->mc_dbistate = bk->mc_dbistate;
+ if (mx) {
+ if (mx != bk->mc_xcursor) {
+ *bk->mc_xcursor = *mx;
+ mx = bk->mc_xcursor;
+ }
+ mx->mx_cursor.mc_txn = bk->mc_txn;
+ }
+ } else {
+ /* Restore from backup, i.e. rollback/abort nested txn */
+ *mc = *bk;
+ if (mx)
+ *mx = *(MDBX_xcursor *)(bk + 1);
+ }
+ bk->mc_signature = 0;
+ mdbx_free(bk);
+ } else {
+ mdbx_ensure(txn->mt_env, stage == MDBX_MC_LIVE);
+ mc->mc_signature = MDBX_MC_READY4CLOSE /* Cursor may be reused */;
+ mc->mc_flags = 0 /* reset C_UNTRACK */;
+ }
+ } while ((mc = next) != NULL);
+ }
+}
- ret.err = MDBX_NOTFOUND;
- if (flags & MDBX_ALLOC_NEW) {
- ret.err = MDBX_MAP_FULL;
- if (next < txn->mt_geo.upper && txn->mt_geo.grow_pv) {
- mdbx_assert(env, next > txn->mt_end_pgno);
- const pgno_t grow_step = pv2pages(txn->mt_geo.grow_pv);
- size_t aligned = pgno_align2os_pgno(
- env, (pgno_t)(next + grow_step - next % grow_step));
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+/* Find largest mvcc-snapshot still referenced by this process. */
+static pgno_t mdbx_find_largest_this(MDBX_env *env, pgno_t largest) {
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (likely(lck != NULL /* exclusive mode */)) {
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ retry:
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease) ==
+ env->me_pid) {
+ /* mdbx_jitter4testing(true); */
+ const pgno_t snap_pages = atomic_load32(
+ &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed);
+ const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (unlikely(
+ snap_pages !=
+ atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used,
+ mo_AcquireRelease) ||
+ snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
+ goto retry;
+ if (largest < snap_pages &&
+ atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease) <=
+ /* ignore pending updates */ snap_txnid &&
+ snap_txnid <= MAX_TXNID)
+ largest = snap_pages;
+ }
+ }
+ }
+ return largest;
+}
- if (aligned > txn->mt_geo.upper)
- aligned = txn->mt_geo.upper;
- mdbx_assert(env, aligned > txn->mt_end_pgno);
+static void mdbx_txn_valgrind(MDBX_env *env, MDBX_txn *txn) {
+#if !defined(__SANITIZE_ADDRESS__)
+ if (!RUNNING_ON_VALGRIND)
+ return;
+#endif
- mdbx_verbose("try growth datafile to %zu pages (+%zu)", aligned,
- aligned - txn->mt_end_pgno);
- ret.err = mdbx_mapresize_implicit(env, txn->mt_next_pgno,
- (pgno_t)aligned, txn->mt_geo.upper);
- if (ret.err == MDBX_SUCCESS) {
- env->me_txn->mt_end_pgno = (pgno_t)aligned;
- goto done;
- }
+ if (txn) { /* transaction start */
+ if (env->me_poison_edge < txn->mt_next_pgno)
+ env->me_poison_edge = txn->mt_next_pgno;
+ VALGRIND_MAKE_MEM_DEFINED(env->me_map, pgno2bytes(env, txn->mt_next_pgno));
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(env->me_map,
+ pgno2bytes(env, txn->mt_next_pgno));
+ /* don't touch more, it should be already poisoned */
+ } else { /* transaction end */
+ bool should_unlock = false;
+ pgno_t last = MAX_PAGENO + 1;
+ if (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self()) {
+ /* inside write-txn */
+ const MDBX_meta *head = constmeta_prefer_last(env);
+ last = head->mm_geo.next;
+ } else if (env->me_flags & MDBX_RDONLY) {
+ /* read-only mode, no write-txn, no wlock mutex */
+ last = NUM_METAS;
+ } else if (mdbx_txn_lock(env, true) == MDBX_SUCCESS) {
+ /* no write-txn */
+ last = NUM_METAS;
+ should_unlock = true;
+ } else {
+ /* write txn is running, therefore shouldn't poison any memory range */
+ return;
+ }
- mdbx_error("unable growth datafile to %zu pages (+%zu), errcode %d",
- aligned, aligned - txn->mt_end_pgno, ret.err);
- } else {
- mdbx_notice("gc-alloc: next %zu > upper %" PRIaPGNO, next,
- txn->mt_geo.upper);
- }
+ last = mdbx_find_largest_this(env, last);
+ const pgno_t edge = env->me_poison_edge;
+ if (edge > last) {
+ mdbx_assert(env, last >= NUM_METAS);
+ env->me_poison_edge = last;
+ VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, last),
+ pgno2bytes(env, edge - last));
+ MDBX_ASAN_POISON_MEMORY_REGION(env->me_map + pgno2bytes(env, last),
+ pgno2bytes(env, edge - last));
}
+ if (should_unlock)
+ mdbx_txn_unlock(env);
+ }
+}
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
- fail:
- mdbx_assert(env,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- if (likely(!(flags & MDBX_ALLOC_FAKE)))
- txn->mt_flags |= MDBX_TXN_ERROR;
- if (num != 1 || ret.err != MDBX_NOTFOUND)
- mdbx_notice("alloc %u pages failed, flags 0x%x, errcode %d", num, flags,
- ret.err);
- else
- mdbx_trace("alloc %u pages failed, flags 0x%x, errcode %d", num, flags,
- ret.err);
- mdbx_assert(env, ret.err != MDBX_SUCCESS);
- ret.page = NULL;
- return ret;
+typedef struct {
+ int err;
+ MDBX_reader *rslot;
+} bind_rslot_result;
+
+static bind_rslot_result bind_rslot(MDBX_env *env, const uintptr_t tid) {
+ mdbx_assert(env, env->me_lck_mmap.lck);
+ mdbx_assert(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC);
+ mdbx_assert(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT);
+
+ bind_rslot_result result = {mdbx_rdt_lock(env), nullptr};
+ if (unlikely(MDBX_IS_ERROR(result.err)))
+ return result;
+ if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
+ mdbx_rdt_unlock(env);
+ result.err = MDBX_PANIC;
+ return result;
+ }
+ if (unlikely(!env->me_map)) {
+ mdbx_rdt_unlock(env);
+ result.err = MDBX_EPERM;
+ return result;
}
-done:
- mdbx_assert(env, !(flags & MDBX_ALLOC_SLOT));
- mdbx_ensure(env, pgno >= NUM_METAS);
- if (unlikely(flags & MDBX_ALLOC_FAKE)) {
- mdbx_debug("return NULL-page for %u pages of %s mode", num,
- "MDBX_ALLOC_FAKE");
- ret.page = NULL;
- ret.err = MDBX_SUCCESS;
- return ret;
+ if (unlikely(env->me_live_reader != env->me_pid)) {
+ result.err = mdbx_rpid_set(env);
+ if (unlikely(result.err != MDBX_SUCCESS)) {
+ mdbx_rdt_unlock(env);
+ return result;
+ }
+ env->me_live_reader = env->me_pid;
}
- if (env->me_flags & MDBX_WRITEMAP) {
- ret.page = pgno2page(env, pgno);
- /* LY: reset no-access flag from mdbx_page_loose() */
- VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));
- MDBX_ASAN_UNPOISON_MEMORY_REGION(ret.page, pgno2bytes(env, num));
- } else {
- ret.page = mdbx_page_malloc(txn, num);
- if (unlikely(!ret.page)) {
- ret.err = MDBX_ENOMEM;
- goto fail;
+ result.err = MDBX_SUCCESS;
+ unsigned slot, nreaders;
+ while (1) {
+ nreaders = atomic_load32(&env->me_lck->mti_numreaders, mo_Relaxed);
+ for (slot = 0; slot < nreaders; slot++)
+ if (atomic_load32(&env->me_lck->mti_readers[slot].mr_pid, mo_Relaxed) ==
+ 0)
+ break;
+
+ if (likely(slot < env->me_maxreaders))
+ break;
+
+ result.err = mdbx_cleanup_dead_readers(env, true, NULL);
+ if (result.err != MDBX_RESULT_TRUE) {
+ mdbx_rdt_unlock(env);
+ result.err =
+ (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
+ return result;
}
}
- if (range_begin) {
- mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
- mdbx_tassert(txn, pgno < txn->mt_next_pgno);
- mdbx_tassert(txn, pgno == re_list[range_begin]);
- /* Cutoff allocated pages from tw.reclaimed_pglist */
-#if MDBX_PNL_ASCENDING
- for (unsigned i = range_begin + num; i <= re_len;)
- re_list[range_begin++] = re_list[i++];
- MDBX_PNL_SIZE(re_list) = re_len = range_begin - 1;
-#else
- MDBX_PNL_SIZE(re_list) = re_len -= num;
- for (unsigned i = range_begin - num; i < re_len;)
- re_list[++i] = re_list[++range_begin];
-#endif
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- } else {
- txn->mt_next_pgno = pgno + num;
- mdbx_assert(env, txn->mt_next_pgno <= txn->mt_end_pgno);
+ result.rslot = &env->me_lck->mti_readers[slot];
+ /* Claim the reader slot, carefully since other code
+ * uses the reader table un-mutexed: First reset the
+ * slot, next publish it in lck->mti_numreaders. After
+ * that, it is safe for mdbx_env_close() to touch it.
+ * When it will be closed, we can finally claim it. */
+ atomic_store32(&result.rslot->mr_pid, 0, mo_Relaxed);
+ safe64_reset(&result.rslot->mr_txnid, true);
+ if (slot == nreaders)
+ atomic_store32(&env->me_lck->mti_numreaders, ++nreaders, mo_Relaxed);
+ atomic_store64(&result.rslot->mr_tid, (env->me_flags & MDBX_NOTLS) ? 0 : tid,
+ mo_Relaxed);
+ atomic_store32(&result.rslot->mr_pid, env->me_pid, mo_Relaxed);
+ mdbx_rdt_unlock(env);
+
+ if (likely(env->me_flags & MDBX_ENV_TXKEY)) {
+ mdbx_assert(env, env->me_live_reader == env->me_pid);
+ thread_rthc_set(env->me_txkey, result.rslot);
}
+ return result;
+}
- if (unlikely(env->me_flags & MDBX_PAGEPERTURB))
- memset(ret.page, -1, pgno2bytes(env, num));
- VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));
+__cold int mdbx_thread_register(const MDBX_env *env) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- ret.page->mp_pgno = pgno;
- ret.page->mp_leaf2_ksize = 0;
- ret.page->mp_flags = 0;
- if ((mdbx_assert_enabled() || mdbx_audit_enabled()) && num > 1) {
- ret.page->mp_pages = num;
- ret.page->mp_flags = P_OVERFLOW;
+ if (unlikely(!env->me_lck_mmap.lck))
+ return (env->me_flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM;
+
+ if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) {
+ mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
+ return MDBX_EINVAL /* MDBX_NOTLS mode */;
}
- ret.err = mdbx_page_dirty(txn, ret.page, num);
- if (unlikely(ret.err != MDBX_SUCCESS))
- goto fail;
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- return ret;
+ mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY |
+ MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY);
+ MDBX_reader *r = thread_rthc_get(env->me_txkey);
+ if (unlikely(r != NULL)) {
+ mdbx_assert(env, r->mr_pid.weak == env->me_pid);
+ mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
+ if (unlikely(r->mr_pid.weak != env->me_pid))
+ return MDBX_BAD_RSLOT;
+ return MDBX_RESULT_TRUE /* already registered */;
+ }
+
+ const uintptr_t tid = mdbx_thread_self();
+ if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == tid))
+ return MDBX_TXN_OVERLAPPING;
+ return bind_rslot((MDBX_env *)env, tid).err;
}
-/* Copy the used portions of a non-overflow page. */
-__hot static void mdbx_page_copy(MDBX_page *dst, const MDBX_page *src,
- size_t psize) {
- STATIC_ASSERT(UINT16_MAX > MAX_PAGESIZE - PAGEHDRSZ);
- STATIC_ASSERT(MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4);
- if ((src->mp_flags & (P_LEAF2 | P_OVERFLOW)) == 0) {
- size_t upper = src->mp_upper, lower = src->mp_lower, unused = upper - lower;
+__cold int mdbx_thread_unregister(const MDBX_env *env) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* If page isn't full, just copy the used portion. Adjust
- * alignment so memcpy may copy words instead of bytes. */
- if (unused >= MDBX_CACHELINE_SIZE * 2) {
- lower = ceil_powerof2(lower + PAGEHDRSZ, sizeof(void *));
- upper = floor_powerof2(upper + PAGEHDRSZ, sizeof(void *));
- memcpy(dst, src, lower);
- dst = (void *)((char *)dst + upper);
- src = (void *)((char *)src + upper);
- psize -= upper;
- }
+ if (unlikely(!env->me_lck_mmap.lck))
+ return MDBX_RESULT_TRUE;
+
+ if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) {
+ mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
+ return MDBX_RESULT_TRUE /* MDBX_NOTLS mode */;
}
- memcpy(dst, src, psize);
+
+ mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY |
+ MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY);
+ MDBX_reader *r = thread_rthc_get(env->me_txkey);
+ if (unlikely(r == NULL))
+ return MDBX_RESULT_TRUE /* not registered */;
+
+ mdbx_assert(env, r->mr_pid.weak == env->me_pid);
+ mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
+ if (unlikely(r->mr_pid.weak != env->me_pid ||
+ r->mr_tid.weak != mdbx_thread_self()))
+ return MDBX_BAD_RSLOT;
+
+ if (unlikely(r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD))
+ return MDBX_BUSY /* transaction is still active */;
+
+ atomic_store32(&r->mr_pid, 0, mo_Relaxed);
+ atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
+ mo_AcquireRelease);
+ thread_rthc_set(env->me_txkey, nullptr);
+ return MDBX_SUCCESS;
}
-/* Pull a page off the txn's spill list, if present.
- *
- * If a page being referenced was spilled to disk in this txn, bring
- * it back and make it dirty/writable again. */
-static struct page_result __must_check_result
-mdbx_page_unspill(MDBX_txn *const txn, const MDBX_page *const mp) {
- mdbx_verbose("unspill page %" PRIaPGNO, mp->mp_pgno);
- mdbx_tassert(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0);
- mdbx_tassert(txn, IS_SPILLED(txn, mp));
- const MDBX_txn *scan = txn;
- struct page_result ret;
- do {
- mdbx_tassert(txn, (scan->mt_flags & MDBX_TXN_SPILLS) != 0);
- const unsigned si = mdbx_search_spilled(scan, mp->mp_pgno);
- if (!si)
- continue;
- const unsigned npages = IS_OVERFLOW(mp) ? mp->mp_pages : 1;
- ret.page = mdbx_page_malloc(txn, npages);
- if (unlikely(!ret.page)) {
- ret.err = MDBX_ENOMEM;
- return ret;
+/* check against todo4recovery://erased_by_github/libmdbx/issues/269 */
+static bool meta_checktxnid(const MDBX_env *env, const MDBX_meta *meta,
+ bool report) {
+ const txnid_t meta_txnid = constmeta_txnid(env, meta);
+ const txnid_t freedb_mod_txnid = meta->mm_dbs[FREE_DBI].md_mod_txnid;
+ const txnid_t maindb_mod_txnid = meta->mm_dbs[MAIN_DBI].md_mod_txnid;
+
+ const pgno_t freedb_root_pgno = meta->mm_dbs[FREE_DBI].md_root;
+ const MDBX_page *freedb_root = (env->me_map && freedb_root_pgno != P_INVALID)
+ ? pgno2page(env, freedb_root_pgno)
+ : nullptr;
+
+ const pgno_t maindb_root_pgno = meta->mm_dbs[MAIN_DBI].md_root;
+ const MDBX_page *maindb_root = (env->me_map && maindb_root_pgno != P_INVALID)
+ ? pgno2page(env, maindb_root_pgno)
+ : nullptr;
+
+ const uint64_t magic_and_version =
+ unaligned_peek_u64(4, &meta->mm_magic_and_version);
+ bool ok = true;
+ if (unlikely(meta_txnid < freedb_mod_txnid ||
+ (!freedb_mod_txnid && freedb_root &&
+ likely(magic_and_version == MDBX_DATA_MAGIC)))) {
+ if (report)
+ mdbx_warning(
+ "catch invalid %sdb_mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
+ " %s",
+ "free", freedb_mod_txnid, meta_txnid,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ ok = false;
+ }
+ if (unlikely(meta_txnid < maindb_mod_txnid ||
+ (!maindb_mod_txnid && maindb_root &&
+ likely(magic_and_version == MDBX_DATA_MAGIC)))) {
+ if (report)
+ mdbx_warning(
+ "catch invalid %sdb_mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
+ " %s",
+ "main", maindb_mod_txnid, meta_txnid,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ ok = false;
+ }
+ if (likely(freedb_root && freedb_mod_txnid)) {
+ VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->mp_txnid));
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root,
+ sizeof(freedb_root->mp_txnid));
+ const txnid_t root_txnid = freedb_root->mp_txnid;
+ if (unlikely(root_txnid != freedb_mod_txnid)) {
+ if (report)
+ mdbx_warning(
+ "catch invalid root_page_txnid %" PRIaTXN
+ " for %sdb_mod_txnid %" PRIaTXN " %s",
+ root_txnid, "free", maindb_mod_txnid,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ ok = false;
+ }
+ }
+ if (likely(maindb_root && maindb_mod_txnid)) {
+ VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->mp_txnid));
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root,
+ sizeof(maindb_root->mp_txnid));
+ const txnid_t root_txnid = maindb_root->mp_txnid;
+ if (unlikely(root_txnid != maindb_mod_txnid)) {
+ if (report)
+ mdbx_warning(
+ "catch invalid root_page_txnid %" PRIaTXN
+ " for %sdb_mod_txnid %" PRIaTXN " %s",
+ root_txnid, "main", maindb_mod_txnid,
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ ok = false;
}
- mdbx_page_copy(ret.page, mp, pgno2bytes(txn->mt_env, npages));
- if (scan == txn) {
- /* If in current txn, this page is no longer spilled.
- * If it happens to be the last page, truncate the spill list.
- * Otherwise mark it as deleted by setting the LSB. */
- mdbx_spill_remove(txn, si, npages);
- } /* otherwise, if belonging to a parent txn, the
- * page remains spilled until child commits */
+ }
+ return ok;
+}
- ret.err = mdbx_page_dirty(txn, ret.page, npages);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
-#if MDBX_ENABLE_PGOP_STAT
- txn->mt_env->me_lck->mti_pgop_stat.unspill.weak += npages;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- ret.page->mp_flags |= (scan == txn) ? 0 : P_SPILLED;
- ret.err = MDBX_SUCCESS;
- return ret;
- } while (likely((scan = scan->mt_parent) != nullptr &&
- (scan->mt_flags & MDBX_TXN_SPILLS) != 0));
- mdbx_error("Page %" PRIaPGNO " mod-txnid %" PRIaTXN
- " not found in the spill-list(s), current txn %" PRIaTXN
- " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN,
- mp->mp_pgno, mp->mp_txnid, txn->mt_txnid, txn->mt_front,
- txn->mt_env->me_txn0->mt_txnid, txn->mt_env->me_txn0->mt_front);
- ret.err = MDBX_PROBLEM;
- ret.page = NULL;
- return ret;
+/* check with timeout as the workaround
+ * for todo4recovery://erased_by_github/libmdbx/issues/269 */
+static int meta_waittxnid(const MDBX_env *env, const MDBX_meta *meta,
+ uint64_t *timestamp) {
+ if (likely(meta_checktxnid(env, (const MDBX_meta *)meta, !*timestamp)))
+ return MDBX_SUCCESS;
+
+ if (!*timestamp)
+ *timestamp = mdbx_osal_monotime();
+ else if (unlikely(mdbx_osal_monotime() - *timestamp > 65536 / 10)) {
+ mdbx_error("bailout waiting for valid snapshot %s",
+ "(workaround for incoherent flaw of unified page/buffer cache)");
+ return MDBX_CORRUPTED;
+ }
+
+#if defined(_WIN32) || defined(_WIN64)
+ SwitchToThread();
+#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
+ sched_yield();
+#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
+ pthread_yield();
+#else
+ usleep(42);
+#endif
+ return MDBX_RESULT_TRUE;
}
-/* Touch a page: make it dirty and re-insert into tree with updated pgno.
- * Set MDBX_TXN_ERROR on failure.
- *
- * [in] mc cursor pointing to the page to be touched
- *
- * Returns 0 on success, non-zero on failure. */
-__hot static int mdbx_page_touch(MDBX_cursor *mc) {
- const MDBX_page *const mp = mc->mc_pg[mc->mc_top];
- MDBX_page *np;
- MDBX_txn *txn = mc->mc_txn;
+/* Common code for mdbx_txn_begin() and mdbx_txn_renew(). */
+static int mdbx_txn_renew0(MDBX_txn *txn, const unsigned flags) {
+ MDBX_env *env = txn->mt_env;
int rc;
- if (mdbx_assert_enabled()) {
- if (mc->mc_flags & C_SUB) {
- MDBX_xcursor *mx = container_of(mc->mc_db, MDBX_xcursor, mx_db);
- MDBX_cursor_couple *couple = container_of(mx, MDBX_cursor_couple, inner);
- mdbx_tassert(txn, mc->mc_db == &couple->outer.mc_xcursor->mx_db);
- mdbx_tassert(txn, mc->mc_dbx == &couple->outer.mc_xcursor->mx_dbx);
- mdbx_tassert(txn, *couple->outer.mc_dbistate & DBI_DIRTY);
- } else {
- mdbx_tassert(txn, *mc->mc_dbistate & DBI_DIRTY);
- }
- mdbx_tassert(txn, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY);
- mdbx_tassert(txn, !IS_OVERFLOW(mp));
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+#if MDBX_ENV_CHECKPID
+ if (unlikely(env->me_pid != mdbx_getpid())) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ return MDBX_PANIC;
}
+#endif /* MDBX_ENV_CHECKPID */
- if (IS_MODIFIABLE(txn, mp) || IS_SUBP(mp))
- return MDBX_SUCCESS;
-
- if (IS_FROZEN(txn, mp)) {
- /* CoW the page */
- rc = mdbx_pnl_need(&txn->tw.retired_pages, 1);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- const struct page_result par = mdbx_page_alloc(mc, 1, MDBX_ALLOC_ALL);
- rc = par.err;
- np = par.page;
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
+ STATIC_ASSERT(sizeof(MDBX_reader) == 32);
+#if MDBX_LOCKING > 0
+ STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_wlock) % MDBX_CACHELINE_SIZE == 0);
+ STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_rlock) % MDBX_CACHELINE_SIZE == 0);
+#else
+ STATIC_ASSERT(
+ offsetof(MDBX_lockinfo, mti_oldest_reader) % MDBX_CACHELINE_SIZE == 0);
+ STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE ==
+ 0);
+#endif /* MDBX_LOCKING */
+ STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE ==
+ 0);
- const pgno_t pgno = np->mp_pgno;
- mdbx_debug("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, DDBI(mc),
- mp->mp_pgno, pgno);
- mdbx_tassert(txn, mp->mp_pgno != pgno);
- mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno);
- /* Update the parent page, if any, to point to the new page */
- if (mc->mc_top) {
- MDBX_page *parent = mc->mc_pg[mc->mc_top - 1];
- MDBX_node *node = page_node(parent, mc->mc_ki[mc->mc_top - 1]);
- node_set_pgno(node, pgno);
+ const uintptr_t tid = mdbx_thread_self();
+ if (flags & MDBX_TXN_RDONLY) {
+ mdbx_assert(env, (flags & ~(MDBX_TXN_RO_BEGIN_FLAGS | MDBX_WRITEMAP)) == 0);
+ txn->mt_flags =
+ MDBX_TXN_RDONLY | (env->me_flags & (MDBX_NOTLS | MDBX_WRITEMAP));
+ MDBX_reader *r = txn->to.reader;
+ STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(r->mr_tid));
+ if (likely(env->me_flags & MDBX_ENV_TXKEY)) {
+ mdbx_assert(env, !(env->me_flags & MDBX_NOTLS));
+ r = thread_rthc_get(env->me_txkey);
+ if (likely(r)) {
+ if (unlikely(!r->mr_pid.weak) &&
+ (mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) {
+ thread_rthc_set(env->me_txkey, nullptr);
+ r = nullptr;
+ } else {
+ mdbx_assert(env, r->mr_pid.weak == env->me_pid);
+ mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
+ }
+ }
} else {
- mc->mc_db->md_root = pgno;
+ mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
}
-#if MDBX_ENABLE_PGOP_STAT
- txn->mt_env->me_lck->mti_pgop_stat.cow.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- mdbx_page_copy(np, mp, txn->mt_env->me_psize);
- np->mp_pgno = pgno;
- np->mp_txnid = txn->mt_front;
- } else if (IS_SPILLED(txn, mp)) {
- struct page_result pur = mdbx_page_unspill(txn, mp);
- np = pur.page;
- rc = pur.err;
- if (likely(rc == MDBX_SUCCESS)) {
- mdbx_tassert(txn, np != nullptr);
- goto done;
+ if (likely(r)) {
+ if (unlikely(r->mr_pid.weak != env->me_pid ||
+ r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD))
+ return MDBX_BAD_RSLOT;
+ } else if (env->me_lck_mmap.lck) {
+ bind_rslot_result brs = bind_rslot(env, tid);
+ if (unlikely(brs.err != MDBX_SUCCESS))
+ return brs.err;
+ r = brs.rslot;
}
- goto fail;
- } else {
- if (unlikely(!txn->mt_parent)) {
- mdbx_error("Unexpected not frozen/modifiable/spilled but shadowed %s "
- "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
- " without parent transaction, current txn %" PRIaTXN
- " front %" PRIaTXN,
- IS_BRANCH(mp) ? "branch" : "leaf", mp->mp_pgno, mp->mp_txnid,
- mc->mc_txn->mt_txnid, mc->mc_txn->mt_front);
- rc = MDBX_PROBLEM;
- goto fail;
+ txn->to.reader = r;
+ if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) {
+ mdbx_assert(env, txn->mt_txnid == 0);
+ mdbx_assert(env, txn->mt_owner == 0);
+ mdbx_assert(env, txn->mt_numdbs == 0);
+ if (likely(r)) {
+ mdbx_assert(env, r->mr_snapshot_pages_used.weak == 0);
+ mdbx_assert(env, r->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD);
+ atomic_store32(&r->mr_snapshot_pages_used, 0, mo_Relaxed);
+ }
+ txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED;
+ return MDBX_SUCCESS;
}
- mdbx_debug("clone db %d page %" PRIaPGNO, DDBI(mc), mp->mp_pgno);
- mdbx_tassert(txn, txn->tw.dirtylist->length <=
- MDBX_PGL_LIMIT + MDBX_PNL_GRANULATE);
- /* No - copy it */
- np = mdbx_page_malloc(txn, 1);
- if (unlikely(!np)) {
- rc = MDBX_ENOMEM;
- goto fail;
+ /* Seek & fetch the last meta */
+ if (likely(/* not recovery mode */ env->me_stuck_meta < 0)) {
+ uint64_t timestamp = 0;
+ while (1) {
+ volatile const MDBX_meta *const meta = meta_prefer_last(env);
+ mdbx_jitter4testing(false);
+ const txnid_t snap = meta_txnid(env, meta);
+ mdbx_jitter4testing(false);
+ if (likely(r)) {
+ safe64_reset(&r->mr_txnid, false);
+ atomic_store32(&r->mr_snapshot_pages_used, meta->mm_geo.next,
+ mo_Relaxed);
+ atomic_store64(&r->mr_snapshot_pages_retired,
+ unaligned_peek_u64_volatile(4, meta->mm_pages_retired),
+ mo_Relaxed);
+ safe64_write(&r->mr_txnid, snap);
+ mdbx_jitter4testing(false);
+ mdbx_assert(env, r->mr_pid.weak == mdbx_getpid());
+ mdbx_assert(
+ env, r->mr_tid.weak ==
+ ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self()));
+ mdbx_assert(env, r->mr_txnid.weak == snap);
+ atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
+ mo_AcquireRelease);
+ } else {
+ /* exclusive mode without lck */
+ }
+ mdbx_jitter4testing(true);
+
+ /* Snap the state from current meta-head */
+ txn->mt_txnid = snap;
+ txn->mt_geo = meta->mm_geo;
+ STATIC_ASSERT(CORE_DBS == 2);
+ txn->mt_dbs[0] = meta->mm_dbs[0];
+ txn->mt_dbs[1] = meta->mm_dbs[1];
+ txn->mt_canary = meta->mm_canary;
+
+ /* LY: Retry on a race, ITS#7970.
+ * The barrier is not needed here since C11-atomics are used,
+ * but it is reasonable paranoia to avoid compiler misoptimization
+ * and makes clarity for code readers. */
+ mdbx_compiler_barrier();
+ if (likely(meta == meta_prefer_last(env) &&
+ snap == meta_txnid(env, meta) &&
+ snap >= atomic_load64(&env->me_lck->mti_oldest_reader,
+ mo_AcquireRelease))) {
+ /* workaround for todo4recovery://erased_by_github/libmdbx/issues/269
+ */
+ rc = meta_waittxnid(env, (const MDBX_meta *)meta, ×tamp);
+ mdbx_jitter4testing(false);
+ if (likely(rc == MDBX_SUCCESS))
+ break;
+ if (likely(rc == MDBX_RESULT_TRUE))
+ continue;
+ goto bailout;
+ }
+ }
+ } else {
+ /* r/o recovery mode */
+ MDBX_meta *const meta = METAPAGE(env, env->me_stuck_meta);
+ txn->mt_txnid = constmeta_txnid(env, meta);
+ txn->mt_geo = meta->mm_geo;
+ memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db));
+ txn->mt_canary = meta->mm_canary;
+ if (likely(r)) {
+ atomic_store32(&r->mr_snapshot_pages_used, meta->mm_geo.next,
+ mo_Relaxed);
+ atomic_store64(&r->mr_snapshot_pages_retired,
+ unaligned_peek_u64(4, meta->mm_pages_retired),
+ mo_Relaxed);
+ atomic_store64(&r->mr_txnid, txn->mt_txnid, mo_Relaxed);
+ mdbx_jitter4testing(false);
+ mdbx_assert(env, r->mr_pid.weak == mdbx_getpid());
+ mdbx_assert(
+ env, r->mr_tid.weak ==
+ ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self()));
+ mdbx_assert(env, r->mr_txnid.weak == txn->mt_txnid);
+ atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
+ mo_Relaxed);
+ }
}
- mdbx_page_copy(np, mp, txn->mt_env->me_psize);
-
- /* insert a clone of parent's dirty page, so don't touch dirtyroom */
- rc = mdbx_page_dirty(txn, np, 1);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
-
-#if MDBX_ENABLE_PGOP_STAT
- txn->mt_env->me_lck->mti_pgop_stat.clone.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- }
-done:
- /* Adjust cursors pointing to mp */
- mc->mc_pg[mc->mc_top] = np;
- MDBX_cursor *m2 = txn->mt_cursors[mc->mc_dbi];
- if (mc->mc_flags & C_SUB) {
- for (; m2; m2 = m2->mc_next) {
- MDBX_cursor *m3 = &m2->mc_xcursor->mx_cursor;
- if (m3->mc_snum < mc->mc_snum)
- continue;
- if (m3->mc_pg[mc->mc_top] == mp)
- m3->mc_pg[mc->mc_top] = np;
+ if (unlikely(txn->mt_txnid < MIN_TXNID || txn->mt_txnid > MAX_TXNID)) {
+ mdbx_error("%s", "environment corrupted by died writer, must shutdown!");
+ rc = MDBX_CORRUPTED;
+ goto bailout;
}
+ mdbx_assert(env, txn->mt_txnid >= env->me_lck->mti_oldest_reader.weak);
+ txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
+ mdbx_ensure(env, txn->mt_txnid >=
+ /* paranoia is appropriate here */ env->me_lck
+ ->mti_oldest_reader.weak);
+ txn->mt_numdbs = env->me_numdbs;
} else {
- for (; m2; m2 = m2->mc_next) {
- if (m2->mc_snum < mc->mc_snum)
- continue;
- if (m2 == mc)
- continue;
- if (m2->mc_pg[mc->mc_top] == mp) {
- m2->mc_pg[mc->mc_top] = np;
- if (XCURSOR_INITED(m2) && IS_LEAF(np))
- XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]);
+ mdbx_assert(env, (flags & ~(MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS |
+ MDBX_WRITEMAP)) == 0);
+ if (unlikely(txn->mt_owner == tid ||
+ /* not recovery mode */ env->me_stuck_meta >= 0))
+ return MDBX_BUSY;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (lck && (env->me_flags & MDBX_NOTLS) == 0 &&
+ (mdbx_runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) {
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) ==
+ env->me_pid &&
+ unlikely(atomic_load64(&lck->mti_readers[i].mr_tid, mo_Relaxed) ==
+ tid)) {
+ const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (txnid >= MIN_TXNID && txnid <= MAX_TXNID)
+ return MDBX_TXN_OVERLAPPING;
+ }
}
}
- }
- return MDBX_SUCCESS;
-
-fail:
- txn->mt_flags |= MDBX_TXN_ERROR;
- return rc;
-}
-
-__cold static int mdbx_env_sync_internal(MDBX_env *env, bool force,
- bool nonblock) {
- bool locked = false;
- int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */;
-
-retry:;
- unsigned flags = env->me_flags & ~(MDBX_NOMETASYNC | MDBX_SHRINK_ALLOWED);
- if (unlikely((flags & (MDBX_RDONLY | MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE)) !=
- MDBX_ENV_ACTIVE)) {
- rc = MDBX_EACCESS;
- if (!(flags & MDBX_ENV_ACTIVE))
- rc = MDBX_EPERM;
- if (flags & MDBX_FATAL_ERROR)
- rc = MDBX_PANIC;
- goto bailout;
- }
-
- const pgno_t unsynced_pages =
- atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed);
- volatile const MDBX_meta *head = meta_prefer_last(env);
- const txnid_t head_txnid = meta_txnid(env, head);
- const uint32_t synched_meta_txnid_u32 =
- atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed);
- if (unsynced_pages == 0 && synched_meta_txnid_u32 == (uint32_t)head_txnid &&
- META_IS_STEADY(head))
- goto bailout;
-
- const pgno_t autosync_threshold =
- atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
- const uint64_t autosync_period =
- atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
- if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) ||
- (autosync_period &&
- mdbx_osal_monotime() -
- atomic_load64(&env->me_lck->mti_sync_timestamp, mo_Relaxed) >=
- autosync_period))
- flags &= MDBX_WRITEMAP /* clear flags for full steady sync */;
- const bool inside_txn = (env->me_txn0->mt_owner == mdbx_thread_self());
- if (!inside_txn) {
- if (!locked) {
- int err;
- unsigned wops = 0;
- /* pre-sync to avoid latency for writer */
- if (unsynced_pages > /* FIXME: define threshold */ 16 &&
- (flags & MDBX_SAFE_NOSYNC) == 0) {
- mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
- if (flags & MDBX_WRITEMAP) {
- /* Acquire guard to avoid collision with remap */
-#if defined(_WIN32) || defined(_WIN64)
- mdbx_srwlock_AcquireShared(&env->me_remap_guard);
-#else
- err = mdbx_fastmutex_acquire(&env->me_remap_guard);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-#endif
- const size_t usedbytes = pgno_align2os_bytes(env, head->mm_geo.next);
- err = mdbx_msync(&env->me_dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA);
+ /* Not yet touching txn == env->me_txn0, it may be active */
+ mdbx_jitter4testing(false);
+ rc = mdbx_txn_lock(env, F_ISSET(flags, MDBX_TXN_TRY));
+ if (unlikely(rc))
+ return rc;
+ if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
+ mdbx_txn_unlock(env);
+ return MDBX_PANIC;
+ }
#if defined(_WIN32) || defined(_WIN64)
- mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
-#else
- int unlock_err = mdbx_fastmutex_release(&env->me_remap_guard);
- if (unlikely(unlock_err != MDBX_SUCCESS) && err == MDBX_SUCCESS)
- err = unlock_err;
-#endif
- } else
- err = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA);
-
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- /* pre-sync done */
- wops = 1;
- rc = MDBX_SUCCESS /* means "some data was synced" */;
- }
-
- err = mdbx_txn_lock(env, nonblock);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ if (unlikely(!env->me_map)) {
+ mdbx_txn_unlock(env);
+ return MDBX_EPERM;
+ }
+#endif /* Windows */
- locked = true;
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += wops;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- goto retry;
+ mdbx_jitter4testing(false);
+ const MDBX_meta *meta = constmeta_prefer_last(env);
+ uint64_t timestamp = 0;
+ while (
+ "workaround for todo4recovery://erased_by_github/libmdbx/issues/269") {
+ rc = meta_waittxnid(env, (const MDBX_meta *)meta, ×tamp);
+ if (likely(rc == MDBX_SUCCESS))
+ break;
+ if (unlikely(rc != MDBX_RESULT_TRUE))
+ goto bailout;
+ }
+ mdbx_jitter4testing(false);
+ txn->mt_canary = meta->mm_canary;
+ const txnid_t snap = constmeta_txnid(env, meta);
+ txn->mt_txnid = safe64_txnid_next(snap);
+ if (unlikely(txn->mt_txnid > MAX_TXNID)) {
+ rc = MDBX_TXN_FULL;
+ mdbx_error("txnid overflow, raise %d", rc);
+ goto bailout;
}
- env->me_txn0->mt_txnid = head_txnid;
- mdbx_assert(env, head_txnid == meta_txnid(env, head));
- mdbx_assert(env, head_txnid == mdbx_recent_committed_txnid(env));
- mdbx_find_oldest(env->me_txn0);
- flags |= MDBX_SHRINK_ALLOWED;
- }
- mdbx_assert(env, inside_txn || locked);
- mdbx_assert(env, !inside_txn || (flags & MDBX_SHRINK_ALLOWED) == 0);
+ txn->mt_flags = flags;
+ txn->mt_child = NULL;
+ txn->tw.loose_pages = NULL;
+ txn->tw.loose_count = 0;
+#if MDBX_ENABLE_REFUND
+ txn->tw.loose_refund_wl = 0;
+#endif /* MDBX_ENABLE_REFUND */
+ MDBX_PNL_SIZE(txn->tw.retired_pages) = 0;
+ txn->tw.spill_pages = NULL;
+ txn->tw.spill_least_removed = 0;
+ txn->tw.last_reclaimed = 0;
+ if (txn->tw.lifo_reclaimed)
+ MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) = 0;
+ env->me_txn = txn;
+ txn->mt_numdbs = env->me_numdbs;
+ memcpy(txn->mt_dbiseqs, env->me_dbiseqs, txn->mt_numdbs * sizeof(unsigned));
+ /* Copy the DB info and flags */
+ memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db));
+ /* Moved to here to avoid a data race in read TXNs */
+ txn->mt_geo = meta->mm_geo;
- if (!META_IS_STEADY(head) ||
- ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) {
- mdbx_debug("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIaPGNO,
- data_page((const void *)head)->mp_pgno, mdbx_durable_str(head),
- unsynced_pages);
- MDBX_meta meta = *head;
- rc = mdbx_sync_locked(env, flags, &meta);
+ rc = mdbx_dpl_alloc(txn);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
+ txn->tw.dirtyroom = txn->mt_env->me_options.dp_limit;
+ txn->tw.dirtylru = MDBX_DEBUG ? ~42u : 0;
+ }
+
+ /* Setup db info */
+ mdbx_compiler_barrier();
+ memset(txn->mt_cursors, 0, sizeof(MDBX_cursor *) * txn->mt_numdbs);
+ for (unsigned i = CORE_DBS; i < txn->mt_numdbs; i++) {
+ const unsigned db_flags = env->me_dbflags[i];
+ txn->mt_dbs[i].md_flags = db_flags & DB_PERSISTENT_FLAGS;
+ txn->mt_dbistate[i] =
+ (db_flags & DB_VALID) ? DBI_VALID | DBI_USRVALID | DBI_STALE : 0;
}
+ txn->mt_dbistate[MAIN_DBI] = DBI_VALID | DBI_USRVALID;
+ txn->mt_dbistate[FREE_DBI] = DBI_VALID;
+ txn->mt_front =
+ txn->mt_txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0);
- /* LY: sync meta-pages if MDBX_NOMETASYNC enabled
- * and someone was not synced above. */
- if (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) !=
- (uint32_t)head_txnid) {
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- rc = (flags & MDBX_WRITEMAP)
- ? mdbx_msync(&env->me_dxb_mmap, 0,
- pgno_align2os_bytes(env, NUM_METAS),
- MDBX_SYNC_DATA | MDBX_SYNC_IODQ)
- : mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- if (likely(rc == MDBX_SUCCESS))
- atomic_store32(&env->me_lck->mti_meta_sync_txnid, (uint32_t)head_txnid,
- mo_Relaxed);
+ if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
+ mdbx_warning("%s", "environment had fatal error, must shutdown!");
+ rc = MDBX_PANIC;
+ } else {
+ const size_t size =
+ pgno2bytes(env, (txn->mt_flags & MDBX_TXN_RDONLY) ? txn->mt_next_pgno
+ : txn->mt_end_pgno);
+ if (unlikely(size > env->me_dxb_mmap.limit)) {
+ if (txn->mt_geo.upper > MAX_PAGENO + 1 ||
+ bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) !=
+ txn->mt_geo.upper) {
+ rc = MDBX_UNABLE_EXTEND_MAPSIZE;
+ goto bailout;
+ }
+ rc = mdbx_mapresize(env, txn->mt_next_pgno, txn->mt_end_pgno,
+ txn->mt_geo.upper,
+ (txn->mt_flags & MDBX_TXN_RDONLY) ? true : false);
+ if (rc != MDBX_SUCCESS)
+ goto bailout;
+ }
+ if (txn->mt_flags & MDBX_TXN_RDONLY) {
+#if defined(_WIN32) || defined(_WIN64)
+ if (((size > env->me_dbgeo.lower && env->me_dbgeo.shrink) ||
+ (mdbx_RunningUnderWine() &&
+ /* under Wine acquisition of remap_guard is always required,
+ * since Wine don't support section extending,
+ * i.e. in both cases unmap+map are required. */
+ size < env->me_dbgeo.upper && env->me_dbgeo.grow)) &&
+ /* avoid recursive use SRW */ (txn->mt_flags & MDBX_NOTLS) == 0) {
+ txn->mt_flags |= MDBX_SHRINK_ALLOWED;
+ mdbx_srwlock_AcquireShared(&env->me_remap_guard);
+ }
+#endif /* Windows */
+ } else {
+ env->me_dxb_mmap.current = size;
+ env->me_dxb_mmap.filesize =
+ (env->me_dxb_mmap.filesize < size) ? size : env->me_dxb_mmap.filesize;
+ }
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ mdbx_txn_valgrind(env, txn);
+#endif
+ txn->mt_owner = tid;
+ return MDBX_SUCCESS;
}
-
bailout:
- if (locked)
- mdbx_txn_unlock(env);
+ mdbx_tassert(txn, rc != MDBX_SUCCESS);
+ mdbx_txn_end(txn, MDBX_END_SLOT | MDBX_END_FAIL_BEGIN);
return rc;
}
-static __inline int check_env(const MDBX_env *env, const bool wanna_active) {
- if (unlikely(!env))
+static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) {
+ if (unlikely(!txn))
return MDBX_EINVAL;
- if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE))
+ if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDBX_EBADSIGN;
-#if MDBX_ENV_CHECKPID
- if (unlikely(env->me_pid != mdbx_getpid())) {
- ((MDBX_env *)env)->me_flags |= MDBX_FATAL_ERROR;
- return MDBX_PANIC;
- }
-#endif /* MDBX_ENV_CHECKPID */
+ if (unlikely(txn->mt_flags & bad_bits))
+ return MDBX_BAD_TXN;
- if (unlikely(env->me_flags & MDBX_FATAL_ERROR))
- return MDBX_PANIC;
+#if MDBX_TXN_CHECKOWNER
+ if ((txn->mt_flags & MDBX_NOTLS) == 0 &&
+ unlikely(txn->mt_owner != mdbx_thread_self()))
+ return txn->mt_owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN;
+#endif /* MDBX_TXN_CHECKOWNER */
- if (wanna_active) {
- if (unlikely((env->me_flags & MDBX_ENV_ACTIVE) == 0))
- return MDBX_EPERM;
- mdbx_assert(env, env->me_map != nullptr);
- }
+ if (unlikely(!txn->mt_env->me_map))
+ return MDBX_EPERM;
return MDBX_SUCCESS;
}
-__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) {
- int rc = check_env(env, true);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+static __always_inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) {
+ int err = check_txn(txn, bad_bits);
+ if (unlikely(err))
+ return err;
- return mdbx_env_sync_internal(env, force, nonblock);
+ if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
+ return MDBX_EACCESS;
+
+ return MDBX_SUCCESS;
}
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); }
+int mdbx_txn_renew(MDBX_txn *txn) {
+ if (unlikely(!txn))
+ return MDBX_EINVAL;
-__cold int mdbx_env_sync_poll(MDBX_env *env) {
- return __inline_mdbx_env_sync_poll(env);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+ if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
+ return MDBX_EBADSIGN;
-/* Back up parent txn's cursors, then grab the originals for tracking */
-static int mdbx_cursor_shadow(MDBX_txn *parent, MDBX_txn *nested) {
- for (int i = parent->mt_numdbs; --i >= 0;) {
- nested->mt_cursors[i] = NULL;
- MDBX_cursor *mc = parent->mt_cursors[i];
- if (mc != NULL) {
- size_t size = mc->mc_xcursor ? sizeof(MDBX_cursor) + sizeof(MDBX_xcursor)
- : sizeof(MDBX_cursor);
- for (MDBX_cursor *bk; mc; mc = bk->mc_next) {
- bk = mc;
- if (mc->mc_signature != MDBX_MC_LIVE)
- continue;
- bk = mdbx_malloc(size);
- if (unlikely(!bk))
- return MDBX_ENOMEM;
-#if MDBX_DEBUG
- memset(bk, 0xCD, size);
- VALGRIND_MAKE_MEM_UNDEFINED(bk, size);
-#endif /* MDBX_DEBUG */
- *bk = *mc;
- mc->mc_backup = bk;
- /* Kill pointers into src to reduce abuse: The
- * user may not use mc until dst ends. But we need a valid
- * txn pointer here for cursor fixups to keep working. */
- mc->mc_txn = nested;
- mc->mc_db = &nested->mt_dbs[i];
- mc->mc_dbistate = &nested->mt_dbistate[i];
- MDBX_xcursor *mx = mc->mc_xcursor;
- if (mx != NULL) {
- *(MDBX_xcursor *)(bk + 1) = *mx;
- mx->mx_cursor.mc_txn = nested;
- }
- mc->mc_next = nested->mt_cursors[i];
- nested->mt_cursors[i] = mc;
- }
- }
- }
- return MDBX_SUCCESS;
-}
+ if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0))
+ return MDBX_EINVAL;
-/* Close this txn's cursors, give parent txn's cursors back to parent.
- *
- * [in] txn the transaction handle.
- * [in] merge true to keep changes to parent cursors, false to revert.
- *
- * Returns 0 on success, non-zero on failure. */
-static void mdbx_cursors_eot(MDBX_txn *txn, const bool merge) {
- for (int i = txn->mt_numdbs; --i >= 0;) {
- MDBX_cursor *next, *mc = txn->mt_cursors[i];
- if (!mc)
- continue;
- txn->mt_cursors[i] = NULL;
- do {
- const unsigned stage = mc->mc_signature;
- MDBX_cursor *bk = mc->mc_backup;
- next = mc->mc_next;
- mdbx_ensure(txn->mt_env,
- stage == MDBX_MC_LIVE || (stage == MDBX_MC_WAIT4EOT && bk));
- mdbx_cassert(mc, mc->mc_dbi == (unsigned)i);
- if (bk) {
- MDBX_xcursor *mx = mc->mc_xcursor;
- mdbx_cassert(mc, mx == bk->mc_xcursor);
- mdbx_tassert(txn, txn->mt_parent != NULL);
- mdbx_ensure(txn->mt_env, bk->mc_signature == MDBX_MC_LIVE);
- if (stage == MDBX_MC_WAIT4EOT /* Cursor was closed by user */)
- mc->mc_signature = stage /* Promote closed state to parent txn */;
- else if (merge) {
- /* Restore pointers to parent txn */
- mc->mc_next = bk->mc_next;
- mc->mc_backup = bk->mc_backup;
- mc->mc_txn = bk->mc_txn;
- mc->mc_db = bk->mc_db;
- mc->mc_dbistate = bk->mc_dbistate;
- if (mx) {
- if (mx != bk->mc_xcursor) {
- *bk->mc_xcursor = *mx;
- mx = bk->mc_xcursor;
- }
- mx->mx_cursor.mc_txn = bk->mc_txn;
- }
- } else {
- /* Restore from backup, i.e. rollback/abort nested txn */
- *mc = *bk;
- if (mx)
- *mx = *(MDBX_xcursor *)(bk + 1);
- }
- bk->mc_signature = 0;
- mdbx_free(bk);
- } else {
- mdbx_ensure(txn->mt_env, stage == MDBX_MC_LIVE);
- mc->mc_signature = MDBX_MC_READY4CLOSE /* Cursor may be reused */;
- mc->mc_flags = 0 /* reset C_UNTRACK */;
- }
- } while ((mc = next) != NULL);
+ int rc;
+ if (unlikely(txn->mt_owner != 0 || !(txn->mt_flags & MDBX_TXN_FINISHED))) {
+ rc = mdbx_txn_reset(txn);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
-}
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
-/* Find largest mvcc-snapshot still referenced by this process. */
-static pgno_t mdbx_find_largest_this(MDBX_env *env, pgno_t largest) {
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (likely(lck != NULL /* exclusive mode */)) {
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- retry:
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease) ==
- env->me_pid) {
- /* mdbx_jitter4testing(true); */
- const pgno_t snap_pages = atomic_load32(
- &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed);
- const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (unlikely(
- snap_pages !=
- atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used,
- mo_AcquireRelease) ||
- snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
- goto retry;
- if (largest < snap_pages &&
- atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease) <=
- /* ignore pending updates */ snap_txnid &&
- snap_txnid <= MAX_TXNID)
- largest = snap_pages;
- }
- }
+ rc = mdbx_txn_renew0(txn, MDBX_TXN_RDONLY);
+ if (rc == MDBX_SUCCESS) {
+ txn->mt_owner = mdbx_thread_self();
+ mdbx_debug("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO
+ "/%" PRIaPGNO,
+ txn->mt_txnid, (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w',
+ (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root,
+ txn->mt_dbs[FREE_DBI].md_root);
}
- return largest;
+ return rc;
}
-static void mdbx_txn_valgrind(MDBX_env *env, MDBX_txn *txn) {
-#if !defined(__SANITIZE_ADDRESS__)
- if (!RUNNING_ON_VALGRIND)
- return;
-#endif
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags,
+ MDBX_txn **ret) {
+ return __inline_mdbx_txn_begin(env, parent, flags, ret);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- if (txn) { /* transaction start */
- if (env->me_poison_edge < txn->mt_next_pgno)
- env->me_poison_edge = txn->mt_next_pgno;
- VALGRIND_MAKE_MEM_DEFINED(env->me_map, pgno2bytes(env, txn->mt_next_pgno));
- MDBX_ASAN_UNPOISON_MEMORY_REGION(env->me_map,
- pgno2bytes(env, txn->mt_next_pgno));
- /* don't touch more, it should be already poisoned */
- } else { /* transaction end */
- bool should_unlock = false;
- pgno_t last = MAX_PAGENO + 1;
- if (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self()) {
- /* inside write-txn */
- const MDBX_meta *head = constmeta_prefer_last(env);
- last = head->mm_geo.next;
- } else if (env->me_flags & MDBX_RDONLY) {
- /* read-only mode, no write-txn, no wlock mutex */
- last = NUM_METAS;
- } else if (mdbx_txn_lock(env, true) == MDBX_SUCCESS) {
- /* no write-txn */
- last = NUM_METAS;
- should_unlock = true;
- } else {
- /* write txn is running, therefore shouldn't poison any memory range */
- return;
- }
+int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) {
+ int rc = check_txn(txn, MDBX_TXN_FINISHED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- last = mdbx_find_largest_this(env, last);
- const pgno_t edge = env->me_poison_edge;
- if (edge > last) {
- mdbx_assert(env, last >= NUM_METAS);
- env->me_poison_edge = last;
- VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, last),
- pgno2bytes(env, edge - last));
- MDBX_ASAN_POISON_MEMORY_REGION(env->me_map + pgno2bytes(env, last),
- pgno2bytes(env, edge - last));
- }
- if (should_unlock)
- mdbx_txn_unlock(env);
- }
+ txn->mt_userctx = ctx;
+ return MDBX_SUCCESS;
}
-#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
-typedef struct {
- int err;
- MDBX_reader *rslot;
-} bind_rslot_result;
+void *mdbx_txn_get_userctx(const MDBX_txn *txn) {
+ return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->mt_userctx;
+}
-static bind_rslot_result bind_rslot(MDBX_env *env, const uintptr_t tid) {
- mdbx_assert(env, env->me_lck_mmap.lck);
- mdbx_assert(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC);
- mdbx_assert(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT);
+int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags,
+ MDBX_txn **ret, void *context) {
+ MDBX_txn *txn;
+ unsigned size, tsize;
- bind_rslot_result result = {mdbx_rdt_lock(env), nullptr};
- if (unlikely(MDBX_IS_ERROR(result.err)))
- return result;
- if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
- mdbx_rdt_unlock(env);
- result.err = MDBX_PANIC;
- return result;
- }
- if (unlikely(!env->me_map)) {
- mdbx_rdt_unlock(env);
- result.err = MDBX_EPERM;
- return result;
- }
+ if (unlikely(!ret))
+ return MDBX_EINVAL;
+ *ret = NULL;
- if (unlikely(env->me_live_reader != env->me_pid)) {
- result.err = mdbx_rpid_set(env);
- if (unlikely(result.err != MDBX_SUCCESS)) {
- mdbx_rdt_unlock(env);
- return result;
+ if (unlikely((flags & ~MDBX_TXN_RW_BEGIN_FLAGS) &&
+ (flags & ~MDBX_TXN_RO_BEGIN_FLAGS)))
+ return MDBX_EINVAL;
+
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(env->me_flags & MDBX_RDONLY &
+ ~flags)) /* write txn in RDONLY env */
+ return MDBX_EACCESS;
+
+ flags |= env->me_flags & MDBX_WRITEMAP;
+
+ if (parent) {
+ /* Nested transactions: Max 1 child, write txns only, no writemap */
+ rc = check_txn_rw(parent,
+ MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (env->me_options.spill_parent4child_denominator) {
+ /* Spill dirty-pages of parent to provide dirtyroom for child txn */
+ rc = mdbx_txn_spill(parent, nullptr,
+ parent->tw.dirtylist->length /
+ env->me_options.spill_parent4child_denominator);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
- env->me_live_reader = env->me_pid;
+ mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
+
+ flags |= parent->mt_flags & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS);
+ } else if (flags & MDBX_TXN_RDONLY) {
+ if (env->me_txn0 &&
+ unlikely(env->me_txn0->mt_owner == mdbx_thread_self()) &&
+ (mdbx_runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0)
+ return MDBX_TXN_OVERLAPPING;
+ } else {
+ /* Reuse preallocated write txn. However, do not touch it until
+ * mdbx_txn_renew0() succeeds, since it currently may be active. */
+ txn = env->me_txn0;
+ goto renew;
}
- result.err = MDBX_SUCCESS;
- unsigned slot, nreaders;
- while (1) {
- nreaders = atomic_load32(&env->me_lck->mti_numreaders, mo_Relaxed);
- for (slot = 0; slot < nreaders; slot++)
- if (atomic_load32(&env->me_lck->mti_readers[slot].mr_pid, mo_Relaxed) ==
- 0)
- break;
+ size = env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + 1);
+ size += tsize = sizeof(MDBX_txn);
+ if (unlikely((txn = mdbx_malloc(size)) == NULL)) {
+ mdbx_debug("calloc: %s", "failed");
+ return MDBX_ENOMEM;
+ }
+#if MDBX_DEBUG
+ memset(txn, 0xCD, size);
+ VALGRIND_MAKE_MEM_UNDEFINED(txn, size);
+#endif /* MDBX_DEBUG */
+ memset(txn, 0, tsize);
+ txn->mt_dbxs = env->me_dbxs; /* static */
+ txn->mt_dbs = (MDBX_db *)((char *)txn + tsize);
+ txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs);
+ txn->mt_dbistate = (uint8_t *)txn + size - env->me_maxdbs;
+ txn->mt_flags = flags;
+ txn->mt_env = env;
- if (likely(slot < env->me_maxreaders))
- break;
+ if (parent) {
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ txn->mt_dbiseqs = parent->mt_dbiseqs;
+ txn->mt_geo = parent->mt_geo;
+ rc = mdbx_dpl_alloc(txn);
+ if (likely(rc == MDBX_SUCCESS)) {
+ const unsigned len =
+ MDBX_PNL_SIZE(parent->tw.reclaimed_pglist) + parent->tw.loose_count;
+ txn->tw.reclaimed_pglist =
+ mdbx_pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL);
+ if (unlikely(!txn->tw.reclaimed_pglist))
+ rc = MDBX_ENOMEM;
+ }
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ nested_failed:
+ mdbx_pnl_free(txn->tw.reclaimed_pglist);
+ mdbx_dpl_free(txn);
+ mdbx_free(txn);
+ return rc;
+ }
- result.err = mdbx_cleanup_dead_readers(env, true, NULL);
- if (result.err != MDBX_RESULT_TRUE) {
- mdbx_rdt_unlock(env);
- result.err =
- (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
- return result;
+ /* Move loose pages to reclaimed list */
+ if (parent->tw.loose_count) {
+ do {
+ MDBX_page *lp = parent->tw.loose_pages;
+ const unsigned di = mdbx_dpl_exist(parent, lp->mp_pgno);
+ mdbx_tassert(parent, di && parent->tw.dirtylist->items[di].ptr == lp);
+ mdbx_tassert(parent, lp->mp_flags == P_LOOSE);
+ rc =
+ mdbx_pnl_insert_range(&parent->tw.reclaimed_pglist, lp->mp_pgno, 1);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto nested_failed;
+ parent->tw.loose_pages = lp->mp_next;
+ /* Remove from dirty list */
+ mdbx_page_wash(parent, di, lp, 1);
+ } while (parent->tw.loose_pages);
+ parent->tw.loose_count = 0;
+#if MDBX_ENABLE_REFUND
+ parent->tw.loose_refund_wl = 0;
+#endif /* MDBX_ENABLE_REFUND */
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
}
- }
+ txn->tw.dirtyroom = parent->tw.dirtyroom;
+ txn->tw.dirtylru = parent->tw.dirtylru;
- result.rslot = &env->me_lck->mti_readers[slot];
- /* Claim the reader slot, carefully since other code
- * uses the reader table un-mutexed: First reset the
- * slot, next publish it in lck->mti_numreaders. After
- * that, it is safe for mdbx_env_close() to touch it.
- * When it will be closed, we can finally claim it. */
- atomic_store32(&result.rslot->mr_pid, 0, mo_Relaxed);
- safe64_reset(&result.rslot->mr_txnid, true);
- if (slot == nreaders)
- atomic_store32(&env->me_lck->mti_numreaders, ++nreaders, mo_Relaxed);
- atomic_store64(&result.rslot->mr_tid, (env->me_flags & MDBX_NOTLS) ? 0 : tid,
- mo_Relaxed);
- atomic_store32(&result.rslot->mr_pid, env->me_pid, mo_Relaxed);
- mdbx_rdt_unlock(env);
+ mdbx_dpl_sort(parent);
+ if (parent->tw.spill_pages)
+ mdbx_spill_purge(parent);
- if (likely(env->me_flags & MDBX_ENV_TXKEY)) {
- mdbx_assert(env, env->me_live_reader == env->me_pid);
- thread_rthc_set(env->me_txkey, result.rslot);
- }
- return result;
-}
+ mdbx_tassert(txn, MDBX_PNL_ALLOCLEN(txn->tw.reclaimed_pglist) >=
+ MDBX_PNL_SIZE(parent->tw.reclaimed_pglist));
+ memcpy(txn->tw.reclaimed_pglist, parent->tw.reclaimed_pglist,
+ MDBX_PNL_SIZEOF(parent->tw.reclaimed_pglist));
+ mdbx_assert(env, mdbx_pnl_check4assert(
+ txn->tw.reclaimed_pglist,
+ (txn->mt_next_pgno /* LY: intentional assignment here,
+ only for assertion */
+ = parent->mt_next_pgno) -
+ MDBX_ENABLE_REFUND));
-__cold int mdbx_thread_register(const MDBX_env *env) {
- int rc = check_env(env, true);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ txn->tw.last_reclaimed = parent->tw.last_reclaimed;
+ if (parent->tw.lifo_reclaimed) {
+ txn->tw.lifo_reclaimed = parent->tw.lifo_reclaimed;
+ parent->tw.lifo_reclaimed =
+ (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.lifo_reclaimed);
+ }
- if (unlikely(!env->me_lck_mmap.lck))
- return (env->me_flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM;
+ txn->tw.retired_pages = parent->tw.retired_pages;
+ parent->tw.retired_pages =
+ (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.retired_pages);
- if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) {
- mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
- return MDBX_EINVAL /* MDBX_NOTLS mode */;
+ txn->mt_txnid = parent->mt_txnid;
+ txn->mt_front = parent->mt_front + 1;
+#if MDBX_ENABLE_REFUND
+ txn->tw.loose_refund_wl = 0;
+#endif /* MDBX_ENABLE_REFUND */
+ txn->mt_canary = parent->mt_canary;
+ parent->mt_flags |= MDBX_TXN_HAS_CHILD;
+ parent->mt_child = txn;
+ txn->mt_parent = parent;
+ txn->mt_numdbs = parent->mt_numdbs;
+ txn->mt_owner = parent->mt_owner;
+ memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db));
+ /* Copy parent's mt_dbistate, but clear DB_NEW */
+ for (unsigned i = 0; i < txn->mt_numdbs; i++)
+ txn->mt_dbistate[i] =
+ parent->mt_dbistate[i] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
+ mdbx_tassert(parent,
+ parent->tw.dirtyroom + parent->tw.dirtylist->length ==
+ (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
+ : parent->mt_env->me_options.dp_limit));
+ mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
+ (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
+ : txn->mt_env->me_options.dp_limit));
+ env->me_txn = txn;
+ rc = mdbx_cursor_shadow(parent, txn);
+ if (mdbx_audit_enabled() && mdbx_assert_enabled()) {
+ txn->mt_signature = MDBX_MT_SIGNATURE;
+ mdbx_tassert(txn, mdbx_audit_ex(txn, 0, false) == 0);
+ }
+ if (unlikely(rc != MDBX_SUCCESS))
+ mdbx_txn_end(txn, MDBX_END_FAIL_BEGINCHILD);
+ } else { /* MDBX_TXN_RDONLY */
+ txn->mt_dbiseqs = env->me_dbiseqs;
+ renew:
+ rc = mdbx_txn_renew0(txn, flags);
}
- mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY |
- MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY);
- MDBX_reader *r = thread_rthc_get(env->me_txkey);
- if (unlikely(r != NULL)) {
- mdbx_assert(env, r->mr_pid.weak == env->me_pid);
- mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
- if (unlikely(r->mr_pid.weak != env->me_pid))
- return MDBX_BAD_RSLOT;
- return MDBX_RESULT_TRUE /* already registered */;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (txn != env->me_txn0)
+ mdbx_free(txn);
+ } else {
+ if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY))
+ mdbx_assert(env, txn->mt_flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED));
+ else if (flags & MDBX_TXN_RDONLY)
+ mdbx_assert(env, (txn->mt_flags &
+ ~(MDBX_NOTLS | MDBX_TXN_RDONLY | MDBX_WRITEMAP |
+ /* Win32: SRWL flag */ MDBX_SHRINK_ALLOWED)) == 0);
+ else {
+ mdbx_assert(env, (txn->mt_flags & ~(MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED |
+ MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC |
+ MDBX_TXN_SPILLS)) == 0);
+ assert(!txn->tw.spill_pages && !txn->tw.spill_least_removed);
+ }
+ txn->mt_signature = MDBX_MT_SIGNATURE;
+ txn->mt_userctx = context;
+ *ret = txn;
+ mdbx_debug("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO
+ "/%" PRIaPGNO,
+ txn->mt_txnid, (flags & MDBX_TXN_RDONLY) ? 'r' : 'w',
+ (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root,
+ txn->mt_dbs[FREE_DBI].md_root);
}
- const uintptr_t tid = mdbx_thread_self();
- if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == tid))
- return MDBX_TXN_OVERLAPPING;
- return bind_rslot((MDBX_env *)env, tid).err;
+ return rc;
}
-__cold int mdbx_thread_unregister(const MDBX_env *env) {
- int rc = check_env(env, true);
+int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
+ int rc = check_txn(txn, MDBX_TXN_FINISHED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!env->me_lck_mmap.lck))
- return MDBX_RESULT_TRUE;
+ if (unlikely(!info))
+ return MDBX_EINVAL;
- if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) {
- mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
- return MDBX_RESULT_TRUE /* MDBX_NOTLS mode */;
+ MDBX_env *const env = txn->mt_env;
+#if MDBX_ENV_CHECKPID
+ if (unlikely(env->me_pid != mdbx_getpid())) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ return MDBX_PANIC;
}
+#endif /* MDBX_ENV_CHECKPID */
- mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY |
- MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY);
- MDBX_reader *r = thread_rthc_get(env->me_txkey);
- if (unlikely(r == NULL))
- return MDBX_RESULT_TRUE /* not registered */;
-
- mdbx_assert(env, r->mr_pid.weak == env->me_pid);
- mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
- if (unlikely(r->mr_pid.weak != env->me_pid ||
- r->mr_tid.weak != mdbx_thread_self()))
- return MDBX_BAD_RSLOT;
-
- if (unlikely(r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD))
- return MDBX_BUSY /* transaction is still active */;
-
- atomic_store32(&r->mr_pid, 0, mo_Relaxed);
- atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
- mo_AcquireRelease);
- thread_rthc_set(env->me_txkey, nullptr);
- return MDBX_SUCCESS;
-}
-
-/* check against todo4recovery://erased_by_github/libmdbx/issues/269 */
-static bool meta_checktxnid(const MDBX_env *env, const MDBX_meta *meta,
- bool report) {
- const txnid_t meta_txnid = constmeta_txnid(env, meta);
- const txnid_t freedb_mod_txnid = meta->mm_dbs[FREE_DBI].md_mod_txnid;
- const txnid_t maindb_mod_txnid = meta->mm_dbs[MAIN_DBI].md_mod_txnid;
+ info->txn_id = txn->mt_txnid;
+ info->txn_space_used = pgno2bytes(env, txn->mt_geo.next);
- const pgno_t freedb_root_pgno = meta->mm_dbs[FREE_DBI].md_root;
- const MDBX_page *freedb_root = (env->me_map && freedb_root_pgno != P_INVALID)
- ? pgno2page(env, freedb_root_pgno)
- : nullptr;
+ if (txn->mt_flags & MDBX_TXN_RDONLY) {
+ volatile const MDBX_meta *head_meta;
+ txnid_t head_txnid;
+ uint64_t head_retired;
+ do {
+ /* fetch info from volatile head */
+ head_meta = meta_prefer_last(env);
+ head_txnid = meta_txnid(env, head_meta);
+ head_retired =
+ unaligned_peek_u64_volatile(4, head_meta->mm_pages_retired);
+ info->txn_space_limit_soft = pgno2bytes(env, head_meta->mm_geo.now);
+ info->txn_space_limit_hard = pgno2bytes(env, head_meta->mm_geo.upper);
+ info->txn_space_leftover =
+ pgno2bytes(env, head_meta->mm_geo.now - head_meta->mm_geo.next);
+ mdbx_compiler_barrier();
+ } while (unlikely(head_meta != meta_prefer_last(env) ||
+ head_txnid != meta_txnid(env, head_meta)));
- const pgno_t maindb_root_pgno = meta->mm_dbs[MAIN_DBI].md_root;
- const MDBX_page *maindb_root = (env->me_map && maindb_root_pgno != P_INVALID)
- ? pgno2page(env, maindb_root_pgno)
- : nullptr;
+ info->txn_reader_lag = head_txnid - info->txn_id;
+ info->txn_space_dirty = info->txn_space_retired = 0;
+ uint64_t reader_snapshot_pages_retired;
+ if (txn->to.reader &&
+ head_retired >
+ (reader_snapshot_pages_retired = atomic_load64(
+ &txn->to.reader->mr_snapshot_pages_retired, mo_Relaxed))) {
+ info->txn_space_dirty = info->txn_space_retired = pgno2bytes(
+ env, (pgno_t)(head_retired - reader_snapshot_pages_retired));
- const uint64_t magic_and_version =
- unaligned_peek_u64(4, &meta->mm_magic_and_version);
- bool ok = true;
- if (unlikely(meta_txnid < freedb_mod_txnid ||
- (!freedb_mod_txnid && freedb_root &&
- likely(magic_and_version == MDBX_DATA_MAGIC)))) {
- if (report)
- mdbx_warning(
- "catch invalid %sdb_mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
- " %s",
- "free", freedb_mod_txnid, meta_txnid,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- ok = false;
- }
- if (unlikely(meta_txnid < maindb_mod_txnid ||
- (!maindb_mod_txnid && maindb_root &&
- likely(magic_and_version == MDBX_DATA_MAGIC)))) {
- if (report)
- mdbx_warning(
- "catch invalid %sdb_mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
- " %s",
- "main", maindb_mod_txnid, meta_txnid,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- ok = false;
- }
- if (likely(freedb_root && freedb_mod_txnid)) {
- VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->mp_txnid));
- MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root,
- sizeof(freedb_root->mp_txnid));
- const txnid_t root_txnid = freedb_root->mp_txnid;
- if (unlikely(root_txnid != freedb_mod_txnid)) {
- if (report)
- mdbx_warning(
- "catch invalid root_page_txnid %" PRIaTXN
- " for %sdb_mod_txnid %" PRIaTXN " %s",
- root_txnid, "free", maindb_mod_txnid,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- ok = false;
+ size_t retired_next_reader = 0;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (scan_rlt && info->txn_reader_lag > 1 && lck) {
+ /* find next more recent reader */
+ txnid_t next_reader = head_txnid;
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ retry:
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
+ mdbx_jitter4testing(true);
+ const txnid_t snap_txnid =
+ safe64_read(&lck->mti_readers[i].mr_txnid);
+ const uint64_t snap_retired =
+ atomic_load64(&lck->mti_readers[i].mr_snapshot_pages_retired,
+ mo_AcquireRelease);
+ if (unlikely(snap_retired !=
+ atomic_load64(
+ &lck->mti_readers[i].mr_snapshot_pages_retired,
+ mo_Relaxed)) ||
+ snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))
+ goto retry;
+ if (snap_txnid <= txn->mt_txnid) {
+ retired_next_reader = 0;
+ break;
+ }
+ if (snap_txnid < next_reader) {
+ next_reader = snap_txnid;
+ retired_next_reader = pgno2bytes(
+ env, (pgno_t)(snap_retired -
+ atomic_load64(
+ &txn->to.reader->mr_snapshot_pages_retired,
+ mo_Relaxed)));
+ }
+ }
+ }
+ }
+ info->txn_space_dirty = retired_next_reader;
}
- }
- if (likely(maindb_root && maindb_mod_txnid)) {
- VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->mp_txnid));
- MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root,
- sizeof(maindb_root->mp_txnid));
- const txnid_t root_txnid = maindb_root->mp_txnid;
- if (unlikely(root_txnid != maindb_mod_txnid)) {
- if (report)
- mdbx_warning(
- "catch invalid root_page_txnid %" PRIaTXN
- " for %sdb_mod_txnid %" PRIaTXN " %s",
- root_txnid, "main", maindb_mod_txnid,
- "(workaround for incoherent flaw of unified page/buffer cache)");
- ok = false;
+ } else {
+ info->txn_space_limit_soft = pgno2bytes(env, txn->mt_geo.now);
+ info->txn_space_limit_hard = pgno2bytes(env, txn->mt_geo.upper);
+ info->txn_space_retired = pgno2bytes(
+ env, txn->mt_child ? (unsigned)(uintptr_t)txn->tw.retired_pages
+ : MDBX_PNL_SIZE(txn->tw.retired_pages));
+ info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom);
+ info->txn_space_dirty =
+ pgno2bytes(env, txn->mt_env->me_options.dp_limit - txn->tw.dirtyroom);
+ info->txn_reader_lag = INT64_MAX;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (scan_rlt && lck) {
+ txnid_t oldest_snapshot = txn->mt_txnid;
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ if (snap_nreaders) {
+ oldest_snapshot = mdbx_find_oldest(txn);
+ if (oldest_snapshot == txn->mt_txnid - 1) {
+ /* check if there is at least one reader */
+ bool exists = false;
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) &&
+ txn->mt_txnid > safe64_read(&lck->mti_readers[i].mr_txnid)) {
+ exists = true;
+ break;
+ }
+ }
+ oldest_snapshot += !exists;
+ }
+ }
+ info->txn_reader_lag = txn->mt_txnid - oldest_snapshot;
}
}
- return ok;
+
+ return MDBX_SUCCESS;
}
-/* check with timeout as the workaround
- * for todo4recovery://erased_by_github/libmdbx/issues/269 */
-static int meta_waittxnid(const MDBX_env *env, const MDBX_meta *meta,
- uint64_t *timestamp) {
- if (likely(meta_checktxnid(env, (const MDBX_meta *)meta, !*timestamp)))
- return MDBX_SUCCESS;
+MDBX_env *mdbx_txn_env(const MDBX_txn *txn) {
+ if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE ||
+ txn->mt_env->me_signature.weak != MDBX_ME_SIGNATURE))
+ return NULL;
+ return txn->mt_env;
+}
- if (!*timestamp)
- *timestamp = mdbx_osal_monotime();
- else if (unlikely(mdbx_osal_monotime() - *timestamp > 65536 / 10)) {
- mdbx_error("bailout waiting for valid snapshot %s",
- "(workaround for incoherent flaw of unified page/buffer cache)");
- return MDBX_CORRUPTED;
- }
+uint64_t mdbx_txn_id(const MDBX_txn *txn) {
+ if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
+ return 0;
+ return txn->mt_txnid;
+}
-#if defined(_WIN32) || defined(_WIN64)
- SwitchToThread();
-#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
- sched_yield();
-#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS)
- pthread_yield();
-#else
- usleep(42);
-#endif
- return MDBX_RESULT_TRUE;
+int mdbx_txn_flags(const MDBX_txn *txn) {
+ if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
+ return -1;
+ return txn->mt_flags;
}
-/* Common code for mdbx_txn_begin() and mdbx_txn_renew(). */
-static int mdbx_txn_renew0(MDBX_txn *txn, const unsigned flags) {
- MDBX_env *env = txn->mt_env;
- int rc;
+/* Check for misused dbi handles */
+#define TXN_DBI_CHANGED(txn, dbi) \
+ ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi])
-#if MDBX_ENV_CHECKPID
- if (unlikely(env->me_pid != mdbx_getpid())) {
- env->me_flags |= MDBX_FATAL_ERROR;
- return MDBX_PANIC;
+static void dbi_import_locked(MDBX_txn *txn) {
+ MDBX_env *const env = txn->mt_env;
+ const unsigned n = env->me_numdbs;
+ for (unsigned i = CORE_DBS; i < n; ++i) {
+ if (i >= txn->mt_numdbs) {
+ txn->mt_dbistate[i] = 0;
+ txn->mt_cursors[i] = NULL;
+ }
+ if ((env->me_dbflags[i] & DB_VALID) &&
+ !(txn->mt_dbistate[i] & DBI_USRVALID)) {
+ txn->mt_dbiseqs[i] = env->me_dbiseqs[i];
+ txn->mt_dbs[i].md_flags = env->me_dbflags[i] & DB_PERSISTENT_FLAGS;
+ txn->mt_dbistate[i] = DBI_VALID | DBI_USRVALID | DBI_STALE;
+ mdbx_tassert(txn, txn->mt_dbxs[i].md_cmp != NULL);
+ mdbx_tassert(txn, txn->mt_dbxs[i].md_name.iov_base != NULL);
+ }
}
-#endif /* MDBX_ENV_CHECKPID */
+ txn->mt_numdbs = n;
+}
- STATIC_ASSERT(sizeof(MDBX_reader) == 32);
-#if MDBX_LOCKING > 0
- STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_wlock) % MDBX_CACHELINE_SIZE == 0);
- STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_rlock) % MDBX_CACHELINE_SIZE == 0);
-#else
- STATIC_ASSERT(
- offsetof(MDBX_lockinfo, mti_oldest_reader) % MDBX_CACHELINE_SIZE == 0);
- STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE ==
- 0);
-#endif /* MDBX_LOCKING */
- STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE ==
- 0);
+/* Import DBI which opened after txn started into context */
+__cold static bool dbi_import(MDBX_txn *txn, MDBX_dbi dbi) {
+ if (dbi < CORE_DBS || dbi >= txn->mt_env->me_numdbs)
+ return false;
- const uintptr_t tid = mdbx_thread_self();
- if (flags & MDBX_TXN_RDONLY) {
- mdbx_assert(env, (flags & ~(MDBX_TXN_RO_BEGIN_FLAGS | MDBX_WRITEMAP)) == 0);
- txn->mt_flags =
- MDBX_TXN_RDONLY | (env->me_flags & (MDBX_NOTLS | MDBX_WRITEMAP));
- MDBX_reader *r = txn->to.reader;
- STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(r->mr_tid));
- if (likely(env->me_flags & MDBX_ENV_TXKEY)) {
- mdbx_assert(env, !(env->me_flags & MDBX_NOTLS));
- r = thread_rthc_get(env->me_txkey);
- if (likely(r)) {
- if (unlikely(!r->mr_pid.weak) &&
- (mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) {
- thread_rthc_set(env->me_txkey, nullptr);
- r = nullptr;
- } else {
- mdbx_assert(env, r->mr_pid.weak == env->me_pid);
- mdbx_assert(env, r->mr_tid.weak == mdbx_thread_self());
+ mdbx_ensure(txn->mt_env, mdbx_fastmutex_acquire(&txn->mt_env->me_dbi_lock) ==
+ MDBX_SUCCESS);
+ dbi_import_locked(txn);
+ mdbx_ensure(txn->mt_env, mdbx_fastmutex_release(&txn->mt_env->me_dbi_lock) ==
+ MDBX_SUCCESS);
+ return txn->mt_dbistate[dbi] & DBI_USRVALID;
+}
+
+/* Export or close DBI handles opened in this txn. */
+static void dbi_update(MDBX_txn *txn, int keep) {
+ mdbx_tassert(txn, !txn->mt_parent && txn == txn->mt_env->me_txn0);
+ MDBX_dbi n = txn->mt_numdbs;
+ if (n) {
+ bool locked = false;
+ MDBX_env *const env = txn->mt_env;
+
+ for (unsigned i = n; --i >= CORE_DBS;) {
+ if (likely((txn->mt_dbistate[i] & DBI_CREAT) == 0))
+ continue;
+ if (!locked) {
+ mdbx_ensure(env,
+ mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS);
+ locked = true;
+ }
+ if (env->me_numdbs <= i || txn->mt_dbiseqs[i] != env->me_dbiseqs[i])
+ continue /* dbi explicitly closed and/or then re-opened by other txn */;
+ if (keep) {
+ env->me_dbflags[i] = txn->mt_dbs[i].md_flags | DB_VALID;
+ } else {
+ char *ptr = env->me_dbxs[i].md_name.iov_base;
+ if (ptr) {
+ env->me_dbxs[i].md_name.iov_len = 0;
+ mdbx_memory_fence(mo_AcquireRelease, true);
+ mdbx_assert(env, env->me_dbflags[i] == 0);
+ env->me_dbiseqs[i]++;
+ env->me_dbxs[i].md_name.iov_base = NULL;
+ mdbx_free(ptr);
}
}
- } else {
- mdbx_assert(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS));
}
- if (likely(r)) {
- if (unlikely(r->mr_pid.weak != env->me_pid ||
- r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD))
- return MDBX_BAD_RSLOT;
- } else if (env->me_lck_mmap.lck) {
- bind_rslot_result brs = bind_rslot(env, tid);
- if (unlikely(brs.err != MDBX_SUCCESS))
- return brs.err;
- r = brs.rslot;
+ n = env->me_numdbs;
+ if (n > CORE_DBS && unlikely(!(env->me_dbflags[n - 1] & DB_VALID))) {
+ if (!locked) {
+ mdbx_ensure(env,
+ mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS);
+ locked = true;
+ }
+
+ n = env->me_numdbs;
+ while (n > CORE_DBS && !(env->me_dbflags[n - 1] & DB_VALID))
+ --n;
+ env->me_numdbs = n;
}
- txn->to.reader = r;
- if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) {
- mdbx_assert(env, txn->mt_txnid == 0);
- mdbx_assert(env, txn->mt_owner == 0);
- mdbx_assert(env, txn->mt_numdbs == 0);
- if (likely(r)) {
- mdbx_assert(env, r->mr_snapshot_pages_used.weak == 0);
- mdbx_assert(env, r->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD);
- atomic_store32(&r->mr_snapshot_pages_used, 0, mo_Relaxed);
+
+ if (unlikely(locked))
+ mdbx_ensure(env,
+ mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
+ }
+}
+
+/* Filter-out pgno list from transaction's dirty-page list */
+static void mdbx_dpl_sift(MDBX_txn *const txn, MDBX_PNL pl,
+ const bool spilled) {
+ if (MDBX_PNL_SIZE(pl) && txn->tw.dirtylist->length) {
+ mdbx_tassert(
+ txn, mdbx_pnl_check4assert(pl, (size_t)txn->mt_next_pgno << spilled));
+ MDBX_dpl *dl = mdbx_dpl_sort(txn);
+
+ /* Scanning in ascend order */
+ const int step = MDBX_PNL_ASCENDING ? 1 : -1;
+ const int begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_SIZE(pl);
+ const int end = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(pl) + 1 : 0;
+ mdbx_tassert(txn, pl[begin] <= pl[end - step]);
+
+ unsigned r = mdbx_dpl_search(txn, pl[begin] >> spilled);
+ mdbx_tassert(txn, dl->sorted == dl->length);
+ for (int i = begin; r <= dl->length;) { /* scan loop */
+ assert(i != end);
+ mdbx_tassert(txn, !spilled || (pl[i] & 1) == 0);
+ pgno_t pl_pgno = pl[i] >> spilled;
+ pgno_t dp_pgno = dl->items[r].pgno;
+ if (likely(dp_pgno != pl_pgno)) {
+ const bool cmp = dp_pgno < pl_pgno;
+ r += cmp;
+ i += cmp ? 0 : step;
+ if (likely(i != end))
+ continue;
+ return;
+ }
+
+ /* update loop */
+ unsigned w = r;
+ remove_dl:
+ if ((txn->mt_env->me_flags & MDBX_WRITEMAP) == 0) {
+ MDBX_page *dp = dl->items[r].ptr;
+ mdbx_dpage_free(txn->mt_env, dp, dpl_npages(dl, r));
+ }
+ ++r;
+ next_i:
+ i += step;
+ if (unlikely(i == end)) {
+ while (r <= dl->length)
+ dl->items[w++] = dl->items[r++];
+ } else {
+ while (r <= dl->length) {
+ assert(i != end);
+ mdbx_tassert(txn, !spilled || (pl[i] & 1) == 0);
+ pl_pgno = pl[i] >> spilled;
+ dp_pgno = dl->items[r].pgno;
+ if (dp_pgno < pl_pgno)
+ dl->items[w++] = dl->items[r++];
+ else if (dp_pgno > pl_pgno)
+ goto next_i;
+ else
+ goto remove_dl;
+ }
}
- txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED;
- return MDBX_SUCCESS;
+ dl->sorted = dpl_setlen(dl, w - 1);
+ txn->tw.dirtyroom += r - w;
+ mdbx_tassert(txn,
+ txn->tw.dirtyroom + txn->tw.dirtylist->length ==
+ (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
+ : txn->mt_env->me_options.dp_limit));
+ return;
}
+ }
+}
- /* Seek & fetch the last meta */
- if (likely(/* not recovery mode */ env->me_stuck_meta < 0)) {
- uint64_t timestamp = 0;
- while (1) {
- volatile const MDBX_meta *const meta = meta_prefer_last(env);
- mdbx_jitter4testing(false);
- const txnid_t snap = meta_txnid(env, meta);
- mdbx_jitter4testing(false);
- if (likely(r)) {
- safe64_reset(&r->mr_txnid, false);
- atomic_store32(&r->mr_snapshot_pages_used, meta->mm_geo.next,
- mo_Relaxed);
- atomic_store64(&r->mr_snapshot_pages_retired,
- unaligned_peek_u64_volatile(4, meta->mm_pages_retired),
- mo_Relaxed);
- safe64_write(&r->mr_txnid, snap);
- mdbx_jitter4testing(false);
- mdbx_assert(env, r->mr_pid.weak == mdbx_getpid());
- mdbx_assert(
- env, r->mr_tid.weak ==
- ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self()));
- mdbx_assert(env, r->mr_txnid.weak == snap);
- atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
- mo_AcquireRelease);
- } else {
- /* exclusive mode without lck */
- }
- mdbx_jitter4testing(true);
+/* End a transaction, except successful commit of a nested transaction.
+ * May be called twice for readonly txns: First reset it, then abort.
+ * [in] txn the transaction handle to end
+ * [in] mode why and how to end the transaction */
+static int mdbx_txn_end(MDBX_txn *txn, const unsigned mode) {
+ MDBX_env *env = txn->mt_env;
+ static const char *const names[] = MDBX_END_NAMES;
- /* Snap the state from current meta-head */
- txn->mt_txnid = snap;
- txn->mt_geo = meta->mm_geo;
- STATIC_ASSERT(CORE_DBS == 2);
- txn->mt_dbs[0] = meta->mm_dbs[0];
- txn->mt_dbs[1] = meta->mm_dbs[1];
- txn->mt_canary = meta->mm_canary;
+#if MDBX_ENV_CHECKPID
+ if (unlikely(txn->mt_env->me_pid != mdbx_getpid())) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ return MDBX_PANIC;
+ }
+#endif /* MDBX_ENV_CHECKPID */
- /* LY: Retry on a race, ITS#7970.
- * The barrier is not needed here since C11-atomics are used,
- * but it is reasonable paranoia to avoid compiler misoptimization
- * and makes clarity for code readers. */
- mdbx_compiler_barrier();
- if (likely(meta == meta_prefer_last(env) &&
- snap == meta_txnid(env, meta) &&
- snap >= atomic_load64(&env->me_lck->mti_oldest_reader,
- mo_AcquireRelease))) {
- /* workaround for todo4recovery://erased_by_github/libmdbx/issues/269
- */
- rc = meta_waittxnid(env, (const MDBX_meta *)meta, ×tamp);
- mdbx_jitter4testing(false);
- if (likely(rc == MDBX_SUCCESS))
- break;
- if (likely(rc == MDBX_RESULT_TRUE))
- continue;
- goto bailout;
- }
- }
- } else {
- /* r/o recovery mode */
- MDBX_meta *const meta = METAPAGE(env, env->me_stuck_meta);
- txn->mt_txnid = constmeta_txnid(env, meta);
- txn->mt_geo = meta->mm_geo;
- memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db));
- txn->mt_canary = meta->mm_canary;
- if (likely(r)) {
- atomic_store32(&r->mr_snapshot_pages_used, meta->mm_geo.next,
- mo_Relaxed);
- atomic_store64(&r->mr_snapshot_pages_retired,
- unaligned_peek_u64(4, meta->mm_pages_retired),
- mo_Relaxed);
- atomic_store64(&r->mr_txnid, txn->mt_txnid, mo_Relaxed);
- mdbx_jitter4testing(false);
- mdbx_assert(env, r->mr_pid.weak == mdbx_getpid());
- mdbx_assert(
- env, r->mr_tid.weak ==
- ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self()));
- mdbx_assert(env, r->mr_txnid.weak == txn->mt_txnid);
+ mdbx_debug("%s txn %" PRIaTXN "%c %p on mdbenv %p, root page %" PRIaPGNO
+ "/%" PRIaPGNO,
+ names[mode & MDBX_END_OPMASK], txn->mt_txnid,
+ (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn,
+ (void *)env, txn->mt_dbs[MAIN_DBI].md_root,
+ txn->mt_dbs[FREE_DBI].md_root);
+
+ mdbx_ensure(env, txn->mt_txnid >=
+ /* paranoia is appropriate here */ env->me_lck
+ ->mti_oldest_reader.weak);
+
+ if (!(mode & MDBX_END_EOTDONE)) /* !(already closed cursors) */
+ mdbx_cursors_eot(txn, false);
+
+ int rc = MDBX_SUCCESS;
+ if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)) {
+ if (txn->to.reader) {
+ MDBX_reader *slot = txn->to.reader;
+ mdbx_assert(env, slot->mr_pid.weak == env->me_pid);
+ if (likely(!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED))) {
+ mdbx_assert(env, txn->mt_txnid == slot->mr_txnid.weak &&
+ slot->mr_txnid.weak >=
+ env->me_lck->mti_oldest_reader.weak);
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ mdbx_txn_valgrind(env, nullptr);
+#endif
+ atomic_store32(&slot->mr_snapshot_pages_used, 0, mo_Relaxed);
+ safe64_reset(&slot->mr_txnid, false);
atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
mo_Relaxed);
+ } else {
+ mdbx_assert(env, slot->mr_pid.weak == env->me_pid);
+ mdbx_assert(env, slot->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD);
+ }
+ if (mode & MDBX_END_SLOT) {
+ if ((env->me_flags & MDBX_ENV_TXKEY) == 0)
+ atomic_store32(&slot->mr_pid, 0, mo_Relaxed);
+ txn->to.reader = NULL;
}
}
+#if defined(_WIN32) || defined(_WIN64)
+ if (txn->mt_flags & MDBX_SHRINK_ALLOWED)
+ mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
+#endif
+ txn->mt_numdbs = 0; /* prevent further DBI activity */
+ txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED;
+ txn->mt_owner = 0;
+ } else if (!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED)) {
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ if (txn == env->me_txn0)
+ mdbx_txn_valgrind(env, nullptr);
+#endif
- if (unlikely(txn->mt_txnid < MIN_TXNID || txn->mt_txnid > MAX_TXNID)) {
- mdbx_error("%s", "environment corrupted by died writer, must shutdown!");
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- mdbx_assert(env, txn->mt_txnid >= env->me_lck->mti_oldest_reader.weak);
- txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
- mdbx_ensure(env, txn->mt_txnid >=
- /* paranoia is appropriate here */ env->me_lck
- ->mti_oldest_reader.weak);
- txn->mt_numdbs = env->me_numdbs;
- } else {
- mdbx_assert(env, (flags & ~(MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS |
- MDBX_WRITEMAP)) == 0);
- if (unlikely(txn->mt_owner == tid ||
- /* not recovery mode */ env->me_stuck_meta >= 0))
- return MDBX_BUSY;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (lck && (env->me_flags & MDBX_NOTLS) == 0 &&
- (mdbx_runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) {
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) ==
- env->me_pid &&
- unlikely(atomic_load64(&lck->mti_readers[i].mr_tid, mo_Relaxed) ==
- tid)) {
- const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (txnid >= MIN_TXNID && txnid <= MAX_TXNID)
- return MDBX_TXN_OVERLAPPING;
+ txn->mt_flags = MDBX_TXN_FINISHED;
+ txn->mt_owner = 0;
+ env->me_txn = txn->mt_parent;
+ mdbx_pnl_free(txn->tw.spill_pages);
+ txn->tw.spill_pages = nullptr;
+ if (txn == env->me_txn0) {
+ mdbx_assert(env, txn->mt_parent == NULL);
+ /* Export or close DBI handles created in this txn */
+ dbi_update(txn, mode & MDBX_END_UPDATE);
+ mdbx_pnl_shrink(&txn->tw.retired_pages);
+ mdbx_pnl_shrink(&txn->tw.reclaimed_pglist);
+ if (!(env->me_flags & MDBX_WRITEMAP))
+ mdbx_dlist_free(txn);
+ /* The writer mutex was locked in mdbx_txn_begin. */
+ mdbx_txn_unlock(env);
+ } else {
+ mdbx_assert(env, txn->mt_parent != NULL);
+ MDBX_txn *const parent = txn->mt_parent;
+ mdbx_assert(env, parent->mt_signature == MDBX_MT_SIGNATURE);
+ mdbx_assert(env, parent->mt_child == txn &&
+ (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0);
+ mdbx_assert(
+ env, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+
+ if (txn->tw.lifo_reclaimed) {
+ mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) >=
+ (unsigned)(uintptr_t)parent->tw.lifo_reclaimed);
+ MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) =
+ (unsigned)(uintptr_t)parent->tw.lifo_reclaimed;
+ parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed;
+ }
+
+ if (txn->tw.retired_pages) {
+ mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.retired_pages) >=
+ (unsigned)(uintptr_t)parent->tw.retired_pages);
+ MDBX_PNL_SIZE(txn->tw.retired_pages) =
+ (unsigned)(uintptr_t)parent->tw.retired_pages;
+ parent->tw.retired_pages = txn->tw.retired_pages;
+ }
+
+ parent->mt_child = nullptr;
+ parent->mt_flags &= ~MDBX_TXN_HAS_CHILD;
+ parent->tw.dirtylru = txn->tw.dirtylru;
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
+ if (!(env->me_flags & MDBX_WRITEMAP))
+ mdbx_dlist_free(txn);
+ mdbx_dpl_free(txn);
+ mdbx_pnl_free(txn->tw.reclaimed_pglist);
+
+ if (parent->mt_geo.upper != txn->mt_geo.upper ||
+ parent->mt_geo.now != txn->mt_geo.now) {
+ /* undo resize performed by child txn */
+ rc = mdbx_mapresize_implicit(env, parent->mt_next_pgno,
+ parent->mt_geo.now, parent->mt_geo.upper);
+ if (rc == MDBX_EPERM) {
+ /* unable undo resize (it is regular for Windows),
+ * therefore promote size changes from child to the parent txn */
+ mdbx_warning("unable undo resize performed by child txn, promote to "
+ "the parent (%u->%u, %u->%u)",
+ txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper,
+ parent->mt_geo.upper);
+ parent->mt_geo.now = txn->mt_geo.now;
+ parent->mt_geo.upper = txn->mt_geo.upper;
+ parent->mt_flags |= MDBX_TXN_DIRTY;
+ rc = MDBX_SUCCESS;
+ } else if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_error("error %d while undo resize performed by child txn, fail "
+ "the parent",
+ rc);
+ parent->mt_flags |= MDBX_TXN_ERROR;
+ if (!env->me_dxb_mmap.address)
+ env->me_flags |= MDBX_FATAL_ERROR;
}
}
}
+ }
- /* Not yet touching txn == env->me_txn0, it may be active */
- mdbx_jitter4testing(false);
- rc = mdbx_txn_lock(env, F_ISSET(flags, MDBX_TXN_TRY));
- if (unlikely(rc))
- return rc;
- if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
- mdbx_txn_unlock(env);
- return MDBX_PANIC;
- }
-#if defined(_WIN32) || defined(_WIN64)
- if (unlikely(!env->me_map)) {
- mdbx_txn_unlock(env);
- return MDBX_EPERM;
- }
-#endif /* Windows */
+ mdbx_assert(env, txn == env->me_txn0 || txn->mt_owner == 0);
+ if ((mode & MDBX_END_FREE) != 0 && txn != env->me_txn0) {
+ txn->mt_signature = 0;
+ mdbx_free(txn);
+ }
- mdbx_jitter4testing(false);
- const MDBX_meta *meta = constmeta_prefer_last(env);
- uint64_t timestamp = 0;
- while (
- "workaround for todo4recovery://erased_by_github/libmdbx/issues/269") {
- rc = meta_waittxnid(env, (const MDBX_meta *)meta, ×tamp);
- if (likely(rc == MDBX_SUCCESS))
- break;
- if (unlikely(rc != MDBX_RESULT_TRUE))
- goto bailout;
- }
- mdbx_jitter4testing(false);
- txn->mt_canary = meta->mm_canary;
- const txnid_t snap = constmeta_txnid(env, meta);
- txn->mt_txnid = safe64_txnid_next(snap);
- if (unlikely(txn->mt_txnid > MAX_TXNID)) {
- rc = MDBX_TXN_FULL;
- mdbx_error("txnid overflow, raise %d", rc);
- goto bailout;
- }
+ return rc;
+}
- txn->mt_flags = flags;
- txn->mt_child = NULL;
- txn->tw.loose_pages = NULL;
- txn->tw.loose_count = 0;
-#if MDBX_ENABLE_REFUND
- txn->tw.loose_refund_wl = 0;
-#endif /* MDBX_ENABLE_REFUND */
- MDBX_PNL_SIZE(txn->tw.retired_pages) = 0;
- txn->tw.spill_pages = NULL;
- txn->tw.spill_least_removed = 0;
- txn->tw.last_reclaimed = 0;
- if (txn->tw.lifo_reclaimed)
- MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) = 0;
- env->me_txn = txn;
- txn->mt_numdbs = env->me_numdbs;
- memcpy(txn->mt_dbiseqs, env->me_dbiseqs, txn->mt_numdbs * sizeof(unsigned));
- /* Copy the DB info and flags */
- memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db));
- /* Moved to here to avoid a data race in read TXNs */
- txn->mt_geo = meta->mm_geo;
+int mdbx_txn_reset(MDBX_txn *txn) {
+ int rc = check_txn(txn, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- rc = mdbx_dpl_alloc(txn);
+ /* This call is only valid for read-only txns */
+ if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0))
+ return MDBX_EINVAL;
+
+ /* LY: don't close DBI-handles */
+ rc = mdbx_txn_end(txn, MDBX_END_RESET | MDBX_END_UPDATE);
+ if (rc == MDBX_SUCCESS) {
+ mdbx_tassert(txn, txn->mt_signature == MDBX_MT_SIGNATURE);
+ mdbx_tassert(txn, txn->mt_owner == 0);
+ }
+ return rc;
+}
+
+int mdbx_txn_break(MDBX_txn *txn) {
+ do {
+ int rc = check_txn(txn, 0);
if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- txn->tw.dirtyroom = txn->mt_env->me_options.dp_limit;
- txn->tw.dirtylru = MDBX_DEBUG ? ~42u : 0;
+ return rc;
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ if (txn->mt_flags & MDBX_TXN_RDONLY)
+ break;
+ txn = txn->mt_child;
+ } while (txn);
+ return MDBX_SUCCESS;
+}
+
+int mdbx_txn_abort(MDBX_txn *txn) {
+ int rc = check_txn(txn, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY))
+ /* LY: don't close DBI-handles */
+ return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_UPDATE | MDBX_END_SLOT |
+ MDBX_END_FREE);
+
+ if (txn->mt_child)
+ mdbx_txn_abort(txn->mt_child);
+
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_SLOT | MDBX_END_FREE);
+}
+
+/* Count all the pages in each DB and in the GC and make sure
+ * it matches the actual number of pages being used. */
+__cold static int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored,
+ bool dont_filter_gc) {
+ pgno_t pending = 0;
+ if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
+ pending = txn->tw.loose_count + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) +
+ (MDBX_PNL_SIZE(txn->tw.retired_pages) - retired_stored);
}
- /* Setup db info */
- mdbx_compiler_barrier();
- memset(txn->mt_cursors, 0, sizeof(MDBX_cursor *) * txn->mt_numdbs);
- for (unsigned i = CORE_DBS; i < txn->mt_numdbs; i++) {
- const unsigned db_flags = env->me_dbflags[i];
- txn->mt_dbs[i].md_flags = db_flags & DB_PERSISTENT_FLAGS;
- txn->mt_dbistate[i] =
- (db_flags & DB_VALID) ? DBI_VALID | DBI_USRVALID | DBI_STALE : 0;
+ MDBX_cursor_couple cx;
+ int rc = mdbx_cursor_init(&cx.outer, txn, FREE_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ pgno_t gc = 0;
+ MDBX_val key, data;
+ while ((rc = mdbx_cursor_get(&cx.outer, &key, &data, MDBX_NEXT)) == 0) {
+ if (!dont_filter_gc) {
+ if (unlikely(key.iov_len != sizeof(txnid_t)))
+ return MDBX_CORRUPTED;
+ txnid_t id = unaligned_peek_u64(4, key.iov_base);
+ if (txn->tw.lifo_reclaimed) {
+ for (unsigned i = 1; i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); ++i)
+ if (id == txn->tw.lifo_reclaimed[i])
+ goto skip;
+ } else if (id <= txn->tw.last_reclaimed)
+ goto skip;
+ }
+
+ gc += *(pgno_t *)data.iov_base;
+ skip:;
}
- txn->mt_dbistate[MAIN_DBI] = DBI_VALID | DBI_USRVALID;
- txn->mt_dbistate[FREE_DBI] = DBI_VALID;
- txn->mt_front =
- txn->mt_txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0);
+ mdbx_tassert(txn, rc == MDBX_NOTFOUND);
- if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) {
- mdbx_warning("%s", "environment had fatal error, must shutdown!");
- rc = MDBX_PANIC;
- } else {
- const size_t size =
- pgno2bytes(env, (txn->mt_flags & MDBX_TXN_RDONLY) ? txn->mt_next_pgno
- : txn->mt_end_pgno);
- if (unlikely(size > env->me_dxb_mmap.limit)) {
- if (txn->mt_geo.upper > MAX_PAGENO + 1 ||
- bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) !=
- txn->mt_geo.upper) {
- rc = MDBX_UNABLE_EXTEND_MAPSIZE;
- goto bailout;
+ for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++)
+ txn->mt_dbistate[i] &= ~DBI_AUDITED;
+
+ pgno_t used = NUM_METAS;
+ for (MDBX_dbi i = FREE_DBI; i <= MAIN_DBI; i++) {
+ if (!(txn->mt_dbistate[i] & DBI_VALID))
+ continue;
+ rc = mdbx_cursor_init(&cx.outer, txn, i);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ txn->mt_dbistate[i] |= DBI_AUDITED;
+ if (txn->mt_dbs[i].md_root == P_INVALID)
+ continue;
+ used += txn->mt_dbs[i].md_branch_pages + txn->mt_dbs[i].md_leaf_pages +
+ txn->mt_dbs[i].md_overflow_pages;
+
+ if (i != MAIN_DBI)
+ continue;
+ rc = mdbx_page_search(&cx.outer, NULL, MDBX_PS_FIRST);
+ while (rc == MDBX_SUCCESS) {
+ MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top];
+ for (unsigned j = 0; j < page_numkeys(mp); j++) {
+ MDBX_node *node = page_node(mp, j);
+ if (node_flags(node) == F_SUBDATA) {
+ if (unlikely(node_ds(node) != sizeof(MDBX_db)))
+ return MDBX_CORRUPTED;
+ MDBX_db db_copy, *db;
+ memcpy(db = &db_copy, node_data(node), sizeof(db_copy));
+ if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
+ for (MDBX_dbi k = txn->mt_numdbs; --k > MAIN_DBI;) {
+ if ((txn->mt_dbistate[k] & DBI_VALID) &&
+ /* txn->mt_dbxs[k].md_name.iov_len > 0 && */
+ node_ks(node) == txn->mt_dbxs[k].md_name.iov_len &&
+ memcmp(node_key(node), txn->mt_dbxs[k].md_name.iov_base,
+ node_ks(node)) == 0) {
+ txn->mt_dbistate[k] |= DBI_AUDITED;
+ if (!(txn->mt_dbistate[k] & MDBX_DBI_STALE))
+ db = txn->mt_dbs + k;
+ break;
+ }
+ }
+ }
+ used +=
+ db->md_branch_pages + db->md_leaf_pages + db->md_overflow_pages;
+ }
}
- rc = mdbx_mapresize(env, txn->mt_next_pgno, txn->mt_end_pgno,
- txn->mt_geo.upper,
- (txn->mt_flags & MDBX_TXN_RDONLY) ? true : false);
- if (rc != MDBX_SUCCESS)
- goto bailout;
+ rc = mdbx_cursor_sibling(&cx.outer, SIBLING_RIGHT);
}
- if (txn->mt_flags & MDBX_TXN_RDONLY) {
-#if defined(_WIN32) || defined(_WIN64)
- if (((size > env->me_dbgeo.lower && env->me_dbgeo.shrink) ||
- (mdbx_RunningUnderWine() &&
- /* under Wine acquisition of remap_guard is always required,
- * since Wine don't support section extending,
- * i.e. in both cases unmap+map are required. */
- size < env->me_dbgeo.upper && env->me_dbgeo.grow)) &&
- /* avoid recursive use SRW */ (txn->mt_flags & MDBX_NOTLS) == 0) {
- txn->mt_flags |= MDBX_SHRINK_ALLOWED;
- mdbx_srwlock_AcquireShared(&env->me_remap_guard);
- }
-#endif /* Windows */
- } else {
- env->me_dxb_mmap.current = size;
- env->me_dxb_mmap.filesize =
- (env->me_dxb_mmap.filesize < size) ? size : env->me_dxb_mmap.filesize;
+ mdbx_tassert(txn, rc == MDBX_NOTFOUND);
+ }
+
+ for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++) {
+ if ((txn->mt_dbistate[i] & (DBI_VALID | DBI_AUDITED | DBI_STALE)) !=
+ DBI_VALID)
+ continue;
+ for (MDBX_txn *t = txn; t; t = t->mt_parent)
+ if (F_ISSET(t->mt_dbistate[i], DBI_DIRTY | DBI_CREAT)) {
+ used += t->mt_dbs[i].md_branch_pages + t->mt_dbs[i].md_leaf_pages +
+ t->mt_dbs[i].md_overflow_pages;
+ txn->mt_dbistate[i] |= DBI_AUDITED;
+ break;
+ }
+ if (!(txn->mt_dbistate[i] & DBI_AUDITED)) {
+ mdbx_warning("audit %s@%" PRIaTXN
+ ": unable account dbi %d / \"%*s\", state 0x%02x",
+ txn->mt_parent ? "nested-" : "", txn->mt_txnid, i,
+ (int)txn->mt_dbxs[i].md_name.iov_len,
+ (const char *)txn->mt_dbxs[i].md_name.iov_base,
+ txn->mt_dbistate[i]);
}
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- mdbx_txn_valgrind(env, txn);
-#endif
- txn->mt_owner = tid;
- return MDBX_SUCCESS;
}
-bailout:
- mdbx_tassert(txn, rc != MDBX_SUCCESS);
- mdbx_txn_end(txn, MDBX_END_SLOT | MDBX_END_FAIL_BEGIN);
- return rc;
-}
-
-static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) {
- if (unlikely(!txn))
- return MDBX_EINVAL;
-
- if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
- return MDBX_EBADSIGN;
-
- if (unlikely(txn->mt_flags & bad_bits))
- return MDBX_BAD_TXN;
-
-#if MDBX_TXN_CHECKOWNER
- if ((txn->mt_flags & MDBX_NOTLS) == 0 &&
- unlikely(txn->mt_owner != mdbx_thread_self()))
- return txn->mt_owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN;
-#endif /* MDBX_TXN_CHECKOWNER */
- if (unlikely(!txn->mt_env->me_map))
- return MDBX_EPERM;
+ if (pending + gc + used == txn->mt_next_pgno)
+ return MDBX_SUCCESS;
- return MDBX_SUCCESS;
+ if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0)
+ mdbx_error("audit @%" PRIaTXN ": %u(pending) = %u(loose) + "
+ "%u(reclaimed) + %u(retired-pending) - %u(retired-stored)",
+ txn->mt_txnid, pending, txn->tw.loose_count,
+ MDBX_PNL_SIZE(txn->tw.reclaimed_pglist),
+ txn->tw.retired_pages ? MDBX_PNL_SIZE(txn->tw.retired_pages) : 0,
+ retired_stored);
+ mdbx_error("audit @%" PRIaTXN ": %" PRIaPGNO "(pending) + %" PRIaPGNO
+ "(gc) + %" PRIaPGNO "(count) = %" PRIaPGNO "(total) <> %" PRIaPGNO
+ "(allocated)",
+ txn->mt_txnid, pending, gc, used, pending + gc + used,
+ txn->mt_next_pgno);
+ return MDBX_PROBLEM;
}
-static __always_inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) {
- int err = check_txn(txn, bad_bits);
- if (unlikely(err))
- return err;
-
- if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
- return MDBX_EACCESS;
-
- return MDBX_SUCCESS;
+static __always_inline unsigned backlog_size(MDBX_txn *txn) {
+ return MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) + txn->tw.loose_count;
}
-int mdbx_txn_renew(MDBX_txn *txn) {
- if (unlikely(!txn))
- return MDBX_EINVAL;
+/* LY: Prepare a backlog of pages to modify GC itself,
+ * while reclaiming is prohibited. It should be enough to prevent search
+ * in mdbx_page_alloc() during a deleting, when GC tree is unbalanced. */
+static int mdbx_prep_backlog(MDBX_txn *txn, MDBX_cursor *gc_cursor,
+ const size_t pnl_bytes, unsigned *retired_stored) {
+ const unsigned linear4list = number_of_ovpages(txn->mt_env, pnl_bytes);
+ const unsigned backlog4cow = txn->mt_dbs[FREE_DBI].md_depth;
+ const unsigned backlog4rebalance = backlog4cow + 1;
- if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
- return MDBX_EBADSIGN;
+ if (likely(linear4list == 1 &&
+ backlog_size(txn) > (pnl_bytes
+ ? backlog4rebalance
+ : (backlog4cow + backlog4rebalance))))
+ return MDBX_SUCCESS;
- if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0))
- return MDBX_EINVAL;
+ mdbx_trace(">> pnl_bytes %zu, backlog %u, 4list %u, 4cow %u, 4rebalance %u",
+ pnl_bytes, backlog_size(txn), linear4list, backlog4cow,
+ backlog4rebalance);
- int rc;
- if (unlikely(txn->mt_owner != 0 || !(txn->mt_flags & MDBX_TXN_FINISHED))) {
- rc = mdbx_txn_reset(txn);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ MDBX_val gc_key, fake_val;
+ int err;
+ if (unlikely(linear4list > 2)) {
+ gc_key.iov_base = fake_val.iov_base = nullptr;
+ gc_key.iov_len = sizeof(txnid_t);
+ fake_val.iov_len = pnl_bytes;
+ err = mdbx_cursor_spill(gc_cursor, &gc_key, &fake_val);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
}
- rc = mdbx_txn_renew0(txn, MDBX_TXN_RDONLY);
- if (rc == MDBX_SUCCESS) {
- txn->mt_owner = mdbx_thread_self();
- mdbx_debug("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO
- "/%" PRIaPGNO,
- txn->mt_txnid, (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w',
- (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root,
- txn->mt_dbs[FREE_DBI].md_root);
- }
- return rc;
-}
+ gc_cursor->mc_flags &= ~C_RECLAIMING;
+ err = mdbx_cursor_touch(gc_cursor);
+ mdbx_trace("== after-touch, backlog %u, err %d", backlog_size(txn), err);
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags,
- MDBX_txn **ret) {
- return __inline_mdbx_txn_begin(env, parent, flags, ret);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+ if (unlikely(linear4list > 1) && err == MDBX_SUCCESS) {
+ if (retired_stored) {
+ gc_key.iov_base = &txn->mt_txnid;
+ gc_key.iov_len = sizeof(txn->mt_txnid);
+ const struct cursor_set_result csr =
+ mdbx_cursor_set(gc_cursor, &gc_key, &fake_val, MDBX_SET);
+ if (csr.err == MDBX_SUCCESS && csr.exact) {
+ *retired_stored = 0;
+ err = mdbx_cursor_del(gc_cursor, 0);
+ mdbx_trace("== clear-4linear, backlog %u, err %d", backlog_size(txn),
+ err);
+ }
+ }
+ err =
+ mdbx_page_alloc(gc_cursor, linear4list, MDBX_ALLOC_GC | MDBX_ALLOC_FAKE)
+ .err;
+ mdbx_trace("== after-4linear, backlog %u, err %d", backlog_size(txn), err);
+ mdbx_cassert(gc_cursor,
+ backlog_size(txn) >= linear4list || err != MDBX_SUCCESS);
+ }
-int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) {
- int rc = check_txn(txn, MDBX_TXN_FINISHED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ while (backlog_size(txn) < backlog4cow + linear4list && err == MDBX_SUCCESS)
+ err = mdbx_page_alloc(gc_cursor, 0,
+ MDBX_ALLOC_GC | MDBX_ALLOC_SLOT | MDBX_ALLOC_FAKE)
+ .err;
- txn->mt_userctx = ctx;
- return MDBX_SUCCESS;
+ gc_cursor->mc_flags |= C_RECLAIMING;
+ mdbx_trace("<< backlog %u, err %d", backlog_size(txn), err);
+ return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS;
}
-void *mdbx_txn_get_userctx(const MDBX_txn *txn) {
- return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->mt_userctx;
+static __inline void clean_reserved_gc_pnl(MDBX_env *env, MDBX_val pnl) {
+ /* PNL is initially empty, zero out at least the length */
+ memset(pnl.iov_base, 0, sizeof(pgno_t));
+ if ((env->me_flags & (MDBX_WRITEMAP | MDBX_NOMEMINIT)) == 0)
+ /* zero out to avoid leaking values from uninitialized malloc'ed memory
+ * to the file in non-writemap mode if length of the saving page-list
+ * was changed during space reservation. */
+ memset(pnl.iov_base, 0, pnl.iov_len);
}
-int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags,
- MDBX_txn **ret, void *context) {
- MDBX_txn *txn;
- unsigned size, tsize;
-
- if (unlikely(!ret))
- return MDBX_EINVAL;
- *ret = NULL;
-
- if (unlikely((flags & ~MDBX_TXN_RW_BEGIN_FLAGS) &&
- (flags & ~MDBX_TXN_RO_BEGIN_FLAGS)))
- return MDBX_EINVAL;
+/* Cleanups reclaimed GC (aka freeDB) records, saves the retired-list (aka
+ * freelist) of current transaction to GC, puts back into GC leftover of the
+ * reclaimed pages with chunking. This recursive changes the reclaimed-list,
+ * loose-list and retired-list. Keep trying until it stabilizes.
+ *
+ * NOTE: This code is a consequence of many iterations of adding crutches (aka
+ * "checks and balances") to partially bypass the fundamental design problems
+ * inherited from LMDB. So do not try to understand it completely in order to
+ * avoid your madness. */
+static int mdbx_update_gc(MDBX_txn *txn) {
+ /* txn->tw.reclaimed_pglist[] can grow and shrink during this call.
+ * txn->tw.last_reclaimed and txn->tw.retired_pages[] can only grow.
+ * Page numbers cannot disappear from txn->tw.retired_pages[]. */
+ MDBX_env *const env = txn->mt_env;
+ const bool lifo = (env->me_flags & MDBX_LIFORECLAIM) != 0;
+ const char *dbg_prefix_mode = lifo ? " lifo" : " fifo";
+ (void)dbg_prefix_mode;
+ mdbx_trace("\n>>> @%" PRIaTXN, txn->mt_txnid);
- int rc = check_env(env, true);
+ unsigned retired_stored = 0, loop = 0;
+ MDBX_cursor_couple couple;
+ int rc = mdbx_cursor_init(&couple.outer, txn, FREE_DBI);
if (unlikely(rc != MDBX_SUCCESS))
- return rc;
-
- if (unlikely(env->me_flags & MDBX_RDONLY &
- ~flags)) /* write txn in RDONLY env */
- return MDBX_EACCESS;
-
- flags |= env->me_flags & MDBX_WRITEMAP;
-
- if (parent) {
- /* Nested transactions: Max 1 child, write txns only, no writemap */
- rc = check_txn_rw(parent,
- MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ goto bailout_notracking;
- if (env->me_options.spill_parent4child_denominator) {
- /* Spill dirty-pages of parent to provide dirtyroom for child txn */
- rc = mdbx_txn_spill(parent, nullptr,
- parent->tw.dirtylist->length /
- env->me_options.spill_parent4child_denominator);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
+ couple.outer.mc_flags |= C_RECLAIMING;
+ couple.outer.mc_next = txn->mt_cursors[FREE_DBI];
+ txn->mt_cursors[FREE_DBI] = &couple.outer;
+ bool dense_gc = false;
- flags |= parent->mt_flags & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS);
- } else if (flags & MDBX_TXN_RDONLY) {
- if (env->me_txn0 &&
- unlikely(env->me_txn0->mt_owner == mdbx_thread_self()) &&
- (mdbx_runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0)
- return MDBX_TXN_OVERLAPPING;
- } else {
- /* Reuse preallocated write txn. However, do not touch it until
- * mdbx_txn_renew0() succeeds, since it currently may be active. */
- txn = env->me_txn0;
- goto renew;
+retry:
+ ++loop;
+ mdbx_trace("%s", " >> restart");
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ if (unlikely(/* paranoia */ loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
+ mdbx_error("too more loops %u, bailout", loop);
+ rc = MDBX_PROBLEM;
+ goto bailout;
}
- size = env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + 1);
- size += tsize = sizeof(MDBX_txn);
- if (unlikely((txn = mdbx_malloc(size)) == NULL)) {
- mdbx_debug("calloc: %s", "failed");
- return MDBX_ENOMEM;
+ if (unlikely(dense_gc) && retired_stored) {
+ rc = mdbx_prep_backlog(txn, &couple.outer,
+ MDBX_PNL_SIZEOF(txn->tw.retired_pages),
+ &retired_stored);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
}
-#if MDBX_DEBUG
- memset(txn, 0xCD, size);
- VALGRIND_MAKE_MEM_UNDEFINED(txn, size);
-#endif /* MDBX_DEBUG */
- memset(txn, 0, tsize);
- txn->mt_dbxs = env->me_dbxs; /* static */
- txn->mt_dbs = (MDBX_db *)((char *)txn + tsize);
- txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs);
- txn->mt_dbistate = (uint8_t *)txn + size - env->me_maxdbs;
- txn->mt_flags = flags;
- txn->mt_env = env;
- if (parent) {
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
- txn->mt_dbiseqs = parent->mt_dbiseqs;
- txn->mt_geo = parent->mt_geo;
- rc = mdbx_dpl_alloc(txn);
- if (likely(rc == MDBX_SUCCESS)) {
- const unsigned len =
- MDBX_PNL_SIZE(parent->tw.reclaimed_pglist) + parent->tw.loose_count;
- txn->tw.reclaimed_pglist =
- mdbx_pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL);
- if (unlikely(!txn->tw.reclaimed_pglist))
- rc = MDBX_ENOMEM;
- }
- if (unlikely(rc != MDBX_SUCCESS)) {
- nested_failed:
- mdbx_pnl_free(txn->tw.reclaimed_pglist);
- mdbx_dpl_free(txn);
- mdbx_free(txn);
- return rc;
+ unsigned settled = 0, cleaned_gc_slot = 0, reused_gc_slot = 0,
+ filled_gc_slot = ~0u;
+ txnid_t cleaned_gc_id = 0, gc_rid = txn->tw.last_reclaimed;
+ while (true) {
+ /* Come back here after each Put() in case retired-list changed */
+ MDBX_val key, data;
+ mdbx_trace("%s", " >> continue");
+
+ if (retired_stored != MDBX_PNL_SIZE(txn->tw.retired_pages) &&
+ MDBX_PNL_SIZE(txn->tw.retired_pages) > env->me_maxgc_ov1page) {
+ rc = mdbx_prep_backlog(txn, &couple.outer,
+ MDBX_PNL_SIZEOF(txn->tw.retired_pages),
+ &retired_stored);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
}
- /* Move loose pages to reclaimed list */
- if (parent->tw.loose_count) {
- do {
- MDBX_page *lp = parent->tw.loose_pages;
- const unsigned di = mdbx_dpl_exist(parent, lp->mp_pgno);
- mdbx_tassert(parent, di && parent->tw.dirtylist->items[di].ptr == lp);
- mdbx_tassert(parent, lp->mp_flags == P_LOOSE);
- rc =
- mdbx_pnl_insert_range(&parent->tw.reclaimed_pglist, lp->mp_pgno, 1);
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ if (lifo) {
+ if (cleaned_gc_slot < (txn->tw.lifo_reclaimed
+ ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
+ : 0)) {
+ settled = 0;
+ cleaned_gc_slot = 0;
+ reused_gc_slot = 0;
+ filled_gc_slot = ~0u;
+ /* LY: cleanup reclaimed records. */
+ do {
+ cleaned_gc_id = txn->tw.lifo_reclaimed[++cleaned_gc_slot];
+ mdbx_tassert(txn,
+ cleaned_gc_slot > 0 &&
+ cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
+ key.iov_base = &cleaned_gc_id;
+ key.iov_len = sizeof(cleaned_gc_id);
+ rc = mdbx_cursor_get(&couple.outer, &key, NULL, MDBX_SET);
+ if (rc == MDBX_NOTFOUND)
+ continue;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ if (likely(!dense_gc)) {
+ rc = mdbx_prep_backlog(txn, &couple.outer, 0, nullptr);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ mdbx_tassert(txn,
+ cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
+ mdbx_trace("%s: cleanup-reclaimed-id [%u]%" PRIaTXN, dbg_prefix_mode,
+ cleaned_gc_slot, cleaned_gc_id);
+ mdbx_tassert(txn, *txn->mt_cursors == &couple.outer);
+ rc = mdbx_cursor_del(&couple.outer, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ } while (cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
+ mdbx_txl_sort(txn->tw.lifo_reclaimed);
+ }
+ } else {
+ /* If using records from GC which we have not yet deleted,
+ * now delete them and any we reserved for tw.reclaimed_pglist. */
+ while (cleaned_gc_id <= txn->tw.last_reclaimed) {
+ rc = mdbx_cursor_first(&couple.outer, &key, NULL);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_NOTFOUND)
+ break;
+ goto bailout;
+ }
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(key.iov_len != sizeof(txnid_t))) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
+ }
+ gc_rid = cleaned_gc_id;
+ settled = 0;
+ reused_gc_slot = 0;
+ cleaned_gc_id = unaligned_peek_u64(4, key.iov_base);
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(cleaned_gc_id < MIN_TXNID || cleaned_gc_id > MAX_TXNID)) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
+ }
+ if (cleaned_gc_id > txn->tw.last_reclaimed)
+ break;
+ if (likely(!dense_gc)) {
+ rc = mdbx_prep_backlog(txn, &couple.outer, 0, nullptr);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ mdbx_tassert(txn, cleaned_gc_id <= txn->tw.last_reclaimed);
+ mdbx_tassert(txn, cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
+ mdbx_trace("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix_mode,
+ cleaned_gc_id);
+ mdbx_tassert(txn, *txn->mt_cursors == &couple.outer);
+ rc = mdbx_cursor_del(&couple.outer, 0);
if (unlikely(rc != MDBX_SUCCESS))
- goto nested_failed;
- parent->tw.loose_pages = lp->mp_next;
- /* Remove from dirty list */
- mdbx_page_wash(parent, di, lp, 1);
- } while (parent->tw.loose_pages);
- parent->tw.loose_count = 0;
-#if MDBX_ENABLE_REFUND
- parent->tw.loose_refund_wl = 0;
-#endif /* MDBX_ENABLE_REFUND */
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ goto bailout;
+ }
}
- txn->tw.dirtyroom = parent->tw.dirtyroom;
- txn->tw.dirtylru = parent->tw.dirtylru;
-
- mdbx_dpl_sort(parent);
- if (parent->tw.spill_pages)
- mdbx_spill_purge(parent);
- mdbx_tassert(txn, MDBX_PNL_ALLOCLEN(txn->tw.reclaimed_pglist) >=
- MDBX_PNL_SIZE(parent->tw.reclaimed_pglist));
- memcpy(txn->tw.reclaimed_pglist, parent->tw.reclaimed_pglist,
- MDBX_PNL_SIZEOF(parent->tw.reclaimed_pglist));
- mdbx_assert(env, mdbx_pnl_check4assert(
- txn->tw.reclaimed_pglist,
- (txn->mt_next_pgno /* LY: intentional assignment here,
- only for assertion */
- = parent->mt_next_pgno) -
- MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_audit_ex(txn, retired_stored, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
- txn->tw.last_reclaimed = parent->tw.last_reclaimed;
- if (parent->tw.lifo_reclaimed) {
- txn->tw.lifo_reclaimed = parent->tw.lifo_reclaimed;
- parent->tw.lifo_reclaimed =
- (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.lifo_reclaimed);
+ /* return suitable into unallocated space */
+ if (mdbx_refund(txn)) {
+ mdbx_tassert(
+ txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_audit_ex(txn, retired_stored, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
}
- txn->tw.retired_pages = parent->tw.retired_pages;
- parent->tw.retired_pages =
- (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.retired_pages);
+ /* handle loose pages - put ones into the reclaimed- or retired-list */
+ if (txn->tw.loose_pages) {
+ /* Return loose page numbers to tw.reclaimed_pglist,
+ * though usually none are left at this point.
+ * The pages themselves remain in dirtylist. */
+ if (unlikely(!txn->tw.lifo_reclaimed && txn->tw.last_reclaimed < 1)) {
+ if (txn->tw.loose_count > 0) {
+ /* Put loose page numbers in tw.retired_pages,
+ * since unable to return them to tw.reclaimed_pglist. */
+ if (unlikely((rc = mdbx_pnl_need(&txn->tw.retired_pages,
+ txn->tw.loose_count)) != 0))
+ goto bailout;
+ for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next)
+ mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno);
+ mdbx_trace("%s: append %u loose-pages to retired-pages",
+ dbg_prefix_mode, txn->tw.loose_count);
+ }
+ } else {
+ /* Room for loose pages + temp PNL with same */
+ rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist,
+ 2 * txn->tw.loose_count + 2);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ MDBX_PNL loose = txn->tw.reclaimed_pglist +
+ MDBX_PNL_ALLOCLEN(txn->tw.reclaimed_pglist) -
+ txn->tw.loose_count - 1;
+ unsigned count = 0;
+ for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next) {
+ mdbx_tassert(txn, mp->mp_flags == P_LOOSE);
+ loose[++count] = mp->mp_pgno;
+ }
+ mdbx_tassert(txn, count == txn->tw.loose_count);
+ MDBX_PNL_SIZE(loose) = count;
+ mdbx_pnl_sort(loose, txn->mt_next_pgno);
+ mdbx_pnl_xmerge(txn->tw.reclaimed_pglist, loose);
+ mdbx_trace("%s: append %u loose-pages to reclaimed-pages",
+ dbg_prefix_mode, txn->tw.loose_count);
+ }
- txn->mt_txnid = parent->mt_txnid;
- txn->mt_front = parent->mt_front + 1;
+ /* filter-out list of dirty-pages from loose-pages */
+ MDBX_dpl *const dl = txn->tw.dirtylist;
+ unsigned w = 0;
+ for (unsigned r = w; ++r <= dl->length;) {
+ MDBX_page *dp = dl->items[r].ptr;
+ mdbx_tassert(txn, dp->mp_flags == P_LOOSE || IS_MODIFIABLE(txn, dp));
+ mdbx_tassert(txn, dpl_endpgno(dl, r) <= txn->mt_next_pgno);
+ if ((dp->mp_flags & P_LOOSE) == 0) {
+ if (++w != r)
+ dl->items[w] = dl->items[r];
+ } else {
+ mdbx_tassert(txn, dp->mp_flags == P_LOOSE);
+ if ((env->me_flags & MDBX_WRITEMAP) == 0)
+ mdbx_dpage_free(env, dp, 1);
+ }
+ }
+ mdbx_trace("%s: filtered-out loose-pages from %u -> %u dirty-pages",
+ dbg_prefix_mode, dl->length, w);
+ mdbx_tassert(txn, txn->tw.loose_count == dl->length - w);
+ dpl_setlen(dl, w);
+ dl->sorted = 0;
+ txn->tw.dirtyroom += txn->tw.loose_count;
+ mdbx_tassert(txn,
+ txn->tw.dirtyroom + txn->tw.dirtylist->length ==
+ (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
+ : txn->mt_env->me_options.dp_limit));
+ txn->tw.loose_pages = NULL;
+ txn->tw.loose_count = 0;
#if MDBX_ENABLE_REFUND
- txn->tw.loose_refund_wl = 0;
+ txn->tw.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
- txn->mt_canary = parent->mt_canary;
- parent->mt_flags |= MDBX_TXN_HAS_CHILD;
- parent->mt_child = txn;
- txn->mt_parent = parent;
- txn->mt_numdbs = parent->mt_numdbs;
- txn->mt_owner = parent->mt_owner;
- memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db));
- /* Copy parent's mt_dbistate, but clear DB_NEW */
- for (unsigned i = 0; i < txn->mt_numdbs; i++)
- txn->mt_dbistate[i] =
- parent->mt_dbistate[i] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
- mdbx_tassert(parent,
- parent->tw.dirtyroom + parent->tw.dirtylist->length ==
- (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
- : parent->mt_env->me_options.dp_limit));
- mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
- (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
- : txn->mt_env->me_options.dp_limit));
- env->me_txn = txn;
- rc = mdbx_cursor_shadow(parent, txn);
- if (mdbx_audit_enabled() && mdbx_assert_enabled()) {
- txn->mt_signature = MDBX_MT_SIGNATURE;
- mdbx_tassert(txn, mdbx_audit_ex(txn, 0, false) == 0);
}
- if (unlikely(rc != MDBX_SUCCESS))
- mdbx_txn_end(txn, MDBX_END_FAIL_BEGINCHILD);
- } else { /* MDBX_TXN_RDONLY */
- txn->mt_dbiseqs = env->me_dbiseqs;
- renew:
- rc = mdbx_txn_renew0(txn, flags);
- }
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (txn != env->me_txn0)
- mdbx_free(txn);
- } else {
- if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY))
- mdbx_assert(env, txn->mt_flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED));
- else if (flags & MDBX_TXN_RDONLY)
- mdbx_assert(env, (txn->mt_flags &
- ~(MDBX_NOTLS | MDBX_TXN_RDONLY | MDBX_WRITEMAP |
- /* Win32: SRWL flag */ MDBX_SHRINK_ALLOWED)) == 0);
- else {
- mdbx_assert(env, (txn->mt_flags & ~(MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED |
- MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC |
- MDBX_TXN_SPILLS)) == 0);
- assert(!txn->tw.spill_pages && !txn->tw.spill_least_removed);
- }
- txn->mt_signature = MDBX_MT_SIGNATURE;
- txn->mt_userctx = context;
- *ret = txn;
- mdbx_debug("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO
- "/%" PRIaPGNO,
- txn->mt_txnid, (flags & MDBX_TXN_RDONLY) ? 'r' : 'w',
- (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root,
- txn->mt_dbs[FREE_DBI].md_root);
- }
+ const unsigned amount = (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist);
+ /* handle retired-list - store ones into single gc-record */
+ if (retired_stored < MDBX_PNL_SIZE(txn->tw.retired_pages)) {
+ if (unlikely(!retired_stored)) {
+ /* Make sure last page of GC is touched and on retired-list */
+ couple.outer.mc_flags &= ~C_RECLAIMING;
+ rc = mdbx_page_search(&couple.outer, NULL,
+ MDBX_PS_LAST | MDBX_PS_MODIFY);
+ couple.outer.mc_flags |= C_RECLAIMING;
+ if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_NOTFOUND)
+ goto bailout;
+ }
+ /* Write to last page of GC */
+ key.iov_len = sizeof(txn->mt_txnid);
+ key.iov_base = &txn->mt_txnid;
+ do {
+ data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages);
+ mdbx_prep_backlog(txn, &couple.outer, data.iov_len, &retired_stored);
+ rc = mdbx_cursor_put(&couple.outer, &key, &data, MDBX_RESERVE);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ /* Retry if tw.retired_pages[] grew during the Put() */
+ } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages));
- return rc;
-}
+ retired_stored = (unsigned)MDBX_PNL_SIZE(txn->tw.retired_pages);
+ mdbx_pnl_sort(txn->tw.retired_pages, txn->mt_next_pgno);
+ mdbx_assert(env, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages));
+ memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len);
-int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
- int rc = check_txn(txn, MDBX_TXN_FINISHED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ mdbx_trace("%s: put-retired #%u @ %" PRIaTXN, dbg_prefix_mode,
+ retired_stored, txn->mt_txnid);
- if (unlikely(!info))
- return MDBX_EINVAL;
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) {
+ unsigned i = retired_stored;
+ mdbx_debug_extra("PNL write txn %" PRIaTXN " root %" PRIaPGNO
+ " num %u, PNL",
+ txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i);
+ for (; i; i--)
+ mdbx_debug_extra_print(" %" PRIaPGNO, txn->tw.retired_pages[i]);
+ mdbx_debug_extra_print("%s\n", ".");
+ }
+ if (unlikely(amount != MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) &&
+ settled)) {
+ mdbx_trace("%s: reclaimed-list changed %u -> %u, retry",
+ dbg_prefix_mode, amount,
+ (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
+ goto retry /* rare case, but avoids GC fragmentation
+ and one cycle. */
+ ;
+ }
+ continue;
+ }
- MDBX_env *const env = txn->mt_env;
-#if MDBX_ENV_CHECKPID
- if (unlikely(env->me_pid != mdbx_getpid())) {
- env->me_flags |= MDBX_FATAL_ERROR;
- return MDBX_PANIC;
- }
-#endif /* MDBX_ENV_CHECKPID */
+ /* handle reclaimed and lost pages - merge and store both into gc */
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn, txn->tw.loose_count == 0);
- info->txn_id = txn->mt_txnid;
- info->txn_space_used = pgno2bytes(env, txn->mt_geo.next);
+ mdbx_trace("%s", " >> reserving");
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_audit_ex(txn, retired_stored, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ const unsigned left = amount - settled;
+ mdbx_trace("%s: amount %u, settled %d, left %d, lifo-reclaimed-slots %u, "
+ "reused-gc-slots %u",
+ dbg_prefix_mode, amount, settled, (int)left,
+ txn->tw.lifo_reclaimed
+ ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
+ : 0,
+ reused_gc_slot);
+ if (0 >= (int)left)
+ break;
- if (txn->mt_flags & MDBX_TXN_RDONLY) {
- volatile const MDBX_meta *head_meta;
- txnid_t head_txnid;
- uint64_t head_retired;
- do {
- /* fetch info from volatile head */
- head_meta = meta_prefer_last(env);
- head_txnid = meta_txnid(env, head_meta);
- head_retired =
- unaligned_peek_u64_volatile(4, head_meta->mm_pages_retired);
- info->txn_space_limit_soft = pgno2bytes(env, head_meta->mm_geo.now);
- info->txn_space_limit_hard = pgno2bytes(env, head_meta->mm_geo.upper);
- info->txn_space_leftover =
- pgno2bytes(env, head_meta->mm_geo.now - head_meta->mm_geo.next);
- mdbx_compiler_barrier();
- } while (unlikely(head_meta != meta_prefer_last(env) ||
- head_txnid != meta_txnid(env, head_meta)));
+ const unsigned prefer_max_scatter = 257;
+ txnid_t reservation_gc_id;
+ if (lifo) {
+ if (txn->tw.lifo_reclaimed == nullptr) {
+ txn->tw.lifo_reclaimed = mdbx_txl_alloc();
+ if (unlikely(!txn->tw.lifo_reclaimed)) {
+ rc = MDBX_ENOMEM;
+ goto bailout;
+ }
+ }
+ if ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <
+ prefer_max_scatter &&
+ left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
+ reused_gc_slot) *
+ env->me_maxgc_ov1page &&
+ !dense_gc) {
+ /* LY: need just a txn-id for save page list. */
+ bool need_cleanup = false;
+ txnid_t snap_oldest;
+ retry_rid:
+ couple.outer.mc_flags &= ~C_RECLAIMING;
+ do {
+ snap_oldest = mdbx_find_oldest(txn);
+ rc =
+ mdbx_page_alloc(&couple.outer, 0,
+ MDBX_ALLOC_GC | MDBX_ALLOC_SLOT | MDBX_ALLOC_FAKE)
+ .err;
+ if (likely(rc == MDBX_SUCCESS)) {
+ mdbx_trace("%s: took @%" PRIaTXN " from GC", dbg_prefix_mode,
+ MDBX_PNL_LAST(txn->tw.lifo_reclaimed));
+ need_cleanup = true;
+ }
+ } while (rc == MDBX_SUCCESS &&
+ (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <
+ prefer_max_scatter &&
+ left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
+ reused_gc_slot) *
+ env->me_maxgc_ov1page);
+ couple.outer.mc_flags |= C_RECLAIMING;
- info->txn_reader_lag = head_txnid - info->txn_id;
- info->txn_space_dirty = info->txn_space_retired = 0;
- uint64_t reader_snapshot_pages_retired;
- if (txn->to.reader &&
- head_retired >
- (reader_snapshot_pages_retired = atomic_load64(
- &txn->to.reader->mr_snapshot_pages_retired, mo_Relaxed))) {
- info->txn_space_dirty = info->txn_space_retired = pgno2bytes(
- env, (pgno_t)(head_retired - reader_snapshot_pages_retired));
+ if (likely(rc == MDBX_SUCCESS)) {
+ mdbx_trace("%s: got enough from GC.", dbg_prefix_mode);
+ continue;
+ } else if (unlikely(rc != MDBX_NOTFOUND))
+ /* LY: some troubles... */
+ goto bailout;
- size_t retired_next_reader = 0;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (scan_rlt && info->txn_reader_lag > 1 && lck) {
- /* find next more recent reader */
- txnid_t next_reader = head_txnid;
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- retry:
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
- mdbx_jitter4testing(true);
- const txnid_t snap_txnid =
- safe64_read(&lck->mti_readers[i].mr_txnid);
- const uint64_t snap_retired =
- atomic_load64(&lck->mti_readers[i].mr_snapshot_pages_retired,
- mo_AcquireRelease);
- if (unlikely(snap_retired !=
- atomic_load64(
- &lck->mti_readers[i].mr_snapshot_pages_retired,
- mo_Relaxed)) ||
- snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))
+ if (MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) {
+ if (need_cleanup) {
+ mdbx_txl_sort(txn->tw.lifo_reclaimed);
+ cleaned_gc_slot = 0;
+ }
+ gc_rid = MDBX_PNL_LAST(txn->tw.lifo_reclaimed);
+ } else {
+ mdbx_tassert(txn, txn->tw.last_reclaimed == 0);
+ if (unlikely(mdbx_find_oldest(txn) != snap_oldest))
+ /* should retry mdbx_page_alloc(MDBX_ALLOC_GC)
+ * if the oldest reader changes since the last attempt */
+ goto retry_rid;
+ /* no reclaimable GC entries,
+ * therefore no entries with ID < mdbx_find_oldest(txn) */
+ txn->tw.last_reclaimed = gc_rid = snap_oldest - 1;
+ mdbx_trace("%s: none recycled yet, set rid to @%" PRIaTXN,
+ dbg_prefix_mode, gc_rid);
+ }
+
+ /* LY: GC is empty, will look any free txn-id in high2low order. */
+ while (MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter &&
+ left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
+ reused_gc_slot) *
+ env->me_maxgc_ov1page) {
+ if (unlikely(gc_rid <= MIN_TXNID)) {
+ if (unlikely(MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <=
+ reused_gc_slot)) {
+ mdbx_notice("** restart: reserve depleted (reused_gc_slot %u >= "
+ "lifo_reclaimed %u" PRIaTXN,
+ reused_gc_slot,
+ (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
goto retry;
- if (snap_txnid <= txn->mt_txnid) {
- retired_next_reader = 0;
+ }
+ break;
+ }
+
+ mdbx_tassert(txn, gc_rid >= MIN_TXNID && gc_rid <= MAX_TXNID);
+ --gc_rid;
+ key.iov_base = &gc_rid;
+ key.iov_len = sizeof(gc_rid);
+ rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_SET_KEY);
+ if (unlikely(rc == MDBX_SUCCESS)) {
+ mdbx_debug("%s: GC's id %" PRIaTXN
+ " is used, continue bottom-up search",
+ dbg_prefix_mode, gc_rid);
+ ++gc_rid;
+ rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_FIRST);
+ if (rc == MDBX_NOTFOUND) {
+ mdbx_debug("%s: GC is empty (going dense-mode)", dbg_prefix_mode);
+ dense_gc = true;
break;
}
- if (snap_txnid < next_reader) {
- next_reader = snap_txnid;
- retired_next_reader = pgno2bytes(
- env, (pgno_t)(snap_retired -
- atomic_load64(
- &txn->to.reader->mr_snapshot_pages_retired,
- mo_Relaxed)));
+ if (unlikely(rc != MDBX_SUCCESS ||
+ key.iov_len != sizeof(mdbx_tid_t))) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
}
- }
- }
- }
- info->txn_space_dirty = retired_next_reader;
- }
- } else {
- info->txn_space_limit_soft = pgno2bytes(env, txn->mt_geo.now);
- info->txn_space_limit_hard = pgno2bytes(env, txn->mt_geo.upper);
- info->txn_space_retired = pgno2bytes(
- env, txn->mt_child ? (unsigned)(uintptr_t)txn->tw.retired_pages
- : MDBX_PNL_SIZE(txn->tw.retired_pages));
- info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom);
- info->txn_space_dirty =
- pgno2bytes(env, txn->mt_env->me_options.dp_limit - txn->tw.dirtyroom);
- info->txn_reader_lag = INT64_MAX;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (scan_rlt && lck) {
- txnid_t oldest_snapshot = txn->mt_txnid;
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- if (snap_nreaders) {
- oldest_snapshot = mdbx_find_oldest(txn);
- if (oldest_snapshot == txn->mt_txnid - 1) {
- /* check if there is at least one reader */
- bool exists = false;
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) &&
- txn->mt_txnid > safe64_read(&lck->mti_readers[i].mr_txnid)) {
- exists = true;
+ txnid_t gc_first = unaligned_peek_u64(4, key.iov_base);
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(gc_first < MIN_TXNID || gc_first > MAX_TXNID)) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
+ }
+ if (gc_first <= MIN_TXNID) {
+ mdbx_debug("%s: no free GC's id(s) less than %" PRIaTXN
+ " (going dense-mode)",
+ dbg_prefix_mode, gc_rid);
+ dense_gc = true;
break;
}
+ gc_rid = gc_first - 1;
}
- oldest_snapshot += !exists;
- }
- }
- info->txn_reader_lag = txn->mt_txnid - oldest_snapshot;
- }
- }
-
- return MDBX_SUCCESS;
-}
-MDBX_env *mdbx_txn_env(const MDBX_txn *txn) {
- if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE ||
- txn->mt_env->me_signature.weak != MDBX_ME_SIGNATURE))
- return NULL;
- return txn->mt_env;
-}
+ mdbx_assert(env, !dense_gc);
+ rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, gc_rid);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
-uint64_t mdbx_txn_id(const MDBX_txn *txn) {
- if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
- return 0;
- return txn->mt_txnid;
-}
+ if (reused_gc_slot)
+ /* rare case, but it is better to clear and re-create GC entries
+ * with less fragmentation. */
+ need_cleanup = true;
+ else
+ cleaned_gc_slot +=
+ 1 /* mark cleanup is not needed for added slot. */;
-int mdbx_txn_flags(const MDBX_txn *txn) {
- if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) {
- assert((-1 & (int)MDBX_TXN_INVALID) != 0);
- return -1;
- }
- assert(0 == (int)(txn->mt_flags & MDBX_TXN_INVALID));
- return txn->mt_flags;
-}
+ mdbx_trace("%s: append @%" PRIaTXN
+ " to lifo-reclaimed, cleaned-gc-slot = %u",
+ dbg_prefix_mode, gc_rid, cleaned_gc_slot);
+ }
-/* Check for misused dbi handles */
-#define TXN_DBI_CHANGED(txn, dbi) \
- ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi])
+ if (need_cleanup || dense_gc) {
+ if (cleaned_gc_slot)
+ mdbx_trace(
+ "%s: restart inner-loop to clear and re-create GC entries",
+ dbg_prefix_mode);
+ cleaned_gc_slot = 0;
+ continue;
+ }
+ }
-static void dbi_import_locked(MDBX_txn *txn) {
- MDBX_env *const env = txn->mt_env;
- const unsigned n = env->me_numdbs;
- for (unsigned i = CORE_DBS; i < n; ++i) {
- if (i >= txn->mt_numdbs) {
- txn->mt_dbistate[i] = 0;
- txn->mt_cursors[i] = NULL;
- }
- if ((env->me_dbflags[i] & DB_VALID) &&
- !(txn->mt_dbistate[i] & DBI_USRVALID)) {
- txn->mt_dbiseqs[i] = env->me_dbiseqs[i];
- txn->mt_dbs[i].md_flags = env->me_dbflags[i] & DB_PERSISTENT_FLAGS;
- txn->mt_dbistate[i] = DBI_VALID | DBI_USRVALID | DBI_STALE;
- mdbx_tassert(txn, txn->mt_dbxs[i].md_cmp != NULL);
- mdbx_tassert(txn, txn->mt_dbxs[i].md_name.iov_base != NULL);
+ const unsigned i =
+ (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot;
+ mdbx_tassert(txn, i > 0 && i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
+ reservation_gc_id = txn->tw.lifo_reclaimed[i];
+ mdbx_trace("%s: take @%" PRIaTXN " from lifo-reclaimed[%u]",
+ dbg_prefix_mode, reservation_gc_id, i);
+ } else {
+ mdbx_tassert(txn, txn->tw.lifo_reclaimed == NULL);
+ if (unlikely(gc_rid == 0)) {
+ gc_rid = mdbx_find_oldest(txn) - 1;
+ rc = mdbx_cursor_get(&couple.outer, &key, NULL, MDBX_FIRST);
+ if (rc == MDBX_SUCCESS) {
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(key.iov_len != sizeof(txnid_t))) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
+ }
+ txnid_t gc_first = unaligned_peek_u64(4, key.iov_base);
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(gc_first < MIN_TXNID || gc_first > MAX_TXNID)) {
+ rc = MDBX_CORRUPTED;
+ goto bailout;
+ }
+ if (gc_rid >= gc_first)
+ gc_rid = gc_first - 1;
+ if (unlikely(gc_rid == 0)) {
+ mdbx_error("%s", "** no GC tail-space to store (going dense-mode)");
+ dense_gc = true;
+ goto retry;
+ }
+ } else if (rc != MDBX_NOTFOUND)
+ goto bailout;
+ txn->tw.last_reclaimed = gc_rid;
+ cleaned_gc_id = gc_rid + 1;
+ }
+ reservation_gc_id = gc_rid--;
+ mdbx_trace("%s: take @%" PRIaTXN " from head-gc-id", dbg_prefix_mode,
+ reservation_gc_id);
}
- }
- txn->mt_numdbs = n;
-}
-
-/* Import DBI which opened after txn started into context */
-__cold static bool dbi_import(MDBX_txn *txn, MDBX_dbi dbi) {
- if (dbi < CORE_DBS || dbi >= txn->mt_env->me_numdbs)
- return false;
-
- mdbx_ensure(txn->mt_env, mdbx_fastmutex_acquire(&txn->mt_env->me_dbi_lock) ==
- MDBX_SUCCESS);
- dbi_import_locked(txn);
- mdbx_ensure(txn->mt_env, mdbx_fastmutex_release(&txn->mt_env->me_dbi_lock) ==
- MDBX_SUCCESS);
- return txn->mt_dbistate[dbi] & DBI_USRVALID;
-}
-
-/* Export or close DBI handles opened in this txn. */
-static void dbi_update(MDBX_txn *txn, int keep) {
- mdbx_tassert(txn, !txn->mt_parent && txn == txn->mt_env->me_txn0);
- MDBX_dbi n = txn->mt_numdbs;
- if (n) {
- bool locked = false;
- MDBX_env *const env = txn->mt_env;
+ ++reused_gc_slot;
- for (unsigned i = n; --i >= CORE_DBS;) {
- if (likely((txn->mt_dbistate[i] & DBI_CREAT) == 0))
- continue;
- if (!locked) {
- mdbx_ensure(env,
- mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS);
- locked = true;
- }
- if (env->me_numdbs <= i || txn->mt_dbiseqs[i] != env->me_dbiseqs[i])
- continue /* dbi explicitly closed and/or then re-opened by other txn */;
- if (keep) {
- env->me_dbflags[i] = txn->mt_dbs[i].md_flags | DB_VALID;
- } else {
- char *ptr = env->me_dbxs[i].md_name.iov_base;
- if (ptr) {
- env->me_dbxs[i].md_name.iov_len = 0;
- mdbx_memory_fence(mo_AcquireRelease, true);
- mdbx_assert(env, env->me_dbflags[i] == 0);
- env->me_dbiseqs[i]++;
- env->me_dbxs[i].md_name.iov_base = NULL;
- mdbx_free(ptr);
+ unsigned chunk = left;
+ if (unlikely(chunk > env->me_maxgc_ov1page)) {
+ const unsigned avail_gc_slots =
+ txn->tw.lifo_reclaimed
+ ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
+ reused_gc_slot + 1
+ : (gc_rid < INT16_MAX) ? (unsigned)gc_rid
+ : INT16_MAX;
+ if (avail_gc_slots > 1) {
+ if (chunk < env->me_maxgc_ov1page * 2)
+ chunk /= 2;
+ else {
+ const unsigned threshold =
+ env->me_maxgc_ov1page * ((avail_gc_slots < prefer_max_scatter)
+ ? avail_gc_slots
+ : prefer_max_scatter);
+ if (left < threshold)
+ chunk = env->me_maxgc_ov1page;
+ else {
+ const unsigned tail = left - threshold + env->me_maxgc_ov1page + 1;
+ unsigned span = 1;
+ unsigned avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) /
+ sizeof(pgno_t)) /* - 1 + span */;
+ if (tail > avail) {
+ for (unsigned i = amount - span; i > 0; --i) {
+ if (MDBX_PNL_ASCENDING
+ ? (txn->tw.reclaimed_pglist[i] + span)
+ : (txn->tw.reclaimed_pglist[i] - span) ==
+ txn->tw.reclaimed_pglist[i + span]) {
+ span += 1;
+ avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) /
+ sizeof(pgno_t)) -
+ 1 + span;
+ if (avail >= tail)
+ break;
+ }
+ }
+ }
+
+ chunk = (avail >= tail) ? tail - span
+ : (avail_gc_slots > 3 &&
+ reused_gc_slot < prefer_max_scatter - 3)
+ ? avail - span
+ : tail;
+ }
}
}
}
+ mdbx_tassert(txn, chunk > 0);
- n = env->me_numdbs;
- if (n > CORE_DBS && unlikely(!(env->me_dbflags[n - 1] & DB_VALID))) {
- if (!locked) {
- mdbx_ensure(env,
- mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS);
- locked = true;
- }
-
- n = env->me_numdbs;
- while (n > CORE_DBS && !(env->me_dbflags[n - 1] & DB_VALID))
- --n;
- env->me_numdbs = n;
- }
+ mdbx_trace("%s: gc_rid %" PRIaTXN ", reused_gc_slot %u, reservation-id "
+ "%" PRIaTXN,
+ dbg_prefix_mode, gc_rid, reused_gc_slot, reservation_gc_id);
- if (unlikely(locked))
- mdbx_ensure(env,
- mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
- }
-}
+ mdbx_trace("%s: chunk %u, gc-per-ovpage %u", dbg_prefix_mode, chunk,
+ env->me_maxgc_ov1page);
-/* Filter-out pgno list from transaction's dirty-page list */
-static void mdbx_dpl_sift(MDBX_txn *const txn, MDBX_PNL pl,
- const bool spilled) {
- if (MDBX_PNL_SIZE(pl) && txn->tw.dirtylist->length) {
- mdbx_tassert(
- txn, mdbx_pnl_check4assert(pl, (size_t)txn->mt_next_pgno << spilled));
- MDBX_dpl *dl = mdbx_dpl_sort(txn);
+ mdbx_tassert(txn, reservation_gc_id < env->me_lck->mti_oldest_reader.weak);
+ if (unlikely(
+ reservation_gc_id < MIN_TXNID ||
+ reservation_gc_id >=
+ atomic_load64(&env->me_lck->mti_oldest_reader, mo_Relaxed))) {
+ mdbx_error("** internal error (reservation_gc_id %" PRIaTXN ")",
+ reservation_gc_id);
+ rc = MDBX_PROBLEM;
+ goto bailout;
+ }
- /* Scanning in ascend order */
- const int step = MDBX_PNL_ASCENDING ? 1 : -1;
- const int begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_SIZE(pl);
- const int end = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(pl) + 1 : 0;
- mdbx_tassert(txn, pl[begin] <= pl[end - step]);
+ key.iov_len = sizeof(reservation_gc_id);
+ key.iov_base = &reservation_gc_id;
+ data.iov_len = (chunk + 1) * sizeof(pgno_t);
+ mdbx_trace("%s: reserve %u [%u...%u) @%" PRIaTXN, dbg_prefix_mode, chunk,
+ settled + 1, settled + chunk + 1, reservation_gc_id);
+ mdbx_prep_backlog(txn, &couple.outer, data.iov_len, nullptr);
+ rc = mdbx_cursor_put(&couple.outer, &key, &data,
+ MDBX_RESERVE | MDBX_NOOVERWRITE);
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
- unsigned r = mdbx_dpl_search(txn, pl[begin] >> spilled);
- mdbx_tassert(txn, dl->sorted == dl->length);
- for (int i = begin; r <= dl->length;) { /* scan loop */
- assert(i != end);
- mdbx_tassert(txn, !spilled || (pl[i] & 1) == 0);
- pgno_t pl_pgno = pl[i] >> spilled;
- pgno_t dp_pgno = dl->items[r].pgno;
- if (likely(dp_pgno != pl_pgno)) {
- const bool cmp = dp_pgno < pl_pgno;
- r += cmp;
- i += cmp ? 0 : step;
- if (likely(i != end))
- continue;
- return;
- }
+ clean_reserved_gc_pnl(env, data);
+ settled += chunk;
+ mdbx_trace("%s: settled %u (+%u), continue", dbg_prefix_mode, settled,
+ chunk);
- /* update loop */
- unsigned w = r;
- remove_dl:
- if ((txn->mt_env->me_flags & MDBX_WRITEMAP) == 0) {
- MDBX_page *dp = dl->items[r].ptr;
- mdbx_dpage_free(txn->mt_env, dp, dpl_npages(dl, r));
- }
- ++r;
- next_i:
- i += step;
- if (unlikely(i == end)) {
- while (r <= dl->length)
- dl->items[w++] = dl->items[r++];
- } else {
- while (r <= dl->length) {
- assert(i != end);
- mdbx_tassert(txn, !spilled || (pl[i] & 1) == 0);
- pl_pgno = pl[i] >> spilled;
- dp_pgno = dl->items[r].pgno;
- if (dp_pgno < pl_pgno)
- dl->items[w++] = dl->items[r++];
- else if (dp_pgno > pl_pgno)
- goto next_i;
- else
- goto remove_dl;
- }
- }
- dl->sorted = dpl_setlen(dl, w - 1);
- txn->tw.dirtyroom += r - w;
- mdbx_tassert(txn,
- txn->tw.dirtyroom + txn->tw.dirtylist->length ==
- (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
- : txn->mt_env->me_options.dp_limit));
- return;
+ if (txn->tw.lifo_reclaimed &&
+ unlikely(amount < MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)) &&
+ (loop < 5 || MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) - amount >
+ env->me_maxgc_ov1page)) {
+ mdbx_notice("** restart: reclaimed-list growth %u -> %u", amount,
+ (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
+ goto retry;
}
+
+ continue;
}
-}
-/* End a transaction, except successful commit of a nested transaction.
- * May be called twice for readonly txns: First reset it, then abort.
- * [in] txn the transaction handle to end
- * [in] mode why and how to end the transaction */
-static int mdbx_txn_end(MDBX_txn *txn, const unsigned mode) {
- MDBX_env *env = txn->mt_env;
- static const char *const names[] = MDBX_END_NAMES;
+ mdbx_tassert(
+ txn,
+ cleaned_gc_slot ==
+ (txn->tw.lifo_reclaimed ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) : 0));
-#if MDBX_ENV_CHECKPID
- if (unlikely(txn->mt_env->me_pid != mdbx_getpid())) {
- env->me_flags |= MDBX_FATAL_ERROR;
- return MDBX_PANIC;
- }
-#endif /* MDBX_ENV_CHECKPID */
+ mdbx_trace("%s", " >> filling");
+ /* Fill in the reserved records */
+ filled_gc_slot =
+ txn->tw.lifo_reclaimed
+ ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot
+ : reused_gc_slot;
+ rc = MDBX_SUCCESS;
+ mdbx_tassert(txn,
+ mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
+ txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ mdbx_tassert(txn, mdbx_dirtylist_check(txn));
+ if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)) {
+ MDBX_val key, data;
+ key.iov_len = data.iov_len = 0; /* avoid MSVC warning */
+ key.iov_base = data.iov_base = NULL;
- mdbx_debug("%s txn %" PRIaTXN "%c %p on mdbenv %p, root page %" PRIaPGNO
- "/%" PRIaPGNO,
- names[mode & MDBX_END_OPMASK], txn->mt_txnid,
- (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn,
- (void *)env, txn->mt_dbs[MAIN_DBI].md_root,
- txn->mt_dbs[FREE_DBI].md_root);
+ const unsigned amount = MDBX_PNL_SIZE(txn->tw.reclaimed_pglist);
+ unsigned left = amount;
+ if (txn->tw.lifo_reclaimed == nullptr) {
+ mdbx_tassert(txn, lifo == 0);
+ rc = mdbx_cursor_first(&couple.outer, &key, &data);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ } else {
+ mdbx_tassert(txn, lifo != 0);
+ }
- mdbx_ensure(env, txn->mt_txnid >=
- /* paranoia is appropriate here */ env->me_lck
- ->mti_oldest_reader.weak);
+ while (true) {
+ txnid_t fill_gc_id;
+ mdbx_trace("%s: left %u of %u", dbg_prefix_mode, left,
+ (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
+ if (txn->tw.lifo_reclaimed == nullptr) {
+ mdbx_tassert(txn, lifo == 0);
+ fill_gc_id = unaligned_peek_u64(4, key.iov_base);
+ if (filled_gc_slot-- == 0 || fill_gc_id > txn->tw.last_reclaimed) {
+ mdbx_notice(
+ "** restart: reserve depleted (filled_slot %u, fill_id %" PRIaTXN
+ " > last_reclaimed %" PRIaTXN,
+ filled_gc_slot, fill_gc_id, txn->tw.last_reclaimed);
+ goto retry;
+ }
+ } else {
+ mdbx_tassert(txn, lifo != 0);
+ if (++filled_gc_slot >
+ (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) {
+ mdbx_notice("** restart: reserve depleted (filled_gc_slot %u > "
+ "lifo_reclaimed %u" PRIaTXN,
+ filled_gc_slot,
+ (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
+ goto retry;
+ }
+ fill_gc_id = txn->tw.lifo_reclaimed[filled_gc_slot];
+ mdbx_trace("%s: seek-reservation @%" PRIaTXN " at lifo_reclaimed[%u]",
+ dbg_prefix_mode, fill_gc_id, filled_gc_slot);
+ key.iov_base = &fill_gc_id;
+ key.iov_len = sizeof(fill_gc_id);
+ rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_SET_KEY);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ mdbx_tassert(txn, cleaned_gc_slot ==
+ (txn->tw.lifo_reclaimed
+ ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
+ : 0));
+ mdbx_tassert(txn, fill_gc_id > 0 &&
+ fill_gc_id < env->me_lck->mti_oldest_reader.weak);
+ key.iov_base = &fill_gc_id;
+ key.iov_len = sizeof(fill_gc_id);
- if (!(mode & MDBX_END_EOTDONE)) /* !(already closed cursors) */
- mdbx_cursors_eot(txn, false);
+ mdbx_tassert(txn, data.iov_len >= sizeof(pgno_t) * 2);
+ couple.outer.mc_flags |= C_GCFREEZE;
+ unsigned chunk = (unsigned)(data.iov_len / sizeof(pgno_t)) - 1;
+ if (unlikely(chunk > left)) {
+ mdbx_trace("%s: chunk %u > left %u, @%" PRIaTXN, dbg_prefix_mode, chunk,
+ left, fill_gc_id);
+ if ((loop < 5 && chunk - left > loop / 2) ||
+ chunk - left > env->me_maxgc_ov1page) {
+ data.iov_len = (left + 1) * sizeof(pgno_t);
+ if (loop < 7)
+ couple.outer.mc_flags &= ~C_GCFREEZE;
+ }
+ chunk = left;
+ }
+ rc = mdbx_cursor_put(&couple.outer, &key, &data,
+ MDBX_CURRENT | MDBX_RESERVE);
+ couple.outer.mc_flags &= ~C_GCFREEZE;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ clean_reserved_gc_pnl(env, data);
- int rc = MDBX_SUCCESS;
- if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)) {
- if (txn->to.reader) {
- MDBX_reader *slot = txn->to.reader;
- mdbx_assert(env, slot->mr_pid.weak == env->me_pid);
- if (likely(!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED))) {
- mdbx_assert(env, txn->mt_txnid == slot->mr_txnid.weak &&
- slot->mr_txnid.weak >=
- env->me_lck->mti_oldest_reader.weak);
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- mdbx_txn_valgrind(env, nullptr);
-#endif
- atomic_store32(&slot->mr_snapshot_pages_used, 0, mo_Relaxed);
- safe64_reset(&slot->mr_txnid, false);
- atomic_store32(&env->me_lck->mti_readers_refresh_flag, true,
- mo_Relaxed);
- } else {
- mdbx_assert(env, slot->mr_pid.weak == env->me_pid);
- mdbx_assert(env, slot->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD);
+ if (unlikely(txn->tw.loose_count ||
+ amount != MDBX_PNL_SIZE(txn->tw.reclaimed_pglist))) {
+ mdbx_notice("** restart: reclaimed-list growth (%u -> %u, loose +%u)",
+ amount, MDBX_PNL_SIZE(txn->tw.reclaimed_pglist),
+ txn->tw.loose_count);
+ goto retry;
}
- if (mode & MDBX_END_SLOT) {
- if ((env->me_flags & MDBX_ENV_TXKEY) == 0)
- atomic_store32(&slot->mr_pid, 0, mo_Relaxed);
- txn->to.reader = NULL;
+ if (unlikely(txn->tw.lifo_reclaimed
+ ? cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
+ : cleaned_gc_id < txn->tw.last_reclaimed)) {
+ mdbx_notice("%s", "** restart: reclaimed-slots changed");
+ goto retry;
+ }
+ if (unlikely(retired_stored != MDBX_PNL_SIZE(txn->tw.retired_pages))) {
+ mdbx_tassert(txn,
+ retired_stored < MDBX_PNL_SIZE(txn->tw.retired_pages));
+ mdbx_notice("** restart: retired-list growth (%u -> %u)",
+ retired_stored, MDBX_PNL_SIZE(txn->tw.retired_pages));
+ goto retry;
}
- }
-#if defined(_WIN32) || defined(_WIN64)
- if (txn->mt_flags & MDBX_SHRINK_ALLOWED)
- mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
-#endif
- txn->mt_numdbs = 0; /* prevent further DBI activity */
- txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED;
- txn->mt_owner = 0;
- } else if (!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED)) {
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- if (txn == env->me_txn0)
- mdbx_txn_valgrind(env, nullptr);
-#endif
- txn->mt_flags = MDBX_TXN_FINISHED;
- txn->mt_owner = 0;
- env->me_txn = txn->mt_parent;
- mdbx_pnl_free(txn->tw.spill_pages);
- txn->tw.spill_pages = nullptr;
- if (txn == env->me_txn0) {
- mdbx_assert(env, txn->mt_parent == NULL);
- /* Export or close DBI handles created in this txn */
- dbi_update(txn, mode & MDBX_END_UPDATE);
- mdbx_pnl_shrink(&txn->tw.retired_pages);
- mdbx_pnl_shrink(&txn->tw.reclaimed_pglist);
- if (!(env->me_flags & MDBX_WRITEMAP))
- mdbx_dlist_free(txn);
- /* The writer mutex was locked in mdbx_txn_begin. */
- mdbx_txn_unlock(env);
- } else {
- mdbx_assert(env, txn->mt_parent != NULL);
- MDBX_txn *const parent = txn->mt_parent;
- mdbx_assert(env, parent->mt_signature == MDBX_MT_SIGNATURE);
- mdbx_assert(env, parent->mt_child == txn &&
- (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0);
- mdbx_assert(
- env, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
+ pgno_t *dst = data.iov_base;
+ *dst++ = chunk;
+ pgno_t *src = MDBX_PNL_BEGIN(txn->tw.reclaimed_pglist) + left - chunk;
+ memcpy(dst, src, chunk * sizeof(pgno_t));
+ pgno_t *from = src, *to = src + chunk;
+ mdbx_trace("%s: fill %u [ %u:%" PRIaPGNO "...%u:%" PRIaPGNO
+ "] @%" PRIaTXN,
+ dbg_prefix_mode, chunk,
+ (unsigned)(from - txn->tw.reclaimed_pglist), from[0],
+ (unsigned)(to - txn->tw.reclaimed_pglist), to[-1], fill_gc_id);
- if (txn->tw.lifo_reclaimed) {
- mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) >=
- (unsigned)(uintptr_t)parent->tw.lifo_reclaimed);
- MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) =
- (unsigned)(uintptr_t)parent->tw.lifo_reclaimed;
- parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed;
+ left -= chunk;
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_audit_ex(txn, retired_stored + amount - left, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
}
-
- if (txn->tw.retired_pages) {
- mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.retired_pages) >=
- (unsigned)(uintptr_t)parent->tw.retired_pages);
- MDBX_PNL_SIZE(txn->tw.retired_pages) =
- (unsigned)(uintptr_t)parent->tw.retired_pages;
- parent->tw.retired_pages = txn->tw.retired_pages;
+ if (left == 0) {
+ rc = MDBX_SUCCESS;
+ break;
}
- parent->mt_child = nullptr;
- parent->mt_flags &= ~MDBX_TXN_HAS_CHILD;
- parent->tw.dirtylru = txn->tw.dirtylru;
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
- mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
- if (!(env->me_flags & MDBX_WRITEMAP))
- mdbx_dlist_free(txn);
- mdbx_dpl_free(txn);
- mdbx_pnl_free(txn->tw.reclaimed_pglist);
-
- if (parent->mt_geo.upper != txn->mt_geo.upper ||
- parent->mt_geo.now != txn->mt_geo.now) {
- /* undo resize performed by child txn */
- rc = mdbx_mapresize_implicit(env, parent->mt_next_pgno,
- parent->mt_geo.now, parent->mt_geo.upper);
- if (rc == MDBX_EPERM) {
- /* unable undo resize (it is regular for Windows),
- * therefore promote size changes from child to the parent txn */
- mdbx_warning("unable undo resize performed by child txn, promote to "
- "the parent (%u->%u, %u->%u)",
- txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper,
- parent->mt_geo.upper);
- parent->mt_geo.now = txn->mt_geo.now;
- parent->mt_geo.upper = txn->mt_geo.upper;
- parent->mt_flags |= MDBX_TXN_DIRTY;
- rc = MDBX_SUCCESS;
- } else if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_error("error %d while undo resize performed by child txn, fail "
- "the parent",
- rc);
- parent->mt_flags |= MDBX_TXN_ERROR;
- if (!env->me_dxb_mmap.address)
- env->me_flags |= MDBX_FATAL_ERROR;
- }
+ if (txn->tw.lifo_reclaimed == nullptr) {
+ mdbx_tassert(txn, lifo == 0);
+ rc = mdbx_cursor_next(&couple.outer, &key, &data, MDBX_NEXT);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ } else {
+ mdbx_tassert(txn, lifo != 0);
}
}
}
- mdbx_assert(env, txn == env->me_txn0 || txn->mt_owner == 0);
- if ((mode & MDBX_END_FREE) != 0 && txn != env->me_txn0) {
- txn->mt_signature = 0;
- mdbx_free(txn);
+ mdbx_tassert(txn, rc == MDBX_SUCCESS);
+ if (unlikely(txn->tw.loose_count != 0)) {
+ mdbx_notice("** restart: got %u loose pages", txn->tw.loose_count);
+ goto retry;
}
+ if (unlikely(filled_gc_slot !=
+ (txn->tw.lifo_reclaimed
+ ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
+ : 0))) {
- return rc;
-}
+ const bool will_retry = loop < 9;
+ mdbx_notice("** %s: reserve excess (filled-slot %u, loop %u)",
+ will_retry ? "restart" : "ignore", filled_gc_slot, loop);
+ if (will_retry)
+ goto retry;
+ }
-int mdbx_txn_reset(MDBX_txn *txn) {
- int rc = check_txn(txn, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ mdbx_tassert(txn,
+ txn->tw.lifo_reclaimed == NULL ||
+ cleaned_gc_slot == MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
- /* This call is only valid for read-only txns */
- if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0))
- return MDBX_EINVAL;
+bailout:
+ txn->mt_cursors[FREE_DBI] = couple.outer.mc_next;
- /* LY: don't close DBI-handles */
- rc = mdbx_txn_end(txn, MDBX_END_RESET | MDBX_END_UPDATE);
- if (rc == MDBX_SUCCESS) {
- mdbx_tassert(txn, txn->mt_signature == MDBX_MT_SIGNATURE);
- mdbx_tassert(txn, txn->mt_owner == 0);
- }
+bailout_notracking:
+ MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) = 0;
+ mdbx_trace("<<< %u loops, rc = %d", loop, rc);
return rc;
}
-int mdbx_txn_break(MDBX_txn *txn) {
- do {
- int rc = check_txn(txn, 0);
+static int mdbx_txn_write(MDBX_txn *txn, struct mdbx_iov_ctx *ctx) {
+ MDBX_dpl *const dl =
+ (txn->mt_flags & MDBX_WRITEMAP) ? txn->tw.dirtylist : mdbx_dpl_sort(txn);
+ int rc = MDBX_SUCCESS;
+ unsigned r, w;
+ for (w = 0, r = 1; r <= dl->length; ++r) {
+ MDBX_page *dp = dl->items[r].ptr;
+ if (dp->mp_flags & P_LOOSE) {
+ dl->items[++w] = dl->items[r];
+ continue;
+ }
+ unsigned npages = dpl_npages(dl, r);
+ rc = iov_page(txn, ctx, dp, npages);
if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- txn->mt_flags |= MDBX_TXN_ERROR;
- if (txn->mt_flags & MDBX_TXN_RDONLY)
break;
- txn = txn->mt_child;
- } while (txn);
- return MDBX_SUCCESS;
-}
-
-int mdbx_txn_abort(MDBX_txn *txn) {
- int rc = check_txn(txn, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ }
- if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY))
- /* LY: don't close DBI-handles */
- return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_UPDATE | MDBX_END_SLOT |
- MDBX_END_FREE);
+ if (ctx->iov_items)
+ rc = mdbx_iov_write(txn, ctx);
- if (txn->mt_child)
- mdbx_txn_abort(txn->mt_child);
+ while (r <= dl->length)
+ dl->items[++w] = dl->items[r++];
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_SLOT | MDBX_END_FREE);
+ dl->sorted = dpl_setlen(dl, w);
+ txn->tw.dirtyroom += r - 1 - w;
+ mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
+ (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
+ : txn->mt_env->me_options.dp_limit));
+ return rc;
}
-/* Count all the pages in each DB and in the GC and make sure
- * it matches the actual number of pages being used. */
-__cold static int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored,
- bool dont_filter_gc) {
- pgno_t pending = 0;
- if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
- pending = txn->tw.loose_count + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) +
- (MDBX_PNL_SIZE(txn->tw.retired_pages) - retired_stored);
- }
-
- MDBX_cursor_couple cx;
- int rc = mdbx_cursor_init(&cx.outer, txn, FREE_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
-
- pgno_t gc = 0;
- MDBX_val key, data;
- while ((rc = mdbx_cursor_get(&cx.outer, &key, &data, MDBX_NEXT)) == 0) {
- if (!dont_filter_gc) {
- if (unlikely(key.iov_len != sizeof(txnid_t)))
- return MDBX_CORRUPTED;
- txnid_t id = unaligned_peek_u64(4, key.iov_base);
- if (txn->tw.lifo_reclaimed) {
- for (unsigned i = 1; i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); ++i)
- if (id == txn->tw.lifo_reclaimed[i])
- goto skip;
- } else if (id <= txn->tw.last_reclaimed)
- goto skip;
- }
+/* Check txn and dbi arguments to a function */
+static __always_inline bool check_dbi(MDBX_txn *txn, MDBX_dbi dbi,
+ unsigned validity) {
+ if (likely(dbi < txn->mt_numdbs))
+ return likely((txn->mt_dbistate[dbi] & validity) &&
+ !TXN_DBI_CHANGED(txn, dbi) &&
+ (txn->mt_dbxs[dbi].md_name.iov_base || dbi < CORE_DBS));
- gc += *(pgno_t *)data.iov_base;
- skip:;
- }
- mdbx_tassert(txn, rc == MDBX_NOTFOUND);
+ return dbi_import(txn, dbi);
+}
- for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++)
- txn->mt_dbistate[i] &= ~DBI_AUDITED;
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); }
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- pgno_t used = NUM_METAS;
- for (MDBX_dbi i = FREE_DBI; i <= MAIN_DBI; i++) {
- if (!(txn->mt_dbistate[i] & DBI_VALID))
- continue;
- rc = mdbx_cursor_init(&cx.outer, txn, i);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- txn->mt_dbistate[i] |= DBI_AUDITED;
- if (txn->mt_dbs[i].md_root == P_INVALID)
- continue;
- used += txn->mt_dbs[i].md_branch_pages + txn->mt_dbs[i].md_leaf_pages +
- txn->mt_dbs[i].md_overflow_pages;
+/* Merge child txn into parent */
+static __inline void mdbx_txn_merge(MDBX_txn *const parent, MDBX_txn *const txn,
+ const unsigned parent_retired_len) {
+ MDBX_dpl *const src = mdbx_dpl_sort(txn);
- if (i != MAIN_DBI)
- continue;
- rc = mdbx_page_search(&cx.outer, NULL, MDBX_PS_FIRST);
- while (rc == MDBX_SUCCESS) {
- MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top];
- for (unsigned j = 0; j < page_numkeys(mp); j++) {
- MDBX_node *node = page_node(mp, j);
- if (node_flags(node) == F_SUBDATA) {
- if (unlikely(node_ds(node) != sizeof(MDBX_db)))
- return MDBX_CORRUPTED;
- MDBX_db db_copy, *db;
- memcpy(db = &db_copy, node_data(node), sizeof(db_copy));
- if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
- for (MDBX_dbi k = txn->mt_numdbs; --k > MAIN_DBI;) {
- if ((txn->mt_dbistate[k] & DBI_VALID) &&
- /* txn->mt_dbxs[k].md_name.iov_len > 0 && */
- node_ks(node) == txn->mt_dbxs[k].md_name.iov_len &&
- memcmp(node_key(node), txn->mt_dbxs[k].md_name.iov_base,
- node_ks(node)) == 0) {
- txn->mt_dbistate[k] |= DBI_AUDITED;
- if (!(txn->mt_dbistate[k] & MDBX_DBI_STALE))
- db = txn->mt_dbs + k;
- break;
- }
- }
- }
- used +=
- db->md_branch_pages + db->md_leaf_pages + db->md_overflow_pages;
- }
+ /* Remove refunded pages from parent's dirty list */
+ MDBX_dpl *const dst = mdbx_dpl_sort(parent);
+ if (MDBX_ENABLE_REFUND) {
+ unsigned n = dst->length;
+ while (n && dst->items[n].pgno >= parent->mt_next_pgno) {
+ if (!(txn->mt_env->me_flags & MDBX_WRITEMAP)) {
+ MDBX_page *dp = dst->items[n].ptr;
+ mdbx_dpage_free(txn->mt_env, dp, dpl_npages(dst, n));
}
- rc = mdbx_cursor_sibling(&cx.outer, SIBLING_RIGHT);
+ --n;
}
- mdbx_tassert(txn, rc == MDBX_NOTFOUND);
+ parent->tw.dirtyroom += dst->sorted - n;
+ dst->sorted = dpl_setlen(dst, n);
+ mdbx_tassert(parent,
+ parent->tw.dirtyroom + parent->tw.dirtylist->length ==
+ (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
+ : parent->mt_env->me_options.dp_limit));
}
- for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++) {
- if ((txn->mt_dbistate[i] & (DBI_VALID | DBI_AUDITED | DBI_STALE)) !=
- DBI_VALID)
- continue;
- for (MDBX_txn *t = txn; t; t = t->mt_parent)
- if (F_ISSET(t->mt_dbistate[i], DBI_DIRTY | DBI_CREAT)) {
- used += t->mt_dbs[i].md_branch_pages + t->mt_dbs[i].md_leaf_pages +
- t->mt_dbs[i].md_overflow_pages;
- txn->mt_dbistate[i] |= DBI_AUDITED;
- break;
+ /* Remove reclaimed pages from parent's dirty list */
+ const MDBX_PNL reclaimed_list = parent->tw.reclaimed_pglist;
+ mdbx_dpl_sift(parent, reclaimed_list, false);
+
+ /* Move retired pages from parent's dirty & spilled list to reclaimed */
+ unsigned r, w, d, s, l;
+ for (r = w = parent_retired_len;
+ ++r <= MDBX_PNL_SIZE(parent->tw.retired_pages);) {
+ const pgno_t pgno = parent->tw.retired_pages[r];
+ const unsigned di = mdbx_dpl_exist(parent, pgno);
+ const unsigned si = !di ? mdbx_search_spilled(parent, pgno) : 0;
+ unsigned npages;
+ const char *kind;
+ if (di) {
+ MDBX_page *dp = dst->items[di].ptr;
+ mdbx_tassert(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH |
+ P_OVERFLOW | P_SPILLED)) == 0);
+ npages = dpl_npages(dst, di);
+ mdbx_page_wash(parent, di, dp, npages);
+ kind = "dirty";
+ l = 1;
+ if (unlikely(npages > l)) {
+ /* OVERFLOW-Ñтраница могла быть переиÑпользована по чаÑÑ‚Ñм. Тогда
+ * в retired-ÑпиÑке может быть только начало поÑледовательноÑти,
+ * а оÑтаток раÑтащен по dirty, spilled и reclaimed ÑпиÑкам. ПоÑтому
+ * переноÑим в reclaimed Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¾Ð¹ на обрыв поÑледовательноÑти.
+ * Ð’ любом Ñлучае, вÑе оÑколки будут учтены и отфильтрованы, Ñ‚.е. еÑли
+ * Ñтраница была разбита на чаÑти, то важно удалить dirty-Ñлемент,
+ * а вÑе оÑколки будут учтены отдельно. */
+
+ /* СпиÑок retired Ñтраниц не Ñортирован, но Ð´Ð»Ñ ÑƒÑÐºÐ¾Ñ€ÐµÐ½Ð¸Ñ Ñортировки
+ * дополнÑетÑÑ Ð² ÑоответÑтвии Ñ MDBX_PNL_ASCENDING */
+#if MDBX_PNL_ASCENDING
+ const unsigned len = MDBX_PNL_SIZE(parent->tw.retired_pages);
+ while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) {
+ ++r;
+ if (++l == npages)
+ break;
+ }
+#else
+ while (w > parent_retired_len &&
+ parent->tw.retired_pages[w - 1] == pgno + l) {
+ --w;
+ if (++l == npages)
+ break;
+ }
+#endif
}
- if (!(txn->mt_dbistate[i] & DBI_AUDITED)) {
- mdbx_warning("audit %s@%" PRIaTXN
- ": unable account dbi %d / \"%*s\", state 0x%02x",
- txn->mt_parent ? "nested-" : "", txn->mt_txnid, i,
- (int)txn->mt_dbxs[i].md_name.iov_len,
- (const char *)txn->mt_dbxs[i].md_name.iov_base,
- txn->mt_dbistate[i]);
+ } else if (unlikely(si)) {
+ l = npages = 1;
+ mdbx_spill_remove(parent, si, 1);
+ kind = "spilled";
+ } else {
+ parent->tw.retired_pages[++w] = pgno;
+ continue;
}
- }
- if (pending + gc + used == txn->mt_next_pgno)
- return MDBX_SUCCESS;
+ mdbx_debug("reclaim retired parent's %u->%u %s page %" PRIaPGNO, npages, l,
+ kind, pgno);
+ int err = mdbx_pnl_insert_range(&parent->tw.reclaimed_pglist, pgno, l);
+ mdbx_ensure(txn->mt_env, err == MDBX_SUCCESS);
+ }
+ MDBX_PNL_SIZE(parent->tw.retired_pages) = w;
- if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0)
- mdbx_error("audit @%" PRIaTXN ": %u(pending) = %u(loose) + "
- "%u(reclaimed) + %u(retired-pending) - %u(retired-stored)",
- txn->mt_txnid, pending, txn->tw.loose_count,
- MDBX_PNL_SIZE(txn->tw.reclaimed_pglist),
- txn->tw.retired_pages ? MDBX_PNL_SIZE(txn->tw.retired_pages) : 0,
- retired_stored);
- mdbx_error("audit @%" PRIaTXN ": %" PRIaPGNO "(pending) + %" PRIaPGNO
- "(gc) + %" PRIaPGNO "(count) = %" PRIaPGNO "(total) <> %" PRIaPGNO
- "(allocated)",
- txn->mt_txnid, pending, gc, used, pending + gc + used,
- txn->mt_next_pgno);
- return MDBX_PROBLEM;
-}
+ /* Filter-out parent spill list */
+ if (parent->tw.spill_pages && MDBX_PNL_SIZE(parent->tw.spill_pages) > 0) {
+ const MDBX_PNL sl = mdbx_spill_purge(parent);
+ unsigned len = MDBX_PNL_SIZE(sl);
+ if (len) {
+ /* Remove refunded pages from parent's spill list */
+ if (MDBX_ENABLE_REFUND &&
+ MDBX_PNL_MOST(sl) >= (parent->mt_next_pgno << 1)) {
+#if MDBX_PNL_ASCENDING
+ unsigned i = MDBX_PNL_SIZE(sl);
+ assert(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl));
+ do {
+ if ((sl[i] & 1) == 0)
+ mdbx_debug("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
+ i -= 1;
+ } while (i && sl[i] >= (parent->mt_next_pgno << 1));
+ MDBX_PNL_SIZE(sl) = i;
+#else
+ assert(MDBX_PNL_MOST(sl) == MDBX_PNL_FIRST(sl));
+ unsigned i = 0;
+ do {
+ ++i;
+ if ((sl[i] & 1) == 0)
+ mdbx_debug("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
+ } while (i < len && sl[i + 1] >= (parent->mt_next_pgno << 1));
+ MDBX_PNL_SIZE(sl) = len -= i;
+ memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0]));
+#endif
+ }
+ mdbx_tassert(
+ txn, mdbx_pnl_check4assert(sl, (size_t)parent->mt_next_pgno << 1));
-static __always_inline unsigned backlog_size(MDBX_txn *txn) {
- return MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) + txn->tw.loose_count;
-}
+ /* Remove reclaimed pages from parent's spill list */
+ s = MDBX_PNL_SIZE(sl), r = MDBX_PNL_SIZE(reclaimed_list);
+ /* Scanning from end to begin */
+ while (s && r) {
+ if (sl[s] & 1) {
+ --s;
+ continue;
+ }
+ const pgno_t spilled_pgno = sl[s] >> 1;
+ const pgno_t reclaimed_pgno = reclaimed_list[r];
+ if (reclaimed_pgno != spilled_pgno) {
+ const bool cmp = MDBX_PNL_ORDERED(spilled_pgno, reclaimed_pgno);
+ s -= !cmp;
+ r -= cmp;
+ } else {
+ mdbx_debug("remove reclaimed parent's spilled page %" PRIaPGNO,
+ reclaimed_pgno);
+ mdbx_spill_remove(parent, s, 1);
+ --s;
+ --r;
+ }
+ }
-/* LY: Prepare a backlog of pages to modify GC itself,
- * while reclaiming is prohibited. It should be enough to prevent search
- * in mdbx_page_alloc() during a deleting, when GC tree is unbalanced. */
-static int mdbx_prep_backlog(MDBX_txn *txn, MDBX_cursor *gc_cursor,
- const size_t pnl_bytes, unsigned *retired_stored) {
- const unsigned linear4list = number_of_ovpages(txn->mt_env, pnl_bytes);
- const unsigned backlog4cow = txn->mt_dbs[FREE_DBI].md_depth;
- const unsigned backlog4rebalance = backlog4cow + 1;
+ /* Remove anything in our dirty list from parent's spill list */
+ /* Scanning spill list in descend order */
+ const int step = MDBX_PNL_ASCENDING ? -1 : 1;
+ s = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(sl) : 1;
+ d = src->length;
+ while (d && (MDBX_PNL_ASCENDING ? s > 0 : s <= MDBX_PNL_SIZE(sl))) {
+ if (sl[s] & 1) {
+ s += step;
+ continue;
+ }
+ const pgno_t spilled_pgno = sl[s] >> 1;
+ const pgno_t dirty_pgno_form = src->items[d].pgno;
+ const unsigned npages = dpl_npages(src, d);
+ const pgno_t dirty_pgno_to = dirty_pgno_form + npages;
+ if (dirty_pgno_form > spilled_pgno) {
+ --d;
+ continue;
+ }
+ if (dirty_pgno_to <= spilled_pgno) {
+ s += step;
+ continue;
+ }
- if (likely(linear4list == 1 &&
- backlog_size(txn) > (pnl_bytes
- ? backlog4rebalance
- : (backlog4cow + backlog4rebalance))))
- return MDBX_SUCCESS;
+ mdbx_debug("remove dirtied parent's spilled %u page %" PRIaPGNO, npages,
+ dirty_pgno_form);
+ mdbx_spill_remove(parent, s, 1);
+ s += step;
+ }
- mdbx_trace(">> pnl_bytes %zu, backlog %u, 4list %u, 4cow %u, 4rebalance %u",
- pnl_bytes, backlog_size(txn), linear4list, backlog4cow,
- backlog4rebalance);
+ /* Squash deleted pagenums if we deleted any */
+ mdbx_spill_purge(parent);
+ }
+ }
- MDBX_val gc_key, fake_val;
- int err;
- if (unlikely(linear4list > 2)) {
- gc_key.iov_base = fake_val.iov_base = nullptr;
- gc_key.iov_len = sizeof(txnid_t);
- fake_val.iov_len = pnl_bytes;
- err = mdbx_cursor_spill(gc_cursor, &gc_key, &fake_val);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ /* Remove anything in our spill list from parent's dirty list */
+ if (txn->tw.spill_pages) {
+ mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.spill_pages,
+ (size_t)parent->mt_next_pgno << 1));
+ mdbx_dpl_sift(parent, txn->tw.spill_pages, true);
+ mdbx_tassert(parent,
+ parent->tw.dirtyroom + parent->tw.dirtylist->length ==
+ (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
+ : parent->mt_env->me_options.dp_limit));
}
- gc_cursor->mc_flags &= ~C_RECLAIMING;
- err = mdbx_cursor_touch(gc_cursor);
- mdbx_trace("== after-touch, backlog %u, err %d", backlog_size(txn), err);
+ /* Find length of merging our dirty list with parent's and release
+ * filter-out pages */
+ for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) {
+ MDBX_page *sp = src->items[s].ptr;
+ mdbx_tassert(parent,
+ (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW |
+ P_LOOSE | P_SPILLED)) == 0);
+ const unsigned s_npages = dpl_npages(src, s);
+ const pgno_t s_pgno = src->items[s].pgno;
- if (unlikely(linear4list > 1) && err == MDBX_SUCCESS) {
- if (retired_stored) {
- gc_key.iov_base = &txn->mt_txnid;
- gc_key.iov_len = sizeof(txn->mt_txnid);
- const struct cursor_set_result csr =
- mdbx_cursor_set(gc_cursor, &gc_key, &fake_val, MDBX_SET);
- if (csr.err == MDBX_SUCCESS && csr.exact) {
- *retired_stored = 0;
- err = mdbx_cursor_del(gc_cursor, 0);
- mdbx_trace("== clear-4linear, backlog %u, err %d", backlog_size(txn),
- err);
+ MDBX_page *dp = dst->items[d].ptr;
+ mdbx_tassert(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH |
+ P_OVERFLOW | P_SPILLED)) == 0);
+ const unsigned d_npages = dpl_npages(dst, d);
+ const pgno_t d_pgno = dst->items[d].pgno;
+
+ if (d_pgno >= s_pgno + s_npages) {
+ --d;
+ ++l;
+ } else if (d_pgno + d_npages <= s_pgno) {
+ if (sp->mp_flags != P_LOOSE) {
+ sp->mp_txnid = parent->mt_front;
+ sp->mp_flags &= ~P_SPILLED;
}
+ --s;
+ ++l;
+ } else {
+ dst->items[d--].ptr = nullptr;
+ if ((txn->mt_flags & MDBX_WRITEMAP) == 0)
+ mdbx_dpage_free(txn->mt_env, dp, d_npages);
}
- err =
- mdbx_page_alloc(gc_cursor, linear4list, MDBX_ALLOC_GC | MDBX_ALLOC_FAKE)
- .err;
- mdbx_trace("== after-4linear, backlog %u, err %d", backlog_size(txn), err);
- mdbx_cassert(gc_cursor,
- backlog_size(txn) >= linear4list || err != MDBX_SUCCESS);
}
+ assert(dst->sorted == dst->length);
+ mdbx_tassert(parent, dst->detent >= l + d + s);
+ dst->sorted = l + d + s; /* the merged length */
- while (backlog_size(txn) < backlog4cow + linear4list && err == MDBX_SUCCESS)
- err = mdbx_page_alloc(gc_cursor, 0,
- MDBX_ALLOC_GC | MDBX_ALLOC_SLOT | MDBX_ALLOC_FAKE)
- .err;
+ while (s > 0) {
+ MDBX_page *sp = src->items[s].ptr;
+ mdbx_tassert(parent,
+ (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW |
+ P_LOOSE | P_SPILLED)) == 0);
+ if (sp->mp_flags != P_LOOSE) {
+ sp->mp_txnid = parent->mt_front;
+ sp->mp_flags &= ~P_SPILLED;
+ }
+ --s;
+ }
- gc_cursor->mc_flags |= C_RECLAIMING;
- mdbx_trace("<< backlog %u, err %d", backlog_size(txn), err);
- return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS;
-}
+ /* Merge our dirty list into parent's, i.e. merge(dst, src) -> dst */
+ if (dst->sorted >= dst->length) {
+ /* from end to begin with dst extending */
+ for (l = dst->sorted, s = src->length, d = dst->length; s > 0 && d > 0;) {
+ if (unlikely(l <= d)) {
+ /* squash to get a gap of free space for merge */
+ for (r = w = 1; r <= d; ++r)
+ if (dst->items[r].ptr) {
+ if (w != r) {
+ dst->items[w] = dst->items[r];
+ dst->items[r].ptr = nullptr;
+ }
+ ++w;
+ }
+ mdbx_notice("squash to begin for extending-merge %u -> %u", d, w - 1);
+ d = w - 1;
+ continue;
+ }
+ assert(l > d);
+ if (dst->items[d].ptr) {
+ dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno)
+ ? dst->items[d--]
+ : src->items[s--];
+ } else
+ --d;
+ }
+ if (s > 0) {
+ assert(l == s);
+ while (d > 0) {
+ assert(dst->items[d].ptr == nullptr);
+ --d;
+ }
+ do {
+ assert(l > 0);
+ dst->items[l--] = src->items[s--];
+ } while (s > 0);
+ } else {
+ assert(l == d);
+ while (l > 0) {
+ assert(dst->items[l].ptr != nullptr);
+ --l;
+ }
+ }
+ } else {
+ /* from begin to end with dst shrinking (a lot of new overflow pages) */
+ for (l = s = d = 1; s <= src->length && d <= dst->length;) {
+ if (unlikely(l >= d)) {
+ /* squash to get a gap of free space for merge */
+ for (r = w = dst->length; r >= d; --r)
+ if (dst->items[r].ptr) {
+ if (w != r) {
+ dst->items[w] = dst->items[r];
+ dst->items[r].ptr = nullptr;
+ }
+ --w;
+ }
+ mdbx_notice("squash to end for shrinking-merge %u -> %u", d, w + 1);
+ d = w + 1;
+ continue;
+ }
+ assert(l < d);
+ if (dst->items[d].ptr) {
+ dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno)
+ ? dst->items[d++]
+ : src->items[s++];
+ } else
+ ++d;
+ }
+ if (s <= src->length) {
+ assert(dst->sorted - l == src->length - s);
+ while (d <= dst->length) {
+ assert(dst->items[d].ptr == nullptr);
+ --d;
+ }
+ do {
+ assert(l <= dst->sorted);
+ dst->items[l++] = src->items[s++];
+ } while (s <= src->length);
+ } else {
+ assert(dst->sorted - l == dst->length - d);
+ while (l <= dst->sorted) {
+ assert(l <= d && d <= dst->length && dst->items[d].ptr);
+ dst->items[l++] = dst->items[d++];
+ }
+ }
+ }
+ parent->tw.dirtyroom -= dst->sorted - dst->length;
+ assert(parent->tw.dirtyroom <= parent->mt_env->me_options.dp_limit);
+ dpl_setlen(dst, dst->sorted);
+ parent->tw.dirtylru = txn->tw.dirtylru;
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ mdbx_dpl_free(txn);
-static __inline void clean_reserved_gc_pnl(MDBX_env *env, MDBX_val pnl) {
- /* PNL is initially empty, zero out at least the length */
- memset(pnl.iov_base, 0, sizeof(pgno_t));
- if ((env->me_flags & (MDBX_WRITEMAP | MDBX_NOMEMINIT)) == 0)
- /* zero out to avoid leaking values from uninitialized malloc'ed memory
- * to the file in non-writemap mode if length of the saving page-list
- * was changed during space reservation. */
- memset(pnl.iov_base, 0, pnl.iov_len);
-}
+ if (txn->tw.spill_pages) {
+ if (parent->tw.spill_pages) {
+ /* Must not fail since space was preserved above. */
+ mdbx_pnl_xmerge(parent->tw.spill_pages, txn->tw.spill_pages);
+ mdbx_pnl_free(txn->tw.spill_pages);
+ } else {
+ parent->tw.spill_pages = txn->tw.spill_pages;
+ parent->tw.spill_least_removed = txn->tw.spill_least_removed;
+ }
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ }
-/* Cleanups reclaimed GC (aka freeDB) records, saves the retired-list (aka
- * freelist) of current transaction to GC, puts back into GC leftover of the
- * reclaimed pages with chunking. This recursive changes the reclaimed-list,
- * loose-list and retired-list. Keep trying until it stabilizes.
- *
- * NOTE: This code is a consequence of many iterations of adding crutches (aka
- * "checks and balances") to partially bypass the fundamental design problems
- * inherited from LMDB. So do not try to understand it completely in order to
- * avoid your madness. */
-static int mdbx_update_gc(MDBX_txn *txn) {
- /* txn->tw.reclaimed_pglist[] can grow and shrink during this call.
- * txn->tw.last_reclaimed and txn->tw.retired_pages[] can only grow.
- * Page numbers cannot disappear from txn->tw.retired_pages[]. */
- MDBX_env *const env = txn->mt_env;
- const bool lifo = (env->me_flags & MDBX_LIFORECLAIM) != 0;
- const char *dbg_prefix_mode = lifo ? " lifo" : " fifo";
- (void)dbg_prefix_mode;
- mdbx_trace("\n>>> @%" PRIaTXN, txn->mt_txnid);
+ parent->mt_flags &= ~MDBX_TXN_HAS_CHILD;
+ if (parent->tw.spill_pages) {
+ assert(mdbx_pnl_check4assert(parent->tw.spill_pages,
+ (size_t)parent->mt_next_pgno << 1));
+ if (MDBX_PNL_SIZE(parent->tw.spill_pages))
+ parent->mt_flags |= MDBX_TXN_SPILLS;
+ }
+}
- unsigned retired_stored = 0, loop = 0;
- MDBX_cursor_couple couple;
- int rc = mdbx_cursor_init(&couple.outer, txn, FREE_DBI);
+int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
+ STATIC_ASSERT(MDBX_TXN_FINISHED ==
+ MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR);
+ const uint64_t ts_0 = latency ? mdbx_osal_monotime() : 0;
+ uint64_t ts_1 = 0, ts_2 = 0, ts_3 = 0, ts_4 = 0;
+ uint32_t audit_duration = 0;
+
+ int rc = check_txn(txn, MDBX_TXN_FINISHED);
if (unlikely(rc != MDBX_SUCCESS))
- goto bailout_notracking;
+ goto provide_latency;
- couple.outer.mc_flags |= C_RECLAIMING;
- couple.outer.mc_next = txn->mt_cursors[FREE_DBI];
- txn->mt_cursors[FREE_DBI] = &couple.outer;
- bool dense_gc = false;
+ if (unlikely(txn->mt_flags & MDBX_TXN_ERROR)) {
+ rc = MDBX_RESULT_TRUE;
+ goto fail;
+ }
-retry:
- ++loop;
- mdbx_trace("%s", " >> restart");
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- if (unlikely(/* paranoia */ loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
- mdbx_error("too more loops %u, bailout", loop);
- rc = MDBX_PROBLEM;
- goto bailout;
+ MDBX_env *env = txn->mt_env;
+#if MDBX_ENV_CHECKPID
+ if (unlikely(env->me_pid != mdbx_getpid())) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ rc = MDBX_PANIC;
+ goto provide_latency;
}
+#endif /* MDBX_ENV_CHECKPID */
- if (unlikely(dense_gc) && retired_stored) {
- rc = mdbx_prep_backlog(txn, &couple.outer,
- MDBX_PNL_SIZEOF(txn->tw.retired_pages),
- &retired_stored);
+ /* mdbx_txn_end() mode for a commit which writes nothing */
+ unsigned end_mode =
+ MDBX_END_PURE_COMMIT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE;
+ if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
+ goto done;
+
+ if (txn->mt_child) {
+ rc = mdbx_txn_commit_ex(txn->mt_child, NULL);
+ mdbx_tassert(txn, txn->mt_child == NULL);
if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ goto fail;
}
- unsigned settled = 0, cleaned_gc_slot = 0, reused_gc_slot = 0,
- filled_gc_slot = ~0u;
- txnid_t cleaned_gc_id = 0, gc_rid = txn->tw.last_reclaimed;
- while (true) {
- /* Come back here after each Put() in case retired-list changed */
- MDBX_val key, data;
- mdbx_trace("%s", " >> continue");
+ if (unlikely(txn != env->me_txn)) {
+ mdbx_debug("%s", "attempt to commit unknown transaction");
+ rc = MDBX_EINVAL;
+ goto fail;
+ }
- if (retired_stored != MDBX_PNL_SIZE(txn->tw.retired_pages) &&
- MDBX_PNL_SIZE(txn->tw.retired_pages) > env->me_maxgc_ov1page) {
- rc = mdbx_prep_backlog(txn, &couple.outer,
- MDBX_PNL_SIZEOF(txn->tw.retired_pages),
- &retired_stored);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ if (txn->mt_parent) {
+ mdbx_tassert(txn, mdbx_audit_ex(txn, 0, false) == 0);
+ mdbx_assert(env, txn != env->me_txn0);
+ MDBX_txn *const parent = txn->mt_parent;
+ mdbx_assert(env, parent->mt_signature == MDBX_MT_SIGNATURE);
+ mdbx_assert(env, parent->mt_child == txn &&
+ (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0);
+ mdbx_assert(env, mdbx_dirtylist_check(txn));
+
+ if (txn->tw.dirtylist->length == 0 && !(txn->mt_flags & MDBX_TXN_DIRTY) &&
+ parent->mt_numdbs == txn->mt_numdbs) {
+ for (int i = txn->mt_numdbs; --i >= 0;) {
+ mdbx_tassert(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0);
+ if ((txn->mt_dbistate[i] & DBI_STALE) &&
+ !(parent->mt_dbistate[i] & DBI_STALE))
+ mdbx_tassert(txn, memcmp(&parent->mt_dbs[i], &txn->mt_dbs[i],
+ sizeof(MDBX_db)) == 0);
+ }
+
+ mdbx_tassert(txn, memcmp(&parent->mt_geo, &txn->mt_geo,
+ sizeof(parent->mt_geo)) == 0);
+ mdbx_tassert(txn, memcmp(&parent->mt_canary, &txn->mt_canary,
+ sizeof(parent->mt_canary)) == 0);
+ mdbx_tassert(txn, !txn->tw.spill_pages ||
+ MDBX_PNL_SIZE(txn->tw.spill_pages) == 0);
+ mdbx_tassert(txn, txn->tw.loose_count == 0);
+
+ /* fast completion of pure nested transaction */
+ end_mode = MDBX_END_PURE_COMMIT | MDBX_END_SLOT | MDBX_END_FREE;
+ goto done;
}
+ /* Preserve space for spill list to avoid parent's state corruption
+ * if allocation fails. */
+ const unsigned parent_retired_len =
+ (unsigned)(uintptr_t)parent->tw.retired_pages;
mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- if (lifo) {
- if (cleaned_gc_slot < (txn->tw.lifo_reclaimed
- ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
- : 0)) {
- settled = 0;
- cleaned_gc_slot = 0;
- reused_gc_slot = 0;
- filled_gc_slot = ~0u;
- /* LY: cleanup reclaimed records. */
- do {
- cleaned_gc_id = txn->tw.lifo_reclaimed[++cleaned_gc_slot];
- mdbx_tassert(txn,
- cleaned_gc_slot > 0 &&
- cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
- key.iov_base = &cleaned_gc_id;
- key.iov_len = sizeof(cleaned_gc_id);
- rc = mdbx_cursor_get(&couple.outer, &key, NULL, MDBX_SET);
- if (rc == MDBX_NOTFOUND)
- continue;
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- if (likely(!dense_gc)) {
- rc = mdbx_prep_backlog(txn, &couple.outer, 0, nullptr);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- mdbx_tassert(txn,
- cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
- mdbx_trace("%s: cleanup-reclaimed-id [%u]%" PRIaTXN, dbg_prefix_mode,
- cleaned_gc_slot, cleaned_gc_id);
- mdbx_tassert(txn, *txn->mt_cursors == &couple.outer);
- rc = mdbx_cursor_del(&couple.outer, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- } while (cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
- mdbx_txl_sort(txn->tw.lifo_reclaimed);
- }
- } else {
- /* If using records from GC which we have not yet deleted,
- * now delete them and any we reserved for tw.reclaimed_pglist. */
- while (cleaned_gc_id <= txn->tw.last_reclaimed) {
- rc = mdbx_cursor_first(&couple.outer, &key, NULL);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_NOTFOUND)
- break;
- goto bailout;
- }
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(key.iov_len != sizeof(txnid_t))) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- gc_rid = cleaned_gc_id;
- settled = 0;
- reused_gc_slot = 0;
- cleaned_gc_id = unaligned_peek_u64(4, key.iov_base);
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(cleaned_gc_id < MIN_TXNID || cleaned_gc_id > MAX_TXNID)) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- if (cleaned_gc_id > txn->tw.last_reclaimed)
- break;
- if (likely(!dense_gc)) {
- rc = mdbx_prep_backlog(txn, &couple.outer, 0, nullptr);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- mdbx_tassert(txn, cleaned_gc_id <= txn->tw.last_reclaimed);
- mdbx_tassert(txn, cleaned_gc_id < env->me_lck->mti_oldest_reader.weak);
- mdbx_trace("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix_mode,
- cleaned_gc_id);
- mdbx_tassert(txn, *txn->mt_cursors == &couple.outer);
- rc = mdbx_cursor_del(&couple.outer, 0);
+ parent_retired_len <= MDBX_PNL_SIZE(txn->tw.retired_pages));
+ const unsigned retired_delta =
+ MDBX_PNL_SIZE(txn->tw.retired_pages) - parent_retired_len;
+ if (retired_delta) {
+ rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, retired_delta);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ }
+
+ if (txn->tw.spill_pages) {
+ if (parent->tw.spill_pages) {
+ rc = mdbx_pnl_need(&parent->tw.spill_pages,
+ MDBX_PNL_SIZE(txn->tw.spill_pages));
if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ goto fail;
}
+ mdbx_spill_purge(txn);
}
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- if (mdbx_audit_enabled()) {
- rc = mdbx_audit_ex(txn, retired_stored, false);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length >
+ parent->tw.dirtylist->detent &&
+ !mdbx_dpl_reserve(parent, txn->tw.dirtylist->length +
+ parent->tw.dirtylist->length))) {
+ rc = MDBX_ENOMEM;
+ goto fail;
+ }
+
+ //-------------------------------------------------------------------------
+
+ parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed;
+ txn->tw.lifo_reclaimed = NULL;
+
+ parent->tw.retired_pages = txn->tw.retired_pages;
+ txn->tw.retired_pages = NULL;
+
+ mdbx_pnl_free(parent->tw.reclaimed_pglist);
+ parent->tw.reclaimed_pglist = txn->tw.reclaimed_pglist;
+ txn->tw.reclaimed_pglist = NULL;
+ parent->tw.last_reclaimed = txn->tw.last_reclaimed;
+
+ parent->mt_geo = txn->mt_geo;
+ parent->mt_canary = txn->mt_canary;
+ parent->mt_flags |= txn->mt_flags & MDBX_TXN_DIRTY;
+
+ /* Move loose pages to parent */
+#if MDBX_ENABLE_REFUND
+ parent->tw.loose_refund_wl = txn->tw.loose_refund_wl;
+#endif /* MDBX_ENABLE_REFUND */
+ parent->tw.loose_count = txn->tw.loose_count;
+ parent->tw.loose_pages = txn->tw.loose_pages;
+
+ /* Merge our cursors into parent's and close them */
+ mdbx_cursors_eot(txn, true);
+ end_mode |= MDBX_END_EOTDONE;
+
+ /* Update parent's DBs array */
+ memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db));
+ parent->mt_numdbs = txn->mt_numdbs;
+ for (unsigned i = 0; i < txn->mt_numdbs; i++) {
+ /* preserve parent's status */
+ const uint8_t state =
+ txn->mt_dbistate[i] |
+ (parent->mt_dbistate[i] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY));
+ mdbx_debug("db %u dbi-state %s 0x%02x -> 0x%02x", i,
+ (parent->mt_dbistate[i] != state) ? "update" : "still",
+ parent->mt_dbistate[i], state);
+ parent->mt_dbistate[i] = state;
}
- /* return suitable into unallocated space */
- if (mdbx_refund(txn)) {
- mdbx_tassert(
- txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- if (mdbx_audit_enabled()) {
- rc = mdbx_audit_ex(txn, retired_stored, false);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
+ ts_1 = latency ? mdbx_osal_monotime() : 0;
+ mdbx_txn_merge(parent, txn, parent_retired_len);
+ ts_2 = latency ? mdbx_osal_monotime() : 0;
+ env->me_txn = parent;
+ parent->mt_child = NULL;
+ mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+
+#if MDBX_ENABLE_REFUND
+ mdbx_refund(parent);
+ if (mdbx_assert_enabled()) {
+ /* Check parent's loose pages not suitable for refund */
+ for (MDBX_page *lp = parent->tw.loose_pages; lp; lp = lp->mp_next)
+ mdbx_tassert(parent, lp->mp_pgno < parent->tw.loose_refund_wl &&
+ lp->mp_pgno + 1 < parent->mt_next_pgno);
+ /* Check parent's reclaimed pages not suitable for refund */
+ if (MDBX_PNL_SIZE(parent->tw.reclaimed_pglist))
+ mdbx_tassert(parent, MDBX_PNL_MOST(parent->tw.reclaimed_pglist) + 1 <
+ parent->mt_next_pgno);
}
+#endif /* MDBX_ENABLE_REFUND */
- /* handle loose pages - put ones into the reclaimed- or retired-list */
- if (txn->tw.loose_pages) {
- /* Return loose page numbers to tw.reclaimed_pglist,
- * though usually none are left at this point.
- * The pages themselves remain in dirtylist. */
- if (unlikely(!txn->tw.lifo_reclaimed && txn->tw.last_reclaimed < 1)) {
- if (txn->tw.loose_count > 0) {
- /* Put loose page numbers in tw.retired_pages,
- * since unable to return them to tw.reclaimed_pglist. */
- if (unlikely((rc = mdbx_pnl_need(&txn->tw.retired_pages,
- txn->tw.loose_count)) != 0))
- goto bailout;
- for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next)
- mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno);
- mdbx_trace("%s: append %u loose-pages to retired-pages",
- dbg_prefix_mode, txn->tw.loose_count);
- }
- } else {
- /* Room for loose pages + temp PNL with same */
- rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist,
- 2 * txn->tw.loose_count + 2);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- MDBX_PNL loose = txn->tw.reclaimed_pglist +
- MDBX_PNL_ALLOCLEN(txn->tw.reclaimed_pglist) -
- txn->tw.loose_count - 1;
- unsigned count = 0;
- for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next) {
- mdbx_tassert(txn, mp->mp_flags == P_LOOSE);
- loose[++count] = mp->mp_pgno;
- }
- mdbx_tassert(txn, count == txn->tw.loose_count);
- MDBX_PNL_SIZE(loose) = count;
- mdbx_pnl_sort(loose, txn->mt_next_pgno);
- mdbx_pnl_xmerge(txn->tw.reclaimed_pglist, loose);
- mdbx_trace("%s: append %u loose-pages to reclaimed-pages",
- dbg_prefix_mode, txn->tw.loose_count);
- }
+ ts_4 = ts_3 = latency ? mdbx_osal_monotime() : 0;
+ txn->mt_signature = 0;
+ mdbx_free(txn);
+ mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
+ rc = MDBX_SUCCESS;
+ goto provide_latency;
+ }
- /* filter-out list of dirty-pages from loose-pages */
- MDBX_dpl *const dl = txn->tw.dirtylist;
- unsigned w = 0;
- for (unsigned r = w; ++r <= dl->length;) {
- MDBX_page *dp = dl->items[r].ptr;
- mdbx_tassert(txn, dp->mp_flags == P_LOOSE || IS_MODIFIABLE(txn, dp));
- mdbx_tassert(txn, dpl_endpgno(dl, r) <= txn->mt_next_pgno);
- if ((dp->mp_flags & P_LOOSE) == 0) {
- if (++w != r)
- dl->items[w] = dl->items[r];
- } else {
- mdbx_tassert(txn, dp->mp_flags == P_LOOSE);
- if ((env->me_flags & MDBX_WRITEMAP) == 0)
- mdbx_dpage_free(env, dp, 1);
+ mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
+ (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
+ : txn->mt_env->me_options.dp_limit));
+ mdbx_cursors_eot(txn, false);
+ end_mode |= MDBX_END_EOTDONE;
+
+ if (txn->tw.dirtylist->length == 0 &&
+ (txn->mt_flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) {
+ for (int i = txn->mt_numdbs; --i >= 0;)
+ mdbx_tassert(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0);
+ rc = MDBX_SUCCESS;
+ goto done;
+ }
+
+ mdbx_debug("committing txn %" PRIaTXN " %p on mdbenv %p, root page %" PRIaPGNO
+ "/%" PRIaPGNO,
+ txn->mt_txnid, (void *)txn, (void *)env,
+ txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root);
+
+ /* Update DB root pointers */
+ if (txn->mt_numdbs > CORE_DBS) {
+ MDBX_cursor_couple couple;
+ MDBX_val data;
+ data.iov_len = sizeof(MDBX_db);
+
+ rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ for (MDBX_dbi i = CORE_DBS; i < txn->mt_numdbs; i++) {
+ if (txn->mt_dbistate[i] & DBI_DIRTY) {
+ if (unlikely(TXN_DBI_CHANGED(txn, i))) {
+ rc = MDBX_BAD_DBI;
+ goto fail;
}
+ MDBX_db *db = &txn->mt_dbs[i];
+ mdbx_debug("update main's entry for sub-db %u, mod_txnid %" PRIaTXN
+ " -> %" PRIaTXN,
+ i, db->md_mod_txnid, txn->mt_txnid);
+ db->md_mod_txnid = txn->mt_txnid;
+ data.iov_base = db;
+ WITH_CURSOR_TRACKING(couple.outer,
+ rc = mdbx_cursor_put(&couple.outer,
+ &txn->mt_dbxs[i].md_name,
+ &data, F_SUBDATA));
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
}
- mdbx_trace("%s: filtered-out loose-pages from %u -> %u dirty-pages",
- dbg_prefix_mode, dl->length, w);
- mdbx_tassert(txn, txn->tw.loose_count == dl->length - w);
- dpl_setlen(dl, w);
- dl->sorted = 0;
- txn->tw.dirtyroom += txn->tw.loose_count;
- mdbx_tassert(txn,
- txn->tw.dirtyroom + txn->tw.dirtylist->length ==
- (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
- : txn->mt_env->me_options.dp_limit));
- txn->tw.loose_pages = NULL;
- txn->tw.loose_count = 0;
-#if MDBX_ENABLE_REFUND
- txn->tw.loose_refund_wl = 0;
-#endif /* MDBX_ENABLE_REFUND */
}
+ }
- const unsigned amount = (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist);
- /* handle retired-list - store ones into single gc-record */
- if (retired_stored < MDBX_PNL_SIZE(txn->tw.retired_pages)) {
- if (unlikely(!retired_stored)) {
- /* Make sure last page of GC is touched and on retired-list */
- couple.outer.mc_flags &= ~C_RECLAIMING;
- rc = mdbx_page_search(&couple.outer, NULL,
- MDBX_PS_LAST | MDBX_PS_MODIFY);
- couple.outer.mc_flags |= C_RECLAIMING;
- if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_NOTFOUND)
- goto bailout;
- }
- /* Write to last page of GC */
- key.iov_len = sizeof(txn->mt_txnid);
- key.iov_base = &txn->mt_txnid;
- do {
- data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages);
- mdbx_prep_backlog(txn, &couple.outer, data.iov_len, &retired_stored);
- rc = mdbx_cursor_put(&couple.outer, &key, &data, MDBX_RESERVE);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- /* Retry if tw.retired_pages[] grew during the Put() */
- } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages));
+ ts_1 = latency ? mdbx_osal_monotime() : 0;
+ rc = mdbx_update_gc(txn);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+
+ txn->mt_dbs[FREE_DBI].md_mod_txnid = (txn->mt_dbistate[FREE_DBI] & DBI_DIRTY)
+ ? txn->mt_txnid
+ : txn->mt_dbs[FREE_DBI].md_mod_txnid;
+
+ txn->mt_dbs[MAIN_DBI].md_mod_txnid = (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY)
+ ? txn->mt_txnid
+ : txn->mt_dbs[MAIN_DBI].md_mod_txnid;
+
+ ts_2 = latency ? mdbx_osal_monotime() : 0;
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_audit_ex(txn, MDBX_PNL_SIZE(txn->tw.retired_pages), true);
+ const uint64_t audit_end = mdbx_osal_monotime();
+ audit_duration = mdbx_osal_monotime_to_16dot16(audit_end - ts_2);
+ ts_2 = audit_end;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ }
+
+ struct mdbx_iov_ctx ctx;
+ mdbx_iov_init(txn, &ctx);
+ rc = mdbx_txn_write(txn, &ctx);
+ if (likely(rc == MDBX_SUCCESS))
+ mdbx_iov_done(txn, &ctx);
+ /* TODO: use ctx.flush_begin & ctx.flush_end for range-sync */
+ ts_3 = latency ? mdbx_osal_monotime() : 0;
+
+ if (likely(rc == MDBX_SUCCESS)) {
+ const MDBX_meta *head = constmeta_prefer_last(env);
+ MDBX_meta meta;
+ memcpy(meta.mm_magic_and_version, head->mm_magic_and_version, 8);
+ meta.mm_extra_flags = head->mm_extra_flags;
+ meta.mm_validator_id = head->mm_validator_id;
+ meta.mm_extra_pagehdr = head->mm_extra_pagehdr;
+ unaligned_poke_u64(4, meta.mm_pages_retired,
+ unaligned_peek_u64(4, head->mm_pages_retired) +
+ MDBX_PNL_SIZE(txn->tw.retired_pages));
+ meta.mm_geo = txn->mt_geo;
+ meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
+ meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
+ meta.mm_canary = txn->mt_canary;
+ meta_set_txnid(env, &meta, txn->mt_txnid);
+
+ rc = mdbx_sync_locked(
+ env, env->me_flags | txn->mt_flags | MDBX_SHRINK_ALLOWED, &meta);
+ }
+ ts_4 = latency ? mdbx_osal_monotime() : 0;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ env->me_flags |= MDBX_FATAL_ERROR;
+ goto fail;
+ }
+
+ end_mode = MDBX_END_COMMITTED | MDBX_END_UPDATE | MDBX_END_EOTDONE;
+
+done:
+ rc = mdbx_txn_end(txn, end_mode);
+
+provide_latency:
+ if (latency) {
+ latency->audit = audit_duration;
+ latency->preparation =
+ ts_1 ? mdbx_osal_monotime_to_16dot16(ts_1 - ts_0) : 0;
+ latency->gc =
+ (ts_1 && ts_2) ? mdbx_osal_monotime_to_16dot16(ts_2 - ts_1) : 0;
+ latency->write =
+ (ts_2 && ts_3) ? mdbx_osal_monotime_to_16dot16(ts_3 - ts_2) : 0;
+ latency->sync =
+ (ts_3 && ts_4) ? mdbx_osal_monotime_to_16dot16(ts_4 - ts_3) : 0;
+ const uint64_t ts_5 = mdbx_osal_monotime();
+ latency->ending = ts_4 ? mdbx_osal_monotime_to_16dot16(ts_5 - ts_4) : 0;
+ latency->whole = mdbx_osal_monotime_to_16dot16(ts_5 - ts_0);
+ }
+ return rc;
+
+fail:
+ mdbx_txn_abort(txn);
+ goto provide_latency;
+}
- retired_stored = (unsigned)MDBX_PNL_SIZE(txn->tw.retired_pages);
- mdbx_pnl_sort(txn->tw.retired_pages, txn->mt_next_pgno);
- mdbx_assert(env, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages));
- memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len);
+static int mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta,
+ const MDBX_page *const page,
+ const unsigned meta_number,
+ unsigned *guess_pagesize) {
+ const uint64_t magic_and_version =
+ unaligned_peek_u64(4, &meta->mm_magic_and_version);
+ if (unlikely(magic_and_version != MDBX_DATA_MAGIC &&
+ magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
+ magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) {
+ mdbx_error("meta[%u] has invalid magic/version %" PRIx64, meta_number,
+ magic_and_version);
+ return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID
+ : MDBX_VERSION_MISMATCH;
+ }
- mdbx_trace("%s: put-retired #%u @ %" PRIaTXN, dbg_prefix_mode,
- retired_stored, txn->mt_txnid);
+ if (unlikely(page->mp_pgno != meta_number)) {
+ mdbx_error("meta[%u] has invalid pageno %" PRIaPGNO, meta_number,
+ page->mp_pgno);
+ return MDBX_INVALID;
+ }
- if (mdbx_log_enabled(MDBX_LOG_EXTRA)) {
- unsigned i = retired_stored;
- mdbx_debug_extra("PNL write txn %" PRIaTXN " root %" PRIaPGNO
- " num %u, PNL",
- txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i);
- for (; i; i--)
- mdbx_debug_extra_print(" %" PRIaPGNO, txn->tw.retired_pages[i]);
- mdbx_debug_extra_print("%s\n", ".");
- }
- if (unlikely(amount != MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) &&
- settled)) {
- mdbx_trace("%s: reclaimed-list changed %u -> %u, retry",
- dbg_prefix_mode, amount,
- (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
- goto retry /* rare case, but avoids GC fragmentation
- and one cycle. */
- ;
- }
- continue;
- }
+ if (unlikely(page->mp_flags != P_META)) {
+ mdbx_error("page #%u not a meta-page", meta_number);
+ return MDBX_INVALID;
+ }
- /* handle reclaimed and lost pages - merge and store both into gc */
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- mdbx_tassert(txn, txn->tw.loose_count == 0);
+ /* LY: check pagesize */
+ if (unlikely(!is_powerof2(meta->mm_psize) || meta->mm_psize < MIN_PAGESIZE ||
+ meta->mm_psize > MAX_PAGESIZE)) {
+ mdbx_warning("meta[%u] has invalid pagesize (%u), skip it", meta_number,
+ meta->mm_psize);
+ return is_powerof2(meta->mm_psize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID;
+ }
- mdbx_trace("%s", " >> reserving");
- if (mdbx_audit_enabled()) {
- rc = mdbx_audit_ex(txn, retired_stored, false);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- const unsigned left = amount - settled;
- mdbx_trace("%s: amount %u, settled %d, left %d, lifo-reclaimed-slots %u, "
- "reused-gc-slots %u",
- dbg_prefix_mode, amount, settled, (int)left,
- txn->tw.lifo_reclaimed
- ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
- : 0,
- reused_gc_slot);
- if (0 >= (int)left)
- break;
+ if (guess_pagesize && *guess_pagesize != meta->mm_psize) {
+ *guess_pagesize = meta->mm_psize;
+ mdbx_verbose("meta[%u] took pagesize %u", meta_number, meta->mm_psize);
+ }
- const unsigned prefer_max_scatter = 257;
- txnid_t reservation_gc_id;
- if (lifo) {
- if (txn->tw.lifo_reclaimed == nullptr) {
- txn->tw.lifo_reclaimed = mdbx_txl_alloc();
- if (unlikely(!txn->tw.lifo_reclaimed)) {
- rc = MDBX_ENOMEM;
- goto bailout;
- }
- }
- if ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <
- prefer_max_scatter &&
- left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
- reused_gc_slot) *
- env->me_maxgc_ov1page &&
- !dense_gc) {
- /* LY: need just a txn-id for save page list. */
- bool need_cleanup = false;
- txnid_t snap_oldest;
- retry_rid:
- couple.outer.mc_flags &= ~C_RECLAIMING;
- do {
- snap_oldest = mdbx_find_oldest(txn);
- rc =
- mdbx_page_alloc(&couple.outer, 0,
- MDBX_ALLOC_GC | MDBX_ALLOC_SLOT | MDBX_ALLOC_FAKE)
- .err;
- if (likely(rc == MDBX_SUCCESS)) {
- mdbx_trace("%s: took @%" PRIaTXN " from GC", dbg_prefix_mode,
- MDBX_PNL_LAST(txn->tw.lifo_reclaimed));
- need_cleanup = true;
- }
- } while (rc == MDBX_SUCCESS &&
- (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <
- prefer_max_scatter &&
- left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
- reused_gc_slot) *
- env->me_maxgc_ov1page);
- couple.outer.mc_flags |= C_RECLAIMING;
+ const txnid_t txnid = unaligned_peek_u64(4, &meta->mm_txnid_a);
+ if (unlikely(txnid != unaligned_peek_u64(4, &meta->mm_txnid_b))) {
+ mdbx_warning("meta[%u] not completely updated, skip it", meta_number);
+ return MDBX_RESULT_TRUE;
+ }
- if (likely(rc == MDBX_SUCCESS)) {
- mdbx_trace("%s: got enough from GC.", dbg_prefix_mode);
- continue;
- } else if (unlikely(rc != MDBX_NOTFOUND))
- /* LY: some troubles... */
- goto bailout;
+ /* LY: check signature as a checksum */
+ if (META_IS_STEADY(meta) &&
+ unlikely(unaligned_peek_u64(4, &meta->mm_datasync_sign) !=
+ meta_sign(meta))) {
+ mdbx_warning("meta[%u] has invalid steady-checksum (0x%" PRIx64
+ " != 0x%" PRIx64 "), skip it",
+ meta_number, unaligned_peek_u64(4, &meta->mm_datasync_sign),
+ meta_sign(meta));
+ return MDBX_RESULT_TRUE;
+ }
- if (MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) {
- if (need_cleanup) {
- mdbx_txl_sort(txn->tw.lifo_reclaimed);
- cleaned_gc_slot = 0;
- }
- gc_rid = MDBX_PNL_LAST(txn->tw.lifo_reclaimed);
- } else {
- mdbx_tassert(txn, txn->tw.last_reclaimed == 0);
- if (unlikely(mdbx_find_oldest(txn) != snap_oldest))
- /* should retry mdbx_page_alloc(MDBX_ALLOC_GC)
- * if the oldest reader changes since the last attempt */
- goto retry_rid;
- /* no reclaimable GC entries,
- * therefore no entries with ID < mdbx_find_oldest(txn) */
- txn->tw.last_reclaimed = gc_rid = snap_oldest - 1;
- mdbx_trace("%s: none recycled yet, set rid to @%" PRIaTXN,
- dbg_prefix_mode, gc_rid);
- }
+ mdbx_debug("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO
+ ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
+ " +%u -%u, txn_id %" PRIaTXN ", %s",
+ page->mp_pgno, meta->mm_dbs[MAIN_DBI].md_root,
+ meta->mm_dbs[FREE_DBI].md_root, meta->mm_geo.lower,
+ meta->mm_geo.next, meta->mm_geo.now, meta->mm_geo.upper,
+ pv2pages(meta->mm_geo.grow_pv), pv2pages(meta->mm_geo.shrink_pv),
+ txnid, mdbx_durable_str(meta));
- /* LY: GC is empty, will look any free txn-id in high2low order. */
- while (MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter &&
- left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
- reused_gc_slot) *
- env->me_maxgc_ov1page) {
- if (unlikely(gc_rid <= MIN_TXNID)) {
- if (unlikely(MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <=
- reused_gc_slot)) {
- mdbx_notice("** restart: reserve depleted (reused_gc_slot %u >= "
- "lifo_reclaimed %u" PRIaTXN,
- reused_gc_slot,
- (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
- goto retry;
- }
- break;
- }
+ if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) {
+ mdbx_warning("meta[%u] has invalid txnid %" PRIaTXN ", skip it",
+ meta_number, txnid);
+ return MDBX_RESULT_TRUE;
+ }
- mdbx_tassert(txn, gc_rid >= MIN_TXNID && gc_rid <= MAX_TXNID);
- --gc_rid;
- key.iov_base = &gc_rid;
- key.iov_len = sizeof(gc_rid);
- rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_SET_KEY);
- if (unlikely(rc == MDBX_SUCCESS)) {
- mdbx_debug("%s: GC's id %" PRIaTXN
- " is used, continue bottom-up search",
- dbg_prefix_mode, gc_rid);
- ++gc_rid;
- rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_FIRST);
- if (rc == MDBX_NOTFOUND) {
- mdbx_debug("%s: GC is empty (going dense-mode)", dbg_prefix_mode);
- dense_gc = true;
- break;
- }
- if (unlikely(rc != MDBX_SUCCESS ||
- key.iov_len != sizeof(mdbx_tid_t))) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- txnid_t gc_first = unaligned_peek_u64(4, key.iov_base);
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(gc_first < MIN_TXNID || gc_first > MAX_TXNID)) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- if (gc_first <= MIN_TXNID) {
- mdbx_debug("%s: no free GC's id(s) less than %" PRIaTXN
- " (going dense-mode)",
- dbg_prefix_mode, gc_rid);
- dense_gc = true;
- break;
- }
- gc_rid = gc_first - 1;
- }
+ /* LY: check min-pages value */
+ if (unlikely(meta->mm_geo.lower < MIN_PAGENO ||
+ meta->mm_geo.lower > MAX_PAGENO + 1)) {
+ mdbx_warning("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it",
+ meta_number, meta->mm_geo.lower);
+ return MDBX_INVALID;
+ }
- mdbx_assert(env, !dense_gc);
- rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, gc_rid);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ /* LY: check max-pages value */
+ if (unlikely(meta->mm_geo.upper < MIN_PAGENO ||
+ meta->mm_geo.upper > MAX_PAGENO + 1 ||
+ meta->mm_geo.upper < meta->mm_geo.lower)) {
+ mdbx_warning("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it",
+ meta_number, meta->mm_geo.upper);
+ return MDBX_INVALID;
+ }
- if (reused_gc_slot)
- /* rare case, but it is better to clear and re-create GC entries
- * with less fragmentation. */
- need_cleanup = true;
- else
- cleaned_gc_slot +=
- 1 /* mark cleanup is not needed for added slot. */;
+ /* LY: check last_pgno */
+ if (unlikely(meta->mm_geo.next < MIN_PAGENO ||
+ meta->mm_geo.next - 1 > MAX_PAGENO)) {
+ mdbx_warning("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it",
+ meta_number, meta->mm_geo.next);
+ return MDBX_CORRUPTED;
+ }
- mdbx_trace("%s: append @%" PRIaTXN
- " to lifo-reclaimed, cleaned-gc-slot = %u",
- dbg_prefix_mode, gc_rid, cleaned_gc_slot);
- }
+ /* LY: check filesize & used_bytes */
+ const uint64_t used_bytes = meta->mm_geo.next * (uint64_t)meta->mm_psize;
+ if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) {
+ /* Here could be a race with DB-shrinking performed by other process */
+ int err = mdbx_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) {
+ mdbx_warning("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64
+ "), skip it",
+ meta_number, used_bytes, env->me_dxb_mmap.filesize);
+ return MDBX_CORRUPTED;
+ }
+ }
+ if (unlikely(meta->mm_geo.next - 1 > MAX_PAGENO ||
+ used_bytes > MAX_MAPSIZE)) {
+ mdbx_warning("meta[%u] has too large used-space (%" PRIu64 "), skip it",
+ meta_number, used_bytes);
+ return MDBX_TOO_LARGE;
+ }
- if (need_cleanup || dense_gc) {
- if (cleaned_gc_slot)
- mdbx_trace(
- "%s: restart inner-loop to clear and re-create GC entries",
- dbg_prefix_mode);
- cleaned_gc_slot = 0;
- continue;
- }
+ /* LY: check mapsize limits */
+ pgno_t geo_lower = meta->mm_geo.lower;
+ uint64_t mapsize_min = geo_lower * (uint64_t)meta->mm_psize;
+ STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MAX_PAGESIZE);
+ STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
+ STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MIN_PAGESIZE % (4ul << 20) == 0);
+ if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) {
+ if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE &&
+ mapsize_min <= MAX_MAPSIZE64) {
+ mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO &&
+ used_bytes <= MAX_MAPSIZE);
+ mdbx_warning("meta[%u] has too large min-mapsize (%" PRIu64 "), "
+ "but size of used space still acceptable (%" PRIu64 ")",
+ meta_number, mapsize_min, used_bytes);
+ geo_lower = (pgno_t)((mapsize_min = MAX_MAPSIZE) / meta->mm_psize);
+ if (geo_lower > MAX_PAGENO + 1) {
+ geo_lower = MAX_PAGENO + 1;
+ mapsize_min = geo_lower * (uint64_t)meta->mm_psize;
}
-
- const unsigned i =
- (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot;
- mdbx_tassert(txn, i > 0 && i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
- reservation_gc_id = txn->tw.lifo_reclaimed[i];
- mdbx_trace("%s: take @%" PRIaTXN " from lifo-reclaimed[%u]",
- dbg_prefix_mode, reservation_gc_id, i);
+ mdbx_warning("meta[%u] consider get-%s pageno is %" PRIaPGNO
+ " instead of wrong %" PRIaPGNO
+ ", will be corrected on next commit(s)",
+ meta_number, "lower", geo_lower, meta->mm_geo.lower);
+ meta->mm_geo.lower = geo_lower;
} else {
- mdbx_tassert(txn, txn->tw.lifo_reclaimed == NULL);
- if (unlikely(gc_rid == 0)) {
- gc_rid = mdbx_find_oldest(txn) - 1;
- rc = mdbx_cursor_get(&couple.outer, &key, NULL, MDBX_FIRST);
- if (rc == MDBX_SUCCESS) {
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(key.iov_len != sizeof(txnid_t))) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- txnid_t gc_first = unaligned_peek_u64(4, key.iov_base);
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(gc_first < MIN_TXNID || gc_first > MAX_TXNID)) {
- rc = MDBX_CORRUPTED;
- goto bailout;
- }
- if (gc_rid >= gc_first)
- gc_rid = gc_first - 1;
- if (unlikely(gc_rid == 0)) {
- mdbx_error("%s", "** no GC tail-space to store (going dense-mode)");
- dense_gc = true;
- goto retry;
- }
- } else if (rc != MDBX_NOTFOUND)
- goto bailout;
- txn->tw.last_reclaimed = gc_rid;
- cleaned_gc_id = gc_rid + 1;
- }
- reservation_gc_id = gc_rid--;
- mdbx_trace("%s: take @%" PRIaTXN " from head-gc-id", dbg_prefix_mode,
- reservation_gc_id);
+ mdbx_warning("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it",
+ meta_number, mapsize_min);
+ return MDBX_VERSION_MISMATCH;
}
- ++reused_gc_slot;
-
- unsigned chunk = left;
- if (unlikely(chunk > env->me_maxgc_ov1page)) {
- const unsigned avail_gc_slots =
- txn->tw.lifo_reclaimed
- ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) -
- reused_gc_slot + 1
- : (gc_rid < INT16_MAX) ? (unsigned)gc_rid
- : INT16_MAX;
- if (avail_gc_slots > 1) {
- if (chunk < env->me_maxgc_ov1page * 2)
- chunk /= 2;
- else {
- const unsigned threshold =
- env->me_maxgc_ov1page * ((avail_gc_slots < prefer_max_scatter)
- ? avail_gc_slots
- : prefer_max_scatter);
- if (left < threshold)
- chunk = env->me_maxgc_ov1page;
- else {
- const unsigned tail = left - threshold + env->me_maxgc_ov1page + 1;
- unsigned span = 1;
- unsigned avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) /
- sizeof(pgno_t)) /* - 1 + span */;
- if (tail > avail) {
- for (unsigned i = amount - span; i > 0; --i) {
- if (MDBX_PNL_ASCENDING
- ? (txn->tw.reclaimed_pglist[i] + span)
- : (txn->tw.reclaimed_pglist[i] - span) ==
- txn->tw.reclaimed_pglist[i + span]) {
- span += 1;
- avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) /
- sizeof(pgno_t)) -
- 1 + span;
- if (avail >= tail)
- break;
- }
- }
- }
+ }
- chunk = (avail >= tail) ? tail - span
- : (avail_gc_slots > 3 &&
- reused_gc_slot < prefer_max_scatter - 3)
- ? avail - span
- : tail;
- }
- }
- }
+ pgno_t geo_upper = meta->mm_geo.upper;
+ uint64_t mapsize_max = geo_upper * (uint64_t)meta->mm_psize;
+ STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
+ if (unlikely(mapsize_max > MAX_MAPSIZE ||
+ (MAX_PAGENO + 1) <
+ ceil_powerof2((size_t)mapsize_max, env->me_os_psize) /
+ (size_t)meta->mm_psize)) {
+ if (mapsize_max > MAX_MAPSIZE64) {
+ mdbx_warning("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it",
+ meta_number, mapsize_max);
+ return MDBX_VERSION_MISMATCH;
}
- mdbx_tassert(txn, chunk > 0);
+ /* allow to open large DB from a 32-bit environment */
+ mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO &&
+ used_bytes <= MAX_MAPSIZE);
+ mdbx_warning("meta[%u] has too large max-mapsize (%" PRIu64 "), "
+ "but size of used space still acceptable (%" PRIu64 ")",
+ meta_number, mapsize_max, used_bytes);
+ geo_upper = (pgno_t)((mapsize_max = MAX_MAPSIZE) / meta->mm_psize);
+ if (geo_upper > MAX_PAGENO + 1) {
+ geo_upper = MAX_PAGENO + 1;
+ mapsize_max = geo_upper * (uint64_t)meta->mm_psize;
+ }
+ mdbx_warning("meta[%u] consider get-%s pageno is %" PRIaPGNO
+ " instead of wrong %" PRIaPGNO
+ ", will be corrected on next commit(s)",
+ meta_number, "upper", geo_upper, meta->mm_geo.upper);
+ meta->mm_geo.upper = geo_upper;
+ }
- mdbx_trace("%s: gc_rid %" PRIaTXN ", reused_gc_slot %u, reservation-id "
- "%" PRIaTXN,
- dbg_prefix_mode, gc_rid, reused_gc_slot, reservation_gc_id);
+ /* LY: check and silently put mm_geo.now into [geo.lower...geo.upper].
+ *
+ * Copy-with-compaction by previous version of libmdbx could produce DB-file
+ * less than meta.geo.lower bound, in case actual filling is low or no data
+ * at all. This is not a problem as there is no damage or loss of data.
+ * Therefore it is better not to consider such situation as an error, but
+ * silently correct it. */
+ pgno_t geo_now = meta->mm_geo.now;
+ if (geo_now < geo_lower)
+ geo_now = geo_lower;
+ if (geo_now > geo_upper && meta->mm_geo.next <= geo_upper)
+ geo_now = geo_upper;
- mdbx_trace("%s: chunk %u, gc-per-ovpage %u", dbg_prefix_mode, chunk,
- env->me_maxgc_ov1page);
+ if (unlikely(meta->mm_geo.next > geo_now)) {
+ mdbx_warning("meta[%u] next-pageno (%" PRIaPGNO
+ ") is beyond end-pgno (%" PRIaPGNO "), skip it",
+ meta_number, meta->mm_geo.next, geo_now);
+ return MDBX_CORRUPTED;
+ }
+ if (meta->mm_geo.now != geo_now) {
+ mdbx_warning("meta[%u] consider geo-%s pageno is %" PRIaPGNO
+ " instead of wrong %" PRIaPGNO
+ ", will be corrected on next commit(s)",
+ meta_number, "now", geo_now, meta->mm_geo.now);
+ meta->mm_geo.now = geo_now;
+ }
- mdbx_tassert(txn, reservation_gc_id < env->me_lck->mti_oldest_reader.weak);
- if (unlikely(
- reservation_gc_id < MIN_TXNID ||
- reservation_gc_id >=
- atomic_load64(&env->me_lck->mti_oldest_reader, mo_Relaxed))) {
- mdbx_error("** internal error (reservation_gc_id %" PRIaTXN ")",
- reservation_gc_id);
- rc = MDBX_PROBLEM;
- goto bailout;
+ /* GC */
+ if (meta->mm_dbs[FREE_DBI].md_root == P_INVALID) {
+ if (unlikely(meta->mm_dbs[FREE_DBI].md_branch_pages ||
+ meta->mm_dbs[FREE_DBI].md_depth ||
+ meta->mm_dbs[FREE_DBI].md_entries ||
+ meta->mm_dbs[FREE_DBI].md_leaf_pages ||
+ meta->mm_dbs[FREE_DBI].md_overflow_pages)) {
+ mdbx_warning("meta[%u] has false-empty %s, skip it", meta_number, "GC");
+ return MDBX_CORRUPTED;
}
+ } else if (unlikely(meta->mm_dbs[FREE_DBI].md_root >= meta->mm_geo.next)) {
+ mdbx_warning("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it",
+ meta_number, "GC", meta->mm_dbs[FREE_DBI].md_root);
+ return MDBX_CORRUPTED;
+ }
- key.iov_len = sizeof(reservation_gc_id);
- key.iov_base = &reservation_gc_id;
- data.iov_len = (chunk + 1) * sizeof(pgno_t);
- mdbx_trace("%s: reserve %u [%u...%u) @%" PRIaTXN, dbg_prefix_mode, chunk,
- settled + 1, settled + chunk + 1, reservation_gc_id);
- mdbx_prep_backlog(txn, &couple.outer, data.iov_len, nullptr);
- rc = mdbx_cursor_put(&couple.outer, &key, &data,
- MDBX_RESERVE | MDBX_NOOVERWRITE);
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
-
- clean_reserved_gc_pnl(env, data);
- settled += chunk;
- mdbx_trace("%s: settled %u (+%u), continue", dbg_prefix_mode, settled,
- chunk);
-
- if (txn->tw.lifo_reclaimed &&
- unlikely(amount < MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)) &&
- (loop < 5 || MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) - amount >
- env->me_maxgc_ov1page)) {
- mdbx_notice("** restart: reclaimed-list growth %u -> %u", amount,
- (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
- goto retry;
+ /* MainDB */
+ if (meta->mm_dbs[MAIN_DBI].md_root == P_INVALID) {
+ if (unlikely(meta->mm_dbs[MAIN_DBI].md_branch_pages ||
+ meta->mm_dbs[MAIN_DBI].md_depth ||
+ meta->mm_dbs[MAIN_DBI].md_entries ||
+ meta->mm_dbs[MAIN_DBI].md_leaf_pages ||
+ meta->mm_dbs[MAIN_DBI].md_overflow_pages)) {
+ mdbx_warning("meta[%u] has false-empty %s", meta_number, "MainDB");
+ return MDBX_CORRUPTED;
}
+ } else if (unlikely(meta->mm_dbs[MAIN_DBI].md_root >= meta->mm_geo.next)) {
+ mdbx_warning("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it",
+ meta_number, "MainDB", meta->mm_dbs[MAIN_DBI].md_root);
+ return MDBX_CORRUPTED;
+ }
- continue;
+ if (unlikely(meta->mm_dbs[FREE_DBI].md_mod_txnid > txnid)) {
+ mdbx_warning("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it",
+ meta_number, meta->mm_dbs[FREE_DBI].md_mod_txnid, "GC");
+ return MDBX_CORRUPTED;
}
- mdbx_tassert(
- txn,
- cleaned_gc_slot ==
- (txn->tw.lifo_reclaimed ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) : 0));
+ if (unlikely(meta->mm_dbs[MAIN_DBI].md_mod_txnid > txnid)) {
+ mdbx_warning("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it",
+ meta_number, meta->mm_dbs[MAIN_DBI].md_mod_txnid, "MainDB");
+ return MDBX_CORRUPTED;
+ }
- mdbx_trace("%s", " >> filling");
- /* Fill in the reserved records */
- filled_gc_slot =
- txn->tw.lifo_reclaimed
- ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot
- : reused_gc_slot;
- rc = MDBX_SUCCESS;
- mdbx_tassert(txn,
- mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
- txn->mt_next_pgno - MDBX_ENABLE_REFUND));
- mdbx_tassert(txn, mdbx_dirtylist_check(txn));
- if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)) {
- MDBX_val key, data;
- key.iov_len = data.iov_len = 0; /* avoid MSVC warning */
- key.iov_base = data.iov_base = NULL;
+ return MDBX_SUCCESS;
+}
- const unsigned amount = MDBX_PNL_SIZE(txn->tw.reclaimed_pglist);
- unsigned left = amount;
- if (txn->tw.lifo_reclaimed == nullptr) {
- mdbx_tassert(txn, lifo == 0);
- rc = mdbx_cursor_first(&couple.outer, &key, &data);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- } else {
- mdbx_tassert(txn, lifo != 0);
- }
+static int mdbx_validate_meta_copy(MDBX_env *env, const MDBX_meta *meta,
+ MDBX_meta *dest) {
+ *dest = *meta;
+ return mdbx_validate_meta(env, dest, data_page(meta),
+ bytes2pgno(env, (uint8_t *)meta - env->me_map),
+ nullptr);
+}
- while (true) {
- txnid_t fill_gc_id;
- mdbx_trace("%s: left %u of %u", dbg_prefix_mode, left,
- (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist));
- if (txn->tw.lifo_reclaimed == nullptr) {
- mdbx_tassert(txn, lifo == 0);
- fill_gc_id = unaligned_peek_u64(4, key.iov_base);
- if (filled_gc_slot-- == 0 || fill_gc_id > txn->tw.last_reclaimed) {
- mdbx_notice(
- "** restart: reserve depleted (filled_slot %u, fill_id %" PRIaTXN
- " > last_reclaimed %" PRIaTXN,
- filled_gc_slot, fill_gc_id, txn->tw.last_reclaimed);
- goto retry;
- }
- } else {
- mdbx_tassert(txn, lifo != 0);
- if (++filled_gc_slot >
- (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) {
- mdbx_notice("** restart: reserve depleted (filled_gc_slot %u > "
- "lifo_reclaimed %u" PRIaTXN,
- filled_gc_slot,
- (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
- goto retry;
- }
- fill_gc_id = txn->tw.lifo_reclaimed[filled_gc_slot];
- mdbx_trace("%s: seek-reservation @%" PRIaTXN " at lifo_reclaimed[%u]",
- dbg_prefix_mode, fill_gc_id, filled_gc_slot);
- key.iov_base = &fill_gc_id;
- key.iov_len = sizeof(fill_gc_id);
- rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_SET_KEY);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- mdbx_tassert(txn, cleaned_gc_slot ==
- (txn->tw.lifo_reclaimed
- ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
- : 0));
- mdbx_tassert(txn, fill_gc_id > 0 &&
- fill_gc_id < env->me_lck->mti_oldest_reader.weak);
- key.iov_base = &fill_gc_id;
- key.iov_len = sizeof(fill_gc_id);
+/* Read the environment parameters of a DB environment
+ * before mapping it into memory. */
+__cold static int mdbx_read_header(MDBX_env *env, MDBX_meta *dest,
+ const int lck_exclusive,
+ const mdbx_mode_t mode_bits) {
+ int rc = mdbx_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- mdbx_tassert(txn, data.iov_len >= sizeof(pgno_t) * 2);
- couple.outer.mc_flags |= C_GCFREEZE;
- unsigned chunk = (unsigned)(data.iov_len / sizeof(pgno_t)) - 1;
- if (unlikely(chunk > left)) {
- mdbx_trace("%s: chunk %u > left %u, @%" PRIaTXN, dbg_prefix_mode, chunk,
- left, fill_gc_id);
- if ((loop < 5 && chunk - left > loop / 2) ||
- chunk - left > env->me_maxgc_ov1page) {
- data.iov_len = (left + 1) * sizeof(pgno_t);
- if (loop < 7)
- couple.outer.mc_flags &= ~C_GCFREEZE;
- }
- chunk = left;
- }
- rc = mdbx_cursor_put(&couple.outer, &key, &data,
- MDBX_CURRENT | MDBX_RESERVE);
- couple.outer.mc_flags &= ~C_GCFREEZE;
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- clean_reserved_gc_pnl(env, data);
+ memset(dest, 0, sizeof(MDBX_meta));
+ unaligned_poke_u64(4, dest->mm_datasync_sign, MDBX_DATASIGN_WEAK);
+ rc = MDBX_CORRUPTED;
- if (unlikely(txn->tw.loose_count ||
- amount != MDBX_PNL_SIZE(txn->tw.reclaimed_pglist))) {
- mdbx_notice("** restart: reclaimed-list growth (%u -> %u, loose +%u)",
- amount, MDBX_PNL_SIZE(txn->tw.reclaimed_pglist),
- txn->tw.loose_count);
- goto retry;
- }
- if (unlikely(txn->tw.lifo_reclaimed
- ? cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
- : cleaned_gc_id < txn->tw.last_reclaimed)) {
- mdbx_notice("%s", "** restart: reclaimed-slots changed");
- goto retry;
- }
- if (unlikely(retired_stored != MDBX_PNL_SIZE(txn->tw.retired_pages))) {
- mdbx_tassert(txn,
- retired_stored < MDBX_PNL_SIZE(txn->tw.retired_pages));
- mdbx_notice("** restart: retired-list growth (%u -> %u)",
- retired_stored, MDBX_PNL_SIZE(txn->tw.retired_pages));
- goto retry;
- }
+ /* Read twice all meta pages so we can find the latest one. */
+ unsigned loop_limit = NUM_METAS * 2;
+ /* We don't know the page size on first time. So, just guess it. */
+ unsigned guess_pagesize = 0;
+ for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) {
+ const unsigned meta_number = loop_count % NUM_METAS;
+ const unsigned offset = (guess_pagesize ? guess_pagesize
+ : (loop_count > NUM_METAS) ? env->me_psize
+ : env->me_os_psize) *
+ meta_number;
- pgno_t *dst = data.iov_base;
- *dst++ = chunk;
- pgno_t *src = MDBX_PNL_BEGIN(txn->tw.reclaimed_pglist) + left - chunk;
- memcpy(dst, src, chunk * sizeof(pgno_t));
- pgno_t *from = src, *to = src + chunk;
- mdbx_trace("%s: fill %u [ %u:%" PRIaPGNO "...%u:%" PRIaPGNO
- "] @%" PRIaTXN,
- dbg_prefix_mode, chunk,
- (unsigned)(from - txn->tw.reclaimed_pglist), from[0],
- (unsigned)(to - txn->tw.reclaimed_pglist), to[-1], fill_gc_id);
+ char buffer[MIN_PAGESIZE];
+ unsigned retryleft = 42;
+ while (1) {
+ mdbx_trace("reading meta[%d]: offset %u, bytes %u, retry-left %u",
+ meta_number, offset, MIN_PAGESIZE, retryleft);
+ int err = mdbx_pread(env->me_lazy_fd, buffer, MIN_PAGESIZE, offset);
+ if (err != MDBX_SUCCESS) {
+ if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 &&
+ env->me_dxb_mmap.filesize == 0 &&
+ mode_bits /* non-zero for DB creation */ != 0)
+ mdbx_notice("read meta: empty file (%d, %s)", err,
+ mdbx_strerror(err));
+ else
+ mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err,
+ mdbx_strerror(err));
+ return err;
+ }
- left -= chunk;
- if (mdbx_audit_enabled()) {
- rc = mdbx_audit_ex(txn, retired_stored + amount - left, true);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ char again[MIN_PAGESIZE];
+ err = mdbx_pread(env->me_lazy_fd, again, MIN_PAGESIZE, offset);
+ if (err != MDBX_SUCCESS) {
+ mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err,
+ mdbx_strerror(err));
+ return err;
}
- if (left == 0) {
- rc = MDBX_SUCCESS;
+
+ if (memcmp(buffer, again, MIN_PAGESIZE) == 0 || --retryleft == 0)
break;
- }
- if (txn->tw.lifo_reclaimed == nullptr) {
- mdbx_tassert(txn, lifo == 0);
- rc = mdbx_cursor_next(&couple.outer, &key, &data, MDBX_NEXT);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- } else {
- mdbx_tassert(txn, lifo != 0);
- }
+ mdbx_verbose("meta[%u] was updated, re-read it", meta_number);
}
- }
- mdbx_tassert(txn, rc == MDBX_SUCCESS);
- if (unlikely(txn->tw.loose_count != 0)) {
- mdbx_notice("** restart: got %u loose pages", txn->tw.loose_count);
- goto retry;
+ if (!retryleft) {
+ mdbx_error("meta[%u] is too volatile, skip it", meta_number);
+ continue;
+ }
+
+ MDBX_page *const page = (MDBX_page *)buffer;
+ MDBX_meta *const meta = page_meta(page);
+ rc = mdbx_validate_meta(env, meta, page, meta_number, &guess_pagesize);
+ if (rc != MDBX_SUCCESS)
+ continue;
+
+ if ((env->me_stuck_meta < 0)
+ ? meta_ot(meta_bootid_match(meta) ? prefer_last : prefer_steady,
+ env, dest, meta)
+ : (meta_number == (unsigned)env->me_stuck_meta)) {
+ *dest = *meta;
+ if (!lck_exclusive && !META_IS_STEADY(dest))
+ loop_limit += 1; /* LY: should re-read to hush race with update */
+ mdbx_verbose("latch meta[%u]", meta_number);
+ }
}
- if (unlikely(filled_gc_slot !=
- (txn->tw.lifo_reclaimed
- ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)
- : 0))) {
- const bool will_retry = loop < 9;
- mdbx_notice("** %s: reserve excess (filled-slot %u, loop %u)",
- will_retry ? "restart" : "ignore", filled_gc_slot, loop);
- if (will_retry)
- goto retry;
+ if (dest->mm_psize == 0 ||
+ (env->me_stuck_meta < 0 &&
+ !(META_IS_STEADY(dest) ||
+ meta_weak_acceptable(env, dest, lck_exclusive)))) {
+ mdbx_error("%s", "no usable meta-pages, database is corrupted");
+ if (rc == MDBX_SUCCESS) {
+ /* TODO: try to restore the database by fully checking b-tree structure
+ * for the each meta page, if the corresponding option was given */
+ return MDBX_CORRUPTED;
+ }
+ return rc;
}
- mdbx_tassert(txn,
- txn->tw.lifo_reclaimed == NULL ||
- cleaned_gc_slot == MDBX_PNL_SIZE(txn->tw.lifo_reclaimed));
-
-bailout:
- txn->mt_cursors[FREE_DBI] = couple.outer.mc_next;
-
-bailout_notracking:
- MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) = 0;
- mdbx_trace("<<< %u loops, rc = %d", loop, rc);
- return rc;
+ return MDBX_SUCCESS;
}
-static int mdbx_txn_write(MDBX_txn *txn, struct mdbx_iov_ctx *ctx) {
- MDBX_dpl *const dl =
- (txn->mt_flags & MDBX_WRITEMAP) ? txn->tw.dirtylist : mdbx_dpl_sort(txn);
- int rc = MDBX_SUCCESS;
- unsigned r, w;
- for (w = 0, r = 1; r <= dl->length; ++r) {
- MDBX_page *dp = dl->items[r].ptr;
- if (dp->mp_flags & P_LOOSE) {
- dl->items[++w] = dl->items[r];
- continue;
- }
- unsigned npages = dpl_npages(dl, r);
- rc = iov_page(txn, ctx, dp, npages);
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- }
+__cold static MDBX_page *mdbx_meta_model(const MDBX_env *env, MDBX_page *model,
+ unsigned num) {
+ mdbx_ensure(env, is_powerof2(env->me_psize));
+ mdbx_ensure(env, env->me_psize >= MIN_PAGESIZE);
+ mdbx_ensure(env, env->me_psize <= MAX_PAGESIZE);
+ mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE);
+ mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE);
+ mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower);
+ mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper);
- if (ctx->iov_items)
- rc = mdbx_iov_write(txn, ctx);
+ memset(model, 0, env->me_psize);
+ model->mp_pgno = num;
+ model->mp_flags = P_META;
+ MDBX_meta *const model_meta = page_meta(model);
+ unaligned_poke_u64(4, model_meta->mm_magic_and_version, MDBX_DATA_MAGIC);
- while (r <= dl->length)
- dl->items[++w] = dl->items[r++];
+ model_meta->mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower);
+ model_meta->mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper);
+ model_meta->mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow));
+ model_meta->mm_geo.shrink_pv =
+ pages2pv(bytes2pgno(env, env->me_dbgeo.shrink));
+ model_meta->mm_geo.now = bytes2pgno(env, env->me_dbgeo.now);
+ model_meta->mm_geo.next = NUM_METAS;
- dl->sorted = dpl_setlen(dl, w);
- txn->tw.dirtyroom += r - 1 - w;
- mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
- (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
- : txn->mt_env->me_options.dp_limit));
- return rc;
+ mdbx_ensure(env, model_meta->mm_geo.lower >= MIN_PAGENO);
+ mdbx_ensure(env, model_meta->mm_geo.upper <= MAX_PAGENO + 1);
+ mdbx_ensure(env, model_meta->mm_geo.now >= model_meta->mm_geo.lower);
+ mdbx_ensure(env, model_meta->mm_geo.now <= model_meta->mm_geo.upper);
+ mdbx_ensure(env, model_meta->mm_geo.next >= MIN_PAGENO);
+ mdbx_ensure(env, model_meta->mm_geo.next <= model_meta->mm_geo.now);
+ mdbx_ensure(env, model_meta->mm_geo.grow_pv ==
+ pages2pv(pv2pages(model_meta->mm_geo.grow_pv)));
+ mdbx_ensure(env, model_meta->mm_geo.shrink_pv ==
+ pages2pv(pv2pages(model_meta->mm_geo.shrink_pv)));
+
+ model_meta->mm_psize = env->me_psize;
+ model_meta->mm_dbs[FREE_DBI].md_flags = MDBX_INTEGERKEY;
+ model_meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
+ model_meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
+ meta_set_txnid(env, model_meta, MIN_TXNID + num);
+ unaligned_poke_u64(4, model_meta->mm_datasync_sign, meta_sign(model_meta));
+ mdbx_assert(env, meta_checktxnid(env, model_meta, true));
+ return (MDBX_page *)((uint8_t *)model + env->me_psize);
}
-/* Check txn and dbi arguments to a function */
-static __always_inline bool check_dbi(MDBX_txn *txn, MDBX_dbi dbi,
- unsigned validity) {
- if (likely(dbi < txn->mt_numdbs))
- return likely((txn->mt_dbistate[dbi] & validity) &&
- !TXN_DBI_CHANGED(txn, dbi) &&
- (txn->mt_dbxs[dbi].md_name.iov_base || dbi < CORE_DBS));
+/* Fill in most of the zeroed meta-pages for an empty database environment.
+ * Return pointer to recently (head) meta-page. */
+__cold static MDBX_meta *mdbx_init_metas(const MDBX_env *env, void *buffer) {
+ MDBX_page *page0 = (MDBX_page *)buffer;
+ MDBX_page *page1 = mdbx_meta_model(env, page0, 0);
+ MDBX_page *page2 = mdbx_meta_model(env, page1, 1);
+ mdbx_meta_model(env, page2, 2);
+ mdbx_assert(env, !meta_eq(env, page_meta(page0), page_meta(page1)));
+ mdbx_assert(env, !meta_eq(env, page_meta(page1), page_meta(page2)));
+ mdbx_assert(env, !meta_eq(env, page_meta(page2), page_meta(page0)));
+ return page_meta(page2);
+}
- return dbi_import(txn, dbi);
+#if MDBX_ENABLE_MADVISE && !(defined(_WIN32) || defined(_WIN64))
+static size_t mdbx_madvise_threshold(const MDBX_env *env,
+ const size_t largest_bytes) {
+ /* TODO: use options */
+ const unsigned factor = 9;
+ const size_t threshold = (largest_bytes < (65536ul << factor))
+ ? 65536 /* minimal threshold */
+ : (largest_bytes > (MEGABYTE * 4 << factor))
+ ? MEGABYTE * 4 /* maximal threshold */
+ : largest_bytes >> factor;
+ return bytes_align2os_bytes(env, threshold);
}
+#endif /* MDBX_ENABLE_MADVISE */
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); }
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+static int mdbx_sync_locked(MDBX_env *env, unsigned flags,
+ MDBX_meta *const pending) {
+ mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0);
+ const MDBX_meta *const meta0 = METAPAGE(env, 0);
+ const MDBX_meta *const meta1 = METAPAGE(env, 1);
+ const MDBX_meta *const meta2 = METAPAGE(env, 2);
+ const MDBX_meta *const head = constmeta_prefer_last(env);
+ int rc;
-/* Merge child txn into parent */
-static __inline void mdbx_txn_merge(MDBX_txn *const parent, MDBX_txn *const txn,
- const unsigned parent_retired_len) {
- MDBX_dpl *const src = mdbx_dpl_sort(txn);
+ mdbx_assert(env, meta_eq_mask(env) == 0);
+ mdbx_assert(env,
+ pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS));
+ mdbx_assert(env, (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0);
+ mdbx_assert(env, pending->mm_geo.next <= pending->mm_geo.now);
- /* Remove refunded pages from parent's dirty list */
- MDBX_dpl *const dst = mdbx_dpl_sort(parent);
- if (MDBX_ENABLE_REFUND) {
- unsigned n = dst->length;
- while (n && dst->items[n].pgno >= parent->mt_next_pgno) {
- if (!(txn->mt_env->me_flags & MDBX_WRITEMAP)) {
- MDBX_page *dp = dst->items[n].ptr;
- mdbx_dpage_free(txn->mt_env, dp, dpl_npages(dst, n));
+ if (flags & MDBX_SAFE_NOSYNC) {
+ /* Check auto-sync conditions */
+ const pgno_t autosync_threshold =
+ atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
+ const uint64_t autosync_period =
+ atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
+ if ((autosync_threshold &&
+ atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed) >=
+ autosync_threshold) ||
+ (autosync_period &&
+ mdbx_osal_monotime() -
+ atomic_load64(&env->me_lck->mti_sync_timestamp, mo_Relaxed) >=
+ autosync_period))
+ flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; /* force steady */
+ }
+
+ pgno_t shrink = 0;
+ if (flags & MDBX_SHRINK_ALLOWED) {
+ /* LY: check conditions to discard unused pages */
+ const pgno_t largest_pgno = mdbx_find_largest(
+ env, (head->mm_geo.next > pending->mm_geo.next) ? head->mm_geo.next
+ : pending->mm_geo.next);
+ mdbx_assert(env, largest_pgno >= NUM_METAS);
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ const pgno_t edge = env->me_poison_edge;
+ if (edge > largest_pgno) {
+ env->me_poison_edge = largest_pgno;
+ VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, largest_pgno),
+ pgno2bytes(env, edge - largest_pgno));
+ MDBX_ASAN_POISON_MEMORY_REGION(env->me_map +
+ pgno2bytes(env, largest_pgno),
+ pgno2bytes(env, edge - largest_pgno));
+ }
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+#if MDBX_ENABLE_MADVISE && \
+ (defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED))
+ const size_t largest_bytes = pgno2bytes(env, largest_pgno);
+ /* threshold to avoid unreasonable frequent madvise() calls */
+ const size_t madvise_threshold = mdbx_madvise_threshold(env, largest_bytes);
+ const size_t discard_edge_bytes = bytes_align2os_bytes(
+ env, ((MDBX_RDONLY &
+ (env->me_lck_mmap.lck ? env->me_lck_mmap.lck->mti_envmode.weak
+ : env->me_flags))
+ ? largest_bytes
+ : largest_bytes + madvise_threshold));
+ const pgno_t discard_edge_pgno = bytes2pgno(env, discard_edge_bytes);
+ const pgno_t prev_discarded_pgno =
+ atomic_load32(&env->me_lck->mti_discarded_tail, mo_Relaxed);
+ if (prev_discarded_pgno >=
+ discard_edge_pgno + bytes2pgno(env, madvise_threshold)) {
+ mdbx_notice("open-MADV_%s %u..%u", "DONTNEED", largest_pgno,
+ prev_discarded_pgno);
+ atomic_store32(&env->me_lck->mti_discarded_tail, discard_edge_pgno,
+ mo_Relaxed);
+ const size_t prev_discarded_bytes =
+ ceil_powerof2(pgno2bytes(env, prev_discarded_pgno), env->me_os_psize);
+ mdbx_ensure(env, prev_discarded_bytes > discard_edge_bytes);
+#if defined(MADV_DONTNEED)
+ int advise = MADV_DONTNEED;
+#if defined(MADV_FREE) && \
+ 0 /* MADV_FREE works for only anonymous vma at the moment */
+ if ((env->me_flags & MDBX_WRITEMAP) &&
+ mdbx_linux_kernel_version > 0x04050000)
+ advise = MADV_FREE;
+#endif /* MADV_FREE */
+ int err = madvise(env->me_map + discard_edge_bytes,
+ prev_discarded_bytes - discard_edge_bytes, advise)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+#else
+ int err = ignore_enosys(posix_madvise(
+ env->me_map + discard_edge_bytes,
+ prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED));
+#endif
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+ }
+#endif /* MDBX_ENABLE_MADVISE && (MADV_DONTNEED || POSIX_MADV_DONTNEED) */
+
+ /* LY: check conditions to shrink datafile */
+ const pgno_t backlog_gap = 3 + pending->mm_dbs[FREE_DBI].md_depth * 3;
+ pgno_t shrink_step = 0;
+ if (pending->mm_geo.shrink_pv &&
+ pending->mm_geo.now - pending->mm_geo.next >
+ (shrink_step = pv2pages(pending->mm_geo.shrink_pv)) + backlog_gap) {
+ if (pending->mm_geo.now > largest_pgno &&
+ pending->mm_geo.now - largest_pgno > shrink_step + backlog_gap) {
+ pgno_t grow_step = 0;
+ const pgno_t aligner =
+ pending->mm_geo.grow_pv
+ ? (grow_step = pv2pages(pending->mm_geo.grow_pv))
+ : shrink_step;
+ const pgno_t with_backlog_gap = largest_pgno + backlog_gap;
+ const pgno_t aligned = pgno_align2os_pgno(
+ env, with_backlog_gap + aligner - with_backlog_gap % aligner);
+ const pgno_t bottom =
+ (aligned > pending->mm_geo.lower) ? aligned : pending->mm_geo.lower;
+ if (pending->mm_geo.now > bottom) {
+ if (META_IS_STEADY(meta_prefer_steady(env)))
+ /* force steady, but only if steady-checkpoint is present */
+ flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED;
+ shrink = pending->mm_geo.now - bottom;
+ pending->mm_geo.now = bottom;
+ if (unlikely(constmeta_txnid(env, head) ==
+ unaligned_peek_u64(4, pending->mm_txnid_a))) {
+ const txnid_t txnid =
+ safe64_txnid_next(unaligned_peek_u64(4, pending->mm_txnid_a));
+ mdbx_notice("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN,
+ unaligned_peek_u64(4, pending->mm_txnid_a), txnid);
+ mdbx_ensure(env, env->me_txn0->mt_owner != mdbx_thread_self() &&
+ !env->me_txn);
+ if (unlikely(txnid > MAX_TXNID)) {
+ rc = MDBX_TXN_FULL;
+ mdbx_error("txnid overflow, raise %d", rc);
+ goto fail;
+ }
+ meta_set_txnid(env, pending, txnid);
+ mdbx_assert(env, meta_checktxnid(env, pending, true));
+ }
+ }
}
- --n;
}
- parent->tw.dirtyroom += dst->sorted - n;
- dst->sorted = dpl_setlen(dst, n);
- mdbx_tassert(parent,
- parent->tw.dirtyroom + parent->tw.dirtylist->length ==
- (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
- : parent->mt_env->me_options.dp_limit));
}
- /* Remove reclaimed pages from parent's dirty list */
- const MDBX_PNL reclaimed_list = parent->tw.reclaimed_pglist;
- mdbx_dpl_sift(parent, reclaimed_list, false);
+ /* LY: step#1 - sync previously written/updated data-pages */
+ rc = MDBX_RESULT_FALSE /* carry steady */;
+ if (atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed)) {
+ mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
+ enum mdbx_syncmode_bits mode_bits = MDBX_SYNC_NONE;
+ if ((flags & MDBX_SAFE_NOSYNC) == 0) {
+ mode_bits = MDBX_SYNC_DATA;
+ if (pending->mm_geo.next > meta_prefer_steady(env)->mm_geo.now)
+ mode_bits |= MDBX_SYNC_SIZE;
+ if (flags & MDBX_NOMETASYNC)
+ mode_bits |= MDBX_SYNC_IODQ;
+ }
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ if (flags & MDBX_WRITEMAP)
+ rc =
+ mdbx_msync(&env->me_dxb_mmap, 0,
+ pgno_align2os_bytes(env, pending->mm_geo.next), mode_bits);
+ else
+ rc = mdbx_fsync(env->me_lazy_fd, mode_bits);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */
+ : MDBX_RESULT_FALSE /* carry steady */;
+ }
+ mdbx_assert(env, meta_checktxnid(env, pending, true));
- /* Move retired pages from parent's dirty & spilled list to reclaimed */
- unsigned r, w, d, s, l;
- for (r = w = parent_retired_len;
- ++r <= MDBX_PNL_SIZE(parent->tw.retired_pages);) {
- const pgno_t pgno = parent->tw.retired_pages[r];
- const unsigned di = mdbx_dpl_exist(parent, pgno);
- const unsigned si = !di ? mdbx_search_spilled(parent, pgno) : 0;
- unsigned npages;
- const char *kind;
- if (di) {
- MDBX_page *dp = dst->items[di].ptr;
- mdbx_tassert(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH |
- P_OVERFLOW | P_SPILLED)) == 0);
- npages = dpl_npages(dst, di);
- mdbx_page_wash(parent, di, dp, npages);
- kind = "dirty";
- l = 1;
- if (unlikely(npages > l)) {
- /* OVERFLOW-Ñтраница могла быть переиÑпользована по чаÑÑ‚Ñм. Тогда
- * в retired-ÑпиÑке может быть только начало поÑледовательноÑти,
- * а оÑтаток раÑтащен по dirty, spilled и reclaimed ÑпиÑкам. ПоÑтому
- * переноÑим в reclaimed Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¾Ð¹ на обрыв поÑледовательноÑти.
- * Ð’ любом Ñлучае, вÑе оÑколки будут учтены и отфильтрованы, Ñ‚.е. еÑли
- * Ñтраница была разбита на чаÑти, то важно удалить dirty-Ñлемент,
- * а вÑе оÑколки будут учтены отдельно. */
+ /* Steady or Weak */
+ if (rc == MDBX_RESULT_FALSE /* carry steady */) {
+ atomic_store64(&env->me_lck->mti_sync_timestamp, mdbx_osal_monotime(),
+ mo_Relaxed);
+ unaligned_poke_u64(4, pending->mm_datasync_sign, meta_sign(pending));
+ atomic_store32(&env->me_lck->mti_unsynced_pages, 0, mo_Relaxed);
+ } else {
+ assert(rc == MDBX_RESULT_TRUE /* carry non-steady */);
+ unaligned_poke_u64(4, pending->mm_datasync_sign, MDBX_DATASIGN_WEAK);
+ }
- /* СпиÑок retired Ñтраниц не Ñортирован, но Ð´Ð»Ñ ÑƒÑÐºÐ¾Ñ€ÐµÐ½Ð¸Ñ Ñортировки
- * дополнÑетÑÑ Ð² ÑоответÑтвии Ñ MDBX_PNL_ASCENDING */
-#if MDBX_PNL_ASCENDING
- const unsigned len = MDBX_PNL_SIZE(parent->tw.retired_pages);
- while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) {
- ++r;
- if (++l == npages)
- break;
- }
-#else
- while (w > parent_retired_len &&
- parent->tw.retired_pages[w - 1] == pgno + l) {
- --w;
- if (++l == npages)
- break;
- }
-#endif
- }
- } else if (unlikely(si)) {
- l = npages = 1;
- mdbx_spill_remove(parent, si, 1);
- kind = "spilled";
- } else {
- parent->tw.retired_pages[++w] = pgno;
- continue;
+ MDBX_meta *target = nullptr;
+ if (constmeta_txnid(env, head) ==
+ unaligned_peek_u64(4, pending->mm_txnid_a)) {
+ mdbx_assert(env, memcmp(&head->mm_dbs, &pending->mm_dbs,
+ sizeof(head->mm_dbs)) == 0);
+ mdbx_assert(env, memcmp(&head->mm_canary, &pending->mm_canary,
+ sizeof(head->mm_canary)) == 0);
+ mdbx_assert(env, memcmp(&head->mm_geo, &pending->mm_geo,
+ sizeof(pending->mm_geo)) == 0);
+ if (!META_IS_STEADY(head) && META_IS_STEADY(pending))
+ target = (MDBX_meta *)head;
+ else {
+ mdbx_ensure(env, meta_eq(env, head, pending));
+ mdbx_debug("%s", "skip update meta");
+ return MDBX_SUCCESS;
}
-
- mdbx_debug("reclaim retired parent's %u->%u %s page %" PRIaPGNO, npages, l,
- kind, pgno);
- int err = mdbx_pnl_insert_range(&parent->tw.reclaimed_pglist, pgno, l);
- mdbx_ensure(txn->mt_env, err == MDBX_SUCCESS);
+ } else if (head == meta0)
+ target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta1, meta2);
+ else if (head == meta1)
+ target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta0, meta2);
+ else {
+ mdbx_assert(env, head == meta2);
+ target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta0, meta1);
}
- MDBX_PNL_SIZE(parent->tw.retired_pages) = w;
- /* Filter-out parent spill list */
- if (parent->tw.spill_pages && MDBX_PNL_SIZE(parent->tw.spill_pages) > 0) {
- const MDBX_PNL sl = mdbx_spill_purge(parent);
- unsigned len = MDBX_PNL_SIZE(sl);
- if (len) {
- /* Remove refunded pages from parent's spill list */
- if (MDBX_ENABLE_REFUND &&
- MDBX_PNL_MOST(sl) >= (parent->mt_next_pgno << 1)) {
-#if MDBX_PNL_ASCENDING
- unsigned i = MDBX_PNL_SIZE(sl);
- assert(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl));
- do {
- if ((sl[i] & 1) == 0)
- mdbx_debug("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
- i -= 1;
- } while (i && sl[i] >= (parent->mt_next_pgno << 1));
- MDBX_PNL_SIZE(sl) = i;
-#else
- assert(MDBX_PNL_MOST(sl) == MDBX_PNL_FIRST(sl));
- unsigned i = 0;
- do {
- ++i;
- if ((sl[i] & 1) == 0)
- mdbx_debug("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
- } while (i < len && sl[i + 1] >= (parent->mt_next_pgno << 1));
- MDBX_PNL_SIZE(sl) = len -= i;
- memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0]));
-#endif
- }
- mdbx_tassert(
- txn, mdbx_pnl_check4assert(sl, (size_t)parent->mt_next_pgno << 1));
+ /* LY: step#2 - update meta-page. */
+ mdbx_debug(
+ "writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO
+ ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
+ " +%u -%u, txn_id %" PRIaTXN ", %s",
+ data_page(target)->mp_pgno, pending->mm_dbs[MAIN_DBI].md_root,
+ pending->mm_dbs[FREE_DBI].md_root, pending->mm_geo.lower,
+ pending->mm_geo.next, pending->mm_geo.now, pending->mm_geo.upper,
+ pv2pages(pending->mm_geo.grow_pv), pv2pages(pending->mm_geo.shrink_pv),
+ unaligned_peek_u64(4, pending->mm_txnid_a), mdbx_durable_str(pending));
- /* Remove reclaimed pages from parent's spill list */
- s = MDBX_PNL_SIZE(sl), r = MDBX_PNL_SIZE(reclaimed_list);
- /* Scanning from end to begin */
- while (s && r) {
- if (sl[s] & 1) {
- --s;
- continue;
- }
- const pgno_t spilled_pgno = sl[s] >> 1;
- const pgno_t reclaimed_pgno = reclaimed_list[r];
- if (reclaimed_pgno != spilled_pgno) {
- const bool cmp = MDBX_PNL_ORDERED(spilled_pgno, reclaimed_pgno);
- s -= !cmp;
- r -= cmp;
- } else {
- mdbx_debug("remove reclaimed parent's spilled page %" PRIaPGNO,
- reclaimed_pgno);
- mdbx_spill_remove(parent, s, 1);
- --s;
- --r;
- }
- }
+ mdbx_debug("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
+ "/%" PRIaPGNO,
+ (meta0 == head) ? "head"
+ : (meta0 == target) ? "tail"
+ : "stay",
+ mdbx_durable_str(meta0), meta_txnid(env, meta0),
+ meta0->mm_dbs[MAIN_DBI].md_root, meta0->mm_dbs[FREE_DBI].md_root);
+ mdbx_debug("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
+ "/%" PRIaPGNO,
+ (meta1 == head) ? "head"
+ : (meta1 == target) ? "tail"
+ : "stay",
+ mdbx_durable_str(meta1), meta_txnid(env, meta1),
+ meta1->mm_dbs[MAIN_DBI].md_root, meta1->mm_dbs[FREE_DBI].md_root);
+ mdbx_debug("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
+ "/%" PRIaPGNO,
+ (meta2 == head) ? "head"
+ : (meta2 == target) ? "tail"
+ : "stay",
+ mdbx_durable_str(meta2), meta_txnid(env, meta2),
+ meta2->mm_dbs[MAIN_DBI].md_root, meta2->mm_dbs[FREE_DBI].md_root);
- /* Remove anything in our dirty list from parent's spill list */
- /* Scanning spill list in descend order */
- const int step = MDBX_PNL_ASCENDING ? -1 : 1;
- s = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(sl) : 1;
- d = src->length;
- while (d && (MDBX_PNL_ASCENDING ? s > 0 : s <= MDBX_PNL_SIZE(sl))) {
- if (sl[s] & 1) {
- s += step;
- continue;
- }
- const pgno_t spilled_pgno = sl[s] >> 1;
- const pgno_t dirty_pgno_form = src->items[d].pgno;
- const unsigned npages = dpl_npages(src, d);
- const pgno_t dirty_pgno_to = dirty_pgno_form + npages;
- if (dirty_pgno_form > spilled_pgno) {
- --d;
- continue;
- }
- if (dirty_pgno_to <= spilled_pgno) {
- s += step;
- continue;
- }
+ mdbx_assert(env, !meta_eq(env, pending, meta0));
+ mdbx_assert(env, !meta_eq(env, pending, meta1));
+ mdbx_assert(env, !meta_eq(env, pending, meta2));
+
+ mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0);
+ mdbx_ensure(env,
+ target == head || constmeta_txnid(env, target) <
+ unaligned_peek_u64(4, pending->mm_txnid_a));
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ if (flags & MDBX_WRITEMAP) {
+ mdbx_jitter4testing(true);
+ if (likely(target != head)) {
+ /* LY: 'invalidate' the meta. */
+ meta_update_begin(env, target,
+ unaligned_peek_u64(4, pending->mm_txnid_a));
+ unaligned_poke_u64(4, target->mm_datasync_sign, MDBX_DATASIGN_WEAK);
+#ifndef NDEBUG
+ /* debug: provoke failure to catch a violators, but don't touch mm_psize
+ * to allow readers catch actual pagesize. */
+ uint8_t *provoke_begin = (uint8_t *)&target->mm_dbs[FREE_DBI].md_root;
+ uint8_t *provoke_end = (uint8_t *)&target->mm_datasync_sign;
+ memset(provoke_begin, 0xCC, provoke_end - provoke_begin);
+ mdbx_jitter4testing(false);
+#endif
- mdbx_debug("remove dirtied parent's spilled %u page %" PRIaPGNO, npages,
- dirty_pgno_form);
- mdbx_spill_remove(parent, s, 1);
- s += step;
- }
+ /* LY: update info */
+ target->mm_geo = pending->mm_geo;
+ target->mm_dbs[FREE_DBI] = pending->mm_dbs[FREE_DBI];
+ target->mm_dbs[MAIN_DBI] = pending->mm_dbs[MAIN_DBI];
+ target->mm_canary = pending->mm_canary;
+ memcpy(target->mm_pages_retired, pending->mm_pages_retired, 8);
+ mdbx_jitter4testing(true);
- /* Squash deleted pagenums if we deleted any */
- mdbx_spill_purge(parent);
+ /* LY: 'commit' the meta */
+ meta_update_end(env, target, unaligned_peek_u64(4, pending->mm_txnid_b));
+ mdbx_jitter4testing(true);
+ mdbx_assert(env, meta_checktxnid(env, target, true));
+ } else {
+ /* dangerous case (target == head), only mm_datasync_sign could
+ * me updated, check assertions once again */
+ mdbx_ensure(env, constmeta_txnid(env, head) ==
+ unaligned_peek_u64(4, pending->mm_txnid_a) &&
+ !META_IS_STEADY(head) && META_IS_STEADY(pending));
+ mdbx_ensure(env, memcmp(&head->mm_geo, &pending->mm_geo,
+ sizeof(head->mm_geo)) == 0);
+ mdbx_ensure(env, memcmp(&head->mm_dbs, &pending->mm_dbs,
+ sizeof(head->mm_dbs)) == 0);
+ mdbx_ensure(env, memcmp(&head->mm_canary, &pending->mm_canary,
+ sizeof(head->mm_canary)) == 0);
+ }
+ memcpy(target->mm_datasync_sign, pending->mm_datasync_sign, 8);
+ mdbx_flush_incoherent_cpu_writeback();
+ mdbx_jitter4testing(true);
+ /* sync meta-pages */
+ rc =
+ mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
+ (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE
+ : MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto fail;
+ } else {
+ const MDBX_meta undo_meta = *target;
+ const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
+ ? env->me_dsync_fd
+ : env->me_lazy_fd;
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ rc = mdbx_pwrite(fd, pending, sizeof(MDBX_meta),
+ (uint8_t *)target - env->me_map);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ undo:
+ mdbx_debug("%s", "write failed, disk error?");
+ /* On a failure, the pagecache still contains the new data.
+ * Try write some old data back, to prevent it from being used. */
+ mdbx_pwrite(fd, &undo_meta, sizeof(MDBX_meta),
+ (uint8_t *)target - env->me_map);
+ goto fail;
+ }
+ mdbx_flush_incoherent_mmap(target, sizeof(MDBX_meta), env->me_os_psize);
+ /* sync meta-pages */
+ if ((flags & MDBX_NOMETASYNC) == 0 && fd == env->me_lazy_fd) {
+ rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ if (rc != MDBX_SUCCESS)
+ goto undo;
}
+ mdbx_assert(env, meta_checktxnid(env, target, true));
}
+ env->me_lck->mti_meta_sync_txnid.weak =
+ (uint32_t)unaligned_peek_u64(4, pending->mm_txnid_a) -
+ ((flags & MDBX_NOMETASYNC) ? UINT32_MAX / 3 : 0);
- /* Remove anything in our spill list from parent's dirty list */
- if (txn->tw.spill_pages) {
- mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.spill_pages,
- (size_t)parent->mt_next_pgno << 1));
- mdbx_dpl_sift(parent, txn->tw.spill_pages, true);
- mdbx_tassert(parent,
- parent->tw.dirtyroom + parent->tw.dirtylist->length ==
- (parent->mt_parent ? parent->mt_parent->tw.dirtyroom
- : parent->mt_env->me_options.dp_limit));
+ /* LY: shrink datafile if needed */
+ if (unlikely(shrink)) {
+ mdbx_verbose("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")",
+ pending->mm_geo.now, shrink);
+ rc = mdbx_mapresize_implicit(env, pending->mm_geo.next, pending->mm_geo.now,
+ pending->mm_geo.upper);
+ if (rc != MDBX_SUCCESS && rc != MDBX_EPERM)
+ goto fail;
+ mdbx_assert(env, meta_checktxnid(env, target, true));
}
- /* Find length of merging our dirty list with parent's and release
- * filter-out pages */
- for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) {
- MDBX_page *sp = src->items[s].ptr;
- mdbx_tassert(parent,
- (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW |
- P_LOOSE | P_SPILLED)) == 0);
- const unsigned s_npages = dpl_npages(src, s);
- const pgno_t s_pgno = src->items[s].pgno;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (likely(lck))
+ /* toggle oldest refresh */
+ atomic_store32(&lck->mti_readers_refresh_flag, false, mo_Relaxed);
- MDBX_page *dp = dst->items[d].ptr;
- mdbx_tassert(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH |
- P_OVERFLOW | P_SPILLED)) == 0);
- const unsigned d_npages = dpl_npages(dst, d);
- const pgno_t d_pgno = dst->items[d].pgno;
+ return MDBX_SUCCESS;
- if (d_pgno >= s_pgno + s_npages) {
- --d;
- ++l;
- } else if (d_pgno + d_npages <= s_pgno) {
- if (sp->mp_flags != P_LOOSE) {
- sp->mp_txnid = parent->mt_front;
- sp->mp_flags &= ~P_SPILLED;
- }
- --s;
- ++l;
- } else {
- dst->items[d--].ptr = nullptr;
- if ((txn->mt_flags & MDBX_WRITEMAP) == 0)
- mdbx_dpage_free(txn->mt_env, dp, d_npages);
- }
- }
- assert(dst->sorted == dst->length);
- mdbx_tassert(parent, dst->detent >= l + d + s);
- dst->sorted = l + d + s; /* the merged length */
+fail:
+ env->me_flags |= MDBX_FATAL_ERROR;
+ return rc;
+}
- while (s > 0) {
- MDBX_page *sp = src->items[s].ptr;
- mdbx_tassert(parent,
- (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW |
- P_LOOSE | P_SPILLED)) == 0);
- if (sp->mp_flags != P_LOOSE) {
- sp->mp_txnid = parent->mt_front;
- sp->mp_flags &= ~P_SPILLED;
- }
- --s;
- }
+static void recalculate_merge_threshold(MDBX_env *env) {
+ const unsigned bytes = page_space(env);
+ env->me_merge_threshold =
+ (uint16_t)(bytes -
+ (bytes * env->me_options.merge_threshold_16dot16_percent >>
+ 16));
+ env->me_merge_threshold_gc =
+ (uint16_t)(bytes -
+ ((env->me_options.merge_threshold_16dot16_percent > 19005)
+ ? bytes / 3 /* 33 % */
+ : bytes / 4 /* 25 % */));
+}
- /* Merge our dirty list into parent's, i.e. merge(dst, src) -> dst */
- if (dst->sorted >= dst->length) {
- /* from end to begin with dst extending */
- for (l = dst->sorted, s = src->length, d = dst->length; s > 0 && d > 0;) {
- if (unlikely(l <= d)) {
- /* squash to get a gap of free space for merge */
- for (r = w = 1; r <= d; ++r)
- if (dst->items[r].ptr) {
- if (w != r) {
- dst->items[w] = dst->items[r];
- dst->items[r].ptr = nullptr;
- }
- ++w;
- }
- mdbx_notice("squash to begin for extending-merge %u -> %u", d, w - 1);
- d = w - 1;
- continue;
- }
- assert(l > d);
- if (dst->items[d].ptr) {
- dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno)
- ? dst->items[d--]
- : src->items[s--];
- } else
- --d;
- }
- if (s > 0) {
- assert(l == s);
- while (d > 0) {
- assert(dst->items[d].ptr == nullptr);
- --d;
- }
- do {
- assert(l > 0);
- dst->items[l--] = src->items[s--];
- } while (s > 0);
- } else {
- assert(l == d);
- while (l > 0) {
- assert(dst->items[l].ptr != nullptr);
- --l;
- }
- }
- } else {
- /* from begin to end with dst shrinking (a lot of new overflow pages) */
- for (l = s = d = 1; s <= src->length && d <= dst->length;) {
- if (unlikely(l >= d)) {
- /* squash to get a gap of free space for merge */
- for (r = w = dst->length; r >= d; --r)
- if (dst->items[r].ptr) {
- if (w != r) {
- dst->items[w] = dst->items[r];
- dst->items[r].ptr = nullptr;
- }
- --w;
- }
- mdbx_notice("squash to end for shrinking-merge %u -> %u", d, w + 1);
- d = w + 1;
- continue;
- }
- assert(l < d);
- if (dst->items[d].ptr) {
- dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno)
- ? dst->items[d++]
- : src->items[s++];
- } else
- ++d;
- }
- if (s <= src->length) {
- assert(dst->sorted - l == src->length - s);
- while (d <= dst->length) {
- assert(dst->items[d].ptr == nullptr);
- --d;
- }
- do {
- assert(l <= dst->sorted);
- dst->items[l++] = src->items[s++];
- } while (s <= src->length);
- } else {
- assert(dst->sorted - l == dst->length - d);
- while (l <= dst->sorted) {
- assert(l <= d && d <= dst->length && dst->items[d].ptr);
- dst->items[l++] = dst->items[d++];
- }
- }
+__cold static void mdbx_setup_pagesize(MDBX_env *env, const size_t pagesize) {
+ STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE);
+ STATIC_ASSERT(MIN_PAGESIZE > sizeof(MDBX_page) + sizeof(MDBX_meta));
+ mdbx_ensure(env, is_powerof2(pagesize));
+ mdbx_ensure(env, pagesize >= MIN_PAGESIZE);
+ mdbx_ensure(env, pagesize <= MAX_PAGESIZE);
+ env->me_psize = (unsigned)pagesize;
+ if (env->me_pbuf) {
+ mdbx_memalign_free(env->me_pbuf);
+ env->me_pbuf = nullptr;
}
- parent->tw.dirtyroom -= dst->sorted - dst->length;
- assert(parent->tw.dirtyroom <= parent->mt_env->me_options.dp_limit);
- dpl_setlen(dst, dst->sorted);
- parent->tw.dirtylru = txn->tw.dirtylru;
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
- mdbx_dpl_free(txn);
- if (txn->tw.spill_pages) {
- if (parent->tw.spill_pages) {
- /* Must not fail since space was preserved above. */
- mdbx_pnl_xmerge(parent->tw.spill_pages, txn->tw.spill_pages);
- mdbx_pnl_free(txn->tw.spill_pages);
- } else {
- parent->tw.spill_pages = txn->tw.spill_pages;
- parent->tw.spill_least_removed = txn->tw.spill_least_removed;
+ STATIC_ASSERT(MAX_GC1OVPAGE(MIN_PAGESIZE) > 4);
+ STATIC_ASSERT(MAX_GC1OVPAGE(MAX_PAGESIZE) < MDBX_PGL_LIMIT);
+ const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
+ mdbx_ensure(env, maxgc_ov1page > 42 &&
+ maxgc_ov1page < (intptr_t)MDBX_PGL_LIMIT / 4);
+ env->me_maxgc_ov1page = (unsigned)maxgc_ov1page;
+
+ STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) > sizeof(MDBX_db) + NODESIZE + 42);
+ STATIC_ASSERT(LEAF_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX);
+ STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) >= BRANCH_NODE_MAX(MIN_PAGESIZE));
+ STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) > NODESIZE + 42);
+ STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX);
+ const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize);
+ const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize);
+ mdbx_ensure(env,
+ branch_nodemax > (intptr_t)(NODESIZE + 42) &&
+ branch_nodemax % 2 == 0 &&
+ leaf_nodemax > (intptr_t)(sizeof(MDBX_db) + NODESIZE + 42) &&
+ leaf_nodemax >= branch_nodemax &&
+ leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0);
+ env->me_leaf_nodemax = (unsigned)leaf_nodemax;
+ env->me_psize2log = (uint8_t)log2n_powerof2(pagesize);
+ mdbx_assert(env, pgno2bytes(env, 1) == pagesize);
+ mdbx_assert(env, bytes2pgno(env, pagesize + pagesize) == 2);
+ recalculate_merge_threshold(env);
+
+ const pgno_t max_pgno = bytes2pgno(env, MAX_MAPSIZE);
+ if (!env->me_options.flags.non_auto.dp_limit) {
+ /* auto-setup dp_limit by "The42" ;-) */
+ intptr_t total_ram_pages, avail_ram_pages;
+ int err = mdbx_get_sysraminfo(nullptr, &total_ram_pages, &avail_ram_pages);
+ if (unlikely(err != MDBX_SUCCESS))
+ mdbx_error("mdbx_get_sysraminfo(), rc %d", err);
+ else {
+ size_t reasonable_dpl_limit =
+ (size_t)(total_ram_pages + avail_ram_pages) / 42;
+ if (pagesize > env->me_os_psize)
+ reasonable_dpl_limit /= pagesize / env->me_os_psize;
+ else if (pagesize < env->me_os_psize)
+ reasonable_dpl_limit *= env->me_os_psize / pagesize;
+ reasonable_dpl_limit = (reasonable_dpl_limit < MDBX_PGL_LIMIT)
+ ? reasonable_dpl_limit
+ : MDBX_PGL_LIMIT;
+ reasonable_dpl_limit = (reasonable_dpl_limit > CURSOR_STACK * 4)
+ ? reasonable_dpl_limit
+ : CURSOR_STACK * 4;
+ env->me_options.dp_limit = (unsigned)reasonable_dpl_limit;
}
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
}
+ if (env->me_options.dp_limit > max_pgno - NUM_METAS)
+ env->me_options.dp_limit = max_pgno - NUM_METAS;
+ if (env->me_options.dp_initial > env->me_options.dp_limit)
+ env->me_options.dp_initial = env->me_options.dp_limit;
+}
- parent->mt_flags &= ~MDBX_TXN_HAS_CHILD;
- if (parent->tw.spill_pages) {
- assert(mdbx_pnl_check4assert(parent->tw.spill_pages,
- (size_t)parent->mt_next_pgno << 1));
- if (MDBX_PNL_SIZE(parent->tw.spill_pages))
- parent->mt_flags |= MDBX_TXN_SPILLS;
- }
+static __inline MDBX_CONST_FUNCTION MDBX_lockinfo *
+lckless_stub(const MDBX_env *env) {
+ uintptr_t stub = (uintptr_t)&env->x_lckless_stub;
+ /* align to avoid false-positive alarm from UndefinedBehaviorSanitizer */
+ stub = (stub + MDBX_CACHELINE_SIZE - 1) & ~(MDBX_CACHELINE_SIZE - 1);
+ return (MDBX_lockinfo *)stub;
}
-int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
- STATIC_ASSERT(MDBX_TXN_FINISHED ==
- MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR);
- const uint64_t ts_0 = latency ? mdbx_osal_monotime() : 0;
- uint64_t ts_1 = 0, ts_2 = 0, ts_3 = 0, ts_4 = 0;
- uint32_t audit_duration = 0;
+__cold int mdbx_env_create(MDBX_env **penv) {
+ MDBX_env *env = mdbx_calloc(1, sizeof(MDBX_env));
+ if (unlikely(!env))
+ return MDBX_ENOMEM;
- int rc = check_txn(txn, MDBX_TXN_FINISHED);
- if (unlikely(rc != MDBX_SUCCESS))
- goto provide_latency;
+ env->me_maxreaders = DEFAULT_READERS;
+ env->me_maxdbs = env->me_numdbs = CORE_DBS;
+ env->me_lazy_fd = INVALID_HANDLE_VALUE;
+ env->me_dsync_fd = INVALID_HANDLE_VALUE;
+ env->me_lfd = INVALID_HANDLE_VALUE;
+ env->me_pid = mdbx_getpid();
+ env->me_stuck_meta = -1;
- if (unlikely(txn->mt_flags & MDBX_TXN_ERROR)) {
- rc = MDBX_RESULT_TRUE;
- goto fail;
- }
+ env->me_options.dp_reserve_limit = 1024;
+ env->me_options.rp_augment_limit = 256 * 1024;
+ env->me_options.dp_limit = 64 * 1024;
+ if (env->me_options.dp_limit > MAX_PAGENO + 1 - NUM_METAS)
+ env->me_options.dp_limit = MAX_PAGENO + 1 - NUM_METAS;
+ env->me_options.dp_initial = MDBX_PNL_INITIAL;
+ if (env->me_options.dp_initial > env->me_options.dp_limit)
+ env->me_options.dp_initial = env->me_options.dp_limit;
+ env->me_options.spill_max_denominator = 8;
+ env->me_options.spill_min_denominator = 8;
+ env->me_options.spill_parent4child_denominator = 0;
+ env->me_options.dp_loose_limit = 64;
+ env->me_options.merge_threshold_16dot16_percent = 65536 / 4 /* 25% */;
- MDBX_env *env = txn->mt_env;
-#if MDBX_ENV_CHECKPID
- if (unlikely(env->me_pid != mdbx_getpid())) {
- env->me_flags |= MDBX_FATAL_ERROR;
- rc = MDBX_PANIC;
- goto provide_latency;
+ int rc;
+ const size_t os_psize = mdbx_syspagesize();
+ if (unlikely(!is_powerof2(os_psize) || os_psize < MIN_PAGESIZE)) {
+ mdbx_error("unsuitable system pagesize %" PRIuPTR, os_psize);
+ rc = MDBX_INCOMPATIBLE;
+ goto bailout;
}
-#endif /* MDBX_ENV_CHECKPID */
+ env->me_os_psize = (unsigned)os_psize;
+ mdbx_setup_pagesize(env, (env->me_os_psize < MAX_PAGESIZE) ? env->me_os_psize
+ : MAX_PAGESIZE);
- /* mdbx_txn_end() mode for a commit which writes nothing */
- unsigned end_mode =
- MDBX_END_PURE_COMMIT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE;
- if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
- goto done;
+ rc = mdbx_fastmutex_init(&env->me_dbi_lock);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
- if (txn->mt_child) {
- rc = mdbx_txn_commit_ex(txn->mt_child, NULL);
- mdbx_tassert(txn, txn->mt_child == NULL);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
+#if defined(_WIN32) || defined(_WIN64)
+ mdbx_srwlock_Init(&env->me_remap_guard);
+ InitializeCriticalSection(&env->me_windowsbug_lock);
+#else
+ rc = mdbx_fastmutex_init(&env->me_remap_guard);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_fastmutex_destroy(&env->me_dbi_lock);
+ goto bailout;
}
- if (unlikely(txn != env->me_txn)) {
- mdbx_debug("%s", "attempt to commit unknown transaction");
- rc = MDBX_EINVAL;
- goto fail;
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV
+ MDBX_lockinfo *const stub = lckless_stub(env);
+ rc = mdbx_ipclock_stub(&stub->mti_wlock);
+#endif /* MDBX_LOCKING */
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_fastmutex_destroy(&env->me_remap_guard);
+ mdbx_fastmutex_destroy(&env->me_dbi_lock);
+ goto bailout;
}
+#endif /* Windows */
- if (txn->mt_parent) {
- mdbx_tassert(txn, mdbx_audit_ex(txn, 0, false) == 0);
- mdbx_assert(env, txn != env->me_txn0);
- MDBX_txn *const parent = txn->mt_parent;
- mdbx_assert(env, parent->mt_signature == MDBX_MT_SIGNATURE);
- mdbx_assert(env, parent->mt_child == txn &&
- (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0);
- mdbx_assert(env, mdbx_dirtylist_check(txn));
-
- if (txn->tw.dirtylist->length == 0 && !(txn->mt_flags & MDBX_TXN_DIRTY) &&
- parent->mt_numdbs == txn->mt_numdbs) {
- for (int i = txn->mt_numdbs; --i >= 0;) {
- mdbx_tassert(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0);
- if ((txn->mt_dbistate[i] & DBI_STALE) &&
- !(parent->mt_dbistate[i] & DBI_STALE))
- mdbx_tassert(txn, memcmp(&parent->mt_dbs[i], &txn->mt_dbs[i],
- sizeof(MDBX_db)) == 0);
- }
+ VALGRIND_CREATE_MEMPOOL(env, 0, 0);
+ env->me_signature.weak = MDBX_ME_SIGNATURE;
+ *penv = env;
+ return MDBX_SUCCESS;
- mdbx_tassert(txn, memcmp(&parent->mt_geo, &txn->mt_geo,
- sizeof(parent->mt_geo)) == 0);
- mdbx_tassert(txn, memcmp(&parent->mt_canary, &txn->mt_canary,
- sizeof(parent->mt_canary)) == 0);
- mdbx_tassert(txn, !txn->tw.spill_pages ||
- MDBX_PNL_SIZE(txn->tw.spill_pages) == 0);
- mdbx_tassert(txn, txn->tw.loose_count == 0);
+bailout:
+ mdbx_free(env);
+ *penv = nullptr;
+ return rc;
+}
- /* fast completion of pure nested transaction */
- end_mode = MDBX_END_PURE_COMMIT | MDBX_END_SLOT | MDBX_END_FREE;
- goto done;
- }
+__cold static intptr_t get_reasonable_db_maxsize(intptr_t *cached_result) {
+ if (*cached_result == 0) {
+ intptr_t pagesize, total_ram_pages;
+ if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) !=
+ MDBX_SUCCESS))
+ return *cached_result = MAX_MAPSIZE32 /* the 32-bit limit is good enough
+ for fallback */
+ ;
- /* Preserve space for spill list to avoid parent's state corruption
- * if allocation fails. */
- const unsigned parent_retired_len =
- (unsigned)(uintptr_t)parent->tw.retired_pages;
- mdbx_tassert(txn,
- parent_retired_len <= MDBX_PNL_SIZE(txn->tw.retired_pages));
- const unsigned retired_delta =
- MDBX_PNL_SIZE(txn->tw.retired_pages) - parent_retired_len;
- if (retired_delta) {
- rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, retired_delta);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- }
+ if (unlikely((size_t)total_ram_pages * 2 > MAX_MAPSIZE / (size_t)pagesize))
+ return *cached_result = MAX_MAPSIZE;
+ assert(MAX_MAPSIZE >= (size_t)(total_ram_pages * pagesize * 2));
- if (txn->tw.spill_pages) {
- if (parent->tw.spill_pages) {
- rc = mdbx_pnl_need(&parent->tw.spill_pages,
- MDBX_PNL_SIZE(txn->tw.spill_pages));
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- }
- mdbx_spill_purge(txn);
- }
+ /* Suggesting should not be more than golden ratio of the size of RAM. */
+ *cached_result = (intptr_t)((size_t)total_ram_pages * 207 >> 7) * pagesize;
- if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length >
- parent->tw.dirtylist->detent &&
- !mdbx_dpl_reserve(parent, txn->tw.dirtylist->length +
- parent->tw.dirtylist->length))) {
- rc = MDBX_ENOMEM;
- goto fail;
+ /* Round to the nearest human-readable granulation. */
+ for (size_t unit = MEGABYTE; unit; unit <<= 5) {
+ const size_t floor = floor_powerof2(*cached_result, unit);
+ const size_t ceil = ceil_powerof2(*cached_result, unit);
+ const size_t threshold = (size_t)*cached_result >> 4;
+ const bool down =
+ *cached_result - floor < ceil - *cached_result || ceil > MAX_MAPSIZE;
+ if (threshold < (down ? *cached_result - floor : ceil - *cached_result))
+ break;
+ *cached_result = down ? floor : ceil;
}
+ }
+ return *cached_result;
+}
- //-------------------------------------------------------------------------
-
- parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed;
- txn->tw.lifo_reclaimed = NULL;
+__cold LIBMDBX_API int
+mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now,
+ intptr_t size_upper, intptr_t growth_step,
+ intptr_t shrink_threshold, intptr_t pagesize) {
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- parent->tw.retired_pages = txn->tw.retired_pages;
- txn->tw.retired_pages = NULL;
+ const bool inside_txn =
+ (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self());
- mdbx_pnl_free(parent->tw.reclaimed_pglist);
- parent->tw.reclaimed_pglist = txn->tw.reclaimed_pglist;
- txn->tw.reclaimed_pglist = NULL;
- parent->tw.last_reclaimed = txn->tw.last_reclaimed;
+#if MDBX_DEBUG
+ if (growth_step < 0) {
+ growth_step = 1;
+ if (shrink_threshold < 0)
+ shrink_threshold = 1;
+ }
+#endif /* MDBX_DEBUG */
- parent->mt_geo = txn->mt_geo;
- parent->mt_canary = txn->mt_canary;
- parent->mt_flags |= txn->mt_flags & MDBX_TXN_DIRTY;
+ intptr_t reasonable_maxsize = 0;
+ bool need_unlock = false;
+ if (env->me_map) {
+ /* env already mapped */
+ if (unlikely(env->me_flags & MDBX_RDONLY))
+ return MDBX_EACCESS;
- /* Move loose pages to parent */
-#if MDBX_ENABLE_REFUND
- parent->tw.loose_refund_wl = txn->tw.loose_refund_wl;
-#endif /* MDBX_ENABLE_REFUND */
- parent->tw.loose_count = txn->tw.loose_count;
- parent->tw.loose_pages = txn->tw.loose_pages;
+ if (!inside_txn) {
+ int err = mdbx_txn_lock(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ need_unlock = true;
+ }
+ const MDBX_meta *head = constmeta_prefer_last(env);
+ if (!inside_txn) {
+ env->me_txn0->mt_txnid = constmeta_txnid(env, head);
+ mdbx_find_oldest(env->me_txn0);
+ }
- /* Merge our cursors into parent's and close them */
- mdbx_cursors_eot(txn, true);
- end_mode |= MDBX_END_EOTDONE;
+ /* get untouched params from DB */
+ if (pagesize <= 0 || pagesize >= INT_MAX)
+ pagesize = env->me_psize;
+ if (size_lower < 0)
+ size_lower = pgno2bytes(env, head->mm_geo.lower);
+ if (size_now < 0)
+ size_now = pgno2bytes(env, head->mm_geo.now);
+ if (size_upper < 0)
+ size_upper = pgno2bytes(env, head->mm_geo.upper);
+ if (growth_step < 0)
+ growth_step = pgno2bytes(env, pv2pages(head->mm_geo.grow_pv));
+ if (shrink_threshold < 0)
+ shrink_threshold = pgno2bytes(env, pv2pages(head->mm_geo.shrink_pv));
- /* Update parent's DBs array */
- memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db));
- parent->mt_numdbs = txn->mt_numdbs;
- for (unsigned i = 0; i < txn->mt_numdbs; i++) {
- /* preserve parent's status */
- const uint8_t state =
- txn->mt_dbistate[i] |
- (parent->mt_dbistate[i] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY));
- mdbx_debug("db %u dbi-state %s 0x%02x -> 0x%02x", i,
- (parent->mt_dbistate[i] != state) ? "update" : "still",
- parent->mt_dbistate[i], state);
- parent->mt_dbistate[i] = state;
+ if (pagesize != (intptr_t)env->me_psize) {
+ rc = MDBX_EINVAL;
+ goto bailout;
+ }
+ const size_t usedbytes =
+ pgno2bytes(env, mdbx_find_largest(env, head->mm_geo.next));
+ if ((size_t)size_upper < usedbytes) {
+ rc = MDBX_MAP_FULL;
+ goto bailout;
}
+ if ((size_t)size_now < usedbytes)
+ size_now = usedbytes;
+ } else {
+ /* env NOT yet mapped */
+ if (unlikely(inside_txn))
+ return MDBX_PANIC;
- ts_1 = latency ? mdbx_osal_monotime() : 0;
- mdbx_txn_merge(parent, txn, parent_retired_len);
- ts_2 = latency ? mdbx_osal_monotime() : 0;
- env->me_txn = parent;
- parent->mt_child = NULL;
- mdbx_tassert(parent, mdbx_dirtylist_check(parent));
+ /* is requested some auto-value for pagesize ? */
+ if (pagesize >= INT_MAX /* maximal */)
+ pagesize = MAX_PAGESIZE;
+ else if (pagesize <= 0) {
+ if (pagesize < 0 /* default */) {
+ pagesize = env->me_os_psize;
+ if ((uintptr_t)pagesize > MAX_PAGESIZE)
+ pagesize = MAX_PAGESIZE;
+ mdbx_assert(env, (uintptr_t)pagesize >= MIN_PAGESIZE);
+ } else if (pagesize == 0 /* minimal */)
+ pagesize = MIN_PAGESIZE;
-#if MDBX_ENABLE_REFUND
- mdbx_refund(parent);
- if (mdbx_assert_enabled()) {
- /* Check parent's loose pages not suitable for refund */
- for (MDBX_page *lp = parent->tw.loose_pages; lp; lp = lp->mp_next)
- mdbx_tassert(parent, lp->mp_pgno < parent->tw.loose_refund_wl &&
- lp->mp_pgno + 1 < parent->mt_next_pgno);
- /* Check parent's reclaimed pages not suitable for refund */
- if (MDBX_PNL_SIZE(parent->tw.reclaimed_pglist))
- mdbx_tassert(parent, MDBX_PNL_MOST(parent->tw.reclaimed_pglist) + 1 <
- parent->mt_next_pgno);
+ /* choose pagesize */
+ intptr_t max_size = (size_now > size_lower) ? size_now : size_lower;
+ max_size = (size_upper > max_size) ? size_upper : max_size;
+ if (max_size < 0 /* default */)
+ max_size = DEFAULT_MAPSIZE;
+ else if (max_size == 0 /* minimal */)
+ max_size = MIN_MAPSIZE;
+ else if (max_size >= (intptr_t)MAX_MAPSIZE /* maximal */)
+ max_size = get_reasonable_db_maxsize(&reasonable_maxsize);
+
+ while (max_size > pagesize * (int64_t)(MAX_PAGENO + 1) &&
+ pagesize < MAX_PAGESIZE)
+ pagesize <<= 1;
}
-#endif /* MDBX_ENABLE_REFUND */
+ }
- ts_4 = ts_3 = latency ? mdbx_osal_monotime() : 0;
- txn->mt_signature = 0;
- mdbx_free(txn);
- mdbx_tassert(parent, mdbx_audit_ex(parent, 0, false) == 0);
- rc = MDBX_SUCCESS;
- goto provide_latency;
+ if (pagesize < (intptr_t)MIN_PAGESIZE || pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2(pagesize)) {
+ rc = MDBX_EINVAL;
+ goto bailout;
}
- mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
- (txn->mt_parent ? txn->mt_parent->tw.dirtyroom
- : txn->mt_env->me_options.dp_limit));
- mdbx_cursors_eot(txn, false);
- end_mode |= MDBX_END_EOTDONE;
+ if (size_lower <= 0) {
+ size_lower = MIN_MAPSIZE;
+ if (MIN_MAPSIZE / pagesize < MIN_PAGENO)
+ size_lower = MIN_PAGENO * pagesize;
+ }
+ if (size_lower >= INTPTR_MAX) {
+ size_lower = get_reasonable_db_maxsize(&reasonable_maxsize);
+ if ((size_t)size_lower / pagesize > MAX_PAGENO + 1)
+ size_lower = pagesize * (MAX_PAGENO + 1);
+ }
- if (txn->tw.dirtylist->length == 0 &&
- (txn->mt_flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) {
- for (int i = txn->mt_numdbs; --i >= 0;)
- mdbx_tassert(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0);
- rc = MDBX_SUCCESS;
- goto done;
+ if (size_now <= 0) {
+ size_now = size_lower;
+ if (size_upper >= size_lower && size_now > size_upper)
+ size_now = size_upper;
+ }
+ if (size_now >= INTPTR_MAX) {
+ size_now = get_reasonable_db_maxsize(&reasonable_maxsize);
+ if ((size_t)size_now / pagesize > MAX_PAGENO + 1)
+ size_now = pagesize * (MAX_PAGENO + 1);
}
- mdbx_debug("committing txn %" PRIaTXN " %p on mdbenv %p, root page %" PRIaPGNO
- "/%" PRIaPGNO,
- txn->mt_txnid, (void *)txn, (void *)env,
- txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root);
+ if (size_upper <= 0) {
+ if (size_now >= get_reasonable_db_maxsize(&reasonable_maxsize) / 2)
+ size_upper = get_reasonable_db_maxsize(&reasonable_maxsize);
+ else if (MAX_MAPSIZE != MAX_MAPSIZE32 &&
+ (size_t)size_now >= MAX_MAPSIZE32 / 2 &&
+ (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3)
+ size_upper = MAX_MAPSIZE32;
+ else {
+ size_upper = size_now + size_now;
+ if ((size_t)size_upper < DEFAULT_MAPSIZE * 2)
+ size_upper = DEFAULT_MAPSIZE * 2;
+ }
+ if ((size_t)size_upper / pagesize > (MAX_PAGENO + 1))
+ size_upper = pagesize * (MAX_PAGENO + 1);
+ } else if (size_upper >= INTPTR_MAX) {
+ size_upper = get_reasonable_db_maxsize(&reasonable_maxsize);
+ if ((size_t)size_upper / pagesize > MAX_PAGENO + 1)
+ size_upper = pagesize * (MAX_PAGENO + 1);
+ }
- /* Update DB root pointers */
- if (txn->mt_numdbs > CORE_DBS) {
- MDBX_cursor_couple couple;
- MDBX_val data;
- data.iov_len = sizeof(MDBX_db);
+ if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) {
+ rc = MDBX_EINVAL;
+ goto bailout;
+ }
- rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- for (MDBX_dbi i = CORE_DBS; i < txn->mt_numdbs; i++) {
- if (txn->mt_dbistate[i] & DBI_DIRTY) {
- if (unlikely(TXN_DBI_CHANGED(txn, i))) {
- rc = MDBX_BAD_DBI;
- goto fail;
- }
- MDBX_db *db = &txn->mt_dbs[i];
- mdbx_debug("update main's entry for sub-db %u, mod_txnid %" PRIaTXN
- " -> %" PRIaTXN,
- i, db->md_mod_txnid, txn->mt_txnid);
- db->md_mod_txnid = txn->mt_txnid;
- data.iov_base = db;
- WITH_CURSOR_TRACKING(couple.outer,
- rc = mdbx_cursor_put(&couple.outer,
- &txn->mt_dbxs[i].md_name,
- &data, F_SUBDATA));
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- }
+ if ((uint64_t)size_lower / pagesize < MIN_PAGENO) {
+ rc = MDBX_EINVAL;
+ goto bailout;
+ }
+
+ if (unlikely((size_t)size_upper > MAX_MAPSIZE ||
+ (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
+ rc = MDBX_TOO_LARGE;
+ goto bailout;
+ }
+
+ const size_t unit = (env->me_os_psize > (size_t)pagesize) ? env->me_os_psize
+ : (size_t)pagesize;
+ size_lower = ceil_powerof2(size_lower, unit);
+ size_upper = ceil_powerof2(size_upper, unit);
+ size_now = ceil_powerof2(size_now, unit);
+
+ /* LY: подбираем значение size_upper:
+ * - кратное размеру Ñтраницы
+ * - без Ð½Ð°Ñ€ÑƒÑˆÐµÐ½Ð¸Ñ MAX_MAPSIZE и MAX_PAGENO */
+ while (unlikely((size_t)size_upper > MAX_MAPSIZE ||
+ (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
+ if ((size_t)size_upper < unit + MIN_MAPSIZE ||
+ (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) {
+ /* паранойа на Ñлучай Ð¿ÐµÑ€ÐµÐ¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€Ð¸ невероÑтных значениÑÑ… */
+ rc = MDBX_EINVAL;
+ goto bailout;
}
+ size_upper -= unit;
+ if ((size_t)size_upper < (size_t)size_lower)
+ size_lower = size_upper;
}
+ mdbx_assert(env, (size_upper - size_lower) % env->me_os_psize == 0);
- ts_1 = latency ? mdbx_osal_monotime() : 0;
- rc = mdbx_update_gc(txn);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
+ if (size_now < size_lower)
+ size_now = size_lower;
+ if (size_now > size_upper)
+ size_now = size_upper;
- txn->mt_dbs[FREE_DBI].md_mod_txnid = (txn->mt_dbistate[FREE_DBI] & DBI_DIRTY)
- ? txn->mt_txnid
- : txn->mt_dbs[FREE_DBI].md_mod_txnid;
+ if (growth_step < 0) {
+ growth_step = ((size_t)(size_upper - size_lower)) / 42;
+ if (growth_step > size_lower && size_lower < (intptr_t)MEGABYTE)
+ growth_step = size_lower;
+ if (growth_step < 65536)
+ growth_step = 65536;
+ if ((size_t)growth_step > MAX_MAPSIZE / 64)
+ growth_step = MAX_MAPSIZE / 64;
+ }
+ if (growth_step == 0 && shrink_threshold > 0)
+ growth_step = 1;
+ growth_step = ceil_powerof2(growth_step, unit);
- txn->mt_dbs[MAIN_DBI].md_mod_txnid = (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY)
- ? txn->mt_txnid
- : txn->mt_dbs[MAIN_DBI].md_mod_txnid;
+ if (shrink_threshold < 0)
+ shrink_threshold = growth_step + growth_step;
+ shrink_threshold = ceil_powerof2(shrink_threshold, unit);
- ts_2 = latency ? mdbx_osal_monotime() : 0;
- if (mdbx_audit_enabled()) {
- rc = mdbx_audit_ex(txn, MDBX_PNL_SIZE(txn->tw.retired_pages), true);
- const uint64_t audit_end = mdbx_osal_monotime();
- audit_duration = mdbx_osal_monotime_to_16dot16(audit_end - ts_2);
- ts_2 = audit_end;
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- }
+ //----------------------------------------------------------------------------
- struct mdbx_iov_ctx ctx;
- mdbx_iov_init(txn, &ctx);
- rc = mdbx_txn_write(txn, &ctx);
- if (likely(rc == MDBX_SUCCESS))
- mdbx_iov_done(txn, &ctx);
- /* TODO: use ctx.flush_begin & ctx.flush_end for range-sync */
- ts_3 = latency ? mdbx_osal_monotime() : 0;
+ if (!env->me_map) {
+ /* save user's geo-params for future open/create */
+ if (pagesize != (intptr_t)env->me_psize)
+ mdbx_setup_pagesize(env, pagesize);
+ env->me_dbgeo.lower = size_lower;
+ env->me_dbgeo.now = size_now;
+ env->me_dbgeo.upper = size_upper;
+ env->me_dbgeo.grow =
+ pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step))));
+ env->me_dbgeo.shrink =
+ pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold))));
- if (likely(rc == MDBX_SUCCESS)) {
- const MDBX_meta *head = constmeta_prefer_last(env);
+ mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE);
+ mdbx_ensure(env, env->me_dbgeo.lower / (unsigned)pagesize >= MIN_PAGENO);
+ mdbx_ensure(env, env->me_dbgeo.lower % (unsigned)pagesize == 0);
+ mdbx_ensure(env, env->me_dbgeo.lower % env->me_os_psize == 0);
+
+ mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE);
+ mdbx_ensure(env,
+ env->me_dbgeo.upper / (unsigned)pagesize <= MAX_PAGENO + 1);
+ mdbx_ensure(env, env->me_dbgeo.upper % (unsigned)pagesize == 0);
+ mdbx_ensure(env, env->me_dbgeo.upper % env->me_os_psize == 0);
+
+ mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower);
+ mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper);
+ mdbx_ensure(env, env->me_dbgeo.now % (unsigned)pagesize == 0);
+ mdbx_ensure(env, env->me_dbgeo.now % env->me_os_psize == 0);
+
+ mdbx_ensure(env, env->me_dbgeo.grow % (unsigned)pagesize == 0);
+ mdbx_ensure(env, env->me_dbgeo.grow % env->me_os_psize == 0);
+ mdbx_ensure(env, env->me_dbgeo.shrink % (unsigned)pagesize == 0);
+ mdbx_ensure(env, env->me_dbgeo.shrink % env->me_os_psize == 0);
+
+ rc = MDBX_SUCCESS;
+ } else {
+ /* apply new params to opened environment */
+ mdbx_ensure(env, pagesize == (intptr_t)env->me_psize);
MDBX_meta meta;
- memcpy(meta.mm_magic_and_version, head->mm_magic_and_version, 8);
- meta.mm_extra_flags = head->mm_extra_flags;
- meta.mm_validator_id = head->mm_validator_id;
- meta.mm_extra_pagehdr = head->mm_extra_pagehdr;
- unaligned_poke_u64(4, meta.mm_pages_retired,
- unaligned_peek_u64(4, head->mm_pages_retired) +
- MDBX_PNL_SIZE(txn->tw.retired_pages));
- meta.mm_geo = txn->mt_geo;
- meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
- meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
- meta.mm_canary = txn->mt_canary;
- meta_set_txnid(env, &meta, txn->mt_txnid);
+ memset(&meta, 0, sizeof(meta));
+ const MDBX_geo *current_geo;
+ if (!inside_txn) {
+ mdbx_assert(env, need_unlock);
+ const MDBX_meta *head = constmeta_prefer_last(env);
- rc = mdbx_sync_locked(
- env, env->me_flags | txn->mt_flags | MDBX_SHRINK_ALLOWED, &meta);
- }
- ts_4 = latency ? mdbx_osal_monotime() : 0;
- if (unlikely(rc != MDBX_SUCCESS)) {
- env->me_flags |= MDBX_FATAL_ERROR;
- goto fail;
- }
+ uint64_t timestamp = 0;
+ while ("workaround for "
+ "todo4recovery://erased_by_github/libmdbx/issues/269") {
+ meta = *head;
+ rc = meta_waittxnid(env, &meta, ×tamp);
+ if (likely(rc == MDBX_SUCCESS))
+ break;
+ if (unlikely(rc != MDBX_RESULT_TRUE))
+ goto bailout;
+ }
+ const txnid_t txnid = safe64_txnid_next(constmeta_txnid(env, &meta));
+ if (unlikely(txnid > MAX_TXNID)) {
+ rc = MDBX_TXN_FULL;
+ mdbx_error("txnid overflow, raise %d", rc);
+ goto bailout;
+ }
+ meta_set_txnid(env, &meta, txnid);
+ current_geo = &meta.mm_geo;
+ } else {
+ current_geo = &env->me_txn->mt_geo;
+ }
- end_mode = MDBX_END_COMMITTED | MDBX_END_UPDATE | MDBX_END_EOTDONE;
+ MDBX_geo new_geo;
+ new_geo.lower = bytes2pgno(env, size_lower);
+ new_geo.now = bytes2pgno(env, size_now);
+ new_geo.upper = bytes2pgno(env, size_upper);
+ new_geo.grow_pv = pages2pv(bytes2pgno(env, growth_step));
+ new_geo.shrink_pv = pages2pv(bytes2pgno(env, shrink_threshold));
+ new_geo.next = current_geo->next;
-done:
- rc = mdbx_txn_end(txn, end_mode);
+ mdbx_ensure(env,
+ pgno_align2os_bytes(env, new_geo.lower) == (size_t)size_lower);
+ mdbx_ensure(env,
+ pgno_align2os_bytes(env, new_geo.upper) == (size_t)size_upper);
+ mdbx_ensure(env, pgno_align2os_bytes(env, new_geo.now) == (size_t)size_now);
+ mdbx_ensure(env, new_geo.grow_pv == pages2pv(pv2pages(new_geo.grow_pv)));
+ mdbx_ensure(env,
+ new_geo.shrink_pv == pages2pv(pv2pages(new_geo.shrink_pv)));
-provide_latency:
- if (latency) {
- latency->audit = audit_duration;
- latency->preparation =
- ts_1 ? mdbx_osal_monotime_to_16dot16(ts_1 - ts_0) : 0;
- latency->gc =
- (ts_1 && ts_2) ? mdbx_osal_monotime_to_16dot16(ts_2 - ts_1) : 0;
- latency->write =
- (ts_2 && ts_3) ? mdbx_osal_monotime_to_16dot16(ts_3 - ts_2) : 0;
- latency->sync =
- (ts_3 && ts_4) ? mdbx_osal_monotime_to_16dot16(ts_4 - ts_3) : 0;
- const uint64_t ts_5 = mdbx_osal_monotime();
- latency->ending = ts_4 ? mdbx_osal_monotime_to_16dot16(ts_5 - ts_4) : 0;
- latency->whole = mdbx_osal_monotime_to_16dot16(ts_5 - ts_0);
- }
- return rc;
+ mdbx_ensure(env, (size_t)size_lower >= MIN_MAPSIZE);
+ mdbx_ensure(env, new_geo.lower >= MIN_PAGENO);
+ mdbx_ensure(env, (size_t)size_upper <= MAX_MAPSIZE);
+ mdbx_ensure(env, new_geo.upper <= MAX_PAGENO + 1);
+ mdbx_ensure(env, new_geo.now >= new_geo.next);
+ mdbx_ensure(env, new_geo.upper >= new_geo.now);
+ mdbx_ensure(env, new_geo.now >= new_geo.lower);
-fail:
- mdbx_txn_abort(txn);
- goto provide_latency;
-}
+ if (memcmp(current_geo, &new_geo, sizeof(MDBX_geo)) != 0) {
+#if defined(_WIN32) || defined(_WIN64)
+ /* Was DB shrinking disabled before and now it will be enabled? */
+ if (new_geo.lower < new_geo.upper && new_geo.shrink_pv &&
+ !(current_geo->lower < current_geo->upper &&
+ current_geo->shrink_pv)) {
+ if (!env->me_lck_mmap.lck) {
+ rc = MDBX_EPERM;
+ goto bailout;
+ }
+ int err = mdbx_rdt_lock(env);
+ if (unlikely(MDBX_IS_ERROR(err))) {
+ rc = err;
+ goto bailout;
+ }
-static int mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta,
- const MDBX_page *const page,
- const unsigned meta_number,
- unsigned *guess_pagesize) {
- const uint64_t magic_and_version =
- unaligned_peek_u64(4, &meta->mm_magic_and_version);
- if (unlikely(magic_and_version != MDBX_DATA_MAGIC &&
- magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
- magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) {
- mdbx_error("meta[%u] has invalid magic/version %" PRIx64, meta_number,
- magic_and_version);
- return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID
- : MDBX_VERSION_MISMATCH;
+ /* Check if there are any reading threads that do not use the SRWL */
+ const size_t CurrentTid = GetCurrentThreadId();
+ const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers;
+ const MDBX_reader *const end =
+ begin + atomic_load32(&env->me_lck_mmap.lck->mti_numreaders,
+ mo_AcquireRelease);
+ for (const MDBX_reader *reader = begin; reader < end; ++reader) {
+ if (reader->mr_pid.weak == env->me_pid && reader->mr_tid.weak &&
+ reader->mr_tid.weak != CurrentTid) {
+ /* At least one thread may don't use SRWL */
+ rc = MDBX_EPERM;
+ break;
+ }
+ }
+
+ mdbx_rdt_unlock(env);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+#endif
+
+ if (new_geo.now != current_geo->now ||
+ new_geo.upper != current_geo->upper) {
+ rc = mdbx_mapresize(env, current_geo->next, new_geo.now, new_geo.upper,
+ false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ if (inside_txn) {
+ env->me_txn->mt_geo = new_geo;
+ env->me_txn->mt_flags |= MDBX_TXN_DIRTY;
+ } else {
+ meta.mm_geo = new_geo;
+ rc = mdbx_sync_locked(env, env->me_flags, &meta);
+ }
+
+ if (likely(rc == MDBX_SUCCESS)) {
+ /* store new geo to env to avoid influences */
+ env->me_dbgeo.now = pgno2bytes(env, new_geo.now);
+ env->me_dbgeo.lower = pgno2bytes(env, new_geo.lower);
+ env->me_dbgeo.upper = pgno2bytes(env, new_geo.upper);
+ env->me_dbgeo.grow = pgno2bytes(env, pv2pages(new_geo.grow_pv));
+ env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(new_geo.shrink_pv));
+ }
+ }
}
- if (unlikely(page->mp_pgno != meta_number)) {
- mdbx_error("meta[%u] has invalid pageno %" PRIaPGNO, meta_number,
- page->mp_pgno);
- return MDBX_INVALID;
- }
+bailout:
+ if (need_unlock)
+ mdbx_txn_unlock(env);
+ return rc;
+}
- if (unlikely(page->mp_flags != P_META)) {
- mdbx_error("page #%u not a meta-page", meta_number);
- return MDBX_INVALID;
- }
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) {
+ return __inline_mdbx_env_set_mapsize(env, size);
+}
- /* LY: check pagesize */
- if (unlikely(!is_powerof2(meta->mm_psize) || meta->mm_psize < MIN_PAGESIZE ||
- meta->mm_psize > MAX_PAGESIZE)) {
- mdbx_warning("meta[%u] has invalid pagesize (%u), skip it", meta_number,
- meta->mm_psize);
- return is_powerof2(meta->mm_psize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID;
- }
+__cold int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) {
+ return __inline_mdbx_env_set_maxdbs(env, dbs);
+}
- if (guess_pagesize && *guess_pagesize != meta->mm_psize) {
- *guess_pagesize = meta->mm_psize;
- mdbx_verbose("meta[%u] took pagesize %u", meta_number, meta->mm_psize);
- }
+__cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) {
+ return __inline_mdbx_env_get_maxdbs(env, dbs);
+}
- const txnid_t txnid = unaligned_peek_u64(4, &meta->mm_txnid_a);
- if (unlikely(txnid != unaligned_peek_u64(4, &meta->mm_txnid_b))) {
- mdbx_warning("meta[%u] not completely updated, skip it", meta_number);
- return MDBX_RESULT_TRUE;
- }
+__cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) {
+ return __inline_mdbx_env_set_maxreaders(env, readers);
+}
- /* LY: check signature as a checksum */
- if (META_IS_STEADY(meta) &&
- unlikely(unaligned_peek_u64(4, &meta->mm_datasync_sign) !=
- meta_sign(meta))) {
- mdbx_warning("meta[%u] has invalid steady-checksum (0x%" PRIx64
- " != 0x%" PRIx64 "), skip it",
- meta_number, unaligned_peek_u64(4, &meta->mm_datasync_sign),
- meta_sign(meta));
- return MDBX_RESULT_TRUE;
- }
+__cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) {
+ return __inline_mdbx_env_get_maxreaders(env, readers);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- mdbx_debug("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO
- ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
- " +%u -%u, txn_id %" PRIaTXN ", %s",
- page->mp_pgno, meta->mm_dbs[MAIN_DBI].md_root,
- meta->mm_dbs[FREE_DBI].md_root, meta->mm_geo.lower,
- meta->mm_geo.next, meta->mm_geo.now, meta->mm_geo.upper,
- pv2pages(meta->mm_geo.grow_pv), pv2pages(meta->mm_geo.shrink_pv),
- txnid, mdbx_durable_str(meta));
+__cold static int alloc_page_buf(MDBX_env *env) {
+ return env->me_pbuf
+ ? MDBX_SUCCESS
+ : mdbx_memalign_alloc(env->me_os_psize, env->me_psize * NUM_METAS,
+ &env->me_pbuf);
+}
- if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) {
- mdbx_warning("meta[%u] has invalid txnid %" PRIaTXN ", skip it",
- meta_number, txnid);
- return MDBX_RESULT_TRUE;
- }
+/* Further setup required for opening an MDBX environment */
+__cold static int mdbx_setup_dxb(MDBX_env *env, const int lck_rc,
+ const mdbx_mode_t mode_bits) {
+ MDBX_meta meta;
+ int rc = MDBX_RESULT_FALSE;
+ int err = mdbx_read_header(env, &meta, lck_rc, mode_bits);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA ||
+ (env->me_flags & MDBX_RDONLY) != 0 ||
+ /* recovery mode */ env->me_stuck_meta >= 0)
+ return err;
- /* LY: check min-pages value */
- if (unlikely(meta->mm_geo.lower < MIN_PAGENO ||
- meta->mm_geo.lower > MAX_PAGENO + 1)) {
- mdbx_warning("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it",
- meta_number, meta->mm_geo.lower);
- return MDBX_INVALID;
- }
+ mdbx_debug("%s", "create new database");
+ rc = /* new database */ MDBX_RESULT_TRUE;
- /* LY: check max-pages value */
- if (unlikely(meta->mm_geo.upper < MIN_PAGENO ||
- meta->mm_geo.upper > MAX_PAGENO + 1 ||
- meta->mm_geo.upper < meta->mm_geo.lower)) {
- mdbx_warning("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it",
- meta_number, meta->mm_geo.upper);
- return MDBX_INVALID;
- }
+ if (!env->me_dbgeo.now) {
+ /* set defaults if not configured */
+ err = mdbx_env_set_geometry(env, 0, -1, DEFAULT_MAPSIZE, -1, -1, -1);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
- /* LY: check last_pgno */
- if (unlikely(meta->mm_geo.next < MIN_PAGENO ||
- meta->mm_geo.next - 1 > MAX_PAGENO)) {
- mdbx_warning("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it",
- meta_number, meta->mm_geo.next);
- return MDBX_CORRUPTED;
- }
+ err = alloc_page_buf(env);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- /* LY: check filesize & used_bytes */
- const uint64_t used_bytes = meta->mm_geo.next * (uint64_t)meta->mm_psize;
- if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) {
- /* Here could be a race with DB-shrinking performed by other process */
- int err = mdbx_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize);
+ meta = *mdbx_init_metas(env, env->me_pbuf);
+ err = mdbx_pwrite(env->me_lazy_fd, env->me_pbuf, env->me_psize * NUM_METAS,
+ 0);
if (unlikely(err != MDBX_SUCCESS))
return err;
- if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) {
- mdbx_warning("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64
- "), skip it",
- meta_number, used_bytes, env->me_dxb_mmap.filesize);
- return MDBX_CORRUPTED;
- }
- }
- if (unlikely(meta->mm_geo.next - 1 > MAX_PAGENO ||
- used_bytes > MAX_MAPSIZE)) {
- mdbx_warning("meta[%u] has too large used-space (%" PRIu64 "), skip it",
- meta_number, used_bytes);
- return MDBX_TOO_LARGE;
+
+ err = mdbx_ftruncate(env->me_lazy_fd,
+ env->me_dxb_mmap.filesize = env->me_dbgeo.now);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+
+#ifndef NDEBUG /* just for checking */
+ err = mdbx_read_header(env, &meta, lck_rc, mode_bits);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+#endif
}
- /* LY: check mapsize limits */
- pgno_t geo_lower = meta->mm_geo.lower;
- uint64_t mapsize_min = geo_lower * (uint64_t)meta->mm_psize;
- STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MAX_PAGESIZE);
- STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
- STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MIN_PAGESIZE % (4ul << 20) == 0);
- if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) {
- if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE &&
- mapsize_min <= MAX_MAPSIZE64) {
- mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO &&
- used_bytes <= MAX_MAPSIZE);
- mdbx_warning("meta[%u] has too large min-mapsize (%" PRIu64 "), "
- "but size of used space still acceptable (%" PRIu64 ")",
- meta_number, mapsize_min, used_bytes);
- geo_lower = (pgno_t)((mapsize_min = MAX_MAPSIZE) / meta->mm_psize);
- if (geo_lower > MAX_PAGENO + 1) {
- geo_lower = MAX_PAGENO + 1;
- mapsize_min = geo_lower * (uint64_t)meta->mm_psize;
+ mdbx_verbose(
+ "header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO
+ "-%" PRIaPGNO "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s",
+ meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root,
+ meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now, meta.mm_geo.upper,
+ pv2pages(meta.mm_geo.grow_pv), pv2pages(meta.mm_geo.shrink_pv),
+ unaligned_peek_u64(4, meta.mm_txnid_a), mdbx_durable_str(&meta));
+
+ if (env->me_psize != meta.mm_psize)
+ mdbx_setup_pagesize(env, meta.mm_psize);
+ const size_t used_bytes = pgno2bytes(env, meta.mm_geo.next);
+ const size_t used_aligned2os_bytes =
+ ceil_powerof2(used_bytes, env->me_os_psize);
+ if ((env->me_flags & MDBX_RDONLY) /* readonly */
+ || lck_rc != MDBX_RESULT_TRUE /* not exclusive */
+ || /* recovery mode */ env->me_stuck_meta >= 0) {
+ /* use present params from db */
+ const size_t pagesize = meta.mm_psize;
+ err = mdbx_env_set_geometry(
+ env, meta.mm_geo.lower * pagesize, meta.mm_geo.now * pagesize,
+ meta.mm_geo.upper * pagesize, pv2pages(meta.mm_geo.grow_pv) * pagesize,
+ pv2pages(meta.mm_geo.shrink_pv) * pagesize, meta.mm_psize);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("%s: err %d", "could not apply preconfigured geometry from db",
+ err);
+ return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
+ }
+ } else if (env->me_dbgeo.now) {
+ /* silently growth to last used page */
+ if (env->me_dbgeo.now < used_aligned2os_bytes)
+ env->me_dbgeo.now = used_aligned2os_bytes;
+ if (env->me_dbgeo.upper < used_aligned2os_bytes)
+ env->me_dbgeo.upper = used_aligned2os_bytes;
+
+ /* apply preconfigured params, but only if substantial changes:
+ * - upper or lower limit changes
+ * - shrink threshold or growth step
+ * But ignore change just a 'now/current' size. */
+ if (bytes_align2os_bytes(env, env->me_dbgeo.upper) !=
+ pgno2bytes(env, meta.mm_geo.upper) ||
+ bytes_align2os_bytes(env, env->me_dbgeo.lower) !=
+ pgno2bytes(env, meta.mm_geo.lower) ||
+ bytes_align2os_bytes(env, env->me_dbgeo.shrink) !=
+ pgno2bytes(env, pv2pages(meta.mm_geo.shrink_pv)) ||
+ bytes_align2os_bytes(env, env->me_dbgeo.grow) !=
+ pgno2bytes(env, pv2pages(meta.mm_geo.grow_pv))) {
+
+ if (env->me_dbgeo.shrink && env->me_dbgeo.now > used_bytes)
+ /* pre-shrink if enabled */
+ env->me_dbgeo.now = used_bytes + env->me_dbgeo.shrink -
+ used_bytes % env->me_dbgeo.shrink;
+
+ err = mdbx_env_set_geometry(env, env->me_dbgeo.lower, env->me_dbgeo.now,
+ env->me_dbgeo.upper, env->me_dbgeo.grow,
+ env->me_dbgeo.shrink, meta.mm_psize);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("%s: err %d", "could not apply preconfigured db-geometry",
+ err);
+ return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
}
- mdbx_warning("meta[%u] consider get-%s pageno is %" PRIaPGNO
- " instead of wrong %" PRIaPGNO
- ", will be corrected on next commit(s)",
- meta_number, "lower", geo_lower, meta->mm_geo.lower);
- meta->mm_geo.lower = geo_lower;
+
+ /* update meta fields */
+ meta.mm_geo.now = bytes2pgno(env, env->me_dbgeo.now);
+ meta.mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower);
+ meta.mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper);
+ meta.mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow));
+ meta.mm_geo.shrink_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.shrink));
+
+ mdbx_verbose("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO
+ "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
+ " +%u -%u, txn_id %" PRIaTXN ", %s",
+ meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root,
+ meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now,
+ meta.mm_geo.upper, pv2pages(meta.mm_geo.grow_pv),
+ pv2pages(meta.mm_geo.shrink_pv),
+ unaligned_peek_u64(4, meta.mm_txnid_a),
+ mdbx_durable_str(&meta));
} else {
- mdbx_warning("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it",
- meta_number, mapsize_min);
- return MDBX_VERSION_MISMATCH;
+ /* fetch back 'now/current' size, since it was ignored during comparison
+ * and may differ. */
+ env->me_dbgeo.now = pgno_align2os_bytes(env, meta.mm_geo.now);
}
+ mdbx_ensure(env, meta.mm_geo.now >= meta.mm_geo.next);
+ } else {
+ /* geo-params are not pre-configured by user,
+ * get current values from the meta. */
+ env->me_dbgeo.now = pgno2bytes(env, meta.mm_geo.now);
+ env->me_dbgeo.lower = pgno2bytes(env, meta.mm_geo.lower);
+ env->me_dbgeo.upper = pgno2bytes(env, meta.mm_geo.upper);
+ env->me_dbgeo.grow = pgno2bytes(env, pv2pages(meta.mm_geo.grow_pv));
+ env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(meta.mm_geo.shrink_pv));
}
- pgno_t geo_upper = meta->mm_geo.upper;
- uint64_t mapsize_max = geo_upper * (uint64_t)meta->mm_psize;
- STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
- if (unlikely(mapsize_max > MAX_MAPSIZE ||
- (MAX_PAGENO + 1) <
- ceil_powerof2((size_t)mapsize_max, env->me_os_psize) /
- (size_t)meta->mm_psize)) {
- if (mapsize_max > MAX_MAPSIZE64) {
- mdbx_warning("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it",
- meta_number, mapsize_max);
- return MDBX_VERSION_MISMATCH;
- }
- /* allow to open large DB from a 32-bit environment */
- mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO &&
- used_bytes <= MAX_MAPSIZE);
- mdbx_warning("meta[%u] has too large max-mapsize (%" PRIu64 "), "
- "but size of used space still acceptable (%" PRIu64 ")",
- meta_number, mapsize_max, used_bytes);
- geo_upper = (pgno_t)((mapsize_max = MAX_MAPSIZE) / meta->mm_psize);
- if (geo_upper > MAX_PAGENO + 1) {
- geo_upper = MAX_PAGENO + 1;
- mapsize_max = geo_upper * (uint64_t)meta->mm_psize;
+ mdbx_ensure(env,
+ pgno_align2os_bytes(env, meta.mm_geo.now) == env->me_dbgeo.now);
+ mdbx_ensure(env, env->me_dbgeo.now >= used_bytes);
+ const uint64_t filesize_before = env->me_dxb_mmap.filesize;
+ if (unlikely(filesize_before != env->me_dbgeo.now)) {
+ if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
+ mdbx_verbose("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO
+ "p, have %" PRIu64 "b/%" PRIaPGNO "p), "
+ "assume other process working",
+ env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now),
+ filesize_before, bytes2pgno(env, (size_t)filesize_before));
+ } else {
+ mdbx_warning("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO
+ "p, have %" PRIu64 "b/%" PRIaPGNO "p)",
+ env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now),
+ filesize_before, bytes2pgno(env, (size_t)filesize_before));
+ if (filesize_before < used_bytes) {
+ mdbx_error("last-page beyond end-of-file (last %" PRIaPGNO
+ ", have %" PRIaPGNO ")",
+ meta.mm_geo.next, bytes2pgno(env, (size_t)filesize_before));
+ return MDBX_CORRUPTED;
+ }
+
+ if (env->me_flags & MDBX_RDONLY) {
+ if (filesize_before & (env->me_os_psize - 1)) {
+ mdbx_error("%s", "filesize should be rounded-up to system page");
+ return MDBX_WANNA_RECOVERY;
+ }
+ mdbx_warning("%s", "ignore filesize mismatch in readonly-mode");
+ } else {
+ mdbx_verbose("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO
+ " pages",
+ env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now));
+ }
}
- mdbx_warning("meta[%u] consider get-%s pageno is %" PRIaPGNO
- " instead of wrong %" PRIaPGNO
- ", will be corrected on next commit(s)",
- meta_number, "upper", geo_upper, meta->mm_geo.upper);
- meta->mm_geo.upper = geo_upper;
}
- /* LY: check and silently put mm_geo.now into [geo.lower...geo.upper].
- *
- * Copy-with-compaction by previous version of libmdbx could produce DB-file
- * less than meta.geo.lower bound, in case actual filling is low or no data
- * at all. This is not a problem as there is no damage or loss of data.
- * Therefore it is better not to consider such situation as an error, but
- * silently correct it. */
- pgno_t geo_now = meta->mm_geo.now;
- if (geo_now < geo_lower)
- geo_now = geo_lower;
- if (geo_now > geo_upper && meta->mm_geo.next <= geo_upper)
- geo_now = geo_upper;
+ mdbx_verbose("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)",
+ bootid.x, bootid.y, (bootid.x | bootid.y) ? "" : "not-");
- if (unlikely(meta->mm_geo.next > geo_now)) {
- mdbx_warning("meta[%u] next-pageno (%" PRIaPGNO
- ") is beyond end-pgno (%" PRIaPGNO "), skip it",
- meta_number, meta->mm_geo.next, geo_now);
- return MDBX_CORRUPTED;
- }
- if (meta->mm_geo.now != geo_now) {
- mdbx_warning("meta[%u] consider geo-%s pageno is %" PRIaPGNO
- " instead of wrong %" PRIaPGNO
- ", will be corrected on next commit(s)",
- meta_number, "now", geo_now, meta->mm_geo.now);
- meta->mm_geo.now = geo_now;
- }
+#if MDBX_ENABLE_MADVISE
+ /* calculate readahead hint before mmap with zero redundant pages */
+ const bool readahead =
+ !(env->me_flags & MDBX_NORDAHEAD) &&
+ mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE;
+#endif /* MDBX_ENABLE_MADVISE */
- /* GC */
- if (meta->mm_dbs[FREE_DBI].md_root == P_INVALID) {
- if (unlikely(meta->mm_dbs[FREE_DBI].md_branch_pages ||
- meta->mm_dbs[FREE_DBI].md_depth ||
- meta->mm_dbs[FREE_DBI].md_entries ||
- meta->mm_dbs[FREE_DBI].md_leaf_pages ||
- meta->mm_dbs[FREE_DBI].md_overflow_pages)) {
- mdbx_warning("meta[%u] has false-empty %s, skip it", meta_number, "GC");
- return MDBX_CORRUPTED;
- }
- } else if (unlikely(meta->mm_dbs[FREE_DBI].md_root >= meta->mm_geo.next)) {
- mdbx_warning("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it",
- meta_number, "GC", meta->mm_dbs[FREE_DBI].md_root);
- return MDBX_CORRUPTED;
- }
+ err = mdbx_mmap(env->me_flags, &env->me_dxb_mmap, env->me_dbgeo.now,
+ env->me_dbgeo.upper, lck_rc ? MMAP_OPTION_TRUNCATE : 0);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- /* MainDB */
- if (meta->mm_dbs[MAIN_DBI].md_root == P_INVALID) {
- if (unlikely(meta->mm_dbs[MAIN_DBI].md_branch_pages ||
- meta->mm_dbs[MAIN_DBI].md_depth ||
- meta->mm_dbs[MAIN_DBI].md_entries ||
- meta->mm_dbs[MAIN_DBI].md_leaf_pages ||
- meta->mm_dbs[MAIN_DBI].md_overflow_pages)) {
- mdbx_warning("meta[%u] has false-empty %s", meta_number, "MainDB");
- return MDBX_CORRUPTED;
- }
- } else if (unlikely(meta->mm_dbs[MAIN_DBI].md_root >= meta->mm_geo.next)) {
- mdbx_warning("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it",
- meta_number, "MainDB", meta->mm_dbs[MAIN_DBI].md_root);
- return MDBX_CORRUPTED;
+#if MDBX_ENABLE_MADVISE
+#if defined(MADV_DONTDUMP)
+ err = madvise(env->me_map, env->me_dxb_mmap.limit, MADV_DONTDUMP)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#endif /* MADV_DONTDUMP */
+#if defined(MADV_DODUMP)
+ if (mdbx_runtime_flags & MDBX_DBG_DUMP) {
+ const size_t meta_length_aligned2os = pgno_align2os_bytes(env, NUM_METAS);
+ err = madvise(env->me_map, meta_length_aligned2os, MADV_DODUMP)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
}
+#endif /* MADV_DODUMP */
+#endif /* MDBX_ENABLE_MADVISE */
- if (unlikely(meta->mm_dbs[FREE_DBI].md_mod_txnid > txnid)) {
- mdbx_warning("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it",
- meta_number, meta->mm_dbs[FREE_DBI].md_mod_txnid, "GC");
- return MDBX_CORRUPTED;
- }
+#ifdef MDBX_USE_VALGRIND
+ env->me_valgrind_handle =
+ VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx");
+#endif /* MDBX_USE_VALGRIND */
- if (unlikely(meta->mm_dbs[MAIN_DBI].md_mod_txnid > txnid)) {
- mdbx_warning("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it",
- meta_number, meta->mm_dbs[MAIN_DBI].md_mod_txnid, "MainDB");
- return MDBX_CORRUPTED;
+ mdbx_assert(env, used_bytes >= pgno2bytes(env, NUM_METAS) &&
+ used_bytes <= env->me_dxb_mmap.limit);
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ if (env->me_dxb_mmap.filesize > used_bytes &&
+ env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit) {
+ VALGRIND_MAKE_MEM_NOACCESS(env->me_map + used_bytes,
+ env->me_dxb_mmap.filesize - used_bytes);
+ MDBX_ASAN_POISON_MEMORY_REGION(env->me_map + used_bytes,
+ env->me_dxb_mmap.filesize - used_bytes);
}
+ env->me_poison_edge =
+ bytes2pgno(env, (env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit)
+ ? env->me_dxb_mmap.filesize
+ : env->me_dxb_mmap.limit);
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
- return MDBX_SUCCESS;
-}
+ //-------------------------------- validate/rollback head & steady meta-pages
+ if (unlikely(env->me_stuck_meta >= 0)) {
+ /* recovery mode */
+ MDBX_meta clone;
+ MDBX_meta const *const target = METAPAGE(env, env->me_stuck_meta);
+ err = mdbx_validate_meta_copy(env, target, &clone);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("target meta[%u] is corrupted",
+ bytes2pgno(env, (uint8_t *)data_page(target) - env->me_map));
+ return MDBX_CORRUPTED;
+ }
+ } else /* not recovery mode */
+ while (1) {
+ const unsigned meta_clash_mask = meta_eq_mask(env);
+ if (unlikely(meta_clash_mask)) {
+ mdbx_error("meta-pages are clashed: mask 0x%d", meta_clash_mask);
+ return MDBX_CORRUPTED;
+ }
-static int mdbx_validate_meta_copy(MDBX_env *env, const MDBX_meta *meta,
- MDBX_meta *dest) {
- *dest = *meta;
- return mdbx_validate_meta(env, dest, data_page(meta),
- bytes2pgno(env, (uint8_t *)meta - env->me_map),
- nullptr);
-}
+ if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
+ /* non-exclusive mode,
+ * meta-pages should be validated by a first process opened the DB */
+ volatile const MDBX_meta *const head = meta_prefer_last(env);
+ volatile const MDBX_meta *const steady = meta_prefer_steady(env);
+ const txnid_t head_txnid = meta_txnid(env, head);
+ const txnid_t steady_txnid = meta_txnid(env, steady);
+ if (head_txnid == steady_txnid)
+ break;
-/* Read the environment parameters of a DB environment
- * before mapping it into memory. */
-__cold static int mdbx_read_header(MDBX_env *env, MDBX_meta *dest,
- const int lck_exclusive,
- const mdbx_mode_t mode_bits) {
- int rc = mdbx_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (!env->me_lck_mmap.lck) {
+ /* LY: without-lck (read-only) mode, so it is impossible that other
+ * process made weak checkpoint. */
+ mdbx_error("%s", "without-lck, unable recovery/rollback");
+ return MDBX_WANNA_RECOVERY;
+ }
- memset(dest, 0, sizeof(MDBX_meta));
- unaligned_poke_u64(4, dest->mm_datasync_sign, MDBX_DATASIGN_WEAK);
- rc = MDBX_CORRUPTED;
+ /* LY: assume just have a collision with other running process,
+ * or someone make a weak checkpoint */
+ mdbx_verbose("%s", "assume collision or online weak checkpoint");
+ break;
+ }
+ mdbx_assert(env, lck_rc == MDBX_RESULT_TRUE);
+ /* exclusive mode */
+
+ MDBX_meta clone;
+ const MDBX_meta *const steady = constmeta_prefer_steady(env);
+ const MDBX_meta *const head = constmeta_prefer_last(env);
+ const txnid_t steady_txnid = meta_txnid(env, steady);
+ if (META_IS_STEADY(steady)) {
+ err = mdbx_validate_meta_copy(env, steady, &clone);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("meta[%u] with %s txnid %" PRIaTXN
+ " is corrupted, %s needed",
+ bytes2pgno(env, (uint8_t *)steady - env->me_map), "steady",
+ steady_txnid, "manual recovery");
+ return MDBX_CORRUPTED;
+ }
+ if (steady == head)
+ break;
+ }
- /* Read twice all meta pages so we can find the latest one. */
- unsigned loop_limit = NUM_METAS * 2;
- /* We don't know the page size on first time. So, just guess it. */
- unsigned guess_pagesize = 0;
- for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) {
- const unsigned meta_number = loop_count % NUM_METAS;
- const unsigned offset = (guess_pagesize ? guess_pagesize
- : (loop_count > NUM_METAS) ? env->me_psize
- : env->me_os_psize) *
- meta_number;
+ const pgno_t pgno = bytes2pgno(env, (uint8_t *)head - env->me_map);
+ const txnid_t head_txnid = meta_txnid(env, head);
+ const bool head_valid =
+ mdbx_validate_meta_copy(env, head, &clone) == MDBX_SUCCESS;
+ mdbx_assert(env, !META_IS_STEADY(steady) || head_txnid != steady_txnid);
+ if (unlikely(!head_valid)) {
+ if (unlikely(!META_IS_STEADY(steady))) {
+ mdbx_error("%s for open or automatic rollback, %s",
+ "there are no suitable meta-pages",
+ "manual recovery is required");
+ return MDBX_CORRUPTED;
+ }
+ mdbx_warning("meta[%u] with last txnid %" PRIaTXN
+ " is corrupted, rollback needed",
+ pgno, head_txnid);
+ goto purge_meta_head;
+ }
- char buffer[MIN_PAGESIZE];
- unsigned retryleft = 42;
- while (1) {
- mdbx_trace("reading meta[%d]: offset %u, bytes %u, retry-left %u",
- meta_number, offset, MIN_PAGESIZE, retryleft);
- int err = mdbx_pread(env->me_lazy_fd, buffer, MIN_PAGESIZE, offset);
- if (err != MDBX_SUCCESS) {
- if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 &&
- env->me_dxb_mmap.filesize == 0 &&
- mode_bits /* non-zero for DB creation */ != 0)
- mdbx_notice("read meta: empty file (%d, %s)", err,
- mdbx_strerror(err));
- else
- mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err,
- mdbx_strerror(err));
- return err;
+ if (meta_bootid_match(head)) {
+ if (env->me_flags & MDBX_RDONLY) {
+ mdbx_error("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
+ "rollback NOT needed, steady-sync NEEDED%s",
+ "opening after an unclean shutdown", bootid.x, bootid.y,
+ ", but unable in read-only mode");
+ return MDBX_WANNA_RECOVERY;
+ }
+ mdbx_warning("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
+ "rollback NOT needed, steady-sync NEEDED%s",
+ "opening after an unclean shutdown", bootid.x, bootid.y,
+ "");
+ meta = clone;
+ atomic_store32(&env->me_lck->mti_unsynced_pages, meta.mm_geo.next,
+ mo_Relaxed);
+ break;
+ }
+ if (unlikely(!META_IS_STEADY(steady))) {
+ mdbx_error("%s, but %s for automatic rollback: %s",
+ "opening after an unclean shutdown",
+ "there are no suitable meta-pages",
+ "manual recovery is required");
+ return MDBX_CORRUPTED;
+ }
+ if (env->me_flags & MDBX_RDONLY) {
+ mdbx_error("%s and rollback needed: (from head %" PRIaTXN
+ " to steady %" PRIaTXN ")%s",
+ "opening after an unclean shutdown", head_txnid,
+ steady_txnid, ", but unable in read-only mode");
+ return MDBX_WANNA_RECOVERY;
}
- char again[MIN_PAGESIZE];
- err = mdbx_pread(env->me_lazy_fd, again, MIN_PAGESIZE, offset);
- if (err != MDBX_SUCCESS) {
- mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err,
- mdbx_strerror(err));
+ purge_meta_head:
+ mdbx_notice("%s and doing automatic rollback: "
+ "purge%s meta[%u] with%s txnid %" PRIaTXN,
+ "opening after an unclean shutdown",
+ head_valid ? "" : " invalid", pgno, head_valid ? " weak" : "",
+ head_txnid);
+ mdbx_ensure(env, META_IS_STEADY(steady));
+ err = mdbx_override_meta(env, pgno, 0, head_valid ? head : steady);
+ if (err) {
+ mdbx_error("rollback: overwrite meta[%u] with txnid %" PRIaTXN
+ ", error %d",
+ pgno, head_txnid, err);
return err;
}
+ mdbx_ensure(env, 0 == meta_txnid(env, head));
+ mdbx_ensure(env, 0 == meta_eq_mask(env));
+ }
- if (memcmp(buffer, again, MIN_PAGESIZE) == 0 || --retryleft == 0)
- break;
-
- mdbx_verbose("meta[%u] was updated, re-read it", meta_number);
+ if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) {
+ //-------------------------------------------------- shrink DB & update geo
+ const MDBX_meta *head = constmeta_prefer_last(env);
+ /* re-check size after mmap */
+ if ((env->me_dxb_mmap.current & (env->me_os_psize - 1)) != 0 ||
+ env->me_dxb_mmap.current < used_bytes) {
+ mdbx_error("unacceptable/unexpected datafile size %" PRIuPTR,
+ env->me_dxb_mmap.current);
+ return MDBX_PROBLEM;
+ }
+ if (env->me_dxb_mmap.current != env->me_dbgeo.now) {
+ meta.mm_geo.now = bytes2pgno(env, env->me_dxb_mmap.current);
+ mdbx_notice("need update meta-geo to filesize %" PRIuPTR
+ " bytes, %" PRIaPGNO " pages",
+ env->me_dxb_mmap.current, meta.mm_geo.now);
}
- if (!retryleft) {
- mdbx_error("meta[%u] is too volatile, skip it", meta_number);
- continue;
+ if (memcmp(&meta.mm_geo, &head->mm_geo, sizeof(meta.mm_geo))) {
+ if ((env->me_flags & MDBX_RDONLY) != 0 ||
+ /* recovery mode */ env->me_stuck_meta >= 0) {
+ mdbx_warning(
+ "skipped update meta.geo in %s mode: from l%" PRIaPGNO
+ "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u, to l%" PRIaPGNO
+ "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u",
+ (env->me_stuck_meta < 0) ? "read-only" : "recovery",
+ head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper,
+ pv2pages(head->mm_geo.shrink_pv), pv2pages(head->mm_geo.grow_pv),
+ meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper,
+ pv2pages(meta.mm_geo.shrink_pv), pv2pages(meta.mm_geo.grow_pv));
+ } else {
+ const txnid_t txnid = constmeta_txnid(env, head);
+ const txnid_t next_txnid = safe64_txnid_next(txnid);
+ if (unlikely(txnid > MAX_TXNID)) {
+ mdbx_error("txnid overflow, raise %d", MDBX_TXN_FULL);
+ return MDBX_TXN_FULL;
+ }
+ mdbx_notice("updating meta.geo: "
+ "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
+ "/s%u-g%u (txn#%" PRIaTXN "), "
+ "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
+ "/s%u-g%u (txn#%" PRIaTXN ")",
+ head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper,
+ pv2pages(head->mm_geo.shrink_pv),
+ pv2pages(head->mm_geo.grow_pv), txnid, meta.mm_geo.lower,
+ meta.mm_geo.now, meta.mm_geo.upper,
+ pv2pages(meta.mm_geo.shrink_pv),
+ pv2pages(meta.mm_geo.grow_pv), next_txnid);
+
+ mdbx_ensure(env, meta_eq(env, &meta, head));
+ meta_set_txnid(env, &meta, next_txnid);
+ err = mdbx_sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &meta);
+ if (err) {
+ mdbx_error("error %d, while updating meta.geo: "
+ "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
+ "/s%u-g%u (txn#%" PRIaTXN "), "
+ "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
+ "/s%u-g%u (txn#%" PRIaTXN ")",
+ err, head->mm_geo.lower, head->mm_geo.now,
+ head->mm_geo.upper, pv2pages(head->mm_geo.shrink_pv),
+ pv2pages(head->mm_geo.grow_pv), txnid, meta.mm_geo.lower,
+ meta.mm_geo.now, meta.mm_geo.upper,
+ pv2pages(meta.mm_geo.shrink_pv),
+ pv2pages(meta.mm_geo.grow_pv), next_txnid);
+ return err;
+ }
+ }
}
- MDBX_page *const page = (MDBX_page *)buffer;
- MDBX_meta *const meta = page_meta(page);
- rc = mdbx_validate_meta(env, meta, page, meta_number, &guess_pagesize);
- if (rc != MDBX_SUCCESS)
- continue;
+ atomic_store32(&env->me_lck->mti_discarded_tail,
+ bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed);
- if ((env->me_stuck_meta < 0)
- ? meta_ot(meta_bootid_match(meta) ? prefer_last : prefer_steady,
- env, dest, meta)
- : (meta_number == (unsigned)env->me_stuck_meta)) {
- *dest = *meta;
- if (!lck_exclusive && !META_IS_STEADY(dest))
- loop_limit += 1; /* LY: should re-read to hush race with update */
- mdbx_verbose("latch meta[%u]", meta_number);
+ if ((env->me_flags & MDBX_RDONLY) == 0 && env->me_stuck_meta < 0 &&
+ (mdbx_runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) {
+ for (int n = 0; n < NUM_METAS; ++n) {
+ MDBX_meta *const pmeta = METAPAGE(env, n);
+ if (unlikely(unaligned_peek_u64(4, &pmeta->mm_magic_and_version) !=
+ MDBX_DATA_MAGIC)) {
+ const txnid_t txnid = meta_txnid(env, pmeta);
+ mdbx_notice("%s %s"
+ "meta[%u], txnid %" PRIaTXN,
+ "updating db-format signature for",
+ META_IS_STEADY(pmeta) ? "stead-" : "weak-", n, txnid);
+ err = mdbx_override_meta(env, n, txnid, pmeta);
+ if (unlikely(err != MDBX_SUCCESS) &&
+ /* Just ignore the MDBX_PROBLEM error, since here it is
+ * returned only in case of the attempt to upgrade an obsolete
+ * meta-page that is invalid for current state of a DB,
+ * e.g. after shrinking DB file */
+ err != MDBX_PROBLEM) {
+ mdbx_error("%s meta[%u], txnid %" PRIaTXN ", error %d",
+ "updating db-format signature for", n, txnid, err);
+ return err;
+ }
+ }
+ }
}
- }
+ } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */
- if (dest->mm_psize == 0 ||
- (env->me_stuck_meta < 0 &&
- !(META_IS_STEADY(dest) ||
- meta_weak_acceptable(env, dest, lck_exclusive)))) {
- mdbx_error("%s", "no usable meta-pages, database is corrupted");
- if (rc == MDBX_SUCCESS) {
- /* TODO: try to restore the database by fully checking b-tree structure
- * for the each meta page, if the corresponding option was given */
- return MDBX_CORRUPTED;
+ //---------------------------------------------------- setup madvise/readahead
+#if MDBX_ENABLE_MADVISE
+ if (used_aligned2os_bytes < env->me_dxb_mmap.current) {
+#if defined(MADV_REMOVE)
+ if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0 &&
+ /* not recovery mode */ env->me_stuck_meta < 0) {
+ mdbx_notice("open-MADV_%s %u..%u", "REMOVE (deallocate file space)",
+ env->me_lck->mti_discarded_tail.weak,
+ bytes2pgno(env, env->me_dxb_mmap.current));
+ err =
+ madvise(env->me_map + used_aligned2os_bytes,
+ env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_REMOVE)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
}
- return rc;
+#endif /* MADV_REMOVE */
+#if defined(MADV_DONTNEED)
+ mdbx_notice("open-MADV_%s %u..%u", "DONTNEED",
+ env->me_lck->mti_discarded_tail.weak,
+ bytes2pgno(env, env->me_dxb_mmap.current));
+ err =
+ madvise(env->me_map + used_aligned2os_bytes,
+ env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_DONTNEED)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_MADV_DONTNEED)
+ err = ignore_enosys(posix_madvise(
+ env->me_map + used_aligned2os_bytes,
+ env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#elif defined(POSIX_FADV_DONTNEED)
+ err = ignore_enosys(posix_fadvise(
+ env->me_lazy_fd, used_aligned2os_bytes,
+ env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+#endif /* MADV_DONTNEED */
}
- return MDBX_SUCCESS;
-}
-
-__cold static MDBX_page *mdbx_meta_model(const MDBX_env *env, MDBX_page *model,
- unsigned num) {
- mdbx_ensure(env, is_powerof2(env->me_psize));
- mdbx_ensure(env, env->me_psize >= MIN_PAGESIZE);
- mdbx_ensure(env, env->me_psize <= MAX_PAGESIZE);
- mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE);
- mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE);
- mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower);
- mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper);
-
- memset(model, 0, env->me_psize);
- model->mp_pgno = num;
- model->mp_flags = P_META;
- MDBX_meta *const model_meta = page_meta(model);
- unaligned_poke_u64(4, model_meta->mm_magic_and_version, MDBX_DATA_MAGIC);
-
- model_meta->mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower);
- model_meta->mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper);
- model_meta->mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow));
- model_meta->mm_geo.shrink_pv =
- pages2pv(bytes2pgno(env, env->me_dbgeo.shrink));
- model_meta->mm_geo.now = bytes2pgno(env, env->me_dbgeo.now);
- model_meta->mm_geo.next = NUM_METAS;
-
- mdbx_ensure(env, model_meta->mm_geo.lower >= MIN_PAGENO);
- mdbx_ensure(env, model_meta->mm_geo.upper <= MAX_PAGENO + 1);
- mdbx_ensure(env, model_meta->mm_geo.now >= model_meta->mm_geo.lower);
- mdbx_ensure(env, model_meta->mm_geo.now <= model_meta->mm_geo.upper);
- mdbx_ensure(env, model_meta->mm_geo.next >= MIN_PAGENO);
- mdbx_ensure(env, model_meta->mm_geo.next <= model_meta->mm_geo.now);
- mdbx_ensure(env, model_meta->mm_geo.grow_pv ==
- pages2pv(pv2pages(model_meta->mm_geo.grow_pv)));
- mdbx_ensure(env, model_meta->mm_geo.shrink_pv ==
- pages2pv(pv2pages(model_meta->mm_geo.shrink_pv)));
-
- model_meta->mm_psize = env->me_psize;
- model_meta->mm_dbs[FREE_DBI].md_flags = MDBX_INTEGERKEY;
- model_meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
- model_meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
- meta_set_txnid(env, model_meta, MIN_TXNID + num);
- unaligned_poke_u64(4, model_meta->mm_datasync_sign, meta_sign(model_meta));
- mdbx_assert(env, meta_checktxnid(env, model_meta, true));
- return (MDBX_page *)((uint8_t *)model + env->me_psize);
-}
+ err = mdbx_set_readahead(env, bytes2pgno(env, used_bytes), readahead, true);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+#endif /* MDBX_ENABLE_MADVISE */
-/* Fill in most of the zeroed meta-pages for an empty database environment.
- * Return pointer to recently (head) meta-page. */
-__cold static MDBX_meta *mdbx_init_metas(const MDBX_env *env, void *buffer) {
- MDBX_page *page0 = (MDBX_page *)buffer;
- MDBX_page *page1 = mdbx_meta_model(env, page0, 0);
- MDBX_page *page2 = mdbx_meta_model(env, page1, 1);
- mdbx_meta_model(env, page2, 2);
- mdbx_assert(env, !meta_eq(env, page_meta(page0), page_meta(page1)));
- mdbx_assert(env, !meta_eq(env, page_meta(page1), page_meta(page2)));
- mdbx_assert(env, !meta_eq(env, page_meta(page2), page_meta(page0)));
- return page_meta(page2);
+ return rc;
}
-#if MDBX_ENABLE_MADVISE && !(defined(_WIN32) || defined(_WIN64))
-static size_t mdbx_madvise_threshold(const MDBX_env *env,
- const size_t largest_bytes) {
- /* TODO: use options */
- const unsigned factor = 9;
- const size_t threshold = (largest_bytes < (65536ul << factor))
- ? 65536 /* minimal threshold */
- : (largest_bytes > (MEGABYTE * 4 << factor))
- ? MEGABYTE * 4 /* maximal threshold */
- : largest_bytes >> factor;
- return bytes_align2os_bytes(env, threshold);
-}
-#endif /* MDBX_ENABLE_MADVISE */
+/******************************************************************************/
-static int mdbx_sync_locked(MDBX_env *env, unsigned flags,
- MDBX_meta *const pending) {
- mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0);
- const MDBX_meta *const meta0 = METAPAGE(env, 0);
- const MDBX_meta *const meta1 = METAPAGE(env, 1);
- const MDBX_meta *const meta2 = METAPAGE(env, 2);
- const MDBX_meta *const head = constmeta_prefer_last(env);
- int rc;
+/* Open and/or initialize the lock region for the environment. */
+__cold static int mdbx_setup_lck(MDBX_env *env, char *lck_pathname,
+ mdbx_mode_t mode) {
+ mdbx_assert(env, env->me_lazy_fd != INVALID_HANDLE_VALUE);
+ mdbx_assert(env, env->me_lfd == INVALID_HANDLE_VALUE);
- mdbx_assert(env, meta_eq_mask(env) == 0);
- mdbx_assert(env,
- pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS));
- mdbx_assert(env, (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0);
- mdbx_assert(env, pending->mm_geo.next <= pending->mm_geo.now);
+ int err = mdbx_openfile(MDBX_OPEN_LCK, env, lck_pathname, &env->me_lfd, mode);
+ if (err != MDBX_SUCCESS) {
+ if (!(err == MDBX_ENOFILE && (env->me_flags & MDBX_EXCLUSIVE)) &&
+ !((err == MDBX_EROFS || err == MDBX_EACCESS || err == MDBX_EPERM) &&
+ (env->me_flags & MDBX_RDONLY)))
+ return err;
- if (flags & MDBX_SAFE_NOSYNC) {
- /* Check auto-sync conditions */
- const pgno_t autosync_threshold =
- atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed);
- const uint64_t autosync_period =
- atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed);
- if ((autosync_threshold &&
- atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed) >=
- autosync_threshold) ||
- (autosync_period &&
- mdbx_osal_monotime() -
- atomic_load64(&env->me_lck->mti_sync_timestamp, mo_Relaxed) >=
- autosync_period))
- flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; /* force steady */
- }
+ /* ensure the file system is read-only */
+ err = mdbx_check_fs_rdonly(env->me_lazy_fd, lck_pathname, err);
+ if (err != MDBX_SUCCESS &&
+ /* ignore ERROR_NOT_SUPPORTED for exclusive mode */
+ !(err == MDBX_ENOSYS && (env->me_flags & MDBX_EXCLUSIVE)))
+ return err;
- pgno_t shrink = 0;
- if (flags & MDBX_SHRINK_ALLOWED) {
- /* LY: check conditions to discard unused pages */
- const pgno_t largest_pgno = mdbx_find_largest(
- env, (head->mm_geo.next > pending->mm_geo.next) ? head->mm_geo.next
- : pending->mm_geo.next);
- mdbx_assert(env, largest_pgno >= NUM_METAS);
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- const pgno_t edge = env->me_poison_edge;
- if (edge > largest_pgno) {
- env->me_poison_edge = largest_pgno;
- VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, largest_pgno),
- pgno2bytes(env, edge - largest_pgno));
- MDBX_ASAN_POISON_MEMORY_REGION(env->me_map +
- pgno2bytes(env, largest_pgno),
- pgno2bytes(env, edge - largest_pgno));
- }
-#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
-#if MDBX_ENABLE_MADVISE && \
- (defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED))
- const size_t largest_bytes = pgno2bytes(env, largest_pgno);
- /* threshold to avoid unreasonable frequent madvise() calls */
- const size_t madvise_threshold = mdbx_madvise_threshold(env, largest_bytes);
- const size_t discard_edge_bytes = bytes_align2os_bytes(
- env, ((MDBX_RDONLY &
- (env->me_lck_mmap.lck ? env->me_lck_mmap.lck->mti_envmode.weak
- : env->me_flags))
- ? largest_bytes
- : largest_bytes + madvise_threshold));
- const pgno_t discard_edge_pgno = bytes2pgno(env, discard_edge_bytes);
- const pgno_t prev_discarded_pgno =
- atomic_load32(&env->me_lck->mti_discarded_tail, mo_Relaxed);
- if (prev_discarded_pgno >=
- discard_edge_pgno + bytes2pgno(env, madvise_threshold)) {
- mdbx_notice("open-MADV_%s %u..%u", "DONTNEED", largest_pgno,
- prev_discarded_pgno);
- atomic_store32(&env->me_lck->mti_discarded_tail, discard_edge_pgno,
- mo_Relaxed);
- const size_t prev_discarded_bytes =
- ceil_powerof2(pgno2bytes(env, prev_discarded_pgno), env->me_os_psize);
- mdbx_ensure(env, prev_discarded_bytes > discard_edge_bytes);
-#if defined(MADV_DONTNEED)
- int advise = MADV_DONTNEED;
-#if defined(MADV_FREE) && \
- 0 /* MADV_FREE works for only anonymous vma at the moment */
- if ((env->me_flags & MDBX_WRITEMAP) &&
- mdbx_linux_kernel_version > 0x04050000)
- advise = MADV_FREE;
-#endif /* MADV_FREE */
- int err = madvise(env->me_map + discard_edge_bytes,
- prev_discarded_bytes - discard_edge_bytes, advise)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
-#else
- int err = ignore_enosys(posix_madvise(
- env->me_map + discard_edge_bytes,
- prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED));
-#endif
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
+ /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
+ /* beginning of a locked section ---------------------------------------- */
+ lcklist_lock();
+ mdbx_assert(env, env->me_lcklist_next == nullptr);
+ env->me_lfd = INVALID_HANDLE_VALUE;
+ const int rc = mdbx_lck_seize(env);
+ if (MDBX_IS_ERROR(rc)) {
+ /* Calling lcklist_detach_locked() is required to restore POSIX-filelock
+ * and this job will be done by mdbx_env_close0(). */
+ lcklist_unlock();
+ return rc;
}
-#endif /* MDBX_ENABLE_MADVISE && (MADV_DONTNEED || POSIX_MADV_DONTNEED) */
+ /* insert into inprocess lck-list */
+ env->me_lcklist_next = inprocess_lcklist_head;
+ inprocess_lcklist_head = env;
+ lcklist_unlock();
+ /* end of a locked section ---------------------------------------------- */
- /* LY: check conditions to shrink datafile */
- const pgno_t backlog_gap = 3 + pending->mm_dbs[FREE_DBI].md_depth * 3;
- pgno_t shrink_step = 0;
- if (pending->mm_geo.shrink_pv &&
- pending->mm_geo.now - pending->mm_geo.next >
- (shrink_step = pv2pages(pending->mm_geo.shrink_pv)) + backlog_gap) {
- if (pending->mm_geo.now > largest_pgno &&
- pending->mm_geo.now - largest_pgno > shrink_step + backlog_gap) {
- pgno_t grow_step = 0;
- const pgno_t aligner =
- pending->mm_geo.grow_pv
- ? (grow_step = pv2pages(pending->mm_geo.grow_pv))
- : shrink_step;
- const pgno_t with_backlog_gap = largest_pgno + backlog_gap;
- const pgno_t aligned = pgno_align2os_pgno(
- env, with_backlog_gap + aligner - with_backlog_gap % aligner);
- const pgno_t bottom =
- (aligned > pending->mm_geo.lower) ? aligned : pending->mm_geo.lower;
- if (pending->mm_geo.now > bottom) {
- if (META_IS_STEADY(meta_prefer_steady(env)))
- /* force steady, but only if steady-checkpoint is present */
- flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED;
- shrink = pending->mm_geo.now - bottom;
- pending->mm_geo.now = bottom;
- if (unlikely(constmeta_txnid(env, head) ==
- unaligned_peek_u64(4, pending->mm_txnid_a))) {
- const txnid_t txnid =
- safe64_txnid_next(unaligned_peek_u64(4, pending->mm_txnid_a));
- mdbx_notice("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN,
- unaligned_peek_u64(4, pending->mm_txnid_a), txnid);
- mdbx_ensure(env, env->me_txn0->mt_owner != mdbx_thread_self() &&
- !env->me_txn);
- if (unlikely(txnid > MAX_TXNID)) {
- rc = MDBX_TXN_FULL;
- mdbx_error("txnid overflow, raise %d", rc);
- goto fail;
- }
- meta_set_txnid(env, pending, txnid);
- mdbx_assert(env, meta_checktxnid(env, pending, true));
- }
- }
- }
- }
+ env->me_lck = lckless_stub(env);
+ env->me_maxreaders = UINT_MAX;
+ mdbx_debug("lck-setup:%s%s%s", " lck-less",
+ (env->me_flags & MDBX_RDONLY) ? " readonly" : "",
+ (rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");
+ return rc;
+ }
+
+ /* beginning of a locked section ------------------------------------------ */
+ lcklist_lock();
+ mdbx_assert(env, env->me_lcklist_next == nullptr);
+
+ /* Try to get exclusive lock. If we succeed, then
+ * nobody is using the lock region and we should initialize it. */
+ err = mdbx_lck_seize(env);
+ if (MDBX_IS_ERROR(err)) {
+ bailout:
+ /* Calling lcklist_detach_locked() is required to restore POSIX-filelock
+ * and this job will be done by mdbx_env_close0(). */
+ lcklist_unlock();
+ return err;
}
- /* LY: step#1 - sync previously written/updated data-pages */
- rc = MDBX_RESULT_FALSE /* carry steady */;
- if (atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed)) {
- mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
- enum mdbx_syncmode_bits mode_bits = MDBX_SYNC_NONE;
- if ((flags & MDBX_SAFE_NOSYNC) == 0) {
- mode_bits = MDBX_SYNC_DATA;
- if (pending->mm_geo.next > meta_prefer_steady(env)->mm_geo.now)
- mode_bits |= MDBX_SYNC_SIZE;
- if (flags & MDBX_NOMETASYNC)
- mode_bits |= MDBX_SYNC_IODQ;
+ MDBX_env *inprocess_neighbor = nullptr;
+ if (err == MDBX_RESULT_TRUE) {
+ err = uniq_check(&env->me_lck_mmap, &inprocess_neighbor);
+ if (MDBX_IS_ERROR(err))
+ goto bailout;
+ if (inprocess_neighbor &&
+ ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 ||
+ (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) != 0)) {
+ err = MDBX_BUSY;
+ goto bailout;
}
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- if (flags & MDBX_WRITEMAP)
- rc =
- mdbx_msync(&env->me_dxb_mmap, 0,
- pgno_align2os_bytes(env, pending->mm_geo.next), mode_bits);
- else
- rc = mdbx_fsync(env->me_lazy_fd, mode_bits);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */
- : MDBX_RESULT_FALSE /* carry steady */;
}
- mdbx_assert(env, meta_checktxnid(env, pending, true));
+ const int lck_seize_rc = err;
- /* Steady or Weak */
- if (rc == MDBX_RESULT_FALSE /* carry steady */) {
- atomic_store64(&env->me_lck->mti_sync_timestamp, mdbx_osal_monotime(),
- mo_Relaxed);
- unaligned_poke_u64(4, pending->mm_datasync_sign, meta_sign(pending));
- atomic_store32(&env->me_lck->mti_unsynced_pages, 0, mo_Relaxed);
- } else {
- assert(rc == MDBX_RESULT_TRUE /* carry non-steady */);
- unaligned_poke_u64(4, pending->mm_datasync_sign, MDBX_DATASIGN_WEAK);
- }
+ mdbx_debug("lck-setup:%s%s%s", " with-lck",
+ (env->me_flags & MDBX_RDONLY) ? " readonly" : "",
+ (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive"
+ : " cooperative");
- MDBX_meta *target = nullptr;
- if (constmeta_txnid(env, head) ==
- unaligned_peek_u64(4, pending->mm_txnid_a)) {
- mdbx_assert(env, memcmp(&head->mm_dbs, &pending->mm_dbs,
- sizeof(head->mm_dbs)) == 0);
- mdbx_assert(env, memcmp(&head->mm_canary, &pending->mm_canary,
- sizeof(head->mm_canary)) == 0);
- mdbx_assert(env, memcmp(&head->mm_geo, &pending->mm_geo,
- sizeof(pending->mm_geo)) == 0);
- if (!META_IS_STEADY(head) && META_IS_STEADY(pending))
- target = (MDBX_meta *)head;
- else {
- mdbx_ensure(env, meta_eq(env, head, pending));
- mdbx_debug("%s", "skip update meta");
- return MDBX_SUCCESS;
+ uint64_t size = 0;
+ err = mdbx_filesize(env->me_lfd, &size);
+ if (unlikely(err != MDBX_SUCCESS))
+ goto bailout;
+
+ if (lck_seize_rc == MDBX_RESULT_TRUE) {
+ size = ceil_powerof2(env->me_maxreaders * sizeof(MDBX_reader) +
+ sizeof(MDBX_lockinfo),
+ env->me_os_psize);
+ mdbx_jitter4testing(false);
+ } else {
+ if (env->me_flags & MDBX_EXCLUSIVE) {
+ err = MDBX_BUSY;
+ goto bailout;
+ }
+ if (size > INT_MAX || (size & (env->me_os_psize - 1)) != 0 ||
+ size < env->me_os_psize) {
+ mdbx_error("lck-file has invalid size %" PRIu64 " bytes", size);
+ err = MDBX_PROBLEM;
+ goto bailout;
}
- } else if (head == meta0)
- target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta1, meta2);
- else if (head == meta1)
- target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta0, meta2);
- else {
- mdbx_assert(env, head == meta2);
- target = (MDBX_meta *)meta_ancient_prefer_weak(env, meta0, meta1);
}
- /* LY: step#2 - update meta-page. */
- mdbx_debug(
- "writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO
- ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
- " +%u -%u, txn_id %" PRIaTXN ", %s",
- data_page(target)->mp_pgno, pending->mm_dbs[MAIN_DBI].md_root,
- pending->mm_dbs[FREE_DBI].md_root, pending->mm_geo.lower,
- pending->mm_geo.next, pending->mm_geo.now, pending->mm_geo.upper,
- pv2pages(pending->mm_geo.grow_pv), pv2pages(pending->mm_geo.shrink_pv),
- unaligned_peek_u64(4, pending->mm_txnid_a), mdbx_durable_str(pending));
-
- mdbx_debug("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
- "/%" PRIaPGNO,
- (meta0 == head) ? "head"
- : (meta0 == target) ? "tail"
- : "stay",
- mdbx_durable_str(meta0), meta_txnid(env, meta0),
- meta0->mm_dbs[MAIN_DBI].md_root, meta0->mm_dbs[FREE_DBI].md_root);
- mdbx_debug("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
- "/%" PRIaPGNO,
- (meta1 == head) ? "head"
- : (meta1 == target) ? "tail"
- : "stay",
- mdbx_durable_str(meta1), meta_txnid(env, meta1),
- meta1->mm_dbs[MAIN_DBI].md_root, meta1->mm_dbs[FREE_DBI].md_root);
- mdbx_debug("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO
- "/%" PRIaPGNO,
- (meta2 == head) ? "head"
- : (meta2 == target) ? "tail"
- : "stay",
- mdbx_durable_str(meta2), meta_txnid(env, meta2),
- meta2->mm_dbs[MAIN_DBI].md_root, meta2->mm_dbs[FREE_DBI].md_root);
+ const size_t maxreaders =
+ ((size_t)size - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader);
+ if (maxreaders < 4) {
+ mdbx_error("lck-size too small (up to %" PRIuPTR " readers)", maxreaders);
+ err = MDBX_PROBLEM;
+ goto bailout;
+ }
+ env->me_maxreaders = (maxreaders <= MDBX_READERS_LIMIT)
+ ? (unsigned)maxreaders
+ : (unsigned)MDBX_READERS_LIMIT;
- mdbx_assert(env, !meta_eq(env, pending, meta0));
- mdbx_assert(env, !meta_eq(env, pending, meta1));
- mdbx_assert(env, !meta_eq(env, pending, meta2));
+ err = mdbx_mmap((env->me_flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP,
+ &env->me_lck_mmap, (size_t)size, (size_t)size,
+ lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE
+ : MMAP_OPTION_SEMAPHORE);
+ if (unlikely(err != MDBX_SUCCESS))
+ goto bailout;
- mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0);
- mdbx_ensure(env,
- target == head || constmeta_txnid(env, target) <
- unaligned_peek_u64(4, pending->mm_txnid_a));
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- if (flags & MDBX_WRITEMAP) {
- mdbx_jitter4testing(true);
- if (likely(target != head)) {
- /* LY: 'invalidate' the meta. */
- meta_update_begin(env, target,
- unaligned_peek_u64(4, pending->mm_txnid_a));
- unaligned_poke_u64(4, target->mm_datasync_sign, MDBX_DATASIGN_WEAK);
-#ifndef NDEBUG
- /* debug: provoke failure to catch a violators, but don't touch mm_psize
- * to allow readers catch actual pagesize. */
- uint8_t *provoke_begin = (uint8_t *)&target->mm_dbs[FREE_DBI].md_root;
- uint8_t *provoke_end = (uint8_t *)&target->mm_datasync_sign;
- memset(provoke_begin, 0xCC, provoke_end - provoke_begin);
- mdbx_jitter4testing(false);
-#endif
+#if MDBX_ENABLE_MADVISE
+#ifdef MADV_DODUMP
+ err = madvise(env->me_lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ goto bailout;
+#endif /* MADV_DODUMP */
- /* LY: update info */
- target->mm_geo = pending->mm_geo;
- target->mm_dbs[FREE_DBI] = pending->mm_dbs[FREE_DBI];
- target->mm_dbs[MAIN_DBI] = pending->mm_dbs[MAIN_DBI];
- target->mm_canary = pending->mm_canary;
- memcpy(target->mm_pages_retired, pending->mm_pages_retired, 8);
- mdbx_jitter4testing(true);
+#ifdef MADV_WILLNEED
+ err = madvise(env->me_lck_mmap.lck, size, MADV_WILLNEED)
+ ? ignore_enosys(errno)
+ : MDBX_SUCCESS;
+ if (unlikely(MDBX_IS_ERROR(err)))
+ goto bailout;
+#elif defined(POSIX_MADV_WILLNEED)
+ err = ignore_enosys(
+ posix_madvise(env->me_lck_mmap.lck, size, POSIX_MADV_WILLNEED));
+ if (unlikely(MDBX_IS_ERROR(err)))
+ goto bailout;
+#endif /* MADV_WILLNEED */
+#endif /* MDBX_ENABLE_MADVISE */
- /* LY: 'commit' the meta */
- meta_update_end(env, target, unaligned_peek_u64(4, pending->mm_txnid_b));
- mdbx_jitter4testing(true);
- mdbx_assert(env, meta_checktxnid(env, target, true));
- } else {
- /* dangerous case (target == head), only mm_datasync_sign could
- * me updated, check assertions once again */
- mdbx_ensure(env, constmeta_txnid(env, head) ==
- unaligned_peek_u64(4, pending->mm_txnid_a) &&
- !META_IS_STEADY(head) && META_IS_STEADY(pending));
- mdbx_ensure(env, memcmp(&head->mm_geo, &pending->mm_geo,
- sizeof(head->mm_geo)) == 0);
- mdbx_ensure(env, memcmp(&head->mm_dbs, &pending->mm_dbs,
- sizeof(head->mm_dbs)) == 0);
- mdbx_ensure(env, memcmp(&head->mm_canary, &pending->mm_canary,
- sizeof(head->mm_canary)) == 0);
- }
- memcpy(target->mm_datasync_sign, pending->mm_datasync_sign, 8);
- mdbx_flush_incoherent_cpu_writeback();
- mdbx_jitter4testing(true);
- /* sync meta-pages */
- rc =
- mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
- (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE
- : MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- if (unlikely(rc != MDBX_SUCCESS))
- goto fail;
- } else {
- const MDBX_meta undo_meta = *target;
- const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
- ? env->me_dsync_fd
- : env->me_lazy_fd;
+ struct MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (lck_seize_rc == MDBX_RESULT_TRUE) {
+ /* LY: exclusive mode, check and reset lck content */
+ memset(lck, 0, (size_t)size);
+ mdbx_jitter4testing(false);
+ lck->mti_magic_and_version = MDBX_LOCK_MAGIC;
+ lck->mti_os_and_format = MDBX_LOCK_FORMAT;
#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
+ lck->mti_pgop_stat.wops.weak = 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
- rc = mdbx_pwrite(fd, pending, sizeof(MDBX_meta),
- (uint8_t *)target - env->me_map);
- if (unlikely(rc != MDBX_SUCCESS)) {
- undo:
- mdbx_debug("%s", "write failed, disk error?");
- /* On a failure, the pagecache still contains the new data.
- * Try write some old data back, to prevent it from being used. */
- mdbx_pwrite(fd, &undo_meta, sizeof(MDBX_meta),
- (uint8_t *)target - env->me_map);
- goto fail;
+ err = mdbx_msync(&env->me_lck_mmap, 0, (size_t)size, MDBX_SYNC_NONE);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("initial-%s for lck-file failed", "msync");
+ goto bailout;
}
- mdbx_flush_incoherent_mmap(target, sizeof(MDBX_meta), env->me_os_psize);
- /* sync meta-pages */
- if ((flags & MDBX_NOMETASYNC) == 0 && fd == env->me_lazy_fd) {
- rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- if (rc != MDBX_SUCCESS)
- goto undo;
+ err = mdbx_fsync(env->me_lck_mmap.fd, MDBX_SYNC_SIZE);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mdbx_error("initial-%s for lck-file failed", "fsync");
+ goto bailout;
+ }
+ } else {
+ if (lck->mti_magic_and_version != MDBX_LOCK_MAGIC) {
+ const bool invalid = (lck->mti_magic_and_version >> 8) != MDBX_MAGIC;
+ mdbx_error(
+ "lock region has %s",
+ invalid
+ ? "invalid magic"
+ : "incompatible version (only applications with nearly or the "
+ "same versions of libmdbx can share the same database)");
+ err = invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
+ goto bailout;
+ }
+ if (lck->mti_os_and_format != MDBX_LOCK_FORMAT) {
+ mdbx_error("lock region has os/format signature 0x%" PRIx32
+ ", expected 0x%" PRIx32,
+ lck->mti_os_and_format, MDBX_LOCK_FORMAT);
+ err = MDBX_VERSION_MISMATCH;
+ goto bailout;
}
- mdbx_assert(env, meta_checktxnid(env, target, true));
}
- env->me_lck->mti_meta_sync_txnid.weak =
- (uint32_t)unaligned_peek_u64(4, pending->mm_txnid_a) -
- ((flags & MDBX_NOMETASYNC) ? UINT32_MAX / 3 : 0);
- /* LY: shrink datafile if needed */
- if (unlikely(shrink)) {
- mdbx_verbose("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")",
- pending->mm_geo.now, shrink);
- rc = mdbx_mapresize_implicit(env, pending->mm_geo.next, pending->mm_geo.now,
- pending->mm_geo.upper);
- if (rc != MDBX_SUCCESS && rc != MDBX_EPERM)
- goto fail;
- mdbx_assert(env, meta_checktxnid(env, target, true));
- }
+ err = mdbx_lck_init(env, inprocess_neighbor, lck_seize_rc);
+ if (MDBX_IS_ERROR(err))
+ goto bailout;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (likely(lck))
- /* toggle oldest refresh */
- atomic_store32(&lck->mti_readers_refresh_flag, false, mo_Relaxed);
+ mdbx_ensure(env, env->me_lcklist_next == nullptr);
+ /* insert into inprocess lck-list */
+ env->me_lcklist_next = inprocess_lcklist_head;
+ inprocess_lcklist_head = env;
+ lcklist_unlock();
+ /* end of a locked section ------------------------------------------------ */
- return MDBX_SUCCESS;
+ mdbx_assert(env, !MDBX_IS_ERROR(lck_seize_rc));
+ env->me_lck = lck;
+ return lck_seize_rc;
+}
-fail:
- env->me_flags |= MDBX_FATAL_ERROR;
- return rc;
+__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
+ if (volume <= 1024 * 1024 * 4ul)
+ return MDBX_RESULT_TRUE;
+
+ intptr_t pagesize, total_ram_pages;
+ int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+
+ const int log2page = log2n_powerof2(pagesize);
+ const intptr_t volume_pages = (volume + pagesize - 1) >> log2page;
+ const intptr_t redundancy_pages =
+ (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page)
+ : (intptr_t)(redundancy + pagesize - 1) >> log2page;
+ if (volume_pages >= total_ram_pages ||
+ volume_pages + redundancy_pages >= total_ram_pages)
+ return MDBX_RESULT_FALSE;
+
+ intptr_t avail_ram_pages;
+ err = mdbx_get_sysraminfo(nullptr, nullptr, &avail_ram_pages);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+
+ return (volume_pages + redundancy_pages >= avail_ram_pages)
+ ? MDBX_RESULT_FALSE
+ : MDBX_RESULT_TRUE;
}
-static void recalculate_merge_threshold(MDBX_env *env) {
- const unsigned bytes = page_space(env);
- env->me_merge_threshold =
- (uint16_t)(bytes -
- (bytes * env->me_options.merge_threshold_16dot16_percent >>
- 16));
- env->me_merge_threshold_gc =
- (uint16_t)(bytes -
- ((env->me_options.merge_threshold_16dot16_percent > 19005)
- ? bytes / 3 /* 33 % */
- : bytes / 4 /* 25 % */));
+/* Merge sync flags */
+static uint32_t merge_sync_flags(const uint32_t a, const uint32_t b) {
+ uint32_t r = a | b;
+
+ /* avoid false MDBX_UTTERLY_NOSYNC */
+ if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
+ !F_ISSET(b, MDBX_UTTERLY_NOSYNC))
+ r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC;
+
+ /* convert MDBX_DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */
+ if ((r & (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC)) ==
+ (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC) &&
+ !F_ISSET(r, MDBX_UTTERLY_NOSYNC))
+ r = (r - MDBX_DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC;
+
+ /* force MDBX_NOMETASYNC if MDBX_SAFE_NOSYNC enabled */
+ if (r & MDBX_SAFE_NOSYNC)
+ r |= MDBX_NOMETASYNC;
+
+ assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) &&
+ !F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
+ !F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
+ return r;
}
-__cold static void mdbx_setup_pagesize(MDBX_env *env, const size_t pagesize) {
- STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE);
- STATIC_ASSERT(MIN_PAGESIZE > sizeof(MDBX_page) + sizeof(MDBX_meta));
- mdbx_ensure(env, is_powerof2(pagesize));
- mdbx_ensure(env, pagesize >= MIN_PAGESIZE);
- mdbx_ensure(env, pagesize <= MAX_PAGESIZE);
- env->me_psize = (unsigned)pagesize;
- if (env->me_pbuf) {
- mdbx_memalign_free(env->me_pbuf);
- env->me_pbuf = nullptr;
+__cold static int __must_check_result mdbx_override_meta(
+ MDBX_env *env, unsigned target, txnid_t txnid, const MDBX_meta *shape) {
+ int rc = alloc_page_buf(env);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ MDBX_page *const page = env->me_pbuf;
+ mdbx_meta_model(env, page, target);
+ MDBX_meta *const model = page_meta(page);
+ meta_set_txnid(env, model, txnid);
+ mdbx_assert(env, meta_checktxnid(env, model, true));
+ if (shape) {
+ if (txnid && unlikely(!meta_checktxnid(env, shape, false))) {
+ mdbx_error("bailout overriding meta-%u since model failed "
+ "freedb/maindb %s-check for txnid #%" PRIaTXN,
+ target, "pre", constmeta_txnid(env, shape));
+ return MDBX_PROBLEM;
+ }
+ if (mdbx_runtime_flags & MDBX_DBG_DONT_UPGRADE)
+ memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version,
+ sizeof(model->mm_magic_and_version));
+ model->mm_extra_flags = shape->mm_extra_flags;
+ model->mm_validator_id = shape->mm_validator_id;
+ model->mm_extra_pagehdr = shape->mm_extra_pagehdr;
+ memcpy(&model->mm_geo, &shape->mm_geo, sizeof(model->mm_geo));
+ memcpy(&model->mm_dbs, &shape->mm_dbs, sizeof(model->mm_dbs));
+ memcpy(&model->mm_canary, &shape->mm_canary, sizeof(model->mm_canary));
+ memcpy(&model->mm_pages_retired, &shape->mm_pages_retired,
+ sizeof(model->mm_pages_retired));
+ if (txnid) {
+ if ((!model->mm_dbs[FREE_DBI].md_mod_txnid &&
+ model->mm_dbs[FREE_DBI].md_root != P_INVALID) ||
+ (!model->mm_dbs[MAIN_DBI].md_mod_txnid &&
+ model->mm_dbs[MAIN_DBI].md_root != P_INVALID))
+ memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version,
+ sizeof(model->mm_magic_and_version));
+ if (unlikely(!meta_checktxnid(env, model, false))) {
+ mdbx_error("bailout overriding meta-%u since model failed "
+ "freedb/maindb %s-check for txnid #%" PRIaTXN,
+ target, "post", txnid);
+ return MDBX_PROBLEM;
+ }
+ }
}
+ unaligned_poke_u64(4, model->mm_datasync_sign, meta_sign(model));
+ rc = mdbx_validate_meta(env, model, page, target, nullptr);
+ if (unlikely(MDBX_IS_ERROR(rc)))
+ return MDBX_PROBLEM;
- STATIC_ASSERT(MAX_GC1OVPAGE(MIN_PAGESIZE) > 4);
- STATIC_ASSERT(MAX_GC1OVPAGE(MAX_PAGESIZE) < MDBX_PGL_LIMIT);
- const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
- mdbx_ensure(env, maxgc_ov1page > 42 &&
- maxgc_ov1page < (intptr_t)MDBX_PGL_LIMIT / 4);
- env->me_maxgc_ov1page = (unsigned)maxgc_ov1page;
-
- STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) > sizeof(MDBX_db) + NODESIZE + 42);
- STATIC_ASSERT(LEAF_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX);
- STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) >= BRANCH_NODE_MAX(MIN_PAGESIZE));
- STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) > NODESIZE + 42);
- STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX);
- const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize);
- const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize);
- mdbx_ensure(env,
- branch_nodemax > (intptr_t)(NODESIZE + 42) &&
- branch_nodemax % 2 == 0 &&
- leaf_nodemax > (intptr_t)(sizeof(MDBX_db) + NODESIZE + 42) &&
- leaf_nodemax >= branch_nodemax &&
- leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0);
- env->me_leaf_nodemax = (unsigned)leaf_nodemax;
- env->me_psize2log = (uint8_t)log2n_powerof2(pagesize);
- mdbx_assert(env, pgno2bytes(env, 1) == pagesize);
- mdbx_assert(env, bytes2pgno(env, pagesize + pagesize) == 2);
- recalculate_merge_threshold(env);
+ if (shape && memcmp(model, shape, sizeof(MDBX_meta)) == 0)
+ return MDBX_SUCCESS;
- const pgno_t max_pgno = bytes2pgno(env, MAX_MAPSIZE);
- if (!env->me_options.flags.non_auto.dp_limit) {
- /* auto-setup dp_limit by "The42" ;-) */
- intptr_t total_ram_pages, avail_ram_pages;
- int err = mdbx_get_sysraminfo(nullptr, &total_ram_pages, &avail_ram_pages);
- if (unlikely(err != MDBX_SUCCESS))
- mdbx_error("mdbx_get_sysraminfo(), rc %d", err);
- else {
- size_t reasonable_dpl_limit =
- (size_t)(total_ram_pages + avail_ram_pages) / 42;
- if (pagesize > env->me_os_psize)
- reasonable_dpl_limit /= pagesize / env->me_os_psize;
- else if (pagesize < env->me_os_psize)
- reasonable_dpl_limit *= env->me_os_psize / pagesize;
- reasonable_dpl_limit = (reasonable_dpl_limit < MDBX_PGL_LIMIT)
- ? reasonable_dpl_limit
- : MDBX_PGL_LIMIT;
- reasonable_dpl_limit = (reasonable_dpl_limit > CURSOR_STACK * 4)
- ? reasonable_dpl_limit
- : CURSOR_STACK * 4;
- env->me_options.dp_limit = (unsigned)reasonable_dpl_limit;
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.wops.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ if (env->me_flags & MDBX_WRITEMAP) {
+ rc = mdbx_msync(&env->me_dxb_mmap, 0,
+ pgno_align2os_bytes(env, model->mm_geo.next),
+ MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ /* mdbx_override_meta() called only while current process have exclusive
+ * lock of a DB file. So meta-page could be updated directly without
+ * clearing consistency flag by mdbx_meta_update_begin() */
+ memcpy(pgno2page(env, target), page, env->me_psize);
+ mdbx_flush_incoherent_cpu_writeback();
+ rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target + 1),
+ MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ } else {
+ const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
+ ? env->me_dsync_fd
+ : env->me_lazy_fd;
+ rc = mdbx_pwrite(fd, page, env->me_psize, pgno2bytes(env, target));
+ if (rc == MDBX_SUCCESS && fd == env->me_lazy_fd)
+ rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ }
+ mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS),
+ env->me_os_psize);
+ return rc;
+}
+
+__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) {
+ if (unlikely(target >= NUM_METAS))
+ return MDBX_EINVAL;
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely((env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) !=
+ MDBX_EXCLUSIVE))
+ return MDBX_EPERM;
+
+ const MDBX_meta *target_meta = METAPAGE(env, target);
+ txnid_t new_txnid = safe64_txnid_next(constmeta_txnid(env, target_meta));
+ for (unsigned n = 0; n < NUM_METAS; ++n) {
+ MDBX_page *page = pgno2page(env, n);
+ MDBX_meta meta = *page_meta(page);
+ if (n == target)
+ continue;
+ if (mdbx_validate_meta(env, &meta, page, n, nullptr) != MDBX_SUCCESS) {
+ int err = mdbx_override_meta(env, n, 0, nullptr);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ } else {
+ txnid_t txnid = constmeta_txnid(env, &meta);
+ if (new_txnid <= txnid)
+ new_txnid = safe64_txnid_next(txnid);
}
}
- if (env->me_options.dp_limit > max_pgno - NUM_METAS)
- env->me_options.dp_limit = max_pgno - NUM_METAS;
- if (env->me_options.dp_initial > env->me_options.dp_limit)
- env->me_options.dp_initial = env->me_options.dp_limit;
-}
-static __inline MDBX_CONST_FUNCTION MDBX_lockinfo *
-lckless_stub(const MDBX_env *env) {
- uintptr_t stub = (uintptr_t)&env->x_lckless_stub;
- /* align to avoid false-positive alarm from UndefinedBehaviorSanitizer */
- stub = (stub + MDBX_CACHELINE_SIZE - 1) & ~(MDBX_CACHELINE_SIZE - 1);
- return (MDBX_lockinfo *)stub;
+ if (unlikely(new_txnid > MAX_TXNID)) {
+ mdbx_error("txnid overflow, raise %d", MDBX_TXN_FULL);
+ return MDBX_TXN_FULL;
+ }
+ return mdbx_override_meta(env, target, new_txnid, target_meta);
}
-__cold int mdbx_env_create(MDBX_env **penv) {
- MDBX_env *env = mdbx_calloc(1, sizeof(MDBX_env));
- if (unlikely(!env))
- return MDBX_ENOMEM;
+__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname,
+ unsigned target_meta, bool writeable) {
+ if (unlikely(target_meta >= NUM_METAS))
+ return MDBX_EINVAL;
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ if (unlikely(env->me_map))
+ return MDBX_EPERM;
- env->me_maxreaders = DEFAULT_READERS;
- env->me_maxdbs = env->me_numdbs = CORE_DBS;
- env->me_lazy_fd = INVALID_HANDLE_VALUE;
- env->me_dsync_fd = INVALID_HANDLE_VALUE;
- env->me_lfd = INVALID_HANDLE_VALUE;
- env->me_pid = mdbx_getpid();
- env->me_stuck_meta = -1;
+ env->me_stuck_meta = (int8_t)target_meta;
+ return mdbx_env_open(
+ env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY,
+ 0);
+}
- env->me_options.dp_reserve_limit = 1024;
- env->me_options.rp_augment_limit = 256 * 1024;
- env->me_options.dp_limit = 64 * 1024;
- if (env->me_options.dp_limit > MAX_PAGENO + 1 - NUM_METAS)
- env->me_options.dp_limit = MAX_PAGENO + 1 - NUM_METAS;
- env->me_options.dp_initial = MDBX_PNL_INITIAL;
- if (env->me_options.dp_initial > env->me_options.dp_limit)
- env->me_options.dp_initial = env->me_options.dp_limit;
- env->me_options.spill_max_denominator = 8;
- env->me_options.spill_min_denominator = 8;
- env->me_options.spill_parent4child_denominator = 0;
- env->me_options.dp_loose_limit = 64;
- env->me_options.merge_threshold_16dot16_percent = 65536 / 4 /* 25% */;
+typedef struct {
+ void *buffer_for_free;
+ char *lck, *dxb;
+ size_t ent_len;
+} MDBX_handle_env_pathname;
+__cold static int mdbx_handle_env_pathname(MDBX_handle_env_pathname *ctx,
+ const char *pathname,
+ MDBX_env_flags_t *flags,
+ const mdbx_mode_t mode) {
int rc;
- const size_t os_psize = mdbx_syspagesize();
- if (unlikely(!is_powerof2(os_psize) || os_psize < MIN_PAGESIZE)) {
- mdbx_error("unsuitable system pagesize %" PRIuPTR, os_psize);
- rc = MDBX_INCOMPATIBLE;
- goto bailout;
- }
- env->me_os_psize = (unsigned)os_psize;
- mdbx_setup_pagesize(env, (env->me_os_psize < MAX_PAGESIZE) ? env->me_os_psize
- : MAX_PAGESIZE);
-
- rc = mdbx_fastmutex_init(&env->me_dbi_lock);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ memset(ctx, 0, sizeof(*ctx));
+ if (unlikely(!pathname))
+ return MDBX_EINVAL;
#if defined(_WIN32) || defined(_WIN64)
- mdbx_srwlock_Init(&env->me_remap_guard);
- InitializeCriticalSection(&env->me_windowsbug_lock);
+ const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
+ if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
+ return ERROR_INVALID_NAME;
+ wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
+ if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
+ return ERROR_INVALID_NAME;
+
+ const DWORD dwAttrib = GetFileAttributesW(pathnameW);
+ if (dwAttrib == INVALID_FILE_ATTRIBUTES) {
+ rc = GetLastError();
+ if (rc != MDBX_ENOFILE)
+ return rc;
+ if (mode == 0 || (*flags & MDBX_RDONLY) != 0)
+ /* can't open existing */
+ return rc;
+
+ /* auto-create directory if requested */
+ if ((*flags & MDBX_NOSUBDIR) == 0 &&
+ !CreateDirectoryW(pathnameW, nullptr)) {
+ rc = GetLastError();
+ if (rc != ERROR_ALREADY_EXISTS)
+ return rc;
+ }
+ } else {
+ /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
+ *flags |= MDBX_NOSUBDIR;
+ if (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)
+ *flags -= MDBX_NOSUBDIR;
+ }
#else
- rc = mdbx_fastmutex_init(&env->me_remap_guard);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_fastmutex_destroy(&env->me_dbi_lock);
- goto bailout;
+ struct stat st;
+ if (stat(pathname, &st)) {
+ rc = errno;
+ if (rc != MDBX_ENOFILE)
+ return rc;
+ if (mode == 0 || (*flags & MDBX_RDONLY) != 0)
+ /* can't open existing */
+ return rc;
+
+ /* auto-create directory if requested */
+ const mdbx_mode_t dir_mode =
+ (/* inherit read/write permissions for group and others */ mode &
+ (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
+ /* always add read/write/search for owner */ S_IRWXU |
+ ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) |
+ ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0);
+ if ((*flags & MDBX_NOSUBDIR) == 0 && mkdir(pathname, dir_mode)) {
+ rc = errno;
+ if (rc != EEXIST)
+ return rc;
+ }
+ } else {
+ /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
+ *flags |= MDBX_NOSUBDIR;
+ if (S_ISDIR(st.st_mode))
+ *flags -= MDBX_NOSUBDIR;
}
+#endif
-#if MDBX_LOCKING > MDBX_LOCKING_SYSV
- MDBX_lockinfo *const stub = lckless_stub(env);
- rc = mdbx_ipclock_stub(&stub->mti_wlock);
-#endif /* MDBX_LOCKING */
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_fastmutex_destroy(&env->me_remap_guard);
- mdbx_fastmutex_destroy(&env->me_dbi_lock);
- goto bailout;
+ static const char dxb_name[] = MDBX_DATANAME;
+ static const size_t dxb_name_len = sizeof(dxb_name) - 1;
+ static const char lck_name[] = MDBX_LOCKNAME;
+ static const char lock_suffix[] = MDBX_LOCK_SUFFIX;
+
+ ctx->ent_len = strlen(pathname);
+ if ((*flags & MDBX_NOSUBDIR) && ctx->ent_len >= dxb_name_len &&
+ !memcmp(dxb_name, pathname + ctx->ent_len - dxb_name_len, dxb_name_len)) {
+ *flags -= MDBX_NOSUBDIR;
+ ctx->ent_len -= dxb_name_len;
}
-#endif /* Windows */
- VALGRIND_CREATE_MEMPOOL(env, 0, 0);
- env->me_signature.weak = MDBX_ME_SIGNATURE;
- *penv = env;
- return MDBX_SUCCESS;
+ const size_t bytes_needed =
+ ctx->ent_len * 2 + ((*flags & MDBX_NOSUBDIR)
+ ? sizeof(lock_suffix) + 1
+ : sizeof(lck_name) + sizeof(dxb_name));
+ ctx->buffer_for_free = mdbx_malloc(bytes_needed);
+ if (!ctx->buffer_for_free)
+ return MDBX_ENOMEM;
-bailout:
- mdbx_free(env);
- *penv = nullptr;
- return rc;
+ ctx->lck = ctx->buffer_for_free;
+ if (*flags & MDBX_NOSUBDIR) {
+ ctx->dxb = ctx->lck + ctx->ent_len + sizeof(lock_suffix);
+ sprintf(ctx->lck, "%s%s", pathname, lock_suffix);
+ strcpy(ctx->dxb, pathname);
+ } else {
+ ctx->dxb = ctx->lck + ctx->ent_len + sizeof(lck_name);
+ sprintf(ctx->lck, "%.*s%s", (int)ctx->ent_len, pathname, lck_name);
+ sprintf(ctx->dxb, "%.*s%s", (int)ctx->ent_len, pathname, dxb_name);
+ }
+
+ return MDBX_SUCCESS;
}
-__cold static intptr_t get_reasonable_db_maxsize(intptr_t *cached_result) {
- if (*cached_result == 0) {
- intptr_t pagesize, total_ram_pages;
- if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) !=
- MDBX_SUCCESS))
- return *cached_result = MAX_MAPSIZE32 /* the 32-bit limit is good enough
- for fallback */
- ;
+__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) {
+ switch (mode) {
+ default:
+ return MDBX_EINVAL;
+ case MDBX_ENV_JUST_DELETE:
+ case MDBX_ENV_ENSURE_UNUSED:
+ case MDBX_ENV_WAIT_FOR_UNUSED:
+ break;
+ }
+
+#ifdef __e2k__ /* https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=6011 */
+ MDBX_env *const dummy_env = alloca(sizeof(MDBX_env));
+#else
+ MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo;
+#endif
+ memset(dummy_env, 0, sizeof(*dummy_env));
+ dummy_env->me_flags =
+ (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS;
+ dummy_env->me_os_psize = (unsigned)mdbx_syspagesize();
+ dummy_env->me_psize = (unsigned)mdbx_default_pagesize();
+ dummy_env->me_pathname = (char *)pathname;
+
+ MDBX_handle_env_pathname env_pathname;
+ STATIC_ASSERT(sizeof(dummy_env->me_flags) == sizeof(MDBX_env_flags_t));
+ int rc = MDBX_RESULT_TRUE,
+ err = mdbx_handle_env_pathname(
+ &env_pathname, pathname, (MDBX_env_flags_t *)&dummy_env->me_flags, 0);
+ if (likely(err == MDBX_SUCCESS)) {
+ mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE,
+ dxb_handle = INVALID_HANDLE_VALUE;
+ if (mode > MDBX_ENV_JUST_DELETE) {
+ err = mdbx_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.dxb,
+ &dxb_handle, 0);
+ err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
+ if (err == MDBX_SUCCESS) {
+ err = mdbx_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.lck,
+ &clk_handle, 0);
+ err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
+ }
+ if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE)
+ err = mdbx_lockfile(clk_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
+ if (err == MDBX_SUCCESS && dxb_handle != INVALID_HANDLE_VALUE)
+ err = mdbx_lockfile(dxb_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
+ }
- if (unlikely((size_t)total_ram_pages * 2 > MAX_MAPSIZE / (size_t)pagesize))
- return *cached_result = MAX_MAPSIZE;
- assert(MAX_MAPSIZE >= (size_t)(total_ram_pages * pagesize * 2));
+ if (err == MDBX_SUCCESS) {
+ err = mdbx_removefile(env_pathname.dxb);
+ if (err == MDBX_SUCCESS)
+ rc = MDBX_SUCCESS;
+ else if (err == MDBX_ENOFILE)
+ err = MDBX_SUCCESS;
+ }
- /* Suggesting should not be more than golden ratio of the size of RAM. */
- *cached_result = (intptr_t)((size_t)total_ram_pages * 207 >> 7) * pagesize;
+ if (err == MDBX_SUCCESS) {
+ err = mdbx_removefile(env_pathname.lck);
+ if (err == MDBX_SUCCESS)
+ rc = MDBX_SUCCESS;
+ else if (err == MDBX_ENOFILE)
+ err = MDBX_SUCCESS;
+ }
- /* Round to the nearest human-readable granulation. */
- for (size_t unit = MEGABYTE; unit; unit <<= 5) {
- const size_t floor = floor_powerof2(*cached_result, unit);
- const size_t ceil = ceil_powerof2(*cached_result, unit);
- const size_t threshold = (size_t)*cached_result >> 4;
- const bool down =
- *cached_result - floor < ceil - *cached_result || ceil > MAX_MAPSIZE;
- if (threshold < (down ? *cached_result - floor : ceil - *cached_result))
- break;
- *cached_result = down ? floor : ceil;
+ if (err == MDBX_SUCCESS && !(dummy_env->me_flags & MDBX_NOSUBDIR)) {
+ err = mdbx_removedirectory(pathname);
+ if (err == MDBX_SUCCESS)
+ rc = MDBX_SUCCESS;
+ else if (err == MDBX_ENOFILE)
+ err = MDBX_SUCCESS;
}
- }
- return *cached_result;
+
+ if (dxb_handle != INVALID_HANDLE_VALUE)
+ mdbx_closefile(dxb_handle);
+ if (clk_handle != INVALID_HANDLE_VALUE)
+ mdbx_closefile(clk_handle);
+ } else if (err == MDBX_ENOFILE)
+ err = MDBX_SUCCESS;
+
+ mdbx_free(env_pathname.buffer_for_free);
+ return (err == MDBX_SUCCESS) ? rc : err;
}
-__cold LIBMDBX_API int
-mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now,
- intptr_t size_upper, intptr_t growth_step,
- intptr_t shrink_threshold, intptr_t pagesize) {
+__cold int mdbx_env_open(MDBX_env *env, const char *pathname,
+ MDBX_env_flags_t flags, mdbx_mode_t mode) {
int rc = check_env(env, false);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- const bool inside_txn =
- (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self());
+ if (unlikely(flags & ~ENV_USABLE_FLAGS))
+ return MDBX_EINVAL;
-#if MDBX_DEBUG
- if (growth_step < 0) {
- growth_step = 1;
- if (shrink_threshold < 0)
- shrink_threshold = 1;
- }
-#endif /* MDBX_DEBUG */
+ if (flags & MDBX_RDONLY)
+ mode = 0;
- intptr_t reasonable_maxsize = 0;
- bool need_unlock = false;
- if (env->me_map) {
- /* env already mapped */
- if (unlikely(env->me_flags & MDBX_RDONLY))
- return MDBX_EACCESS;
+ if (env->me_lazy_fd != INVALID_HANDLE_VALUE ||
+ (env->me_flags & MDBX_ENV_ACTIVE) != 0 || env->me_map)
+ return MDBX_EPERM;
- if (!inside_txn) {
- int err = mdbx_txn_lock(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- need_unlock = true;
- }
- const MDBX_meta *head = constmeta_prefer_last(env);
- if (!inside_txn) {
- env->me_txn0->mt_txnid = constmeta_txnid(env, head);
- mdbx_find_oldest(env->me_txn0);
- }
+ /* pickup previously mdbx_env_set_flags(),
+ * but avoid MDBX_UTTERLY_NOSYNC by disjunction */
+ const uint32_t saved_me_flags = env->me_flags;
+ flags = merge_sync_flags(flags, env->me_flags);
- /* get untouched params from DB */
- if (pagesize <= 0 || pagesize >= INT_MAX)
- pagesize = env->me_psize;
- if (size_lower < 0)
- size_lower = pgno2bytes(env, head->mm_geo.lower);
- if (size_now < 0)
- size_now = pgno2bytes(env, head->mm_geo.now);
- if (size_upper < 0)
- size_upper = pgno2bytes(env, head->mm_geo.upper);
- if (growth_step < 0)
- growth_step = pgno2bytes(env, pv2pages(head->mm_geo.grow_pv));
- if (shrink_threshold < 0)
- shrink_threshold = pgno2bytes(env, pv2pages(head->mm_geo.shrink_pv));
+ MDBX_handle_env_pathname env_pathname;
+ rc = mdbx_handle_env_pathname(&env_pathname, pathname, &flags, mode);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
- if (pagesize != (intptr_t)env->me_psize) {
- rc = MDBX_EINVAL;
- goto bailout;
+ if (flags & MDBX_RDONLY) {
+ /* LY: silently ignore irrelevant flags when
+ * we're only getting read access */
+ flags &= ~(MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC |
+ MDBX_NOMETASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM |
+ MDBX_NOMEMINIT | MDBX_ACCEDE);
+ } else {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ /* Temporary `workaround` for OpenBSD kernel's flaw.
+ * See todo4recovery://erased_by_github/libmdbx/issues/67 */
+ if ((flags & MDBX_WRITEMAP) == 0) {
+ if (flags & MDBX_ACCEDE)
+ flags |= MDBX_WRITEMAP;
+ else {
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__,
+ "System (i.e. OpenBSD) requires MDBX_WRITEMAP because "
+ "of an internal flaw(s) in a file/buffer/page cache.\n");
+ rc = 42 /* ENOPROTOOPT */;
+ goto bailout;
+ }
}
- const size_t usedbytes =
- pgno2bytes(env, mdbx_find_largest(env, head->mm_geo.next));
- if ((size_t)size_upper < usedbytes) {
- rc = MDBX_MAP_FULL;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+ }
+
+ env->me_flags = (flags & ~MDBX_FATAL_ERROR) | MDBX_ENV_ACTIVE;
+ env->me_pathname = mdbx_calloc(env_pathname.ent_len + 1, 1);
+ env->me_dbxs = mdbx_calloc(env->me_maxdbs, sizeof(MDBX_dbx));
+ env->me_dbflags = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbflags[0]));
+ env->me_dbiseqs = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbiseqs[0]));
+ if (!(env->me_dbxs && env->me_pathname && env->me_dbflags &&
+ env->me_dbiseqs)) {
+ rc = MDBX_ENOMEM;
+ goto bailout;
+ }
+ memcpy(env->me_pathname, env_pathname.dxb, env_pathname.ent_len);
+ env->me_dbxs[FREE_DBI].md_cmp = cmp_int_align4; /* aligned MDBX_INTEGERKEY */
+ env->me_dbxs[FREE_DBI].md_dcmp = cmp_lenfast;
+
+ rc = mdbx_openfile(F_ISSET(flags, MDBX_RDONLY) ? MDBX_OPEN_DXB_READ
+ : MDBX_OPEN_DXB_LAZY,
+ env, env_pathname.dxb, &env->me_lazy_fd, mode);
+ if (rc != MDBX_SUCCESS)
+ goto bailout;
+
+ mdbx_assert(env, env->me_dsync_fd == INVALID_HANDLE_VALUE);
+ if ((flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC)) == 0) {
+ rc = mdbx_openfile(MDBX_OPEN_DXB_DSYNC, env, env_pathname.dxb,
+ &env->me_dsync_fd, 0);
+ mdbx_ensure(env, (rc != MDBX_SUCCESS) ==
+ (env->me_dsync_fd == INVALID_HANDLE_VALUE));
+ }
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ env->me_sysv_ipc.key = ftok(env_pathname.dxb, 42);
+ if (env->me_sysv_ipc.key == -1) {
+ rc = errno;
+ goto bailout;
+ }
+#endif /* MDBX_LOCKING */
+
+#if !(defined(_WIN32) || defined(_WIN64))
+ if (mode == 0) {
+ /* pickup mode for lck-file */
+ struct stat st;
+ if (fstat(env->me_lazy_fd, &st)) {
+ rc = errno;
goto bailout;
}
- if ((size_t)size_now < usedbytes)
- size_now = usedbytes;
- } else {
- /* env NOT yet mapped */
- if (unlikely(inside_txn))
- return MDBX_PANIC;
+ mode = st.st_mode;
+ }
+ mode = (/* inherit read permissions for group and others */ mode &
+ (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
+ /* always add read/write/search for owner */ S_IRUSR | S_IWUSR |
+ ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) |
+ ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0);
+#endif /* !Windows */
+ const int lck_rc = mdbx_setup_lck(env, env_pathname.lck, mode);
+ if (MDBX_IS_ERROR(lck_rc)) {
+ rc = lck_rc;
+ goto bailout;
+ }
- /* is requested some auto-value for pagesize ? */
- if (pagesize >= INT_MAX /* maximal */)
- pagesize = MAX_PAGESIZE;
- else if (pagesize <= 0) {
- if (pagesize < 0 /* default */) {
- pagesize = env->me_os_psize;
- if ((uintptr_t)pagesize > MAX_PAGESIZE)
- pagesize = MAX_PAGESIZE;
- mdbx_assert(env, (uintptr_t)pagesize >= MIN_PAGESIZE);
- } else if (pagesize == 0 /* minimal */)
- pagesize = MIN_PAGESIZE;
+ /* Set the position in files outside of the data to avoid corruption
+ * due to erroneous use of file descriptors in the application code. */
+ mdbx_fseek(env->me_lfd, UINT64_C(1) << 63);
+ mdbx_fseek(env->me_lazy_fd, UINT64_C(1) << 63);
+ if (env->me_dsync_fd != INVALID_HANDLE_VALUE)
+ mdbx_fseek(env->me_dsync_fd, UINT64_C(1) << 63);
- /* choose pagesize */
- intptr_t max_size = (size_now > size_lower) ? size_now : size_lower;
- max_size = (size_upper > max_size) ? size_upper : max_size;
- if (max_size < 0 /* default */)
- max_size = DEFAULT_MAPSIZE;
- else if (max_size == 0 /* minimal */)
- max_size = MIN_MAPSIZE;
- else if (max_size >= (intptr_t)MAX_MAPSIZE /* maximal */)
- max_size = get_reasonable_db_maxsize(&reasonable_maxsize);
+ const MDBX_env_flags_t rigorous_flags =
+ MDBX_SAFE_NOSYNC | MDBX_DEPRECATED_MAPASYNC;
+ const MDBX_env_flags_t mode_flags = rigorous_flags | MDBX_NOMETASYNC |
+ MDBX_LIFORECLAIM | MDBX_COALESCE |
+ MDBX_NORDAHEAD;
+
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (lck && lck_rc != MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
+ while (atomic_load32(&lck->mti_envmode, mo_AcquireRelease) == MDBX_RDONLY) {
+ if (atomic_cas32(&lck->mti_envmode, MDBX_RDONLY,
+ env->me_flags & mode_flags)) {
+ /* The case:
+ * - let's assume that for some reason the DB file is smaller
+ * than it should be according to the geometry,
+ * but not smaller than the last page used;
+ * - the first process that opens the database (lc_rc = true)
+ * does this in readonly mode and therefore cannot bring
+ * the file size back to normal;
+ * - some next process (lc_rc = false) opens the DB in read-write
+ * mode and now is here.
+ *
+ * FIXME: Should we re-check and set the size of DB-file right here? */
+ break;
+ }
+ atomic_yield();
+ }
- while (max_size > pagesize * (int64_t)(MAX_PAGENO + 1) &&
- pagesize < MAX_PAGESIZE)
- pagesize <<= 1;
+ if (env->me_flags & MDBX_ACCEDE) {
+ /* pickup current mode-flags, including MDBX_LIFORECLAIM |
+ * MDBX_COALESCE | MDBX_NORDAHEAD */
+ const unsigned diff =
+ (lck->mti_envmode.weak ^ env->me_flags) & mode_flags;
+ mdbx_notice("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->me_flags,
+ env->me_flags ^ diff);
+ env->me_flags ^= diff;
+ }
+
+ if ((lck->mti_envmode.weak ^ env->me_flags) & rigorous_flags) {
+ mdbx_error("%s", "current mode/flags incompatible with requested");
+ rc = MDBX_INCOMPATIBLE;
+ goto bailout;
}
}
- if (pagesize < (intptr_t)MIN_PAGESIZE || pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2(pagesize)) {
- rc = MDBX_EINVAL;
+ const int dxb_rc = mdbx_setup_dxb(env, lck_rc, mode);
+ if (MDBX_IS_ERROR(dxb_rc)) {
+ rc = dxb_rc;
goto bailout;
}
- if (size_lower <= 0) {
- size_lower = MIN_MAPSIZE;
- if (MIN_MAPSIZE / pagesize < MIN_PAGENO)
- size_lower = MIN_PAGENO * pagesize;
- }
- if (size_lower >= INTPTR_MAX) {
- size_lower = get_reasonable_db_maxsize(&reasonable_maxsize);
- if ((size_t)size_lower / pagesize > MAX_PAGENO + 1)
- size_lower = pagesize * (MAX_PAGENO + 1);
+ if (unlikely(/* recovery mode */ env->me_stuck_meta >= 0) &&
+ (lck_rc != /* exclusive */ MDBX_RESULT_TRUE ||
+ (flags & MDBX_EXCLUSIVE) == 0)) {
+ mdbx_error("%s", "recovery requires exclusive mode");
+ rc = MDBX_BUSY;
+ goto bailout;
}
- if (size_now <= 0) {
- size_now = size_lower;
- if (size_upper >= size_lower && size_now > size_upper)
- size_now = size_upper;
- }
- if (size_now >= INTPTR_MAX) {
- size_now = get_reasonable_db_maxsize(&reasonable_maxsize);
- if ((size_t)size_now / pagesize > MAX_PAGENO + 1)
- size_now = pagesize * (MAX_PAGENO + 1);
+ mdbx_debug("opened dbenv %p", (void *)env);
+ if (lck) {
+ if (lck_rc == MDBX_RESULT_TRUE) {
+ lck->mti_envmode.weak = env->me_flags & (mode_flags | MDBX_RDONLY);
+ rc = mdbx_lck_downgrade(env);
+ mdbx_debug("lck-downgrade-%s: rc %i",
+ (env->me_flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc);
+ if (rc != MDBX_SUCCESS)
+ goto bailout;
+ } else {
+ rc = mdbx_cleanup_dead_readers(env, false, NULL);
+ if (MDBX_IS_ERROR(rc))
+ goto bailout;
+ }
+
+ if ((env->me_flags & MDBX_NOTLS) == 0) {
+ rc = mdbx_rthc_alloc(&env->me_txkey, &lck->mti_readers[0],
+ &lck->mti_readers[env->me_maxreaders]);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ env->me_flags |= MDBX_ENV_TXKEY;
+ }
}
- if (size_upper <= 0) {
- if (size_now >= get_reasonable_db_maxsize(&reasonable_maxsize) / 2)
- size_upper = get_reasonable_db_maxsize(&reasonable_maxsize);
- else if (MAX_MAPSIZE != MAX_MAPSIZE32 &&
- (size_t)size_now >= MAX_MAPSIZE32 / 2 &&
- (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3)
- size_upper = MAX_MAPSIZE32;
- else {
- size_upper = size_now + size_now;
- if ((size_t)size_upper < DEFAULT_MAPSIZE * 2)
- size_upper = DEFAULT_MAPSIZE * 2;
+ if ((flags & MDBX_RDONLY) == 0) {
+ const size_t tsize = sizeof(MDBX_txn),
+ size = tsize + env->me_maxdbs *
+ (sizeof(MDBX_db) + sizeof(MDBX_cursor *) +
+ sizeof(unsigned) + 1);
+ rc = alloc_page_buf(env);
+ if (rc == MDBX_SUCCESS) {
+ memset(env->me_pbuf, -1, env->me_psize * 2);
+ MDBX_txn *txn = mdbx_calloc(1, size);
+ if (txn) {
+ txn->mt_dbs = (MDBX_db *)((char *)txn + tsize);
+ txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs);
+ txn->mt_dbiseqs = (unsigned *)(txn->mt_cursors + env->me_maxdbs);
+ txn->mt_dbistate = (uint8_t *)(txn->mt_dbiseqs + env->me_maxdbs);
+ txn->mt_env = env;
+ txn->mt_dbxs = env->me_dbxs;
+ txn->mt_flags = MDBX_TXN_FINISHED;
+ env->me_txn0 = txn;
+ txn->tw.retired_pages = mdbx_pnl_alloc(MDBX_PNL_INITIAL);
+ txn->tw.reclaimed_pglist = mdbx_pnl_alloc(MDBX_PNL_INITIAL);
+ if (unlikely(!txn->tw.retired_pages || !txn->tw.reclaimed_pglist))
+ rc = MDBX_ENOMEM;
+ } else
+ rc = MDBX_ENOMEM;
}
- if ((size_t)size_upper / pagesize > (MAX_PAGENO + 1))
- size_upper = pagesize * (MAX_PAGENO + 1);
- } else if (size_upper >= INTPTR_MAX) {
- size_upper = get_reasonable_db_maxsize(&reasonable_maxsize);
- if ((size_t)size_upper / pagesize > MAX_PAGENO + 1)
- size_upper = pagesize * (MAX_PAGENO + 1);
}
- if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) {
- rc = MDBX_EINVAL;
- goto bailout;
+#if MDBX_DEBUG
+ if (rc == MDBX_SUCCESS) {
+ const MDBX_meta *meta = constmeta_prefer_last(env);
+ const MDBX_db *db = &meta->mm_dbs[MAIN_DBI];
+
+ mdbx_debug("opened database version %u, pagesize %u",
+ (uint8_t)unaligned_peek_u64(4, meta->mm_magic_and_version),
+ env->me_psize);
+ mdbx_debug("using meta page %" PRIaPGNO ", txn %" PRIaTXN,
+ data_page(meta)->mp_pgno, meta_txnid(env, meta));
+ mdbx_debug("depth: %u", db->md_depth);
+ mdbx_debug("entries: %" PRIu64, db->md_entries);
+ mdbx_debug("branch pages: %" PRIaPGNO, db->md_branch_pages);
+ mdbx_debug("leaf pages: %" PRIaPGNO, db->md_leaf_pages);
+ mdbx_debug("overflow pages: %" PRIaPGNO, db->md_overflow_pages);
+ mdbx_debug("root: %" PRIaPGNO, db->md_root);
+ mdbx_debug("schema_altered: %" PRIaTXN, db->md_mod_txnid);
}
+#endif
- if ((uint64_t)size_lower / pagesize < MIN_PAGENO) {
- rc = MDBX_EINVAL;
- goto bailout;
+bailout:
+ if (rc != MDBX_SUCCESS) {
+ rc = mdbx_env_close0(env) ? MDBX_PANIC : rc;
+ env->me_flags =
+ saved_me_flags | ((rc != MDBX_PANIC) ? 0 : MDBX_FATAL_ERROR);
+ } else {
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ mdbx_txn_valgrind(env, nullptr);
+#endif
}
+ mdbx_free(env_pathname.buffer_for_free);
+ return rc;
+}
- if (unlikely((size_t)size_upper > MAX_MAPSIZE ||
- (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
- rc = MDBX_TOO_LARGE;
- goto bailout;
+/* Destroy resources from mdbx_env_open(), clear our readers & DBIs */
+__cold static int mdbx_env_close0(MDBX_env *env) {
+ const unsigned flags = env->me_flags;
+ if (!(flags & MDBX_ENV_ACTIVE)) {
+ mdbx_ensure(env, env->me_lcklist_next == nullptr);
+ return MDBX_SUCCESS;
}
- const size_t unit = (env->me_os_psize > (size_t)pagesize) ? env->me_os_psize
- : (size_t)pagesize;
- size_lower = ceil_powerof2(size_lower, unit);
- size_upper = ceil_powerof2(size_upper, unit);
- size_now = ceil_powerof2(size_now, unit);
+ env->me_flags &= ~ENV_INTERNAL_FLAGS;
+ env->me_lck = nullptr;
+ if (flags & MDBX_ENV_TXKEY) {
+ mdbx_rthc_remove(env->me_txkey);
+ env->me_txkey = (mdbx_thread_key_t)0;
+ }
- /* LY: подбираем значение size_upper:
- * - кратное размеру Ñтраницы
- * - без Ð½Ð°Ñ€ÑƒÑˆÐµÐ½Ð¸Ñ MAX_MAPSIZE и MAX_PAGENO */
- while (unlikely((size_t)size_upper > MAX_MAPSIZE ||
- (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
- if ((size_t)size_upper < unit + MIN_MAPSIZE ||
- (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) {
- /* паранойа на Ñлучай Ð¿ÐµÑ€ÐµÐ¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€Ð¸ невероÑтных значениÑÑ… */
- rc = MDBX_EINVAL;
- goto bailout;
- }
- size_upper -= unit;
- if ((size_t)size_upper < (size_t)size_lower)
- size_lower = size_upper;
+ lcklist_lock();
+ const int rc = lcklist_detach_locked(env);
+ lcklist_unlock();
+
+ if (env->me_map) {
+ mdbx_munmap(&env->me_dxb_mmap);
+#ifdef MDBX_USE_VALGRIND
+ VALGRIND_DISCARD(env->me_valgrind_handle);
+ env->me_valgrind_handle = -1;
+#endif
}
- mdbx_assert(env, (size_upper - size_lower) % env->me_os_psize == 0);
- if (size_now < size_lower)
- size_now = size_lower;
- if (size_now > size_upper)
- size_now = size_upper;
+ if (env->me_dsync_fd != INVALID_HANDLE_VALUE) {
+ (void)mdbx_closefile(env->me_dsync_fd);
+ env->me_dsync_fd = INVALID_HANDLE_VALUE;
+ }
- if (growth_step < 0) {
- growth_step = ((size_t)(size_upper - size_lower)) / 42;
- if (growth_step > size_lower && size_lower < (intptr_t)MEGABYTE)
- growth_step = size_lower;
- if (growth_step < 65536)
- growth_step = 65536;
- if ((size_t)growth_step > MAX_MAPSIZE / 64)
- growth_step = MAX_MAPSIZE / 64;
+ if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
+ (void)mdbx_closefile(env->me_lazy_fd);
+ env->me_lazy_fd = INVALID_HANDLE_VALUE;
}
- if (growth_step == 0 && shrink_threshold > 0)
- growth_step = 1;
- growth_step = ceil_powerof2(growth_step, unit);
- if (shrink_threshold < 0)
- shrink_threshold = growth_step + growth_step;
- shrink_threshold = ceil_powerof2(shrink_threshold, unit);
+ if (env->me_lck_mmap.lck)
+ mdbx_munmap(&env->me_lck_mmap);
- //----------------------------------------------------------------------------
+ if (env->me_lfd != INVALID_HANDLE_VALUE) {
+ (void)mdbx_closefile(env->me_lfd);
+ env->me_lfd = INVALID_HANDLE_VALUE;
+ }
+
+ if (env->me_dbxs) {
+ for (unsigned i = env->me_numdbs; --i >= CORE_DBS;)
+ mdbx_free(env->me_dbxs[i].md_name.iov_base);
+ mdbx_free(env->me_dbxs);
+ env->me_dbxs = nullptr;
+ }
+ if (env->me_pbuf) {
+ mdbx_memalign_free(env->me_pbuf);
+ env->me_pbuf = nullptr;
+ }
+ if (env->me_dbiseqs) {
+ mdbx_free(env->me_dbiseqs);
+ env->me_dbiseqs = nullptr;
+ }
+ if (env->me_dbflags) {
+ mdbx_free(env->me_dbflags);
+ env->me_dbflags = nullptr;
+ }
+ if (env->me_pathname) {
+ mdbx_free(env->me_pathname);
+ env->me_pathname = nullptr;
+ }
+ if (env->me_txn0) {
+ mdbx_dpl_free(env->me_txn0);
+ mdbx_txl_free(env->me_txn0->tw.lifo_reclaimed);
+ mdbx_pnl_free(env->me_txn0->tw.retired_pages);
+ mdbx_pnl_free(env->me_txn0->tw.spill_pages);
+ mdbx_pnl_free(env->me_txn0->tw.reclaimed_pglist);
+ mdbx_free(env->me_txn0);
+ env->me_txn0 = nullptr;
+ }
+ env->me_stuck_meta = -1;
+ return rc;
+}
- if (!env->me_map) {
- /* save user's geo-params for future open/create */
- if (pagesize != (intptr_t)env->me_psize)
- mdbx_setup_pagesize(env, pagesize);
- env->me_dbgeo.lower = size_lower;
- env->me_dbgeo.now = size_now;
- env->me_dbgeo.upper = size_upper;
- env->me_dbgeo.grow =
- pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step))));
- env->me_dbgeo.shrink =
- pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold))));
+__cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
+ MDBX_page *dp;
+ int rc = MDBX_SUCCESS;
- mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE);
- mdbx_ensure(env, env->me_dbgeo.lower / (unsigned)pagesize >= MIN_PAGENO);
- mdbx_ensure(env, env->me_dbgeo.lower % (unsigned)pagesize == 0);
- mdbx_ensure(env, env->me_dbgeo.lower % env->me_os_psize == 0);
+ if (unlikely(!env))
+ return MDBX_EINVAL;
- mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE);
- mdbx_ensure(env,
- env->me_dbgeo.upper / (unsigned)pagesize <= MAX_PAGENO + 1);
- mdbx_ensure(env, env->me_dbgeo.upper % (unsigned)pagesize == 0);
- mdbx_ensure(env, env->me_dbgeo.upper % env->me_os_psize == 0);
+ if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE))
+ return MDBX_EBADSIGN;
- mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower);
- mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper);
- mdbx_ensure(env, env->me_dbgeo.now % (unsigned)pagesize == 0);
- mdbx_ensure(env, env->me_dbgeo.now % env->me_os_psize == 0);
+#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64))
+ /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows
+ * platforms (i.e. where fork() is available).
+ * This is required to legitimize a call after fork()
+ * from a child process, that should be allowed to free resources. */
+ if (unlikely(env->me_pid != mdbx_getpid()))
+ env->me_flags |= MDBX_FATAL_ERROR;
+#endif /* MDBX_ENV_CHECKPID */
- mdbx_ensure(env, env->me_dbgeo.grow % (unsigned)pagesize == 0);
- mdbx_ensure(env, env->me_dbgeo.grow % env->me_os_psize == 0);
- mdbx_ensure(env, env->me_dbgeo.shrink % (unsigned)pagesize == 0);
- mdbx_ensure(env, env->me_dbgeo.shrink % env->me_os_psize == 0);
+ if (env->me_map && (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0 &&
+ env->me_txn0) {
+ if (env->me_txn0->mt_owner && env->me_txn0->mt_owner != mdbx_thread_self())
+ return MDBX_BUSY;
+ } else
+ dont_sync = true;
- rc = MDBX_SUCCESS;
- } else {
- /* apply new params to opened environment */
- mdbx_ensure(env, pagesize == (intptr_t)env->me_psize);
- MDBX_meta meta;
- memset(&meta, 0, sizeof(meta));
- const MDBX_geo *current_geo;
- if (!inside_txn) {
- mdbx_assert(env, need_unlock);
- const MDBX_meta *head = constmeta_prefer_last(env);
+ if (!atomic_cas32(&env->me_signature, MDBX_ME_SIGNATURE, 0))
+ return MDBX_EBADSIGN;
- uint64_t timestamp = 0;
- while ("workaround for "
- "todo4recovery://erased_by_github/libmdbx/issues/269") {
- meta = *head;
- rc = meta_waittxnid(env, &meta, ×tamp);
- if (likely(rc == MDBX_SUCCESS))
- break;
- if (unlikely(rc != MDBX_RESULT_TRUE))
- goto bailout;
- }
- const txnid_t txnid = safe64_txnid_next(constmeta_txnid(env, &meta));
- if (unlikely(txnid > MAX_TXNID)) {
- rc = MDBX_TXN_FULL;
- mdbx_error("txnid overflow, raise %d", rc);
- goto bailout;
- }
- meta_set_txnid(env, &meta, txnid);
- current_geo = &meta.mm_geo;
- } else {
- current_geo = &env->me_txn->mt_geo;
+ if (!dont_sync) {
+#if defined(_WIN32) || defined(_WIN64)
+ /* On windows, without blocking is impossible to determine whether another
+ * process is running a writing transaction or not.
+ * Because in the "owner died" condition kernel don't release
+ * file lock immediately. */
+ rc = mdbx_env_sync_internal(env, true, false);
+ rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
+#else
+ struct stat st;
+ if (unlikely(fstat(env->me_lazy_fd, &st)))
+ rc = errno;
+ else if (st.st_nlink > 0 /* don't sync deleted files */) {
+ rc = mdbx_env_sync_internal(env, true, true);
+ rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY ||
+ rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE)
+ ? MDBX_SUCCESS
+ : rc;
}
+#endif
+ }
- MDBX_geo new_geo;
- new_geo.lower = bytes2pgno(env, size_lower);
- new_geo.now = bytes2pgno(env, size_now);
- new_geo.upper = bytes2pgno(env, size_upper);
- new_geo.grow_pv = pages2pv(bytes2pgno(env, growth_step));
- new_geo.shrink_pv = pages2pv(bytes2pgno(env, shrink_threshold));
- new_geo.next = current_geo->next;
-
- mdbx_ensure(env,
- pgno_align2os_bytes(env, new_geo.lower) == (size_t)size_lower);
- mdbx_ensure(env,
- pgno_align2os_bytes(env, new_geo.upper) == (size_t)size_upper);
- mdbx_ensure(env, pgno_align2os_bytes(env, new_geo.now) == (size_t)size_now);
- mdbx_ensure(env, new_geo.grow_pv == pages2pv(pv2pages(new_geo.grow_pv)));
- mdbx_ensure(env,
- new_geo.shrink_pv == pages2pv(pv2pages(new_geo.shrink_pv)));
-
- mdbx_ensure(env, (size_t)size_lower >= MIN_MAPSIZE);
- mdbx_ensure(env, new_geo.lower >= MIN_PAGENO);
- mdbx_ensure(env, (size_t)size_upper <= MAX_MAPSIZE);
- mdbx_ensure(env, new_geo.upper <= MAX_PAGENO + 1);
- mdbx_ensure(env, new_geo.now >= new_geo.next);
- mdbx_ensure(env, new_geo.upper >= new_geo.now);
- mdbx_ensure(env, new_geo.now >= new_geo.lower);
-
- if (memcmp(current_geo, &new_geo, sizeof(MDBX_geo)) != 0) {
+ mdbx_assert(env, env->me_signature.weak == 0);
+ rc = mdbx_env_close0(env) ? MDBX_PANIC : rc;
+ mdbx_ensure(env, mdbx_fastmutex_destroy(&env->me_dbi_lock) == MDBX_SUCCESS);
#if defined(_WIN32) || defined(_WIN64)
- /* Was DB shrinking disabled before and now it will be enabled? */
- if (new_geo.lower < new_geo.upper && new_geo.shrink_pv &&
- !(current_geo->lower < current_geo->upper &&
- current_geo->shrink_pv)) {
- if (!env->me_lck_mmap.lck) {
- rc = MDBX_EPERM;
- goto bailout;
- }
- int err = mdbx_rdt_lock(env);
- if (unlikely(MDBX_IS_ERROR(err))) {
- rc = err;
- goto bailout;
- }
+ /* me_remap_guard don't have destructor (Slim Reader/Writer Lock) */
+ DeleteCriticalSection(&env->me_windowsbug_lock);
+#else
+ mdbx_ensure(env,
+ mdbx_fastmutex_destroy(&env->me_remap_guard) == MDBX_SUCCESS);
+#endif /* Windows */
- /* Check if there are any reading threads that do not use the SRWL */
- const size_t CurrentTid = GetCurrentThreadId();
- const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers;
- const MDBX_reader *const end =
- begin + atomic_load32(&env->me_lck_mmap.lck->mti_numreaders,
- mo_AcquireRelease);
- for (const MDBX_reader *reader = begin; reader < end; ++reader) {
- if (reader->mr_pid.weak == env->me_pid && reader->mr_tid.weak &&
- reader->mr_tid.weak != CurrentTid) {
- /* At least one thread may don't use SRWL */
- rc = MDBX_EPERM;
- break;
- }
- }
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV
+ MDBX_lockinfo *const stub = lckless_stub(env);
+ mdbx_ensure(env, mdbx_ipclock_destroy(&stub->mti_wlock) == 0);
+#endif /* MDBX_LOCKING */
- mdbx_rdt_unlock(env);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
-#endif
+ while ((dp = env->me_dp_reserve) != NULL) {
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize);
+ VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
+ env->me_dp_reserve = dp->mp_next;
+ mdbx_free(dp);
+ }
+ VALGRIND_DESTROY_MEMPOOL(env);
+ mdbx_ensure(env, env->me_lcklist_next == nullptr);
+ env->me_pid = 0;
+ mdbx_free(env);
- if (new_geo.now != current_geo->now ||
- new_geo.upper != current_geo->upper) {
- rc = mdbx_mapresize(env, current_geo->next, new_geo.now, new_geo.upper,
- false);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- if (inside_txn) {
- env->me_txn->mt_geo = new_geo;
- env->me_txn->mt_flags |= MDBX_TXN_DIRTY;
- } else {
- meta.mm_geo = new_geo;
- rc = mdbx_sync_locked(env, env->me_flags, &meta);
- }
+ return rc;
+}
- if (likely(rc == MDBX_SUCCESS)) {
- /* store new geo to env to avoid influences */
- env->me_dbgeo.now = pgno2bytes(env, new_geo.now);
- env->me_dbgeo.lower = pgno2bytes(env, new_geo.lower);
- env->me_dbgeo.upper = pgno2bytes(env, new_geo.upper);
- env->me_dbgeo.grow = pgno2bytes(env, pv2pages(new_geo.grow_pv));
- env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(new_geo.shrink_pv));
- }
- }
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_close(MDBX_env *env) {
+ return __inline_mdbx_env_close(env);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+
+/* Compare two items pointing at aligned unsigned int's. */
+static int __hot cmp_int_align4(const MDBX_val *a, const MDBX_val *b) {
+ mdbx_assert(NULL, a->iov_len == b->iov_len);
+ switch (a->iov_len) {
+ case 4:
+ return CMP2INT(unaligned_peek_u32(4, a->iov_base),
+ unaligned_peek_u32(4, b->iov_base));
+ case 8:
+ return CMP2INT(unaligned_peek_u64(4, a->iov_base),
+ unaligned_peek_u64(4, b->iov_base));
+ default:
+ mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
+ __LINE__);
+ return 0;
}
+}
-bailout:
- if (need_unlock)
- mdbx_txn_unlock(env);
- return rc;
+/* Compare two items pointing at 2-byte aligned unsigned int's. */
+static int __hot cmp_int_align2(const MDBX_val *a, const MDBX_val *b) {
+ mdbx_assert(NULL, a->iov_len == b->iov_len);
+ switch (a->iov_len) {
+ case 4:
+ return CMP2INT(unaligned_peek_u32(2, a->iov_base),
+ unaligned_peek_u32(2, b->iov_base));
+ case 8:
+ return CMP2INT(unaligned_peek_u64(2, a->iov_base),
+ unaligned_peek_u64(2, b->iov_base));
+ default:
+ mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
+ __LINE__);
+ return 0;
+ }
}
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) {
- return __inline_mdbx_env_set_mapsize(env, size);
+/* Compare two items pointing at unsigned values with unknown alignment.
+ *
+ * This is also set as MDBX_INTEGERDUP|MDBX_DUPFIXED's MDBX_dbx.md_dcmp. */
+static int __hot cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) {
+ mdbx_assert(NULL, a->iov_len == b->iov_len);
+ switch (a->iov_len) {
+ case 4:
+ return CMP2INT(unaligned_peek_u32(1, a->iov_base),
+ unaligned_peek_u32(1, b->iov_base));
+ case 8:
+ return CMP2INT(unaligned_peek_u64(1, a->iov_base),
+ unaligned_peek_u64(1, b->iov_base));
+ default:
+ mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
+ __LINE__);
+ return 0;
+ }
}
-__cold int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) {
- return __inline_mdbx_env_set_maxdbs(env, dbs);
-}
+/* Compare two items lexically */
+static int __hot cmp_lexical(const MDBX_val *a, const MDBX_val *b) {
+ if (a->iov_len == b->iov_len)
+ return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0;
-__cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) {
- return __inline_mdbx_env_get_maxdbs(env, dbs);
+ const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1;
+ const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
+ int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0;
+ return likely(diff_data) ? diff_data : diff_len;
}
-__cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) {
- return __inline_mdbx_env_set_maxreaders(env, readers);
+/* Compare two items in reverse byte order */
+static int __hot cmp_reverse(const MDBX_val *a, const MDBX_val *b) {
+ const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
+ if (likely(shortest)) {
+ const uint8_t *pa = (const uint8_t *)a->iov_base + a->iov_len;
+ const uint8_t *pb = (const uint8_t *)b->iov_base + b->iov_len;
+ const uint8_t *const end = pa - shortest;
+ do {
+ int diff = *--pa - *--pb;
+ if (likely(diff))
+ return diff;
+ } while (pa != end);
+ }
+ return CMP2INT(a->iov_len, b->iov_len);
}
-__cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) {
- return __inline_mdbx_env_get_maxreaders(env, readers);
+/* Fast non-lexically comparator */
+static int __hot cmp_lenfast(const MDBX_val *a, const MDBX_val *b) {
+ int diff = CMP2INT(a->iov_len, b->iov_len);
+ return likely(diff || a->iov_len == 0)
+ ? diff
+ : memcmp(a->iov_base, b->iov_base, a->iov_len);
}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
-__cold static int alloc_page_buf(MDBX_env *env) {
- return env->me_pbuf
- ? MDBX_SUCCESS
- : mdbx_memalign_alloc(env->me_os_psize, env->me_psize * NUM_METAS,
- &env->me_pbuf);
+static bool unsure_equal(MDBX_cmp_func cmp, const MDBX_val *a,
+ const MDBX_val *b) {
+ /* checking for the use of a known good comparator
+ * or/otherwise for a full byte-to-byte match */
+ return cmp == cmp_lenfast || cmp == cmp_lexical || cmp == cmp_reverse ||
+ cmp == cmp_int_unaligned || cmp_lenfast(a, b) == 0;
}
-/* Further setup required for opening an MDBX environment */
-__cold static int mdbx_setup_dxb(MDBX_env *env, const int lck_rc,
- const mdbx_mode_t mode_bits) {
- MDBX_meta meta;
- int rc = MDBX_RESULT_FALSE;
- int err = mdbx_read_header(env, &meta, lck_rc, mode_bits);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA ||
- (env->me_flags & MDBX_RDONLY) != 0 ||
- /* recovery mode */ env->me_stuck_meta >= 0)
- return err;
-
- mdbx_debug("%s", "create new database");
- rc = /* new database */ MDBX_RESULT_TRUE;
-
- if (!env->me_dbgeo.now) {
- /* set defaults if not configured */
- err = mdbx_env_set_geometry(env, 0, -1, DEFAULT_MAPSIZE, -1, -1, -1);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
-
- err = alloc_page_buf(env);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- meta = *mdbx_init_metas(env, env->me_pbuf);
- err = mdbx_pwrite(env->me_lazy_fd, env->me_pbuf, env->me_psize * NUM_METAS,
- 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- err = mdbx_ftruncate(env->me_lazy_fd,
- env->me_dxb_mmap.filesize = env->me_dbgeo.now);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
-#ifndef NDEBUG /* just for checking */
- err = mdbx_read_header(env, &meta, lck_rc, mode_bits);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-#endif
- }
-
- mdbx_verbose(
- "header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO
- "-%" PRIaPGNO "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s",
- meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root,
- meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now, meta.mm_geo.upper,
- pv2pages(meta.mm_geo.grow_pv), pv2pages(meta.mm_geo.shrink_pv),
- unaligned_peek_u64(4, meta.mm_txnid_a), mdbx_durable_str(&meta));
-
- if (env->me_psize != meta.mm_psize)
- mdbx_setup_pagesize(env, meta.mm_psize);
- const size_t used_bytes = pgno2bytes(env, meta.mm_geo.next);
- const size_t used_aligned2os_bytes =
- ceil_powerof2(used_bytes, env->me_os_psize);
- if ((env->me_flags & MDBX_RDONLY) /* readonly */
- || lck_rc != MDBX_RESULT_TRUE /* not exclusive */
- || /* recovery mode */ env->me_stuck_meta >= 0) {
- /* use present params from db */
- const size_t pagesize = meta.mm_psize;
- err = mdbx_env_set_geometry(
- env, meta.mm_geo.lower * pagesize, meta.mm_geo.now * pagesize,
- meta.mm_geo.upper * pagesize, pv2pages(meta.mm_geo.grow_pv) * pagesize,
- pv2pages(meta.mm_geo.shrink_pv) * pagesize, meta.mm_psize);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("%s: err %d", "could not apply preconfigured geometry from db",
- err);
- return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
- }
- } else if (env->me_dbgeo.now) {
- /* silently growth to last used page */
- if (env->me_dbgeo.now < used_aligned2os_bytes)
- env->me_dbgeo.now = used_aligned2os_bytes;
- if (env->me_dbgeo.upper < used_aligned2os_bytes)
- env->me_dbgeo.upper = used_aligned2os_bytes;
-
- /* apply preconfigured params, but only if substantial changes:
- * - upper or lower limit changes
- * - shrink threshold or growth step
- * But ignore change just a 'now/current' size. */
- if (bytes_align2os_bytes(env, env->me_dbgeo.upper) !=
- pgno2bytes(env, meta.mm_geo.upper) ||
- bytes_align2os_bytes(env, env->me_dbgeo.lower) !=
- pgno2bytes(env, meta.mm_geo.lower) ||
- bytes_align2os_bytes(env, env->me_dbgeo.shrink) !=
- pgno2bytes(env, pv2pages(meta.mm_geo.shrink_pv)) ||
- bytes_align2os_bytes(env, env->me_dbgeo.grow) !=
- pgno2bytes(env, pv2pages(meta.mm_geo.grow_pv))) {
-
- if (env->me_dbgeo.shrink && env->me_dbgeo.now > used_bytes)
- /* pre-shrink if enabled */
- env->me_dbgeo.now = used_bytes + env->me_dbgeo.shrink -
- used_bytes % env->me_dbgeo.shrink;
-
- err = mdbx_env_set_geometry(env, env->me_dbgeo.lower, env->me_dbgeo.now,
- env->me_dbgeo.upper, env->me_dbgeo.grow,
- env->me_dbgeo.shrink, meta.mm_psize);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("%s: err %d", "could not apply preconfigured db-geometry",
- err);
- return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
- }
+/* Search for key within a page, using binary search.
+ * Returns the smallest entry larger or equal to the key.
+ * Updates the cursor index with the index of the found entry.
+ * If no entry larger or equal to the key is found, returns NULL. */
+static struct node_result __hot mdbx_node_search(MDBX_cursor *mc,
+ const MDBX_val *key) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ const int nkeys = page_numkeys(mp);
+ DKBUF_DEBUG;
- /* update meta fields */
- meta.mm_geo.now = bytes2pgno(env, env->me_dbgeo.now);
- meta.mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower);
- meta.mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper);
- meta.mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow));
- meta.mm_geo.shrink_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.shrink));
+ mdbx_debug("searching %u keys in %s %spage %" PRIaPGNO, nkeys,
+ IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "",
+ mp->mp_pgno);
- mdbx_verbose("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO
- "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
- " +%u -%u, txn_id %" PRIaTXN ", %s",
- meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root,
- meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now,
- meta.mm_geo.upper, pv2pages(meta.mm_geo.grow_pv),
- pv2pages(meta.mm_geo.shrink_pv),
- unaligned_peek_u64(4, meta.mm_txnid_a),
- mdbx_durable_str(&meta));
- } else {
- /* fetch back 'now/current' size, since it was ignored during comparison
- * and may differ. */
- env->me_dbgeo.now = pgno_align2os_bytes(env, meta.mm_geo.now);
- }
- mdbx_ensure(env, meta.mm_geo.now >= meta.mm_geo.next);
- } else {
- /* geo-params are not pre-configured by user,
- * get current values from the meta. */
- env->me_dbgeo.now = pgno2bytes(env, meta.mm_geo.now);
- env->me_dbgeo.lower = pgno2bytes(env, meta.mm_geo.lower);
- env->me_dbgeo.upper = pgno2bytes(env, meta.mm_geo.upper);
- env->me_dbgeo.grow = pgno2bytes(env, pv2pages(meta.mm_geo.grow_pv));
- env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(meta.mm_geo.shrink_pv));
+ struct node_result ret;
+ ret.exact = false;
+ STATIC_ASSERT(P_BRANCH == 1);
+ int low = mp->mp_flags & P_BRANCH;
+ int high = nkeys - 1;
+ if (unlikely(high < low)) {
+ mc->mc_ki[mc->mc_top] = 0;
+ ret.node = NULL;
+ return ret;
}
- mdbx_ensure(env,
- pgno_align2os_bytes(env, meta.mm_geo.now) == env->me_dbgeo.now);
- mdbx_ensure(env, env->me_dbgeo.now >= used_bytes);
- const uint64_t filesize_before = env->me_dxb_mmap.filesize;
- if (unlikely(filesize_before != env->me_dbgeo.now)) {
- if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
- mdbx_verbose("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO
- "p, have %" PRIu64 "b/%" PRIaPGNO "p), "
- "assume other process working",
- env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now),
- filesize_before, bytes2pgno(env, (size_t)filesize_before));
- } else {
- mdbx_warning("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO
- "p, have %" PRIu64 "b/%" PRIaPGNO "p)",
- env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now),
- filesize_before, bytes2pgno(env, (size_t)filesize_before));
- if (filesize_before < used_bytes) {
- mdbx_error("last-page beyond end-of-file (last %" PRIaPGNO
- ", have %" PRIaPGNO ")",
- meta.mm_geo.next, bytes2pgno(env, (size_t)filesize_before));
- return MDBX_CORRUPTED;
+ int cr = 0, i = 0;
+ MDBX_cmp_func *cmp = mc->mc_dbx->md_cmp;
+ MDBX_val nodekey;
+ if (unlikely(IS_LEAF2(mp))) {
+ mdbx_cassert(mc, mp->mp_leaf2_ksize == mc->mc_db->md_xsize);
+ nodekey.iov_len = mp->mp_leaf2_ksize;
+ do {
+ i = (low + high) >> 1;
+ nodekey.iov_base = page_leaf2key(mp, i, nodekey.iov_len);
+ mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >=
+ (char *)nodekey.iov_base + nodekey.iov_len);
+ cr = cmp(key, &nodekey);
+ mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY_DEBUG(&nodekey),
+ cr);
+ if (unlikely(cr == 0)) {
+ ret.exact = true;
+ break;
}
+ low = (cr < 0) ? low : i + 1;
+ high = (cr < 0) ? i - 1 : high;
+ } while (likely(low <= high));
- if (env->me_flags & MDBX_RDONLY) {
- if (filesize_before & (env->me_os_psize - 1)) {
- mdbx_error("%s", "filesize should be rounded-up to system page");
- return MDBX_WANNA_RECOVERY;
- }
- mdbx_warning("%s", "ignore filesize mismatch in readonly-mode");
- } else {
- mdbx_verbose("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO
- " pages",
- env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now));
- }
- }
+ /* Found entry is less than the key. */
+ /* Skip to get the smallest entry larger than key. */
+ i += cr > 0;
+
+ /* store the key index */
+ mc->mc_ki[mc->mc_top] = (indx_t)i;
+ ret.node = (i < nkeys)
+ ? /* fake for LEAF2 */ (MDBX_node *)(intptr_t)-1
+ : /* There is no entry larger or equal to the key. */ NULL;
+ return ret;
}
- mdbx_verbose("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)",
- bootid.x, bootid.y, (bootid.x | bootid.y) ? "" : "not-");
+ if (IS_BRANCH(mp) && cmp == cmp_int_align2)
+ /* Branch pages have no data, so if using integer keys,
+ * alignment is guaranteed. Use faster cmp_int_align4(). */
+ cmp = cmp_int_align4;
-#if MDBX_ENABLE_MADVISE
- /* calculate readahead hint before mmap with zero redundant pages */
- const bool readahead =
- !(env->me_flags & MDBX_NORDAHEAD) &&
- mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE;
-#endif /* MDBX_ENABLE_MADVISE */
+ MDBX_node *node;
+ do {
+ i = (low + high) >> 1;
- err = mdbx_mmap(env->me_flags, &env->me_dxb_mmap, env->me_dbgeo.now,
- env->me_dbgeo.upper, lck_rc ? MMAP_OPTION_TRUNCATE : 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ node = page_node(mp, i);
+ nodekey.iov_len = node_ks(node);
+ nodekey.iov_base = node_key(node);
+ mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >=
+ (char *)nodekey.iov_base + nodekey.iov_len);
-#if MDBX_ENABLE_MADVISE
-#if defined(MADV_DONTDUMP)
- err = madvise(env->me_map, env->me_dxb_mmap.limit, MADV_DONTDUMP)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#endif /* MADV_DONTDUMP */
-#if defined(MADV_DODUMP)
- if (mdbx_runtime_flags & MDBX_DBG_DUMP) {
- const size_t meta_length_aligned2os = pgno_align2os_bytes(env, NUM_METAS);
- err = madvise(env->me_map, meta_length_aligned2os, MADV_DODUMP)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
- }
-#endif /* MADV_DODUMP */
-#endif /* MDBX_ENABLE_MADVISE */
+ cr = cmp(key, &nodekey);
+ if (IS_LEAF(mp))
+ mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY_DEBUG(&nodekey),
+ cr);
+ else
+ mdbx_debug("found branch index %u [%s -> %" PRIaPGNO "], rc = %i", i,
+ DKEY_DEBUG(&nodekey), node_pgno(node), cr);
+ if (unlikely(cr == 0)) {
+ ret.exact = true;
+ break;
+ }
+ low = (cr < 0) ? low : i + 1;
+ high = (cr < 0) ? i - 1 : high;
+ } while (likely(low <= high));
-#ifdef MDBX_USE_VALGRIND
- env->me_valgrind_handle =
- VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx");
-#endif /* MDBX_USE_VALGRIND */
+ /* Found entry is less than the key. */
+ /* Skip to get the smallest entry larger than key. */
+ i += cr > 0;
- mdbx_assert(env, used_bytes >= pgno2bytes(env, NUM_METAS) &&
- used_bytes <= env->me_dxb_mmap.limit);
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- if (env->me_dxb_mmap.filesize > used_bytes &&
- env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit) {
- VALGRIND_MAKE_MEM_NOACCESS(env->me_map + used_bytes,
- env->me_dxb_mmap.filesize - used_bytes);
- MDBX_ASAN_POISON_MEMORY_REGION(env->me_map + used_bytes,
- env->me_dxb_mmap.filesize - used_bytes);
- }
- env->me_poison_edge =
- bytes2pgno(env, (env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit)
- ? env->me_dxb_mmap.filesize
- : env->me_dxb_mmap.limit);
-#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+ /* store the key index */
+ mc->mc_ki[mc->mc_top] = (indx_t)i;
+ ret.node = (i < nkeys)
+ ? page_node(mp, i)
+ : /* There is no entry larger or equal to the key. */ NULL;
+ return ret;
+}
- //-------------------------------- validate/rollback head & steady meta-pages
- if (unlikely(env->me_stuck_meta >= 0)) {
- /* recovery mode */
- MDBX_meta clone;
- MDBX_meta const *const target = METAPAGE(env, env->me_stuck_meta);
- err = mdbx_validate_meta_copy(env, target, &clone);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("target meta[%u] is corrupted",
- bytes2pgno(env, (uint8_t *)data_page(target) - env->me_map));
- return MDBX_CORRUPTED;
+/* Pop a page off the top of the cursor's stack. */
+static __inline void mdbx_cursor_pop(MDBX_cursor *mc) {
+ if (mc->mc_snum) {
+ mdbx_debug("popped page %" PRIaPGNO " off db %d cursor %p",
+ mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *)mc);
+ if (--mc->mc_snum) {
+ mc->mc_top--;
+ } else {
+ mc->mc_flags &= ~C_INITIALIZED;
}
- } else /* not recovery mode */
- while (1) {
- const unsigned meta_clash_mask = meta_eq_mask(env);
- if (unlikely(meta_clash_mask)) {
- mdbx_error("meta-pages are clashed: mask 0x%d", meta_clash_mask);
- return MDBX_CORRUPTED;
- }
-
- if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
- /* non-exclusive mode,
- * meta-pages should be validated by a first process opened the DB */
- volatile const MDBX_meta *const head = meta_prefer_last(env);
- volatile const MDBX_meta *const steady = meta_prefer_steady(env);
- const txnid_t head_txnid = meta_txnid(env, head);
- const txnid_t steady_txnid = meta_txnid(env, steady);
- if (head_txnid == steady_txnid)
- break;
+ }
+}
- if (!env->me_lck_mmap.lck) {
- /* LY: without-lck (read-only) mode, so it is impossible that other
- * process made weak checkpoint. */
- mdbx_error("%s", "without-lck, unable recovery/rollback");
- return MDBX_WANNA_RECOVERY;
- }
+/* Push a page onto the top of the cursor's stack.
+ * Set MDBX_TXN_ERROR on failure. */
+static __inline int mdbx_cursor_push(MDBX_cursor *mc, MDBX_page *mp) {
+ mdbx_debug("pushing page %" PRIaPGNO " on db %d cursor %p", mp->mp_pgno,
+ DDBI(mc), (void *)mc);
- /* LY: assume just have a collision with other running process,
- * or someone make a weak checkpoint */
- mdbx_verbose("%s", "assume collision or online weak checkpoint");
- break;
- }
- mdbx_assert(env, lck_rc == MDBX_RESULT_TRUE);
- /* exclusive mode */
+ if (unlikely(mc->mc_snum >= CURSOR_STACK)) {
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_CURSOR_FULL;
+ }
- MDBX_meta clone;
- const MDBX_meta *const steady = constmeta_prefer_steady(env);
- const MDBX_meta *const head = constmeta_prefer_last(env);
- const txnid_t steady_txnid = meta_txnid(env, steady);
- if (META_IS_STEADY(steady)) {
- err = mdbx_validate_meta_copy(env, steady, &clone);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("meta[%u] with %s txnid %" PRIaTXN
- " is corrupted, %s needed",
- bytes2pgno(env, (uint8_t *)steady - env->me_map), "steady",
- steady_txnid, "manual recovery");
- return MDBX_CORRUPTED;
- }
- if (steady == head)
- break;
- }
+ mdbx_cassert(mc, mc->mc_snum < UINT16_MAX);
+ mc->mc_top = mc->mc_snum++;
+ mc->mc_pg[mc->mc_top] = mp;
+ mc->mc_ki[mc->mc_top] = 0;
- const pgno_t pgno = bytes2pgno(env, (uint8_t *)head - env->me_map);
- const txnid_t head_txnid = meta_txnid(env, head);
- const bool head_valid =
- mdbx_validate_meta_copy(env, head, &clone) == MDBX_SUCCESS;
- mdbx_assert(env, !META_IS_STEADY(steady) || head_txnid != steady_txnid);
- if (unlikely(!head_valid)) {
- if (unlikely(!META_IS_STEADY(steady))) {
- mdbx_error("%s for open or automatic rollback, %s",
- "there are no suitable meta-pages",
- "manual recovery is required");
- return MDBX_CORRUPTED;
- }
- mdbx_warning("meta[%u] with last txnid %" PRIaTXN
- " is corrupted, rollback needed",
- pgno, head_txnid);
- goto purge_meta_head;
- }
+ return MDBX_SUCCESS;
+}
- if (meta_bootid_match(head)) {
- if (env->me_flags & MDBX_RDONLY) {
- mdbx_error("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
- "rollback NOT needed, steady-sync NEEDED%s",
- "opening after an unclean shutdown", bootid.x, bootid.y,
- ", but unable in read-only mode");
- return MDBX_WANNA_RECOVERY;
- }
- mdbx_warning("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
- "rollback NOT needed, steady-sync NEEDED%s",
- "opening after an unclean shutdown", bootid.x, bootid.y,
- "");
- meta = clone;
- atomic_store32(&env->me_lck->mti_unsynced_pages, meta.mm_geo.next,
- mo_Relaxed);
- break;
- }
- if (unlikely(!META_IS_STEADY(steady))) {
- mdbx_error("%s, but %s for automatic rollback: %s",
- "opening after an unclean shutdown",
- "there are no suitable meta-pages",
- "manual recovery is required");
- return MDBX_CORRUPTED;
- }
- if (env->me_flags & MDBX_RDONLY) {
- mdbx_error("%s and rollback needed: (from head %" PRIaTXN
- " to steady %" PRIaTXN ")%s",
- "opening after an unclean shutdown", head_txnid,
- steady_txnid, ", but unable in read-only mode");
- return MDBX_WANNA_RECOVERY;
- }
+__hot static struct page_result
+mdbx_page_get_ex(MDBX_cursor *const mc, const pgno_t pgno,
+ /* TODO: use parent-page ptr */ txnid_t front) {
+ struct page_result ret;
+ MDBX_txn *const txn = mc->mc_txn;
+ mdbx_tassert(txn, front <= txn->mt_front);
+ if (unlikely(pgno >= txn->mt_next_pgno)) {
+ mdbx_error("page #%" PRIaPGNO " beyond next-pgno", pgno);
+ notfound:
+ ret.page = nullptr;
+ ret.err = MDBX_PAGE_NOTFOUND;
+ bailout:
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return ret;
+ }
- purge_meta_head:
- mdbx_notice("%s and doing automatic rollback: "
- "purge%s meta[%u] with%s txnid %" PRIaTXN,
- "opening after an unclean shutdown",
- head_valid ? "" : " invalid", pgno, head_valid ? " weak" : "",
- head_txnid);
- mdbx_ensure(env, META_IS_STEADY(steady));
- err = mdbx_override_meta(env, pgno, 0, head_valid ? head : steady);
- if (err) {
- mdbx_error("rollback: overwrite meta[%u] with txnid %" PRIaTXN
- ", error %d",
- pgno, head_txnid, err);
- return err;
+ MDBX_env *const env = txn->mt_env;
+ mdbx_assert(env, ((txn->mt_flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
+ if (unlikely((txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0)) {
+ const MDBX_txn *spiller = txn;
+ do {
+ /* Spilled pages were dirtied in this txn and flushed
+ * because the dirty list got full. Bring this page
+ * back in from the map (but don't unspill it here,
+ * leave that unless page_touch happens again). */
+ if (unlikely(spiller->mt_flags & MDBX_TXN_SPILLS) &&
+ mdbx_search_spilled(spiller, pgno)) {
+ goto spilled;
}
- mdbx_ensure(env, 0 == meta_txnid(env, head));
- mdbx_ensure(env, 0 == meta_eq_mask(env));
- }
- if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) {
- //-------------------------------------------------- shrink DB & update geo
- const MDBX_meta *head = constmeta_prefer_last(env);
- /* re-check size after mmap */
- if ((env->me_dxb_mmap.current & (env->me_os_psize - 1)) != 0 ||
- env->me_dxb_mmap.current < used_bytes) {
- mdbx_error("unacceptable/unexpected datafile size %" PRIuPTR,
- env->me_dxb_mmap.current);
- return MDBX_PROBLEM;
- }
- if (env->me_dxb_mmap.current != env->me_dbgeo.now) {
- meta.mm_geo.now = bytes2pgno(env, env->me_dxb_mmap.current);
- mdbx_notice("need update meta-geo to filesize %" PRIuPTR
- " bytes, %" PRIaPGNO " pages",
- env->me_dxb_mmap.current, meta.mm_geo.now);
- }
+ const unsigned i = mdbx_dpl_search(spiller, pgno);
+ assert((int)i > 0);
+ if (spiller->tw.dirtylist->items[i].pgno == pgno) {
+ ret.page = spiller->tw.dirtylist->items[i].ptr;
+ spiller->tw.dirtylist->items[i].lru = txn->tw.dirtylru++;
+ goto dirty;
+ }
- if (memcmp(&meta.mm_geo, &head->mm_geo, sizeof(meta.mm_geo))) {
- if ((env->me_flags & MDBX_RDONLY) != 0 ||
- /* recovery mode */ env->me_stuck_meta >= 0) {
- mdbx_warning(
- "skipped update meta.geo in %s mode: from l%" PRIaPGNO
- "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u, to l%" PRIaPGNO
- "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u",
- (env->me_stuck_meta < 0) ? "read-only" : "recovery",
- head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper,
- pv2pages(head->mm_geo.shrink_pv), pv2pages(head->mm_geo.grow_pv),
- meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper,
- pv2pages(meta.mm_geo.shrink_pv), pv2pages(meta.mm_geo.grow_pv));
- } else {
- const txnid_t txnid = constmeta_txnid(env, head);
- const txnid_t next_txnid = safe64_txnid_next(txnid);
- if (unlikely(txnid > MAX_TXNID)) {
- mdbx_error("txnid overflow, raise %d", MDBX_TXN_FULL);
- return MDBX_TXN_FULL;
- }
- mdbx_notice("updating meta.geo: "
- "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
- "/s%u-g%u (txn#%" PRIaTXN "), "
- "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
- "/s%u-g%u (txn#%" PRIaTXN ")",
- head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper,
- pv2pages(head->mm_geo.shrink_pv),
- pv2pages(head->mm_geo.grow_pv), txnid, meta.mm_geo.lower,
- meta.mm_geo.now, meta.mm_geo.upper,
- pv2pages(meta.mm_geo.shrink_pv),
- pv2pages(meta.mm_geo.grow_pv), next_txnid);
+ spiller = spiller->mt_parent;
+ } while (spiller != NULL);
+ }
- mdbx_ensure(env, meta_eq(env, &meta, head));
- meta_set_txnid(env, &meta, next_txnid);
- err = mdbx_sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &meta);
- if (err) {
- mdbx_error("error %d, while updating meta.geo: "
- "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
- "/s%u-g%u (txn#%" PRIaTXN "), "
- "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
- "/s%u-g%u (txn#%" PRIaTXN ")",
- err, head->mm_geo.lower, head->mm_geo.now,
- head->mm_geo.upper, pv2pages(head->mm_geo.shrink_pv),
- pv2pages(head->mm_geo.grow_pv), txnid, meta.mm_geo.lower,
- meta.mm_geo.now, meta.mm_geo.upper,
- pv2pages(meta.mm_geo.shrink_pv),
- pv2pages(meta.mm_geo.grow_pv), next_txnid);
- return err;
- }
- }
- }
+spilled:
+ ret.page = pgno2page(env, pgno);
- atomic_store32(&env->me_lck->mti_discarded_tail,
- bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed);
+dirty:
+ if (unlikely(ret.page->mp_pgno != pgno)) {
+ bad_page(ret.page,
+ "mismatch actual pgno (%" PRIaPGNO ") != expected (%" PRIaPGNO
+ ")\n",
+ ret.page->mp_pgno, pgno);
+ goto notfound;
+ }
- if ((env->me_flags & MDBX_RDONLY) == 0 && env->me_stuck_meta < 0 &&
- (mdbx_runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) {
- for (int n = 0; n < NUM_METAS; ++n) {
- MDBX_meta *const pmeta = METAPAGE(env, n);
- if (unlikely(unaligned_peek_u64(4, &pmeta->mm_magic_and_version) !=
- MDBX_DATA_MAGIC)) {
- const txnid_t txnid = meta_txnid(env, pmeta);
- mdbx_notice("%s %s"
- "meta[%u], txnid %" PRIaTXN,
- "updating db-format signature for",
- META_IS_STEADY(pmeta) ? "stead-" : "weak-", n, txnid);
- err = mdbx_override_meta(env, n, txnid, pmeta);
- if (unlikely(err != MDBX_SUCCESS) &&
- /* Just ignore the MDBX_PROBLEM error, since here it is
- * returned only in case of the attempt to upgrade an obsolete
- * meta-page that is invalid for current state of a DB,
- * e.g. after shrinking DB file */
- err != MDBX_PROBLEM) {
- mdbx_error("%s meta[%u], txnid %" PRIaTXN ", error %d",
- "updating db-format signature for", n, txnid, err);
- return err;
- }
- }
- }
- }
- } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */
+#if !MDBX_DISABLE_PAGECHECKS
+ if (unlikely(ret.page->mp_flags & P_ILL_BITS)) {
+ ret.err =
+ bad_page(ret.page, "invalid page's flags (%u)\n", ret.page->mp_flags);
+ goto bailout;
+ }
- //---------------------------------------------------- setup madvise/readahead
-#if MDBX_ENABLE_MADVISE
- if (used_aligned2os_bytes < env->me_dxb_mmap.current) {
-#if defined(MADV_REMOVE)
- if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0 &&
- /* not recovery mode */ env->me_stuck_meta < 0) {
- mdbx_notice("open-MADV_%s %u..%u", "REMOVE (deallocate file space)",
- env->me_lck->mti_discarded_tail.weak,
- bytes2pgno(env, env->me_dxb_mmap.current));
- err =
- madvise(env->me_map + used_aligned2os_bytes,
- env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_REMOVE)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
- }
-#endif /* MADV_REMOVE */
-#if defined(MADV_DONTNEED)
- mdbx_notice("open-MADV_%s %u..%u", "DONTNEED",
- env->me_lck->mti_discarded_tail.weak,
- bytes2pgno(env, env->me_dxb_mmap.current));
- err =
- madvise(env->me_map + used_aligned2os_bytes,
- env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_DONTNEED)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_MADV_DONTNEED)
- err = ignore_enosys(posix_madvise(
- env->me_map + used_aligned2os_bytes,
- env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#elif defined(POSIX_FADV_DONTNEED)
- err = ignore_enosys(posix_fadvise(
- env->me_lazy_fd, used_aligned2os_bytes,
- env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED));
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
-#endif /* MADV_DONTNEED */
+ if (unlikely(ret.page->mp_txnid > front) &&
+ unlikely(ret.page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) {
+ ret.err = bad_page(
+ ret.page,
+ "invalid page txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
+ ret.page->mp_txnid,
+ (front == txn->mt_front && front != txn->mt_txnid) ? "front-txn"
+ : "parent-page",
+ front);
+ goto bailout;
}
- err = mdbx_set_readahead(env, bytes2pgno(env, used_bytes), readahead, true);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-#endif /* MDBX_ENABLE_MADVISE */
+ if (unlikely((ret.page->mp_upper < ret.page->mp_lower ||
+ ((ret.page->mp_lower | ret.page->mp_upper) & 1) ||
+ PAGEHDRSZ + ret.page->mp_upper > env->me_psize) &&
+ !IS_OVERFLOW(ret.page))) {
+ ret.err =
+ bad_page(ret.page, "invalid page lower(%u)/upper(%u) with limit (%u)\n",
+ ret.page->mp_lower, ret.page->mp_upper, page_space(env));
+ goto bailout;
+ }
+#endif /* !MDBX_DISABLE_PAGECHECKS */
- return rc;
+ ret.err = MDBX_SUCCESS;
+ if (mdbx_audit_enabled())
+ ret.err = mdbx_page_check(mc, ret.page, C_UPDATING);
+ return ret;
}
-/******************************************************************************/
+/* Finish mdbx_page_search() / mdbx_page_search_lowest().
+ * The cursor is at the root page, set up the rest of it. */
+__hot static int mdbx_page_search_root(MDBX_cursor *mc, const MDBX_val *key,
+ int flags) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ int rc;
+ DKBUF_DEBUG;
-/* Open and/or initialize the lock region for the environment. */
-__cold static int mdbx_setup_lck(MDBX_env *env, char *lck_pathname,
- mdbx_mode_t mode) {
- mdbx_assert(env, env->me_lazy_fd != INVALID_HANDLE_VALUE);
- mdbx_assert(env, env->me_lfd == INVALID_HANDLE_VALUE);
+ while (IS_BRANCH(mp)) {
+ MDBX_node *node;
+ int i;
+
+ mdbx_debug("branch page %" PRIaPGNO " has %u keys", mp->mp_pgno,
+ page_numkeys(mp));
+ /* Don't assert on branch pages in the GC. We can get here
+ * while in the process of rebalancing a GC branch page; we must
+ * let that proceed. ITS#8336 */
+ mdbx_cassert(mc, !mc->mc_dbi || page_numkeys(mp) > 1);
+ mdbx_debug("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0)));
+
+ if (flags & (MDBX_PS_FIRST | MDBX_PS_LAST)) {
+ i = 0;
+ if (flags & MDBX_PS_LAST) {
+ i = page_numkeys(mp) - 1;
+ /* if already init'd, see if we're already in right place */
+ if (mc->mc_flags & C_INITIALIZED) {
+ if (mc->mc_ki[mc->mc_top] == i) {
+ mc->mc_top = mc->mc_snum++;
+ mp = mc->mc_pg[mc->mc_top];
+ goto ready;
+ }
+ }
+ }
+ } else {
+ const struct node_result nsr = mdbx_node_search(mc, key);
+ if (nsr.node)
+ i = mc->mc_ki[mc->mc_top] + nsr.exact - 1;
+ else
+ i = page_numkeys(mp) - 1;
+ mdbx_debug("following index %u for key [%s]", i, DKEY_DEBUG(key));
+ }
- int err = mdbx_openfile(MDBX_OPEN_LCK, env, lck_pathname, &env->me_lfd, mode);
- if (err != MDBX_SUCCESS) {
- if (!(err == MDBX_ENOFILE && (env->me_flags & MDBX_EXCLUSIVE)) &&
- !((err == MDBX_EROFS || err == MDBX_EACCESS || err == MDBX_EPERM) &&
- (env->me_flags & MDBX_RDONLY)))
- return err;
+ mdbx_cassert(mc, i >= 0 && i < (int)page_numkeys(mp));
+ node = page_node(mp, i);
- /* ensure the file system is read-only */
- err = mdbx_check_fs_rdonly(env->me_lazy_fd, lck_pathname, err);
- if (err != MDBX_SUCCESS &&
- /* ignore ERROR_NOT_SUPPORTED for exclusive mode */
- !(err == MDBX_ENOSYS && (env->me_flags & MDBX_EXCLUSIVE)))
- return err;
+ if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
+ pp_txnid4chk(mp, mc->mc_txn))) != 0))
+ return rc;
- /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
- /* beginning of a locked section ---------------------------------------- */
- lcklist_lock();
- mdbx_assert(env, env->me_lcklist_next == nullptr);
- env->me_lfd = INVALID_HANDLE_VALUE;
- const int rc = mdbx_lck_seize(env);
- if (MDBX_IS_ERROR(rc)) {
- /* Calling lcklist_detach_locked() is required to restore POSIX-filelock
- * and this job will be done by mdbx_env_close0(). */
- lcklist_unlock();
+ mc->mc_ki[mc->mc_top] = (indx_t)i;
+ if (unlikely(rc = mdbx_cursor_push(mc, mp)))
return rc;
+
+ ready:
+ if (flags & MDBX_PS_MODIFY) {
+ if (unlikely((rc = mdbx_page_touch(mc)) != 0))
+ return rc;
+ mp = mc->mc_pg[mc->mc_top];
}
- /* insert into inprocess lck-list */
- env->me_lcklist_next = inprocess_lcklist_head;
- inprocess_lcklist_head = env;
- lcklist_unlock();
- /* end of a locked section ---------------------------------------------- */
+ }
- env->me_lck = lckless_stub(env);
- env->me_maxreaders = UINT_MAX;
- mdbx_debug("lck-setup:%s%s%s", " lck-less",
- (env->me_flags & MDBX_RDONLY) ? " readonly" : "",
- (rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");
- return rc;
+#if !MDBX_DISABLE_PAGECHECKS
+ if (unlikely(!IS_LEAF(mp))) {
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return bad_page(mp, "index points to a page with 0x%02x flags\n",
+ mp->mp_flags);
}
+#endif /* !MDBX_DISABLE_PAGECHECKS */
- /* beginning of a locked section ------------------------------------------ */
- lcklist_lock();
- mdbx_assert(env, env->me_lcklist_next == nullptr);
+ mdbx_debug("found leaf page %" PRIaPGNO " for key [%s]", mp->mp_pgno,
+ DKEY_DEBUG(key));
+ mc->mc_flags |= C_INITIALIZED;
+ mc->mc_flags &= ~C_EOF;
- /* Try to get exclusive lock. If we succeed, then
- * nobody is using the lock region and we should initialize it. */
- err = mdbx_lck_seize(env);
- if (MDBX_IS_ERROR(err)) {
- bailout:
- /* Calling lcklist_detach_locked() is required to restore POSIX-filelock
- * and this job will be done by mdbx_env_close0(). */
- lcklist_unlock();
- return err;
- }
+ return MDBX_SUCCESS;
+}
- MDBX_env *inprocess_neighbor = nullptr;
- if (err == MDBX_RESULT_TRUE) {
- err = uniq_check(&env->me_lck_mmap, &inprocess_neighbor);
- if (MDBX_IS_ERROR(err))
- goto bailout;
- if (inprocess_neighbor &&
- ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 ||
- (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) != 0)) {
- err = MDBX_BUSY;
- goto bailout;
- }
+static int mdbx_setup_dbx(MDBX_dbx *const dbx, const MDBX_db *const db,
+ const unsigned pagesize) {
+ if (unlikely(!dbx->md_cmp)) {
+ dbx->md_cmp = get_default_keycmp(db->md_flags);
+ dbx->md_dcmp = get_default_datacmp(db->md_flags);
}
- const int lck_seize_rc = err;
- mdbx_debug("lck-setup:%s%s%s", " with-lck",
- (env->me_flags & MDBX_RDONLY) ? " readonly" : "",
- (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive"
- : " cooperative");
+ dbx->md_klen_min =
+ (db->md_flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0;
+ dbx->md_klen_max = keysize_max(pagesize, db->md_flags);
+ assert(dbx->md_klen_max != (unsigned)-1);
- uint64_t size = 0;
- err = mdbx_filesize(env->me_lfd, &size);
- if (unlikely(err != MDBX_SUCCESS))
- goto bailout;
+ dbx->md_vlen_min = (db->md_flags & MDBX_INTEGERDUP)
+ ? 4 /* sizeof(uint32_t) */
+ : ((db->md_flags & MDBX_DUPFIXED) ? 1 : 0);
+ dbx->md_vlen_max = valsize_max(pagesize, db->md_flags);
+ assert(dbx->md_vlen_max != (unsigned)-1);
- if (lck_seize_rc == MDBX_RESULT_TRUE) {
- size = ceil_powerof2(env->me_maxreaders * sizeof(MDBX_reader) +
- sizeof(MDBX_lockinfo),
- env->me_os_psize);
- mdbx_jitter4testing(false);
- } else {
- if (env->me_flags & MDBX_EXCLUSIVE) {
- err = MDBX_BUSY;
- goto bailout;
- }
- if (size > INT_MAX || (size & (env->me_os_psize - 1)) != 0 ||
- size < env->me_os_psize) {
- mdbx_error("lck-file has invalid size %" PRIu64 " bytes", size);
- err = MDBX_PROBLEM;
- goto bailout;
+ if ((db->md_flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->md_xsize) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(db->md_xsize < dbx->md_vlen_min ||
+ db->md_xsize > dbx->md_vlen_max)) {
+ mdbx_error("db.md_xsize (%u) <> min/max value-length (%zu/%zu)",
+ db->md_xsize, dbx->md_vlen_min, dbx->md_vlen_max);
+ return MDBX_CORRUPTED;
}
+ dbx->md_vlen_min = dbx->md_vlen_max = db->md_xsize;
}
+ return MDBX_SUCCESS;
+}
- const size_t maxreaders =
- ((size_t)size - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader);
- if (maxreaders < 4) {
- mdbx_error("lck-size too small (up to %" PRIuPTR " readers)", maxreaders);
- err = MDBX_PROBLEM;
- goto bailout;
- }
- env->me_maxreaders = (maxreaders <= MDBX_READERS_LIMIT)
- ? (unsigned)maxreaders
- : (unsigned)MDBX_READERS_LIMIT;
+static int mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi) {
+ MDBX_cursor_couple couple;
+ if (unlikely(TXN_DBI_CHANGED(txn, dbi)))
+ return MDBX_BAD_DBI;
+ int rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- err = mdbx_mmap((env->me_flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP,
- &env->me_lck_mmap, (size_t)size, (size_t)size,
- lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE
- : MMAP_OPTION_SEMAPHORE);
- if (unlikely(err != MDBX_SUCCESS))
- goto bailout;
+ MDBX_dbx *const dbx = &txn->mt_dbxs[dbi];
+ rc = mdbx_page_search(&couple.outer, &dbx->md_name, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc;
-#if MDBX_ENABLE_MADVISE
-#ifdef MADV_DODUMP
- err = madvise(env->me_lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- goto bailout;
-#endif /* MADV_DODUMP */
+ MDBX_val data;
+ struct node_result nsr = mdbx_node_search(&couple.outer, &dbx->md_name);
+ if (unlikely(!nsr.exact))
+ return MDBX_BAD_DBI;
+ if (unlikely((node_flags(nsr.node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA))
+ return MDBX_INCOMPATIBLE; /* not a named DB */
-#ifdef MADV_WILLNEED
- err = madvise(env->me_lck_mmap.lck, size, MADV_WILLNEED)
- ? ignore_enosys(errno)
- : MDBX_SUCCESS;
- if (unlikely(MDBX_IS_ERROR(err)))
- goto bailout;
-#elif defined(POSIX_MADV_WILLNEED)
- err = ignore_enosys(
- posix_madvise(env->me_lck_mmap.lck, size, POSIX_MADV_WILLNEED));
- if (unlikely(MDBX_IS_ERROR(err)))
- goto bailout;
-#endif /* MADV_WILLNEED */
-#endif /* MDBX_ENABLE_MADVISE */
+ const txnid_t pp_txnid =
+ pp_txnid4chk(couple.outer.mc_pg[couple.outer.mc_top], txn);
+ rc = mdbx_node_read(&couple.outer, nsr.node, &data, pp_txnid);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- struct MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (lck_seize_rc == MDBX_RESULT_TRUE) {
- /* LY: exclusive mode, check and reset lck content */
- memset(lck, 0, (size_t)size);
- mdbx_jitter4testing(false);
- lck->mti_magic_and_version = MDBX_LOCK_MAGIC;
- lck->mti_os_and_format = MDBX_LOCK_FORMAT;
-#if MDBX_ENABLE_PGOP_STAT
- lck->mti_pgop_stat.wops.weak = 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- err = mdbx_msync(&env->me_lck_mmap, 0, (size_t)size, MDBX_SYNC_NONE);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("initial-%s for lck-file failed", "msync");
- goto bailout;
- }
- err = mdbx_fsync(env->me_lck_mmap.fd, MDBX_SYNC_SIZE);
- if (unlikely(err != MDBX_SUCCESS)) {
- mdbx_error("initial-%s for lck-file failed", "fsync");
- goto bailout;
- }
- } else {
- if (lck->mti_magic_and_version != MDBX_LOCK_MAGIC) {
- const bool invalid = (lck->mti_magic_and_version >> 8) != MDBX_MAGIC;
- mdbx_error(
- "lock region has %s",
- invalid
- ? "invalid magic"
- : "incompatible version (only applications with nearly or the "
- "same versions of libmdbx can share the same database)");
- err = invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
- goto bailout;
- }
- if (lck->mti_os_and_format != MDBX_LOCK_FORMAT) {
- mdbx_error("lock region has os/format signature 0x%" PRIx32
- ", expected 0x%" PRIx32,
- lck->mti_os_and_format, MDBX_LOCK_FORMAT);
- err = MDBX_VERSION_MISMATCH;
- goto bailout;
- }
+ if (unlikely(data.iov_len != sizeof(MDBX_db)))
+ return MDBX_INCOMPATIBLE; /* not a named DB */
+
+ uint16_t md_flags = UNALIGNED_PEEK_16(data.iov_base, MDBX_db, md_flags);
+ /* The txn may not know this DBI, or another process may
+ * have dropped and recreated the DB with other flags. */
+ MDBX_db *const db = &txn->mt_dbs[dbi];
+ if (unlikely((db->md_flags & DB_PERSISTENT_FLAGS) != md_flags))
+ return MDBX_INCOMPATIBLE;
+
+ memcpy(db, data.iov_base, sizeof(MDBX_db));
+#if !MDBX_DISABLE_PAGECHECKS
+ mdbx_tassert(txn, txn->mt_front >= pp_txnid);
+ if (unlikely(db->md_mod_txnid > pp_txnid)) {
+ mdbx_error("db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")",
+ db->md_mod_txnid, pp_txnid);
+ return MDBX_CORRUPTED;
}
+#endif /* !MDBX_DISABLE_PAGECHECKS */
+ rc = mdbx_setup_dbx(dbx, db, txn->mt_env->me_psize);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- err = mdbx_lck_init(env, inprocess_neighbor, lck_seize_rc);
- if (MDBX_IS_ERROR(err))
- goto bailout;
+ txn->mt_dbistate[dbi] &= ~DBI_STALE;
+ return MDBX_SUCCESS;
+}
- mdbx_ensure(env, env->me_lcklist_next == nullptr);
- /* insert into inprocess lck-list */
- env->me_lcklist_next = inprocess_lcklist_head;
- inprocess_lcklist_head = env;
- lcklist_unlock();
- /* end of a locked section ------------------------------------------------ */
+/* Search for the lowest key under the current branch page.
+ * This just bypasses a numkeys check in the current page
+ * before calling mdbx_page_search_root(), because the callers
+ * are all in situations where the current page is known to
+ * be underfilled. */
+__hot static int mdbx_page_search_lowest(MDBX_cursor *mc) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ mdbx_cassert(mc, IS_BRANCH(mp));
+ MDBX_node *node = page_node(mp, 0);
+ int rc;
- mdbx_assert(env, !MDBX_IS_ERROR(lck_seize_rc));
- env->me_lck = lck;
- return lck_seize_rc;
+ if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
+ pp_txnid4chk(mp, mc->mc_txn))) != 0))
+ return rc;
+
+ mc->mc_ki[mc->mc_top] = 0;
+ if (unlikely(rc = mdbx_cursor_push(mc, mp)))
+ return rc;
+ return mdbx_page_search_root(mc, NULL, MDBX_PS_FIRST);
}
-__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
- if (volume <= 1024 * 1024 * 4ul)
- return MDBX_RESULT_TRUE;
+/* Search for the page a given key should be in.
+ * Push it and its parent pages on the cursor stack.
+ *
+ * [in,out] mc the cursor for this operation.
+ * [in] key the key to search for, or NULL for first/last page.
+ * [in] flags If MDBX_PS_MODIFY is set, visited pages in the DB
+ * are touched (updated with new page numbers).
+ * If MDBX_PS_FIRST or MDBX_PS_LAST is set, find first or last
+ * leaf.
+ * This is used by mdbx_cursor_first() and mdbx_cursor_last().
+ * If MDBX_PS_ROOTONLY set, just fetch root node, no further
+ * lookups.
+ *
+ * Returns 0 on success, non-zero on failure. */
+__hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key,
+ int flags) {
+ int rc;
+ pgno_t root;
- intptr_t pagesize, total_ram_pages;
- int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ /* Make sure the txn is still viable, then find the root from
+ * the txn's db table and set it as the root of the cursor's stack. */
+ if (unlikely(mc->mc_txn->mt_flags & MDBX_TXN_BLOCKED)) {
+ mdbx_debug("%s", "transaction has failed, must abort");
+ return MDBX_BAD_TXN;
+ }
- const int log2page = log2n_powerof2(pagesize);
- const intptr_t volume_pages = (volume + pagesize - 1) >> log2page;
- const intptr_t redundancy_pages =
- (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page)
- : (intptr_t)(redundancy + pagesize - 1) >> log2page;
- if (volume_pages >= total_ram_pages ||
- volume_pages + redundancy_pages >= total_ram_pages)
- return MDBX_RESULT_FALSE;
+ /* Make sure we're using an up-to-date root */
+ if (unlikely(*mc->mc_dbistate & DBI_STALE)) {
+ rc = mdbx_fetch_sdb(mc->mc_txn, mc->mc_dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+ root = mc->mc_db->md_root;
- intptr_t avail_ram_pages;
- err = mdbx_get_sysraminfo(nullptr, nullptr, &avail_ram_pages);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ if (unlikely(root == P_INVALID)) { /* Tree is empty. */
+ mdbx_debug("%s", "tree is empty");
+ return MDBX_NOTFOUND;
+ }
- return (volume_pages + redundancy_pages >= avail_ram_pages)
- ? MDBX_RESULT_FALSE
- : MDBX_RESULT_TRUE;
-}
+ mdbx_cassert(mc, root >= NUM_METAS);
+ if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) {
+ txnid_t pp_txnid = mc->mc_db->md_mod_txnid;
+ pp_txnid = /* mc->mc_db->md_mod_txnid maybe zero in a legacy DB */ pp_txnid
+ ? pp_txnid
+ : mc->mc_txn->mt_txnid;
+ if ((mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
+ MDBX_txn *scan = mc->mc_txn;
+ do
+ if ((scan->mt_flags & MDBX_TXN_DIRTY) &&
+ (mc->mc_dbi == MAIN_DBI ||
+ (scan->mt_dbistate[mc->mc_dbi] & DBI_DIRTY))) {
+ pp_txnid = scan->mt_front;
+ break;
+ }
+ while (unlikely((scan = scan->mt_parent) != nullptr));
+ }
+ if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], pp_txnid)) != 0))
+ return rc;
+ }
-/* Merge sync flags */
-static uint32_t merge_sync_flags(const uint32_t a, const uint32_t b) {
- uint32_t r = a | b;
+ mc->mc_snum = 1;
+ mc->mc_top = 0;
- /* avoid false MDBX_UTTERLY_NOSYNC */
- if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
- !F_ISSET(b, MDBX_UTTERLY_NOSYNC))
- r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC;
+ mdbx_debug("db %d root page %" PRIaPGNO " has flags 0x%X", DDBI(mc), root,
+ mc->mc_pg[0]->mp_flags);
- /* convert MDBX_DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */
- if ((r & (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC)) ==
- (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC) &&
- !F_ISSET(r, MDBX_UTTERLY_NOSYNC))
- r = (r - MDBX_DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC;
+ if (flags & MDBX_PS_MODIFY) {
+ if (!(*mc->mc_dbistate & DBI_DIRTY) && unlikely(rc = mdbx_touch_dbi(mc)))
+ return rc;
+ if (unlikely(rc = mdbx_page_touch(mc)))
+ return rc;
+ }
- /* force MDBX_NOMETASYNC if MDBX_SAFE_NOSYNC enabled */
- if (r & MDBX_SAFE_NOSYNC)
- r |= MDBX_NOMETASYNC;
+ if (flags & MDBX_PS_ROOTONLY)
+ return MDBX_SUCCESS;
- assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) &&
- !F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
- !F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
- return r;
+ return mdbx_page_search_root(mc, key, flags);
}
-__cold static int __must_check_result mdbx_override_meta(
- MDBX_env *env, unsigned target, txnid_t txnid, const MDBX_meta *shape) {
- int rc = alloc_page_buf(env);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- MDBX_page *const page = env->me_pbuf;
- mdbx_meta_model(env, page, target);
- MDBX_meta *const model = page_meta(page);
- meta_set_txnid(env, model, txnid);
- mdbx_assert(env, meta_checktxnid(env, model, true));
- if (shape) {
- if (txnid && unlikely(!meta_checktxnid(env, shape, false))) {
- mdbx_error("bailout overriding meta-%u since model failed "
- "freedb/maindb %s-check for txnid #%" PRIaTXN,
- target, "pre", constmeta_txnid(env, shape));
- return MDBX_PROBLEM;
- }
- if (mdbx_runtime_flags & MDBX_DBG_DONT_UPGRADE)
- memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version,
- sizeof(model->mm_magic_and_version));
- model->mm_extra_flags = shape->mm_extra_flags;
- model->mm_validator_id = shape->mm_validator_id;
- model->mm_extra_pagehdr = shape->mm_extra_pagehdr;
- memcpy(&model->mm_geo, &shape->mm_geo, sizeof(model->mm_geo));
- memcpy(&model->mm_dbs, &shape->mm_dbs, sizeof(model->mm_dbs));
- memcpy(&model->mm_canary, &shape->mm_canary, sizeof(model->mm_canary));
- memcpy(&model->mm_pages_retired, &shape->mm_pages_retired,
- sizeof(model->mm_pages_retired));
- if (txnid) {
- if ((!model->mm_dbs[FREE_DBI].md_mod_txnid &&
- model->mm_dbs[FREE_DBI].md_root != P_INVALID) ||
- (!model->mm_dbs[MAIN_DBI].md_mod_txnid &&
- model->mm_dbs[MAIN_DBI].md_root != P_INVALID))
- memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version,
- sizeof(model->mm_magic_and_version));
- if (unlikely(!meta_checktxnid(env, model, false))) {
- mdbx_error("bailout overriding meta-%u since model failed "
- "freedb/maindb %s-check for txnid #%" PRIaTXN,
- target, "post", txnid);
- return MDBX_PROBLEM;
- }
+/* Return the data associated with a given node.
+ *
+ * [in] mc The cursor for this operation.
+ * [in] leaf The node being read.
+ * [out] data Updated to point to the node's data.
+ *
+ * Returns 0 on success, non-zero on failure. */
+static __always_inline int mdbx_node_read(MDBX_cursor *mc,
+ const MDBX_node *node, MDBX_val *data,
+ const txnid_t front) {
+ data->iov_len = node_ds(node);
+ data->iov_base = node_data(node);
+ if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) {
+ /* Read overflow data. */
+ MDBX_page *omp; /* overflow page */
+ int rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, front);
+ if (unlikely((rc != MDBX_SUCCESS))) {
+ mdbx_debug("read overflow page %" PRIaPGNO " failed",
+ node_largedata_pgno(node));
+ return rc;
}
+ data->iov_base = page_data(omp);
}
- unaligned_poke_u64(4, model->mm_datasync_sign, meta_sign(model));
- rc = mdbx_validate_meta(env, model, page, target, nullptr);
- if (unlikely(MDBX_IS_ERROR(rc)))
- return MDBX_PROBLEM;
+ return MDBX_SUCCESS;
+}
- if (shape && memcmp(model, shape, sizeof(MDBX_meta)) == 0)
- return MDBX_SUCCESS;
+int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) {
+ DKBUF_DEBUG;
+ mdbx_debug("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.wops.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- if (env->me_flags & MDBX_WRITEMAP) {
- rc = mdbx_msync(&env->me_dxb_mmap, 0,
- pgno_align2os_bytes(env, model->mm_geo.next),
- MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- /* mdbx_override_meta() called only while current process have exclusive
- * lock of a DB file. So meta-page could be updated directly without
- * clearing consistency flag by mdbx_meta_update_begin() */
- memcpy(pgno2page(env, target), page, env->me_psize);
- mdbx_flush_incoherent_cpu_writeback();
- rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target + 1),
- MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- } else {
- const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
- ? env->me_dsync_fd
- : env->me_lazy_fd;
- rc = mdbx_pwrite(fd, page, env->me_psize, pgno2bytes(env, target));
- if (rc == MDBX_SUCCESS && fd == env->me_lazy_fd)
- rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
- }
- mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS),
- env->me_os_psize);
- return rc;
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(!key || !data))
+ return MDBX_EINVAL;
+
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
+
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ return mdbx_cursor_set(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err;
}
-__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) {
- if (unlikely(target >= NUM_METAS))
- return MDBX_EINVAL;
- int rc = check_env(env, true);
+int mdbx_get_equal_or_great(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
+ MDBX_val *data) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely((env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) !=
- MDBX_EXCLUSIVE))
- return MDBX_EPERM;
+ if (unlikely(!key || !data))
+ return MDBX_EINVAL;
- const MDBX_meta *target_meta = METAPAGE(env, target);
- txnid_t new_txnid = safe64_txnid_next(constmeta_txnid(env, target_meta));
- for (unsigned n = 0; n < NUM_METAS; ++n) {
- MDBX_page *page = pgno2page(env, n);
- MDBX_meta meta = *page_meta(page);
- if (n == target)
- continue;
- if (mdbx_validate_meta(env, &meta, page, n, nullptr) != MDBX_SUCCESS) {
- int err = mdbx_override_meta(env, n, 0, nullptr);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- } else {
- txnid_t txnid = constmeta_txnid(env, &meta);
- if (new_txnid <= txnid)
- new_txnid = safe64_txnid_next(txnid);
- }
- }
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
- if (unlikely(new_txnid > MAX_TXNID)) {
- mdbx_error("txnid overflow, raise %d", MDBX_TXN_FULL);
- return MDBX_TXN_FULL;
- }
- return mdbx_override_meta(env, target, new_txnid, target_meta);
-}
+ if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED))
+ return MDBX_BAD_TXN;
-__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname,
- unsigned target_meta, bool writeable) {
- if (unlikely(target_meta >= NUM_METAS))
- return MDBX_EINVAL;
- int rc = check_env(env, false);
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(env->me_map))
- return MDBX_EPERM;
- env->me_stuck_meta = (int8_t)target_meta;
- return mdbx_env_open(
- env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY,
- 0);
+ return mdbx_cursor_get(&cx.outer, key, data, MDBX_SET_LOWERBOUND);
}
-typedef struct {
- void *buffer_for_free;
- char *lck, *dxb;
- size_t ent_len;
-} MDBX_handle_env_pathname;
+int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
+ size_t *values_count) {
+ DKBUF_DEBUG;
+ mdbx_debug("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
-__cold static int mdbx_handle_env_pathname(MDBX_handle_env_pathname *ctx,
- const char *pathname,
- MDBX_env_flags_t *flags,
- const mdbx_mode_t mode) {
- int rc;
- memset(ctx, 0, sizeof(*ctx));
- if (unlikely(!pathname))
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(!key || !data))
return MDBX_EINVAL;
-#if defined(_WIN32) || defined(_WIN64)
- const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
- if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
- return ERROR_INVALID_NAME;
- wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
- if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
- return ERROR_INVALID_NAME;
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
- const DWORD dwAttrib = GetFileAttributesW(pathnameW);
- if (dwAttrib == INVALID_FILE_ATTRIBUTES) {
- rc = GetLastError();
- if (rc != MDBX_ENOFILE)
- return rc;
- if (mode == 0 || (*flags & MDBX_RDONLY) != 0)
- /* can't open existing */
- return rc;
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* auto-create directory if requested */
- if ((*flags & MDBX_NOSUBDIR) == 0 &&
- !CreateDirectoryW(pathnameW, nullptr)) {
- rc = GetLastError();
- if (rc != ERROR_ALREADY_EXISTS)
- return rc;
- }
- } else {
- /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
- *flags |= MDBX_NOSUBDIR;
- if (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)
- *flags -= MDBX_NOSUBDIR;
+ rc = mdbx_cursor_set(&cx.outer, key, data, MDBX_SET_KEY).err;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_NOTFOUND && values_count)
+ *values_count = 0;
+ return rc;
}
-#else
- struct stat st;
- if (stat(pathname, &st)) {
- rc = errno;
- if (rc != MDBX_ENOFILE)
- return rc;
- if (mode == 0 || (*flags & MDBX_RDONLY) != 0)
- /* can't open existing */
- return rc;
- /* auto-create directory if requested */
- const mdbx_mode_t dir_mode =
- (/* inherit read/write permissions for group and others */ mode &
- (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
- /* always add read/write/search for owner */ S_IRWXU |
- ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) |
- ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0);
- if ((*flags & MDBX_NOSUBDIR) == 0 && mkdir(pathname, dir_mode)) {
- rc = errno;
- if (rc != EEXIST)
- return rc;
+ if (values_count) {
+ *values_count = 1;
+ if (cx.outer.mc_xcursor != NULL) {
+ MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
+ cx.outer.mc_ki[cx.outer.mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ // coverity[uninit_use : FALSE]
+ mdbx_tassert(txn, cx.outer.mc_xcursor == &cx.inner &&
+ (cx.inner.mx_cursor.mc_flags & C_INITIALIZED));
+ // coverity[uninit_use : FALSE]
+ *values_count =
+ (sizeof(*values_count) >= sizeof(cx.inner.mx_db.md_entries) ||
+ cx.inner.mx_db.md_entries <= PTRDIFF_MAX)
+ ? (size_t)cx.inner.mx_db.md_entries
+ : PTRDIFF_MAX;
+ }
}
- } else {
- /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
- *flags |= MDBX_NOSUBDIR;
- if (S_ISDIR(st.st_mode))
- *flags -= MDBX_NOSUBDIR;
}
-#endif
+ return MDBX_SUCCESS;
+}
- static const char dxb_name[] = MDBX_DATANAME;
- static const size_t dxb_name_len = sizeof(dxb_name) - 1;
- static const char lck_name[] = MDBX_LOCKNAME;
- static const char lock_suffix[] = MDBX_LOCK_SUFFIX;
+/* Find a sibling for a page.
+ * Replaces the page at the top of the cursor's stack with the specified
+ * sibling, if one exists.
+ *
+ * [in] mc The cursor for this operation.
+ * [in] dir SIBLING_LEFT or SIBLING_RIGHT.
+ *
+ * Returns 0 on success, non-zero on failure. */
+static int mdbx_cursor_sibling(MDBX_cursor *mc, int dir) {
+ int rc;
+ MDBX_node *node;
+ MDBX_page *mp;
+ assert(dir == SIBLING_LEFT || dir == SIBLING_RIGHT);
- ctx->ent_len = strlen(pathname);
- if ((*flags & MDBX_NOSUBDIR) && ctx->ent_len >= dxb_name_len &&
- !memcmp(dxb_name, pathname + ctx->ent_len - dxb_name_len, dxb_name_len)) {
- *flags -= MDBX_NOSUBDIR;
- ctx->ent_len -= dxb_name_len;
- }
+ if (unlikely(mc->mc_snum < 2))
+ return MDBX_NOTFOUND; /* root has no siblings */
- const size_t bytes_needed =
- ctx->ent_len * 2 + ((*flags & MDBX_NOSUBDIR)
- ? sizeof(lock_suffix) + 1
- : sizeof(lck_name) + sizeof(dxb_name));
- ctx->buffer_for_free = mdbx_malloc(bytes_needed);
- if (!ctx->buffer_for_free)
- return MDBX_ENOMEM;
+ mdbx_cursor_pop(mc);
+ mdbx_debug("parent page is page %" PRIaPGNO ", index %u",
+ mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]);
- ctx->lck = ctx->buffer_for_free;
- if (*flags & MDBX_NOSUBDIR) {
- ctx->dxb = ctx->lck + ctx->ent_len + sizeof(lock_suffix);
- sprintf(ctx->lck, "%s%s", pathname, lock_suffix);
- strcpy(ctx->dxb, pathname);
+ if ((dir == SIBLING_RIGHT)
+ ? (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mc->mc_pg[mc->mc_top]))
+ : (mc->mc_ki[mc->mc_top] == 0)) {
+ mdbx_debug("no more keys aside, moving to next %s sibling",
+ dir ? "right" : "left");
+ if (unlikely((rc = mdbx_cursor_sibling(mc, dir)) != MDBX_SUCCESS)) {
+ /* undo cursor_pop before returning */
+ mc->mc_top++;
+ mc->mc_snum++;
+ return rc;
+ }
} else {
- ctx->dxb = ctx->lck + ctx->ent_len + sizeof(lck_name);
- sprintf(ctx->lck, "%.*s%s", (int)ctx->ent_len, pathname, lck_name);
- sprintf(ctx->dxb, "%.*s%s", (int)ctx->ent_len, pathname, dxb_name);
+ assert((dir - 1) == -1 || (dir - 1) == 1);
+ mc->mc_ki[mc->mc_top] += (indx_t)(dir - 1);
+ mdbx_debug("just moving to %s index key %u",
+ (dir == SIBLING_RIGHT) ? "right" : "left",
+ mc->mc_ki[mc->mc_top]);
+ }
+ mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top]));
+
+ node = page_node(mp = mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
+ pp_txnid4chk(mp, mc->mc_txn))) != 0)) {
+ /* mc will be inconsistent if caller does mc_snum++ as above */
+ mc->mc_flags &= ~(C_INITIALIZED | C_EOF);
+ return rc;
}
+ rc = mdbx_cursor_push(mc, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ mc->mc_ki[mc->mc_top] =
+ (indx_t)((dir == SIBLING_LEFT) ? page_numkeys(mp) - 1 : 0);
return MDBX_SUCCESS;
}
-__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) {
- switch (mode) {
- default:
- return MDBX_EINVAL;
- case MDBX_ENV_JUST_DELETE:
- case MDBX_ENV_ENSURE_UNUSED:
- case MDBX_ENV_WAIT_FOR_UNUSED:
- break;
- }
+/* Move the cursor to the next data item. */
+static int mdbx_cursor_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
+ MDBX_cursor_op op) {
+ MDBX_page *mp;
+ MDBX_node *node;
+ int rc;
+
+ if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_NEXT_DUP)
+ return MDBX_NOTFOUND;
+
+ if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
+ return mdbx_cursor_first(mc, key, data);
-#ifdef __e2k__ /* https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=6011 */
- MDBX_env *const dummy_env = alloca(sizeof(MDBX_env));
-#else
- MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo;
-#endif
- memset(dummy_env, 0, sizeof(*dummy_env));
- dummy_env->me_flags =
- (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS;
- dummy_env->me_os_psize = (unsigned)mdbx_syspagesize();
- dummy_env->me_psize = (unsigned)mdbx_default_pagesize();
- dummy_env->me_pathname = (char *)pathname;
+ mp = mc->mc_pg[mc->mc_top];
+ if (unlikely(mc->mc_flags & C_EOF)) {
+ if (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mp))
+ return MDBX_NOTFOUND;
+ mc->mc_flags ^= C_EOF;
+ }
- MDBX_handle_env_pathname env_pathname;
- STATIC_ASSERT(sizeof(dummy_env->me_flags) == sizeof(MDBX_env_flags_t));
- int rc = MDBX_RESULT_TRUE,
- err = mdbx_handle_env_pathname(
- &env_pathname, pathname, (MDBX_env_flags_t *)&dummy_env->me_flags, 0);
- if (likely(err == MDBX_SUCCESS)) {
- mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE,
- dxb_handle = INVALID_HANDLE_VALUE;
- if (mode > MDBX_ENV_JUST_DELETE) {
- err = mdbx_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.dxb,
- &dxb_handle, 0);
- err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
- if (err == MDBX_SUCCESS) {
- err = mdbx_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.lck,
- &clk_handle, 0);
- err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
+ if (mc->mc_db->md_flags & MDBX_DUPSORT) {
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ if (op == MDBX_NEXT || op == MDBX_NEXT_DUP) {
+ rc =
+ mdbx_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_NEXT);
+ if (op != MDBX_NEXT || rc != MDBX_NOTFOUND) {
+ if (likely(rc == MDBX_SUCCESS))
+ get_key_optional(node, key);
+ return rc;
+ }
}
- if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE)
- err = mdbx_lockfile(clk_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
- if (err == MDBX_SUCCESS && dxb_handle != INVALID_HANDLE_VALUE)
- err = mdbx_lockfile(dxb_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
+ } else {
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+ if (op == MDBX_NEXT_DUP)
+ return MDBX_NOTFOUND;
}
+ }
- if (err == MDBX_SUCCESS) {
- err = mdbx_removefile(env_pathname.dxb);
- if (err == MDBX_SUCCESS)
- rc = MDBX_SUCCESS;
- else if (err == MDBX_ENOFILE)
- err = MDBX_SUCCESS;
- }
+ mdbx_debug("cursor_next: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno,
+ (void *)mc);
+ if (mc->mc_flags & C_DEL) {
+ mc->mc_flags ^= C_DEL;
+ goto skip;
+ }
- if (err == MDBX_SUCCESS) {
- err = mdbx_removefile(env_pathname.lck);
- if (err == MDBX_SUCCESS)
- rc = MDBX_SUCCESS;
- else if (err == MDBX_ENOFILE)
- err = MDBX_SUCCESS;
+ int ki = mc->mc_ki[mc->mc_top];
+ mc->mc_ki[mc->mc_top] = (indx_t)++ki;
+ const int numkeys = page_numkeys(mp);
+ if (unlikely(ki >= numkeys)) {
+ mdbx_debug("%s", "=====> move to next sibling page");
+ mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1);
+ if (unlikely((rc = mdbx_cursor_sibling(mc, SIBLING_RIGHT)) !=
+ MDBX_SUCCESS)) {
+ mc->mc_flags |= C_EOF;
+ return rc;
}
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
+ mc->mc_ki[mc->mc_top]);
+ }
- if (err == MDBX_SUCCESS && !(dummy_env->me_flags & MDBX_NOSUBDIR)) {
- err = mdbx_removedirectory(pathname);
- if (err == MDBX_SUCCESS)
- rc = MDBX_SUCCESS;
- else if (err == MDBX_ENOFILE)
- err = MDBX_SUCCESS;
+skip:
+ mdbx_debug("==> cursor points to page %" PRIaPGNO
+ " with %u keys, key index %u",
+ mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]);
+
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
+ return MDBX_CORRUPTED;
+
+ if (IS_LEAF2(mp)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mp->mp_pgno);
+ return MDBX_CORRUPTED;
+ } else if (likely(key)) {
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
}
+ return MDBX_SUCCESS;
+ }
- if (dxb_handle != INVALID_HANDLE_VALUE)
- mdbx_closefile(dxb_handle);
- if (clk_handle != INVALID_HANDLE_VALUE)
- mdbx_closefile(clk_handle);
- } else if (err == MDBX_ENOFILE)
- err = MDBX_SUCCESS;
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ rc = mdbx_xcursor_init1(mc, node, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ } else if (likely(data)) {
+ if (unlikely((rc = mdbx_node_read(mc, node, data,
+ pp_txnid4chk(mp, mc->mc_txn))) !=
+ MDBX_SUCCESS))
+ return rc;
+ }
- mdbx_free(env_pathname.buffer_for_free);
- return (err == MDBX_SUCCESS) ? rc : err;
+ get_key_optional(node, key);
+ return MDBX_SUCCESS;
}
-__cold int mdbx_env_open(MDBX_env *env, const char *pathname,
- MDBX_env_flags_t flags, mdbx_mode_t mode) {
- int rc = check_env(env, false);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/* Move the cursor to the previous data item. */
+static int mdbx_cursor_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
+ MDBX_cursor_op op) {
+ MDBX_page *mp;
+ MDBX_node *node;
+ int rc;
- if (unlikely(flags & ~ENV_USABLE_FLAGS))
- return MDBX_EINVAL;
+ if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_PREV_DUP)
+ return MDBX_NOTFOUND;
- if (flags & MDBX_RDONLY)
- mode = 0;
+ if (unlikely(!(mc->mc_flags & C_INITIALIZED))) {
+ rc = mdbx_cursor_last(mc, key, data);
+ if (unlikely(rc))
+ return rc;
+ mc->mc_ki[mc->mc_top]++;
+ }
- if (env->me_lazy_fd != INVALID_HANDLE_VALUE ||
- (env->me_flags & MDBX_ENV_ACTIVE) != 0 || env->me_map)
- return MDBX_EPERM;
+ mp = mc->mc_pg[mc->mc_top];
+ if ((mc->mc_db->md_flags & MDBX_DUPSORT) &&
+ mc->mc_ki[mc->mc_top] < page_numkeys(mp)) {
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ if (op == MDBX_PREV || op == MDBX_PREV_DUP) {
+ rc =
+ mdbx_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_PREV);
+ if (op != MDBX_PREV || rc != MDBX_NOTFOUND) {
+ if (likely(rc == MDBX_SUCCESS)) {
+ get_key_optional(node, key);
+ mc->mc_flags &= ~C_EOF;
+ }
+ return rc;
+ }
+ }
+ } else {
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+ if (op == MDBX_PREV_DUP)
+ return MDBX_NOTFOUND;
+ }
+ }
- /* pickup previously mdbx_env_set_flags(),
- * but avoid MDBX_UTTERLY_NOSYNC by disjunction */
- const uint32_t saved_me_flags = env->me_flags;
- flags = merge_sync_flags(flags, env->me_flags);
+ mdbx_debug("cursor_prev: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno,
+ (void *)mc);
- MDBX_handle_env_pathname env_pathname;
- rc = mdbx_handle_env_pathname(&env_pathname, pathname, &flags, mode);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ mc->mc_flags &= ~(C_EOF | C_DEL);
- if (flags & MDBX_RDONLY) {
- /* LY: silently ignore irrelevant flags when
- * we're only getting read access */
- flags &= ~(MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC |
- MDBX_NOMETASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM |
- MDBX_NOMEMINIT | MDBX_ACCEDE);
- } else {
-#if MDBX_MMAP_INCOHERENT_FILE_WRITE
- /* Temporary `workaround` for OpenBSD kernel's flaw.
- * See todo4recovery://erased_by_github/libmdbx/issues/67 */
- if ((flags & MDBX_WRITEMAP) == 0) {
- if (flags & MDBX_ACCEDE)
- flags |= MDBX_WRITEMAP;
- else {
- mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__,
- "System (i.e. OpenBSD) requires MDBX_WRITEMAP because "
- "of an internal flaw(s) in a file/buffer/page cache.\n");
- rc = 42 /* ENOPROTOOPT */;
- goto bailout;
- }
+ int ki = mc->mc_ki[mc->mc_top];
+ mc->mc_ki[mc->mc_top] = (indx_t)--ki;
+ if (unlikely(ki < 0)) {
+ mc->mc_ki[mc->mc_top] = 0;
+ mdbx_debug("%s", "=====> move to prev sibling page");
+ if ((rc = mdbx_cursor_sibling(mc, SIBLING_LEFT)) != MDBX_SUCCESS)
+ return rc;
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_debug("prev page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
+ mc->mc_ki[mc->mc_top]);
+ }
+ mdbx_debug("==> cursor points to page %" PRIaPGNO
+ " with %u keys, key index %u",
+ mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]);
+
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
+ return MDBX_CORRUPTED;
+
+ if (IS_LEAF2(mp)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mp->mp_pgno);
+ return MDBX_CORRUPTED;
+ } else if (likely(key)) {
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
}
-#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+ return MDBX_SUCCESS;
}
- env->me_flags = (flags & ~MDBX_FATAL_ERROR) | MDBX_ENV_ACTIVE;
- env->me_pathname = mdbx_calloc(env_pathname.ent_len + 1, 1);
- env->me_dbxs = mdbx_calloc(env->me_maxdbs, sizeof(MDBX_dbx));
- env->me_dbflags = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbflags[0]));
- env->me_dbiseqs = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbiseqs[0]));
- if (!(env->me_dbxs && env->me_pathname && env->me_dbflags &&
- env->me_dbiseqs)) {
- rc = MDBX_ENOMEM;
- goto bailout;
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ rc = mdbx_xcursor_init1(mc, node, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ } else if (likely(data)) {
+ if (unlikely((rc = mdbx_node_read(mc, node, data,
+ pp_txnid4chk(mp, mc->mc_txn))) !=
+ MDBX_SUCCESS))
+ return rc;
}
- memcpy(env->me_pathname, env_pathname.dxb, env_pathname.ent_len);
- env->me_dbxs[FREE_DBI].md_cmp = cmp_int_align4; /* aligned MDBX_INTEGERKEY */
- env->me_dbxs[FREE_DBI].md_dcmp = cmp_lenfast;
- rc = mdbx_openfile(F_ISSET(flags, MDBX_RDONLY) ? MDBX_OPEN_DXB_READ
- : MDBX_OPEN_DXB_LAZY,
- env, env_pathname.dxb, &env->me_lazy_fd, mode);
- if (rc != MDBX_SUCCESS)
- goto bailout;
+ get_key_optional(node, key);
+ return MDBX_SUCCESS;
+}
- mdbx_assert(env, env->me_dsync_fd == INVALID_HANDLE_VALUE);
- if ((flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC)) == 0) {
- rc = mdbx_openfile(MDBX_OPEN_DXB_DSYNC, env, env_pathname.dxb,
- &env->me_dsync_fd, 0);
- mdbx_ensure(env, (rc != MDBX_SUCCESS) ==
- (env->me_dsync_fd == INVALID_HANDLE_VALUE));
- }
+/* Set the cursor on a specific data item. */
+static struct cursor_set_result mdbx_cursor_set(MDBX_cursor *mc, MDBX_val *key,
+ MDBX_val *data,
+ MDBX_cursor_op op) {
+ MDBX_page *mp;
+ MDBX_node *node = NULL;
+ DKBUF_DEBUG;
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- env->me_sysv_ipc.key = ftok(env_pathname.dxb, 42);
- if (env->me_sysv_ipc.key == -1) {
- rc = errno;
- goto bailout;
+ struct cursor_set_result ret;
+ ret.exact = false;
+ if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min ||
+ key->iov_len > mc->mc_dbx->md_klen_max)) {
+ mdbx_cassert(mc, !"Invalid key-size");
+ ret.err = MDBX_BAD_VALSIZE;
+ return ret;
}
-#endif /* MDBX_LOCKING */
-#if !(defined(_WIN32) || defined(_WIN64))
- if (mode == 0) {
- /* pickup mode for lck-file */
- struct stat st;
- if (fstat(env->me_lazy_fd, &st)) {
- rc = errno;
- goto bailout;
+ MDBX_val aligned_key = *key;
+ uint64_t aligned_keybytes;
+ if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
+ switch (aligned_key.iov_len) {
+ default:
+ mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY");
+ ret.err = MDBX_BAD_VALSIZE;
+ return ret;
+ case 4:
+ if (unlikely(3 & (uintptr_t)aligned_key.iov_base))
+ /* copy instead of return error to avoid break compatibility */
+ aligned_key.iov_base =
+ memcpy(&aligned_keybytes, aligned_key.iov_base, 4);
+ break;
+ case 8:
+ if (unlikely(7 & (uintptr_t)aligned_key.iov_base))
+ /* copy instead of return error to avoid break compatibility */
+ aligned_key.iov_base =
+ memcpy(&aligned_keybytes, aligned_key.iov_base, 8);
+ break;
}
- mode = st.st_mode;
- }
- mode = (/* inherit read permissions for group and others */ mode &
- (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
- /* always add read/write/search for owner */ S_IRUSR | S_IWUSR |
- ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) |
- ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0);
-#endif /* !Windows */
- const int lck_rc = mdbx_setup_lck(env, env_pathname.lck, mode);
- if (MDBX_IS_ERROR(lck_rc)) {
- rc = lck_rc;
- goto bailout;
}
- /* Set the position in files outside of the data to avoid corruption
- * due to erroneous use of file descriptors in the application code. */
- mdbx_fseek(env->me_lfd, UINT64_C(1) << 63);
- mdbx_fseek(env->me_lazy_fd, UINT64_C(1) << 63);
- if (env->me_dsync_fd != INVALID_HANDLE_VALUE)
- mdbx_fseek(env->me_dsync_fd, UINT64_C(1) << 63);
+ if (mc->mc_xcursor)
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
- const MDBX_env_flags_t rigorous_flags =
- MDBX_SAFE_NOSYNC | MDBX_DEPRECATED_MAPASYNC;
- const MDBX_env_flags_t mode_flags = rigorous_flags | MDBX_NOMETASYNC |
- MDBX_LIFORECLAIM | MDBX_COALESCE |
- MDBX_NORDAHEAD;
+ /* See if we're already on the right page */
+ if (mc->mc_flags & C_INITIALIZED) {
+ MDBX_val nodekey;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (lck && lck_rc != MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
- while (atomic_load32(&lck->mti_envmode, mo_AcquireRelease) == MDBX_RDONLY) {
- if (atomic_cas32(&lck->mti_envmode, MDBX_RDONLY,
- env->me_flags & mode_flags)) {
- /* The case:
- * - let's assume that for some reason the DB file is smaller
- * than it should be according to the geometry,
- * but not smaller than the last page used;
- * - the first process that opens the database (lc_rc = true)
- * does this in readonly mode and therefore cannot bring
- * the file size back to normal;
- * - some next process (lc_rc = false) opens the DB in read-write
- * mode and now is here.
- *
- * FIXME: Should we re-check and set the size of DB-file right here? */
- break;
+ mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
+ mp = mc->mc_pg[mc->mc_top];
+ if (unlikely(!page_numkeys(mp))) {
+ mc->mc_ki[mc->mc_top] = 0;
+ mc->mc_flags |= C_EOF;
+ ret.err = MDBX_NOTFOUND;
+ return ret;
+ }
+ if (IS_LEAF2(mp)) {
+ nodekey.iov_len = mc->mc_db->md_xsize;
+ nodekey.iov_base = page_leaf2key(mp, 0, nodekey.iov_len);
+ } else {
+ node = page_node(mp, 0);
+ get_key(node, &nodekey);
+ }
+ int cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
+ if (unlikely(cmp == 0)) {
+ /* Probably happens rarely, but first node on the page
+ * was the one we wanted. */
+ mc->mc_ki[mc->mc_top] = 0;
+ ret.exact = true;
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ goto got_node;
+ }
+ if (cmp > 0) {
+ const unsigned nkeys = page_numkeys(mp);
+ if (nkeys > 1) {
+ if (IS_LEAF2(mp)) {
+ nodekey.iov_base = page_leaf2key(mp, nkeys - 1, nodekey.iov_len);
+ } else {
+ node = page_node(mp, nkeys - 1);
+ get_key(node, &nodekey);
+ }
+ cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
+ if (cmp == 0) {
+ /* last node was the one we wanted */
+ mdbx_cassert(mc, nkeys >= 1 && nkeys <= UINT16_MAX + 1);
+ mc->mc_ki[mc->mc_top] = (indx_t)(nkeys - 1);
+ ret.exact = true;
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ goto got_node;
+ }
+ if (cmp < 0) {
+ if (mc->mc_ki[mc->mc_top] < page_numkeys(mp)) {
+ /* This is definitely the right page, skip search_page */
+ if (IS_LEAF2(mp)) {
+ nodekey.iov_base =
+ page_leaf2key(mp, mc->mc_ki[mc->mc_top], nodekey.iov_len);
+ } else {
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ get_key(node, &nodekey);
+ }
+ cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
+ if (cmp == 0) {
+ /* current node was the one we wanted */
+ ret.exact = true;
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ goto got_node;
+ }
+ }
+ mc->mc_flags &= ~C_EOF;
+ goto search_node;
+ }
+ }
+ /* If any parents have right-sibs, search.
+ * Otherwise, there's nothing further. */
+ unsigned i;
+ for (i = 0; i < mc->mc_top; i++)
+ if (mc->mc_ki[i] < page_numkeys(mc->mc_pg[i]) - 1)
+ break;
+ if (i == mc->mc_top) {
+ /* There are no other pages */
+ mdbx_cassert(mc, nkeys <= UINT16_MAX);
+ mc->mc_ki[mc->mc_top] = (uint16_t)nkeys;
+ mc->mc_flags |= C_EOF;
+ ret.err = MDBX_NOTFOUND;
+ return ret;
}
- atomic_yield();
}
+ if (!mc->mc_top) {
+ /* There are no other pages */
+ mc->mc_ki[mc->mc_top] = 0;
+ if (op == MDBX_SET_RANGE)
+ goto got_node;
- if (env->me_flags & MDBX_ACCEDE) {
- /* pickup current mode-flags, including MDBX_LIFORECLAIM |
- * MDBX_COALESCE | MDBX_NORDAHEAD */
- const unsigned diff =
- (lck->mti_envmode.weak ^ env->me_flags) & mode_flags;
- mdbx_notice("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->me_flags,
- env->me_flags ^ diff);
- env->me_flags ^= diff;
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ ret.err = MDBX_NOTFOUND;
+ return ret;
}
+ } else {
+ mc->mc_pg[0] = 0;
+ }
- if ((lck->mti_envmode.weak ^ env->me_flags) & rigorous_flags) {
- mdbx_error("%s", "current mode/flags incompatible with requested");
- rc = MDBX_INCOMPATIBLE;
- goto bailout;
+ ret.err = mdbx_page_search(mc, &aligned_key, 0);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
+
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_cassert(mc, IS_LEAF(mp));
+
+search_node:;
+ struct node_result nsr = mdbx_node_search(mc, &aligned_key);
+ node = nsr.node;
+ ret.exact = nsr.exact;
+ if (!ret.exact) {
+ if (op != MDBX_SET_RANGE) {
+ /* MDBX_SET specified and not an exact match. */
+ if (unlikely(mc->mc_ki[mc->mc_top] >=
+ page_numkeys(mc->mc_pg[mc->mc_top])))
+ mc->mc_flags |= C_EOF;
+ ret.err = MDBX_NOTFOUND;
+ return ret;
+ }
+
+ if (node == NULL) {
+ mdbx_debug("%s", "===> inexact leaf not found, goto sibling");
+ ret.err = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
+ if (unlikely(ret.err != MDBX_SUCCESS)) {
+ mc->mc_flags |= C_EOF;
+ return ret; /* no entries matched */
+ }
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_cassert(mc, IS_LEAF(mp));
+ if (!IS_LEAF2(mp))
+ node = page_node(mp, 0);
}
}
+ mdbx_cassert(mc,
+ mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
- const int dxb_rc = mdbx_setup_dxb(env, lck_rc, mode);
- if (MDBX_IS_ERROR(dxb_rc)) {
- rc = dxb_rc;
- goto bailout;
- }
+got_node:
+ mc->mc_flags |= C_INITIALIZED;
+ mc->mc_flags &= ~C_EOF;
- if (unlikely(/* recovery mode */ env->me_stuck_meta >= 0) &&
- (lck_rc != /* exclusive */ MDBX_RESULT_TRUE ||
- (flags & MDBX_EXCLUSIVE) == 0)) {
- mdbx_error("%s", "recovery requires exclusive mode");
- rc = MDBX_BUSY;
- goto bailout;
+ if (IS_LEAF2(mp)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mp->mp_pgno);
+ ret.err = MDBX_CORRUPTED;
+ } else {
+ if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) {
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
+ }
+ ret.err = MDBX_SUCCESS;
+ }
+ return ret;
}
- mdbx_debug("opened dbenv %p", (void *)env);
- if (lck) {
- if (lck_rc == MDBX_RESULT_TRUE) {
- lck->mti_envmode.weak = env->me_flags & (mode_flags | MDBX_RDONLY);
- rc = mdbx_lck_downgrade(env);
- mdbx_debug("lck-downgrade-%s: rc %i",
- (env->me_flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc);
- if (rc != MDBX_SUCCESS)
- goto bailout;
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ ret.err = mdbx_xcursor_init1(mc, node, mp);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
+ if (op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_SET_RANGE) {
+ ret.err = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
} else {
- rc = mdbx_cleanup_dead_readers(env, false, NULL);
- if (MDBX_IS_ERROR(rc))
- goto bailout;
+ ret = mdbx_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL,
+ MDBX_SET_RANGE);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
+ if (op == MDBX_GET_BOTH && !ret.exact) {
+ ret.err = MDBX_NOTFOUND;
+ return ret;
+ }
}
-
- if ((env->me_flags & MDBX_NOTLS) == 0) {
- rc = mdbx_rthc_alloc(&env->me_txkey, &lck->mti_readers[0],
- &lck->mti_readers[env->me_maxreaders]);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- env->me_flags |= MDBX_ENV_TXKEY;
+ } else if (likely(data)) {
+ if (op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE) {
+ if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min ||
+ data->iov_len > mc->mc_dbx->md_vlen_max)) {
+ mdbx_cassert(mc, !"Invalid data-size");
+ ret.err = MDBX_BAD_VALSIZE;
+ return ret;
+ }
+ MDBX_val aligned_data = *data;
+ uint64_t aligned_databytes;
+ if (mc->mc_db->md_flags & MDBX_INTEGERDUP) {
+ switch (aligned_data.iov_len) {
+ default:
+ mdbx_cassert(mc, !"data-size is invalid for MDBX_INTEGERDUP");
+ ret.err = MDBX_BAD_VALSIZE;
+ return ret;
+ case 4:
+ if (unlikely(3 & (uintptr_t)aligned_data.iov_base))
+ /* copy instead of return error to avoid break compatibility */
+ aligned_data.iov_base =
+ memcpy(&aligned_databytes, aligned_data.iov_base, 4);
+ break;
+ case 8:
+ if (unlikely(7 & (uintptr_t)aligned_data.iov_base))
+ /* copy instead of return error to avoid break compatibility */
+ aligned_data.iov_base =
+ memcpy(&aligned_databytes, aligned_data.iov_base, 8);
+ break;
+ }
+ }
+ MDBX_val actual_data;
+ ret.err = mdbx_node_read(mc, node, &actual_data,
+ pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
+ const int cmp = mc->mc_dbx->md_dcmp(&aligned_data, &actual_data);
+ if (cmp) {
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ if (op != MDBX_GET_BOTH_RANGE || cmp > 0) {
+ ret.err = MDBX_NOTFOUND;
+ return ret;
+ }
+ }
+ *data = actual_data;
+ } else {
+ ret.err = mdbx_node_read(mc, node, data,
+ pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
}
}
- if ((flags & MDBX_RDONLY) == 0) {
- const size_t tsize = sizeof(MDBX_txn),
- size = tsize + env->me_maxdbs *
- (sizeof(MDBX_db) + sizeof(MDBX_cursor *) +
- sizeof(unsigned) + 1);
- rc = alloc_page_buf(env);
- if (rc == MDBX_SUCCESS) {
- memset(env->me_pbuf, -1, env->me_psize * 2);
- MDBX_txn *txn = mdbx_calloc(1, size);
- if (txn) {
- txn->mt_dbs = (MDBX_db *)((char *)txn + tsize);
- txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs);
- txn->mt_dbiseqs = (unsigned *)(txn->mt_cursors + env->me_maxdbs);
- txn->mt_dbistate = (uint8_t *)(txn->mt_dbiseqs + env->me_maxdbs);
- txn->mt_env = env;
- txn->mt_dbxs = env->me_dbxs;
- txn->mt_flags = MDBX_TXN_FINISHED;
- env->me_txn0 = txn;
- txn->tw.retired_pages = mdbx_pnl_alloc(MDBX_PNL_INITIAL);
- txn->tw.reclaimed_pglist = mdbx_pnl_alloc(MDBX_PNL_INITIAL);
- if (unlikely(!txn->tw.retired_pages || !txn->tw.reclaimed_pglist))
- rc = MDBX_ENOMEM;
- } else
- rc = MDBX_ENOMEM;
- }
- }
+ /* The key already matches in all other cases */
+ if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY)
+ get_key_optional(node, key);
-#if MDBX_DEBUG
- if (rc == MDBX_SUCCESS) {
- const MDBX_meta *meta = constmeta_prefer_last(env);
- const MDBX_db *db = &meta->mm_dbs[MAIN_DBI];
+ mdbx_debug("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key),
+ DVAL_DEBUG(data));
+ ret.err = MDBX_SUCCESS;
+ return ret;
+}
- mdbx_debug("opened database version %u, pagesize %u",
- (uint8_t)unaligned_peek_u64(4, meta->mm_magic_and_version),
- env->me_psize);
- mdbx_debug("using meta page %" PRIaPGNO ", txn %" PRIaTXN,
- data_page(meta)->mp_pgno, meta_txnid(env, meta));
- mdbx_debug("depth: %u", db->md_depth);
- mdbx_debug("entries: %" PRIu64, db->md_entries);
- mdbx_debug("branch pages: %" PRIaPGNO, db->md_branch_pages);
- mdbx_debug("leaf pages: %" PRIaPGNO, db->md_leaf_pages);
- mdbx_debug("overflow pages: %" PRIaPGNO, db->md_overflow_pages);
- mdbx_debug("root: %" PRIaPGNO, db->md_root);
- mdbx_debug("schema_altered: %" PRIaTXN, db->md_mod_txnid);
- }
-#endif
+/* Move the cursor to the first item in the database. */
+static int mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
+ int rc;
-bailout:
- if (rc != MDBX_SUCCESS) {
- rc = mdbx_env_close0(env) ? MDBX_PANIC : rc;
- env->me_flags =
- saved_me_flags | ((rc != MDBX_PANIC) ? 0 : MDBX_FATAL_ERROR);
- } else {
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- mdbx_txn_valgrind(env, nullptr);
-#endif
+ if (mc->mc_xcursor)
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+
+ if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
+ rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
- mdbx_free(env_pathname.buffer_for_free);
- return rc;
-}
-/* Destroy resources from mdbx_env_open(), clear our readers & DBIs */
-__cold static int mdbx_env_close0(MDBX_env *env) {
- const unsigned flags = env->me_flags;
- if (!(flags & MDBX_ENV_ACTIVE)) {
- mdbx_ensure(env, env->me_lcklist_next == nullptr);
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top])))
+ return MDBX_CORRUPTED;
+
+ mc->mc_flags |= C_INITIALIZED;
+ mc->mc_flags &= ~C_EOF;
+ mc->mc_ki[mc->mc_top] = 0;
+
+ if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mc->mc_pg[mc->mc_top]->mp_pgno);
+ return MDBX_CORRUPTED;
+ } else if (likely(key)) {
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top], 0, key->iov_len);
+ }
return MDBX_SUCCESS;
}
- env->me_flags &= ~ENV_INTERNAL_FLAGS;
- env->me_lck = nullptr;
- if (flags & MDBX_ENV_TXKEY) {
- mdbx_rthc_remove(env->me_txkey);
- env->me_txkey = (mdbx_thread_key_t)0;
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], 0);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(rc))
+ return rc;
+ } else if (likely(data)) {
+ if (unlikely((rc = mdbx_node_read(
+ mc, node, data,
+ pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) !=
+ MDBX_SUCCESS))
+ return rc;
}
- lcklist_lock();
- const int rc = lcklist_detach_locked(env);
- lcklist_unlock();
+ get_key_optional(node, key);
+ return MDBX_SUCCESS;
+}
- if (env->me_map) {
- mdbx_munmap(&env->me_dxb_mmap);
-#ifdef MDBX_USE_VALGRIND
- VALGRIND_DISCARD(env->me_valgrind_handle);
- env->me_valgrind_handle = -1;
-#endif
- }
+/* Move the cursor to the last item in the database. */
+static int mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
+ int rc;
- if (env->me_dsync_fd != INVALID_HANDLE_VALUE) {
- (void)mdbx_closefile(env->me_dsync_fd);
- env->me_dsync_fd = INVALID_HANDLE_VALUE;
+ if (mc->mc_xcursor)
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+
+ if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
+ rc = mdbx_page_search(mc, NULL, MDBX_PS_LAST);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
- if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
- (void)mdbx_closefile(env->me_lazy_fd);
- env->me_lazy_fd = INVALID_HANDLE_VALUE;
- }
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top])))
+ return MDBX_CORRUPTED;
- if (env->me_lck_mmap.lck)
- mdbx_munmap(&env->me_lck_mmap);
+ mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]) - 1;
+ mc->mc_flags |= C_INITIALIZED | C_EOF;
- if (env->me_lfd != INVALID_HANDLE_VALUE) {
- (void)mdbx_closefile(env->me_lfd);
- env->me_lfd = INVALID_HANDLE_VALUE;
+ if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mc->mc_pg[mc->mc_top]->mp_pgno);
+ return MDBX_CORRUPTED;
+ } else if (likely(key)) {
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top],
+ mc->mc_ki[mc->mc_top], key->iov_len);
+ }
+ return MDBX_SUCCESS;
}
- if (env->me_dbxs) {
- for (unsigned i = env->me_numdbs; --i >= CORE_DBS;)
- mdbx_free(env->me_dbxs[i].md_name.iov_base);
- mdbx_free(env->me_dbxs);
- env->me_dbxs = nullptr;
- }
- if (env->me_pbuf) {
- mdbx_memalign_free(env->me_pbuf);
- env->me_pbuf = nullptr;
- }
- if (env->me_dbiseqs) {
- mdbx_free(env->me_dbiseqs);
- env->me_dbiseqs = nullptr;
- }
- if (env->me_dbflags) {
- mdbx_free(env->me_dbflags);
- env->me_dbflags = nullptr;
- }
- if (env->me_pathname) {
- mdbx_free(env->me_pathname);
- env->me_pathname = nullptr;
- }
- if (env->me_txn0) {
- mdbx_dpl_free(env->me_txn0);
- mdbx_txl_free(env->me_txn0->tw.lifo_reclaimed);
- mdbx_pnl_free(env->me_txn0->tw.retired_pages);
- mdbx_pnl_free(env->me_txn0->tw.spill_pages);
- mdbx_pnl_free(env->me_txn0->tw.reclaimed_pglist);
- mdbx_free(env->me_txn0);
- env->me_txn0 = nullptr;
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(rc))
+ return rc;
+ } else if (likely(data)) {
+ if (unlikely((rc = mdbx_node_read(
+ mc, node, data,
+ pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) !=
+ MDBX_SUCCESS))
+ return rc;
}
- env->me_stuck_meta = -1;
- return rc;
-}
-__cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
- MDBX_page *dp;
- int rc = MDBX_SUCCESS;
+ get_key_optional(node, key);
+ return MDBX_SUCCESS;
+}
- if (unlikely(!env))
+int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
+ MDBX_cursor_op op) {
+ if (unlikely(mc == NULL))
return MDBX_EINVAL;
- if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE))
- return MDBX_EBADSIGN;
-
-#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64))
- /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows
- * platforms (i.e. where fork() is available).
- * This is required to legitimize a call after fork()
- * from a child process, that should be allowed to free resources. */
- if (unlikely(env->me_pid != mdbx_getpid()))
- env->me_flags |= MDBX_FATAL_ERROR;
-#endif /* MDBX_ENV_CHECKPID */
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- if (env->me_map && (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0 &&
- env->me_txn0) {
- if (env->me_txn0->mt_owner && env->me_txn0->mt_owner != mdbx_thread_self())
- return MDBX_BUSY;
- } else
- dont_sync = true;
+ int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- if (!atomic_cas32(&env->me_signature, MDBX_ME_SIGNATURE, 0))
- return MDBX_EBADSIGN;
+ int (*mfunc)(MDBX_cursor * mc, MDBX_val * key, MDBX_val * data);
+ switch (op) {
+ case MDBX_GET_CURRENT: {
+ if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
+ return MDBX_ENODATA;
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ const unsigned nkeys = page_numkeys(mp);
+ if (unlikely(mc->mc_ki[mc->mc_top] >= nkeys)) {
+ mdbx_cassert(mc, nkeys <= UINT16_MAX);
+ if (mc->mc_flags & C_EOF)
+ return MDBX_ENODATA;
+ mc->mc_ki[mc->mc_top] = (uint16_t)nkeys;
+ mc->mc_flags |= C_EOF;
+ return MDBX_NOTFOUND;
+ }
+ mdbx_cassert(mc, nkeys > 0);
- if (!dont_sync) {
-#if defined(_WIN32) || defined(_WIN64)
- /* On windows, without blocking is impossible to determine whether another
- * process is running a writing transaction or not.
- * Because in the "owner died" condition kernel don't release
- * file lock immediately. */
- rc = mdbx_env_sync_internal(env, true, false);
- rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
-#else
- struct stat st;
- if (unlikely(fstat(env->me_lazy_fd, &st)))
- rc = errno;
- else if (st.st_nlink > 0 /* don't sync deleted files */) {
- rc = mdbx_env_sync_internal(env, true, true);
- rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY ||
- rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE)
- ? MDBX_SUCCESS
- : rc;
+ rc = MDBX_SUCCESS;
+ if (IS_LEAF2(mp)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mp->mp_pgno);
+ return MDBX_CORRUPTED;
+ }
+ key->iov_len = mc->mc_db->md_xsize;
+ key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
+ } else {
+ MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ get_key_optional(node, key);
+ if (data) {
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) {
+ rc = mdbx_xcursor_init1(mc, node, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+ if (unlikely(rc))
+ return rc;
+ } else {
+ rc = mdbx_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL,
+ MDBX_GET_CURRENT);
+ if (unlikely(rc))
+ return rc;
+ }
+ } else {
+ rc = mdbx_node_read(mc, node, data, pp_txnid4chk(mp, mc->mc_txn));
+ if (unlikely(rc))
+ return rc;
+ }
+ }
+ }
+ break;
+ }
+ case MDBX_GET_BOTH:
+ case MDBX_GET_BOTH_RANGE:
+ if (unlikely(data == NULL))
+ return MDBX_EINVAL;
+ if (unlikely(mc->mc_xcursor == NULL))
+ return MDBX_INCOMPATIBLE;
+ /* fall through */
+ __fallthrough;
+ case MDBX_SET:
+ case MDBX_SET_KEY:
+ case MDBX_SET_RANGE:
+ if (unlikely(key == NULL))
+ return MDBX_EINVAL;
+ rc = mdbx_cursor_set(mc, key, data, op).err;
+ if (mc->mc_flags & C_INITIALIZED) {
+ mdbx_cassert(mc, mc->mc_snum > 0 && mc->mc_top < mc->mc_snum);
+ mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
+ page_numkeys(mc->mc_pg[mc->mc_top]) ||
+ (mc->mc_flags & C_EOF));
+ }
+ break;
+ case MDBX_GET_MULTIPLE:
+ if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED)))
+ return MDBX_EINVAL;
+ if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED)))
+ return MDBX_INCOMPATIBLE;
+ rc = MDBX_SUCCESS;
+ if ((mc->mc_xcursor->mx_cursor.mc_flags & (C_INITIALIZED | C_EOF)) !=
+ C_INITIALIZED)
+ break;
+ goto fetchm;
+ case MDBX_NEXT_MULTIPLE:
+ if (unlikely(data == NULL))
+ return MDBX_EINVAL;
+ if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED)))
+ return MDBX_INCOMPATIBLE;
+ rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_DUP);
+ if (rc == MDBX_SUCCESS) {
+ if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+ MDBX_cursor *mx;
+ fetchm:
+ mx = &mc->mc_xcursor->mx_cursor;
+ data->iov_len =
+ page_numkeys(mx->mc_pg[mx->mc_top]) * mx->mc_db->md_xsize;
+ data->iov_base = page_data(mx->mc_pg[mx->mc_top]);
+ mx->mc_ki[mx->mc_top] = (indx_t)page_numkeys(mx->mc_pg[mx->mc_top]) - 1;
+ } else {
+ rc = MDBX_NOTFOUND;
+ }
+ }
+ break;
+ case MDBX_PREV_MULTIPLE:
+ if (data == NULL)
+ return MDBX_EINVAL;
+ if (!(mc->mc_db->md_flags & MDBX_DUPFIXED))
+ return MDBX_INCOMPATIBLE;
+ rc = MDBX_SUCCESS;
+ if (!(mc->mc_flags & C_INITIALIZED))
+ rc = mdbx_cursor_last(mc, key, data);
+ if (rc == MDBX_SUCCESS) {
+ MDBX_cursor *mx = &mc->mc_xcursor->mx_cursor;
+ if (mx->mc_flags & C_INITIALIZED) {
+ rc = mdbx_cursor_sibling(mx, SIBLING_LEFT);
+ if (rc == MDBX_SUCCESS)
+ goto fetchm;
+ } else {
+ rc = MDBX_NOTFOUND;
+ }
+ }
+ break;
+ case MDBX_NEXT:
+ case MDBX_NEXT_DUP:
+ case MDBX_NEXT_NODUP:
+ rc = mdbx_cursor_next(mc, key, data, op);
+ break;
+ case MDBX_PREV:
+ case MDBX_PREV_DUP:
+ case MDBX_PREV_NODUP:
+ rc = mdbx_cursor_prev(mc, key, data, op);
+ break;
+ case MDBX_FIRST:
+ rc = mdbx_cursor_first(mc, key, data);
+ break;
+ case MDBX_FIRST_DUP:
+ mfunc = mdbx_cursor_first;
+ move:
+ if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED)))
+ return MDBX_EINVAL;
+ if (unlikely(mc->mc_xcursor == NULL))
+ return MDBX_INCOMPATIBLE;
+ if (mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])) {
+ mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]);
+ mc->mc_flags |= C_EOF;
+ return MDBX_NOTFOUND;
}
-#endif
+ {
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ if (!F_ISSET(node_flags(node), F_DUPDATA)) {
+ get_key_optional(node, key);
+ rc = mdbx_node_read(mc, node, data,
+ pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
+ break;
+ }
+ }
+ if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)))
+ return MDBX_EINVAL;
+ rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL);
+ break;
+ case MDBX_LAST:
+ rc = mdbx_cursor_last(mc, key, data);
+ break;
+ case MDBX_LAST_DUP:
+ mfunc = mdbx_cursor_last;
+ goto move;
+ case MDBX_SET_UPPERBOUND: /* mostly same as MDBX_SET_LOWERBOUND */
+ case MDBX_SET_LOWERBOUND: {
+ if (unlikely(key == NULL || data == NULL))
+ return MDBX_EINVAL;
+ MDBX_val save_data = *data;
+ struct cursor_set_result csr =
+ mdbx_cursor_set(mc, key, data, MDBX_SET_RANGE);
+ rc = csr.err;
+ if (rc == MDBX_SUCCESS && csr.exact && mc->mc_xcursor) {
+ mc->mc_flags &= ~C_DEL;
+ csr.exact = false;
+ if (!save_data.iov_base && (mc->mc_db->md_flags & MDBX_DUPFIXED)) {
+ /* Avoiding search nested dupfixed hive if no data provided.
+ * This is changes the semantic of MDBX_SET_LOWERBOUND but avoid
+ * returning MDBX_BAD_VALSIZE. */
+ } else if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+ *data = save_data;
+ csr = mdbx_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL,
+ MDBX_SET_RANGE);
+ rc = csr.err;
+ if (rc == MDBX_NOTFOUND) {
+ mdbx_cassert(mc, !csr.exact);
+ rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_NODUP);
+ }
+ } else {
+ int cmp = mc->mc_dbx->md_dcmp(&save_data, data);
+ csr.exact = (cmp == 0);
+ if (cmp > 0)
+ rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_NODUP);
+ }
+ }
+ if (rc == MDBX_SUCCESS && !csr.exact)
+ rc = MDBX_RESULT_TRUE;
+ if (unlikely(op == MDBX_SET_UPPERBOUND)) {
+ /* minor fixups for MDBX_SET_UPPERBOUND */
+ if (rc == MDBX_RESULT_TRUE)
+ /* already at great-than by MDBX_SET_LOWERBOUND */
+ rc = MDBX_SUCCESS;
+ else if (rc == MDBX_SUCCESS)
+ /* exactly match, going next */
+ rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT);
+ }
+ break;
}
-
- mdbx_assert(env, env->me_signature.weak == 0);
- rc = mdbx_env_close0(env) ? MDBX_PANIC : rc;
- mdbx_ensure(env, mdbx_fastmutex_destroy(&env->me_dbi_lock) == MDBX_SUCCESS);
-#if defined(_WIN32) || defined(_WIN64)
- /* me_remap_guard don't have destructor (Slim Reader/Writer Lock) */
- DeleteCriticalSection(&env->me_windowsbug_lock);
-#else
- mdbx_ensure(env,
- mdbx_fastmutex_destroy(&env->me_remap_guard) == MDBX_SUCCESS);
-#endif /* Windows */
-
-#if MDBX_LOCKING > MDBX_LOCKING_SYSV
- MDBX_lockinfo *const stub = lckless_stub(env);
- mdbx_ensure(env, mdbx_ipclock_destroy(&stub->mti_wlock) == 0);
-#endif /* MDBX_LOCKING */
-
- while ((dp = env->me_dp_reserve) != NULL) {
- MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize);
- VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
- env->me_dp_reserve = dp->mp_next;
- mdbx_free(dp);
+ default:
+ mdbx_debug("unhandled/unimplemented cursor operation %u", op);
+ return MDBX_EINVAL;
}
- VALGRIND_DESTROY_MEMPOOL(env);
- mdbx_ensure(env, env->me_lcklist_next == nullptr);
- env->me_pid = 0;
- mdbx_free(env);
+ mc->mc_flags &= ~C_DEL;
return rc;
}
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_close(MDBX_env *env) {
- return __inline_mdbx_env_close(env);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
-
-/* Compare two items pointing at aligned unsigned int's. */
-static int __hot cmp_int_align4(const MDBX_val *a, const MDBX_val *b) {
- mdbx_assert(NULL, a->iov_len == b->iov_len);
- switch (a->iov_len) {
- case 4:
- return CMP2INT(unaligned_peek_u32(4, a->iov_base),
- unaligned_peek_u32(4, b->iov_base));
- case 8:
- return CMP2INT(unaligned_peek_u64(4, a->iov_base),
- unaligned_peek_u64(4, b->iov_base));
- default:
- mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
- __LINE__);
- return 0;
- }
-}
-
-/* Compare two items pointing at 2-byte aligned unsigned int's. */
-static int __hot cmp_int_align2(const MDBX_val *a, const MDBX_val *b) {
- mdbx_assert(NULL, a->iov_len == b->iov_len);
- switch (a->iov_len) {
- case 4:
- return CMP2INT(unaligned_peek_u32(2, a->iov_base),
- unaligned_peek_u32(2, b->iov_base));
- case 8:
- return CMP2INT(unaligned_peek_u64(2, a->iov_base),
- unaligned_peek_u64(2, b->iov_base));
- default:
- mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
- __LINE__);
- return 0;
+static int cursor_first_batch(MDBX_cursor *mc) {
+ if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
+ int err = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
}
-}
+ mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
-/* Compare two items pointing at unsigned values with unknown alignment.
- *
- * This is also set as MDBX_INTEGERDUP|MDBX_DUPFIXED's MDBX_dbx.md_dcmp. */
-static int __hot cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) {
- mdbx_assert(NULL, a->iov_len == b->iov_len);
- switch (a->iov_len) {
- case 4:
- return CMP2INT(unaligned_peek_u32(1, a->iov_base),
- unaligned_peek_u32(1, b->iov_base));
- case 8:
- return CMP2INT(unaligned_peek_u64(1, a->iov_base),
- unaligned_peek_u64(1, b->iov_base));
- default:
- mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__,
- __LINE__);
- return 0;
- }
+ mc->mc_flags |= C_INITIALIZED;
+ mc->mc_flags &= ~C_EOF;
+ mc->mc_ki[mc->mc_top] = 0;
+ return MDBX_SUCCESS;
}
-/* Compare two items lexically */
-static int __hot cmp_lexical(const MDBX_val *a, const MDBX_val *b) {
- if (a->iov_len == b->iov_len)
- return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0;
-
- const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1;
- const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
- int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0;
- return likely(diff_data) ? diff_data : diff_len;
-}
+static int cursor_next_batch(MDBX_cursor *mc) {
+ if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
+ return cursor_first_batch(mc);
-/* Compare two items in reverse byte order */
-static int __hot cmp_reverse(const MDBX_val *a, const MDBX_val *b) {
- const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
- if (likely(shortest)) {
- const uint8_t *pa = (const uint8_t *)a->iov_base + a->iov_len;
- const uint8_t *pb = (const uint8_t *)b->iov_base + b->iov_len;
- const uint8_t *const end = pa - shortest;
- do {
- int diff = *--pa - *--pb;
- if (likely(diff))
- return diff;
- } while (pa != end);
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ if (unlikely(mc->mc_flags & C_EOF)) {
+ if ((unsigned)mc->mc_ki[mc->mc_top] + 1 >= page_numkeys(mp))
+ return MDBX_NOTFOUND;
+ mc->mc_flags ^= C_EOF;
}
- return CMP2INT(a->iov_len, b->iov_len);
-}
-
-/* Fast non-lexically comparator */
-static int __hot cmp_lenfast(const MDBX_val *a, const MDBX_val *b) {
- int diff = CMP2INT(a->iov_len, b->iov_len);
- return likely(diff || a->iov_len == 0)
- ? diff
- : memcmp(a->iov_base, b->iov_base, a->iov_len);
-}
-static bool unsure_equal(MDBX_cmp_func cmp, const MDBX_val *a,
- const MDBX_val *b) {
- /* checking for the use of a known good comparator
- * or/otherwise for a full byte-to-byte match */
- return cmp == cmp_lenfast || cmp == cmp_lexical || cmp == cmp_reverse ||
- cmp == cmp_int_unaligned || cmp_lenfast(a, b) == 0;
+ int ki = mc->mc_ki[mc->mc_top];
+ mc->mc_ki[mc->mc_top] = (indx_t)++ki;
+ const int numkeys = page_numkeys(mp);
+ if (likely(ki >= numkeys)) {
+ mdbx_debug("%s", "=====> move to next sibling page");
+ mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1);
+ int err = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ mc->mc_flags |= C_EOF;
+ return err;
+ }
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
+ mc->mc_ki[mc->mc_top]);
+ }
+ return MDBX_SUCCESS;
}
-/* Search for key within a page, using binary search.
- * Returns the smallest entry larger or equal to the key.
- * Updates the cursor index with the index of the found entry.
- * If no entry larger or equal to the key is found, returns NULL. */
-static struct node_result __hot mdbx_node_search(MDBX_cursor *mc,
- const MDBX_val *key) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- const int nkeys = page_numkeys(mp);
- DKBUF_DEBUG;
+int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
+ size_t limit, MDBX_cursor_op op) {
+ if (unlikely(mc == NULL || count == NULL || limit < 4))
+ return MDBX_EINVAL;
- mdbx_debug("searching %u keys in %s %spage %" PRIaPGNO, nkeys,
- IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "",
- mp->mp_pgno);
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- struct node_result ret;
- ret.exact = false;
- STATIC_ASSERT(P_BRANCH == 1);
- int low = mp->mp_flags & P_BRANCH;
- int high = nkeys - 1;
- if (unlikely(high < low)) {
- mc->mc_ki[mc->mc_top] = 0;
- ret.node = NULL;
- return ret;
- }
+ int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- int cr = 0, i = 0;
- MDBX_cmp_func *cmp = mc->mc_dbx->md_cmp;
- MDBX_val nodekey;
- if (unlikely(IS_LEAF2(mp))) {
- mdbx_cassert(mc, mp->mp_leaf2_ksize == mc->mc_db->md_xsize);
- nodekey.iov_len = mp->mp_leaf2_ksize;
- do {
- i = (low + high) >> 1;
- nodekey.iov_base = page_leaf2key(mp, i, nodekey.iov_len);
- mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >=
- (char *)nodekey.iov_base + nodekey.iov_len);
- cr = cmp(key, &nodekey);
- mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY_DEBUG(&nodekey),
- cr);
- if (unlikely(cr == 0)) {
- ret.exact = true;
- break;
- }
- low = (cr < 0) ? low : i + 1;
- high = (cr < 0) ? i - 1 : high;
- } while (likely(low <= high));
+ if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT))
+ return MDBX_INCOMPATIBLE /* must be a non-dupsort subDB */;
- /* Found entry is less than the key. */
- /* Skip to get the smallest entry larger than key. */
- i += cr > 0;
+ switch (op) {
+ case MDBX_FIRST:
+ rc = cursor_first_batch(mc);
+ break;
+ case MDBX_NEXT:
+ rc = cursor_next_batch(mc);
+ break;
+ case MDBX_GET_CURRENT:
+ rc = likely(mc->mc_flags & C_INITIALIZED) ? MDBX_SUCCESS : MDBX_ENODATA;
+ break;
+ default:
+ mdbx_debug("unhandled/unimplemented cursor operation %u", op);
+ rc = EINVAL;
+ break;
+ }
- /* store the key index */
- mc->mc_ki[mc->mc_top] = (indx_t)i;
- ret.node = (i < nkeys)
- ? /* fake for LEAF2 */ (MDBX_node *)(intptr_t)-1
- : /* There is no entry larger or equal to the key. */ NULL;
- return ret;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ *count = 0;
+ return rc;
}
- if (IS_BRANCH(mp) && cmp == cmp_int_align2)
- /* Branch pages have no data, so if using integer keys,
- * alignment is guaranteed. Use faster cmp_int_align4(). */
- cmp = cmp_int_align4;
+ const MDBX_page *const page = mc->mc_pg[mc->mc_top];
+ const unsigned nkeys = page_numkeys(page);
+ unsigned i = mc->mc_ki[mc->mc_top], n = 0;
+ if (unlikely(i >= nkeys)) {
+ mdbx_cassert(mc, op == MDBX_GET_CURRENT);
+ mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
+ *count = 0;
+ if (mc->mc_flags & C_EOF) {
+ mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
+ return MDBX_ENODATA;
+ }
+ if (mdbx_cursor_on_last(mc) != MDBX_RESULT_TRUE)
+ return MDBX_EINVAL /* again MDBX_GET_CURRENT after MDBX_GET_CURRENT */;
+ mc->mc_flags |= C_EOF;
+ return MDBX_NOTFOUND;
+ }
- MDBX_node *node;
+ const txnid_t pp_txnid = pp_txnid4chk(page, mc->mc_txn);
do {
- i = (low + high) >> 1;
-
- node = page_node(mp, i);
- nodekey.iov_len = node_ks(node);
- nodekey.iov_base = node_key(node);
- mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >=
- (char *)nodekey.iov_base + nodekey.iov_len);
-
- cr = cmp(key, &nodekey);
- if (IS_LEAF(mp))
- mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY_DEBUG(&nodekey),
- cr);
- else
- mdbx_debug("found branch index %u [%s -> %" PRIaPGNO "], rc = %i", i,
- DKEY_DEBUG(&nodekey), node_pgno(node), cr);
- if (unlikely(cr == 0)) {
- ret.exact = true;
+ if (unlikely(n + 2 > limit)) {
+ rc = MDBX_RESULT_TRUE;
break;
}
- low = (cr < 0) ? low : i + 1;
- high = (cr < 0) ? i - 1 : high;
- } while (likely(low <= high));
-
- /* Found entry is less than the key. */
- /* Skip to get the smallest entry larger than key. */
- i += cr > 0;
+ const MDBX_node *leaf = page_node(page, i);
+ get_key(leaf, &pairs[n]);
+ rc = mdbx_node_read(mc, leaf, &pairs[n + 1], pp_txnid);
+ if (unlikely(rc != MDBX_SUCCESS))
+ break;
+ n += 2;
+ } while (++i < nkeys);
- /* store the key index */
mc->mc_ki[mc->mc_top] = (indx_t)i;
- ret.node = (i < nkeys)
- ? page_node(mp, i)
- : /* There is no entry larger or equal to the key. */ NULL;
- return ret;
+ *count = n;
+ return rc;
}
-/* Pop a page off the top of the cursor's stack. */
-static __inline void mdbx_cursor_pop(MDBX_cursor *mc) {
- if (mc->mc_snum) {
- mdbx_debug("popped page %" PRIaPGNO " off db %d cursor %p",
- mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *)mc);
- if (--mc->mc_snum) {
- mc->mc_top--;
- } else {
- mc->mc_flags &= ~C_INITIALIZED;
- }
+static int mdbx_touch_dbi(MDBX_cursor *mc) {
+ mdbx_cassert(mc, (*mc->mc_dbistate & DBI_DIRTY) == 0);
+ *mc->mc_dbistate |= DBI_DIRTY;
+ mc->mc_txn->mt_flags |= MDBX_TXN_DIRTY;
+ if (mc->mc_dbi >= CORE_DBS) {
+ mdbx_cassert(mc, (mc->mc_flags & C_RECLAIMING) == 0);
+ /* Touch DB record of named DB */
+ MDBX_cursor_couple cx;
+ int rc = mdbx_cursor_init(&cx.outer, mc->mc_txn, MAIN_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ mc->mc_txn->mt_dbistate[MAIN_DBI] |= DBI_DIRTY;
+ rc = mdbx_page_search(&cx.outer, &mc->mc_dbx->md_name, MDBX_PS_MODIFY);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
+ return MDBX_SUCCESS;
}
-/* Push a page onto the top of the cursor's stack.
- * Set MDBX_TXN_ERROR on failure. */
-static __inline int mdbx_cursor_push(MDBX_cursor *mc, MDBX_page *mp) {
- mdbx_debug("pushing page %" PRIaPGNO " on db %d cursor %p", mp->mp_pgno,
- DDBI(mc), (void *)mc);
-
- if (unlikely(mc->mc_snum >= CURSOR_STACK)) {
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return MDBX_CURSOR_FULL;
+/* Touch all the pages in the cursor stack. Set mc_top.
+ * Makes sure all the pages are writable, before attempting a write operation.
+ * [in] mc The cursor to operate on. */
+static int mdbx_cursor_touch(MDBX_cursor *mc) {
+ int rc = MDBX_SUCCESS;
+ if (unlikely((*mc->mc_dbistate & DBI_DIRTY) == 0)) {
+ rc = mdbx_touch_dbi(mc);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+ if (likely(mc->mc_snum)) {
+ mc->mc_top = 0;
+ do {
+ rc = mdbx_page_touch(mc);
+ } while (!rc && ++(mc->mc_top) < mc->mc_snum);
+ mc->mc_top = mc->mc_snum - 1;
}
+ return rc;
+}
- mdbx_cassert(mc, mc->mc_snum < UINT16_MAX);
- mc->mc_top = mc->mc_snum++;
- mc->mc_pg[mc->mc_top] = mp;
- mc->mc_ki[mc->mc_top] = 0;
+int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
+ unsigned flags) {
+ MDBX_env *env;
+ MDBX_page *sub_root = NULL;
+ MDBX_val xdata, *rdata, dkey, olddata;
+ MDBX_db nested_dupdb;
+ int err;
+ DKBUF_DEBUG;
- return MDBX_SUCCESS;
-}
+ if (unlikely(mc == NULL || key == NULL || data == NULL))
+ return MDBX_EINVAL;
-__hot static struct page_result
-mdbx_page_get_ex(MDBX_cursor *const mc, const pgno_t pgno,
- /* TODO: use parent-page ptr */ txnid_t front) {
- struct page_result ret;
- MDBX_txn *const txn = mc->mc_txn;
- mdbx_tassert(txn, front <= txn->mt_front);
- if (unlikely(pgno >= txn->mt_next_pgno)) {
- mdbx_error("page #%" PRIaPGNO " beyond next-pgno", pgno);
- notfound:
- ret.page = nullptr;
- ret.err = MDBX_PAGE_NOTFOUND;
- bailout:
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return ret;
- }
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- MDBX_env *const env = txn->mt_env;
- mdbx_assert(env, ((txn->mt_flags ^ env->me_flags) & MDBX_WRITEMAP) == 0);
- if (unlikely((txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0)) {
- const MDBX_txn *spiller = txn;
- do {
- /* Spilled pages were dirtied in this txn and flushed
- * because the dirty list got full. Bring this page
- * back in from the map (but don't unspill it here,
- * leave that unless page_touch happens again). */
- if (unlikely(spiller->mt_flags & MDBX_TXN_SPILLS) &&
- mdbx_search_spilled(spiller, pgno)) {
- goto spilled;
- }
+ int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- const unsigned i = mdbx_dpl_search(spiller, pgno);
- assert((int)i > 0);
- if (spiller->tw.dirtylist->items[i].pgno == pgno) {
- ret.page = spiller->tw.dirtylist->items[i].ptr;
- spiller->tw.dirtylist->items[i].lru = txn->tw.dirtylru++;
- goto dirty;
- }
+ if (unlikely(TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)))
+ return MDBX_BAD_DBI;
- spiller = spiller->mt_parent;
- } while (spiller != NULL);
+ mdbx_cassert(mc, cursor_is_tracked(mc));
+ env = mc->mc_txn->mt_env;
+
+ /* Check this first so counter will always be zero on any early failures. */
+ size_t mcount = 0, dcount = 0;
+ if (unlikely(flags & MDBX_MULTIPLE)) {
+ if (unlikely(flags & MDBX_RESERVE))
+ return MDBX_EINVAL;
+ if (unlikely(!F_ISSET(mc->mc_db->md_flags, MDBX_DUPFIXED)))
+ return MDBX_INCOMPATIBLE;
+ dcount = data[1].iov_len;
+ if (unlikely(dcount < 2 || data->iov_len == 0))
+ return MDBX_BAD_VALSIZE;
+ if (unlikely(mc->mc_db->md_xsize != data->iov_len) && mc->mc_db->md_xsize)
+ return MDBX_BAD_VALSIZE;
+ if (unlikely(dcount > MAX_MAPSIZE / 2 /
+ (BRANCH_NODE_MAX(MAX_PAGESIZE) - NODESIZE))) {
+ /* checking for multiplication overflow */
+ if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len))
+ return MDBX_TOO_LARGE;
+ }
+ data[1].iov_len = 0 /* reset done item counter */;
}
-spilled:
- ret.page = pgno2page(env, pgno);
-
-dirty:
- if (unlikely(ret.page->mp_pgno != pgno)) {
- bad_page(ret.page,
- "mismatch actual pgno (%" PRIaPGNO ") != expected (%" PRIaPGNO
- ")\n",
- ret.page->mp_pgno, pgno);
- goto notfound;
+ if (flags & MDBX_RESERVE) {
+ if (unlikely(mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_REVERSEDUP |
+ MDBX_INTEGERDUP | MDBX_DUPFIXED)))
+ return MDBX_INCOMPATIBLE;
+ data->iov_base = nullptr;
}
-#if !MDBX_DISABLE_PAGECHECKS
- if (unlikely(ret.page->mp_flags & P_ILL_BITS)) {
- ret.err =
- bad_page(ret.page, "invalid page's flags (%u)\n", ret.page->mp_flags);
- goto bailout;
- }
+ const unsigned nospill = flags & MDBX_NOSPILL;
+ flags -= nospill;
- if (unlikely(ret.page->mp_txnid > front) &&
- unlikely(ret.page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) {
- ret.err = bad_page(
- ret.page,
- "invalid page txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
- ret.page->mp_txnid,
- (front == txn->mt_front && front != txn->mt_txnid) ? "front-txn"
- : "parent-page",
- front);
- goto bailout;
- }
+ if (unlikely(mc->mc_txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
+ return (mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS
+ : MDBX_BAD_TXN;
- if (unlikely((ret.page->mp_upper < ret.page->mp_lower ||
- ((ret.page->mp_lower | ret.page->mp_upper) & 1) ||
- PAGEHDRSZ + ret.page->mp_upper > env->me_psize) &&
- !IS_OVERFLOW(ret.page))) {
- ret.err =
- bad_page(ret.page, "invalid page lower(%u)/upper(%u) with limit (%u)\n",
- ret.page->mp_lower, ret.page->mp_upper, page_space(env));
- goto bailout;
+ uint64_t aligned_keybytes, aligned_databytes;
+ MDBX_val aligned_key, aligned_data;
+ if (likely((mc->mc_flags & C_SUB) == 0)) {
+ if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min ||
+ key->iov_len > mc->mc_dbx->md_klen_max)) {
+ mdbx_cassert(mc, !"Invalid key-size");
+ return MDBX_BAD_VALSIZE;
+ }
+ if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min ||
+ data->iov_len > mc->mc_dbx->md_vlen_max)) {
+ mdbx_cassert(mc, !"Invalid data-size");
+ return MDBX_BAD_VALSIZE;
+ }
+
+ if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
+ switch (key->iov_len) {
+ default:
+ mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY");
+ return MDBX_BAD_VALSIZE;
+ case 4:
+ if (unlikely(3 & (uintptr_t)key->iov_base)) {
+ /* copy instead of return error to avoid break compatibility */
+ aligned_key.iov_base =
+ memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 4);
+ key = &aligned_key;
+ }
+ break;
+ case 8:
+ if (unlikely(7 & (uintptr_t)key->iov_base)) {
+ /* copy instead of return error to avoid break compatibility */
+ aligned_key.iov_base =
+ memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 8);
+ key = &aligned_key;
+ }
+ break;
+ }
+ }
+ if (mc->mc_db->md_flags & MDBX_INTEGERDUP) {
+ switch (data->iov_len) {
+ default:
+ mdbx_cassert(mc, !"data-size is invalid for MDBX_INTEGERKEY");
+ return MDBX_BAD_VALSIZE;
+ case 4:
+ if (unlikely(3 & (uintptr_t)data->iov_base)) {
+ if (unlikely(flags & MDBX_MULTIPLE))
+ return MDBX_BAD_VALSIZE;
+ /* copy instead of return error to avoid break compatibility */
+ aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base,
+ aligned_data.iov_len = 4);
+ data = &aligned_data;
+ }
+ break;
+ case 8:
+ if (unlikely(7 & (uintptr_t)data->iov_base)) {
+ if (unlikely(flags & MDBX_MULTIPLE))
+ return MDBX_BAD_VALSIZE;
+ /* copy instead of return error to avoid break compatibility */
+ aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base,
+ aligned_data.iov_len = 8);
+ data = &aligned_data;
+ }
+ break;
+ }
+ }
}
-#endif /* !MDBX_DISABLE_PAGECHECKS */
- ret.err = MDBX_SUCCESS;
- if (mdbx_audit_enabled())
- ret.err = mdbx_page_check(mc, ret.page, C_UPDATING);
- return ret;
-}
+ mdbx_debug(
+ "==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR,
+ DDBI(mc), DKEY_DEBUG(key), key->iov_len,
+ DVAL_DEBUG((flags & MDBX_RESERVE) ? nullptr : data), data->iov_len);
-/* Finish mdbx_page_search() / mdbx_page_search_lowest().
- * The cursor is at the root page, set up the rest of it. */
-__hot static int mdbx_page_search_root(MDBX_cursor *mc, const MDBX_val *key,
- int flags) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- int rc;
- DKBUF_DEBUG;
+ int dupdata_flag = 0;
+ if ((flags & MDBX_CURRENT) != 0 && (mc->mc_flags & C_SUB) == 0) {
+ if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE)))
+ return MDBX_EINVAL;
+ /* ÐžÐ¿Ñ†Ð¸Ñ MDBX_CURRENT означает, что запрошено обновление текущей запиÑи,
+ * на которой ÑÐµÐ¹Ñ‡Ð°Ñ Ñтоит курÑор. ПроверÑем что переданный ключ Ñовпадает
+ * Ñо значением в текущей позиции курÑора.
+ * ЗдеÑÑŒ проще вызвать mdbx_cursor_get(), так как Ð´Ð»Ñ Ð¾Ð±ÑÐ»ÑƒÐ¶Ð¸Ð²Ð°Ð½Ð¸Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ†
+ * Ñ MDBX_DUPSORT также требуетÑÑ Ñ‚ÐµÐºÑƒÑ‰Ð¸Ð¹ размер данных. */
+ MDBX_val current_key, current_data;
+ rc = mdbx_cursor_get(mc, ¤t_key, ¤t_data, MDBX_GET_CURRENT);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ if (mc->mc_dbx->md_cmp(key, ¤t_key) != 0)
+ return MDBX_EKEYMISMATCH;
- while (IS_BRANCH(mp)) {
- MDBX_node *node;
- int i;
+ if (unlikely((flags & MDBX_MULTIPLE)))
+ goto drop_current;
- mdbx_debug("branch page %" PRIaPGNO " has %u keys", mp->mp_pgno,
- page_numkeys(mp));
- /* Don't assert on branch pages in the GC. We can get here
- * while in the process of rebalancing a GC branch page; we must
- * let that proceed. ITS#8336 */
- mdbx_cassert(mc, !mc->mc_dbi || page_numkeys(mp) > 1);
- mdbx_debug("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0)));
+ if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) {
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ mdbx_cassert(mc,
+ mc->mc_xcursor != NULL &&
+ (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED));
+ /* ЕÑли за ключом более одного значениÑ, либо еÑли размер данных
+ * отличаетÑÑ, то вмеÑто Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ñ‚Ñ€ÐµÐ±ÑƒÐµÑ‚ÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ и
+ * поÑÐ»ÐµÐ´ÑƒÑŽÑ‰Ð°Ñ Ð²Ñтавка. */
+ if (mc->mc_xcursor->mx_db.md_entries > 1 ||
+ current_data.iov_len != data->iov_len) {
+ drop_current:
+ rc = mdbx_cursor_del(mc, flags & MDBX_ALLDUPS);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ flags -= MDBX_CURRENT;
+ goto skip_check_samedata;
+ }
+ } else if (unlikely(node_size(key, data) > env->me_leaf_nodemax)) {
+ rc = mdbx_cursor_del(mc, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ flags -= MDBX_CURRENT;
+ goto skip_check_samedata;
+ }
+ }
+ if (!(flags & MDBX_RESERVE) &&
+ unlikely(cmp_lenfast(¤t_data, data) == 0))
+ return MDBX_SUCCESS /* the same data, nothing to update */;
+ skip_check_samedata:;
+ }
- if (flags & (MDBX_PS_FIRST | MDBX_PS_LAST)) {
- i = 0;
- if (flags & MDBX_PS_LAST) {
- i = page_numkeys(mp) - 1;
- /* if already init'd, see if we're already in right place */
- if (mc->mc_flags & C_INITIALIZED) {
- if (mc->mc_ki[mc->mc_top] == i) {
- mc->mc_top = mc->mc_snum++;
- mp = mc->mc_pg[mc->mc_top];
- goto ready;
+ if (mc->mc_db->md_root == P_INVALID) {
+ /* new database, cursor has nothing to point to */
+ mc->mc_snum = 0;
+ mc->mc_top = 0;
+ mc->mc_flags &= ~C_INITIALIZED;
+ rc = MDBX_NO_ROOT;
+ } else if ((flags & MDBX_CURRENT) == 0) {
+ bool exact = false;
+ if ((flags & MDBX_APPEND) && mc->mc_db->md_entries > 0) {
+ rc = mdbx_cursor_last(mc, &dkey, &olddata);
+ if (likely(rc == MDBX_SUCCESS)) {
+ rc = mc->mc_dbx->md_cmp(key, &dkey);
+ if (likely(rc > 0)) {
+ mc->mc_ki[mc->mc_top]++; /* step forward for appending */
+ rc = MDBX_NOTFOUND;
+ } else {
+ if (unlikely(rc != MDBX_SUCCESS || !(flags & MDBX_APPENDDUP)))
+ /* new-key < last-key
+ * or new-key == last-key without MDBX_APPENDDUP */
+ return MDBX_EKEYMISMATCH;
+ exact = true;
+ }
+ }
+ } else {
+ struct cursor_set_result csr =
+ /* olddata may not be updated in case LEAF2-page of dupfixed-subDB */
+ mdbx_cursor_set(mc, (MDBX_val *)key, &olddata, MDBX_SET);
+ rc = csr.err;
+ exact = csr.exact;
+ }
+ if (likely(rc == MDBX_SUCCESS)) {
+ if (exact) {
+ if (unlikely(flags & MDBX_NOOVERWRITE)) {
+ mdbx_debug("duplicate key [%s]", DKEY_DEBUG(key));
+ *data = olddata;
+ return MDBX_KEYEXIST;
+ }
+ if (unlikely(mc->mc_flags & C_SUB)) {
+ /* nested subtree of DUPSORT-database with the same key,
+ * nothing to update */
+ mdbx_assert(env, data->iov_len == 0 &&
+ (olddata.iov_len == 0 ||
+ /* olddata may not be updated in case LEAF2-page
+ of dupfixed-subDB */
+ (mc->mc_db->md_flags & MDBX_DUPFIXED)));
+ return MDBX_SUCCESS;
+ }
+ if (unlikely(flags & MDBX_ALLDUPS) && mc->mc_xcursor &&
+ (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+ rc = mdbx_cursor_del(mc, MDBX_ALLDUPS);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ flags -= MDBX_ALLDUPS;
+ rc = MDBX_NOTFOUND;
+ exact = false;
+ } else /* checking for early exit without dirtying pages */
+ if (!(flags & (MDBX_RESERVE | MDBX_MULTIPLE)) &&
+ unlikely(mc->mc_dbx->md_dcmp(data, &olddata) == 0)) {
+ if (!mc->mc_xcursor)
+ /* the same data, nothing to update */
+ return MDBX_SUCCESS;
+ if (flags & MDBX_NODUPDATA)
+ return MDBX_KEYEXIST;
+ if (flags & MDBX_APPENDDUP)
+ return MDBX_EKEYMISMATCH;
+ if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata)))
+ /* data is match exactly byte-to-byte, nothing to update */
+ return MDBX_SUCCESS;
+ else {
+ /* The data has differences, but the user-provided comparator
+ * considers them equal. So continue update since called without.
+ * Continue to update since was called without MDBX_NODUPDATA. */
+ }
}
- }
}
- } else {
- const struct node_result nsr = mdbx_node_search(mc, key);
- if (nsr.node)
- i = mc->mc_ki[mc->mc_top] + nsr.exact - 1;
- else
- i = page_numkeys(mp) - 1;
- mdbx_debug("following index %u for key [%s]", i, DKEY_DEBUG(key));
- }
-
- mdbx_cassert(mc, i >= 0 && i < (int)page_numkeys(mp));
- node = page_node(mp, i);
-
- if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
- pp_txnid4chk(mp, mc->mc_txn))) != 0))
+ } else if (unlikely(rc != MDBX_NOTFOUND))
return rc;
+ }
- mc->mc_ki[mc->mc_top] = (indx_t)i;
- if (unlikely(rc = mdbx_cursor_push(mc, mp)))
- return rc;
+ mc->mc_flags &= ~C_DEL;
- ready:
- if (flags & MDBX_PS_MODIFY) {
- if (unlikely((rc = mdbx_page_touch(mc)) != 0))
- return rc;
- mp = mc->mc_pg[mc->mc_top];
+ /* Cursor is positioned, check for room in the dirty list */
+ if (!nospill) {
+ rdata = data;
+ if (unlikely(flags & MDBX_MULTIPLE)) {
+ rdata = &xdata;
+ xdata.iov_len = data->iov_len * dcount;
}
+ if (unlikely(err = mdbx_cursor_spill(mc, key, rdata)))
+ return err;
}
-#if !MDBX_DISABLE_PAGECHECKS
- if (unlikely(!IS_LEAF(mp))) {
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return bad_page(mp, "index points to a page with 0x%02x flags\n",
- mp->mp_flags);
- }
-#endif /* !MDBX_DISABLE_PAGECHECKS */
-
- mdbx_debug("found leaf page %" PRIaPGNO " for key [%s]", mp->mp_pgno,
- DKEY_DEBUG(key));
- mc->mc_flags |= C_INITIALIZED;
- mc->mc_flags &= ~C_EOF;
-
- return MDBX_SUCCESS;
-}
-
-static int mdbx_setup_dbx(MDBX_dbx *const dbx, const MDBX_db *const db,
- const unsigned pagesize) {
- if (unlikely(!dbx->md_cmp)) {
- dbx->md_cmp = get_default_keycmp(db->md_flags);
- dbx->md_dcmp = get_default_datacmp(db->md_flags);
+ if (unlikely(rc == MDBX_NO_ROOT)) {
+ /* new database, write a root leaf page */
+ mdbx_debug("%s", "allocating new root leaf page");
+ if (unlikely((*mc->mc_dbistate & DBI_DIRTY) == 0)) {
+ err = mdbx_touch_dbi(mc);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ struct page_result npr = mdbx_page_new(mc, P_LEAF, 1);
+ if (unlikely(npr.err != MDBX_SUCCESS))
+ return npr.err;
+ npr.err = mdbx_cursor_push(mc, npr.page);
+ if (unlikely(npr.err != MDBX_SUCCESS))
+ return npr.err;
+ mc->mc_db->md_root = npr.page->mp_pgno;
+ mc->mc_db->md_depth++;
+ if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
+ assert(key->iov_len >= mc->mc_dbx->md_klen_min &&
+ key->iov_len <= mc->mc_dbx->md_klen_max);
+ mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = key->iov_len;
+ }
+ if (mc->mc_db->md_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) {
+ assert(data->iov_len >= mc->mc_dbx->md_vlen_min &&
+ data->iov_len <= mc->mc_dbx->md_vlen_max);
+ assert(mc->mc_xcursor != NULL);
+ mc->mc_db->md_xsize = mc->mc_xcursor->mx_db.md_xsize =
+ (unsigned)(mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max =
+ mc->mc_xcursor->mx_dbx.md_klen_min =
+ mc->mc_xcursor->mx_dbx.md_klen_max =
+ data->iov_len);
+ }
+ if ((mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_DUPFIXED)) == MDBX_DUPFIXED)
+ npr.page->mp_flags |= P_LEAF2;
+ mc->mc_flags |= C_INITIALIZED;
+ } else {
+ /* make sure all cursor pages are writable */
+ err = mdbx_cursor_touch(mc);
+ if (unlikely(err))
+ return err;
}
- dbx->md_klen_min =
- (db->md_flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0;
- dbx->md_klen_max = keysize_max(pagesize, db->md_flags);
- assert(dbx->md_klen_max != (unsigned)-1);
-
- dbx->md_vlen_min = (db->md_flags & MDBX_INTEGERDUP)
- ? 4 /* sizeof(uint32_t) */
- : ((db->md_flags & MDBX_DUPFIXED) ? 1 : 0);
- dbx->md_vlen_max = valsize_max(pagesize, db->md_flags);
- assert(dbx->md_vlen_max != (unsigned)-1);
+ bool insert_key, insert_data, do_sub = false;
+ insert_key = insert_data = (rc != MDBX_SUCCESS);
+ uint16_t fp_flags = P_LEAF;
+ MDBX_page *fp = env->me_pbuf;
+ fp->mp_txnid = mc->mc_txn->mt_front;
+ if (insert_key) {
+ /* The key does not exist */
+ mdbx_debug("inserting key at index %i", mc->mc_ki[mc->mc_top]);
+ if ((mc->mc_db->md_flags & MDBX_DUPSORT) &&
+ node_size(key, data) > env->me_leaf_nodemax) {
+ /* Too big for a node, insert in sub-DB. Set up an empty
+ * "old sub-page" for prep_subDB to expand to a full page. */
+ fp->mp_leaf2_ksize =
+ (mc->mc_db->md_flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0;
+ fp->mp_lower = fp->mp_upper = 0;
+ olddata.iov_len = PAGEHDRSZ;
+ goto prep_subDB;
+ }
+ } else {
+ /* there's only a key anyway, so this is a no-op */
+ if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
+ char *ptr;
+ unsigned ksize = mc->mc_db->md_xsize;
+ if (unlikely(key->iov_len != ksize))
+ return MDBX_BAD_VALSIZE;
+ ptr = page_leaf2key(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize);
+ memcpy(ptr, key->iov_base, ksize);
+ fix_parent:
+ /* if overwriting slot 0 of leaf, need to
+ * update branch key if there is a parent page */
+ if (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
+ unsigned dtop = 1;
+ mc->mc_top--;
+ /* slot 0 is always an empty key, find real slot */
+ while (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
+ mc->mc_top--;
+ dtop++;
+ }
+ err = MDBX_SUCCESS;
+ if (mc->mc_ki[mc->mc_top])
+ err = mdbx_update_key(mc, key);
+ mdbx_cassert(mc, mc->mc_top + dtop < UINT16_MAX);
+ mc->mc_top += (uint16_t)dtop;
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
- if ((db->md_flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->md_xsize) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(db->md_xsize < dbx->md_vlen_min ||
- db->md_xsize > dbx->md_vlen_max)) {
- mdbx_error("db.md_xsize (%u) <> min/max value-length (%zu/%zu)",
- db->md_xsize, dbx->md_vlen_min, dbx->md_vlen_max);
- return MDBX_CORRUPTED;
+ if (mdbx_audit_enabled()) {
+ err = mdbx_cursor_check(mc, 0);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ return MDBX_SUCCESS;
}
- dbx->md_vlen_min = dbx->md_vlen_max = db->md_xsize;
- }
- return MDBX_SUCCESS;
-}
-static int mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi) {
- MDBX_cursor_couple couple;
- if (unlikely(TXN_DBI_CHANGED(txn, dbi))) {
- mdbx_notice("dbi %u was changed", dbi);
- return MDBX_BAD_DBI;
- }
- int rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ more:;
+ if (mdbx_audit_enabled()) {
+ err = mdbx_cursor_check(mc, 0);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- MDBX_dbx *const dbx = &txn->mt_dbxs[dbi];
- rc = mdbx_page_search(&couple.outer, &dbx->md_name, 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_notice("dbi %u refs to inaccessible subDB (err %d)", dbi, rc);
- return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc;
- }
+ /* Large/Overflow page overwrites need special handling */
+ if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) {
+ int dpages = (node_size(key, data) > env->me_leaf_nodemax)
+ ? number_of_ovpages(env, data->iov_len)
+ : 0;
- MDBX_val data;
- struct node_result nsr = mdbx_node_search(&couple.outer, &dbx->md_name);
- if (unlikely(!nsr.exact))
- return MDBX_BAD_DBI;
- if (unlikely((node_flags(nsr.node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) {
- mdbx_notice("dbi %u refs to not a named subDB (%s)", dbi, "wrong flags");
- return MDBX_INCOMPATIBLE; /* not a named DB */
- }
+ const pgno_t pgno = node_largedata_pgno(node);
+ struct page_result pgr = mdbx_page_get_ex(
+ mc, pgno, pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
+ if (unlikely(pgr.err != MDBX_SUCCESS))
+ return pgr.err;
+ if (unlikely(!IS_OVERFLOW(pgr.page)))
+ return MDBX_CORRUPTED;
- const txnid_t pp_txnid =
- pp_txnid4chk(couple.outer.mc_pg[couple.outer.mc_top], txn);
- rc = mdbx_node_read(&couple.outer, nsr.node, &data, pp_txnid);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ /* Is the ov page from this txn (or a parent) and big enough? */
+ int ovpages = pgr.page->mp_pages;
+ if (!IS_FROZEN(mc->mc_txn, pgr.page) &&
+ (unlikely(mc->mc_flags & C_GCFREEZE)
+ ? (ovpages >= dpages)
+ : (ovpages ==
+ /* LY: add configurable threshold to keep reserve space */
+ dpages))) {
+ /* yes, overwrite it. */
+ if (!IS_MODIFIABLE(mc->mc_txn, pgr.page)) {
+ if (IS_SPILLED(mc->mc_txn, pgr.page)) {
+ pgr = /* TODO: avoid search and get txn & spill-index from
+ page_result */
+ mdbx_page_unspill(mc->mc_txn, pgr.page);
+ if (unlikely(pgr.err))
+ return pgr.err;
+ } else {
+ if (unlikely(!mc->mc_txn->mt_parent)) {
+ mdbx_error(
+ "Unexpected not frozen/modifiable/spilled but shadowed %s "
+ "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
+ " without parent transaction, current txn %" PRIaTXN
+ " front %" PRIaTXN,
+ "overflow/large", pgno, pgr.page->mp_txnid,
+ mc->mc_txn->mt_txnid, mc->mc_txn->mt_front);
+ return MDBX_PROBLEM;
+ }
- if (unlikely(data.iov_len != sizeof(MDBX_db))) {
- mdbx_notice("dbi %u refs to not a named subDB (%s)", dbi, "wrong rec-size");
- return MDBX_INCOMPATIBLE; /* not a named DB */
- }
+ /* It is writable only in a parent txn */
+ MDBX_page *np = mdbx_page_malloc(mc->mc_txn, ovpages);
+ if (unlikely(!np))
+ return MDBX_ENOMEM;
- uint16_t md_flags = UNALIGNED_PEEK_16(data.iov_base, MDBX_db, md_flags);
- /* The txn may not know this DBI, or another process may
- * have dropped and recreated the DB with other flags. */
- MDBX_db *const db = &txn->mt_dbs[dbi];
- if (unlikely((db->md_flags & DB_PERSISTENT_FLAGS) != md_flags)) {
- mdbx_notice("dbi %u refs to the re-created subDB with different flags "
- "(present 0x%x != wanna 0x%x)",
- dbi, db->md_flags & DB_PERSISTENT_FLAGS, md_flags);
- return MDBX_INCOMPATIBLE;
- }
+ memcpy(np, pgr.page, PAGEHDRSZ); /* Copy header of page */
+ err = mdbx_page_dirty(mc->mc_txn, pgr.page = np, ovpages);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- memcpy(db, data.iov_base, sizeof(MDBX_db));
-#if !MDBX_DISABLE_PAGECHECKS
- mdbx_tassert(txn, txn->mt_front >= pp_txnid);
- if (unlikely(db->md_mod_txnid > pp_txnid)) {
- mdbx_error("db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")",
- db->md_mod_txnid, pp_txnid);
- return MDBX_CORRUPTED;
- }
-#endif /* !MDBX_DISABLE_PAGECHECKS */
- rc = mdbx_setup_dbx(dbx, db, txn->mt_env->me_psize);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#if MDBX_ENABLE_PGOP_STAT
+ mc->mc_txn->mt_env->me_lck->mti_pgop_stat.clone.weak += ovpages;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+ mdbx_cassert(mc, mdbx_dirtylist_check(mc->mc_txn));
+ }
+ }
+ node_set_ds(node, data->iov_len);
+ if (F_ISSET(flags, MDBX_RESERVE))
+ data->iov_base = page_data(pgr.page);
+ else
+ memcpy(page_data(pgr.page), data->iov_base, data->iov_len);
- txn->mt_dbistate[dbi] &= ~DBI_STALE;
- return MDBX_SUCCESS;
-}
+ if (mdbx_audit_enabled()) {
+ err = mdbx_cursor_check(mc, 0);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ return MDBX_SUCCESS;
+ }
-/* Search for the lowest key under the current branch page.
- * This just bypasses a numkeys check in the current page
- * before calling mdbx_page_search_root(), because the callers
- * are all in situations where the current page is known to
- * be underfilled. */
-__hot static int mdbx_page_search_lowest(MDBX_cursor *mc) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- mdbx_cassert(mc, IS_BRANCH(mp));
- MDBX_node *node = page_node(mp, 0);
- int rc;
+ if ((err = mdbx_page_retire(mc, pgr.page)) != MDBX_SUCCESS)
+ return err;
+ } else {
+ olddata.iov_len = node_ds(node);
+ olddata.iov_base = node_data(node);
+ mdbx_cassert(mc, (char *)olddata.iov_base + olddata.iov_len <=
+ (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize);
- if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
- pp_txnid4chk(mp, mc->mc_txn))) != 0))
- return rc;
+ /* DB has dups? */
+ if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) {
+ /* Prepare (sub-)page/sub-DB to accept the new item, if needed.
+ * fp: old sub-page or a header faking it.
+ * mp: new (sub-)page. offset: growth in page size.
+ * xdata: node data with new page or DB. */
+ unsigned i;
+ size_t offset = 0;
+ MDBX_page *mp = fp = xdata.iov_base = env->me_pbuf;
+ mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno;
- mc->mc_ki[mc->mc_top] = 0;
- if (unlikely(rc = mdbx_cursor_push(mc, mp)))
- return rc;
- return mdbx_page_search_root(mc, NULL, MDBX_PS_FIRST);
-}
+ /* Was a single item before, must convert now */
+ if (!F_ISSET(node_flags(node), F_DUPDATA)) {
-/* Search for the page a given key should be in.
- * Push it and its parent pages on the cursor stack.
- *
- * [in,out] mc the cursor for this operation.
- * [in] key the key to search for, or NULL for first/last page.
- * [in] flags If MDBX_PS_MODIFY is set, visited pages in the DB
- * are touched (updated with new page numbers).
- * If MDBX_PS_FIRST or MDBX_PS_LAST is set, find first or last
- * leaf.
- * This is used by mdbx_cursor_first() and mdbx_cursor_last().
- * If MDBX_PS_ROOTONLY set, just fetch root node, no further
- * lookups.
- *
- * Returns 0 on success, non-zero on failure. */
-__hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key,
- int flags) {
- int rc;
- pgno_t root;
+ /* does data match? */
+ const int cmp = mc->mc_dbx->md_dcmp(data, &olddata);
+ if ((flags & MDBX_APPENDDUP) && unlikely(cmp <= 0))
+ return MDBX_EKEYMISMATCH;
+ if (cmp == 0) {
+ if (flags & MDBX_NODUPDATA)
+ return MDBX_KEYEXIST;
+ if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata))) {
+ /* data is match exactly byte-to-byte, nothing to update */
+ if (unlikely(flags & MDBX_MULTIPLE)) {
+ rc = MDBX_SUCCESS;
+ goto continue_multiple;
+ }
+ return MDBX_SUCCESS;
+ } else {
+ /* The data has differences, but the user-provided comparator
+ * considers them equal. So continue update since called without.
+ * Continue to update since was called without MDBX_NODUPDATA. */
+ }
+ mdbx_cassert(mc, node_size(key, data) <= env->me_leaf_nodemax);
+ goto current;
+ }
- /* Make sure the txn is still viable, then find the root from
- * the txn's db table and set it as the root of the cursor's stack. */
- if (unlikely(mc->mc_txn->mt_flags & MDBX_TXN_BLOCKED)) {
- mdbx_debug("%s", "transaction has failed, must abort");
- return MDBX_BAD_TXN;
- }
+ /* Just overwrite the current item */
+ if (flags & MDBX_CURRENT) {
+ mdbx_cassert(mc, node_size(key, data) <= env->me_leaf_nodemax);
+ goto current;
+ }
- /* Make sure we're using an up-to-date root */
- if (unlikely(*mc->mc_dbistate & DBI_STALE)) {
- rc = mdbx_fetch_sdb(mc->mc_txn, mc->mc_dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- root = mc->mc_db->md_root;
+ /* Back up original data item */
+ memcpy(dkey.iov_base = fp + 1, olddata.iov_base,
+ dkey.iov_len = olddata.iov_len);
+ dupdata_flag = 1;
- if (unlikely(root == P_INVALID)) { /* Tree is empty. */
- mdbx_debug("%s", "tree is empty");
- return MDBX_NOTFOUND;
- }
+ /* Make sub-page header for the dup items, with dummy body */
+ fp->mp_flags = P_LEAF | P_SUBP;
+ fp->mp_lower = 0;
+ xdata.iov_len = PAGEHDRSZ + dkey.iov_len + data->iov_len;
+ if (mc->mc_db->md_flags & MDBX_DUPFIXED) {
+ fp->mp_flags |= P_LEAF2;
+ fp->mp_leaf2_ksize = (uint16_t)data->iov_len;
+ xdata.iov_len += 2 * data->iov_len; /* leave space for 2 more */
+ mdbx_cassert(mc, xdata.iov_len <= env->me_psize);
+ } else {
+ xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) +
+ (dkey.iov_len & 1) + (data->iov_len & 1);
+ mdbx_cassert(mc, xdata.iov_len <= env->me_psize);
+ }
+ fp->mp_upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ);
+ olddata.iov_len = xdata.iov_len; /* pretend olddata is fp */
+ } else if (node_flags(node) & F_SUBDATA) {
+ /* Data is on sub-DB, just store it */
+ flags |= F_DUPDATA | F_SUBDATA;
+ goto put_sub;
+ } else {
+ /* Data is on sub-page */
+ fp = olddata.iov_base;
+ switch (flags) {
+ default:
+ if (!(mc->mc_db->md_flags & MDBX_DUPFIXED)) {
+ offset = node_size(data, nullptr) + sizeof(indx_t);
+ break;
+ }
+ offset = fp->mp_leaf2_ksize;
+ if (page_room(fp) < offset) {
+ offset *= 4; /* space for 4 more */
+ break;
+ }
+ /* FALLTHRU: Big enough MDBX_DUPFIXED sub-page */
+ __fallthrough;
+ case MDBX_CURRENT | MDBX_NODUPDATA:
+ case MDBX_CURRENT:
+ fp->mp_txnid = mc->mc_txn->mt_front;
+ fp->mp_pgno = mp->mp_pgno;
+ mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;
+ flags |= F_DUPDATA;
+ goto put_sub;
+ }
+ xdata.iov_len = olddata.iov_len + offset;
+ }
- mdbx_cassert(mc, root >= NUM_METAS);
- if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) {
- txnid_t pp_txnid = mc->mc_db->md_mod_txnid;
- pp_txnid = /* mc->mc_db->md_mod_txnid maybe zero in a legacy DB */ pp_txnid
- ? pp_txnid
- : mc->mc_txn->mt_txnid;
- if ((mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) == 0) {
- MDBX_txn *scan = mc->mc_txn;
- do
- if ((scan->mt_flags & MDBX_TXN_DIRTY) &&
- (mc->mc_dbi == MAIN_DBI ||
- (scan->mt_dbistate[mc->mc_dbi] & DBI_DIRTY))) {
- pp_txnid = scan->mt_front;
- break;
+ fp_flags = fp->mp_flags;
+ if (node_size_len(node_ks(node), xdata.iov_len) >
+ env->me_leaf_nodemax) {
+ /* Too big for a sub-page, convert to sub-DB */
+ fp_flags &= ~P_SUBP;
+ prep_subDB:
+ nested_dupdb.md_xsize = 0;
+ nested_dupdb.md_flags = flags_db2sub(mc->mc_db->md_flags);
+ if (mc->mc_db->md_flags & MDBX_DUPFIXED) {
+ fp_flags |= P_LEAF2;
+ nested_dupdb.md_xsize = fp->mp_leaf2_ksize;
+ }
+ nested_dupdb.md_depth = 1;
+ nested_dupdb.md_branch_pages = 0;
+ nested_dupdb.md_leaf_pages = 1;
+ nested_dupdb.md_overflow_pages = 0;
+ nested_dupdb.md_entries = page_numkeys(fp);
+ xdata.iov_len = sizeof(nested_dupdb);
+ xdata.iov_base = &nested_dupdb;
+ const struct page_result par = mdbx_page_alloc(mc, 1, MDBX_ALLOC_ALL);
+ mp = par.page;
+ if (unlikely(par.err != MDBX_SUCCESS))
+ return par.err;
+ mc->mc_db->md_leaf_pages += 1;
+ mdbx_cassert(mc, env->me_psize > olddata.iov_len);
+ offset = env->me_psize - (unsigned)olddata.iov_len;
+ flags |= F_DUPDATA | F_SUBDATA;
+ nested_dupdb.md_root = mp->mp_pgno;
+ nested_dupdb.md_seq = 0;
+ nested_dupdb.md_mod_txnid = mc->mc_txn->mt_txnid;
+ sub_root = mp;
+ }
+ if (mp != fp) {
+ mp->mp_flags = fp_flags;
+ mp->mp_txnid = mc->mc_txn->mt_front;
+ mp->mp_leaf2_ksize = fp->mp_leaf2_ksize;
+ mp->mp_lower = fp->mp_lower;
+ mdbx_cassert(mc, fp->mp_upper + offset <= UINT16_MAX);
+ mp->mp_upper = (indx_t)(fp->mp_upper + offset);
+ if (unlikely(fp_flags & P_LEAF2)) {
+ memcpy(page_data(mp), page_data(fp),
+ page_numkeys(fp) * fp->mp_leaf2_ksize);
+ } else {
+ memcpy((char *)mp + mp->mp_upper + PAGEHDRSZ,
+ (char *)fp + fp->mp_upper + PAGEHDRSZ,
+ olddata.iov_len - fp->mp_upper - PAGEHDRSZ);
+ memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs),
+ page_numkeys(fp) * sizeof(mp->mp_ptrs[0]));
+ for (i = 0; i < page_numkeys(fp); i++) {
+ mdbx_cassert(mc, mp->mp_ptrs[i] + offset <= UINT16_MAX);
+ mp->mp_ptrs[i] += (indx_t)offset;
+ }
+ }
}
- while (unlikely((scan = scan->mt_parent) != nullptr));
- }
- if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], pp_txnid)) != 0))
- return rc;
- }
-
- mc->mc_snum = 1;
- mc->mc_top = 0;
- mdbx_debug("db %d root page %" PRIaPGNO " has flags 0x%X", DDBI(mc), root,
- mc->mc_pg[0]->mp_flags);
-
- if (flags & MDBX_PS_MODIFY) {
- if (!(*mc->mc_dbistate & DBI_DIRTY) && unlikely(rc = mdbx_touch_dbi(mc)))
- return rc;
- if (unlikely(rc = mdbx_page_touch(mc)))
- return rc;
- }
+ rdata = &xdata;
+ flags |= F_DUPDATA;
+ do_sub = true;
+ if (!insert_key)
+ mdbx_node_del(mc, 0);
+ goto new_sub;
+ }
- if (flags & MDBX_PS_ROOTONLY)
- return MDBX_SUCCESS;
+ /* MDBX passes F_SUBDATA in 'flags' to write a DB record */
+ if (unlikely((node_flags(node) ^ flags) & F_SUBDATA))
+ return MDBX_INCOMPATIBLE;
- return mdbx_page_search_root(mc, key, flags);
-}
+ current:
+ if (data->iov_len == olddata.iov_len) {
+ mdbx_cassert(mc, EVEN(key->iov_len) == EVEN(node_ks(node)));
+ /* same size, just replace it. Note that we could
+ * also reuse this node if the new data is smaller,
+ * but instead we opt to shrink the node in that case. */
+ if (F_ISSET(flags, MDBX_RESERVE))
+ data->iov_base = olddata.iov_base;
+ else if (!(mc->mc_flags & C_SUB))
+ memcpy(olddata.iov_base, data->iov_base, data->iov_len);
+ else {
+ mdbx_cassert(mc, page_numkeys(mc->mc_pg[mc->mc_top]) == 1);
+ mdbx_cassert(mc, PAGETYPE(mc->mc_pg[mc->mc_top]) == P_LEAF);
+ mdbx_cassert(mc, node_ds(node) == 0);
+ mdbx_cassert(mc, node_flags(node) == 0);
+ mdbx_cassert(mc, key->iov_len < UINT16_MAX);
+ node_set_ks(node, key->iov_len);
+ memcpy(node_key(node), key->iov_base, key->iov_len);
+ mdbx_cassert(mc, (char *)node_key(node) + node_ds(node) <
+ (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize);
+ goto fix_parent;
+ }
-/* Return the data associated with a given node.
- *
- * [in] mc The cursor for this operation.
- * [in] leaf The node being read.
- * [out] data Updated to point to the node's data.
- *
- * Returns 0 on success, non-zero on failure. */
-static __always_inline int mdbx_node_read(MDBX_cursor *mc,
- const MDBX_node *node, MDBX_val *data,
- const txnid_t front) {
- data->iov_len = node_ds(node);
- data->iov_base = node_data(node);
- if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) {
- /* Read overflow data. */
- MDBX_page *omp; /* overflow page */
- int rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, front);
- if (unlikely((rc != MDBX_SUCCESS))) {
- mdbx_debug("read overflow page %" PRIaPGNO " failed",
- node_largedata_pgno(node));
- return rc;
+ if (mdbx_audit_enabled()) {
+ err = mdbx_cursor_check(mc, 0);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ return MDBX_SUCCESS;
+ }
}
- data->iov_base = page_data(omp);
+ mdbx_node_del(mc, 0);
}
- return MDBX_SUCCESS;
-}
-int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) {
- DKBUF_DEBUG;
- mdbx_debug("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
-
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ rdata = data;
- if (unlikely(!key || !data))
- return MDBX_EINVAL;
+new_sub:;
+ unsigned nflags = flags & NODE_ADD_FLAGS;
+ size_t nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->iov_len
+ : leaf_size(env, key, rdata);
+ if (page_room(mc->mc_pg[mc->mc_top]) < nsize) {
+ if (!insert_key)
+ nflags |= MDBX_SPLIT_REPLACE;
+ rc = mdbx_page_split(mc, key, rdata, P_INVALID, nflags);
+ if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
+ rc = mdbx_cursor_check(mc, 0);
+ } else {
+ /* There is room already in this leaf page. */
+ if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
+ mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0 &&
+ rdata->iov_len == 0);
+ rc = mdbx_node_add_leaf2(mc, mc->mc_ki[mc->mc_top], key);
+ } else
+ rc = mdbx_node_add_leaf(mc, mc->mc_ki[mc->mc_top], key, rdata, nflags);
+ if (likely(rc == 0)) {
+ /* Adjust other cursors pointing to mp */
+ const MDBX_dbi dbi = mc->mc_dbi;
+ const unsigned i = mc->mc_top;
+ MDBX_page *const mp = mc->mc_pg[i];
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2;
+ m2 = m2->mc_next) {
+ MDBX_cursor *m3 =
+ (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp)
+ continue;
+ if (m3->mc_ki[i] >= mc->mc_ki[i])
+ m3->mc_ki[i] += insert_key;
+ if (XCURSOR_INITED(m3))
+ XCURSOR_REFRESH(m3, mp, m3->mc_ki[i]);
+ }
+ }
+ }
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
+ if (likely(rc == MDBX_SUCCESS)) {
+ /* Now store the actual data in the child DB. Note that we're
+ * storing the user data in the keys field, so there are strict
+ * size limits on dupdata. The actual data fields of the child
+ * DB are all zero size. */
+ if (do_sub) {
+ int xflags;
+ size_t ecount;
+ put_sub:
+ xdata.iov_len = 0;
+ xdata.iov_base = nullptr;
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+#define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1
+ STATIC_ASSERT(
+ (MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) ==
+ MDBX_NOOVERWRITE);
+ xflags = MDBX_CURRENT | MDBX_NOSPILL |
+ ((flags & MDBX_NODUPDATA) >>
+ SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE);
+ if ((flags & MDBX_CURRENT) == 0) {
+ xflags -= MDBX_CURRENT;
+ err = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ if (sub_root)
+ mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root;
+ /* converted, write the original data first */
+ if (dupdata_flag) {
+ rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
+ if (unlikely(rc))
+ goto bad_sub;
+ /* we've done our job */
+ dkey.iov_len = 0;
+ }
+ if (!(node_flags(node) & F_SUBDATA) || sub_root) {
+ /* Adjust other cursors pointing to mp */
+ MDBX_cursor *m2;
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ unsigned i = mc->mc_top;
+ MDBX_page *mp = mc->mc_pg[i];
+ const int nkeys = page_numkeys(mp);
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
+ for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) {
+ if (m2 == mc || m2->mc_snum < mc->mc_snum)
+ continue;
+ if (!(m2->mc_flags & C_INITIALIZED))
+ continue;
+ if (m2->mc_pg[i] == mp) {
+ if (m2->mc_ki[i] == mc->mc_ki[i]) {
+ err = mdbx_xcursor_init2(m2, mx, dupdata_flag);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ } else if (!insert_key && m2->mc_ki[i] < nkeys) {
+ XCURSOR_REFRESH(m2, mp, m2->mc_ki[i]);
+ }
+ }
+ }
+ }
+ mdbx_cassert(mc, mc->mc_xcursor->mx_db.md_entries < PTRDIFF_MAX);
+ ecount = (size_t)mc->mc_xcursor->mx_db.md_entries;
+#define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1
+ STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) ==
+ MDBX_APPEND);
+ xflags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND;
+ rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
+ if (flags & F_SUBDATA) {
+ void *db = node_data(node);
+ mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid;
+ memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db));
+ }
+ insert_data = (ecount != (size_t)mc->mc_xcursor->mx_db.md_entries);
+ }
+ /* Increment count unless we just replaced an existing item. */
+ if (insert_data)
+ mc->mc_db->md_entries++;
+ if (insert_key) {
+ /* Invalidate txn if we created an empty sub-DB */
+ if (unlikely(rc))
+ goto bad_sub;
+ /* If we succeeded and the key didn't exist before,
+ * make sure the cursor is marked valid. */
+ mc->mc_flags |= C_INITIALIZED;
+ }
+ if (unlikely(flags & MDBX_MULTIPLE)) {
+ if (likely(rc == MDBX_SUCCESS)) {
+ continue_multiple:
+ mcount++;
+ /* let caller know how many succeeded, if any */
+ data[1].iov_len = mcount;
+ if (mcount < dcount) {
+ data[0].iov_base = (char *)data[0].iov_base + data[0].iov_len;
+ insert_key = insert_data = false;
+ goto more;
+ }
+ }
+ }
+ if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
+ rc = mdbx_cursor_check(mc, 0);
return rc;
-
- return mdbx_cursor_set(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err;
+ bad_sub:
+ if (unlikely(rc == MDBX_KEYEXIST)) {
+ /* should not happen, we deleted that item */
+ mdbx_error("Unexpected %i error while put to nested dupsort's hive", rc);
+ rc = MDBX_PROBLEM;
+ }
+ }
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return rc;
}
-int mdbx_get_equal_or_great(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
- MDBX_val *data) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
-
- if (unlikely(!key || !data))
+int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) {
+ if (unlikely(!mc))
return MDBX_EINVAL;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
-
- if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED))
- return MDBX_BAD_TXN;
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- return mdbx_cursor_get(&cx.outer, key, data, MDBX_SET_LOWERBOUND);
-}
-
-int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
- size_t *values_count) {
- DKBUF_DEBUG;
- mdbx_debug("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
+ if (unlikely(TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)))
+ return MDBX_BAD_DBI;
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
+ return MDBX_ENODATA;
- if (unlikely(!key || !data))
- return MDBX_EINVAL;
+ if (unlikely(mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])))
+ return MDBX_NOTFOUND;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
+ if (likely((flags & MDBX_NOSPILL) == 0) &&
+ unlikely(rc = mdbx_cursor_spill(mc, NULL, NULL)))
+ return rc;
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ rc = mdbx_cursor_touch(mc);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- rc = mdbx_cursor_set(&cx.outer, key, data, MDBX_SET_KEY).err;
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_NOTFOUND && values_count)
- *values_count = 0;
- return rc;
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
+ return MDBX_CORRUPTED;
+ if (IS_LEAF2(mp)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
+ mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
+ mp->mp_pgno);
+ return MDBX_CORRUPTED;
+ }
+ goto del_key;
}
- if (values_count) {
- *values_count = 1;
- if (cx.outer.mc_xcursor != NULL) {
- MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
- cx.outer.mc_ki[cx.outer.mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- // coverity[uninit_use : FALSE]
- mdbx_tassert(txn, cx.outer.mc_xcursor == &cx.inner &&
- (cx.inner.mx_cursor.mc_flags & C_INITIALIZED));
- // coverity[uninit_use : FALSE]
- *values_count =
- (sizeof(*values_count) >= sizeof(cx.inner.mx_db.md_entries) ||
- cx.inner.mx_db.md_entries <= PTRDIFF_MAX)
- ? (size_t)cx.inner.mx_db.md_entries
- : PTRDIFF_MAX;
+ MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ if (flags & (MDBX_ALLDUPS | /* for compatibility */ MDBX_NODUPDATA)) {
+ /* mdbx_cursor_del0() will subtract the final entry */
+ mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1;
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
+ } else {
+ if (!F_ISSET(node_flags(node), F_SUBDATA))
+ mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
+ rc = mdbx_cursor_del(&mc->mc_xcursor->mx_cursor, MDBX_NOSPILL);
+ if (unlikely(rc))
+ return rc;
+ /* If sub-DB still has entries, we're done */
+ if (mc->mc_xcursor->mx_db.md_entries) {
+ if (node_flags(node) & F_SUBDATA) {
+ /* update subDB info */
+ void *db = node_data(node);
+ mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid;
+ memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db));
+ } else {
+ MDBX_cursor *m2;
+ /* shrink fake page */
+ mdbx_node_shrink(mp, mc->mc_ki[mc->mc_top]);
+ node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
+ /* fix other sub-DB cursors pointed at fake pages on this page */
+ for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) {
+ if (m2 == mc || m2->mc_snum < mc->mc_snum)
+ continue;
+ if (!(m2->mc_flags & C_INITIALIZED))
+ continue;
+ if (m2->mc_pg[mc->mc_top] == mp) {
+ MDBX_node *inner = node;
+ if (m2->mc_ki[mc->mc_top] >= page_numkeys(mp))
+ continue;
+ if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) {
+ inner = page_node(mp, m2->mc_ki[mc->mc_top]);
+ if (node_flags(inner) & F_SUBDATA)
+ continue;
+ }
+ m2->mc_xcursor->mx_cursor.mc_pg[0] = node_data(inner);
+ }
+ }
+ }
+ mc->mc_db->md_entries--;
+ mdbx_cassert(mc, mc->mc_db->md_entries > 0 && mc->mc_db->md_depth > 0 &&
+ mc->mc_db->md_root != P_INVALID);
+ return rc;
+ } else {
+ mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
}
+ /* otherwise fall thru and delete the sub-DB */
}
- }
- return MDBX_SUCCESS;
-}
-
-/* Find a sibling for a page.
- * Replaces the page at the top of the cursor's stack with the specified
- * sibling, if one exists.
- *
- * [in] mc The cursor for this operation.
- * [in] dir SIBLING_LEFT or SIBLING_RIGHT.
- *
- * Returns 0 on success, non-zero on failure. */
-static int mdbx_cursor_sibling(MDBX_cursor *mc, int dir) {
- int rc;
- MDBX_node *node;
- MDBX_page *mp;
- assert(dir == SIBLING_LEFT || dir == SIBLING_RIGHT);
- if (unlikely(mc->mc_snum < 2))
- return MDBX_NOTFOUND; /* root has no siblings */
-
- mdbx_cursor_pop(mc);
- mdbx_debug("parent page is page %" PRIaPGNO ", index %u",
- mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]);
-
- if ((dir == SIBLING_RIGHT)
- ? (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mc->mc_pg[mc->mc_top]))
- : (mc->mc_ki[mc->mc_top] == 0)) {
- mdbx_debug("no more keys aside, moving to next %s sibling",
- dir ? "right" : "left");
- if (unlikely((rc = mdbx_cursor_sibling(mc, dir)) != MDBX_SUCCESS)) {
- /* undo cursor_pop before returning */
- mc->mc_top++;
- mc->mc_snum++;
- return rc;
+ if (node_flags(node) & F_SUBDATA) {
+ /* add all the child DB's pages to the free list */
+ rc = mdbx_drop_tree(&mc->mc_xcursor->mx_cursor, false);
+ if (unlikely(rc))
+ goto fail;
}
- } else {
- assert((dir - 1) == -1 || (dir - 1) == 1);
- mc->mc_ki[mc->mc_top] += (indx_t)(dir - 1);
- mdbx_debug("just moving to %s index key %u",
- (dir == SIBLING_RIGHT) ? "right" : "left",
- mc->mc_ki[mc->mc_top]);
}
- mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top]));
+ /* MDBX passes F_SUBDATA in 'flags' to delete a DB record */
+ else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA))
+ return MDBX_INCOMPATIBLE;
- node = page_node(mp = mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp,
- pp_txnid4chk(mp, mc->mc_txn))) != 0)) {
- /* mc will be inconsistent if caller does mc_snum++ as above */
- mc->mc_flags &= ~(C_INITIALIZED | C_EOF);
- return rc;
+ /* add overflow pages to free list */
+ if (F_ISSET(node_flags(node), F_BIGDATA)) {
+ MDBX_page *omp;
+ if (unlikely((rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp,
+ pp_txnid4chk(mp, mc->mc_txn))) ||
+ (rc = mdbx_page_retire(mc, omp))))
+ goto fail;
}
- rc = mdbx_cursor_push(mc, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+del_key:
+ return mdbx_cursor_del0(mc);
- mc->mc_ki[mc->mc_top] =
- (indx_t)((dir == SIBLING_LEFT) ? page_numkeys(mp) - 1 : 0);
- return MDBX_SUCCESS;
+fail:
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return rc;
}
-/* Move the cursor to the next data item. */
-static int mdbx_cursor_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
- MDBX_cursor_op op) {
- MDBX_page *mp;
- MDBX_node *node;
- int rc;
-
- if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_NEXT_DUP)
- return MDBX_NOTFOUND;
-
- if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
- return mdbx_cursor_first(mc, key, data);
-
- mp = mc->mc_pg[mc->mc_top];
- if (unlikely(mc->mc_flags & C_EOF)) {
- if (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mp))
- return MDBX_NOTFOUND;
- mc->mc_flags ^= C_EOF;
- }
-
- if (mc->mc_db->md_flags & MDBX_DUPSORT) {
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- if (op == MDBX_NEXT || op == MDBX_NEXT_DUP) {
- rc =
- mdbx_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_NEXT);
- if (op != MDBX_NEXT || rc != MDBX_NOTFOUND) {
- if (likely(rc == MDBX_SUCCESS))
- get_key_optional(node, key);
- return rc;
- }
- }
- } else {
- mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
- if (op == MDBX_NEXT_DUP)
- return MDBX_NOTFOUND;
- }
- }
+/* Allocate and initialize new pages for a database.
+ * Set MDBX_TXN_ERROR on failure.
+ *
+ * [in] mc a cursor on the database being added to.
+ * [in] flags flags defining what type of page is being allocated.
+ * [in] num the number of pages to allocate. This is usually 1,
+ * unless allocating overflow pages for a large record.
+ * [out] mp Address of a page, or NULL on failure.
+ *
+ * Returns 0 on success, non-zero on failure. */
+static struct page_result mdbx_page_new(MDBX_cursor *mc, const unsigned flags,
+ const unsigned npages) {
+ struct page_result ret = mdbx_page_alloc(mc, npages, MDBX_ALLOC_ALL);
+ if (unlikely(ret.err != MDBX_SUCCESS))
+ return ret;
- mdbx_debug("cursor_next: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno,
- (void *)mc);
- if (mc->mc_flags & C_DEL) {
- mc->mc_flags ^= C_DEL;
- goto skip;
- }
+ mdbx_debug("db %u allocated new page %" PRIaPGNO ", num %u", mc->mc_dbi,
+ ret.page->mp_pgno, npages);
+ ret.page->mp_flags = (uint16_t)flags;
+ ret.page->mp_txnid = mc->mc_txn->mt_front;
+ mdbx_cassert(mc, *mc->mc_dbistate & DBI_DIRTY);
+ mdbx_cassert(mc, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY);
+#if MDBX_ENABLE_PGOP_STAT
+ mc->mc_txn->mt_env->me_lck->mti_pgop_stat.newly.weak += npages;
+#endif /* MDBX_ENABLE_PGOP_STAT */
- int ki = mc->mc_ki[mc->mc_top];
- mc->mc_ki[mc->mc_top] = (indx_t)++ki;
- const int numkeys = page_numkeys(mp);
- if (unlikely(ki >= numkeys)) {
- mdbx_debug("%s", "=====> move to next sibling page");
- mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1);
- if (unlikely((rc = mdbx_cursor_sibling(mc, SIBLING_RIGHT)) !=
- MDBX_SUCCESS)) {
- mc->mc_flags |= C_EOF;
- return rc;
+ if (likely((flags & P_OVERFLOW) == 0)) {
+ STATIC_ASSERT(P_BRANCH == 1);
+ const bool is_branch = flags & P_BRANCH;
+ ret.page->mp_lower = 0;
+ ret.page->mp_upper = (indx_t)(mc->mc_txn->mt_env->me_psize - PAGEHDRSZ);
+ mc->mc_db->md_branch_pages += is_branch;
+ mc->mc_db->md_leaf_pages += 1 - is_branch;
+ if (unlikely(mc->mc_flags & C_SUB)) {
+ MDBX_db *outer = mdbx_outer_db(mc);
+ outer->md_branch_pages += is_branch;
+ outer->md_leaf_pages += 1 - is_branch;
}
- mp = mc->mc_pg[mc->mc_top];
- mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
- mc->mc_ki[mc->mc_top]);
+ } else {
+ mc->mc_db->md_overflow_pages += npages;
+ ret.page->mp_pages = npages;
+ mdbx_cassert(mc, !(mc->mc_flags & C_SUB));
}
-skip:
- mdbx_debug("==> cursor points to page %" PRIaPGNO
- " with %u keys, key index %u",
- mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]);
+ return ret;
+}
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
- return MDBX_CORRUPTED;
+static int __must_check_result mdbx_node_add_leaf2(MDBX_cursor *mc,
+ unsigned indx,
+ const MDBX_val *key) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ DKBUF_DEBUG;
+ mdbx_debug("add to leaf2-%spage %" PRIaPGNO " index %i, "
+ " key size %" PRIuPTR " [%s]",
+ IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx,
+ key ? key->iov_len : 0, DKEY_DEBUG(key));
- if (IS_LEAF2(mp)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mp->mp_pgno);
- return MDBX_CORRUPTED;
- } else if (likely(key)) {
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
- }
- return MDBX_SUCCESS;
- }
+ mdbx_cassert(mc, key);
+ mdbx_cassert(mc, PAGETYPE(mp) == (P_LEAF | P_LEAF2));
+ const unsigned ksize = mc->mc_db->md_xsize;
+ mdbx_cassert(mc, ksize == key->iov_len);
+ const unsigned nkeys = page_numkeys(mp);
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- rc = mdbx_xcursor_init1(mc, node, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- } else if (likely(data)) {
- if (unlikely((rc = mdbx_node_read(mc, node, data,
- pp_txnid4chk(mp, mc->mc_txn))) !=
- MDBX_SUCCESS))
- return rc;
+ /* Just using these for counting */
+ const intptr_t lower = mp->mp_lower + sizeof(indx_t);
+ const intptr_t upper = mp->mp_upper - (ksize - sizeof(indx_t));
+ if (unlikely(lower > upper)) {
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_PAGE_FULL;
}
+ mp->mp_lower = (indx_t)lower;
+ mp->mp_upper = (indx_t)upper;
- get_key_optional(node, key);
+ char *const ptr = page_leaf2key(mp, indx, ksize);
+ mdbx_cassert(mc, nkeys >= indx);
+ const unsigned diff = nkeys - indx;
+ if (likely(diff > 0))
+ /* Move higher keys up one slot. */
+ memmove(ptr + ksize, ptr, diff * ksize);
+ /* insert new key */
+ memcpy(ptr, key->iov_base, ksize);
return MDBX_SUCCESS;
}
-/* Move the cursor to the previous data item. */
-static int mdbx_cursor_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
- MDBX_cursor_op op) {
- MDBX_page *mp;
- MDBX_node *node;
- int rc;
+static int __must_check_result mdbx_node_add_branch(MDBX_cursor *mc,
+ unsigned indx,
+ const MDBX_val *key,
+ pgno_t pgno) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ DKBUF_DEBUG;
+ mdbx_debug("add to branch-%spage %" PRIaPGNO " index %i, node-pgno %" PRIaPGNO
+ " key size %" PRIuPTR " [%s]",
+ IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, pgno,
+ key ? key->iov_len : 0, DKEY_DEBUG(key));
- if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_PREV_DUP)
- return MDBX_NOTFOUND;
+ mdbx_cassert(mc, PAGETYPE(mp) == P_BRANCH);
+ STATIC_ASSERT(NODESIZE % 2 == 0);
- if (unlikely(!(mc->mc_flags & C_INITIALIZED))) {
- rc = mdbx_cursor_last(mc, key, data);
- if (unlikely(rc))
- return rc;
- mc->mc_ki[mc->mc_top]++;
- }
+ /* Move higher pointers up one slot. */
+ const unsigned nkeys = page_numkeys(mp);
+ mdbx_cassert(mc, nkeys >= indx);
+ for (unsigned i = nkeys; i > indx; --i)
+ mp->mp_ptrs[i] = mp->mp_ptrs[i - 1];
- mp = mc->mc_pg[mc->mc_top];
- if ((mc->mc_db->md_flags & MDBX_DUPSORT) &&
- mc->mc_ki[mc->mc_top] < page_numkeys(mp)) {
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- if (op == MDBX_PREV || op == MDBX_PREV_DUP) {
- rc =
- mdbx_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_PREV);
- if (op != MDBX_PREV || rc != MDBX_NOTFOUND) {
- if (likely(rc == MDBX_SUCCESS)) {
- get_key_optional(node, key);
- mc->mc_flags &= ~C_EOF;
- }
- return rc;
- }
- }
- } else {
- mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
- if (op == MDBX_PREV_DUP)
- return MDBX_NOTFOUND;
- }
+ /* Adjust free space offsets. */
+ const size_t branch_bytes = branch_size(mc->mc_txn->mt_env, key);
+ const intptr_t lower = mp->mp_lower + sizeof(indx_t);
+ const intptr_t upper = mp->mp_upper - (branch_bytes - sizeof(indx_t));
+ if (unlikely(lower > upper)) {
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_PAGE_FULL;
}
+ mp->mp_lower = (indx_t)lower;
+ mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper;
- mdbx_debug("cursor_prev: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno,
- (void *)mc);
+ /* Write the node data. */
+ MDBX_node *node = page_node(mp, indx);
+ node_set_pgno(node, pgno);
+ node_set_flags(node, 0);
+ UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0);
+ node_set_ks(node, 0);
+ if (likely(key != NULL)) {
+ node_set_ks(node, key->iov_len);
+ memcpy(node_key(node), key->iov_base, key->iov_len);
+ }
+ return MDBX_SUCCESS;
+}
- mc->mc_flags &= ~(C_EOF | C_DEL);
+static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc,
+ unsigned indx,
+ const MDBX_val *key,
+ MDBX_val *data,
+ unsigned flags) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ DKBUF_DEBUG;
+ mdbx_debug("add to leaf-%spage %" PRIaPGNO " index %i, data size %" PRIuPTR
+ " key size %" PRIuPTR " [%s]",
+ IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx,
+ data ? data->iov_len : 0, key ? key->iov_len : 0, DKEY_DEBUG(key));
+ mdbx_cassert(mc, key != NULL && data != NULL);
+ mdbx_cassert(mc, PAGETYPE(mp) == P_LEAF);
+ mdbx_cassert(mc, page_room(mp) >= leaf_size(mc->mc_txn->mt_env, key, data));
+ MDBX_page *largepage = NULL;
- int ki = mc->mc_ki[mc->mc_top];
- mc->mc_ki[mc->mc_top] = (indx_t)--ki;
- if (unlikely(ki < 0)) {
- mc->mc_ki[mc->mc_top] = 0;
- mdbx_debug("%s", "=====> move to prev sibling page");
- if ((rc = mdbx_cursor_sibling(mc, SIBLING_LEFT)) != MDBX_SUCCESS)
- return rc;
- mp = mc->mc_pg[mc->mc_top];
- mdbx_debug("prev page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
- mc->mc_ki[mc->mc_top]);
+ size_t node_bytes;
+ if (unlikely(flags & F_BIGDATA)) {
+ /* Data already on overflow page. */
+ STATIC_ASSERT(sizeof(pgno_t) % 2 == 0);
+ node_bytes =
+ node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
+ } else if (unlikely(node_size(key, data) >
+ mc->mc_txn->mt_env->me_leaf_nodemax)) {
+ /* Put data on overflow page. */
+ if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT)) {
+ mdbx_error("Unexpected target %s flags 0x%x for large data-item",
+ "dupsort-db", mc->mc_db->md_flags);
+ return MDBX_PROBLEM;
+ }
+ if (unlikely(flags & (F_DUPDATA | F_SUBDATA))) {
+ mdbx_error("Unexpected target %s flags 0x%x for large data-item", "node",
+ flags);
+ return MDBX_PROBLEM;
+ }
+ const pgno_t ovpages = number_of_ovpages(mc->mc_txn->mt_env, data->iov_len);
+ const struct page_result npr = mdbx_page_new(mc, P_OVERFLOW, ovpages);
+ if (unlikely(npr.err != MDBX_SUCCESS))
+ return npr.err;
+ largepage = npr.page;
+ mdbx_debug("allocated %u overflow page(s) %" PRIaPGNO "for %" PRIuPTR
+ " data bytes",
+ largepage->mp_pages, largepage->mp_pgno, data->iov_len);
+ flags |= F_BIGDATA;
+ node_bytes =
+ node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
+ } else {
+ node_bytes = node_size(key, data) + sizeof(indx_t);
}
- mdbx_debug("==> cursor points to page %" PRIaPGNO
- " with %u keys, key index %u",
- mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]);
+ mdbx_cassert(mc, node_bytes == leaf_size(mc->mc_txn->mt_env, key, data));
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
- return MDBX_CORRUPTED;
+ /* Move higher pointers up one slot. */
+ const unsigned nkeys = page_numkeys(mp);
+ mdbx_cassert(mc, nkeys >= indx);
+ for (unsigned i = nkeys; i > indx; --i)
+ mp->mp_ptrs[i] = mp->mp_ptrs[i - 1];
- if (IS_LEAF2(mp)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mp->mp_pgno);
- return MDBX_CORRUPTED;
- } else if (likely(key)) {
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
- }
- return MDBX_SUCCESS;
+ /* Adjust free space offsets. */
+ const intptr_t lower = mp->mp_lower + sizeof(indx_t);
+ const intptr_t upper = mp->mp_upper - (node_bytes - sizeof(indx_t));
+ if (unlikely(lower > upper)) {
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_PAGE_FULL;
}
+ mp->mp_lower = (indx_t)lower;
+ mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper;
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ /* Write the node data. */
+ MDBX_node *node = page_node(mp, indx);
+ node_set_ks(node, key->iov_len);
+ node_set_flags(node, (uint8_t)flags);
+ UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0);
+ node_set_ds(node, data->iov_len);
+ memcpy(node_key(node), key->iov_base, key->iov_len);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- rc = mdbx_xcursor_init1(mc, node, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- } else if (likely(data)) {
- if (unlikely((rc = mdbx_node_read(mc, node, data,
- pp_txnid4chk(mp, mc->mc_txn))) !=
- MDBX_SUCCESS))
- return rc;
+ void *nodedata = node_data(node);
+ if (likely(largepage == NULL)) {
+ if (unlikely(flags & F_BIGDATA))
+ memcpy(nodedata, data->iov_base, sizeof(pgno_t));
+ else if (unlikely(flags & MDBX_RESERVE))
+ data->iov_base = nodedata;
+ else if (likely(nodedata != data->iov_base &&
+ data->iov_len /* to avoid UBSAN traps*/ != 0))
+ memcpy(nodedata, data->iov_base, data->iov_len);
+ } else {
+ poke_pgno(nodedata, largepage->mp_pgno);
+ nodedata = page_data(largepage);
+ if (unlikely(flags & MDBX_RESERVE))
+ data->iov_base = nodedata;
+ else if (likely(nodedata != data->iov_base &&
+ data->iov_len /* to avoid UBSAN traps*/ != 0))
+ memcpy(nodedata, data->iov_base, data->iov_len);
}
-
- get_key_optional(node, key);
return MDBX_SUCCESS;
}
-/* Set the cursor on a specific data item. */
-static struct cursor_set_result mdbx_cursor_set(MDBX_cursor *mc, MDBX_val *key,
- MDBX_val *data,
- MDBX_cursor_op op) {
- MDBX_page *mp;
- MDBX_node *node = NULL;
- DKBUF_DEBUG;
+/* Delete the specified node from a page.
+ * [in] mc Cursor pointing to the node to delete.
+ * [in] ksize The size of a node. Only used if the page is
+ * part of a MDBX_DUPFIXED database. */
+static void mdbx_node_del(MDBX_cursor *mc, size_t ksize) {
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ int indx = mc->mc_ki[mc->mc_top];
+ int i, j, nkeys, ptr;
+ MDBX_node *node;
+ char *base;
- struct cursor_set_result ret;
- ret.exact = false;
- if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min ||
- key->iov_len > mc->mc_dbx->md_klen_max)) {
- mdbx_cassert(mc, !"Invalid key-size");
- ret.err = MDBX_BAD_VALSIZE;
- return ret;
+ mdbx_debug("delete node %u on %s page %" PRIaPGNO, indx,
+ IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno);
+ nkeys = page_numkeys(mp);
+ mdbx_cassert(mc, indx < nkeys);
+
+ if (IS_LEAF2(mp)) {
+ mdbx_cassert(mc, ksize >= sizeof(indx_t));
+ unsigned diff = nkeys - 1 - indx;
+ base = page_leaf2key(mp, indx, ksize);
+ if (diff)
+ memmove(base, base + ksize, diff * ksize);
+ mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t));
+ mp->mp_lower -= sizeof(indx_t);
+ mdbx_cassert(mc,
+ (size_t)UINT16_MAX - mp->mp_upper >= ksize - sizeof(indx_t));
+ mp->mp_upper += (indx_t)(ksize - sizeof(indx_t));
+ return;
}
- MDBX_val aligned_key = *key;
- uint64_t aligned_keybytes;
- if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
- switch (aligned_key.iov_len) {
- default:
- mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY");
- ret.err = MDBX_BAD_VALSIZE;
- return ret;
- case 4:
- if (unlikely(3 & (uintptr_t)aligned_key.iov_base))
- /* copy instead of return error to avoid break compatibility */
- aligned_key.iov_base =
- memcpy(&aligned_keybytes, aligned_key.iov_base, 4);
- break;
- case 8:
- if (unlikely(7 & (uintptr_t)aligned_key.iov_base))
- /* copy instead of return error to avoid break compatibility */
- aligned_key.iov_base =
- memcpy(&aligned_keybytes, aligned_key.iov_base, 8);
- break;
+ node = page_node(mp, indx);
+ mdbx_cassert(mc, !IS_BRANCH(mp) || indx || node_ks(node) == 0);
+ size_t sz = NODESIZE + node_ks(node);
+ if (IS_LEAF(mp)) {
+ if (F_ISSET(node_flags(node), F_BIGDATA))
+ sz += sizeof(pgno_t);
+ else
+ sz += node_ds(node);
+ }
+ sz = EVEN(sz);
+
+ ptr = mp->mp_ptrs[indx];
+ for (i = j = 0; i < nkeys; i++) {
+ if (i != indx) {
+ mp->mp_ptrs[j] = mp->mp_ptrs[i];
+ if (mp->mp_ptrs[i] < ptr) {
+ mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_ptrs[j] >= sz);
+ mp->mp_ptrs[j] += (indx_t)sz;
+ }
+ j++;
}
}
- if (mc->mc_xcursor)
- mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+ base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
+ memmove(base + sz, base, ptr - mp->mp_upper);
- /* See if we're already on the right page */
- if (mc->mc_flags & C_INITIALIZED) {
- MDBX_val nodekey;
+ mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t));
+ mp->mp_lower -= sizeof(indx_t);
+ mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_upper >= sz);
+ mp->mp_upper += (indx_t)sz;
- mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
- mp = mc->mc_pg[mc->mc_top];
- if (unlikely(!page_numkeys(mp))) {
- mc->mc_ki[mc->mc_top] = 0;
- mc->mc_flags |= C_EOF;
- ret.err = MDBX_NOTFOUND;
- return ret;
- }
- if (IS_LEAF2(mp)) {
- nodekey.iov_len = mc->mc_db->md_xsize;
- nodekey.iov_base = page_leaf2key(mp, 0, nodekey.iov_len);
- } else {
- node = page_node(mp, 0);
- get_key(node, &nodekey);
- }
- int cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
- if (unlikely(cmp == 0)) {
- /* Probably happens rarely, but first node on the page
- * was the one we wanted. */
- mc->mc_ki[mc->mc_top] = 0;
- ret.exact = true;
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- goto got_node;
- }
- if (cmp > 0) {
- const unsigned nkeys = page_numkeys(mp);
- if (nkeys > 1) {
- if (IS_LEAF2(mp)) {
- nodekey.iov_base = page_leaf2key(mp, nkeys - 1, nodekey.iov_len);
- } else {
- node = page_node(mp, nkeys - 1);
- get_key(node, &nodekey);
- }
- cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
- if (cmp == 0) {
- /* last node was the one we wanted */
- mdbx_cassert(mc, nkeys >= 1 && nkeys <= UINT16_MAX + 1);
- mc->mc_ki[mc->mc_top] = (indx_t)(nkeys - 1);
- ret.exact = true;
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- goto got_node;
- }
- if (cmp < 0) {
- if (mc->mc_ki[mc->mc_top] < page_numkeys(mp)) {
- /* This is definitely the right page, skip search_page */
- if (IS_LEAF2(mp)) {
- nodekey.iov_base =
- page_leaf2key(mp, mc->mc_ki[mc->mc_top], nodekey.iov_len);
- } else {
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
- get_key(node, &nodekey);
- }
- cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey);
- if (cmp == 0) {
- /* current node was the one we wanted */
- ret.exact = true;
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- goto got_node;
- }
- }
- mc->mc_flags &= ~C_EOF;
- goto search_node;
- }
- }
- /* If any parents have right-sibs, search.
- * Otherwise, there's nothing further. */
- unsigned i;
- for (i = 0; i < mc->mc_top; i++)
- if (mc->mc_ki[i] < page_numkeys(mc->mc_pg[i]) - 1)
- break;
- if (i == mc->mc_top) {
- /* There are no other pages */
- mdbx_cassert(mc, nkeys <= UINT16_MAX);
- mc->mc_ki[mc->mc_top] = (uint16_t)nkeys;
- mc->mc_flags |= C_EOF;
- ret.err = MDBX_NOTFOUND;
- return ret;
- }
- }
- if (!mc->mc_top) {
- /* There are no other pages */
- mc->mc_ki[mc->mc_top] = 0;
- if (op == MDBX_SET_RANGE)
- goto got_node;
+#if MDBX_DEBUG > 0
+ if (mdbx_audit_enabled()) {
+ int page_check_err = mdbx_page_check(mc, mp, C_UPDATING);
+ mdbx_cassert(mc, page_check_err == MDBX_SUCCESS);
+ }
+#endif
+}
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- ret.err = MDBX_NOTFOUND;
- return ret;
- }
+/* Compact the main page after deleting a node on a subpage.
+ * [in] mp The main page to operate on.
+ * [in] indx The index of the subpage on the main page. */
+static void mdbx_node_shrink(MDBX_page *mp, unsigned indx) {
+ MDBX_node *node;
+ MDBX_page *sp, *xp;
+ char *base;
+ size_t nsize, delta, len, ptr;
+ int i;
+
+ node = page_node(mp, indx);
+ sp = (MDBX_page *)node_data(node);
+ delta = page_room(sp);
+ assert(delta > 0);
+
+ /* Prepare to shift upward, set len = length(subpage part to shift) */
+ if (IS_LEAF2(sp)) {
+ delta &= /* do not make the node uneven-sized */ ~(size_t)1;
+ if (unlikely(delta) == 0)
+ return;
+ nsize = node_ds(node) - delta;
+ assert(nsize % 1 == 0);
+ len = nsize;
} else {
- mc->mc_pg[0] = 0;
+ xp = (MDBX_page *)((char *)sp + delta); /* destination subpage */
+ for (i = page_numkeys(sp); --i >= 0;) {
+ assert(sp->mp_ptrs[i] >= delta);
+ xp->mp_ptrs[i] = (indx_t)(sp->mp_ptrs[i] - delta);
+ }
+ nsize = node_ds(node) - delta;
+ len = PAGEHDRSZ;
}
+ sp->mp_upper = sp->mp_lower;
+ sp->mp_pgno = mp->mp_pgno;
+ node_set_ds(node, nsize);
- ret.err = mdbx_page_search(mc, &aligned_key, 0);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
-
- mp = mc->mc_pg[mc->mc_top];
- mdbx_cassert(mc, IS_LEAF(mp));
+ /* Shift <lower nodes...initial part of subpage> upward */
+ base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
+ memmove(base + delta, base, (char *)sp + len - base);
-search_node:;
- struct node_result nsr = mdbx_node_search(mc, &aligned_key);
- node = nsr.node;
- ret.exact = nsr.exact;
- if (!ret.exact) {
- if (op != MDBX_SET_RANGE) {
- /* MDBX_SET specified and not an exact match. */
- if (unlikely(mc->mc_ki[mc->mc_top] >=
- page_numkeys(mc->mc_pg[mc->mc_top])))
- mc->mc_flags |= C_EOF;
- ret.err = MDBX_NOTFOUND;
- return ret;
+ ptr = mp->mp_ptrs[indx];
+ for (i = page_numkeys(mp); --i >= 0;) {
+ if (mp->mp_ptrs[i] <= ptr) {
+ assert((size_t)UINT16_MAX - mp->mp_ptrs[i] >= delta);
+ mp->mp_ptrs[i] += (indx_t)delta;
}
+ }
+ assert((size_t)UINT16_MAX - mp->mp_upper >= delta);
+ mp->mp_upper += (indx_t)delta;
+}
- if (node == NULL) {
- mdbx_debug("%s", "===> inexact leaf not found, goto sibling");
- ret.err = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
- if (unlikely(ret.err != MDBX_SUCCESS)) {
- mc->mc_flags |= C_EOF;
- return ret; /* no entries matched */
- }
- mp = mc->mc_pg[mc->mc_top];
- mdbx_cassert(mc, IS_LEAF(mp));
- if (!IS_LEAF2(mp))
- node = page_node(mp, 0);
- }
+/* Initial setup of a sorted-dups cursor.
+ *
+ * Sorted duplicates are implemented as a sub-database for the given key.
+ * The duplicate data items are actually keys of the sub-database.
+ * Operations on the duplicate data items are performed using a sub-cursor
+ * initialized when the sub-database is first accessed. This function does
+ * the preliminary setup of the sub-cursor, filling in the fields that
+ * depend only on the parent DB.
+ *
+ * [in] mc The main cursor whose sorted-dups cursor is to be initialized. */
+static int mdbx_xcursor_init0(MDBX_cursor *mc) {
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
+ mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
+ mc->mc_dbi);
+ return MDBX_CORRUPTED;
}
- mdbx_cassert(mc,
- mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
-got_node:
- mc->mc_flags |= C_INITIALIZED;
- mc->mc_flags &= ~C_EOF;
+ mx->mx_cursor.mc_xcursor = NULL;
+ mx->mx_cursor.mc_next = NULL;
+ mx->mx_cursor.mc_txn = mc->mc_txn;
+ mx->mx_cursor.mc_db = &mx->mx_db;
+ mx->mx_cursor.mc_dbx = &mx->mx_dbx;
+ mx->mx_cursor.mc_dbi = mc->mc_dbi;
+ mx->mx_cursor.mc_dbistate = mc->mc_dbistate;
+ mx->mx_cursor.mc_snum = 0;
+ mx->mx_cursor.mc_top = 0;
+ mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
+ mx->mx_dbx.md_name.iov_len = 0;
+ mx->mx_dbx.md_name.iov_base = NULL;
+ mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp;
+ mx->mx_dbx.md_dcmp = NULL;
+ mx->mx_dbx.md_klen_min = INT_MAX;
+ mx->mx_dbx.md_vlen_min = mx->mx_dbx.md_klen_max = mx->mx_dbx.md_vlen_max = 0;
+ return MDBX_SUCCESS;
+}
- if (IS_LEAF2(mp)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mp->mp_pgno);
- ret.err = MDBX_CORRUPTED;
- } else {
- if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) {
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
- }
- ret.err = MDBX_SUCCESS;
+/* Final setup of a sorted-dups cursor.
+ * Sets up the fields that depend on the data from the main cursor.
+ * [in] mc The main cursor whose sorted-dups cursor is to be initialized.
+ * [in] node The data containing the MDBX_db record for the sorted-dup database.
+ */
+static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node,
+ const MDBX_page *mp) {
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
+ mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
+ mc->mc_dbi);
+ return MDBX_CORRUPTED;
+ }
+
+ const uint8_t flags = node_flags(node);
+ switch (flags) {
+ default:
+ mdbx_error("invalid node flags %u", flags);
+ return MDBX_CORRUPTED;
+ case F_DUPDATA | F_SUBDATA:
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(node_ds(node) != sizeof(MDBX_db))) {
+ mdbx_error("invalid nested-db record size %zu", node_ds(node));
+ return MDBX_CORRUPTED;
}
- return ret;
+ memcpy(&mx->mx_db, node_data(node), sizeof(MDBX_db));
+ const txnid_t pp_txnid = mp->mp_txnid;
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(mx->mx_db.md_mod_txnid > pp_txnid)) {
+ mdbx_error("nested-db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN
+ ")",
+ mx->mx_db.md_mod_txnid, pp_txnid);
+ return MDBX_CORRUPTED;
+ }
+ mx->mx_cursor.mc_pg[0] = 0;
+ mx->mx_cursor.mc_snum = 0;
+ mx->mx_cursor.mc_top = 0;
+ mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
+ break;
+ case F_DUPDATA:
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(node_ds(node) <= PAGEHDRSZ)) {
+ mdbx_error("invalid nested-page size %zu", node_ds(node));
+ return MDBX_CORRUPTED;
+ }
+ MDBX_page *fp = node_data(node);
+ mx->mx_db.md_depth = 1;
+ mx->mx_db.md_branch_pages = 0;
+ mx->mx_db.md_leaf_pages = 1;
+ mx->mx_db.md_overflow_pages = 0;
+ mx->mx_db.md_entries = page_numkeys(fp);
+ mx->mx_db.md_root = fp->mp_pgno;
+ mx->mx_db.md_mod_txnid = mp->mp_txnid;
+ mx->mx_cursor.mc_snum = 1;
+ mx->mx_cursor.mc_top = 0;
+ mx->mx_cursor.mc_flags =
+ C_INITIALIZED | C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
+ mx->mx_cursor.mc_pg[0] = fp;
+ mx->mx_cursor.mc_ki[0] = 0;
+ mx->mx_db.md_flags = flags_db2sub(mc->mc_db->md_flags);
+ mx->mx_db.md_xsize =
+ (mc->mc_db->md_flags & MDBX_DUPFIXED) ? fp->mp_leaf2_ksize : 0;
+ break;
}
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- ret.err = mdbx_xcursor_init1(mc, node, mp);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
- if (op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_SET_RANGE) {
- ret.err = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
- } else {
- ret = mdbx_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL,
- MDBX_SET_RANGE);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
- if (op == MDBX_GET_BOTH && !ret.exact) {
- ret.err = MDBX_NOTFOUND;
- return ret;
- }
+ if (unlikely(mx->mx_db.md_xsize != mc->mc_db->md_xsize)) {
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(mc->mc_db->md_xsize != 0)) {
+ mdbx_error("cursor mismatched nested-db md_xsize %u",
+ mc->mc_db->md_xsize);
+ return MDBX_CORRUPTED;
}
- } else if (likely(data)) {
- if (op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE) {
- if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min ||
- data->iov_len > mc->mc_dbx->md_vlen_max)) {
- mdbx_cassert(mc, !"Invalid data-size");
- ret.err = MDBX_BAD_VALSIZE;
- return ret;
- }
- MDBX_val aligned_data = *data;
- uint64_t aligned_databytes;
- if (mc->mc_db->md_flags & MDBX_INTEGERDUP) {
- switch (aligned_data.iov_len) {
- default:
- mdbx_cassert(mc, !"data-size is invalid for MDBX_INTEGERDUP");
- ret.err = MDBX_BAD_VALSIZE;
- return ret;
- case 4:
- if (unlikely(3 & (uintptr_t)aligned_data.iov_base))
- /* copy instead of return error to avoid break compatibility */
- aligned_data.iov_base =
- memcpy(&aligned_databytes, aligned_data.iov_base, 4);
- break;
- case 8:
- if (unlikely(7 & (uintptr_t)aligned_data.iov_base))
- /* copy instead of return error to avoid break compatibility */
- aligned_data.iov_base =
- memcpy(&aligned_databytes, aligned_data.iov_base, 8);
- break;
- }
- }
- MDBX_val actual_data;
- ret.err = mdbx_node_read(mc, node, &actual_data,
- pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
- const int cmp = mc->mc_dbx->md_dcmp(&aligned_data, &actual_data);
- if (cmp) {
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- if (op != MDBX_GET_BOTH_RANGE || cmp > 0) {
- ret.err = MDBX_NOTFOUND;
- return ret;
- }
- }
- *data = actual_data;
- } else {
- ret.err = mdbx_node_read(mc, node, data,
- pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) {
+ mdbx_error("mismatched nested-db md_flags %u", mc->mc_db->md_flags);
+ return MDBX_CORRUPTED;
+ }
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(mx->mx_db.md_xsize < mc->mc_dbx->md_vlen_min ||
+ mx->mx_db.md_xsize > mc->mc_dbx->md_vlen_max)) {
+ mdbx_error("mismatched nested-db.md_xsize (%u) <> min/max value-length "
+ "(%zu/%zu)",
+ mx->mx_db.md_xsize, mc->mc_dbx->md_vlen_min,
+ mc->mc_dbx->md_vlen_max);
+ return MDBX_CORRUPTED;
}
+ mc->mc_db->md_xsize = mx->mx_db.md_xsize;
+ mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = mx->mx_db.md_xsize;
}
+ mx->mx_dbx.md_klen_min = mc->mc_dbx->md_vlen_min;
+ mx->mx_dbx.md_klen_max = mc->mc_dbx->md_vlen_max;
- /* The key already matches in all other cases */
- if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY)
- get_key_optional(node, key);
-
- mdbx_debug("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key),
- DVAL_DEBUG(data));
- ret.err = MDBX_SUCCESS;
- return ret;
+ mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi,
+ mx->mx_db.md_root);
+ return MDBX_SUCCESS;
}
-/* Move the cursor to the first item in the database. */
-static int mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
- int rc;
-
- if (mc->mc_xcursor)
- mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+/* Fixup a sorted-dups cursor due to underlying update.
+ * Sets up some fields that depend on the data from the main cursor.
+ * Almost the same as init1, but skips initialization steps if the
+ * xcursor had already been used.
+ * [in] mc The main cursor whose sorted-dups cursor is to be fixed up.
+ * [in] src_mx The xcursor of an up-to-date cursor.
+ * [in] new_dupdata True if converting from a non-F_DUPDATA item. */
+static int mdbx_xcursor_init2(MDBX_cursor *mc, MDBX_xcursor *src_mx,
+ bool new_dupdata) {
+ MDBX_xcursor *mx = mc->mc_xcursor;
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
+ mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
+ mc->mc_dbi);
+ return MDBX_CORRUPTED;
+ }
- if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
- rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (new_dupdata) {
+ mx->mx_cursor.mc_snum = 1;
+ mx->mx_cursor.mc_top = 0;
+ mx->mx_cursor.mc_flags |= C_INITIALIZED;
+ mx->mx_cursor.mc_ki[0] = 0;
}
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top])))
- return MDBX_CORRUPTED;
+ mx->mx_dbx.md_klen_min = src_mx->mx_dbx.md_klen_min;
+ mx->mx_dbx.md_klen_max = src_mx->mx_dbx.md_klen_max;
+ mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
+ mx->mx_db = src_mx->mx_db;
+ mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
+ if (mx->mx_cursor.mc_flags & C_INITIALIZED) {
+ mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi,
+ mx->mx_db.md_root);
+ }
+ return MDBX_SUCCESS;
+}
- mc->mc_flags |= C_INITIALIZED;
- mc->mc_flags &= ~C_EOF;
- mc->mc_ki[mc->mc_top] = 0;
+static __inline int mdbx_couple_init(MDBX_cursor_couple *couple,
+ const MDBX_dbi dbi, MDBX_txn *const txn,
+ MDBX_db *const db, MDBX_dbx *const dbx,
+ uint8_t *const dbstate) {
+ couple->outer.mc_signature = MDBX_MC_LIVE;
+ couple->outer.mc_next = NULL;
+ couple->outer.mc_backup = NULL;
+ couple->outer.mc_dbi = dbi;
+ couple->outer.mc_txn = txn;
+ couple->outer.mc_db = db;
+ couple->outer.mc_dbx = dbx;
+ couple->outer.mc_dbistate = dbstate;
+ couple->outer.mc_snum = 0;
+ couple->outer.mc_top = 0;
+ couple->outer.mc_pg[0] = 0;
+ couple->outer.mc_flags = 0;
+ couple->outer.mc_ki[0] = 0;
+ couple->outer.mc_xcursor = NULL;
- if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mc->mc_pg[mc->mc_top]->mp_pgno);
- return MDBX_CORRUPTED;
- } else if (likely(key)) {
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top], 0, key->iov_len);
- }
- return MDBX_SUCCESS;
+ int rc = MDBX_SUCCESS;
+ if (unlikely(*couple->outer.mc_dbistate & DBI_STALE)) {
+ rc = mdbx_page_search(&couple->outer, NULL, MDBX_PS_ROOTONLY);
+ rc = (rc != MDBX_NOTFOUND) ? rc : MDBX_SUCCESS;
+ } else if (unlikely(couple->outer.mc_dbx->md_klen_max == 0)) {
+ rc = mdbx_setup_dbx(couple->outer.mc_dbx, couple->outer.mc_db,
+ txn->mt_env->me_psize);
}
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], 0);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
+ if (couple->outer.mc_db->md_flags & MDBX_DUPSORT) {
+ couple->inner.mx_cursor.mc_signature = MDBX_MC_LIVE;
+ couple->outer.mc_xcursor = &couple->inner;
+ rc = mdbx_xcursor_init0(&couple->outer);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(rc))
- return rc;
- } else if (likely(data)) {
- if (unlikely((rc = mdbx_node_read(
- mc, node, data,
- pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) !=
- MDBX_SUCCESS))
- return rc;
+ couple->inner.mx_dbx.md_klen_min = couple->outer.mc_dbx->md_vlen_min;
+ couple->inner.mx_dbx.md_klen_max = couple->outer.mc_dbx->md_vlen_max;
}
+ return rc;
+}
- get_key_optional(node, key);
+/* Initialize a cursor for a given transaction and database. */
+static int mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, MDBX_dbi dbi) {
+ STATIC_ASSERT(offsetof(MDBX_cursor_couple, outer) == 0);
+ if (unlikely(TXN_DBI_CHANGED(txn, dbi)))
+ return MDBX_BAD_DBI;
+
+ return mdbx_couple_init(container_of(mc, MDBX_cursor_couple, outer), dbi, txn,
+ &txn->mt_dbs[dbi], &txn->mt_dbxs[dbi],
+ &txn->mt_dbistate[dbi]);
+}
+
+MDBX_cursor *mdbx_cursor_create(void *context) {
+ MDBX_cursor_couple *couple = mdbx_calloc(1, sizeof(MDBX_cursor_couple));
+ if (unlikely(!couple))
+ return nullptr;
+
+ couple->outer.mc_signature = MDBX_MC_READY4CLOSE;
+ couple->outer.mc_dbi = UINT_MAX;
+ couple->mc_userctx = context;
+ return &couple->outer;
+}
+
+int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) {
+ if (unlikely(!mc))
+ return MDBX_EINVAL;
+
+ if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
+ mc->mc_signature != MDBX_MC_LIVE))
+ return MDBX_EBADSIGN;
+
+ MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer);
+ couple->mc_userctx = ctx;
return MDBX_SUCCESS;
}
-/* Move the cursor to the last item in the database. */
-static int mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
- int rc;
+void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) {
+ if (unlikely(!mc))
+ return nullptr;
- if (mc->mc_xcursor)
- mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+ if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
+ mc->mc_signature != MDBX_MC_LIVE))
+ return nullptr;
- if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
- rc = mdbx_page_search(mc, NULL, MDBX_PS_LAST);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
+ MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer);
+ return couple->mc_userctx;
+}
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top])))
- return MDBX_CORRUPTED;
+int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) {
+ if (unlikely(!mc))
+ return MDBX_EINVAL;
- mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]) - 1;
- mc->mc_flags |= C_INITIALIZED | C_EOF;
+ if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
+ mc->mc_signature != MDBX_MC_LIVE))
+ return MDBX_EBADSIGN;
- if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mc->mc_pg[mc->mc_top]->mp_pgno);
- return MDBX_CORRUPTED;
- } else if (likely(key)) {
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top],
- mc->mc_ki[mc->mc_top], key->iov_len);
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
+ return MDBX_BAD_DBI;
+
+ if (unlikely(dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
+ return MDBX_EACCESS;
+
+ if (unlikely(mc->mc_backup)) /* Cursor from parent transaction */ {
+ mdbx_cassert(mc, mc->mc_signature == MDBX_MC_LIVE);
+ if (unlikely(mc->mc_dbi != dbi ||
+ /* paranoia */ mc->mc_signature != MDBX_MC_LIVE ||
+ mc->mc_txn != txn))
+ return MDBX_EINVAL;
+
+ assert(mc->mc_db == &txn->mt_dbs[dbi]);
+ assert(mc->mc_dbx == &txn->mt_dbxs[dbi]);
+ assert(mc->mc_dbi == dbi);
+ assert(mc->mc_dbistate == &txn->mt_dbistate[dbi]);
+ return likely(mc->mc_dbi == dbi &&
+ /* paranoia */ mc->mc_signature == MDBX_MC_LIVE &&
+ mc->mc_txn == txn)
+ ? MDBX_SUCCESS
+ : MDBX_EINVAL /* Disallow change DBI in nested transactions */;
+ }
+
+ if (mc->mc_signature == MDBX_MC_LIVE) {
+ if (unlikely(!mc->mc_txn ||
+ mc->mc_txn->mt_signature != MDBX_MT_SIGNATURE)) {
+ mdbx_error("Wrong cursor's transaction %p 0x%x",
+ __Wpedantic_format_voidptr(mc->mc_txn),
+ mc->mc_txn ? mc->mc_txn->mt_signature : 0);
+ return MDBX_PROBLEM;
}
- return MDBX_SUCCESS;
+ if (mc->mc_flags & C_UNTRACK) {
+ MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi];
+ while (*prev && *prev != mc)
+ prev = &(*prev)->mc_next;
+ mdbx_cassert(mc, *prev == mc);
+ *prev = mc->mc_next;
+ }
+ mc->mc_signature = MDBX_MC_READY4CLOSE;
+ mc->mc_flags = 0;
+ mc->mc_dbi = UINT_MAX;
+ mc->mc_next = NULL;
+ mc->mc_db = NULL;
+ mc->mc_dbx = NULL;
+ mc->mc_dbistate = NULL;
}
+ mdbx_cassert(mc, !(mc->mc_flags & C_UNTRACK));
+
+ rc = mdbx_cursor_init(mc, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ mc->mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = mc;
+ mc->mc_flags |= C_UNTRACK;
+
+ return MDBX_SUCCESS;
+}
+
+int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) {
+ if (unlikely(!ret))
+ return MDBX_EINVAL;
+ *ret = NULL;
+
+ MDBX_cursor *const mc = mdbx_cursor_create(nullptr);
+ if (unlikely(!mc))
+ return MDBX_ENOMEM;
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(rc))
- return rc;
- } else if (likely(data)) {
- if (unlikely((rc = mdbx_node_read(
- mc, node, data,
- pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) !=
- MDBX_SUCCESS))
- return rc;
+ int rc = mdbx_cursor_bind(txn, mc, dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_cursor_close(mc);
+ return rc;
}
- get_key_optional(node, key);
+ *ret = mc;
return MDBX_SUCCESS;
}
-int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
- MDBX_cursor_op op) {
- if (unlikely(mc == NULL))
- return MDBX_EINVAL;
+int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) {
+ return likely(mc) ? mdbx_cursor_bind(txn, mc, mc->mc_dbi) : MDBX_EINVAL;
+}
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) {
+ if (unlikely(!src))
+ return MDBX_EINVAL;
+ if (unlikely(src->mc_signature != MDBX_MC_LIVE))
+ return (src->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
+ int rc = mdbx_cursor_bind(src->mc_txn, dest, src->mc_dbi);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- int (*mfunc)(MDBX_cursor * mc, MDBX_val * key, MDBX_val * data);
- switch (op) {
- case MDBX_GET_CURRENT: {
- if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
- return MDBX_ENODATA;
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- const unsigned nkeys = page_numkeys(mp);
- if (unlikely(mc->mc_ki[mc->mc_top] >= nkeys)) {
- mdbx_cassert(mc, nkeys <= UINT16_MAX);
- if (mc->mc_flags & C_EOF)
- return MDBX_ENODATA;
- mc->mc_ki[mc->mc_top] = (uint16_t)nkeys;
- mc->mc_flags |= C_EOF;
- return MDBX_NOTFOUND;
- }
- mdbx_cassert(mc, nkeys > 0);
+ assert(dest->mc_db == src->mc_db);
+ assert(dest->mc_dbi == src->mc_dbi);
+ assert(dest->mc_dbx == src->mc_dbx);
+ assert(dest->mc_dbistate == src->mc_dbistate);
+again:
+ assert(dest->mc_txn == src->mc_txn);
+ dest->mc_flags ^= (dest->mc_flags ^ src->mc_flags) & ~C_UNTRACK;
+ dest->mc_top = src->mc_top;
+ dest->mc_snum = src->mc_snum;
+ for (unsigned i = 0; i < src->mc_snum; ++i) {
+ dest->mc_ki[i] = src->mc_ki[i];
+ dest->mc_pg[i] = src->mc_pg[i];
+ }
- rc = MDBX_SUCCESS;
- if (IS_LEAF2(mp)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mp->mp_pgno);
- return MDBX_CORRUPTED;
- }
- key->iov_len = mc->mc_db->md_xsize;
- key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len);
- } else {
- MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
- get_key_optional(node, key);
- if (data) {
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) {
- rc = mdbx_xcursor_init1(mc, node, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
- if (unlikely(rc))
- return rc;
- } else {
- rc = mdbx_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL,
- MDBX_GET_CURRENT);
- if (unlikely(rc))
- return rc;
- }
- } else {
- rc = mdbx_node_read(mc, node, data, pp_txnid4chk(mp, mc->mc_txn));
- if (unlikely(rc))
- return rc;
- }
- }
- }
- break;
+ if (src->mc_xcursor) {
+ dest->mc_xcursor->mx_db = src->mc_xcursor->mx_db;
+ dest->mc_xcursor->mx_dbx = src->mc_xcursor->mx_dbx;
+ src = &src->mc_xcursor->mx_cursor;
+ dest = &dest->mc_xcursor->mx_cursor;
+ goto again;
}
- case MDBX_GET_BOTH:
- case MDBX_GET_BOTH_RANGE:
- if (unlikely(data == NULL))
- return MDBX_EINVAL;
- if (unlikely(mc->mc_xcursor == NULL))
- return MDBX_INCOMPATIBLE;
- /* fall through */
- __fallthrough;
- case MDBX_SET:
- case MDBX_SET_KEY:
- case MDBX_SET_RANGE:
- if (unlikely(key == NULL))
- return MDBX_EINVAL;
- rc = mdbx_cursor_set(mc, key, data, op).err;
- if (mc->mc_flags & C_INITIALIZED) {
- mdbx_cassert(mc, mc->mc_snum > 0 && mc->mc_top < mc->mc_snum);
- mdbx_cassert(mc, mc->mc_ki[mc->mc_top] <
- page_numkeys(mc->mc_pg[mc->mc_top]) ||
- (mc->mc_flags & C_EOF));
- }
- break;
- case MDBX_GET_MULTIPLE:
- if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED)))
- return MDBX_EINVAL;
- if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED)))
- return MDBX_INCOMPATIBLE;
- rc = MDBX_SUCCESS;
- if ((mc->mc_xcursor->mx_cursor.mc_flags & (C_INITIALIZED | C_EOF)) !=
- C_INITIALIZED)
- break;
- goto fetchm;
- case MDBX_NEXT_MULTIPLE:
- if (unlikely(data == NULL))
- return MDBX_EINVAL;
- if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED)))
- return MDBX_INCOMPATIBLE;
- rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_DUP);
- if (rc == MDBX_SUCCESS) {
- if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
- MDBX_cursor *mx;
- fetchm:
- mx = &mc->mc_xcursor->mx_cursor;
- data->iov_len =
- page_numkeys(mx->mc_pg[mx->mc_top]) * mx->mc_db->md_xsize;
- data->iov_base = page_data(mx->mc_pg[mx->mc_top]);
- mx->mc_ki[mx->mc_top] = (indx_t)page_numkeys(mx->mc_pg[mx->mc_top]) - 1;
- } else {
- rc = MDBX_NOTFOUND;
- }
- }
- break;
- case MDBX_PREV_MULTIPLE:
- if (data == NULL)
- return MDBX_EINVAL;
- if (!(mc->mc_db->md_flags & MDBX_DUPFIXED))
- return MDBX_INCOMPATIBLE;
- rc = MDBX_SUCCESS;
- if (!(mc->mc_flags & C_INITIALIZED))
- rc = mdbx_cursor_last(mc, key, data);
- if (rc == MDBX_SUCCESS) {
- MDBX_cursor *mx = &mc->mc_xcursor->mx_cursor;
- if (mx->mc_flags & C_INITIALIZED) {
- rc = mdbx_cursor_sibling(mx, SIBLING_LEFT);
- if (rc == MDBX_SUCCESS)
- goto fetchm;
- } else {
- rc = MDBX_NOTFOUND;
- }
- }
- break;
- case MDBX_NEXT:
- case MDBX_NEXT_DUP:
- case MDBX_NEXT_NODUP:
- rc = mdbx_cursor_next(mc, key, data, op);
- break;
- case MDBX_PREV:
- case MDBX_PREV_DUP:
- case MDBX_PREV_NODUP:
- rc = mdbx_cursor_prev(mc, key, data, op);
- break;
- case MDBX_FIRST:
- rc = mdbx_cursor_first(mc, key, data);
- break;
- case MDBX_FIRST_DUP:
- mfunc = mdbx_cursor_first;
- move:
- if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED)))
- return MDBX_EINVAL;
- if (unlikely(mc->mc_xcursor == NULL))
- return MDBX_INCOMPATIBLE;
- if (mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])) {
- mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]);
- mc->mc_flags |= C_EOF;
- return MDBX_NOTFOUND;
- }
- {
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- if (!F_ISSET(node_flags(node), F_DUPDATA)) {
- get_key_optional(node, key);
- rc = mdbx_node_read(mc, node, data,
- pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
- break;
- }
- }
- if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)))
- return MDBX_EINVAL;
- rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL);
- break;
- case MDBX_LAST:
- rc = mdbx_cursor_last(mc, key, data);
- break;
- case MDBX_LAST_DUP:
- mfunc = mdbx_cursor_last;
- goto move;
- case MDBX_SET_UPPERBOUND: /* mostly same as MDBX_SET_LOWERBOUND */
- case MDBX_SET_LOWERBOUND: {
- if (unlikely(key == NULL || data == NULL))
- return MDBX_EINVAL;
- MDBX_val save_data = *data;
- struct cursor_set_result csr =
- mdbx_cursor_set(mc, key, data, MDBX_SET_RANGE);
- rc = csr.err;
- if (rc == MDBX_SUCCESS && csr.exact && mc->mc_xcursor) {
- mc->mc_flags &= ~C_DEL;
- csr.exact = false;
- if (!save_data.iov_base && (mc->mc_db->md_flags & MDBX_DUPFIXED)) {
- /* Avoiding search nested dupfixed hive if no data provided.
- * This is changes the semantic of MDBX_SET_LOWERBOUND but avoid
- * returning MDBX_BAD_VALSIZE. */
- } else if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
- *data = save_data;
- csr = mdbx_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL,
- MDBX_SET_RANGE);
- rc = csr.err;
- if (rc == MDBX_NOTFOUND) {
- mdbx_cassert(mc, !csr.exact);
- rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_NODUP);
- }
- } else {
- int cmp = mc->mc_dbx->md_dcmp(&save_data, data);
- csr.exact = (cmp == 0);
- if (cmp > 0)
- rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_NODUP);
+
+ return MDBX_SUCCESS;
+}
+
+void mdbx_cursor_close(MDBX_cursor *mc) {
+ if (likely(mc)) {
+ mdbx_ensure(NULL, mc->mc_signature == MDBX_MC_LIVE ||
+ mc->mc_signature == MDBX_MC_READY4CLOSE);
+ MDBX_txn *const txn = mc->mc_txn;
+ if (!mc->mc_backup) {
+ mc->mc_txn = NULL;
+ /* Unlink from txn, if tracked. */
+ if (mc->mc_flags & C_UNTRACK) {
+ mdbx_ensure(txn->mt_env, check_txn(txn, 0) == MDBX_SUCCESS);
+ MDBX_cursor **prev = &txn->mt_cursors[mc->mc_dbi];
+ while (*prev && *prev != mc)
+ prev = &(*prev)->mc_next;
+ mdbx_tassert(txn, *prev == mc);
+ *prev = mc->mc_next;
}
+ mc->mc_signature = 0;
+ mc->mc_next = mc;
+ mdbx_free(mc);
+ } else {
+ /* Cursor closed before nested txn ends */
+ mdbx_tassert(txn, mc->mc_signature == MDBX_MC_LIVE);
+ mdbx_ensure(txn->mt_env, check_txn_rw(txn, 0) == MDBX_SUCCESS);
+ mc->mc_signature = MDBX_MC_WAIT4EOT;
}
- if (rc == MDBX_SUCCESS && !csr.exact)
- rc = MDBX_RESULT_TRUE;
- if (unlikely(op == MDBX_SET_UPPERBOUND)) {
- /* minor fixups for MDBX_SET_UPPERBOUND */
- if (rc == MDBX_RESULT_TRUE)
- /* already at great-than by MDBX_SET_LOWERBOUND */
- rc = MDBX_SUCCESS;
- else if (rc == MDBX_SUCCESS)
- /* exactly match, going next */
- rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT);
- }
- break;
}
- default:
- mdbx_debug("unhandled/unimplemented cursor operation %u", op);
+}
+
+MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) {
+ if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE))
+ return NULL;
+ MDBX_txn *txn = mc->mc_txn;
+ if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
+ return NULL;
+ if (unlikely(txn->mt_flags & MDBX_TXN_FINISHED))
+ return NULL;
+ return txn;
+}
+
+MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) {
+ if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE))
+ return UINT_MAX;
+ return mc->mc_dbi;
+}
+
+/* Return the count of duplicate data items for the current key */
+int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) {
+ if (unlikely(mc == NULL))
+ return MDBX_EINVAL;
+
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
+
+ int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(countp == NULL || !(mc->mc_flags & C_INITIALIZED)))
return MDBX_EINVAL;
+
+ if (!mc->mc_snum) {
+ *countp = 0;
+ return MDBX_NOTFOUND;
}
- mc->mc_flags &= ~C_DEL;
- return rc;
+ MDBX_page *mp = mc->mc_pg[mc->mc_top];
+ if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= page_numkeys(mp)) {
+ *countp = 0;
+ return MDBX_NOTFOUND;
+ }
+
+ *countp = 1;
+ if (mc->mc_xcursor != NULL) {
+ MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ mdbx_cassert(mc, mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags &
+ C_INITIALIZED));
+ *countp = unlikely(mc->mc_xcursor->mx_db.md_entries > PTRDIFF_MAX)
+ ? PTRDIFF_MAX
+ : (size_t)mc->mc_xcursor->mx_db.md_entries;
+ }
+ }
+ return MDBX_SUCCESS;
}
-static int cursor_first_batch(MDBX_cursor *mc) {
- if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
- int err = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+/* Replace the key for a branch node with a new key.
+ * Set MDBX_TXN_ERROR on failure.
+ * [in] mc Cursor pointing to the node to operate on.
+ * [in] key The new key to use.
+ * Returns 0 on success, non-zero on failure. */
+static int mdbx_update_key(MDBX_cursor *mc, const MDBX_val *key) {
+ MDBX_page *mp;
+ MDBX_node *node;
+ char *base;
+ size_t len;
+ int delta, ksize, oksize;
+ int ptr, i, nkeys, indx;
+ DKBUF_DEBUG;
+
+ mdbx_cassert(mc, cursor_is_tracked(mc));
+ indx = mc->mc_ki[mc->mc_top];
+ mp = mc->mc_pg[mc->mc_top];
+ node = page_node(mp, indx);
+ ptr = mp->mp_ptrs[indx];
+#if MDBX_DEBUG
+ MDBX_val k2;
+ k2.iov_base = node_key(node);
+ k2.iov_len = node_ks(node);
+ mdbx_debug("update key %u (offset %u) [%s] to [%s] on page %" PRIaPGNO, indx,
+ ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), mp->mp_pgno);
+#endif /* MDBX_DEBUG */
+
+ /* Sizes must be 2-byte aligned. */
+ ksize = EVEN(key->iov_len);
+ oksize = EVEN(node_ks(node));
+ delta = ksize - oksize;
+
+ /* Shift node contents if EVEN(key length) changed. */
+ if (delta) {
+ if (delta > (int)page_room(mp)) {
+ /* not enough space left, do a delete and split */
+ mdbx_debug("Not enough room, delta = %d, splitting...", delta);
+ pgno_t pgno = node_pgno(node);
+ mdbx_node_del(mc, 0);
+ int rc = mdbx_page_split(mc, key, NULL, pgno, MDBX_SPLIT_REPLACE);
+ if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ return rc;
+ }
+
+ nkeys = page_numkeys(mp);
+ for (i = 0; i < nkeys; i++) {
+ if (mp->mp_ptrs[i] <= ptr) {
+ mdbx_cassert(mc, mp->mp_ptrs[i] >= delta);
+ mp->mp_ptrs[i] -= (indx_t)delta;
+ }
+ }
+
+ base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
+ len = ptr - mp->mp_upper + NODESIZE;
+ memmove(base - delta, base, len);
+ mdbx_cassert(mc, mp->mp_upper >= delta);
+ mp->mp_upper -= (indx_t)delta;
+
+ node = page_node(mp, indx);
}
- mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
- mc->mc_flags |= C_INITIALIZED;
- mc->mc_flags &= ~C_EOF;
- mc->mc_ki[mc->mc_top] = 0;
+ /* But even if no shift was needed, update ksize */
+ node_set_ks(node, key->iov_len);
+
+ if (likely(key->iov_len /* to avoid UBSAN traps*/ != 0))
+ memcpy(node_key(node), key->iov_base, key->iov_len);
return MDBX_SUCCESS;
}
-static int cursor_next_batch(MDBX_cursor *mc) {
- if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
- return cursor_first_batch(mc);
+/* Move a node from csrc to cdst. */
+static int mdbx_node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
+ int rc;
+ DKBUF_DEBUG;
+
+ MDBX_page *psrc = csrc->mc_pg[csrc->mc_top];
+ MDBX_page *pdst = cdst->mc_pg[cdst->mc_top];
+ mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
+ mdbx_cassert(csrc, csrc->mc_dbi == cdst->mc_dbi);
+ mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top);
+ if (unlikely(PAGETYPE(psrc) != PAGETYPE(pdst))) {
+ bailout:
+ mdbx_error("Wrong or mismatch pages's types (src %d, dst %d) to move node",
+ PAGETYPE(psrc), PAGETYPE(pdst));
+ csrc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_PROBLEM;
+ }
+
+ MDBX_val key4move;
+ switch (PAGETYPE(psrc)) {
+ case P_BRANCH: {
+ const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]);
+ mdbx_cassert(csrc, node_flags(srcnode) == 0);
+ const pgno_t srcpg = node_pgno(srcnode);
+ key4move.iov_len = node_ks(srcnode);
+ key4move.iov_base = node_key(srcnode);
+
+ if (csrc->mc_ki[csrc->mc_top] == 0) {
+ const unsigned snum = csrc->mc_snum;
+ mdbx_cassert(csrc, snum > 0);
+ /* must find the lowest key below src */
+ rc = mdbx_page_search_lowest(csrc);
+ MDBX_page *lowest_page = csrc->mc_pg[csrc->mc_top];
+ if (unlikely(rc))
+ return rc;
+ mdbx_cassert(csrc, IS_LEAF(lowest_page));
+ if (unlikely(!IS_LEAF(lowest_page)))
+ goto bailout;
+ if (IS_LEAF2(lowest_page)) {
+ key4move.iov_len = csrc->mc_db->md_xsize;
+ key4move.iov_base = page_leaf2key(lowest_page, 0, key4move.iov_len);
+ } else {
+ const MDBX_node *lowest_node = page_node(lowest_page, 0);
+ key4move.iov_len = node_ks(lowest_node);
+ key4move.iov_base = node_key(lowest_node);
+ }
+
+ /* restore cursor after mdbx_page_search_lowest() */
+ csrc->mc_snum = snum;
+ csrc->mc_top = snum - 1;
+ csrc->mc_ki[csrc->mc_top] = 0;
+
+ /* paranoia */
+ mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
+ mdbx_cassert(csrc, IS_BRANCH(psrc));
+ if (unlikely(!IS_BRANCH(psrc)))
+ goto bailout;
+ }
+
+ if (cdst->mc_ki[cdst->mc_top] == 0) {
+ const unsigned snum = cdst->mc_snum;
+ mdbx_cassert(csrc, snum > 0);
+ MDBX_cursor mn;
+ cursor_copy(cdst, &mn);
+ /* must find the lowest key below dst */
+ rc = mdbx_page_search_lowest(&mn);
+ if (unlikely(rc))
+ return rc;
+ MDBX_page *const lowest_page = mn.mc_pg[mn.mc_top];
+ mdbx_cassert(cdst, IS_LEAF(lowest_page));
+ if (unlikely(!IS_LEAF(lowest_page)))
+ goto bailout;
+ MDBX_val key;
+ if (IS_LEAF2(lowest_page)) {
+ key.iov_len = mn.mc_db->md_xsize;
+ key.iov_base = page_leaf2key(lowest_page, 0, key.iov_len);
+ } else {
+ MDBX_node *lowest_node = page_node(lowest_page, 0);
+ key.iov_len = node_ks(lowest_node);
+ key.iov_base = node_key(lowest_node);
+ }
+
+ /* restore cursor after mdbx_page_search_lowest() */
+ mn.mc_snum = snum;
+ mn.mc_top = snum - 1;
+ mn.mc_ki[mn.mc_top] = 0;
+
+ const intptr_t delta =
+ EVEN(key.iov_len) - EVEN(node_ks(page_node(mn.mc_pg[mn.mc_top], 0)));
+ const intptr_t needed =
+ branch_size(cdst->mc_txn->mt_env, &key4move) + delta;
+ const intptr_t have = page_room(pdst);
+ if (unlikely(needed > have))
+ return MDBX_RESULT_TRUE;
+
+ if (unlikely((rc = mdbx_page_touch(csrc)) ||
+ (rc = mdbx_page_touch(cdst))))
+ return rc;
+ psrc = csrc->mc_pg[csrc->mc_top];
+ pdst = cdst->mc_pg[cdst->mc_top];
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- if (unlikely(mc->mc_flags & C_EOF)) {
- if ((unsigned)mc->mc_ki[mc->mc_top] + 1 >= page_numkeys(mp))
- return MDBX_NOTFOUND;
- mc->mc_flags ^= C_EOF;
- }
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
+ if (unlikely(rc))
+ return rc;
+ } else {
+ const size_t needed = branch_size(cdst->mc_txn->mt_env, &key4move);
+ const size_t have = page_room(pdst);
+ if (unlikely(needed > have))
+ return MDBX_RESULT_TRUE;
- int ki = mc->mc_ki[mc->mc_top];
- mc->mc_ki[mc->mc_top] = (indx_t)++ki;
- const int numkeys = page_numkeys(mp);
- if (likely(ki >= numkeys)) {
- mdbx_debug("%s", "=====> move to next sibling page");
- mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1);
- int err = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
- if (unlikely(err != MDBX_SUCCESS)) {
- mc->mc_flags |= C_EOF;
- return err;
+ if (unlikely((rc = mdbx_page_touch(csrc)) ||
+ (rc = mdbx_page_touch(cdst))))
+ return rc;
+ psrc = csrc->mc_pg[csrc->mc_top];
+ pdst = cdst->mc_pg[cdst->mc_top];
}
- mp = mc->mc_pg[mc->mc_top];
- mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
- mc->mc_ki[mc->mc_top]);
- }
- return MDBX_SUCCESS;
-}
-
-int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
- size_t limit, MDBX_cursor_op op) {
- if (unlikely(mc == NULL || count == NULL || limit < 4))
- return MDBX_EINVAL;
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
+ " to node %u on page %" PRIaPGNO,
+ "branch", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
+ psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
+ /* Add the node to the destination page. */
+ rc =
+ mdbx_node_add_branch(cdst, cdst->mc_ki[cdst->mc_top], &key4move, srcpg);
+ } break;
- int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ case P_LEAF: {
+ /* Mark src and dst as dirty. */
+ if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst))))
+ return rc;
+ psrc = csrc->mc_pg[csrc->mc_top];
+ pdst = cdst->mc_pg[cdst->mc_top];
+ const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]);
+ MDBX_val data;
+ data.iov_len = node_ds(srcnode);
+ data.iov_base = node_data(srcnode);
+ key4move.iov_len = node_ks(srcnode);
+ key4move.iov_base = node_key(srcnode);
+ mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
+ " to node %u on page %" PRIaPGNO,
+ "leaf", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
+ psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
+ /* Add the node to the destination page. */
+ rc = mdbx_node_add_leaf(cdst, cdst->mc_ki[cdst->mc_top], &key4move, &data,
+ node_flags(srcnode));
+ } break;
- if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT))
- return MDBX_INCOMPATIBLE /* must be a non-dupsort subDB */;
+ case P_LEAF | P_LEAF2: {
+ /* Mark src and dst as dirty. */
+ if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst))))
+ return rc;
+ psrc = csrc->mc_pg[csrc->mc_top];
+ pdst = cdst->mc_pg[cdst->mc_top];
+ key4move.iov_len = csrc->mc_db->md_xsize;
+ key4move.iov_base =
+ page_leaf2key(psrc, csrc->mc_ki[csrc->mc_top], key4move.iov_len);
+ mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
+ " to node %u on page %" PRIaPGNO,
+ "leaf2", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
+ psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
+ /* Add the node to the destination page. */
+ rc = mdbx_node_add_leaf2(cdst, cdst->mc_ki[cdst->mc_top], &key4move);
+ } break;
- switch (op) {
- case MDBX_FIRST:
- rc = cursor_first_batch(mc);
- break;
- case MDBX_NEXT:
- rc = cursor_next_batch(mc);
- break;
- case MDBX_GET_CURRENT:
- rc = likely(mc->mc_flags & C_INITIALIZED) ? MDBX_SUCCESS : MDBX_ENODATA;
- break;
default:
- mdbx_debug("unhandled/unimplemented cursor operation %u", op);
- rc = EINVAL;
- break;
+ goto bailout;
}
- if (unlikely(rc != MDBX_SUCCESS)) {
- *count = 0;
+ if (unlikely(rc != MDBX_SUCCESS))
return rc;
- }
- const MDBX_page *const page = mc->mc_pg[mc->mc_top];
- const unsigned nkeys = page_numkeys(page);
- unsigned i = mc->mc_ki[mc->mc_top], n = 0;
- if (unlikely(i >= nkeys)) {
- mdbx_cassert(mc, op == MDBX_GET_CURRENT);
- mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
- *count = 0;
- if (mc->mc_flags & C_EOF) {
- mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
- return MDBX_ENODATA;
+ /* Delete the node from the source page. */
+ mdbx_node_del(csrc, key4move.iov_len);
+
+ mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
+ mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
+ mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
+
+ {
+ /* Adjust other cursors pointing to mp */
+ MDBX_cursor *m2, *m3;
+ const MDBX_dbi dbi = csrc->mc_dbi;
+ mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top);
+ if (fromleft) {
+ /* If we're adding on the left, bump others up */
+ for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
+ m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
+ continue;
+ if (m3 != cdst && m3->mc_pg[csrc->mc_top] == pdst &&
+ m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) {
+ m3->mc_ki[csrc->mc_top]++;
+ }
+ if (m3 != csrc && m3->mc_pg[csrc->mc_top] == psrc &&
+ m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) {
+ m3->mc_pg[csrc->mc_top] = pdst;
+ m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+ mdbx_cassert(csrc, csrc->mc_top > 0);
+ m3->mc_ki[csrc->mc_top - 1]++;
+ }
+ if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
+ XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
+ }
+ } else {
+ /* Adding on the right, bump others down */
+ for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
+ m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == csrc)
+ continue;
+ if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
+ continue;
+ if (m3->mc_pg[csrc->mc_top] == psrc) {
+ if (!m3->mc_ki[csrc->mc_top]) {
+ m3->mc_pg[csrc->mc_top] = pdst;
+ m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+ mdbx_cassert(csrc, csrc->mc_top > 0);
+ m3->mc_ki[csrc->mc_top - 1]--;
+ } else {
+ m3->mc_ki[csrc->mc_top]--;
+ }
+ if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
+ XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top],
+ m3->mc_ki[csrc->mc_top]);
+ }
+ }
}
- if (mdbx_cursor_on_last(mc) != MDBX_RESULT_TRUE)
- return MDBX_EINVAL /* again MDBX_GET_CURRENT after MDBX_GET_CURRENT */;
- mc->mc_flags |= C_EOF;
- return MDBX_NOTFOUND;
}
- const txnid_t pp_txnid = pp_txnid4chk(page, mc->mc_txn);
- do {
- if (unlikely(n + 2 > limit)) {
- rc = MDBX_RESULT_TRUE;
- break;
+ /* Update the parent separators. */
+ if (csrc->mc_ki[csrc->mc_top] == 0) {
+ mdbx_cassert(csrc, csrc->mc_top > 0);
+ if (csrc->mc_ki[csrc->mc_top - 1] != 0) {
+ MDBX_val key;
+ if (IS_LEAF2(psrc)) {
+ key.iov_len = psrc->mp_leaf2_ksize;
+ key.iov_base = page_leaf2key(psrc, 0, key.iov_len);
+ } else {
+ MDBX_node *srcnode = page_node(psrc, 0);
+ key.iov_len = node_ks(srcnode);
+ key.iov_base = node_key(srcnode);
+ }
+ mdbx_debug("update separator for source page %" PRIaPGNO " to [%s]",
+ psrc->mp_pgno, DKEY_DEBUG(&key));
+ MDBX_cursor mn;
+ cursor_copy(csrc, &mn);
+ mdbx_cassert(csrc, mn.mc_snum > 0);
+ mn.mc_snum--;
+ mn.mc_top--;
+ /* We want mdbx_rebalance to find mn when doing fixups */
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+ if (IS_BRANCH(psrc)) {
+ const MDBX_val nullkey = {0, 0};
+ const indx_t ix = csrc->mc_ki[csrc->mc_top];
+ csrc->mc_ki[csrc->mc_top] = 0;
+ rc = mdbx_update_key(csrc, &nullkey);
+ csrc->mc_ki[csrc->mc_top] = ix;
+ mdbx_cassert(csrc, rc == MDBX_SUCCESS);
}
- const MDBX_node *leaf = page_node(page, i);
- get_key(leaf, &pairs[n]);
- rc = mdbx_node_read(mc, leaf, &pairs[n + 1], pp_txnid);
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- n += 2;
- } while (++i < nkeys);
-
- mc->mc_ki[mc->mc_top] = (indx_t)i;
- *count = n;
- return rc;
-}
-
-static int mdbx_touch_dbi(MDBX_cursor *mc) {
- mdbx_cassert(mc, (*mc->mc_dbistate & DBI_DIRTY) == 0);
- *mc->mc_dbistate |= DBI_DIRTY;
- mc->mc_txn->mt_flags |= MDBX_TXN_DIRTY;
- if (mc->mc_dbi >= CORE_DBS) {
- mdbx_cassert(mc, (mc->mc_flags & C_RECLAIMING) == 0);
- /* Touch DB record of named DB */
- MDBX_cursor_couple cx;
- int rc = mdbx_cursor_init(&cx.outer, mc->mc_txn, MAIN_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- mc->mc_txn->mt_dbistate[MAIN_DBI] |= DBI_DIRTY;
- rc = mdbx_page_search(&cx.outer, &mc->mc_dbx->md_name, MDBX_PS_MODIFY);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
}
- return MDBX_SUCCESS;
-}
-/* Touch all the pages in the cursor stack. Set mc_top.
- * Makes sure all the pages are writable, before attempting a write operation.
- * [in] mc The cursor to operate on. */
-static int mdbx_cursor_touch(MDBX_cursor *mc) {
- int rc = MDBX_SUCCESS;
- if (unlikely((*mc->mc_dbistate & DBI_DIRTY) == 0)) {
- rc = mdbx_touch_dbi(mc);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- if (likely(mc->mc_snum)) {
- mc->mc_top = 0;
- do {
- rc = mdbx_page_touch(mc);
- } while (!rc && ++(mc->mc_top) < mc->mc_snum);
- mc->mc_top = mc->mc_snum - 1;
+ if (cdst->mc_ki[cdst->mc_top] == 0) {
+ mdbx_cassert(cdst, cdst->mc_top > 0);
+ if (cdst->mc_ki[cdst->mc_top - 1] != 0) {
+ MDBX_val key;
+ if (IS_LEAF2(pdst)) {
+ key.iov_len = pdst->mp_leaf2_ksize;
+ key.iov_base = page_leaf2key(pdst, 0, key.iov_len);
+ } else {
+ MDBX_node *srcnode = page_node(pdst, 0);
+ key.iov_len = node_ks(srcnode);
+ key.iov_base = node_key(srcnode);
+ }
+ mdbx_debug("update separator for destination page %" PRIaPGNO " to [%s]",
+ pdst->mp_pgno, DKEY_DEBUG(&key));
+ MDBX_cursor mn;
+ cursor_copy(cdst, &mn);
+ mdbx_cassert(cdst, mn.mc_snum > 0);
+ mn.mc_snum--;
+ mn.mc_top--;
+ /* We want mdbx_rebalance to find mn when doing fixups */
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+ if (IS_BRANCH(pdst)) {
+ const MDBX_val nullkey = {0, 0};
+ const indx_t ix = cdst->mc_ki[cdst->mc_top];
+ cdst->mc_ki[cdst->mc_top] = 0;
+ rc = mdbx_update_key(cdst, &nullkey);
+ cdst->mc_ki[cdst->mc_top] = ix;
+ mdbx_cassert(cdst, rc == MDBX_SUCCESS);
+ }
}
- return rc;
-}
-
-int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
- unsigned flags) {
- MDBX_env *env;
- MDBX_page *sub_root = NULL;
- MDBX_val xdata, *rdata, dkey, olddata;
- MDBX_db nested_dupdb;
- int err;
- DKBUF_DEBUG;
-
- if (unlikely(mc == NULL || key == NULL || data == NULL))
- return MDBX_EINVAL;
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ return MDBX_SUCCESS;
+}
- int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/* Merge one page into another.
+ *
+ * The nodes from the page pointed to by csrc will be copied to the page
+ * pointed to by cdst and then the csrc page will be freed.
+ *
+ * [in] csrc Cursor pointing to the source page.
+ * [in] cdst Cursor pointing to the destination page.
+ *
+ * Returns 0 on success, non-zero on failure. */
+static int mdbx_page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
+ MDBX_val key;
+ int rc;
- if (unlikely(TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)))
- return MDBX_BAD_DBI;
+ mdbx_cassert(csrc, csrc != cdst);
+ mdbx_cassert(csrc, cursor_is_tracked(csrc));
+ mdbx_cassert(cdst, cursor_is_tracked(cdst));
+ const MDBX_page *const psrc = csrc->mc_pg[csrc->mc_top];
+ MDBX_page *pdst = cdst->mc_pg[cdst->mc_top];
+ mdbx_debug("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->mp_pgno,
+ pdst->mp_pgno);
- mdbx_cassert(mc, cursor_is_tracked(mc));
- env = mc->mc_txn->mt_env;
+ mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
+ mdbx_cassert(csrc,
+ csrc->mc_dbi == cdst->mc_dbi && csrc->mc_db == cdst->mc_db);
+ mdbx_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */
+ mdbx_cassert(cdst, cdst->mc_snum > 1);
+ mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
+ IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
+ mdbx_cassert(csrc, csrc->mc_snum < csrc->mc_db->md_depth ||
+ IS_LEAF(csrc->mc_pg[csrc->mc_db->md_depth - 1]));
+ mdbx_cassert(cdst, page_room(pdst) >= page_used(cdst->mc_txn->mt_env, psrc));
+ const int pagetype = PAGETYPE(psrc);
- /* Check this first so counter will always be zero on any early failures. */
- size_t mcount = 0, dcount = 0;
- if (unlikely(flags & MDBX_MULTIPLE)) {
- if (unlikely(flags & MDBX_RESERVE))
- return MDBX_EINVAL;
- if (unlikely(!F_ISSET(mc->mc_db->md_flags, MDBX_DUPFIXED)))
- return MDBX_INCOMPATIBLE;
- dcount = data[1].iov_len;
- if (unlikely(dcount < 2 || data->iov_len == 0))
- return MDBX_BAD_VALSIZE;
- if (unlikely(mc->mc_db->md_xsize != data->iov_len) && mc->mc_db->md_xsize)
- return MDBX_BAD_VALSIZE;
- if (unlikely(dcount > MAX_MAPSIZE / 2 /
- (BRANCH_NODE_MAX(MAX_PAGESIZE) - NODESIZE))) {
- /* checking for multiplication overflow */
- if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len))
- return MDBX_TOO_LARGE;
- }
- data[1].iov_len = 0 /* reset done item counter */;
- }
+ /* Move all nodes from src to dst */
+ const unsigned dst_nkeys = page_numkeys(pdst);
+ const unsigned src_nkeys = page_numkeys(psrc);
+ mdbx_cassert(cdst, dst_nkeys + src_nkeys >= (IS_LEAF(psrc) ? 1u : 2u));
+ if (likely(src_nkeys)) {
+ unsigned j = dst_nkeys;
+ if (unlikely(pagetype & P_LEAF2)) {
+ /* Mark dst as dirty. */
+ if (unlikely(rc = mdbx_page_touch(cdst)))
+ return rc;
- if (flags & MDBX_RESERVE) {
- if (unlikely(mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_REVERSEDUP |
- MDBX_INTEGERDUP | MDBX_DUPFIXED)))
- return MDBX_INCOMPATIBLE;
- data->iov_base = nullptr;
- }
+ key.iov_len = csrc->mc_db->md_xsize;
+ key.iov_base = page_data(psrc);
+ unsigned i = 0;
+ do {
+ rc = mdbx_node_add_leaf2(cdst, j++, &key);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ key.iov_base = (char *)key.iov_base + key.iov_len;
+ } while (++i != src_nkeys);
+ } else {
+ MDBX_node *srcnode = page_node(psrc, 0);
+ key.iov_len = node_ks(srcnode);
+ key.iov_base = node_key(srcnode);
+ if (pagetype & P_BRANCH) {
+ MDBX_cursor mn;
+ cursor_copy(csrc, &mn);
+ /* must find the lowest key below src */
+ rc = mdbx_page_search_lowest(&mn);
+ if (unlikely(rc))
+ return rc;
- const unsigned nospill = flags & MDBX_NOSPILL;
- flags -= nospill;
+ const MDBX_page *mp = mn.mc_pg[mn.mc_top];
+ if (likely(!IS_LEAF2(mp))) {
+ mdbx_cassert(&mn, IS_LEAF(mp));
+ const MDBX_node *lowest = page_node(mp, 0);
+ key.iov_len = node_ks(lowest);
+ key.iov_base = node_key(lowest);
+ } else {
+ mdbx_cassert(&mn, mn.mc_top > csrc->mc_top);
+ key.iov_len = mp->mp_leaf2_ksize;
+ key.iov_base = page_leaf2key(mp, mn.mc_ki[mn.mc_top], key.iov_len);
+ }
+ mdbx_cassert(&mn, key.iov_len >= csrc->mc_dbx->md_klen_min);
+ mdbx_cassert(&mn, key.iov_len <= csrc->mc_dbx->md_klen_max);
- if (unlikely(mc->mc_txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
- return (mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS
- : MDBX_BAD_TXN;
+ const size_t dst_room = page_room(pdst);
+ const size_t src_used = page_used(cdst->mc_txn->mt_env, psrc);
+ const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len;
+ if (unlikely(space_needed > dst_room))
+ return MDBX_RESULT_TRUE;
+ }
- uint64_t aligned_keybytes, aligned_databytes;
- MDBX_val aligned_key, aligned_data;
- if (likely((mc->mc_flags & C_SUB) == 0)) {
- if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min ||
- key->iov_len > mc->mc_dbx->md_klen_max)) {
- mdbx_cassert(mc, !"Invalid key-size");
- return MDBX_BAD_VALSIZE;
- }
- if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min ||
- data->iov_len > mc->mc_dbx->md_vlen_max)) {
- mdbx_cassert(mc, !"Invalid data-size");
- return MDBX_BAD_VALSIZE;
- }
+ /* Mark dst as dirty. */
+ if (unlikely(rc = mdbx_page_touch(cdst)))
+ return rc;
- if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
- switch (key->iov_len) {
- default:
- mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY");
- return MDBX_BAD_VALSIZE;
- case 4:
- if (unlikely(3 & (uintptr_t)key->iov_base)) {
- /* copy instead of return error to avoid break compatibility */
- aligned_key.iov_base =
- memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 4);
- key = &aligned_key;
- }
- break;
- case 8:
- if (unlikely(7 & (uintptr_t)key->iov_base)) {
- /* copy instead of return error to avoid break compatibility */
- aligned_key.iov_base =
- memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 8);
- key = &aligned_key;
- }
- break;
- }
- }
- if (mc->mc_db->md_flags & MDBX_INTEGERDUP) {
- switch (data->iov_len) {
- default:
- mdbx_cassert(mc, !"data-size is invalid for MDBX_INTEGERKEY");
- return MDBX_BAD_VALSIZE;
- case 4:
- if (unlikely(3 & (uintptr_t)data->iov_base)) {
- if (unlikely(flags & MDBX_MULTIPLE))
- return MDBX_BAD_VALSIZE;
- /* copy instead of return error to avoid break compatibility */
- aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base,
- aligned_data.iov_len = 4);
- data = &aligned_data;
- }
- break;
- case 8:
- if (unlikely(7 & (uintptr_t)data->iov_base)) {
- if (unlikely(flags & MDBX_MULTIPLE))
- return MDBX_BAD_VALSIZE;
- /* copy instead of return error to avoid break compatibility */
- aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base,
- aligned_data.iov_len = 8);
- data = &aligned_data;
+ unsigned i = 0;
+ while (true) {
+ if (pagetype & P_LEAF) {
+ MDBX_val data;
+ data.iov_len = node_ds(srcnode);
+ data.iov_base = node_data(srcnode);
+ rc = mdbx_node_add_leaf(cdst, j++, &key, &data, node_flags(srcnode));
+ } else {
+ mdbx_cassert(csrc, node_flags(srcnode) == 0);
+ rc = mdbx_node_add_branch(cdst, j++, &key, node_pgno(srcnode));
}
- break;
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (++i == src_nkeys)
+ break;
+ srcnode = page_node(psrc, i);
+ key.iov_len = node_ks(srcnode);
+ key.iov_base = node_key(srcnode);
}
}
- }
- mdbx_debug(
- "==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR,
- DDBI(mc), DKEY_DEBUG(key), key->iov_len,
- DVAL_DEBUG((flags & MDBX_RESERVE) ? nullptr : data), data->iov_len);
+ pdst = cdst->mc_pg[cdst->mc_top];
+ mdbx_debug("dst page %" PRIaPGNO " now has %u keys (%.1f%% filled)",
+ pdst->mp_pgno, page_numkeys(pdst),
+ page_fill(cdst->mc_txn->mt_env, pdst));
- int dupdata_flag = 0;
- if ((flags & MDBX_CURRENT) != 0 && (mc->mc_flags & C_SUB) == 0) {
- if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE)))
- return MDBX_EINVAL;
- /* ÐžÐ¿Ñ†Ð¸Ñ MDBX_CURRENT означает, что запрошено обновление текущей запиÑи,
- * на которой ÑÐµÐ¹Ñ‡Ð°Ñ Ñтоит курÑор. ПроверÑем что переданный ключ Ñовпадает
- * Ñо значением в текущей позиции курÑора.
- * ЗдеÑÑŒ проще вызвать mdbx_cursor_get(), так как Ð´Ð»Ñ Ð¾Ð±ÑÐ»ÑƒÐ¶Ð¸Ð²Ð°Ð½Ð¸Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ†
- * Ñ MDBX_DUPSORT также требуетÑÑ Ñ‚ÐµÐºÑƒÑ‰Ð¸Ð¹ размер данных. */
- MDBX_val current_key, current_data;
- rc = mdbx_cursor_get(mc, ¤t_key, ¤t_data, MDBX_GET_CURRENT);
- if (unlikely(rc != MDBX_SUCCESS))
+ mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
+ mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
+ }
+
+ /* Unlink the src page from parent and add to free list. */
+ csrc->mc_top--;
+ mdbx_node_del(csrc, 0);
+ if (csrc->mc_ki[csrc->mc_top] == 0) {
+ const MDBX_val nullkey = {0, 0};
+ rc = mdbx_update_key(csrc, &nullkey);
+ if (unlikely(rc)) {
+ csrc->mc_top++;
return rc;
- if (mc->mc_dbx->md_cmp(key, ¤t_key) != 0)
- return MDBX_EKEYMISMATCH;
+ }
+ }
+ csrc->mc_top++;
- if (unlikely((flags & MDBX_MULTIPLE)))
- goto drop_current;
+ mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
+ mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
- if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) {
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- mdbx_cassert(mc,
- mc->mc_xcursor != NULL &&
- (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED));
- /* ЕÑли за ключом более одного значениÑ, либо еÑли размер данных
- * отличаетÑÑ, то вмеÑто Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ñ‚Ñ€ÐµÐ±ÑƒÐµÑ‚ÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ и
- * поÑÐ»ÐµÐ´ÑƒÑŽÑ‰Ð°Ñ Ð²Ñтавка. */
- if (mc->mc_xcursor->mx_db.md_entries > 1 ||
- current_data.iov_len != data->iov_len) {
- drop_current:
- rc = mdbx_cursor_del(mc, flags & MDBX_ALLDUPS);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- flags -= MDBX_CURRENT;
- goto skip_check_samedata;
- }
- } else if (unlikely(node_size(key, data) > env->me_leaf_nodemax)) {
- rc = mdbx_cursor_del(mc, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- flags -= MDBX_CURRENT;
- goto skip_check_samedata;
+ {
+ /* Adjust other cursors pointing to mp */
+ MDBX_cursor *m2, *m3;
+ const MDBX_dbi dbi = csrc->mc_dbi;
+ const unsigned top = csrc->mc_top;
+
+ for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
+ m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == csrc || top >= m3->mc_snum)
+ continue;
+ if (m3->mc_pg[top] == psrc) {
+ m3->mc_pg[top] = pdst;
+ mdbx_cassert(m3, dst_nkeys + m3->mc_ki[top] <= UINT16_MAX);
+ m3->mc_ki[top] += (indx_t)dst_nkeys;
+ m3->mc_ki[top - 1] = cdst->mc_ki[top - 1];
+ } else if (m3->mc_pg[top - 1] == csrc->mc_pg[top - 1] &&
+ m3->mc_ki[top - 1] > csrc->mc_ki[top - 1]) {
+ m3->mc_ki[top - 1]--;
}
+ if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
+ XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]);
}
- if (!(flags & MDBX_RESERVE) &&
- unlikely(cmp_lenfast(¤t_data, data) == 0))
- return MDBX_SUCCESS /* the same data, nothing to update */;
- skip_check_samedata:;
}
- if (mc->mc_db->md_root == P_INVALID) {
- /* new database, cursor has nothing to point to */
- mc->mc_snum = 0;
- mc->mc_top = 0;
- mc->mc_flags &= ~C_INITIALIZED;
- rc = MDBX_NO_ROOT;
- } else if ((flags & MDBX_CURRENT) == 0) {
- bool exact = false;
- if ((flags & MDBX_APPEND) && mc->mc_db->md_entries > 0) {
- rc = mdbx_cursor_last(mc, &dkey, &olddata);
- if (likely(rc == MDBX_SUCCESS)) {
- rc = mc->mc_dbx->md_cmp(key, &dkey);
- if (likely(rc > 0)) {
- mc->mc_ki[mc->mc_top]++; /* step forward for appending */
- rc = MDBX_NOTFOUND;
- } else {
- if (unlikely(rc != MDBX_SUCCESS || !(flags & MDBX_APPENDDUP)))
- /* new-key < last-key
- * or new-key == last-key without MDBX_APPENDDUP */
- return MDBX_EKEYMISMATCH;
- exact = true;
- }
- }
- } else {
- struct cursor_set_result csr =
- /* olddata may not be updated in case LEAF2-page of dupfixed-subDB */
- mdbx_cursor_set(mc, (MDBX_val *)key, &olddata, MDBX_SET);
- rc = csr.err;
- exact = csr.exact;
- }
- if (likely(rc == MDBX_SUCCESS)) {
- if (exact) {
- if (unlikely(flags & MDBX_NOOVERWRITE)) {
- mdbx_debug("duplicate key [%s]", DKEY_DEBUG(key));
- *data = olddata;
- return MDBX_KEYEXIST;
- }
- if (unlikely(mc->mc_flags & C_SUB)) {
- /* nested subtree of DUPSORT-database with the same key,
- * nothing to update */
- mdbx_assert(env, data->iov_len == 0 &&
- (olddata.iov_len == 0 ||
- /* olddata may not be updated in case LEAF2-page
- of dupfixed-subDB */
- (mc->mc_db->md_flags & MDBX_DUPFIXED)));
- return MDBX_SUCCESS;
- }
- if (unlikely(flags & MDBX_ALLDUPS) && mc->mc_xcursor &&
- (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
- rc = mdbx_cursor_del(mc, MDBX_ALLDUPS);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- flags -= MDBX_ALLDUPS;
- rc = MDBX_NOTFOUND;
- exact = false;
- } else /* checking for early exit without dirtying pages */
- if (!(flags & (MDBX_RESERVE | MDBX_MULTIPLE)) &&
- unlikely(mc->mc_dbx->md_dcmp(data, &olddata) == 0)) {
- if (!mc->mc_xcursor)
- /* the same data, nothing to update */
- return MDBX_SUCCESS;
- if (flags & MDBX_NODUPDATA)
- return MDBX_KEYEXIST;
- if (flags & MDBX_APPENDDUP)
- return MDBX_EKEYMISMATCH;
- if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata)))
- /* data is match exactly byte-to-byte, nothing to update */
- return MDBX_SUCCESS;
- else {
- /* The data has differences, but the user-provided comparator
- * considers them equal. So continue update since called without.
- * Continue to update since was called without MDBX_NODUPDATA. */
- }
- }
- }
- } else if (unlikely(rc != MDBX_NOTFOUND))
- return rc;
+ /* If not operating on GC, allow this page to be reused
+ * in this txn. Otherwise just add to free list. */
+ rc = mdbx_page_retire(csrc, (MDBX_page *)psrc);
+ if (unlikely(rc))
+ return rc;
+
+ mdbx_cassert(cdst, cdst->mc_db->md_entries > 0);
+ mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth);
+ mdbx_cassert(cdst, cdst->mc_top > 0);
+ mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1);
+ MDBX_page *const top_page = cdst->mc_pg[cdst->mc_top];
+ const indx_t top_indx = cdst->mc_ki[cdst->mc_top];
+ const unsigned save_snum = cdst->mc_snum;
+ const uint16_t save_depth = cdst->mc_db->md_depth;
+ mdbx_cursor_pop(cdst);
+ rc = mdbx_rebalance(cdst);
+ if (unlikely(rc))
+ return rc;
+
+ mdbx_cassert(cdst, cdst->mc_db->md_entries > 0);
+ mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth);
+ mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1);
+
+#if MDBX_ENABLE_PGOP_STAT
+ cdst->mc_txn->mt_env->me_lck->mti_pgop_stat.merge.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+ if (IS_LEAF(cdst->mc_pg[cdst->mc_top])) {
+ /* LY: don't touch cursor if top-page is a LEAF */
+ mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
+ PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
+ return MDBX_SUCCESS;
+ }
+
+ mdbx_cassert(cdst, page_numkeys(top_page) == dst_nkeys + src_nkeys);
+
+ if (unlikely(pagetype != PAGETYPE(top_page))) {
+ /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */
+ goto bailout;
+ }
+
+ if (top_page == cdst->mc_pg[cdst->mc_top]) {
+ /* LY: don't touch cursor if prev top-page already on the top */
+ mdbx_cassert(cdst, cdst->mc_ki[cdst->mc_top] == top_indx);
+ mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
+ PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
+ return MDBX_SUCCESS;
+ }
+
+ const int new_snum = save_snum - save_depth + cdst->mc_db->md_depth;
+ if (unlikely(new_snum < 1 || new_snum > cdst->mc_db->md_depth)) {
+ /* LY: out of range, unable restore cursor's stack */
+ goto bailout;
+ }
+
+ if (top_page == cdst->mc_pg[new_snum - 1]) {
+ mdbx_cassert(cdst, cdst->mc_ki[new_snum - 1] == top_indx);
+ /* LY: restore cursor stack */
+ cdst->mc_snum = (uint16_t)new_snum;
+ cdst->mc_top = (uint16_t)new_snum - 1;
+ mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
+ IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
+ mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
+ PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
+ return MDBX_SUCCESS;
+ }
+
+ MDBX_page *const stub_page = (MDBX_page *)(~(uintptr_t)top_page);
+ const indx_t stub_indx = top_indx;
+ if (save_depth > cdst->mc_db->md_depth &&
+ ((cdst->mc_pg[save_snum - 1] == top_page &&
+ cdst->mc_ki[save_snum - 1] == top_indx) ||
+ (cdst->mc_pg[save_snum - 1] == stub_page &&
+ cdst->mc_ki[save_snum - 1] == stub_indx))) {
+ /* LY: restore cursor stack */
+ cdst->mc_pg[new_snum - 1] = top_page;
+ cdst->mc_ki[new_snum - 1] = top_indx;
+ cdst->mc_pg[new_snum] = (MDBX_page *)(~(uintptr_t)cdst->mc_pg[new_snum]);
+ cdst->mc_ki[new_snum] = ~cdst->mc_ki[new_snum];
+ cdst->mc_snum = (uint16_t)new_snum;
+ cdst->mc_top = (uint16_t)new_snum - 1;
+ mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
+ IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
+ mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
+ PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
+ return MDBX_SUCCESS;
}
- mc->mc_flags &= ~C_DEL;
+bailout:
+ /* LY: unable restore cursor's stack */
+ cdst->mc_flags &= ~C_INITIALIZED;
+ return MDBX_CURSOR_FULL;
+}
- /* Cursor is positioned, check for room in the dirty list */
- if (!nospill) {
- rdata = data;
- if (unlikely(flags & MDBX_MULTIPLE)) {
- rdata = &xdata;
- xdata.iov_len = data->iov_len * dcount;
- }
- if (unlikely(err = mdbx_cursor_spill(mc, key, rdata)))
- return err;
- }
+static void cursor_restore(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
+ mdbx_cassert(cdst, cdst->mc_dbi == csrc->mc_dbi);
+ mdbx_cassert(cdst, cdst->mc_txn == csrc->mc_txn);
+ mdbx_cassert(cdst, cdst->mc_db == csrc->mc_db);
+ mdbx_cassert(cdst, cdst->mc_dbx == csrc->mc_dbx);
+ mdbx_cassert(cdst, cdst->mc_dbistate == csrc->mc_dbistate);
+ cdst->mc_snum = csrc->mc_snum;
+ cdst->mc_top = csrc->mc_top;
+ cdst->mc_flags = csrc->mc_flags;
- if (unlikely(rc == MDBX_NO_ROOT)) {
- /* new database, write a root leaf page */
- mdbx_debug("%s", "allocating new root leaf page");
- if (unlikely((*mc->mc_dbistate & DBI_DIRTY) == 0)) {
- err = mdbx_touch_dbi(mc);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- struct page_result npr = mdbx_page_new(mc, P_LEAF, 1);
- if (unlikely(npr.err != MDBX_SUCCESS))
- return npr.err;
- npr.err = mdbx_cursor_push(mc, npr.page);
- if (unlikely(npr.err != MDBX_SUCCESS))
- return npr.err;
- mc->mc_db->md_root = npr.page->mp_pgno;
- mc->mc_db->md_depth++;
- if (mc->mc_db->md_flags & MDBX_INTEGERKEY) {
- assert(key->iov_len >= mc->mc_dbx->md_klen_min &&
- key->iov_len <= mc->mc_dbx->md_klen_max);
- mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = key->iov_len;
- }
- if (mc->mc_db->md_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) {
- assert(data->iov_len >= mc->mc_dbx->md_vlen_min &&
- data->iov_len <= mc->mc_dbx->md_vlen_max);
- assert(mc->mc_xcursor != NULL);
- mc->mc_db->md_xsize = mc->mc_xcursor->mx_db.md_xsize =
- (unsigned)(mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max =
- mc->mc_xcursor->mx_dbx.md_klen_min =
- mc->mc_xcursor->mx_dbx.md_klen_max =
- data->iov_len);
- }
- if ((mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_DUPFIXED)) == MDBX_DUPFIXED)
- npr.page->mp_flags |= P_LEAF2;
- mc->mc_flags |= C_INITIALIZED;
- } else {
- /* make sure all cursor pages are writable */
- err = mdbx_cursor_touch(mc);
- if (unlikely(err))
- return err;
+ for (unsigned i = 0; i < csrc->mc_snum; i++) {
+ cdst->mc_pg[i] = csrc->mc_pg[i];
+ cdst->mc_ki[i] = csrc->mc_ki[i];
}
+}
- bool insert_key, insert_data, do_sub = false;
- insert_key = insert_data = (rc != MDBX_SUCCESS);
- uint16_t fp_flags = P_LEAF;
- MDBX_page *fp = env->me_pbuf;
- fp->mp_txnid = mc->mc_txn->mt_front;
- if (insert_key) {
- /* The key does not exist */
- mdbx_debug("inserting key at index %i", mc->mc_ki[mc->mc_top]);
- if ((mc->mc_db->md_flags & MDBX_DUPSORT) &&
- node_size(key, data) > env->me_leaf_nodemax) {
- /* Too big for a node, insert in sub-DB. Set up an empty
- * "old sub-page" for prep_subDB to expand to a full page. */
- fp->mp_leaf2_ksize =
- (mc->mc_db->md_flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0;
- fp->mp_lower = fp->mp_upper = 0;
- olddata.iov_len = PAGEHDRSZ;
- goto prep_subDB;
- }
- } else {
- /* there's only a key anyway, so this is a no-op */
- if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
- char *ptr;
- unsigned ksize = mc->mc_db->md_xsize;
- if (unlikely(key->iov_len != ksize))
- return MDBX_BAD_VALSIZE;
- ptr = page_leaf2key(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize);
- memcpy(ptr, key->iov_base, ksize);
- fix_parent:
- /* if overwriting slot 0 of leaf, need to
- * update branch key if there is a parent page */
- if (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
- unsigned dtop = 1;
- mc->mc_top--;
- /* slot 0 is always an empty key, find real slot */
- while (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
- mc->mc_top--;
- dtop++;
- }
- err = MDBX_SUCCESS;
- if (mc->mc_ki[mc->mc_top])
- err = mdbx_update_key(mc, key);
- mdbx_cassert(mc, mc->mc_top + dtop < UINT16_MAX);
- mc->mc_top += (uint16_t)dtop;
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
+/* Copy the contents of a cursor.
+ * [in] csrc The cursor to copy from.
+ * [out] cdst The cursor to copy to. */
+static void cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
+ mdbx_cassert(csrc, csrc->mc_txn->mt_txnid >=
+ csrc->mc_txn->mt_env->me_lck->mti_oldest_reader.weak);
+ cdst->mc_dbi = csrc->mc_dbi;
+ cdst->mc_next = NULL;
+ cdst->mc_backup = NULL;
+ cdst->mc_xcursor = NULL;
+ cdst->mc_txn = csrc->mc_txn;
+ cdst->mc_db = csrc->mc_db;
+ cdst->mc_dbx = csrc->mc_dbx;
+ cdst->mc_dbistate = csrc->mc_dbistate;
+ cursor_restore(csrc, cdst);
+}
- if (mdbx_audit_enabled()) {
- err = mdbx_cursor_check(mc, 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- return MDBX_SUCCESS;
- }
+/* Rebalance the tree after a delete operation.
+ * [in] mc Cursor pointing to the page where rebalancing should begin.
+ * Returns 0 on success, non-zero on failure. */
+static int mdbx_rebalance(MDBX_cursor *mc) {
+ mdbx_cassert(mc, cursor_is_tracked(mc));
+ mdbx_cassert(mc, mc->mc_snum > 0);
+ mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth ||
+ IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1]));
+ const int pagetype = PAGETYPE(mc->mc_pg[mc->mc_top]);
- more:;
- if (mdbx_audit_enabled()) {
- err = mdbx_cursor_check(mc, 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ STATIC_ASSERT(P_BRANCH == 1);
+ const unsigned minkeys = (pagetype & P_BRANCH) + 1;
- /* Large/Overflow page overwrites need special handling */
- if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) {
- int dpages = (node_size(key, data) > env->me_leaf_nodemax)
- ? number_of_ovpages(env, data->iov_len)
- : 0;
+ /* Pages emptier than this are candidates for merging. */
+ unsigned room_threshold = likely(mc->mc_dbi != FREE_DBI)
+ ? mc->mc_txn->mt_env->me_merge_threshold
+ : mc->mc_txn->mt_env->me_merge_threshold_gc;
- const pgno_t pgno = node_largedata_pgno(node);
- struct page_result pgr = mdbx_page_get_ex(
- mc, pgno, pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn));
- if (unlikely(pgr.err != MDBX_SUCCESS))
- return pgr.err;
- if (unlikely(!IS_OVERFLOW(pgr.page)))
- return MDBX_CORRUPTED;
+ const MDBX_page *const tp = mc->mc_pg[mc->mc_top];
+ const unsigned numkeys = page_numkeys(tp);
+ const unsigned room = page_room(tp);
+ mdbx_debug("rebalancing %s page %" PRIaPGNO
+ " (has %u keys, full %.1f%%, used %u, room %u bytes )",
+ (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys,
+ page_fill(mc->mc_txn->mt_env, tp),
+ page_used(mc->mc_txn->mt_env, tp), room);
- /* Is the ov page from this txn (or a parent) and big enough? */
- int ovpages = pgr.page->mp_pages;
- if (!IS_FROZEN(mc->mc_txn, pgr.page) &&
- (unlikely(mc->mc_flags & C_GCFREEZE)
- ? (ovpages >= dpages)
- : (ovpages ==
- /* LY: add configurable threshold to keep reserve space */
- dpages))) {
- /* yes, overwrite it. */
- if (!IS_MODIFIABLE(mc->mc_txn, pgr.page)) {
- if (IS_SPILLED(mc->mc_txn, pgr.page)) {
- pgr = /* TODO: avoid search and get txn & spill-index from
- page_result */
- mdbx_page_unspill(mc->mc_txn, pgr.page);
- if (unlikely(pgr.err))
- return pgr.err;
- } else {
- if (unlikely(!mc->mc_txn->mt_parent)) {
- mdbx_error(
- "Unexpected not frozen/modifiable/spilled but shadowed %s "
- "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
- " without parent transaction, current txn %" PRIaTXN
- " front %" PRIaTXN,
- "overflow/large", pgno, pgr.page->mp_txnid,
- mc->mc_txn->mt_txnid, mc->mc_txn->mt_front);
- return MDBX_PROBLEM;
- }
+ if (unlikely(numkeys < minkeys)) {
+ mdbx_debug("page %" PRIaPGNO " must be merged due keys < %u threshold",
+ tp->mp_pgno, minkeys);
+ } else if (unlikely(room > room_threshold)) {
+ mdbx_debug("page %" PRIaPGNO " should be merged due room %u > %u threshold",
+ tp->mp_pgno, room, room_threshold);
+ } else {
+ mdbx_debug("no need to rebalance page %" PRIaPGNO
+ ", room %u < %u threshold",
+ tp->mp_pgno, room, room_threshold);
+ mdbx_cassert(mc, mc->mc_db->md_entries > 0);
+ return MDBX_SUCCESS;
+ }
- /* It is writable only in a parent txn */
- MDBX_page *np = mdbx_page_malloc(mc->mc_txn, ovpages);
- if (unlikely(!np))
- return MDBX_ENOMEM;
+ int rc;
+ if (mc->mc_snum < 2) {
+ MDBX_page *const mp = mc->mc_pg[0];
+ const unsigned nkeys = page_numkeys(mp);
+ mdbx_cassert(mc, (mc->mc_db->md_entries == 0) == (nkeys == 0));
+ if (IS_SUBP(mp)) {
+ mdbx_debug("%s", "Can't rebalance a subpage, ignoring");
+ mdbx_cassert(mc, pagetype & P_LEAF);
+ return MDBX_SUCCESS;
+ }
+ if (nkeys == 0) {
+ mdbx_cassert(mc, IS_LEAF(mp));
+ mdbx_debug("%s", "tree is completely empty");
+ mdbx_cassert(mc, (*mc->mc_dbistate & DBI_DIRTY) != 0);
+ mc->mc_db->md_root = P_INVALID;
+ mc->mc_db->md_depth = 0;
+ mdbx_cassert(mc, mc->mc_db->md_branch_pages == 0 &&
+ mc->mc_db->md_overflow_pages == 0 &&
+ mc->mc_db->md_leaf_pages == 1);
+ /* Adjust cursors pointing to mp */
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
+ m2 = m2->mc_next) {
+ MDBX_cursor *m3 =
+ (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == mc || !(m3->mc_flags & C_INITIALIZED))
+ continue;
+ if (m3->mc_pg[0] == mp) {
+ m3->mc_snum = 0;
+ m3->mc_top = 0;
+ m3->mc_flags &= ~C_INITIALIZED;
+ }
+ }
+ mc->mc_snum = 0;
+ mc->mc_top = 0;
+ mc->mc_flags &= ~C_INITIALIZED;
- memcpy(np, pgr.page, PAGEHDRSZ); /* Copy header of page */
- err = mdbx_page_dirty(mc->mc_txn, pgr.page = np, ovpages);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ rc = mdbx_page_retire(mc, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ } else if (IS_BRANCH(mp) && nkeys == 1) {
+ mdbx_debug("%s", "collapsing root page!");
+ mc->mc_db->md_root = node_pgno(page_node(mp, 0));
+ rc = mdbx_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0],
+ pp_txnid4chk(mp, mc->mc_txn));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ mc->mc_db->md_depth--;
+ mc->mc_ki[0] = mc->mc_ki[1];
+ for (int i = 1; i < mc->mc_db->md_depth; i++) {
+ mc->mc_pg[i] = mc->mc_pg[i + 1];
+ mc->mc_ki[i] = mc->mc_ki[i + 1];
+ }
-#if MDBX_ENABLE_PGOP_STAT
- mc->mc_txn->mt_env->me_lck->mti_pgop_stat.clone.weak += ovpages;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- mdbx_cassert(mc, mdbx_dirtylist_check(mc->mc_txn));
+ /* Adjust other cursors pointing to mp */
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
+ m2 = m2->mc_next) {
+ MDBX_cursor *m3 =
+ (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == mc || !(m3->mc_flags & C_INITIALIZED))
+ continue;
+ if (m3->mc_pg[0] == mp) {
+ for (int i = 0; i < mc->mc_db->md_depth; i++) {
+ m3->mc_pg[i] = m3->mc_pg[i + 1];
+ m3->mc_ki[i] = m3->mc_ki[i + 1];
}
+ m3->mc_snum--;
+ m3->mc_top--;
}
- node_set_ds(node, data->iov_len);
- if (F_ISSET(flags, MDBX_RESERVE))
- data->iov_base = page_data(pgr.page);
- else
- memcpy(page_data(pgr.page), data->iov_base, data->iov_len);
-
- if (mdbx_audit_enabled()) {
- err = mdbx_cursor_check(mc, 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- return MDBX_SUCCESS;
}
+ mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) ||
+ PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype);
+ mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth ||
+ IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1]));
- if ((err = mdbx_page_retire(mc, pgr.page)) != MDBX_SUCCESS)
- return err;
+ rc = mdbx_page_retire(mc, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
} else {
- olddata.iov_len = node_ds(node);
- olddata.iov_base = node_data(node);
- mdbx_cassert(mc, (char *)olddata.iov_base + olddata.iov_len <=
- (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize);
+ mdbx_debug("root page %" PRIaPGNO
+ " doesn't need rebalancing (flags 0x%x)",
+ mp->mp_pgno, mp->mp_flags);
+ }
+ return MDBX_SUCCESS;
+ }
- /* DB has dups? */
- if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) {
- /* Prepare (sub-)page/sub-DB to accept the new item, if needed.
- * fp: old sub-page or a header faking it.
- * mp: new (sub-)page. offset: growth in page size.
- * xdata: node data with new page or DB. */
- unsigned i;
- size_t offset = 0;
- MDBX_page *mp = fp = xdata.iov_base = env->me_pbuf;
- mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno;
+ /* The parent (branch page) must have at least 2 pointers,
+ * otherwise the tree is invalid. */
+ const unsigned pre_top = mc->mc_top - 1;
+ mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[pre_top]));
+ mdbx_cassert(mc, !IS_SUBP(mc->mc_pg[0]));
+ mdbx_cassert(mc, page_numkeys(mc->mc_pg[pre_top]) > 1);
- /* Was a single item before, must convert now */
- if (!F_ISSET(node_flags(node), F_DUPDATA)) {
+ /* Leaf page fill factor is below the threshold.
+ * Try to move keys from left or right neighbor, or
+ * merge with a neighbor page. */
- /* does data match? */
- const int cmp = mc->mc_dbx->md_dcmp(data, &olddata);
- if ((flags & MDBX_APPENDDUP) && unlikely(cmp <= 0))
- return MDBX_EKEYMISMATCH;
- if (cmp == 0) {
- if (flags & MDBX_NODUPDATA)
- return MDBX_KEYEXIST;
- if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata))) {
- /* data is match exactly byte-to-byte, nothing to update */
- if (unlikely(flags & MDBX_MULTIPLE)) {
- rc = MDBX_SUCCESS;
- goto continue_multiple;
- }
- return MDBX_SUCCESS;
- } else {
- /* The data has differences, but the user-provided comparator
- * considers them equal. So continue update since called without.
- * Continue to update since was called without MDBX_NODUPDATA. */
- }
- mdbx_cassert(mc, node_size(key, data) <= env->me_leaf_nodemax);
- goto current;
- }
+ /* Find neighbors. */
+ MDBX_cursor mn;
+ cursor_copy(mc, &mn);
- /* Just overwrite the current item */
- if (flags & MDBX_CURRENT) {
- mdbx_cassert(mc, node_size(key, data) <= env->me_leaf_nodemax);
- goto current;
- }
+ MDBX_page *left = nullptr, *right = nullptr;
+ if (mn.mc_ki[pre_top] > 0) {
+ rc = mdbx_page_get(
+ &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] - 1)),
+ &left, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ mdbx_cassert(mc, PAGETYPE(left) == PAGETYPE(mc->mc_pg[mc->mc_top]));
+ }
+ if (mn.mc_ki[pre_top] + 1u < page_numkeys(mn.mc_pg[pre_top])) {
+ rc = mdbx_page_get(
+ &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] + 1)),
+ &right, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ mdbx_cassert(mc, PAGETYPE(right) == PAGETYPE(mc->mc_pg[mc->mc_top]));
+ }
+ mdbx_cassert(mc, left || right);
+
+ const unsigned ki_top = mc->mc_ki[mc->mc_top];
+ const unsigned ki_pre_top = mn.mc_ki[pre_top];
+ const unsigned nkeys = page_numkeys(mn.mc_pg[mn.mc_top]);
+
+ const unsigned left_room = left ? page_room(left) : 0;
+ const unsigned right_room = right ? page_room(right) : 0;
+ const unsigned left_nkeys = left ? page_numkeys(left) : 0;
+ const unsigned right_nkeys = right ? page_numkeys(right) : 0;
+retry:
+ if (left_room > room_threshold && left_room >= right_room) {
+ /* try merge with left */
+ mdbx_cassert(mc, left_nkeys >= minkeys);
+ mn.mc_pg[mn.mc_top] = left;
+ mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1);
+ mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1);
+ mc->mc_ki[mc->mc_top] = 0;
+ const unsigned new_ki = ki_top + left_nkeys;
+ mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
+ /* We want mdbx_rebalance to find mn when doing fixups */
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_page_merge(mc, &mn));
+ if (likely(rc != MDBX_RESULT_TRUE)) {
+ cursor_restore(&mn, mc);
+ mc->mc_ki[mc->mc_top] = (indx_t)new_ki;
+ mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
+ return rc;
+ }
+ }
+ if (right_room > room_threshold) {
+ /* try merge with right */
+ mdbx_cassert(mc, right_nkeys >= minkeys);
+ mn.mc_pg[mn.mc_top] = right;
+ mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1);
+ mn.mc_ki[mn.mc_top] = 0;
+ mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_page_merge(&mn, mc));
+ if (likely(rc != MDBX_RESULT_TRUE)) {
+ mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
+ mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
+ return rc;
+ }
+ }
+
+ if (left_nkeys > minkeys &&
+ (right_nkeys <= left_nkeys || right_room >= left_room)) {
+ /* try move from left */
+ mn.mc_pg[mn.mc_top] = left;
+ mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1);
+ mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1);
+ mc->mc_ki[mc->mc_top] = 0;
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_node_move(&mn, mc, true));
+ if (likely(rc != MDBX_RESULT_TRUE)) {
+ mc->mc_ki[mc->mc_top] = (indx_t)(ki_top + 1);
+ mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
+ return rc;
+ }
+ }
+ if (right_nkeys > minkeys) {
+ /* try move from right */
+ mn.mc_pg[mn.mc_top] = right;
+ mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1);
+ mn.mc_ki[mn.mc_top] = 0;
+ mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
+ WITH_CURSOR_TRACKING(mn, rc = mdbx_node_move(&mn, mc, false));
+ if (likely(rc != MDBX_RESULT_TRUE)) {
+ mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
+ mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
+ return rc;
+ }
+ }
+
+ if (nkeys >= minkeys) {
+ mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
+ if (!mdbx_audit_enabled())
+ return MDBX_SUCCESS;
+ return mdbx_cursor_check(mc, C_UPDATING);
+ }
+
+ if (likely(room_threshold > 0)) {
+ room_threshold = 0;
+ goto retry;
+ }
+ mdbx_error("Unable to merge/rebalance %s page %" PRIaPGNO
+ " (has %u keys, full %.1f%%, used %u, room %u bytes )",
+ (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys,
+ page_fill(mc->mc_txn->mt_env, tp),
+ page_used(mc->mc_txn->mt_env, tp), room);
+ return MDBX_PROBLEM;
+}
+
+__cold static int mdbx_page_check(MDBX_cursor *const mc,
+ const MDBX_page *const mp, unsigned options) {
+ DKBUF;
+ options |= mc->mc_flags;
+ MDBX_env *const env = mc->mc_txn->mt_env;
+ const unsigned nkeys = page_numkeys(mp);
+ char *const end_of_page = (char *)mp + env->me_psize;
+ if (unlikely(mp->mp_pgno < MIN_PAGENO || mp->mp_pgno > MAX_PAGENO))
+ return bad_page(mp, "invalid pgno (%u)\n", mp->mp_pgno);
+ if (IS_OVERFLOW(mp)) {
+ if (unlikely(mp->mp_pages < 1 && mp->mp_pages >= MAX_PAGENO / 2))
+ return bad_page(mp, "invalid overflow n-pages (%u)\n", mp->mp_pages);
+ if (unlikely(mp->mp_pgno + mp->mp_pages > mc->mc_txn->mt_next_pgno))
+ return bad_page(mp, "overflow page beyond (%u) next-pgno\n",
+ mp->mp_pgno + mp->mp_pages);
+ if (unlikely((options & (C_SUB | C_COPYING)) == C_SUB))
+ return bad_page(mp,
+ "unexpected overflow-page for dupsort db (flags 0x%x)\n",
+ mc->mc_db->md_flags);
+ return MDBX_SUCCESS;
+ }
- /* Back up original data item */
- memcpy(dkey.iov_base = fp + 1, olddata.iov_base,
- dkey.iov_len = olddata.iov_len);
- dupdata_flag = 1;
+ int rc = MDBX_SUCCESS;
+ if ((options & C_UPDATING) == 0 || !IS_MODIFIABLE(mc->mc_txn, mp)) {
+ if (unlikely(nkeys < 2 && IS_BRANCH(mp)))
+ rc = bad_page(mp, "branch-page nkey (%u) < 2\n", nkeys);
+ }
+ if (IS_LEAF2(mp) && unlikely((options & (C_SUB | C_COPYING)) == 0))
+ rc = bad_page(mp, "unexpected leaf2-page (db flags 0x%x)\n",
+ mc->mc_db->md_flags);
- /* Make sub-page header for the dup items, with dummy body */
- fp->mp_flags = P_LEAF | P_SUBP;
- fp->mp_lower = 0;
- xdata.iov_len = PAGEHDRSZ + dkey.iov_len + data->iov_len;
- if (mc->mc_db->md_flags & MDBX_DUPFIXED) {
- fp->mp_flags |= P_LEAF2;
- fp->mp_leaf2_ksize = (uint16_t)data->iov_len;
- xdata.iov_len += 2 * data->iov_len; /* leave space for 2 more */
- mdbx_cassert(mc, xdata.iov_len <= env->me_psize);
- } else {
- xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) +
- (dkey.iov_len & 1) + (data->iov_len & 1);
- mdbx_cassert(mc, xdata.iov_len <= env->me_psize);
- }
- fp->mp_upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ);
- olddata.iov_len = xdata.iov_len; /* pretend olddata is fp */
- } else if (node_flags(node) & F_SUBDATA) {
- /* Data is on sub-DB, just store it */
- flags |= F_DUPDATA | F_SUBDATA;
- goto put_sub;
- } else {
- /* Data is on sub-page */
- fp = olddata.iov_base;
- switch (flags) {
- default:
- if (!(mc->mc_db->md_flags & MDBX_DUPFIXED)) {
- offset = node_size(data, nullptr) + sizeof(indx_t);
- break;
- }
- offset = fp->mp_leaf2_ksize;
- if (page_room(fp) < offset) {
- offset *= 4; /* space for 4 more */
- break;
- }
- /* FALLTHRU: Big enough MDBX_DUPFIXED sub-page */
- __fallthrough;
- case MDBX_CURRENT | MDBX_NODUPDATA:
- case MDBX_CURRENT:
- fp->mp_txnid = mc->mc_txn->mt_front;
- fp->mp_pgno = mp->mp_pgno;
- mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;
- flags |= F_DUPDATA;
- goto put_sub;
- }
- xdata.iov_len = olddata.iov_len + offset;
- }
+ MDBX_val here, prev = {0, 0};
+ for (unsigned i = 0; i < nkeys; ++i) {
+ if (IS_LEAF2(mp)) {
+ const size_t ksize = mp->mp_leaf2_ksize;
+ char *const key = page_leaf2key(mp, i, ksize);
+ if (unlikely(end_of_page < key + ksize)) {
+ rc = bad_page(mp, "leaf2-key beyond (%zu) page-end\n",
+ key + ksize - end_of_page);
+ continue;
+ }
- fp_flags = fp->mp_flags;
- if (node_size_len(node_ks(node), xdata.iov_len) >
- env->me_leaf_nodemax) {
- /* Too big for a sub-page, convert to sub-DB */
- fp_flags &= ~P_SUBP;
- prep_subDB:
- nested_dupdb.md_xsize = 0;
- nested_dupdb.md_flags = flags_db2sub(mc->mc_db->md_flags);
- if (mc->mc_db->md_flags & MDBX_DUPFIXED) {
- fp_flags |= P_LEAF2;
- nested_dupdb.md_xsize = fp->mp_leaf2_ksize;
- }
- nested_dupdb.md_depth = 1;
- nested_dupdb.md_branch_pages = 0;
- nested_dupdb.md_leaf_pages = 1;
- nested_dupdb.md_overflow_pages = 0;
- nested_dupdb.md_entries = page_numkeys(fp);
- xdata.iov_len = sizeof(nested_dupdb);
- xdata.iov_base = &nested_dupdb;
- const struct page_result par = mdbx_page_alloc(mc, 1, MDBX_ALLOC_ALL);
- mp = par.page;
- if (unlikely(par.err != MDBX_SUCCESS))
- return par.err;
- mc->mc_db->md_leaf_pages += 1;
- mdbx_cassert(mc, env->me_psize > olddata.iov_len);
- offset = env->me_psize - (unsigned)olddata.iov_len;
- flags |= F_DUPDATA | F_SUBDATA;
- nested_dupdb.md_root = mp->mp_pgno;
- nested_dupdb.md_seq = 0;
- nested_dupdb.md_mod_txnid = mc->mc_txn->mt_txnid;
- sub_root = mp;
+ if ((options & C_COPYING) == 0) {
+ if (unlikely(ksize != mc->mc_dbx->md_klen_min)) {
+ if (unlikely(ksize < mc->mc_dbx->md_klen_min ||
+ ksize > mc->mc_dbx->md_klen_max))
+ rc = bad_page(
+ mp, "leaf2-key size (%zu) <> min/max key-length (%zu/%zu)\n",
+ ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max);
+ else
+ mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = ksize;
}
- if (mp != fp) {
- mp->mp_flags = fp_flags;
- mp->mp_txnid = mc->mc_txn->mt_front;
- mp->mp_leaf2_ksize = fp->mp_leaf2_ksize;
- mp->mp_lower = fp->mp_lower;
- mdbx_cassert(mc, fp->mp_upper + offset <= UINT16_MAX);
- mp->mp_upper = (indx_t)(fp->mp_upper + offset);
- if (unlikely(fp_flags & P_LEAF2)) {
- memcpy(page_data(mp), page_data(fp),
- page_numkeys(fp) * fp->mp_leaf2_ksize);
- } else {
- memcpy((char *)mp + mp->mp_upper + PAGEHDRSZ,
- (char *)fp + fp->mp_upper + PAGEHDRSZ,
- olddata.iov_len - fp->mp_upper - PAGEHDRSZ);
- memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs),
- page_numkeys(fp) * sizeof(mp->mp_ptrs[0]));
- for (i = 0; i < page_numkeys(fp); i++) {
- mdbx_cassert(mc, mp->mp_ptrs[i] + offset <= UINT16_MAX);
- mp->mp_ptrs[i] += (indx_t)offset;
- }
- }
+ if ((options & C_SKIPORD) == 0) {
+ here.iov_len = ksize;
+ here.iov_base = key;
+ if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0))
+ rc = bad_page(mp, "leaf2-key #%u wrong order (%s >= %s)\n", i,
+ DKEY(&prev), DVAL(&here));
+ prev = here;
}
-
- rdata = &xdata;
- flags |= F_DUPDATA;
- do_sub = true;
- if (!insert_key)
- mdbx_node_del(mc, 0);
- goto new_sub;
}
-
- /* MDBX passes F_SUBDATA in 'flags' to write a DB record */
- if (unlikely((node_flags(node) ^ flags) & F_SUBDATA))
- return MDBX_INCOMPATIBLE;
-
- current:
- if (data->iov_len == olddata.iov_len) {
- mdbx_cassert(mc, EVEN(key->iov_len) == EVEN(node_ks(node)));
- /* same size, just replace it. Note that we could
- * also reuse this node if the new data is smaller,
- * but instead we opt to shrink the node in that case. */
- if (F_ISSET(flags, MDBX_RESERVE))
- data->iov_base = olddata.iov_base;
- else if (!(mc->mc_flags & C_SUB))
- memcpy(olddata.iov_base, data->iov_base, data->iov_len);
- else {
- mdbx_cassert(mc, page_numkeys(mc->mc_pg[mc->mc_top]) == 1);
- mdbx_cassert(mc, PAGETYPE(mc->mc_pg[mc->mc_top]) == P_LEAF);
- mdbx_cassert(mc, node_ds(node) == 0);
- mdbx_cassert(mc, node_flags(node) == 0);
- mdbx_cassert(mc, key->iov_len < UINT16_MAX);
- node_set_ks(node, key->iov_len);
- memcpy(node_key(node), key->iov_base, key->iov_len);
- mdbx_cassert(mc, (char *)node_key(node) + node_ds(node) <
- (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize);
- goto fix_parent;
- }
-
- if (mdbx_audit_enabled()) {
- err = mdbx_cursor_check(mc, 0);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ } else {
+ const MDBX_node *const node = page_node(mp, i);
+ const char *node_end = (char *)node + NODESIZE;
+ if (unlikely(node_end > end_of_page)) {
+ rc = bad_page(mp, "node[%u] (%zu) beyond page-end\n", i,
+ node_end - end_of_page);
+ continue;
+ }
+ size_t ksize = node_ks(node);
+ char *key = node_key(node);
+ if (unlikely(end_of_page < key + ksize)) {
+ rc = bad_page(mp, "node[%u] key (%zu) beyond page-end\n", i,
+ key + ksize - end_of_page);
+ continue;
+ }
+ if ((IS_LEAF(mp) || i > 0) && (options & C_COPYING) == 0) {
+ if (unlikely(ksize < mc->mc_dbx->md_klen_min ||
+ ksize > mc->mc_dbx->md_klen_max))
+ rc = bad_page(
+ mp, "node[%u] key size (%zu) <> min/max key-length (%zu/%zu)\n",
+ i, ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max);
+ if ((options & C_SKIPORD) == 0) {
+ here.iov_base = key;
+ here.iov_len = ksize;
+ if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0))
+ rc = bad_page(mp, "node[%u] key wrong order (%s >= %s)\n", i,
+ DKEY(&prev), DVAL(&here));
+ prev = here;
}
- return MDBX_SUCCESS;
}
- }
- mdbx_node_del(mc, 0);
- }
-
- rdata = data;
-
-new_sub:;
- unsigned nflags = flags & NODE_ADD_FLAGS;
- size_t nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->iov_len
- : leaf_size(env, key, rdata);
- if (page_room(mc->mc_pg[mc->mc_top]) < nsize) {
- if (!insert_key)
- nflags |= MDBX_SPLIT_REPLACE;
- rc = mdbx_page_split(mc, key, rdata, P_INVALID, nflags);
- if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
- rc = mdbx_cursor_check(mc, 0);
- } else {
- /* There is room already in this leaf page. */
- if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
- mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0 &&
- rdata->iov_len == 0);
- rc = mdbx_node_add_leaf2(mc, mc->mc_ki[mc->mc_top], key);
- } else
- rc = mdbx_node_add_leaf(mc, mc->mc_ki[mc->mc_top], key, rdata, nflags);
- if (likely(rc == 0)) {
- /* Adjust other cursors pointing to mp */
- const MDBX_dbi dbi = mc->mc_dbi;
- const unsigned i = mc->mc_top;
- MDBX_page *const mp = mc->mc_pg[i];
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2;
- m2 = m2->mc_next) {
- MDBX_cursor *m3 =
- (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp)
- continue;
- if (m3->mc_ki[i] >= mc->mc_ki[i])
- m3->mc_ki[i] += insert_key;
- if (XCURSOR_INITED(m3))
- XCURSOR_REFRESH(m3, mp, m3->mc_ki[i]);
+ if (IS_BRANCH(mp)) {
+ if ((options & C_UPDATING) == 0 && i == 0 && unlikely(ksize != 0))
+ rc = bad_page(mp, "branch-node[%u] wrong 0-node key-length (%zu)\n",
+ i, ksize);
+ if ((options & C_RETIRING) == 0) {
+ const pgno_t ref = node_pgno(node);
+ if (unlikely(ref < MIN_PAGENO || ref >= mc->mc_txn->mt_next_pgno))
+ rc = bad_page(mp, "branch-node[%u] wrong pgno (%u)\n", i, ref);
+ }
+ if (unlikely(node_flags(node)))
+ rc = bad_page(mp, "branch-node[%u] wrong flags (%u)\n", i,
+ node_flags(node));
+ continue;
}
- }
- }
- if (likely(rc == MDBX_SUCCESS)) {
- /* Now store the actual data in the child DB. Note that we're
- * storing the user data in the keys field, so there are strict
- * size limits on dupdata. The actual data fields of the child
- * DB are all zero size. */
- if (do_sub) {
- int xflags;
- size_t ecount;
- put_sub:
- xdata.iov_len = 0;
- xdata.iov_base = nullptr;
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-#define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1
- STATIC_ASSERT(
- (MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) ==
- MDBX_NOOVERWRITE);
- xflags = MDBX_CURRENT | MDBX_NOSPILL |
- ((flags & MDBX_NODUPDATA) >>
- SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE);
- if ((flags & MDBX_CURRENT) == 0) {
- xflags -= MDBX_CURRENT;
- err = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- if (sub_root)
- mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root;
- /* converted, write the original data first */
- if (dupdata_flag) {
- rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
- if (unlikely(rc))
- goto bad_sub;
- /* we've done our job */
- dkey.iov_len = 0;
+ switch (node_flags(node)) {
+ default:
+ rc = bad_page(mp, "invalid node[%u] flags (%u)\n", i, node_flags(node));
+ break;
+ case F_BIGDATA /* data on large-page */:
+ case 0 /* usual */:
+ case F_SUBDATA /* sub-db */:
+ case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
+ case F_DUPDATA /* short sub-page */:
+ break;
}
- if (!(node_flags(node) & F_SUBDATA) || sub_root) {
- /* Adjust other cursors pointing to mp */
- MDBX_cursor *m2;
- MDBX_xcursor *mx = mc->mc_xcursor;
- unsigned i = mc->mc_top;
- MDBX_page *mp = mc->mc_pg[i];
- const int nkeys = page_numkeys(mp);
- for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) {
- if (m2 == mc || m2->mc_snum < mc->mc_snum)
- continue;
- if (!(m2->mc_flags & C_INITIALIZED))
+ const size_t dsize = node_ds(node);
+ const char *const data = node_data(node);
+ if (node_flags(node) & F_BIGDATA) {
+ if (unlikely(end_of_page < data + sizeof(pgno_t))) {
+ rc = bad_page(
+ mp, "node-%s(%u of %u, %zu bytes) beyond (%zu) page-end\n",
+ "bigdata-pgno", i, nkeys, dsize, data + dsize - end_of_page);
+ continue;
+ }
+ if ((options & C_COPYING) == 0) {
+ if (unlikely(dsize <= mc->mc_dbx->md_vlen_min ||
+ dsize > mc->mc_dbx->md_vlen_max))
+ rc = bad_page(
+ mp,
+ "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n",
+ dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max);
+ }
+ if ((options & C_RETIRING) == 0) {
+ MDBX_page *lp;
+ int err = mdbx_page_get(mc, node_largedata_pgno(node), &lp,
+ pp_txnid4chk(mp, mc->mc_txn));
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ if (unlikely(!IS_OVERFLOW(lp))) {
+ rc = bad_page(mp, "big-node refs to non-overflow page (%u)\n",
+ lp->mp_pgno);
continue;
- if (m2->mc_pg[i] == mp) {
- if (m2->mc_ki[i] == mc->mc_ki[i]) {
- err = mdbx_xcursor_init2(m2, mx, dupdata_flag);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- } else if (!insert_key && m2->mc_ki[i] < nkeys) {
- XCURSOR_REFRESH(m2, mp, m2->mc_ki[i]);
- }
}
+ if (unlikely(number_of_ovpages(env, dsize) > lp->mp_pages))
+ rc =
+ bad_page(mp, "big-node size (%zu) mismatch n-pages size (%u)\n",
+ dsize, lp->mp_pages);
}
+ continue;
}
- mdbx_cassert(mc, mc->mc_xcursor->mx_db.md_entries < PTRDIFF_MAX);
- ecount = (size_t)mc->mc_xcursor->mx_db.md_entries;
-#define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1
- STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) ==
- MDBX_APPEND);
- xflags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND;
- rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
- if (flags & F_SUBDATA) {
- void *db = node_data(node);
- mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid;
- memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db));
+
+ if (unlikely(end_of_page < data + dsize)) {
+ rc =
+ bad_page(mp, "node-%s(%u of %u, %zu bytes) beyond (%zu) page-end\n",
+ "data", i, nkeys, dsize, data + dsize - end_of_page);
+ continue;
}
- insert_data = (ecount != (size_t)mc->mc_xcursor->mx_db.md_entries);
- }
- /* Increment count unless we just replaced an existing item. */
- if (insert_data)
- mc->mc_db->md_entries++;
- if (insert_key) {
- /* Invalidate txn if we created an empty sub-DB */
- if (unlikely(rc))
- goto bad_sub;
- /* If we succeeded and the key didn't exist before,
- * make sure the cursor is marked valid. */
- mc->mc_flags |= C_INITIALIZED;
- }
- if (unlikely(flags & MDBX_MULTIPLE)) {
- if (likely(rc == MDBX_SUCCESS)) {
- continue_multiple:
- mcount++;
- /* let caller know how many succeeded, if any */
- data[1].iov_len = mcount;
- if (mcount < dcount) {
- data[0].iov_base = (char *)data[0].iov_base + data[0].iov_len;
- insert_key = insert_data = false;
- goto more;
+
+ switch (node_flags(node)) {
+ default:
+ /* wrong, but already handled */
+ continue;
+ case 0 /* usual */:
+ if ((options & C_COPYING) == 0) {
+ if (unlikely(dsize < mc->mc_dbx->md_vlen_min ||
+ dsize > mc->mc_dbx->md_vlen_max)) {
+ rc = bad_page(
+ mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n",
+ dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max);
+ continue;
+ }
+ }
+ break;
+ case F_SUBDATA /* sub-db */:
+ if (unlikely(dsize != sizeof(MDBX_db))) {
+ rc = bad_page(mp, "invalid sub-db record size (%zu)\n", dsize);
+ continue;
+ }
+ break;
+ case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
+ if (unlikely(dsize != sizeof(MDBX_db))) {
+ rc = bad_page(mp, "invalid nested-db record size (%zu)\n", dsize);
+ continue;
+ }
+ break;
+ case F_DUPDATA /* short sub-page */:
+ if (unlikely(dsize <= PAGEHDRSZ)) {
+ rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n",
+ dsize);
+ continue;
+ } else {
+ const MDBX_page *const sp = (MDBX_page *)data;
+ const char *const end_of_subpage = data + dsize;
+ const int nsubkeys = page_numkeys(sp);
+ switch (sp->mp_flags & /* ignore legacy P_DIRTY flag */ ~0x10) {
+ case P_LEAF | P_SUBP:
+ case P_LEAF | P_LEAF2 | P_SUBP:
+ break;
+ default:
+ rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n",
+ sp->mp_flags);
+ continue;
+ }
+
+ MDBX_val sub_here, sub_prev = {0, 0};
+ for (int j = 0; j < nsubkeys; j++) {
+ if (IS_LEAF2(sp)) {
+ /* LEAF2 pages have no mp_ptrs[] or node headers */
+ size_t sub_ksize = sp->mp_leaf2_ksize;
+ char *sub_key = page_leaf2key(sp, j, sub_ksize);
+ if (unlikely(end_of_subpage < sub_key + sub_ksize)) {
+ rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n",
+ sub_key + sub_ksize - end_of_subpage);
+ continue;
+ }
+
+ if ((options & C_COPYING) == 0) {
+ if (unlikely(sub_ksize != mc->mc_dbx->md_vlen_min)) {
+ if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min ||
+ sub_ksize > mc->mc_dbx->md_vlen_max)) {
+ rc = bad_page(mp,
+ "nested-leaf2-key size (%zu) <> min/max "
+ "value-length (%zu/%zu)\n",
+ sub_ksize, mc->mc_dbx->md_vlen_min,
+ mc->mc_dbx->md_vlen_max);
+ continue;
+ }
+ mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = sub_ksize;
+ }
+ if ((options & C_SKIPORD) == 0) {
+ sub_here.iov_len = sub_ksize;
+ sub_here.iov_base = sub_key;
+ if (sub_prev.iov_base &&
+ unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0))
+ rc = bad_page(
+ mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", j,
+ DKEY(&sub_prev), DVAL(&sub_here));
+ sub_prev = sub_here;
+ }
+ }
+ } else {
+ const MDBX_node *const sub_node = page_node(sp, j);
+ const char *sub_node_end = (char *)sub_node + NODESIZE;
+ if (unlikely(sub_node_end > end_of_subpage)) {
+ rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n",
+ end_of_subpage - sub_node_end);
+ continue;
+ }
+ if (unlikely(node_flags(sub_node) != 0))
+ rc = bad_page(mp, "nested-node invalid flags (%u)\n",
+ node_flags(sub_node));
+
+ size_t sub_ksize = node_ks(sub_node);
+ char *sub_key = node_key(sub_node);
+ size_t sub_dsize = node_ds(sub_node);
+ /* char *sub_data = node_data(sub_node); */
+
+ if ((options & C_COPYING) == 0) {
+ if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min ||
+ sub_ksize > mc->mc_dbx->md_vlen_max))
+ rc = bad_page(mp,
+ "nested-node-key size (%zu) <> min/max "
+ "value-length (%zu/%zu)\n",
+ sub_ksize, mc->mc_dbx->md_vlen_min,
+ mc->mc_dbx->md_vlen_max);
+
+ if ((options & C_SKIPORD) == 0) {
+ sub_here.iov_len = sub_ksize;
+ sub_here.iov_base = sub_key;
+ if (sub_prev.iov_base &&
+ unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0))
+ rc = bad_page(
+ mp, "nested-node-key #%u wrong order (%s >= %s)\n", j,
+ DKEY(&sub_prev), DVAL(&sub_here));
+ sub_prev = sub_here;
+ }
+ }
+ if (unlikely(sub_dsize != 0))
+ rc = bad_page(mp, "nested-node non-empty data size (%zu)\n",
+ sub_dsize);
+ if (unlikely(end_of_subpage < sub_key + sub_ksize))
+ rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n",
+ sub_key + sub_ksize - end_of_subpage);
+ }
+ }
}
+ break;
}
}
- if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
- rc = mdbx_cursor_check(mc, 0);
- return rc;
- bad_sub:
- if (unlikely(rc == MDBX_KEYEXIST)) {
- /* should not happen, we deleted that item */
- mdbx_error("Unexpected %i error while put to nested dupsort's hive", rc);
- rc = MDBX_PROBLEM;
- }
}
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
return rc;
}
-int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) {
- if (unlikely(!mc))
- return MDBX_EINVAL;
+__cold static int mdbx_cursor_check(MDBX_cursor *mc, unsigned options) {
+ mdbx_cassert(mc,
+ mc->mc_txn->tw.dirtyroom + mc->mc_txn->tw.dirtylist->length ==
+ (mc->mc_txn->mt_parent
+ ? mc->mc_txn->mt_parent->tw.dirtyroom
+ : mc->mc_txn->mt_env->me_options.dp_limit));
+ mdbx_cassert(mc, mc->mc_top == mc->mc_snum - 1 || (options & C_UPDATING));
+ if (unlikely(mc->mc_top != mc->mc_snum - 1) && (options & C_UPDATING) == 0)
+ return MDBX_CURSOR_FULL;
+ mdbx_cassert(mc, (options & C_UPDATING) ? mc->mc_snum <= mc->mc_db->md_depth
+ : mc->mc_snum == mc->mc_db->md_depth);
+ if (unlikely((options & C_UPDATING) ? mc->mc_snum > mc->mc_db->md_depth
+ : mc->mc_snum != mc->mc_db->md_depth))
+ return MDBX_CURSOR_FULL;
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ for (int n = 0; n < (int)mc->mc_snum; ++n) {
+ MDBX_page *mp = mc->mc_pg[n];
+ const unsigned nkeys = page_numkeys(mp);
+ const bool expect_branch = (n < mc->mc_db->md_depth - 1) ? true : false;
+ const bool expect_nested_leaf =
+ (n + 1 == mc->mc_db->md_depth - 1) ? true : false;
+ const bool branch = IS_BRANCH(mp) ? true : false;
+ mdbx_cassert(mc, branch == expect_branch);
+ if (unlikely(branch != expect_branch))
+ return MDBX_CURSOR_FULL;
+ if ((options & C_UPDATING) == 0) {
+ mdbx_cassert(mc,
+ nkeys > mc->mc_ki[n] || (!branch && nkeys == mc->mc_ki[n] &&
+ (mc->mc_flags & C_EOF) != 0));
+ if (unlikely(nkeys <= mc->mc_ki[n] &&
+ !(!branch && nkeys == mc->mc_ki[n] &&
+ (mc->mc_flags & C_EOF) != 0)))
+ return MDBX_CURSOR_FULL;
+ } else {
+ mdbx_cassert(mc, nkeys + 1 >= mc->mc_ki[n]);
+ if (unlikely(nkeys + 1 < mc->mc_ki[n]))
+ return MDBX_CURSOR_FULL;
+ }
- int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ int err = mdbx_page_check(mc, mp, options);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- if (unlikely(TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)))
- return MDBX_BAD_DBI;
+ for (unsigned i = 0; i < nkeys; ++i) {
+ if (branch) {
+ MDBX_node *node = page_node(mp, i);
+ mdbx_cassert(mc, node_flags(node) == 0);
+ if (unlikely(node_flags(node) != 0))
+ return MDBX_CURSOR_FULL;
+ pgno_t pgno = node_pgno(node);
+ MDBX_page *np;
+ int rc = mdbx_page_get(mc, pgno, &np, pp_txnid4chk(mp, mc->mc_txn));
+ mdbx_cassert(mc, rc == MDBX_SUCCESS);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ const bool nested_leaf = IS_LEAF(np) ? true : false;
+ mdbx_cassert(mc, nested_leaf == expect_nested_leaf);
+ if (unlikely(nested_leaf != expect_nested_leaf))
+ return MDBX_CURSOR_FULL;
+ err = mdbx_page_check(mc, np, options);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ }
+ }
+ return MDBX_SUCCESS;
+}
- if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
- return MDBX_ENODATA;
+/* Complete a delete operation started by mdbx_cursor_del(). */
+static int mdbx_cursor_del0(MDBX_cursor *mc) {
+ int rc;
+ MDBX_page *mp;
+ indx_t ki;
+ unsigned nkeys;
+ MDBX_dbi dbi = mc->mc_dbi;
- if (unlikely(mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])))
- return MDBX_NOTFOUND;
+ mdbx_cassert(mc, cursor_is_tracked(mc));
+ mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
+ ki = mc->mc_ki[mc->mc_top];
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_node_del(mc, mc->mc_db->md_xsize);
+ mc->mc_db->md_entries--;
- if (likely((flags & MDBX_NOSPILL) == 0) &&
- unlikely(rc = mdbx_cursor_spill(mc, NULL, NULL)))
- return rc;
+ /* Adjust other cursors pointing to mp */
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
+ MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == mc || !(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
+ continue;
+ if (m3->mc_snum < mc->mc_snum)
+ continue;
+ if (m3->mc_pg[mc->mc_top] == mp) {
+ if (m3->mc_ki[mc->mc_top] == ki) {
+ m3->mc_flags |= C_DEL;
+ if (mc->mc_db->md_flags & MDBX_DUPSORT) {
+ /* Sub-cursor referred into dataset which is gone */
+ m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
+ }
+ continue;
+ } else if (m3->mc_ki[mc->mc_top] > ki) {
+ m3->mc_ki[mc->mc_top]--;
+ }
+ if (XCURSOR_INITED(m3))
+ XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+ }
+ }
- rc = mdbx_cursor_touch(mc);
+ rc = mdbx_rebalance(mc);
if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ goto bailout;
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(!IS_LEAF(mp)))
- return MDBX_CORRUPTED;
- if (IS_LEAF2(mp)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely((mc->mc_flags & C_SUB) == 0)) {
- mdbx_error("unexpected LEAF2-page %" PRIaPGNO "for non-dupsort cursor",
- mp->mp_pgno);
- return MDBX_CORRUPTED;
- }
- goto del_key;
+ if (unlikely(!mc->mc_snum)) {
+ /* DB is totally empty now, just bail out.
+ * Other cursors adjustments were already done
+ * by mdbx_rebalance and aren't needed here. */
+ mdbx_cassert(mc, mc->mc_db->md_entries == 0 && mc->mc_db->md_depth == 0 &&
+ mc->mc_db->md_root == P_INVALID);
+ mc->mc_flags |= C_EOF;
+ return MDBX_SUCCESS;
}
- MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- if (flags & (MDBX_ALLDUPS | /* for compatibility */ MDBX_NODUPDATA)) {
- /* mdbx_cursor_del0() will subtract the final entry */
- mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1;
- mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
- } else {
- if (!F_ISSET(node_flags(node), F_SUBDATA))
- mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
- rc = mdbx_cursor_del(&mc->mc_xcursor->mx_cursor, MDBX_NOSPILL);
- if (unlikely(rc))
- return rc;
- /* If sub-DB still has entries, we're done */
- if (mc->mc_xcursor->mx_db.md_entries) {
- if (node_flags(node) & F_SUBDATA) {
- /* update subDB info */
- void *db = node_data(node);
- mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid;
- memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db));
- } else {
- MDBX_cursor *m2;
- /* shrink fake page */
- mdbx_node_shrink(mp, mc->mc_ki[mc->mc_top]);
- node = page_node(mp, mc->mc_ki[mc->mc_top]);
- mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
- /* fix other sub-DB cursors pointed at fake pages on this page */
- for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) {
- if (m2 == mc || m2->mc_snum < mc->mc_snum)
- continue;
- if (!(m2->mc_flags & C_INITIALIZED))
- continue;
- if (m2->mc_pg[mc->mc_top] == mp) {
- MDBX_node *inner = node;
- if (m2->mc_ki[mc->mc_top] >= page_numkeys(mp))
- continue;
- if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) {
- inner = page_node(mp, m2->mc_ki[mc->mc_top]);
- if (node_flags(inner) & F_SUBDATA)
- continue;
- }
- m2->mc_xcursor->mx_cursor.mc_pg[0] = node_data(inner);
+ ki = mc->mc_ki[mc->mc_top];
+ mp = mc->mc_pg[mc->mc_top];
+ mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
+ nkeys = page_numkeys(mp);
+ mdbx_cassert(mc, (mc->mc_db->md_entries > 0 && nkeys > 0) ||
+ ((mc->mc_flags & C_SUB) && mc->mc_db->md_entries == 0 &&
+ nkeys == 0));
+
+ /* Adjust this and other cursors pointing to mp */
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
+ MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
+ continue;
+ if (m3->mc_snum < mc->mc_snum)
+ continue;
+ if (m3->mc_pg[mc->mc_top] == mp) {
+ /* if m3 points past last node in page, find next sibling */
+ if (m3->mc_ki[mc->mc_top] >= nkeys) {
+ rc = mdbx_cursor_sibling(m3, SIBLING_RIGHT);
+ if (rc == MDBX_NOTFOUND) {
+ m3->mc_flags |= C_EOF;
+ rc = MDBX_SUCCESS;
+ continue;
+ }
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ if (m3->mc_ki[mc->mc_top] >= ki ||
+ /* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) {
+ if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
+ MDBX_node *node =
+ page_node(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
+ /* If this node has dupdata, it may need to be reinited
+ * because its data has moved.
+ * If the xcursor was not inited it must be reinited.
+ * Else if node points to a subDB, nothing is needed. */
+ if (node_flags(node) & F_DUPDATA) {
+ if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+ if (!(node_flags(node) & F_SUBDATA))
+ m3->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
+ } else {
+ rc = mdbx_xcursor_init1(m3, node, m3->mc_pg[m3->mc_top]);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ rc = mdbx_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
}
}
- }
- mc->mc_db->md_entries--;
- mdbx_cassert(mc, mc->mc_db->md_entries > 0 && mc->mc_db->md_depth > 0 &&
- mc->mc_db->md_root != P_INVALID);
- return rc;
- } else {
- mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
+ m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
+ }
+ m3->mc_flags |= C_DEL;
}
- /* otherwise fall thru and delete the sub-DB */
- }
-
- if (node_flags(node) & F_SUBDATA) {
- /* add all the child DB's pages to the free list */
- rc = mdbx_drop_tree(&mc->mc_xcursor->mx_cursor, false);
- if (unlikely(rc))
- goto fail;
}
}
- /* MDBX passes F_SUBDATA in 'flags' to delete a DB record */
- else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA))
- return MDBX_INCOMPATIBLE;
-
- /* add overflow pages to free list */
- if (F_ISSET(node_flags(node), F_BIGDATA)) {
- MDBX_page *omp;
- if (unlikely((rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp,
- pp_txnid4chk(mp, mc->mc_txn))) ||
- (rc = mdbx_page_retire(mc, omp))))
- goto fail;
- }
-del_key:
- return mdbx_cursor_del0(mc);
+ mdbx_cassert(mc, rc == MDBX_SUCCESS);
+ if (mdbx_audit_enabled())
+ rc = mdbx_cursor_check(mc, 0);
+ return rc;
-fail:
+bailout:
mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
return rc;
}
-/* Allocate and initialize new pages for a database.
- * Set MDBX_TXN_ERROR on failure.
- *
- * [in] mc a cursor on the database being added to.
- * [in] flags flags defining what type of page is being allocated.
- * [in] num the number of pages to allocate. This is usually 1,
- * unless allocating overflow pages for a large record.
- * [out] mp Address of a page, or NULL on failure.
- *
- * Returns 0 on success, non-zero on failure. */
-static struct page_result mdbx_page_new(MDBX_cursor *mc, const unsigned flags,
- const unsigned npages) {
- struct page_result ret = mdbx_page_alloc(mc, npages, MDBX_ALLOC_ALL);
- if (unlikely(ret.err != MDBX_SUCCESS))
- return ret;
+int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
+ const MDBX_val *data) {
+ int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- mdbx_debug("db %u allocated new page %" PRIaPGNO ", num %u", mc->mc_dbi,
- ret.page->mp_pgno, npages);
- ret.page->mp_flags = (uint16_t)flags;
- ret.page->mp_txnid = mc->mc_txn->mt_front;
- mdbx_cassert(mc, *mc->mc_dbistate & DBI_DIRTY);
- mdbx_cassert(mc, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY);
-#if MDBX_ENABLE_PGOP_STAT
- mc->mc_txn->mt_env->me_lck->mti_pgop_stat.newly.weak += npages;
-#endif /* MDBX_ENABLE_PGOP_STAT */
+ if (unlikely(!key))
+ return MDBX_EINVAL;
- if (likely((flags & P_OVERFLOW) == 0)) {
- STATIC_ASSERT(P_BRANCH == 1);
- const bool is_branch = flags & P_BRANCH;
- ret.page->mp_lower = 0;
- ret.page->mp_upper = (indx_t)(mc->mc_txn->mt_env->me_psize - PAGEHDRSZ);
- mc->mc_db->md_branch_pages += is_branch;
- mc->mc_db->md_leaf_pages += 1 - is_branch;
- if (unlikely(mc->mc_flags & C_SUB)) {
- MDBX_db *outer = mdbx_outer_db(mc);
- outer->md_branch_pages += is_branch;
- outer->md_leaf_pages += 1 - is_branch;
- }
- } else {
- mc->mc_db->md_overflow_pages += npages;
- ret.page->mp_pages = npages;
- mdbx_cassert(mc, !(mc->mc_flags & C_SUB));
- }
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
- return ret;
+ if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
+ return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
+
+ return mdbx_del0(txn, dbi, key, data, 0);
}
-static int __must_check_result mdbx_node_add_leaf2(MDBX_cursor *mc,
- unsigned indx,
- const MDBX_val *key) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
+static int mdbx_del0(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
+ const MDBX_val *data, unsigned flags) {
+ MDBX_cursor_couple cx;
+ MDBX_cursor_op op;
+ MDBX_val rdata;
+ int rc;
DKBUF_DEBUG;
- mdbx_debug("add to leaf2-%spage %" PRIaPGNO " index %i, "
- " key size %" PRIuPTR " [%s]",
- IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx,
- key ? key->iov_len : 0, DKEY_DEBUG(key));
- mdbx_cassert(mc, key);
- mdbx_cassert(mc, PAGETYPE(mp) == (P_LEAF | P_LEAF2));
- const unsigned ksize = mc->mc_db->md_xsize;
- mdbx_cassert(mc, ksize == key->iov_len);
- const unsigned nkeys = page_numkeys(mp);
+ mdbx_debug("====> delete db %u key [%s], data [%s]", dbi, DKEY_DEBUG(key),
+ DVAL_DEBUG(data));
- /* Just using these for counting */
- const intptr_t lower = mp->mp_lower + sizeof(indx_t);
- const intptr_t upper = mp->mp_upper - (ksize - sizeof(indx_t));
- if (unlikely(lower > upper)) {
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return MDBX_PAGE_FULL;
- }
- mp->mp_lower = (indx_t)lower;
- mp->mp_upper = (indx_t)upper;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- char *const ptr = page_leaf2key(mp, indx, ksize);
- mdbx_cassert(mc, nkeys >= indx);
- const unsigned diff = nkeys - indx;
- if (likely(diff > 0))
- /* Move higher keys up one slot. */
- memmove(ptr + ksize, ptr, diff * ksize);
- /* insert new key */
- memcpy(ptr, key->iov_base, ksize);
- return MDBX_SUCCESS;
+ if (data) {
+ op = MDBX_GET_BOTH;
+ rdata = *data;
+ data = &rdata;
+ } else {
+ op = MDBX_SET;
+ flags |= MDBX_ALLDUPS;
+ }
+ rc = mdbx_cursor_set(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op).err;
+ if (likely(rc == MDBX_SUCCESS)) {
+ /* let mdbx_page_split know about this cursor if needed:
+ * delete will trigger a rebalance; if it needs to move
+ * a node from one page to another, it will have to
+ * update the parent's separator key(s). If the new sepkey
+ * is larger than the current one, the parent page may
+ * run out of space, triggering a split. We need this
+ * cursor to be consistent until the end of the rebalance. */
+ cx.outer.mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = &cx.outer;
+ rc = mdbx_cursor_del(&cx.outer, flags);
+ txn->mt_cursors[dbi] = cx.outer.mc_next;
+ }
+ return rc;
}
-static int __must_check_result mdbx_node_add_branch(MDBX_cursor *mc,
- unsigned indx,
- const MDBX_val *key,
- pgno_t pgno) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- DKBUF_DEBUG;
- mdbx_debug("add to branch-%spage %" PRIaPGNO " index %i, node-pgno %" PRIaPGNO
- " key size %" PRIuPTR " [%s]",
- IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, pgno,
- key ? key->iov_len : 0, DKEY_DEBUG(key));
-
- mdbx_cassert(mc, PAGETYPE(mp) == P_BRANCH);
- STATIC_ASSERT(NODESIZE % 2 == 0);
-
- /* Move higher pointers up one slot. */
- const unsigned nkeys = page_numkeys(mp);
- mdbx_cassert(mc, nkeys >= indx);
- for (unsigned i = nkeys; i > indx; --i)
- mp->mp_ptrs[i] = mp->mp_ptrs[i - 1];
+/* Split a page and insert a new node.
+ * Set MDBX_TXN_ERROR on failure.
+ * [in,out] mc Cursor pointing to the page and desired insertion index.
+ * The cursor will be updated to point to the actual page and index where
+ * the node got inserted after the split.
+ * [in] newkey The key for the newly inserted node.
+ * [in] newdata The data for the newly inserted node.
+ * [in] newpgno The page number, if the new node is a branch node.
+ * [in] nflags The NODE_ADD_FLAGS for the new node.
+ * Returns 0 on success, non-zero on failure. */
+static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
+ MDBX_val *const newdata, pgno_t newpgno,
+ unsigned nflags) {
+ unsigned flags;
+ int rc = MDBX_SUCCESS, foliage = 0;
+ unsigned i, ptop;
+ MDBX_env *const env = mc->mc_txn->mt_env;
+ MDBX_val sepkey, rkey, xdata;
+ MDBX_page *tmp_ki_copy = NULL;
+ DKBUF;
- /* Adjust free space offsets. */
- const size_t branch_bytes = branch_size(mc->mc_txn->mt_env, key);
- const intptr_t lower = mp->mp_lower + sizeof(indx_t);
- const intptr_t upper = mp->mp_upper - (branch_bytes - sizeof(indx_t));
- if (unlikely(lower > upper)) {
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return MDBX_PAGE_FULL;
+ MDBX_page *const mp = mc->mc_pg[mc->mc_top];
+ const unsigned newindx = mc->mc_ki[mc->mc_top];
+ unsigned nkeys = page_numkeys(mp);
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
- mp->mp_lower = (indx_t)lower;
- mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper;
+ STATIC_ASSERT(P_BRANCH == 1);
+ const unsigned minkeys = (mp->mp_flags & P_BRANCH) + 1;
- /* Write the node data. */
- MDBX_node *node = page_node(mp, indx);
- node_set_pgno(node, pgno);
- node_set_flags(node, 0);
- UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0);
- node_set_ks(node, 0);
- if (likely(key != NULL)) {
- node_set_ks(node, key->iov_len);
- memcpy(node_key(node), key->iov_base, key->iov_len);
- }
- return MDBX_SUCCESS;
-}
+ mdbx_debug(">> splitting %s-page %" PRIaPGNO
+ " and adding %zu+%zu [%s] at %i, nkeys %i",
+ IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, newkey->iov_len,
+ newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey),
+ mc->mc_ki[mc->mc_top], nkeys);
+ mdbx_cassert(mc, nkeys + 1 >= minkeys * 2);
-static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc,
- unsigned indx,
- const MDBX_val *key,
- MDBX_val *data,
- unsigned flags) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- DKBUF_DEBUG;
- mdbx_debug("add to leaf-%spage %" PRIaPGNO " index %i, data size %" PRIuPTR
- " key size %" PRIuPTR " [%s]",
- IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx,
- data ? data->iov_len : 0, key ? key->iov_len : 0, DKEY_DEBUG(key));
- mdbx_cassert(mc, key != NULL && data != NULL);
- mdbx_cassert(mc, PAGETYPE(mp) == P_LEAF);
- mdbx_cassert(mc, page_room(mp) >= leaf_size(mc->mc_txn->mt_env, key, data));
- MDBX_page *largepage = NULL;
+ /* Create a new sibling page. */
+ struct page_result npr = mdbx_page_new(mc, mp->mp_flags, 1);
+ if (unlikely(npr.err != MDBX_SUCCESS))
+ return npr.err;
+ MDBX_page *const sister = npr.page;
+ sister->mp_leaf2_ksize = mp->mp_leaf2_ksize;
+ mdbx_debug("new sibling: page %" PRIaPGNO, sister->mp_pgno);
+
+ /* Usually when splitting the root page, the cursor
+ * height is 1. But when called from mdbx_update_key,
+ * the cursor height may be greater because it walks
+ * up the stack while finding the branch slot to update. */
+ if (mc->mc_top < 1) {
+ npr = mdbx_page_new(mc, P_BRANCH, 1);
+ rc = npr.err;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ MDBX_page *const pp = npr.page;
+ /* shift current top to make room for new parent */
+ mdbx_cassert(mc, mc->mc_snum < 2 && mc->mc_db->md_depth > 0);
+#if MDBX_DEBUG
+ memset(mc->mc_pg + 3, 0, sizeof(mc->mc_pg) - sizeof(mc->mc_pg[0]) * 3);
+ memset(mc->mc_ki + 3, -1, sizeof(mc->mc_ki) - sizeof(mc->mc_ki[0]) * 3);
+#endif
+ mc->mc_pg[2] = mc->mc_pg[1];
+ mc->mc_ki[2] = mc->mc_ki[1];
+ mc->mc_pg[1] = mc->mc_pg[0];
+ mc->mc_ki[1] = mc->mc_ki[0];
+ mc->mc_pg[0] = pp;
+ mc->mc_ki[0] = 0;
+ mc->mc_db->md_root = pp->mp_pgno;
+ mdbx_debug("root split! new root = %" PRIaPGNO, pp->mp_pgno);
+ foliage = mc->mc_db->md_depth++;
- size_t node_bytes;
- if (unlikely(flags & F_BIGDATA)) {
- /* Data already on overflow page. */
- STATIC_ASSERT(sizeof(pgno_t) % 2 == 0);
- node_bytes =
- node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
- } else if (unlikely(node_size(key, data) >
- mc->mc_txn->mt_env->me_leaf_nodemax)) {
- /* Put data on overflow page. */
- if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT)) {
- mdbx_error("Unexpected target %s flags 0x%x for large data-item",
- "dupsort-db", mc->mc_db->md_flags);
- return MDBX_PROBLEM;
+ /* Add left (implicit) pointer. */
+ rc = mdbx_node_add_branch(mc, 0, NULL, mp->mp_pgno);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ /* undo the pre-push */
+ mc->mc_pg[0] = mc->mc_pg[1];
+ mc->mc_ki[0] = mc->mc_ki[1];
+ mc->mc_db->md_root = mp->mp_pgno;
+ mc->mc_db->md_depth--;
+ goto done;
}
- if (unlikely(flags & (F_DUPDATA | F_SUBDATA))) {
- mdbx_error("Unexpected target %s flags 0x%x for large data-item", "node",
- flags);
- return MDBX_PROBLEM;
+ mc->mc_snum++;
+ mc->mc_top++;
+ ptop = 0;
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
}
- const pgno_t ovpages = number_of_ovpages(mc->mc_txn->mt_env, data->iov_len);
- const struct page_result npr = mdbx_page_new(mc, P_OVERFLOW, ovpages);
- if (unlikely(npr.err != MDBX_SUCCESS))
- return npr.err;
- largepage = npr.page;
- mdbx_debug("allocated %u overflow page(s) %" PRIaPGNO "for %" PRIuPTR
- " data bytes",
- largepage->mp_pages, largepage->mp_pgno, data->iov_len);
- flags |= F_BIGDATA;
- node_bytes =
- node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
} else {
- node_bytes = node_size(key, data) + sizeof(indx_t);
+ ptop = mc->mc_top - 1;
+ mdbx_debug("parent branch page is %" PRIaPGNO, mc->mc_pg[ptop]->mp_pgno);
}
- mdbx_cassert(mc, node_bytes == leaf_size(mc->mc_txn->mt_env, key, data));
- /* Move higher pointers up one slot. */
- const unsigned nkeys = page_numkeys(mp);
- mdbx_cassert(mc, nkeys >= indx);
- for (unsigned i = nkeys; i > indx; --i)
- mp->mp_ptrs[i] = mp->mp_ptrs[i - 1];
+ MDBX_cursor mn;
+ cursor_copy(mc, &mn);
+ mn.mc_pg[mn.mc_top] = sister;
+ mn.mc_ki[mn.mc_top] = 0;
+ mn.mc_ki[ptop] = mc->mc_ki[ptop] + 1;
- /* Adjust free space offsets. */
- const intptr_t lower = mp->mp_lower + sizeof(indx_t);
- const intptr_t upper = mp->mp_upper - (node_bytes - sizeof(indx_t));
- if (unlikely(lower > upper)) {
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return MDBX_PAGE_FULL;
- }
- mp->mp_lower = (indx_t)lower;
- mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper;
+ unsigned split_indx =
+ (newindx < nkeys)
+ ? /* split at the middle */ (nkeys + 1) / 2
+ : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1;
- /* Write the node data. */
- MDBX_node *node = page_node(mp, indx);
- node_set_ks(node, key->iov_len);
- node_set_flags(node, (uint8_t)flags);
- UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0);
- node_set_ds(node, data->iov_len);
- memcpy(node_key(node), key->iov_base, key->iov_len);
+ mdbx_cassert(mc, !IS_BRANCH(mp) || newindx > 0);
+ /* It is reasonable and possible to split the page at the begin */
+ if (unlikely(newindx < minkeys)) {
+ split_indx = minkeys;
+ if (newindx == 0 && foliage == 0 && !(nflags & MDBX_SPLIT_REPLACE)) {
+ split_indx = 0;
+ /* Checking for ability of splitting by the left-side insertion
+ * of a pure page with the new key */
+ for (i = 0; i < mc->mc_top; ++i)
+ if (mc->mc_ki[i]) {
+ get_key(page_node(mc->mc_pg[i], mc->mc_ki[i]), &sepkey);
+ if (mc->mc_dbx->md_cmp(newkey, &sepkey) >= 0)
+ split_indx = minkeys;
+ break;
+ }
+ if (split_indx == 0) {
+ /* Save the current first key which was omitted on the parent branch
+ * page and should be updated if the new first entry will be added */
+ if (IS_LEAF2(mp)) {
+ sepkey.iov_len = mp->mp_leaf2_ksize;
+ sepkey.iov_base = page_leaf2key(mp, 0, sepkey.iov_len);
+ } else
+ get_key(page_node(mp, 0), &sepkey);
+ mdbx_cassert(mc, mc->mc_dbx->md_cmp(newkey, &sepkey) < 0);
+ /* Avoiding rare complex cases of split the parent page */
+ if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey))
+ split_indx = minkeys;
+ }
+ }
+ }
- void *nodedata = node_data(node);
- if (likely(largepage == NULL)) {
- if (unlikely(flags & F_BIGDATA))
- memcpy(nodedata, data->iov_base, sizeof(pgno_t));
- else if (unlikely(flags & MDBX_RESERVE))
- data->iov_base = nodedata;
- else if (likely(nodedata != data->iov_base &&
- data->iov_len /* to avoid UBSAN traps*/ != 0))
- memcpy(nodedata, data->iov_base, data->iov_len);
+ const bool pure_right = split_indx == nkeys;
+ const bool pure_left = split_indx == 0;
+ if (unlikely(pure_right)) {
+ /* newindx == split_indx == nkeys */
+ mdbx_trace("no-split, but add new pure page at the %s", "right/after");
+ mdbx_cassert(mc, newindx == nkeys && split_indx == nkeys && minkeys == 1);
+ sepkey = *newkey;
+ } else if (unlikely(pure_left)) {
+ /* newindx == split_indx == 0 */
+ mdbx_trace("no-split, but add new pure page at the %s", "left/before");
+ mdbx_cassert(mc, newindx == 0 && split_indx == 0 && minkeys == 1);
+ mdbx_trace("old-first-key is %s", DKEY_DEBUG(&sepkey));
} else {
- poke_pgno(nodedata, largepage->mp_pgno);
- nodedata = page_data(largepage);
- if (unlikely(flags & MDBX_RESERVE))
- data->iov_base = nodedata;
- else if (likely(nodedata != data->iov_base &&
- data->iov_len /* to avoid UBSAN traps*/ != 0))
- memcpy(nodedata, data->iov_base, data->iov_len);
- }
- return MDBX_SUCCESS;
-}
+ if (IS_LEAF2(sister)) {
+ char *split, *ins;
+ unsigned lsize, rsize, ksize;
+ /* Move half of the keys to the right sibling */
+ const int x = mc->mc_ki[mc->mc_top] - split_indx;
+ ksize = mc->mc_db->md_xsize;
+ split = page_leaf2key(mp, split_indx, ksize);
+ rsize = (nkeys - split_indx) * ksize;
+ lsize = (nkeys - split_indx) * sizeof(indx_t);
+ mdbx_cassert(mc, mp->mp_lower >= lsize);
+ mp->mp_lower -= (indx_t)lsize;
+ mdbx_cassert(mc, sister->mp_lower + lsize <= UINT16_MAX);
+ sister->mp_lower += (indx_t)lsize;
+ mdbx_cassert(mc, mp->mp_upper + rsize - lsize <= UINT16_MAX);
+ mp->mp_upper += (indx_t)(rsize - lsize);
+ mdbx_cassert(mc, sister->mp_upper >= rsize - lsize);
+ sister->mp_upper -= (indx_t)(rsize - lsize);
+ sepkey.iov_len = ksize;
+ sepkey.iov_base = (newindx != split_indx) ? split : newkey->iov_base;
+ if (x < 0) {
+ mdbx_cassert(mc, ksize >= sizeof(indx_t));
+ ins = page_leaf2key(mp, mc->mc_ki[mc->mc_top], ksize);
+ memcpy(sister->mp_ptrs, split, rsize);
+ sepkey.iov_base = sister->mp_ptrs;
+ memmove(ins + ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize);
+ memcpy(ins, newkey->iov_base, ksize);
+ mdbx_cassert(mc, UINT16_MAX - mp->mp_lower >= (int)sizeof(indx_t));
+ mp->mp_lower += sizeof(indx_t);
+ mdbx_cassert(mc, mp->mp_upper >= ksize - sizeof(indx_t));
+ mp->mp_upper -= (indx_t)(ksize - sizeof(indx_t));
+ } else {
+ memcpy(sister->mp_ptrs, split, x * ksize);
+ ins = page_leaf2key(sister, x, ksize);
+ memcpy(ins, newkey->iov_base, ksize);
+ memcpy(ins + ksize, split + x * ksize, rsize - x * ksize);
+ mdbx_cassert(mc, UINT16_MAX - sister->mp_lower >= (int)sizeof(indx_t));
+ sister->mp_lower += sizeof(indx_t);
+ mdbx_cassert(mc, sister->mp_upper >= ksize - sizeof(indx_t));
+ sister->mp_upper -= (indx_t)(ksize - sizeof(indx_t));
+ mdbx_cassert(mc, x <= (int)UINT16_MAX);
+ mc->mc_ki[mc->mc_top] = (indx_t)x;
+ }
+
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ rc = mdbx_cursor_check(&mn, C_UPDATING);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ }
+ } else {
+ /* Maximum free space in an empty page */
+ const unsigned max_space = page_space(env);
+ const size_t new_size = IS_LEAF(mp) ? leaf_size(env, newkey, newdata)
+ : branch_size(env, newkey);
+
+ /* grab a page to hold a temporary copy */
+ tmp_ki_copy = mdbx_page_malloc(mc->mc_txn, 1);
+ if (unlikely(tmp_ki_copy == NULL)) {
+ rc = MDBX_ENOMEM;
+ goto done;
+ }
+
+ /* prepare to insert */
+ for (unsigned j = i = 0; i < nkeys; ++i, ++j) {
+ tmp_ki_copy->mp_ptrs[j] = 0;
+ j += (i == newindx);
+ tmp_ki_copy->mp_ptrs[j] = mp->mp_ptrs[i];
+ }
+ tmp_ki_copy->mp_pgno = mp->mp_pgno;
+ tmp_ki_copy->mp_flags = mp->mp_flags;
+ tmp_ki_copy->mp_txnid = INVALID_TXNID;
+ tmp_ki_copy->mp_lower = 0;
+ tmp_ki_copy->mp_upper = (indx_t)max_space;
+
+ /* When items are relatively large the split point needs
+ * to be checked, because being off-by-one will make the
+ * difference between success or failure in mdbx_node_add.
+ *
+ * It's also relevant if a page happens to be laid out
+ * such that one half of its nodes are all "small" and
+ * the other half of its nodes are "large". If the new
+ * item is also "large" and falls on the half with
+ * "large" nodes, it also may not fit.
+ *
+ * As a final tweak, if the new item goes on the last
+ * spot on the page (and thus, onto the new page), bias
+ * the split so the new page is emptier than the old page.
+ * This yields better packing during sequential inserts. */
+
+ if (nkeys < 32 || new_size > max_space / 16) {
+ /* Find split point */
+ int dir;
+ if (newindx <= split_indx) {
+ i = 0;
+ dir = 1;
+ } else {
+ i = nkeys;
+ dir = -1;
+ }
+ size_t before = 0, after = new_size + page_used(env, mp);
+ int best = split_indx;
+ int best_offset = nkeys + 1;
-/* Delete the specified node from a page.
- * [in] mc Cursor pointing to the node to delete.
- * [in] ksize The size of a node. Only used if the page is
- * part of a MDBX_DUPFIXED database. */
-static void mdbx_node_del(MDBX_cursor *mc, size_t ksize) {
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- int indx = mc->mc_ki[mc->mc_top];
- int i, j, nkeys, ptr;
- MDBX_node *node;
- char *base;
+ mdbx_trace("seek separator from %u, step %i, default %u, new-idx %u, "
+ "new-size %zu",
+ i, dir, split_indx, newindx, new_size);
+ do {
+ mdbx_cassert(mc, i <= nkeys);
+ size_t size = new_size;
+ if (i != newindx) {
+ MDBX_node *node =
+ (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ);
+ size = NODESIZE + node_ks(node) + sizeof(indx_t);
+ if (IS_LEAF(mp))
+ size += F_ISSET(node_flags(node), F_BIGDATA) ? sizeof(pgno_t)
+ : node_ds(node);
+ size = EVEN(size);
+ }
- mdbx_debug("delete node %u on %s page %" PRIaPGNO, indx,
- IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno);
- nkeys = page_numkeys(mp);
- mdbx_cassert(mc, indx < nkeys);
+ before += size;
+ after -= size;
+ mdbx_trace("step %u, size %zu, before %zu, after %zu, max %u", i,
+ size, before, after, max_space);
- if (IS_LEAF2(mp)) {
- mdbx_cassert(mc, ksize >= sizeof(indx_t));
- unsigned diff = nkeys - 1 - indx;
- base = page_leaf2key(mp, indx, ksize);
- if (diff)
- memmove(base, base + ksize, diff * ksize);
- mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t));
- mp->mp_lower -= sizeof(indx_t);
- mdbx_cassert(mc,
- (size_t)UINT16_MAX - mp->mp_upper >= ksize - sizeof(indx_t));
- mp->mp_upper += (indx_t)(ksize - sizeof(indx_t));
- return;
- }
+ if (before <= max_space && after <= max_space) {
+ int offset = branchless_abs(split_indx - i);
+ if (offset >= best_offset)
+ break;
+ best_offset = offset;
+ best = i;
+ }
+ i += dir;
+ } while (i < nkeys);
- node = page_node(mp, indx);
- mdbx_cassert(mc, !IS_BRANCH(mp) || indx || node_ks(node) == 0);
- size_t sz = NODESIZE + node_ks(node);
- if (IS_LEAF(mp)) {
- if (F_ISSET(node_flags(node), F_BIGDATA))
- sz += sizeof(pgno_t);
- else
- sz += node_ds(node);
- }
- sz = EVEN(sz);
+ split_indx = best + (dir > 0);
+ split_indx = (split_indx <= nkeys - minkeys + 1) ? split_indx
+ : nkeys - minkeys + 1;
+ split_indx = (split_indx >= minkeys) ? split_indx : minkeys;
+ mdbx_trace("chosen %u", split_indx);
+ }
- ptr = mp->mp_ptrs[indx];
- for (i = j = 0; i < nkeys; i++) {
- if (i != indx) {
- mp->mp_ptrs[j] = mp->mp_ptrs[i];
- if (mp->mp_ptrs[i] < ptr) {
- mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_ptrs[j] >= sz);
- mp->mp_ptrs[j] += (indx_t)sz;
+ sepkey.iov_len = newkey->iov_len;
+ sepkey.iov_base = newkey->iov_base;
+ if (split_indx != newindx) {
+ MDBX_node *node =
+ (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[split_indx] +
+ PAGEHDRSZ);
+ sepkey.iov_len = node_ks(node);
+ sepkey.iov_base = node_key(node);
}
- j++;
}
}
+ mdbx_debug("separator is %d [%s]", split_indx, DKEY_DEBUG(&sepkey));
- base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
- memmove(base + sz, base, ptr - mp->mp_upper);
+ bool did_split_parent = false;
+ /* Copy separator key to the parent. */
+ if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey)) {
+ mdbx_trace("need split parent branch-page for key %s", DKEY_DEBUG(&sepkey));
+ mdbx_cassert(mc, page_numkeys(mn.mc_pg[ptop]) > 2);
+ mdbx_cassert(mc, !pure_left);
+ const int snum = mc->mc_snum;
+ const int depth = mc->mc_db->md_depth;
+ mn.mc_snum--;
+ mn.mc_top--;
+ did_split_parent = true;
+ /* We want other splits to find mn when doing fixups */
+ WITH_CURSOR_TRACKING(
+ mn, rc = mdbx_page_split(&mn, &sepkey, NULL, sister->mp_pgno, 0));
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ mdbx_cassert(mc, (int)mc->mc_snum - snum == mc->mc_db->md_depth - depth);
+ if (mdbx_audit_enabled()) {
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ }
- mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t));
- mp->mp_lower -= sizeof(indx_t);
- mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_upper >= sz);
- mp->mp_upper += (indx_t)sz;
+ /* root split? */
+ ptop += mc->mc_snum - snum;
-#if MDBX_DEBUG > 0
- if (mdbx_audit_enabled()) {
- int page_check_err = mdbx_page_check(mc, mp, C_UPDATING);
- mdbx_cassert(mc, page_check_err == MDBX_SUCCESS);
- }
-#endif
-}
+ /* Right page might now have changed parent.
+ * Check if left page also changed parent. */
+ if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
+ mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
+ for (i = 0; i < ptop; i++) {
+ mc->mc_pg[i] = mn.mc_pg[i];
+ mc->mc_ki[i] = mn.mc_ki[i];
+ }
+ mc->mc_pg[ptop] = mn.mc_pg[ptop];
+ if (mn.mc_ki[ptop]) {
+ mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1;
+ } else {
+ /* find right page's left sibling */
+ mc->mc_ki[ptop] = mn.mc_ki[ptop];
+ rc = mdbx_cursor_sibling(mc, SIBLING_LEFT);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_NOTFOUND) /* improper mdbx_cursor_sibling() result */ {
+ mdbx_error("unexpected %i error going left sibling", rc);
+ rc = MDBX_PROBLEM;
+ }
+ goto done;
+ }
+ }
+ }
+ } else if (unlikely(pure_left)) {
+ MDBX_page *ptop_page = mc->mc_pg[ptop];
+ mdbx_debug("adding to parent page %u node[%u] left-leaf page #%u key %s",
+ ptop_page->mp_pgno, mc->mc_ki[ptop], sister->mp_pgno,
+ DKEY(mc->mc_ki[ptop] ? newkey : NULL));
+ mc->mc_top--;
+ rc = mdbx_node_add_branch(mc, mc->mc_ki[ptop],
+ mc->mc_ki[ptop] ? newkey : NULL, sister->mp_pgno);
+ mdbx_cassert(mc, mp == mc->mc_pg[ptop + 1] &&
+ newindx == mc->mc_ki[ptop + 1] && ptop == mc->mc_top);
-/* Compact the main page after deleting a node on a subpage.
- * [in] mp The main page to operate on.
- * [in] indx The index of the subpage on the main page. */
-static void mdbx_node_shrink(MDBX_page *mp, unsigned indx) {
- MDBX_node *node;
- MDBX_page *sp, *xp;
- char *base;
- size_t nsize, delta, len, ptr;
- int i;
+ if (likely(rc == MDBX_SUCCESS) && mc->mc_ki[ptop] == 0) {
+ mdbx_debug("update prev-first key on parent %s", DKEY(&sepkey));
+ MDBX_node *node = page_node(mc->mc_pg[ptop], 1);
+ mdbx_cassert(mc, node_ks(node) == 0 && node_pgno(node) == mp->mp_pgno);
+ mdbx_cassert(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 0);
+ mc->mc_ki[ptop] = 1;
+ rc = mdbx_update_key(mc, &sepkey);
+ mdbx_cassert(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 1);
+ mdbx_cassert(mc,
+ mp == mc->mc_pg[ptop + 1] && newindx == mc->mc_ki[ptop + 1]);
+ mc->mc_ki[ptop] = 0;
+ }
- node = page_node(mp, indx);
- sp = (MDBX_page *)node_data(node);
- delta = page_room(sp);
- assert(delta > 0);
+ mc->mc_top++;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
- /* Prepare to shift upward, set len = length(subpage part to shift) */
- if (IS_LEAF2(sp)) {
- delta &= /* do not make the node uneven-sized */ ~(size_t)1;
- if (unlikely(delta) == 0)
- return;
- nsize = node_ds(node) - delta;
- assert(nsize % 1 == 0);
- len = nsize;
+ MDBX_node *node = page_node(mc->mc_pg[ptop], mc->mc_ki[ptop] + 1);
+ mdbx_cassert(mc, node_pgno(node) == mp->mp_pgno &&
+ mc->mc_pg[ptop] == ptop_page);
} else {
- xp = (MDBX_page *)((char *)sp + delta); /* destination subpage */
- for (i = page_numkeys(sp); --i >= 0;) {
- assert(sp->mp_ptrs[i] >= delta);
- xp->mp_ptrs[i] = (indx_t)(sp->mp_ptrs[i] - delta);
+ mn.mc_top--;
+ mdbx_trace("add-to-parent the right-entry[%u] for new sibling-page",
+ mn.mc_ki[ptop]);
+ rc = mdbx_node_add_branch(&mn, mn.mc_ki[ptop], &sepkey, sister->mp_pgno);
+ mn.mc_top++;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ }
+
+ if (unlikely(pure_left | pure_right)) {
+ mc->mc_pg[mc->mc_top] = sister;
+ mc->mc_ki[mc->mc_top] = 0;
+ switch (PAGETYPE(sister)) {
+ case P_LEAF: {
+ mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID);
+ rc = mdbx_node_add_leaf(mc, 0, newkey, newdata, nflags);
+ } break;
+ case P_LEAF | P_LEAF2: {
+ mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
+ mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID);
+ rc = mdbx_node_add_leaf2(mc, 0, newkey);
+ } break;
+ default:
+ rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE(sister));
+ }
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+
+ if (pure_right) {
+ for (i = 0; i < mc->mc_top; i++)
+ mc->mc_ki[i] = mn.mc_ki[i];
+ } else if (mc->mc_ki[mc->mc_top - 1] == 0) {
+ for (i = 2; i <= mc->mc_top; ++i)
+ if (mc->mc_ki[mc->mc_top - i]) {
+ get_key(
+ page_node(mc->mc_pg[mc->mc_top - i], mc->mc_ki[mc->mc_top - i]),
+ &sepkey);
+ if (mc->mc_dbx->md_cmp(newkey, &sepkey) < 0) {
+ mc->mc_top -= i;
+ mdbx_debug("update new-first on parent [%i] page %u key %s",
+ mc->mc_ki[mc->mc_top], mc->mc_pg[mc->mc_top]->mp_pgno,
+ DKEY(newkey));
+ rc = mdbx_update_key(mc, newkey);
+ mc->mc_top += i;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ }
+ break;
+ }
}
- nsize = node_ds(node) - delta;
- len = PAGEHDRSZ;
- }
- sp->mp_upper = sp->mp_lower;
- sp->mp_pgno = mp->mp_pgno;
- node_set_ds(node, nsize);
-
- /* Shift <lower nodes...initial part of subpage> upward */
- base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
- memmove(base + delta, base, (char *)sp + len - base);
+ } else if (!IS_LEAF2(mp)) {
+ /* Move nodes */
+ mc->mc_pg[mc->mc_top] = sister;
+ i = split_indx;
+ unsigned n = 0;
+ pgno_t pgno = 0;
+ do {
+ mdbx_trace("i %u, nkeys %u => n %u, rp #%u", i, nkeys, n,
+ sister->mp_pgno);
+ MDBX_val *rdata = NULL;
+ if (i == newindx) {
+ rkey.iov_base = newkey->iov_base;
+ rkey.iov_len = newkey->iov_len;
+ if (IS_LEAF(mp))
+ rdata = newdata;
+ else
+ pgno = newpgno;
+ flags = nflags;
+ /* Update index for the new key. */
+ mc->mc_ki[mc->mc_top] = (indx_t)n;
+ } else {
+ MDBX_node *node =
+ (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ);
+ rkey.iov_base = node_key(node);
+ rkey.iov_len = node_ks(node);
+ if (IS_LEAF(mp)) {
+ xdata.iov_base = node_data(node);
+ xdata.iov_len = node_ds(node);
+ rdata = &xdata;
+ } else
+ pgno = node_pgno(node);
+ flags = node_flags(node);
+ }
- ptr = mp->mp_ptrs[indx];
- for (i = page_numkeys(mp); --i >= 0;) {
- if (mp->mp_ptrs[i] <= ptr) {
- assert((size_t)UINT16_MAX - mp->mp_ptrs[i] >= delta);
- mp->mp_ptrs[i] += (indx_t)delta;
- }
- }
- assert((size_t)UINT16_MAX - mp->mp_upper >= delta);
- mp->mp_upper += (indx_t)delta;
-}
+ switch (PAGETYPE(sister)) {
+ case P_BRANCH: {
+ mdbx_cassert(mc, 0 == (uint16_t)flags);
+ /* First branch index doesn't need key data. */
+ rc = mdbx_node_add_branch(mc, n, n ? &rkey : NULL, pgno);
+ } break;
+ case P_LEAF: {
+ mdbx_cassert(mc, pgno == 0);
+ mdbx_cassert(mc, rdata != NULL);
+ rc = mdbx_node_add_leaf(mc, n, &rkey, rdata, flags);
+ } break;
+ /* case P_LEAF | P_LEAF2: {
+ mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
+ mdbx_cassert(mc, gno == 0);
+ rc = mdbx_node_add_leaf2(mc, n, &rkey);
+ } break; */
+ default:
+ rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE(sister));
+ }
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
-/* Initial setup of a sorted-dups cursor.
- *
- * Sorted duplicates are implemented as a sub-database for the given key.
- * The duplicate data items are actually keys of the sub-database.
- * Operations on the duplicate data items are performed using a sub-cursor
- * initialized when the sub-database is first accessed. This function does
- * the preliminary setup of the sub-cursor, filling in the fields that
- * depend only on the parent DB.
- *
- * [in] mc The main cursor whose sorted-dups cursor is to be initialized. */
-static int mdbx_xcursor_init0(MDBX_cursor *mc) {
- MDBX_xcursor *mx = mc->mc_xcursor;
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
- mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
- mc->mc_dbi);
- return MDBX_CORRUPTED;
- }
+ ++n;
+ if (++i > nkeys) {
+ i = 0;
+ n = 0;
+ mc->mc_pg[mc->mc_top] = tmp_ki_copy;
+ mdbx_trace("switch to mp #%u", tmp_ki_copy->mp_pgno);
+ }
+ } while (i != split_indx);
- mx->mx_cursor.mc_xcursor = NULL;
- mx->mx_cursor.mc_next = NULL;
- mx->mx_cursor.mc_txn = mc->mc_txn;
- mx->mx_cursor.mc_db = &mx->mx_db;
- mx->mx_cursor.mc_dbx = &mx->mx_dbx;
- mx->mx_cursor.mc_dbi = mc->mc_dbi;
- mx->mx_cursor.mc_dbistate = mc->mc_dbistate;
- mx->mx_cursor.mc_snum = 0;
- mx->mx_cursor.mc_top = 0;
- mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
- mx->mx_dbx.md_name.iov_len = 0;
- mx->mx_dbx.md_name.iov_base = NULL;
- mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp;
- mx->mx_dbx.md_dcmp = NULL;
- mx->mx_dbx.md_klen_min = INT_MAX;
- mx->mx_dbx.md_vlen_min = mx->mx_dbx.md_klen_max = mx->mx_dbx.md_vlen_max = 0;
- return MDBX_SUCCESS;
-}
+ mdbx_trace("i %u, nkeys %u, n %u, pgno #%u", i, nkeys, n,
+ mc->mc_pg[mc->mc_top]->mp_pgno);
-/* Final setup of a sorted-dups cursor.
- * Sets up the fields that depend on the data from the main cursor.
- * [in] mc The main cursor whose sorted-dups cursor is to be initialized.
- * [in] node The data containing the MDBX_db record for the sorted-dup database.
- */
-static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node,
- const MDBX_page *mp) {
- MDBX_xcursor *mx = mc->mc_xcursor;
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
- mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
- mc->mc_dbi);
- return MDBX_CORRUPTED;
- }
+ nkeys = page_numkeys(tmp_ki_copy);
+ for (i = 0; i < nkeys; i++)
+ mp->mp_ptrs[i] = tmp_ki_copy->mp_ptrs[i];
+ mp->mp_lower = tmp_ki_copy->mp_lower;
+ mp->mp_upper = tmp_ki_copy->mp_upper;
+ memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1),
+ env->me_psize - tmp_ki_copy->mp_upper - PAGEHDRSZ);
- const uint8_t flags = node_flags(node);
- switch (flags) {
- default:
- mdbx_error("invalid node flags %u", flags);
- return MDBX_CORRUPTED;
- case F_DUPDATA | F_SUBDATA:
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(node_ds(node) != sizeof(MDBX_db))) {
- mdbx_error("invalid nested-db record size %zu", node_ds(node));
- return MDBX_CORRUPTED;
- }
- memcpy(&mx->mx_db, node_data(node), sizeof(MDBX_db));
- const txnid_t pp_txnid = mp->mp_txnid;
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(mx->mx_db.md_mod_txnid > pp_txnid)) {
- mdbx_error("nested-db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN
- ")",
- mx->mx_db.md_mod_txnid, pp_txnid);
- return MDBX_CORRUPTED;
+ /* reset back to original page */
+ if (newindx < split_indx) {
+ mc->mc_pg[mc->mc_top] = mp;
+ } else {
+ mc->mc_pg[mc->mc_top] = sister;
+ mc->mc_ki[ptop]++;
+ /* Make sure mc_ki is still valid. */
+ if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
+ mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
+ for (i = 0; i <= ptop; i++) {
+ mc->mc_pg[i] = mn.mc_pg[i];
+ mc->mc_ki[i] = mn.mc_ki[i];
+ }
+ }
}
- mx->mx_cursor.mc_pg[0] = 0;
- mx->mx_cursor.mc_snum = 0;
- mx->mx_cursor.mc_top = 0;
- mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
- break;
- case F_DUPDATA:
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(node_ds(node) <= PAGEHDRSZ)) {
- mdbx_error("invalid nested-page size %zu", node_ds(node));
- return MDBX_CORRUPTED;
+ } else if (newindx >= split_indx) {
+ mc->mc_pg[mc->mc_top] = sister;
+ mc->mc_ki[ptop]++;
+ /* Make sure mc_ki is still valid. */
+ if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
+ mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
+ for (i = 0; i <= ptop; i++) {
+ mc->mc_pg[i] = mn.mc_pg[i];
+ mc->mc_ki[i] = mn.mc_ki[i];
+ }
}
- MDBX_page *fp = node_data(node);
- mx->mx_db.md_depth = 1;
- mx->mx_db.md_branch_pages = 0;
- mx->mx_db.md_leaf_pages = 1;
- mx->mx_db.md_overflow_pages = 0;
- mx->mx_db.md_entries = page_numkeys(fp);
- mx->mx_db.md_root = fp->mp_pgno;
- mx->mx_db.md_mod_txnid = mp->mp_txnid;
- mx->mx_cursor.mc_snum = 1;
- mx->mx_cursor.mc_top = 0;
- mx->mx_cursor.mc_flags =
- C_INITIALIZED | C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD));
- mx->mx_cursor.mc_pg[0] = fp;
- mx->mx_cursor.mc_ki[0] = 0;
- mx->mx_db.md_flags = flags_db2sub(mc->mc_db->md_flags);
- mx->mx_db.md_xsize =
- (mc->mc_db->md_flags & MDBX_DUPFIXED) ? fp->mp_leaf2_ksize : 0;
- break;
}
- if (unlikely(mx->mx_db.md_xsize != mc->mc_db->md_xsize)) {
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(mc->mc_db->md_xsize != 0)) {
- mdbx_error("cursor mismatched nested-db md_xsize %u",
- mc->mc_db->md_xsize);
- return MDBX_CORRUPTED;
+ /* Adjust other cursors pointing to mp and/or to parent page */
+ nkeys = page_numkeys(mp);
+ for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
+ m2 = m2->mc_next) {
+ MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+ if (m3 == mc)
+ continue;
+ if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
+ continue;
+ if (foliage) {
+ /* sub cursors may be on different DB */
+ if (m3->mc_pg[0] != mp)
+ continue;
+ /* root split */
+ for (int k = foliage; k >= 0; k--) {
+ m3->mc_ki[k + 1] = m3->mc_ki[k];
+ m3->mc_pg[k + 1] = m3->mc_pg[k];
+ }
+ m3->mc_ki[0] = (m3->mc_ki[0] >= nkeys) ? 1 : 0;
+ m3->mc_pg[0] = mc->mc_pg[0];
+ m3->mc_snum++;
+ m3->mc_top++;
}
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) {
- mdbx_error("mismatched nested-db md_flags %u", mc->mc_db->md_flags);
- return MDBX_CORRUPTED;
+
+ if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp && !pure_left) {
+ if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDBX_SPLIT_REPLACE))
+ m3->mc_ki[mc->mc_top]++;
+ if (m3->mc_ki[mc->mc_top] >= nkeys) {
+ m3->mc_pg[mc->mc_top] = sister;
+ mdbx_cassert(mc, m3->mc_ki[mc->mc_top] >= nkeys);
+ m3->mc_ki[mc->mc_top] -= (indx_t)nkeys;
+ for (i = 0; i < mc->mc_top; i++) {
+ m3->mc_ki[i] = mn.mc_ki[i];
+ m3->mc_pg[i] = mn.mc_pg[i];
+ }
+ }
+ } else if (!did_split_parent && m3->mc_top >= ptop &&
+ m3->mc_pg[ptop] == mc->mc_pg[ptop] &&
+ m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
+ m3->mc_ki[ptop]++; /* also for the `pure-left` case */
}
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(mx->mx_db.md_xsize < mc->mc_dbx->md_vlen_min ||
- mx->mx_db.md_xsize > mc->mc_dbx->md_vlen_max)) {
- mdbx_error("mismatched nested-db.md_xsize (%u) <> min/max value-length "
- "(%zu/%zu)",
- mx->mx_db.md_xsize, mc->mc_dbx->md_vlen_min,
- mc->mc_dbx->md_vlen_max);
- return MDBX_CORRUPTED;
+ if (XCURSOR_INITED(m3) && IS_LEAF(mp))
+ XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+ }
+ mdbx_trace("mp #%u left: %d, sister #%u left: %d", mp->mp_pgno, page_room(mp),
+ sister->mp_pgno, page_room(sister));
+
+done:
+ if (tmp_ki_copy)
+ mdbx_dpage_free(env, tmp_ki_copy, 1);
+
+ if (unlikely(rc != MDBX_SUCCESS))
+ mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
+ else {
+ if (mdbx_audit_enabled())
+ rc = mdbx_cursor_check(mc, C_UPDATING);
+ if (unlikely(nflags & MDBX_RESERVE)) {
+ MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+ if (!(node_flags(node) & F_BIGDATA))
+ newdata->iov_base = node_data(node);
}
- mc->mc_db->md_xsize = mx->mx_db.md_xsize;
- mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = mx->mx_db.md_xsize;
+#if MDBX_ENABLE_PGOP_STAT
+ env->me_lck->mti_pgop_stat.split.weak += 1;
+#endif /* MDBX_ENABLE_PGOP_STAT */
}
- mx->mx_dbx.md_klen_min = mc->mc_dbx->md_vlen_min;
- mx->mx_dbx.md_klen_max = mc->mc_dbx->md_vlen_max;
- mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi,
- mx->mx_db.md_root);
- return MDBX_SUCCESS;
+ mdbx_debug("<< mp #%u, rc %d", mp->mp_pgno, rc);
+ return rc;
}
-/* Fixup a sorted-dups cursor due to underlying update.
- * Sets up some fields that depend on the data from the main cursor.
- * Almost the same as init1, but skips initialization steps if the
- * xcursor had already been used.
- * [in] mc The main cursor whose sorted-dups cursor is to be fixed up.
- * [in] src_mx The xcursor of an up-to-date cursor.
- * [in] new_dupdata True if converting from a non-F_DUPDATA item. */
-static int mdbx_xcursor_init2(MDBX_cursor *mc, MDBX_xcursor *src_mx,
- bool new_dupdata) {
- MDBX_xcursor *mx = mc->mc_xcursor;
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(mx == nullptr)) {
- mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)",
- mc->mc_dbi);
- return MDBX_CORRUPTED;
- }
+int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
+ unsigned flags) {
+ int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- if (new_dupdata) {
- mx->mx_cursor.mc_snum = 1;
- mx->mx_cursor.mc_top = 0;
- mx->mx_cursor.mc_flags |= C_INITIALIZED;
- mx->mx_cursor.mc_ki[0] = 0;
- }
+ if (unlikely(!key || !data))
+ return MDBX_EINVAL;
- mx->mx_dbx.md_klen_min = src_mx->mx_dbx.md_klen_min;
- mx->mx_dbx.md_klen_max = src_mx->mx_dbx.md_klen_max;
- mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
- mx->mx_db = src_mx->mx_db;
- mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
- if (mx->mx_cursor.mc_flags & C_INITIALIZED) {
- mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi,
- mx->mx_db.md_root);
- }
- return MDBX_SUCCESS;
-}
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
-static __inline int mdbx_couple_init(MDBX_cursor_couple *couple,
- const MDBX_dbi dbi, MDBX_txn *const txn,
- MDBX_db *const db, MDBX_dbx *const dbx,
- uint8_t *const dbstate) {
- couple->outer.mc_signature = MDBX_MC_LIVE;
- couple->outer.mc_next = NULL;
- couple->outer.mc_backup = NULL;
- couple->outer.mc_dbi = dbi;
- couple->outer.mc_txn = txn;
- couple->outer.mc_db = db;
- couple->outer.mc_dbx = dbx;
- couple->outer.mc_dbistate = dbstate;
- couple->outer.mc_snum = 0;
- couple->outer.mc_top = 0;
- couple->outer.mc_pg[0] = 0;
- couple->outer.mc_flags = 0;
- couple->outer.mc_ki[0] = 0;
- couple->outer.mc_xcursor = NULL;
+ if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
+ MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
+ MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE)))
+ return MDBX_EINVAL;
- int rc = MDBX_SUCCESS;
- if (unlikely(*couple->outer.mc_dbistate & DBI_STALE)) {
- rc = mdbx_page_search(&couple->outer, NULL, MDBX_PS_ROOTONLY);
- rc = (rc != MDBX_NOTFOUND) ? rc : MDBX_SUCCESS;
- } else if (unlikely(couple->outer.mc_dbx->md_klen_max == 0)) {
- rc = mdbx_setup_dbx(couple->outer.mc_dbx, couple->outer.mc_db,
- txn->mt_env->me_psize);
- }
+ if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
+ return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
- if (couple->outer.mc_db->md_flags & MDBX_DUPSORT) {
- couple->inner.mx_cursor.mc_signature = MDBX_MC_LIVE;
- couple->outer.mc_xcursor = &couple->inner;
- rc = mdbx_xcursor_init0(&couple->outer);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- couple->inner.mx_dbx.md_klen_min = couple->outer.mc_dbx->md_vlen_min;
- couple->inner.mx_dbx.md_klen_max = couple->outer.mc_dbx->md_vlen_max;
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ cx.outer.mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = &cx.outer;
+
+ /* LY: support for update (explicit overwrite) */
+ if (flags & MDBX_CURRENT) {
+ rc = mdbx_cursor_get(&cx.outer, (MDBX_val *)key, NULL, MDBX_SET);
+ if (likely(rc == MDBX_SUCCESS) &&
+ (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) &&
+ (flags & MDBX_ALLDUPS) == 0) {
+ /* LY: allows update (explicit overwrite) only for unique keys */
+ MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
+ cx.outer.mc_ki[cx.outer.mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) &&
+ cx.outer.mc_xcursor->mx_db.md_entries > 1);
+ rc = MDBX_EMULTIVAL;
+ }
+ }
}
- return rc;
-}
-/* Initialize a cursor for a given transaction and database. */
-static int mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, MDBX_dbi dbi) {
- STATIC_ASSERT(offsetof(MDBX_cursor_couple, outer) == 0);
- if (unlikely(TXN_DBI_CHANGED(txn, dbi)))
- return MDBX_BAD_DBI;
+ if (likely(rc == MDBX_SUCCESS))
+ rc = mdbx_cursor_put(&cx.outer, key, data, flags);
+ txn->mt_cursors[dbi] = cx.outer.mc_next;
- return mdbx_couple_init(container_of(mc, MDBX_cursor_couple, outer), dbi, txn,
- &txn->mt_dbs[dbi], &txn->mt_dbxs[dbi],
- &txn->mt_dbistate[dbi]);
+ return rc;
}
-MDBX_cursor *mdbx_cursor_create(void *context) {
- MDBX_cursor_couple *couple = mdbx_calloc(1, sizeof(MDBX_cursor_couple));
- if (unlikely(!couple))
- return nullptr;
-
- couple->outer.mc_signature = MDBX_MC_READY4CLOSE;
- couple->outer.mc_dbi = UINT_MAX;
- couple->mc_userctx = context;
- return &couple->outer;
-}
+/**** COPYING *****************************************************************/
-int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) {
- if (unlikely(!mc))
- return MDBX_EINVAL;
+/* State needed for a double-buffering compacting copy. */
+typedef struct mdbx_copy {
+ MDBX_env *mc_env;
+ MDBX_txn *mc_txn;
+ mdbx_condpair_t mc_condpair;
+ uint8_t *mc_wbuf[2];
+ uint8_t *mc_over[2];
+ size_t mc_wlen[2];
+ size_t mc_olen[2];
+ mdbx_filehandle_t mc_fd;
+ /* Error code. Never cleared if set. Both threads can set nonzero
+ * to fail the copy. Not mutex-protected, MDBX expects atomic int. */
+ volatile int mc_error;
+ pgno_t mc_next_pgno;
+ volatile unsigned mc_head;
+ volatile unsigned mc_tail;
+} mdbx_copy;
- if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
- mc->mc_signature != MDBX_MC_LIVE))
- return MDBX_EBADSIGN;
+/* Dedicated writer thread for compacting copy. */
+__cold static THREAD_RESULT THREAD_CALL mdbx_env_copythr(void *arg) {
+ mdbx_copy *my = arg;
- MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer);
- couple->mc_userctx = ctx;
- return MDBX_SUCCESS;
-}
+#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
+ sigset_t sigset;
+ sigemptyset(&sigset);
+ sigaddset(&sigset, SIGPIPE);
+ my->mc_error = pthread_sigmask(SIG_BLOCK, &sigset, NULL);
+#endif /* EPIPE */
-void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) {
- if (unlikely(!mc))
- return nullptr;
+ mdbx_condpair_lock(&my->mc_condpair);
+ while (!my->mc_error) {
+ while (my->mc_tail == my->mc_head && !my->mc_error) {
+ int err = mdbx_condpair_wait(&my->mc_condpair, true);
+ if (err != MDBX_SUCCESS) {
+ my->mc_error = err;
+ goto bailout;
+ }
+ }
+ const unsigned toggle = my->mc_tail & 1;
+ size_t wsize = my->mc_wlen[toggle];
+ if (wsize == 0) {
+ my->mc_tail += 1;
+ break /* EOF */;
+ }
+ my->mc_wlen[toggle] = 0;
+ uint8_t *ptr = my->mc_wbuf[toggle];
+ again:
+ if (!my->mc_error) {
+ int err = mdbx_write(my->mc_fd, ptr, wsize);
+ if (err != MDBX_SUCCESS) {
+#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
+ if (err == EPIPE) {
+ /* Collect the pending SIGPIPE,
+ * otherwise at least OS X gives it to the process on thread-exit. */
+ int unused;
+ sigwait(&sigset, &unused);
+ }
+#endif /* EPIPE */
+ my->mc_error = err;
+ goto bailout;
+ }
+ }
- if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
- mc->mc_signature != MDBX_MC_LIVE))
- return nullptr;
+ /* If there's an overflow page tail, write it too */
+ wsize = my->mc_olen[toggle];
+ if (wsize) {
+ my->mc_olen[toggle] = 0;
+ ptr = my->mc_over[toggle];
+ goto again;
+ }
+ my->mc_tail += 1;
+ mdbx_condpair_signal(&my->mc_condpair, false);
+ }
+bailout:
+ mdbx_condpair_unlock(&my->mc_condpair);
+ return (THREAD_RESULT)0;
+}
- MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer);
- return couple->mc_userctx;
+/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. */
+__cold static int mdbx_env_cthr_toggle(mdbx_copy *my) {
+ mdbx_condpair_lock(&my->mc_condpair);
+ mdbx_assert(my->mc_env, my->mc_head - my->mc_tail < 2 || my->mc_error);
+ my->mc_head += 1;
+ mdbx_condpair_signal(&my->mc_condpair, true);
+ while (!my->mc_error &&
+ my->mc_head - my->mc_tail == 2 /* both buffers in use */) {
+ int err = mdbx_condpair_wait(&my->mc_condpair, false);
+ if (err != MDBX_SUCCESS)
+ my->mc_error = err;
+ }
+ mdbx_condpair_unlock(&my->mc_condpair);
+ return my->mc_error;
}
-int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) {
- if (unlikely(!mc))
- return MDBX_EINVAL;
+/* Depth-first tree traversal for compacting copy.
+ * [in] my control structure.
+ * [in,out] pg database root.
+ * [in] flags includes F_DUPDATA if it is a sorted-duplicate sub-DB. */
+__cold static int mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) {
+ MDBX_cursor_couple couple;
+ MDBX_page *mo, *mp, *leaf;
+ char *buf, *ptr;
+ int rc;
+ unsigned i;
- if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE &&
- mc->mc_signature != MDBX_MC_LIVE))
- return MDBX_EBADSIGN;
+ /* Empty DB, nothing to do */
+ if (*pg == P_INVALID)
+ return MDBX_SUCCESS;
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ memset(&couple, 0, sizeof(couple));
+ couple.outer.mc_snum = 1;
+ couple.outer.mc_txn = my->mc_txn;
+ couple.outer.mc_flags = couple.inner.mx_cursor.mc_flags =
+ C_COPYING | C_SKIPORD;
+
+ rc = mdbx_page_get(&couple.outer, *pg, &couple.outer.mc_pg[0],
+ my->mc_txn->mt_txnid);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
-
- if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
- return MDBX_BAD_DBI;
-
- if (unlikely(dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)))
- return MDBX_EACCESS;
-
- if (unlikely(mc->mc_backup)) /* Cursor from parent transaction */ {
- mdbx_cassert(mc, mc->mc_signature == MDBX_MC_LIVE);
- if (unlikely(mc->mc_dbi != dbi ||
- /* paranoia */ mc->mc_signature != MDBX_MC_LIVE ||
- mc->mc_txn != txn))
- return MDBX_EINVAL;
-
- assert(mc->mc_db == &txn->mt_dbs[dbi]);
- assert(mc->mc_dbx == &txn->mt_dbxs[dbi]);
- assert(mc->mc_dbi == dbi);
- assert(mc->mc_dbistate == &txn->mt_dbistate[dbi]);
- return likely(mc->mc_dbi == dbi &&
- /* paranoia */ mc->mc_signature == MDBX_MC_LIVE &&
- mc->mc_txn == txn)
- ? MDBX_SUCCESS
- : MDBX_EINVAL /* Disallow change DBI in nested transactions */;
- }
-
- if (mc->mc_signature == MDBX_MC_LIVE) {
- if (unlikely(!mc->mc_txn ||
- mc->mc_txn->mt_signature != MDBX_MT_SIGNATURE)) {
- mdbx_error("Wrong cursor's transaction %p 0x%x",
- __Wpedantic_format_voidptr(mc->mc_txn),
- mc->mc_txn ? mc->mc_txn->mt_signature : 0);
- return MDBX_PROBLEM;
- }
- if (mc->mc_flags & C_UNTRACK) {
- MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi];
- while (*prev && *prev != mc)
- prev = &(*prev)->mc_next;
- mdbx_cassert(mc, *prev == mc);
- *prev = mc->mc_next;
- }
- mc->mc_signature = MDBX_MC_READY4CLOSE;
- mc->mc_flags = 0;
- mc->mc_dbi = UINT_MAX;
- mc->mc_next = NULL;
- mc->mc_db = NULL;
- mc->mc_dbx = NULL;
- mc->mc_dbistate = NULL;
- }
- mdbx_cassert(mc, !(mc->mc_flags & C_UNTRACK));
-
- rc = mdbx_cursor_init(mc, txn, dbi);
+ rc = mdbx_page_search_root(&couple.outer, NULL, MDBX_PS_FIRST);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- mc->mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = mc;
- mc->mc_flags |= C_UNTRACK;
-
- return MDBX_SUCCESS;
-}
-
-int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) {
- if (unlikely(!ret))
- return MDBX_EINVAL;
- *ret = NULL;
-
- MDBX_cursor *const mc = mdbx_cursor_create(nullptr);
- if (unlikely(!mc))
+ /* Make cursor pages writable */
+ buf = ptr = mdbx_malloc(pgno2bytes(my->mc_env, couple.outer.mc_snum));
+ if (buf == NULL)
return MDBX_ENOMEM;
- int rc = mdbx_cursor_bind(txn, mc, dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_cursor_close(mc);
- return rc;
+ for (i = 0; i < couple.outer.mc_top; i++) {
+ mdbx_page_copy((MDBX_page *)ptr, couple.outer.mc_pg[i],
+ my->mc_env->me_psize);
+ couple.outer.mc_pg[i] = (MDBX_page *)ptr;
+ ptr += my->mc_env->me_psize;
}
- *ret = mc;
- return MDBX_SUCCESS;
-}
-
-int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) {
- return likely(mc) ? mdbx_cursor_bind(txn, mc, mc->mc_dbi) : MDBX_EINVAL;
-}
+ /* This is writable space for a leaf page. Usually not needed. */
+ leaf = (MDBX_page *)ptr;
-int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) {
- if (unlikely(!src))
- return MDBX_EINVAL;
- if (unlikely(src->mc_signature != MDBX_MC_LIVE))
- return (src->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ while (couple.outer.mc_snum > 0) {
+ mp = couple.outer.mc_pg[couple.outer.mc_top];
+ unsigned n = page_numkeys(mp);
- int rc = mdbx_cursor_bind(src->mc_txn, dest, src->mc_dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (IS_LEAF(mp)) {
+ if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) {
+ for (i = 0; i < n; i++) {
+ MDBX_node *node = page_node(mp, i);
+ if (node_flags(node) & F_BIGDATA) {
+ MDBX_page *omp;
- assert(dest->mc_db == src->mc_db);
- assert(dest->mc_dbi == src->mc_dbi);
- assert(dest->mc_dbx == src->mc_dbx);
- assert(dest->mc_dbistate == src->mc_dbistate);
-again:
- assert(dest->mc_txn == src->mc_txn);
- dest->mc_flags ^= (dest->mc_flags ^ src->mc_flags) & ~C_UNTRACK;
- dest->mc_top = src->mc_top;
- dest->mc_snum = src->mc_snum;
- for (unsigned i = 0; i < src->mc_snum; ++i) {
- dest->mc_ki[i] = src->mc_ki[i];
- dest->mc_pg[i] = src->mc_pg[i];
- }
+ /* Need writable leaf */
+ if (mp != leaf) {
+ couple.outer.mc_pg[couple.outer.mc_top] = leaf;
+ mdbx_page_copy(leaf, mp, my->mc_env->me_psize);
+ mp = leaf;
+ node = page_node(mp, i);
+ }
- if (src->mc_xcursor) {
- dest->mc_xcursor->mx_db = src->mc_xcursor->mx_db;
- dest->mc_xcursor->mx_dbx = src->mc_xcursor->mx_dbx;
- src = &src->mc_xcursor->mx_cursor;
- dest = &dest->mc_xcursor->mx_cursor;
- goto again;
- }
+ const pgno_t pgno = node_largedata_pgno(node);
+ poke_pgno(node_data(node), my->mc_next_pgno);
+ rc = mdbx_page_get(&couple.outer, pgno, &omp,
+ pp_txnid4chk(mp, my->mc_txn));
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ unsigned toggle = my->mc_head & 1;
+ if (my->mc_wlen[toggle] + my->mc_env->me_psize >
+ ((size_t)(MDBX_ENVCOPY_WRITEBUF))) {
+ rc = mdbx_env_cthr_toggle(my);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ toggle = my->mc_head & 1;
+ }
+ mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
+ memcpy(mo, omp, my->mc_env->me_psize);
+ mo->mp_pgno = my->mc_next_pgno;
+ my->mc_next_pgno += omp->mp_pages;
+ my->mc_wlen[toggle] += my->mc_env->me_psize;
+ if (omp->mp_pages > 1) {
+ my->mc_olen[toggle] = pgno2bytes(my->mc_env, omp->mp_pages - 1);
+ my->mc_over[toggle] = (uint8_t *)omp + my->mc_env->me_psize;
+ rc = mdbx_env_cthr_toggle(my);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ toggle = my->mc_head & 1;
+ }
+ } else if (node_flags(node) & F_SUBDATA) {
+ if (!MDBX_DISABLE_PAGECHECKS &&
+ unlikely(node_ds(node) != sizeof(MDBX_db))) {
+ rc = MDBX_CORRUPTED;
+ goto done;
+ }
- return MDBX_SUCCESS;
-}
+ /* Need writable leaf */
+ if (mp != leaf) {
+ couple.outer.mc_pg[couple.outer.mc_top] = leaf;
+ mdbx_page_copy(leaf, mp, my->mc_env->me_psize);
+ mp = leaf;
+ node = page_node(mp, i);
+ }
-void mdbx_cursor_close(MDBX_cursor *mc) {
- if (likely(mc)) {
- mdbx_ensure(NULL, mc->mc_signature == MDBX_MC_LIVE ||
- mc->mc_signature == MDBX_MC_READY4CLOSE);
- MDBX_txn *const txn = mc->mc_txn;
- if (!mc->mc_backup) {
- mc->mc_txn = NULL;
- /* Unlink from txn, if tracked. */
- if (mc->mc_flags & C_UNTRACK) {
- mdbx_ensure(txn->mt_env, check_txn(txn, 0) == MDBX_SUCCESS);
- MDBX_cursor **prev = &txn->mt_cursors[mc->mc_dbi];
- while (*prev && *prev != mc)
- prev = &(*prev)->mc_next;
- mdbx_tassert(txn, *prev == mc);
- *prev = mc->mc_next;
+ MDBX_db db;
+ memcpy(&db, node_data(node), sizeof(MDBX_db));
+ rc = mdbx_env_cwalk(my, &db.md_root, node_flags(node) & F_DUPDATA);
+ if (rc)
+ goto done;
+ memcpy(node_data(node), &db, sizeof(MDBX_db));
+ }
+ }
+ }
+ } else {
+ couple.outer.mc_ki[couple.outer.mc_top]++;
+ if (couple.outer.mc_ki[couple.outer.mc_top] < n) {
+ again:
+ rc = mdbx_page_get(
+ &couple.outer,
+ node_pgno(page_node(mp, couple.outer.mc_ki[couple.outer.mc_top])),
+ &mp, pp_txnid4chk(mp, my->mc_txn));
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ couple.outer.mc_top++;
+ couple.outer.mc_snum++;
+ couple.outer.mc_ki[couple.outer.mc_top] = 0;
+ if (IS_BRANCH(mp)) {
+ /* Whenever we advance to a sibling branch page,
+ * we must proceed all the way down to its first leaf. */
+ mdbx_page_copy(couple.outer.mc_pg[couple.outer.mc_top], mp,
+ my->mc_env->me_psize);
+ goto again;
+ } else
+ couple.outer.mc_pg[couple.outer.mc_top] = mp;
+ continue;
}
- mc->mc_signature = 0;
- mc->mc_next = mc;
- mdbx_free(mc);
+ }
+ unsigned toggle = my->mc_head & 1;
+ if (my->mc_wlen[toggle] + my->mc_wlen[toggle] >
+ ((size_t)(MDBX_ENVCOPY_WRITEBUF))) {
+ rc = mdbx_env_cthr_toggle(my);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto done;
+ toggle = my->mc_head & 1;
+ }
+ mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
+ mdbx_page_copy(mo, mp, my->mc_env->me_psize);
+ mo->mp_pgno = my->mc_next_pgno++;
+ my->mc_wlen[toggle] += my->mc_env->me_psize;
+ if (couple.outer.mc_top) {
+ /* Update parent if there is one */
+ node_set_pgno(page_node(couple.outer.mc_pg[couple.outer.mc_top - 1],
+ couple.outer.mc_ki[couple.outer.mc_top - 1]),
+ mo->mp_pgno);
+ mdbx_cursor_pop(&couple.outer);
} else {
- /* Cursor closed before nested txn ends */
- mdbx_tassert(txn, mc->mc_signature == MDBX_MC_LIVE);
- mdbx_ensure(txn->mt_env, check_txn_rw(txn, 0) == MDBX_SUCCESS);
- mc->mc_signature = MDBX_MC_WAIT4EOT;
+ /* Otherwise we're done */
+ *pg = mo->mp_pgno;
+ break;
}
}
+done:
+ mdbx_free(buf);
+ return rc;
}
-MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) {
- if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE))
- return NULL;
- MDBX_txn *txn = mc->mc_txn;
- if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
- return NULL;
- if (unlikely(txn->mt_flags & MDBX_TXN_FINISHED))
- return NULL;
- return txn;
-}
-
-MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) {
- if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE))
- return UINT_MAX;
- return mc->mc_dbi;
-}
+__cold static void compact_fixup_meta(MDBX_env *env, MDBX_meta *meta) {
+ /* Calculate filesize taking in account shrink/growing thresholds */
+ if (meta->mm_geo.next != meta->mm_geo.now) {
+ meta->mm_geo.now = meta->mm_geo.next;
+ const pgno_t aligner = pv2pages(
+ meta->mm_geo.grow_pv ? meta->mm_geo.grow_pv : meta->mm_geo.shrink_pv);
+ if (aligner) {
+ const pgno_t aligned = pgno_align2os_pgno(
+ env, meta->mm_geo.next + aligner - meta->mm_geo.next % aligner);
+ meta->mm_geo.now = aligned;
+ }
+ }
-/* Return the count of duplicate data items for the current key */
-int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) {
- if (unlikely(mc == NULL))
- return MDBX_EINVAL;
+ if (meta->mm_geo.now < meta->mm_geo.lower)
+ meta->mm_geo.now = meta->mm_geo.lower;
+ if (meta->mm_geo.now > meta->mm_geo.upper)
+ meta->mm_geo.now = meta->mm_geo.upper;
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ /* Update signature */
+ assert(meta->mm_geo.now >= meta->mm_geo.next);
+ unaligned_poke_u64(4, meta->mm_datasync_sign, meta_sign(meta));
+}
- int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/* Make resizeable */
+__cold static void make_sizeable(MDBX_meta *meta) {
+ meta->mm_geo.lower = MIN_PAGENO;
+ if (meta->mm_geo.grow_pv == 0) {
+ const pgno_t step = 1 + (meta->mm_geo.upper - meta->mm_geo.lower) / 42;
+ meta->mm_geo.grow_pv = pages2pv(step);
+ }
+ if (meta->mm_geo.shrink_pv == 0) {
+ const pgno_t step = pv2pages(meta->mm_geo.grow_pv) << 1;
+ meta->mm_geo.shrink_pv = pages2pv(step);
+ }
+}
- if (unlikely(countp == NULL || !(mc->mc_flags & C_INITIALIZED)))
- return MDBX_EINVAL;
+/* Copy environment with compaction. */
+__cold static int mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn,
+ mdbx_filehandle_t fd, uint8_t *buffer,
+ const bool dest_is_pipe, const int flags) {
+ const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
+ uint8_t *const data_buffer =
+ buffer + ceil_powerof2(meta_bytes, env->me_os_psize);
+ MDBX_meta *const meta = mdbx_init_metas(env, buffer);
+ meta_set_txnid(env, meta, read_txn->mt_txnid);
- if (!mc->mc_snum) {
- *countp = 0;
- return MDBX_NOTFOUND;
- }
+ if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
+ make_sizeable(meta);
- MDBX_page *mp = mc->mc_pg[mc->mc_top];
- if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= page_numkeys(mp)) {
- *countp = 0;
- return MDBX_NOTFOUND;
+ /* copy canary sequences if present */
+ if (read_txn->mt_canary.v) {
+ meta->mm_canary = read_txn->mt_canary;
+ meta->mm_canary.v = constmeta_txnid(env, meta);
}
- *countp = 1;
- if (mc->mc_xcursor != NULL) {
- MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- mdbx_cassert(mc, mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags &
- C_INITIALIZED));
- *countp = unlikely(mc->mc_xcursor->mx_db.md_entries > PTRDIFF_MAX)
- ? PTRDIFF_MAX
- : (size_t)mc->mc_xcursor->mx_db.md_entries;
+ /* Set metapage 1 with current main DB */
+ pgno_t new_root, root = read_txn->mt_dbs[MAIN_DBI].md_root;
+ if ((new_root = root) == P_INVALID) {
+ /* When the DB is empty, handle it specially to
+ * fix any breakage like page leaks from ITS#8174. */
+ meta->mm_dbs[MAIN_DBI].md_flags = read_txn->mt_dbs[MAIN_DBI].md_flags;
+ compact_fixup_meta(env, meta);
+ if (dest_is_pipe) {
+ int rc = mdbx_write(fd, buffer, meta_bytes);
+ if (rc != MDBX_SUCCESS)
+ return rc;
}
- }
- return MDBX_SUCCESS;
-}
+ } else {
+ /* Count free pages + GC pages. Subtract from last_pg
+ * to find the new last_pg, which also becomes the new root. */
+ pgno_t freecount = 0;
+ MDBX_cursor_couple couple;
+ MDBX_val key, data;
-/* Replace the key for a branch node with a new key.
- * Set MDBX_TXN_ERROR on failure.
- * [in] mc Cursor pointing to the node to operate on.
- * [in] key The new key to use.
- * Returns 0 on success, non-zero on failure. */
-static int mdbx_update_key(MDBX_cursor *mc, const MDBX_val *key) {
- MDBX_page *mp;
- MDBX_node *node;
- char *base;
- size_t len;
- int delta, ksize, oksize;
- int ptr, i, nkeys, indx;
- DKBUF_DEBUG;
+ int rc = mdbx_cursor_init(&couple.outer, read_txn, FREE_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ while ((rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_NEXT)) == 0)
+ freecount += *(pgno_t *)data.iov_base;
+ if (unlikely(rc != MDBX_NOTFOUND))
+ return rc;
- mdbx_cassert(mc, cursor_is_tracked(mc));
- indx = mc->mc_ki[mc->mc_top];
- mp = mc->mc_pg[mc->mc_top];
- node = page_node(mp, indx);
- ptr = mp->mp_ptrs[indx];
-#if MDBX_DEBUG
- MDBX_val k2;
- k2.iov_base = node_key(node);
- k2.iov_len = node_ks(node);
- mdbx_debug("update key %u (offset %u) [%s] to [%s] on page %" PRIaPGNO, indx,
- ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), mp->mp_pgno);
-#endif /* MDBX_DEBUG */
+ freecount += read_txn->mt_dbs[FREE_DBI].md_branch_pages +
+ read_txn->mt_dbs[FREE_DBI].md_leaf_pages +
+ read_txn->mt_dbs[FREE_DBI].md_overflow_pages;
- /* Sizes must be 2-byte aligned. */
- ksize = EVEN(key->iov_len);
- oksize = EVEN(node_ks(node));
- delta = ksize - oksize;
+ new_root = read_txn->mt_next_pgno - 1 - freecount;
+ meta->mm_geo.next = new_root + 1;
+ meta->mm_dbs[MAIN_DBI] = read_txn->mt_dbs[MAIN_DBI];
+ meta->mm_dbs[MAIN_DBI].md_root = new_root;
- /* Shift node contents if EVEN(key length) changed. */
- if (delta) {
- if (delta > (int)page_room(mp)) {
- /* not enough space left, do a delete and split */
- mdbx_debug("Not enough room, delta = %d, splitting...", delta);
- pgno_t pgno = node_pgno(node);
- mdbx_node_del(mc, 0);
- int rc = mdbx_page_split(mc, key, NULL, pgno, MDBX_SPLIT_REPLACE);
- if (rc == MDBX_SUCCESS && mdbx_audit_enabled())
- rc = mdbx_cursor_check(mc, C_UPDATING);
+ mdbx_copy ctx;
+ memset(&ctx, 0, sizeof(ctx));
+ rc = mdbx_condpair_init(&ctx.mc_condpair);
+ if (unlikely(rc != MDBX_SUCCESS))
return rc;
+
+ memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)) * 2);
+ ctx.mc_wbuf[0] = data_buffer;
+ ctx.mc_wbuf[1] = data_buffer + ((size_t)(MDBX_ENVCOPY_WRITEBUF));
+ ctx.mc_next_pgno = NUM_METAS;
+ ctx.mc_env = env;
+ ctx.mc_fd = fd;
+ ctx.mc_txn = read_txn;
+
+ mdbx_thread_t thread;
+ int thread_err = mdbx_thread_create(&thread, mdbx_env_copythr, &ctx);
+ if (likely(thread_err == MDBX_SUCCESS)) {
+ if (dest_is_pipe) {
+ compact_fixup_meta(env, meta);
+ rc = mdbx_write(fd, buffer, meta_bytes);
+ }
+ if (rc == MDBX_SUCCESS)
+ rc = mdbx_env_cwalk(&ctx, &root, 0);
+ mdbx_env_cthr_toggle(&ctx);
+ mdbx_env_cthr_toggle(&ctx);
+ thread_err = mdbx_thread_join(thread);
+ mdbx_assert(env, (ctx.mc_tail == ctx.mc_head &&
+ ctx.mc_wlen[ctx.mc_head & 1] == 0) ||
+ ctx.mc_error);
+ mdbx_condpair_destroy(&ctx.mc_condpair);
}
+ if (unlikely(thread_err != MDBX_SUCCESS))
+ return thread_err;
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ if (unlikely(ctx.mc_error != MDBX_SUCCESS))
+ return ctx.mc_error;
- nkeys = page_numkeys(mp);
- for (i = 0; i < nkeys; i++) {
- if (mp->mp_ptrs[i] <= ptr) {
- mdbx_cassert(mc, mp->mp_ptrs[i] >= delta);
- mp->mp_ptrs[i] -= (indx_t)delta;
+ if (dest_is_pipe) {
+ if (unlikely(root != new_root)) {
+ mdbx_error("post-compactification root %" PRIaPGNO
+ " NE expected %" PRIaPGNO
+ " (source DB corrupted or has a page leak(s))",
+ root, new_root);
+ return MDBX_CORRUPTED; /* page leak or corrupt DB */
+ }
+ } else {
+ if (unlikely(root > new_root)) {
+ mdbx_error("post-compactification root %" PRIaPGNO
+ " GT expected %" PRIaPGNO " (source DB corrupted)",
+ root, new_root);
+ return MDBX_CORRUPTED; /* page leak or corrupt DB */
+ }
+ if (unlikely(root < new_root)) {
+ mdbx_warning("post-compactification root %" PRIaPGNO
+ " LT expected %" PRIaPGNO " (page leak(s) in source DB)",
+ root, new_root);
+ /* fixup meta */
+ meta->mm_dbs[MAIN_DBI].md_root = root;
+ meta->mm_geo.next = root + 1;
}
+ compact_fixup_meta(env, meta);
}
-
- base = (char *)mp + mp->mp_upper + PAGEHDRSZ;
- len = ptr - mp->mp_upper + NODESIZE;
- memmove(base - delta, base, len);
- mdbx_cassert(mc, mp->mp_upper >= delta);
- mp->mp_upper -= (indx_t)delta;
-
- node = page_node(mp, indx);
}
- /* But even if no shift was needed, update ksize */
- node_set_ks(node, key->iov_len);
+ /* Extend file if required */
+ if (meta->mm_geo.now != meta->mm_geo.next) {
+ const size_t whole_size = pgno2bytes(env, meta->mm_geo.now);
+ if (!dest_is_pipe)
+ return mdbx_ftruncate(fd, whole_size);
- if (likely(key->iov_len /* to avoid UBSAN traps*/ != 0))
- memcpy(node_key(node), key->iov_base, key->iov_len);
+ const size_t used_size = pgno2bytes(env, meta->mm_geo.next);
+ memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)));
+ for (size_t offset = used_size; offset < whole_size;) {
+ const size_t chunk =
+ (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < whole_size - offset)
+ ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
+ : whole_size - offset;
+ /* copy to avoid EFAULT in case swapped-out */
+ int rc = mdbx_write(fd, data_buffer, chunk);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ offset += chunk;
+ }
+ }
return MDBX_SUCCESS;
}
-/* Move a node from csrc to cdst. */
-static int mdbx_node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
- int rc;
- DKBUF_DEBUG;
+/* Copy environment as-is. */
+__cold static int mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn,
+ mdbx_filehandle_t fd, uint8_t *buffer,
+ const bool dest_is_pipe, const int flags) {
+ /* We must start the actual read txn after blocking writers */
+ int rc = mdbx_txn_end(read_txn, MDBX_END_RESET_TMP);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- MDBX_page *psrc = csrc->mc_pg[csrc->mc_top];
- MDBX_page *pdst = cdst->mc_pg[cdst->mc_top];
- mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
- mdbx_cassert(csrc, csrc->mc_dbi == cdst->mc_dbi);
- mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top);
- if (unlikely(PAGETYPE(psrc) != PAGETYPE(pdst))) {
- bailout:
- mdbx_error("Wrong or mismatch pages's types (src %d, dst %d) to move node",
- PAGETYPE(psrc), PAGETYPE(pdst));
- csrc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return MDBX_PROBLEM;
+ /* Temporarily block writers until we snapshot the meta pages */
+ rc = mdbx_txn_lock(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ rc = mdbx_txn_renew0(read_txn, MDBX_TXN_RDONLY);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_txn_unlock(env);
+ return rc;
}
- MDBX_val key4move;
- switch (PAGETYPE(psrc)) {
- case P_BRANCH: {
- const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]);
- mdbx_cassert(csrc, node_flags(srcnode) == 0);
- const pgno_t srcpg = node_pgno(srcnode);
- key4move.iov_len = node_ks(srcnode);
- key4move.iov_base = node_key(srcnode);
+ mdbx_jitter4testing(false);
+ const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
+ /* Make a snapshot of meta-pages,
+ * but writing ones after the data was flushed */
+ memcpy(buffer, env->me_map, meta_bytes);
+ MDBX_meta *const headcopy = /* LY: get pointer to the snapshot copy */
+ (MDBX_meta *)(buffer + ((uint8_t *)meta_prefer_last(env) - env->me_map));
+ mdbx_txn_unlock(env);
- if (csrc->mc_ki[csrc->mc_top] == 0) {
- const unsigned snum = csrc->mc_snum;
- mdbx_cassert(csrc, snum > 0);
- /* must find the lowest key below src */
- rc = mdbx_page_search_lowest(csrc);
- MDBX_page *lowest_page = csrc->mc_pg[csrc->mc_top];
- if (unlikely(rc))
- return rc;
- mdbx_cassert(csrc, IS_LEAF(lowest_page));
- if (unlikely(!IS_LEAF(lowest_page)))
- goto bailout;
- if (IS_LEAF2(lowest_page)) {
- key4move.iov_len = csrc->mc_db->md_xsize;
- key4move.iov_base = page_leaf2key(lowest_page, 0, key4move.iov_len);
- } else {
- const MDBX_node *lowest_node = page_node(lowest_page, 0);
- key4move.iov_len = node_ks(lowest_node);
- key4move.iov_base = node_key(lowest_node);
- }
+ if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
+ make_sizeable(headcopy);
+ /* Update signature to steady */
+ unaligned_poke_u64(4, headcopy->mm_datasync_sign, meta_sign(headcopy));
- /* restore cursor after mdbx_page_search_lowest() */
- csrc->mc_snum = snum;
- csrc->mc_top = snum - 1;
- csrc->mc_ki[csrc->mc_top] = 0;
+ /* Copy the data */
+ const size_t whole_size = pgno_align2os_bytes(env, read_txn->mt_end_pgno);
+ const size_t used_size = pgno2bytes(env, read_txn->mt_next_pgno);
+ mdbx_jitter4testing(false);
- /* paranoia */
- mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
- mdbx_cassert(csrc, IS_BRANCH(psrc));
- if (unlikely(!IS_BRANCH(psrc)))
- goto bailout;
+ if (dest_is_pipe)
+ rc = mdbx_write(fd, buffer, meta_bytes);
+
+ uint8_t *const data_buffer =
+ buffer + ceil_powerof2(meta_bytes, env->me_os_psize);
+#if MDBX_USE_COPYFILERANGE
+ static bool copyfilerange_unavailable;
+ bool not_the_same_filesystem = false;
+#endif /* MDBX_USE_COPYFILERANGE */
+ for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) {
+#if MDBX_USE_SENDFILE
+ static bool sendfile_unavailable;
+ if (dest_is_pipe && likely(!sendfile_unavailable)) {
+ off_t in_offset = offset;
+ const ssize_t written =
+ sendfile(fd, env->me_lazy_fd, &in_offset, used_size - offset);
+ if (likely(written > 0)) {
+ offset = in_offset;
+ continue;
+ }
+ rc = MDBX_ENODATA;
+ if (written == 0 || ignore_enosys(rc = errno) != MDBX_RESULT_TRUE)
+ break;
+ sendfile_unavailable = true;
}
+#endif /* MDBX_USE_SENDFILE */
- if (cdst->mc_ki[cdst->mc_top] == 0) {
- const unsigned snum = cdst->mc_snum;
- mdbx_cassert(csrc, snum > 0);
- MDBX_cursor mn;
- cursor_copy(cdst, &mn);
- /* must find the lowest key below dst */
- rc = mdbx_page_search_lowest(&mn);
- if (unlikely(rc))
- return rc;
- MDBX_page *const lowest_page = mn.mc_pg[mn.mc_top];
- mdbx_cassert(cdst, IS_LEAF(lowest_page));
- if (unlikely(!IS_LEAF(lowest_page)))
- goto bailout;
- MDBX_val key;
- if (IS_LEAF2(lowest_page)) {
- key.iov_len = mn.mc_db->md_xsize;
- key.iov_base = page_leaf2key(lowest_page, 0, key.iov_len);
- } else {
- MDBX_node *lowest_node = page_node(lowest_page, 0);
- key.iov_len = node_ks(lowest_node);
- key.iov_base = node_key(lowest_node);
+#if MDBX_USE_COPYFILERANGE
+ if (!dest_is_pipe && !not_the_same_filesystem &&
+ likely(!copyfilerange_unavailable)) {
+ off_t in_offset = offset, out_offset = offset;
+ ssize_t bytes_copied = copy_file_range(
+ env->me_lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0);
+ if (likely(bytes_copied > 0)) {
+ offset = in_offset;
+ continue;
+ }
+ rc = MDBX_ENODATA;
+ if (bytes_copied == 0)
+ break;
+ rc = errno;
+ if (rc == EXDEV)
+ not_the_same_filesystem = true;
+ else if (ignore_enosys(rc) == MDBX_RESULT_TRUE)
+ copyfilerange_unavailable = true;
+ else
+ break;
+ }
+#endif /* MDBX_USE_COPYFILERANGE */
+
+ /* fallback to portable */
+ const size_t chunk =
+ (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < used_size - offset)
+ ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
+ : used_size - offset;
+ /* copy to avoid EFAULT in case swapped-out */
+ memcpy(data_buffer, env->me_map + offset, chunk);
+ rc = mdbx_write(fd, data_buffer, chunk);
+ offset += chunk;
+ }
+
+ /* Extend file if required */
+ if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) {
+ if (!dest_is_pipe)
+ rc = mdbx_ftruncate(fd, whole_size);
+ else {
+ memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)));
+ for (size_t offset = used_size;
+ rc == MDBX_SUCCESS && offset < whole_size;) {
+ const size_t chunk =
+ (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < whole_size - offset)
+ ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
+ : whole_size - offset;
+ /* copy to avoid EFAULT in case swapped-out */
+ rc = mdbx_write(fd, data_buffer, chunk);
+ offset += chunk;
}
+ }
+ }
- /* restore cursor after mdbx_page_search_lowest() */
- mn.mc_snum = snum;
- mn.mc_top = snum - 1;
- mn.mc_ki[mn.mc_top] = 0;
+ return rc;
+}
- const intptr_t delta =
- EVEN(key.iov_len) - EVEN(node_ks(page_node(mn.mc_pg[mn.mc_top], 0)));
- const intptr_t needed =
- branch_size(cdst->mc_txn->mt_env, &key4move) + delta;
- const intptr_t have = page_room(pdst);
- if (unlikely(needed > have))
- return MDBX_RESULT_TRUE;
+__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd,
+ unsigned flags) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ const int dest_is_pipe = mdbx_is_pipe(fd);
+ if (MDBX_IS_ERROR(dest_is_pipe))
+ return dest_is_pipe;
+
+ if (!dest_is_pipe) {
+ rc = mdbx_fseek(fd, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+
+ const size_t buffer_size =
+ pgno_align2os_bytes(env, NUM_METAS) +
+ ceil_powerof2(((flags & MDBX_CP_COMPACT)
+ ? ((size_t)(MDBX_ENVCOPY_WRITEBUF)) * 2
+ : ((size_t)(MDBX_ENVCOPY_WRITEBUF))),
+ env->me_os_psize);
- if (unlikely((rc = mdbx_page_touch(csrc)) ||
- (rc = mdbx_page_touch(cdst))))
- return rc;
- psrc = csrc->mc_pg[csrc->mc_top];
- pdst = cdst->mc_pg[cdst->mc_top];
+ uint8_t *buffer = NULL;
+ rc = mdbx_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
- if (unlikely(rc))
- return rc;
- } else {
- const size_t needed = branch_size(cdst->mc_txn->mt_env, &key4move);
- const size_t have = page_room(pdst);
- if (unlikely(needed > have))
- return MDBX_RESULT_TRUE;
+ MDBX_txn *read_txn = NULL;
+ /* Do the lock/unlock of the reader mutex before starting the
+ * write txn. Otherwise other read txns could block writers. */
+ rc = mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &read_txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_memalign_free(buffer);
+ return rc;
+ }
- if (unlikely((rc = mdbx_page_touch(csrc)) ||
- (rc = mdbx_page_touch(cdst))))
- return rc;
- psrc = csrc->mc_pg[csrc->mc_top];
- pdst = cdst->mc_pg[cdst->mc_top];
- }
+ if (!dest_is_pipe) {
+ /* Firstly write a stub to meta-pages.
+ * Now we sure to incomplete copy will not be used. */
+ memset(buffer, -1, pgno2bytes(env, NUM_METAS));
+ rc = mdbx_write(fd, buffer, pgno2bytes(env, NUM_METAS));
+ }
- mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
- " to node %u on page %" PRIaPGNO,
- "branch", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
- psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
- /* Add the node to the destination page. */
- rc =
- mdbx_node_add_branch(cdst, cdst->mc_ki[cdst->mc_top], &key4move, srcpg);
- } break;
+ if (likely(rc == MDBX_SUCCESS)) {
+ memset(buffer, 0, pgno2bytes(env, NUM_METAS));
+ rc = ((flags & MDBX_CP_COMPACT) ? mdbx_env_compact : mdbx_env_copy_asis)(
+ env, read_txn, fd, buffer, dest_is_pipe, flags);
+ }
+ mdbx_txn_abort(read_txn);
- case P_LEAF: {
- /* Mark src and dst as dirty. */
- if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst))))
- return rc;
- psrc = csrc->mc_pg[csrc->mc_top];
- pdst = cdst->mc_pg[cdst->mc_top];
- const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]);
- MDBX_val data;
- data.iov_len = node_ds(srcnode);
- data.iov_base = node_data(srcnode);
- key4move.iov_len = node_ks(srcnode);
- key4move.iov_base = node_key(srcnode);
- mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
- " to node %u on page %" PRIaPGNO,
- "leaf", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
- psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
- /* Add the node to the destination page. */
- rc = mdbx_node_add_leaf(cdst, cdst->mc_ki[cdst->mc_top], &key4move, &data,
- node_flags(srcnode));
- } break;
+ if (!dest_is_pipe) {
+ if (likely(rc == MDBX_SUCCESS))
+ rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE);
- case P_LEAF | P_LEAF2: {
- /* Mark src and dst as dirty. */
- if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst))))
- return rc;
- psrc = csrc->mc_pg[csrc->mc_top];
- pdst = cdst->mc_pg[cdst->mc_top];
- key4move.iov_len = csrc->mc_db->md_xsize;
- key4move.iov_base =
- page_leaf2key(psrc, csrc->mc_ki[csrc->mc_top], key4move.iov_len);
- mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO
- " to node %u on page %" PRIaPGNO,
- "leaf2", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move),
- psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno);
- /* Add the node to the destination page. */
- rc = mdbx_node_add_leaf2(cdst, cdst->mc_ki[cdst->mc_top], &key4move);
- } break;
+ /* Write actual meta */
+ if (likely(rc == MDBX_SUCCESS))
+ rc = mdbx_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0);
- default:
- goto bailout;
+ if (likely(rc == MDBX_SUCCESS))
+ rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
}
+ mdbx_memalign_free(buffer);
+ return rc;
+}
+
+__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path,
+ MDBX_copy_flags_t flags) {
+ int rc = check_env(env, true);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- /* Delete the node from the source page. */
- mdbx_node_del(csrc, key4move.iov_len);
+ if (unlikely(!dest_path))
+ return MDBX_EINVAL;
- mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
- mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
- mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
+ /* The destination path must exist, but the destination file must not.
+ * We don't want the OS to cache the writes, since the source data is
+ * already in the OS cache. */
+ mdbx_filehandle_t newfd;
+ rc = mdbx_openfile(MDBX_OPEN_COPY, env, dest_path, &newfd,
+#if defined(_WIN32) || defined(_WIN64)
+ (mdbx_mode_t)-1
+#else
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
+#endif
+ );
- {
- /* Adjust other cursors pointing to mp */
- MDBX_cursor *m2, *m3;
- const MDBX_dbi dbi = csrc->mc_dbi;
- mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top);
- if (fromleft) {
- /* If we're adding on the left, bump others up */
- for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
- m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
- continue;
- if (m3 != cdst && m3->mc_pg[csrc->mc_top] == pdst &&
- m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) {
- m3->mc_ki[csrc->mc_top]++;
- }
- if (m3 != csrc && m3->mc_pg[csrc->mc_top] == psrc &&
- m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) {
- m3->mc_pg[csrc->mc_top] = pdst;
- m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
- mdbx_cassert(csrc, csrc->mc_top > 0);
- m3->mc_ki[csrc->mc_top - 1]++;
- }
- if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
- XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
- }
- } else {
- /* Adding on the right, bump others down */
- for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
- m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == csrc)
- continue;
- if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
- continue;
- if (m3->mc_pg[csrc->mc_top] == psrc) {
- if (!m3->mc_ki[csrc->mc_top]) {
- m3->mc_pg[csrc->mc_top] = pdst;
- m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
- mdbx_cassert(csrc, csrc->mc_top > 0);
- m3->mc_ki[csrc->mc_top - 1]--;
- } else {
- m3->mc_ki[csrc->mc_top]--;
- }
- if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
- XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top],
- m3->mc_ki[csrc->mc_top]);
- }
- }
- }
+ if (rc == MDBX_SUCCESS) {
+#if defined(_WIN32) || defined(_WIN64)
+ OVERLAPPED ov;
+ memset(&ov, 0, sizeof(ov));
+ if (!LockFileEx(newfd, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
+ 0, 0, INT32_MAX, &ov))
+ rc = GetLastError();
+#else
+ struct flock lock_op;
+ memset(&lock_op, 0, sizeof(lock_op));
+ lock_op.l_type = F_WRLCK;
+ lock_op.l_whence = SEEK_SET;
+ lock_op.l_start = 0;
+ lock_op.l_len =
+ (sizeof(lock_op.l_len) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff;
+ if (fcntl(newfd, F_SETLK, &lock_op)
+#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \
+ (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24)
+ || flock(newfd, LOCK_EX | LOCK_NB)
+#endif /* Linux */
+ )
+ rc = errno;
+#endif /* Windows / POSIX */
}
- /* Update the parent separators. */
- if (csrc->mc_ki[csrc->mc_top] == 0) {
- mdbx_cassert(csrc, csrc->mc_top > 0);
- if (csrc->mc_ki[csrc->mc_top - 1] != 0) {
- MDBX_val key;
- if (IS_LEAF2(psrc)) {
- key.iov_len = psrc->mp_leaf2_ksize;
- key.iov_base = page_leaf2key(psrc, 0, key.iov_len);
- } else {
- MDBX_node *srcnode = page_node(psrc, 0);
- key.iov_len = node_ks(srcnode);
- key.iov_base = node_key(srcnode);
- }
- mdbx_debug("update separator for source page %" PRIaPGNO " to [%s]",
- psrc->mp_pgno, DKEY_DEBUG(&key));
- MDBX_cursor mn;
- cursor_copy(csrc, &mn);
- mdbx_cassert(csrc, mn.mc_snum > 0);
- mn.mc_snum--;
- mn.mc_top--;
- /* We want mdbx_rebalance to find mn when doing fixups */
- WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- if (IS_BRANCH(psrc)) {
- const MDBX_val nullkey = {0, 0};
- const indx_t ix = csrc->mc_ki[csrc->mc_top];
- csrc->mc_ki[csrc->mc_top] = 0;
- rc = mdbx_update_key(csrc, &nullkey);
- csrc->mc_ki[csrc->mc_top] = ix;
- mdbx_cassert(csrc, rc == MDBX_SUCCESS);
- }
+ if (rc == MDBX_SUCCESS)
+ rc = mdbx_env_copy2fd(env, newfd, flags);
+
+ if (newfd != INVALID_HANDLE_VALUE) {
+ int err = mdbx_closefile(newfd);
+ if (rc == MDBX_SUCCESS && err != rc)
+ rc = err;
+ if (rc != MDBX_SUCCESS)
+ (void)mdbx_removefile(dest_path);
}
- if (cdst->mc_ki[cdst->mc_top] == 0) {
- mdbx_cassert(cdst, cdst->mc_top > 0);
- if (cdst->mc_ki[cdst->mc_top - 1] != 0) {
- MDBX_val key;
- if (IS_LEAF2(pdst)) {
- key.iov_len = pdst->mp_leaf2_ksize;
- key.iov_base = page_leaf2key(pdst, 0, key.iov_len);
- } else {
- MDBX_node *srcnode = page_node(pdst, 0);
- key.iov_len = node_ks(srcnode);
- key.iov_base = node_key(srcnode);
- }
- mdbx_debug("update separator for destination page %" PRIaPGNO " to [%s]",
- pdst->mp_pgno, DKEY_DEBUG(&key));
- MDBX_cursor mn;
- cursor_copy(cdst, &mn);
- mdbx_cassert(cdst, mn.mc_snum > 0);
- mn.mc_snum--;
- mn.mc_top--;
- /* We want mdbx_rebalance to find mn when doing fixups */
- WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key));
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- if (IS_BRANCH(pdst)) {
- const MDBX_val nullkey = {0, 0};
- const indx_t ix = cdst->mc_ki[cdst->mc_top];
- cdst->mc_ki[cdst->mc_top] = 0;
- rc = mdbx_update_key(cdst, &nullkey);
- cdst->mc_ki[cdst->mc_top] = ix;
- mdbx_cassert(cdst, rc == MDBX_SUCCESS);
- }
+ return rc;
+}
+
+/******************************************************************************/
+
+__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags,
+ bool onoff) {
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ if (unlikely(flags &
+ ((env->me_flags & MDBX_ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS
+ : ~ENV_USABLE_FLAGS)))
+ return MDBX_EPERM;
+
+ if (unlikely(env->me_flags & MDBX_RDONLY))
+ return MDBX_EACCESS;
+
+ if ((env->me_flags & MDBX_ENV_ACTIVE) &&
+ unlikely(env->me_txn0->mt_owner == mdbx_thread_self()))
+ return MDBX_BUSY;
+
+ const bool lock_needed = (env->me_flags & MDBX_ENV_ACTIVE) &&
+ env->me_txn0->mt_owner != mdbx_thread_self();
+ bool should_unlock = false;
+ if (lock_needed) {
+ rc = mdbx_txn_lock(env, false);
+ if (unlikely(rc))
+ return rc;
+ should_unlock = true;
}
+ if (onoff)
+ env->me_flags = merge_sync_flags(env->me_flags, flags);
+ else
+ env->me_flags &= ~flags;
+
+ if (should_unlock)
+ mdbx_txn_unlock(env);
return MDBX_SUCCESS;
}
-/* Merge one page into another.
- *
- * The nodes from the page pointed to by csrc will be copied to the page
- * pointed to by cdst and then the csrc page will be freed.
- *
- * [in] csrc Cursor pointing to the source page.
- * [in] cdst Cursor pointing to the destination page.
- *
- * Returns 0 on success, non-zero on failure. */
-static int mdbx_page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
- MDBX_val key;
- int rc;
+__cold int mdbx_env_get_flags(const MDBX_env *env, unsigned *arg) {
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- mdbx_cassert(csrc, csrc != cdst);
- mdbx_cassert(csrc, cursor_is_tracked(csrc));
- mdbx_cassert(cdst, cursor_is_tracked(cdst));
- const MDBX_page *const psrc = csrc->mc_pg[csrc->mc_top];
- MDBX_page *pdst = cdst->mc_pg[cdst->mc_top];
- mdbx_debug("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->mp_pgno,
- pdst->mp_pgno);
+ if (unlikely(!arg))
+ return MDBX_EINVAL;
- mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst));
- mdbx_cassert(csrc,
- csrc->mc_dbi == cdst->mc_dbi && csrc->mc_db == cdst->mc_db);
- mdbx_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */
- mdbx_cassert(cdst, cdst->mc_snum > 1);
- mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
- IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
- mdbx_cassert(csrc, csrc->mc_snum < csrc->mc_db->md_depth ||
- IS_LEAF(csrc->mc_pg[csrc->mc_db->md_depth - 1]));
- mdbx_cassert(cdst, page_room(pdst) >= page_used(cdst->mc_txn->mt_env, psrc));
- const int pagetype = PAGETYPE(psrc);
+ *arg = env->me_flags & ENV_USABLE_FLAGS;
+ return MDBX_SUCCESS;
+}
- /* Move all nodes from src to dst */
- const unsigned dst_nkeys = page_numkeys(pdst);
- const unsigned src_nkeys = page_numkeys(psrc);
- mdbx_cassert(cdst, dst_nkeys + src_nkeys >= (IS_LEAF(psrc) ? 1u : 2u));
- if (likely(src_nkeys)) {
- unsigned j = dst_nkeys;
- if (unlikely(pagetype & P_LEAF2)) {
- /* Mark dst as dirty. */
- if (unlikely(rc = mdbx_page_touch(cdst)))
- return rc;
+__cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) {
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- key.iov_len = csrc->mc_db->md_xsize;
- key.iov_base = page_data(psrc);
- unsigned i = 0;
- do {
- rc = mdbx_node_add_leaf2(cdst, j++, &key);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- key.iov_base = (char *)key.iov_base + key.iov_len;
- } while (++i != src_nkeys);
- } else {
- MDBX_node *srcnode = page_node(psrc, 0);
- key.iov_len = node_ks(srcnode);
- key.iov_base = node_key(srcnode);
- if (pagetype & P_BRANCH) {
- MDBX_cursor mn;
- cursor_copy(csrc, &mn);
- /* must find the lowest key below src */
- rc = mdbx_page_search_lowest(&mn);
- if (unlikely(rc))
- return rc;
+ env->me_userctx = ctx;
+ return MDBX_SUCCESS;
+}
- const MDBX_page *mp = mn.mc_pg[mn.mc_top];
- if (likely(!IS_LEAF2(mp))) {
- mdbx_cassert(&mn, IS_LEAF(mp));
- const MDBX_node *lowest = page_node(mp, 0);
- key.iov_len = node_ks(lowest);
- key.iov_base = node_key(lowest);
- } else {
- mdbx_cassert(&mn, mn.mc_top > csrc->mc_top);
- key.iov_len = mp->mp_leaf2_ksize;
- key.iov_base = page_leaf2key(mp, mn.mc_ki[mn.mc_top], key.iov_len);
- }
- mdbx_cassert(&mn, key.iov_len >= csrc->mc_dbx->md_klen_min);
- mdbx_cassert(&mn, key.iov_len <= csrc->mc_dbx->md_klen_max);
+__cold void *mdbx_env_get_userctx(const MDBX_env *env) {
+ return env ? env->me_userctx : NULL;
+}
- const size_t dst_room = page_room(pdst);
- const size_t src_used = page_used(cdst->mc_txn->mt_env, psrc);
- const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len;
- if (unlikely(space_needed > dst_room))
- return MDBX_RESULT_TRUE;
- }
+__cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) {
+ int rc = check_env(env, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* Mark dst as dirty. */
- if (unlikely(rc = mdbx_page_touch(cdst)))
- return rc;
+#if MDBX_DEBUG
+ env->me_assert_func = func;
+ return MDBX_SUCCESS;
+#else
+ (void)func;
+ return MDBX_ENOSYS;
+#endif
+}
- unsigned i = 0;
- while (true) {
- if (pagetype & P_LEAF) {
- MDBX_val data;
- data.iov_len = node_ds(srcnode);
- data.iov_base = node_data(srcnode);
- rc = mdbx_node_add_leaf(cdst, j++, &key, &data, node_flags(srcnode));
- } else {
- mdbx_cassert(csrc, node_flags(srcnode) == 0);
- rc = mdbx_node_add_branch(cdst, j++, &key, node_pgno(srcnode));
- }
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+__cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- if (++i == src_nkeys)
- break;
- srcnode = page_node(psrc, i);
- key.iov_len = node_ks(srcnode);
- key.iov_base = node_key(srcnode);
- }
- }
+ if (unlikely(!arg))
+ return MDBX_EINVAL;
- pdst = cdst->mc_pg[cdst->mc_top];
- mdbx_debug("dst page %" PRIaPGNO " now has %u keys (%.1f%% filled)",
- pdst->mp_pgno, page_numkeys(pdst),
- page_fill(cdst->mc_txn->mt_env, pdst));
+ *arg = env->me_pathname;
+ return MDBX_SUCCESS;
+}
- mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
- mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
- }
+__cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* Unlink the src page from parent and add to free list. */
- csrc->mc_top--;
- mdbx_node_del(csrc, 0);
- if (csrc->mc_ki[csrc->mc_top] == 0) {
- const MDBX_val nullkey = {0, 0};
- rc = mdbx_update_key(csrc, &nullkey);
- if (unlikely(rc)) {
- csrc->mc_top++;
- return rc;
- }
- }
- csrc->mc_top++;
+ if (unlikely(!arg))
+ return MDBX_EINVAL;
- mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]);
- mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]);
+ *arg = env->me_lazy_fd;
+ return MDBX_SUCCESS;
+}
- {
- /* Adjust other cursors pointing to mp */
- MDBX_cursor *m2, *m3;
- const MDBX_dbi dbi = csrc->mc_dbi;
- const unsigned top = csrc->mc_top;
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) {
+ return __inline_mdbx_env_stat(env, stat, bytes);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
- m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == csrc || top >= m3->mc_snum)
- continue;
- if (m3->mc_pg[top] == psrc) {
- m3->mc_pg[top] = pdst;
- mdbx_cassert(m3, dst_nkeys + m3->mc_ki[top] <= UINT16_MAX);
- m3->mc_ki[top] += (indx_t)dst_nkeys;
- m3->mc_ki[top - 1] = cdst->mc_ki[top - 1];
- } else if (m3->mc_pg[top - 1] == csrc->mc_pg[top - 1] &&
- m3->mc_ki[top - 1] > csrc->mc_ki[top - 1]) {
- m3->mc_ki[top - 1]--;
- }
- if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
- XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]);
- }
- }
+static void stat_get(const MDBX_db *db, MDBX_stat *st, size_t bytes) {
+ st->ms_depth = db->md_depth;
+ st->ms_branch_pages = db->md_branch_pages;
+ st->ms_leaf_pages = db->md_leaf_pages;
+ st->ms_overflow_pages = db->md_overflow_pages;
+ st->ms_entries = db->md_entries;
+ if (likely(bytes >=
+ offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
+ st->ms_mod_txnid = db->md_mod_txnid;
+}
- /* If not operating on GC, allow this page to be reused
- * in this txn. Otherwise just add to free list. */
- rc = mdbx_page_retire(csrc, (MDBX_page *)psrc);
- if (unlikely(rc))
- return rc;
+static void stat_add(const MDBX_db *db, MDBX_stat *const st,
+ const size_t bytes) {
+ st->ms_depth += db->md_depth;
+ st->ms_branch_pages += db->md_branch_pages;
+ st->ms_leaf_pages += db->md_leaf_pages;
+ st->ms_overflow_pages += db->md_overflow_pages;
+ st->ms_entries += db->md_entries;
+ if (likely(bytes >=
+ offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
+ st->ms_mod_txnid = (st->ms_mod_txnid > db->md_mod_txnid) ? st->ms_mod_txnid
+ : db->md_mod_txnid;
+}
- mdbx_cassert(cdst, cdst->mc_db->md_entries > 0);
- mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth);
- mdbx_cassert(cdst, cdst->mc_top > 0);
- mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1);
- MDBX_page *const top_page = cdst->mc_pg[cdst->mc_top];
- const indx_t top_indx = cdst->mc_ki[cdst->mc_top];
- const unsigned save_snum = cdst->mc_snum;
- const uint16_t save_depth = cdst->mc_db->md_depth;
- mdbx_cursor_pop(cdst);
- rc = mdbx_rebalance(cdst);
- if (unlikely(rc))
- return rc;
+__cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
+ int err = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- mdbx_cassert(cdst, cdst->mc_db->md_entries > 0);
- mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth);
- mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1);
+ st->ms_psize = txn->mt_env->me_psize;
+#if 1
+ /* assuming GC is internal and not subject for accounting */
+ stat_get(&txn->mt_dbs[MAIN_DBI], st, bytes);
+#else
+ stat_get(&txn->mt_dbs[FREE_DBI], st, bytes);
+ stat_add(&txn->mt_dbs[MAIN_DBI], st, bytes);
+#endif
-#if MDBX_ENABLE_PGOP_STAT
- cdst->mc_txn->mt_env->me_lck->mti_pgop_stat.merge.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
+ /* account opened named subDBs */
+ for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++)
+ if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID)
+ stat_add(txn->mt_dbs + dbi, st, bytes);
- if (IS_LEAF(cdst->mc_pg[cdst->mc_top])) {
- /* LY: don't touch cursor if top-page is a LEAF */
- mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
- PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
- return MDBX_SUCCESS;
- }
+ if (!(txn->mt_dbs[MAIN_DBI].md_flags & (MDBX_DUPSORT | MDBX_INTEGERKEY)) &&
+ txn->mt_dbs[MAIN_DBI].md_entries /* TODO: use `md_subs` field */) {
+ MDBX_cursor_couple cx;
+ err = mdbx_cursor_init(&cx.outer, (MDBX_txn *)txn, MAIN_DBI);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- mdbx_cassert(cdst, page_numkeys(top_page) == dst_nkeys + src_nkeys);
+ /* scan and account not opened named subDBs */
+ err = mdbx_page_search(&cx.outer, NULL, MDBX_PS_FIRST);
+ while (err == MDBX_SUCCESS) {
+ const MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top];
+ for (unsigned i = 0; i < page_numkeys(mp); i++) {
+ const MDBX_node *node = page_node(mp, i);
+ if (node_flags(node) != F_SUBDATA)
+ continue;
+ if (unlikely(node_ds(node) != sizeof(MDBX_db)))
+ return MDBX_CORRUPTED;
- if (unlikely(pagetype != PAGETYPE(top_page))) {
- /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */
- goto bailout;
- }
+ /* skip opened and already accounted */
+ for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++)
+ if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID &&
+ node_ks(node) == txn->mt_dbxs[dbi].md_name.iov_len &&
+ memcmp(node_key(node), txn->mt_dbxs[dbi].md_name.iov_base,
+ node_ks(node)) == 0) {
+ node = NULL;
+ break;
+ }
- if (top_page == cdst->mc_pg[cdst->mc_top]) {
- /* LY: don't touch cursor if prev top-page already on the top */
- mdbx_cassert(cdst, cdst->mc_ki[cdst->mc_top] == top_indx);
- mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
- PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
- return MDBX_SUCCESS;
+ if (node) {
+ MDBX_db db;
+ memcpy(&db, node_data(node), sizeof(db));
+ stat_add(&db, st, bytes);
+ }
+ }
+ err = mdbx_cursor_sibling(&cx.outer, SIBLING_RIGHT);
+ }
+ if (unlikely(err != MDBX_NOTFOUND))
+ return err;
}
- const int new_snum = save_snum - save_depth + cdst->mc_db->md_depth;
- if (unlikely(new_snum < 1 || new_snum > cdst->mc_db->md_depth)) {
- /* LY: out of range, unable restore cursor's stack */
- goto bailout;
- }
+ return MDBX_SUCCESS;
+}
- if (top_page == cdst->mc_pg[new_snum - 1]) {
- mdbx_cassert(cdst, cdst->mc_ki[new_snum - 1] == top_indx);
- /* LY: restore cursor stack */
- cdst->mc_snum = (uint16_t)new_snum;
- cdst->mc_top = (uint16_t)new_snum - 1;
- mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
- IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
- mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
- PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
- return MDBX_SUCCESS;
- }
+__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn,
+ MDBX_stat *dest, size_t bytes) {
+ if (unlikely(!dest))
+ return MDBX_EINVAL;
+ const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
+ if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
+ return MDBX_EINVAL;
- MDBX_page *const stub_page = (MDBX_page *)(~(uintptr_t)top_page);
- const indx_t stub_indx = top_indx;
- if (save_depth > cdst->mc_db->md_depth &&
- ((cdst->mc_pg[save_snum - 1] == top_page &&
- cdst->mc_ki[save_snum - 1] == top_indx) ||
- (cdst->mc_pg[save_snum - 1] == stub_page &&
- cdst->mc_ki[save_snum - 1] == stub_indx))) {
- /* LY: restore cursor stack */
- cdst->mc_pg[new_snum - 1] = top_page;
- cdst->mc_ki[new_snum - 1] = top_indx;
- cdst->mc_pg[new_snum] = (MDBX_page *)(~(uintptr_t)cdst->mc_pg[new_snum]);
- cdst->mc_ki[new_snum] = ~cdst->mc_ki[new_snum];
- cdst->mc_snum = (uint16_t)new_snum;
- cdst->mc_top = (uint16_t)new_snum - 1;
- mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth ||
- IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1]));
- mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) ||
- PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype);
- return MDBX_SUCCESS;
+ if (likely(txn)) {
+ if (env && unlikely(txn->mt_env != env))
+ return MDBX_EINVAL;
+ return stat_acc(txn, dest, bytes);
}
-bailout:
- /* LY: unable restore cursor's stack */
- cdst->mc_flags &= ~C_INITIALIZED;
- return MDBX_CURSOR_FULL;
-}
+ int err = check_env(env, true);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
-static void cursor_restore(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
- mdbx_cassert(cdst, cdst->mc_dbi == csrc->mc_dbi);
- mdbx_cassert(cdst, cdst->mc_txn == csrc->mc_txn);
- mdbx_cassert(cdst, cdst->mc_db == csrc->mc_db);
- mdbx_cassert(cdst, cdst->mc_dbx == csrc->mc_dbx);
- mdbx_cassert(cdst, cdst->mc_dbistate == csrc->mc_dbistate);
- cdst->mc_snum = csrc->mc_snum;
- cdst->mc_top = csrc->mc_top;
- cdst->mc_flags = csrc->mc_flags;
+ if (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self())
+ /* inside write-txn */
+ return stat_acc(env->me_txn, dest, bytes);
- for (unsigned i = 0; i < csrc->mc_snum; i++) {
- cdst->mc_pg[i] = csrc->mc_pg[i];
- cdst->mc_ki[i] = csrc->mc_ki[i];
- }
-}
+ MDBX_txn *tmp_txn;
+ err = mdbx_txn_begin((MDBX_env *)env, NULL, MDBX_TXN_RDONLY, &tmp_txn);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
-/* Copy the contents of a cursor.
- * [in] csrc The cursor to copy from.
- * [out] cdst The cursor to copy to. */
-static void cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
- mdbx_cassert(csrc, csrc->mc_txn->mt_txnid >=
- csrc->mc_txn->mt_env->me_lck->mti_oldest_reader.weak);
- cdst->mc_dbi = csrc->mc_dbi;
- cdst->mc_next = NULL;
- cdst->mc_backup = NULL;
- cdst->mc_xcursor = NULL;
- cdst->mc_txn = csrc->mc_txn;
- cdst->mc_db = csrc->mc_db;
- cdst->mc_dbx = csrc->mc_dbx;
- cdst->mc_dbistate = csrc->mc_dbistate;
- cursor_restore(csrc, cdst);
+ const int rc = stat_acc(tmp_txn, dest, bytes);
+ err = mdbx_txn_abort(tmp_txn);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ return rc;
}
-/* Rebalance the tree after a delete operation.
- * [in] mc Cursor pointing to the page where rebalancing should begin.
- * Returns 0 on success, non-zero on failure. */
-static int mdbx_rebalance(MDBX_cursor *mc) {
- mdbx_cassert(mc, cursor_is_tracked(mc));
- mdbx_cassert(mc, mc->mc_snum > 0);
- mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth ||
- IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1]));
- const int pagetype = PAGETYPE(mc->mc_pg[mc->mc_top]);
-
- STATIC_ASSERT(P_BRANCH == 1);
- const unsigned minkeys = (pagetype & P_BRANCH) + 1;
+__cold int mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi,
+ uint32_t *mask) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* Pages emptier than this are candidates for merging. */
- unsigned room_threshold = likely(mc->mc_dbi != FREE_DBI)
- ? mc->mc_txn->mt_env->me_merge_threshold
- : mc->mc_txn->mt_env->me_merge_threshold_gc;
+ if (unlikely(!mask))
+ return MDBX_EINVAL;
- const MDBX_page *const tp = mc->mc_pg[mc->mc_top];
- const unsigned numkeys = page_numkeys(tp);
- const unsigned room = page_room(tp);
- mdbx_debug("rebalancing %s page %" PRIaPGNO
- " (has %u keys, full %.1f%%, used %u, room %u bytes )",
- (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys,
- page_fill(mc->mc_txn->mt_env, tp),
- page_used(mc->mc_txn->mt_env, tp), room);
+ if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
+ return MDBX_BAD_DBI;
- if (unlikely(numkeys < minkeys)) {
- mdbx_debug("page %" PRIaPGNO " must be merged due keys < %u threshold",
- tp->mp_pgno, minkeys);
- } else if (unlikely(room > room_threshold)) {
- mdbx_debug("page %" PRIaPGNO " should be merged due room %u > %u threshold",
- tp->mp_pgno, room, room_threshold);
- } else {
- mdbx_debug("no need to rebalance page %" PRIaPGNO
- ", room %u < %u threshold",
- tp->mp_pgno, room, room_threshold);
- mdbx_cassert(mc, mc->mc_db->md_entries > 0);
- return MDBX_SUCCESS;
- }
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ if ((cx.outer.mc_db->md_flags & MDBX_DUPSORT) == 0)
+ return MDBX_RESULT_TRUE;
- int rc;
- if (mc->mc_snum < 2) {
- MDBX_page *const mp = mc->mc_pg[0];
- const unsigned nkeys = page_numkeys(mp);
- mdbx_cassert(mc, (mc->mc_db->md_entries == 0) == (nkeys == 0));
- if (IS_SUBP(mp)) {
- mdbx_debug("%s", "Can't rebalance a subpage, ignoring");
- mdbx_cassert(mc, pagetype & P_LEAF);
- return MDBX_SUCCESS;
+ MDBX_val key, data;
+ rc = mdbx_cursor_first(&cx.outer, &key, &data);
+ *mask = 0;
+ while (rc == MDBX_SUCCESS) {
+ const MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
+ cx.outer.mc_ki[cx.outer.mc_top]);
+ const MDBX_db *db = node_data(node);
+ const unsigned flags = node_flags(node);
+ switch (flags) {
+ case F_BIGDATA:
+ case 0:
+ /* single-value entry, deep = 0 */
+ *mask |= 1 << 0;
+ break;
+ case F_DUPDATA:
+ /* single sub-page, deep = 1 */
+ *mask |= 1 << 1;
+ break;
+ case F_DUPDATA | F_SUBDATA:
+ /* sub-tree */
+ *mask |= 1 << UNALIGNED_PEEK_16(db, MDBX_db, md_depth);
+ break;
+ default:
+ mdbx_error("wrong node-flags %u", flags);
+ return MDBX_CORRUPTED;
}
- if (nkeys == 0) {
- mdbx_cassert(mc, IS_LEAF(mp));
- mdbx_debug("%s", "tree is completely empty");
- mdbx_cassert(mc, (*mc->mc_dbistate & DBI_DIRTY) != 0);
- mc->mc_db->md_root = P_INVALID;
- mc->mc_db->md_depth = 0;
- mdbx_cassert(mc, mc->mc_db->md_branch_pages == 0 &&
- mc->mc_db->md_overflow_pages == 0 &&
- mc->mc_db->md_leaf_pages == 1);
- /* Adjust cursors pointing to mp */
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
- m2 = m2->mc_next) {
- MDBX_cursor *m3 =
- (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == mc || !(m3->mc_flags & C_INITIALIZED))
- continue;
- if (m3->mc_pg[0] == mp) {
- m3->mc_snum = 0;
- m3->mc_top = 0;
- m3->mc_flags &= ~C_INITIALIZED;
- }
- }
- mc->mc_snum = 0;
- mc->mc_top = 0;
- mc->mc_flags &= ~C_INITIALIZED;
+ rc = mdbx_cursor_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP);
+ }
- rc = mdbx_page_retire(mc, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- } else if (IS_BRANCH(mp) && nkeys == 1) {
- mdbx_debug("%s", "collapsing root page!");
- mc->mc_db->md_root = node_pgno(page_node(mp, 0));
- rc = mdbx_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0],
- pp_txnid4chk(mp, mc->mc_txn));
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- mc->mc_db->md_depth--;
- mc->mc_ki[0] = mc->mc_ki[1];
- for (int i = 1; i < mc->mc_db->md_depth; i++) {
- mc->mc_pg[i] = mc->mc_pg[i + 1];
- mc->mc_ki[i] = mc->mc_ki[i + 1];
- }
+ return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
+}
- /* Adjust other cursors pointing to mp */
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
- m2 = m2->mc_next) {
- MDBX_cursor *m3 =
- (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == mc || !(m3->mc_flags & C_INITIALIZED))
- continue;
- if (m3->mc_pg[0] == mp) {
- for (int i = 0; i < mc->mc_db->md_depth; i++) {
- m3->mc_pg[i] = m3->mc_pg[i + 1];
- m3->mc_ki[i] = m3->mc_ki[i + 1];
- }
- m3->mc_snum--;
- m3->mc_top--;
- }
- }
- mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) ||
- PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype);
- mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth ||
- IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1]));
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info,
+ size_t bytes) {
+ return __inline_mdbx_env_info(env, info, bytes);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- rc = mdbx_page_retire(mc, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- } else {
- mdbx_debug("root page %" PRIaPGNO
- " doesn't need rebalancing (flags 0x%x)",
- mp->mp_pgno, mp->mp_flags);
+__cold static int fetch_envinfo_ex(const MDBX_env *env, const MDBX_txn *txn,
+ MDBX_envinfo *arg, const size_t bytes) {
+
+ const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
+ const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
+
+ /* is the environment open?
+ * (todo4recovery://erased_by_github/libmdbx/issues/171) */
+ if (unlikely(!env->me_map)) {
+ /* environment not yet opened */
+#if 1
+ /* default behavior: returns the available info but zeroed the rest */
+ memset(arg, 0, bytes);
+ arg->mi_geo.lower = env->me_dbgeo.lower;
+ arg->mi_geo.upper = env->me_dbgeo.upper;
+ arg->mi_geo.shrink = env->me_dbgeo.shrink;
+ arg->mi_geo.grow = env->me_dbgeo.grow;
+ arg->mi_geo.current = env->me_dbgeo.now;
+ arg->mi_maxreaders = env->me_maxreaders;
+ arg->mi_dxb_pagesize = env->me_psize;
+ arg->mi_sys_pagesize = env->me_os_psize;
+ if (likely(bytes > size_before_bootid)) {
+ arg->mi_bootid.current.x = bootid.x;
+ arg->mi_bootid.current.y = bootid.y;
}
return MDBX_SUCCESS;
+#else
+ /* some users may prefer this behavior: return appropriate error */
+ return MDBX_EPERM;
+#endif
}
- /* The parent (branch page) must have at least 2 pointers,
- * otherwise the tree is invalid. */
- const unsigned pre_top = mc->mc_top - 1;
- mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[pre_top]));
- mdbx_cassert(mc, !IS_SUBP(mc->mc_pg[0]));
- mdbx_cassert(mc, page_numkeys(mc->mc_pg[pre_top]) > 1);
+ const MDBX_meta *const meta0 = METAPAGE(env, 0);
+ const MDBX_meta *const meta1 = METAPAGE(env, 1);
+ const MDBX_meta *const meta2 = METAPAGE(env, 2);
+ if (unlikely(env->me_flags & MDBX_FATAL_ERROR))
+ return MDBX_PANIC;
- /* Leaf page fill factor is below the threshold.
- * Try to move keys from left or right neighbor, or
- * merge with a neighbor page. */
+ volatile const MDBX_meta *const recent_meta = meta_prefer_last(env);
+ arg->mi_recent_txnid = meta_txnid(env, recent_meta);
+ arg->mi_meta0_txnid = meta_txnid(env, meta0);
+ arg->mi_meta0_sign = unaligned_peek_u64(4, meta0->mm_datasync_sign);
+ arg->mi_meta1_txnid = meta_txnid(env, meta1);
+ arg->mi_meta1_sign = unaligned_peek_u64(4, meta1->mm_datasync_sign);
+ arg->mi_meta2_txnid = meta_txnid(env, meta2);
+ arg->mi_meta2_sign = unaligned_peek_u64(4, meta2->mm_datasync_sign);
+ if (likely(bytes > size_before_bootid)) {
+ memcpy(&arg->mi_bootid.meta0, &meta0->mm_bootid, 16);
+ memcpy(&arg->mi_bootid.meta1, &meta1->mm_bootid, 16);
+ memcpy(&arg->mi_bootid.meta2, &meta2->mm_bootid, 16);
+ }
- /* Find neighbors. */
- MDBX_cursor mn;
- cursor_copy(mc, &mn);
+ volatile const MDBX_meta *txn_meta = recent_meta;
+ arg->mi_last_pgno = txn_meta->mm_geo.next - 1;
+ arg->mi_geo.current = pgno2bytes(env, txn_meta->mm_geo.now);
+ if (txn) {
+ arg->mi_last_pgno = txn->mt_next_pgno - 1;
+ arg->mi_geo.current = pgno2bytes(env, txn->mt_end_pgno);
- MDBX_page *left = nullptr, *right = nullptr;
- if (mn.mc_ki[pre_top] > 0) {
- rc = mdbx_page_get(
- &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] - 1)),
- &left, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn));
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- mdbx_cassert(mc, PAGETYPE(left) == PAGETYPE(mc->mc_pg[mc->mc_top]));
- }
- if (mn.mc_ki[pre_top] + 1u < page_numkeys(mn.mc_pg[pre_top])) {
- rc = mdbx_page_get(
- &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] + 1)),
- &right, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn));
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- mdbx_cassert(mc, PAGETYPE(right) == PAGETYPE(mc->mc_pg[mc->mc_top]));
+ const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_TXN_RDONLY)
+ ? txn->mt_txnid
+ : txn->mt_txnid - xMDBX_TXNID_STEP;
+ txn_meta = (arg->mi_meta0_txnid == wanna_meta_txnid) ? meta0 : txn_meta;
+ txn_meta = (arg->mi_meta1_txnid == wanna_meta_txnid) ? meta1 : txn_meta;
+ txn_meta = (arg->mi_meta2_txnid == wanna_meta_txnid) ? meta2 : txn_meta;
}
- mdbx_cassert(mc, left || right);
+ arg->mi_geo.lower = pgno2bytes(env, txn_meta->mm_geo.lower);
+ arg->mi_geo.upper = pgno2bytes(env, txn_meta->mm_geo.upper);
+ arg->mi_geo.shrink = pgno2bytes(env, pv2pages(txn_meta->mm_geo.shrink_pv));
+ arg->mi_geo.grow = pgno2bytes(env, pv2pages(txn_meta->mm_geo.grow_pv));
+ const pgno_t unsynced_pages =
+ atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed) +
+ (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) !=
+ (uint32_t)arg->mi_recent_txnid);
- const unsigned ki_top = mc->mc_ki[mc->mc_top];
- const unsigned ki_pre_top = mn.mc_ki[pre_top];
- const unsigned nkeys = page_numkeys(mn.mc_pg[mn.mc_top]);
+ arg->mi_mapsize = env->me_dxb_mmap.limit;
- const unsigned left_room = left ? page_room(left) : 0;
- const unsigned right_room = right ? page_room(right) : 0;
- const unsigned left_nkeys = left ? page_numkeys(left) : 0;
- const unsigned right_nkeys = right ? page_numkeys(right) : 0;
-retry:
- if (left_room > room_threshold && left_room >= right_room) {
- /* try merge with left */
- mdbx_cassert(mc, left_nkeys >= minkeys);
- mn.mc_pg[mn.mc_top] = left;
- mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1);
- mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1);
- mc->mc_ki[mc->mc_top] = 0;
- const unsigned new_ki = ki_top + left_nkeys;
- mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
- /* We want mdbx_rebalance to find mn when doing fixups */
- WITH_CURSOR_TRACKING(mn, rc = mdbx_page_merge(mc, &mn));
- if (likely(rc != MDBX_RESULT_TRUE)) {
- cursor_restore(&mn, mc);
- mc->mc_ki[mc->mc_top] = (indx_t)new_ki;
- mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
- return rc;
- }
- }
- if (right_room > room_threshold) {
- /* try merge with right */
- mdbx_cassert(mc, right_nkeys >= minkeys);
- mn.mc_pg[mn.mc_top] = right;
- mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1);
- mn.mc_ki[mn.mc_top] = 0;
- mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
- WITH_CURSOR_TRACKING(mn, rc = mdbx_page_merge(&mn, mc));
- if (likely(rc != MDBX_RESULT_TRUE)) {
- mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
- mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
- return rc;
- }
- }
+ const MDBX_lockinfo *const lck = env->me_lck;
+ arg->mi_maxreaders = env->me_maxreaders;
+ arg->mi_numreaders = env->me_lck_mmap.lck
+ ? atomic_load32(&lck->mti_numreaders, mo_Relaxed)
+ : INT32_MAX;
+ arg->mi_dxb_pagesize = env->me_psize;
+ arg->mi_sys_pagesize = env->me_os_psize;
- if (left_nkeys > minkeys &&
- (right_nkeys <= left_nkeys || right_room >= left_room)) {
- /* try move from left */
- mn.mc_pg[mn.mc_top] = left;
- mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1);
- mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1);
- mc->mc_ki[mc->mc_top] = 0;
- WITH_CURSOR_TRACKING(mn, rc = mdbx_node_move(&mn, mc, true));
- if (likely(rc != MDBX_RESULT_TRUE)) {
- mc->mc_ki[mc->mc_top] = (indx_t)(ki_top + 1);
- mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
- return rc;
- }
- }
- if (right_nkeys > minkeys) {
- /* try move from right */
- mn.mc_pg[mn.mc_top] = right;
- mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1);
- mn.mc_ki[mn.mc_top] = 0;
- mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
- WITH_CURSOR_TRACKING(mn, rc = mdbx_node_move(&mn, mc, false));
- if (likely(rc != MDBX_RESULT_TRUE)) {
- mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
- mdbx_cassert(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys);
- return rc;
- }
+ if (likely(bytes > size_before_bootid)) {
+ arg->mi_unsync_volume = pgno2bytes(env, unsynced_pages);
+ const uint64_t monotime_now = mdbx_osal_monotime();
+ uint64_t ts = atomic_load64(&lck->mti_sync_timestamp, mo_Relaxed);
+ arg->mi_since_sync_seconds16dot16 =
+ ts ? mdbx_osal_monotime_to_16dot16(monotime_now - ts) : 0;
+ ts = atomic_load64(&lck->mti_reader_check_timestamp, mo_Relaxed);
+ arg->mi_since_reader_check_seconds16dot16 =
+ ts ? mdbx_osal_monotime_to_16dot16(monotime_now - ts) : 0;
+ arg->mi_autosync_threshold = pgno2bytes(
+ env, atomic_load32(&lck->mti_autosync_threshold, mo_Relaxed));
+ arg->mi_autosync_period_seconds16dot16 = mdbx_osal_monotime_to_16dot16(
+ atomic_load64(&lck->mti_autosync_period, mo_Relaxed));
+ arg->mi_bootid.current.x = bootid.x;
+ arg->mi_bootid.current.y = bootid.y;
+ arg->mi_mode = env->me_lck_mmap.lck ? lck->mti_envmode.weak : env->me_flags;
}
- if (nkeys >= minkeys) {
- mc->mc_ki[mc->mc_top] = (indx_t)ki_top;
- if (!mdbx_audit_enabled())
- return MDBX_SUCCESS;
- return mdbx_cursor_check(mc, C_UPDATING);
+ if (likely(bytes > size_before_pgop_stat)) {
+#if MDBX_ENABLE_PGOP_STAT
+ arg->mi_pgop_stat.newly =
+ atomic_load64(&lck->mti_pgop_stat.newly, mo_Relaxed);
+ arg->mi_pgop_stat.cow = atomic_load64(&lck->mti_pgop_stat.cow, mo_Relaxed);
+ arg->mi_pgop_stat.clone =
+ atomic_load64(&lck->mti_pgop_stat.clone, mo_Relaxed);
+ arg->mi_pgop_stat.split =
+ atomic_load64(&lck->mti_pgop_stat.split, mo_Relaxed);
+ arg->mi_pgop_stat.merge =
+ atomic_load64(&lck->mti_pgop_stat.merge, mo_Relaxed);
+ arg->mi_pgop_stat.spill =
+ atomic_load64(&lck->mti_pgop_stat.spill, mo_Relaxed);
+ arg->mi_pgop_stat.unspill =
+ atomic_load64(&lck->mti_pgop_stat.unspill, mo_Relaxed);
+ arg->mi_pgop_stat.wops =
+ atomic_load64(&lck->mti_pgop_stat.wops, mo_Relaxed);
+#else
+ memset(&arg->mi_pgop_stat, 0, sizeof(arg->mi_pgop_stat));
+#endif /* MDBX_ENABLE_PGOP_STAT*/
}
- if (likely(room_threshold > 0)) {
- room_threshold = 0;
- goto retry;
+ arg->mi_self_latter_reader_txnid = arg->mi_latter_reader_txnid =
+ arg->mi_recent_txnid;
+ for (unsigned i = 0; i < arg->mi_numreaders; ++i) {
+ const uint32_t pid =
+ atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease);
+ if (pid) {
+ const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (arg->mi_latter_reader_txnid > txnid)
+ arg->mi_latter_reader_txnid = txnid;
+ if (pid == env->me_pid && arg->mi_self_latter_reader_txnid > txnid)
+ arg->mi_self_latter_reader_txnid = txnid;
+ }
}
- mdbx_error("Unable to merge/rebalance %s page %" PRIaPGNO
- " (has %u keys, full %.1f%%, used %u, room %u bytes )",
- (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys,
- page_fill(mc->mc_txn->mt_env, tp),
- page_used(mc->mc_txn->mt_env, tp), room);
- return MDBX_PROBLEM;
+
+ mdbx_compiler_barrier();
+ return MDBX_SUCCESS;
}
-__cold static int mdbx_page_check(MDBX_cursor *const mc,
- const MDBX_page *const mp, unsigned options) {
- DKBUF;
- options |= mc->mc_flags;
- MDBX_env *const env = mc->mc_txn->mt_env;
- const unsigned nkeys = page_numkeys(mp);
- char *const end_of_page = (char *)mp + env->me_psize;
- if (unlikely(mp->mp_pgno < MIN_PAGENO || mp->mp_pgno > MAX_PAGENO))
- return bad_page(mp, "invalid pgno (%u)\n", mp->mp_pgno);
- if (IS_OVERFLOW(mp)) {
- if (unlikely(mp->mp_pages < 1 && mp->mp_pages >= MAX_PAGENO / 2))
- return bad_page(mp, "invalid overflow n-pages (%u)\n", mp->mp_pages);
- if (unlikely(mp->mp_pgno + mp->mp_pages > mc->mc_txn->mt_next_pgno))
- return bad_page(mp, "overflow page beyond (%u) next-pgno\n",
- mp->mp_pgno + mp->mp_pages);
- if (unlikely((options & (C_SUB | C_COPYING)) == C_SUB))
- return bad_page(mp,
- "unexpected overflow-page for dupsort db (flags 0x%x)\n",
- mc->mc_db->md_flags);
- return MDBX_SUCCESS;
+__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn,
+ MDBX_envinfo *arg, size_t bytes) {
+ if (unlikely((env == NULL && txn == NULL) || arg == NULL))
+ return MDBX_EINVAL;
+
+ if (txn) {
+ int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ }
+ if (env) {
+ int err = check_env(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ if (txn && unlikely(txn->mt_env != env))
+ return MDBX_EINVAL;
+ } else {
+ env = txn->mt_env;
}
- int rc = MDBX_SUCCESS;
- if ((options & C_UPDATING) == 0 || !IS_MODIFIABLE(mc->mc_txn, mp)) {
- if (unlikely(nkeys < 2 && IS_BRANCH(mp)))
- rc = bad_page(mp, "branch-page nkey (%u) < 2\n", nkeys);
+ const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
+ const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
+ if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid &&
+ bytes != size_before_pgop_stat)
+ return MDBX_EINVAL;
+
+ MDBX_envinfo snap;
+ int rc = fetch_envinfo_ex(env, txn, &snap, sizeof(snap));
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ while (1) {
+ rc = fetch_envinfo_ex(env, txn, arg, bytes);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ snap.mi_since_sync_seconds16dot16 = arg->mi_since_sync_seconds16dot16;
+ snap.mi_since_reader_check_seconds16dot16 =
+ arg->mi_since_reader_check_seconds16dot16;
+ if (likely(memcmp(&snap, arg, bytes) == 0))
+ return MDBX_SUCCESS;
+ memcpy(&snap, arg, bytes);
}
- if (IS_LEAF2(mp) && unlikely((options & (C_SUB | C_COPYING)) == 0))
- rc = bad_page(mp, "unexpected leaf2-page (db flags 0x%x)\n",
- mc->mc_db->md_flags);
+}
- MDBX_val here, prev = {0, 0};
- for (unsigned i = 0; i < nkeys; ++i) {
- if (IS_LEAF2(mp)) {
- const size_t ksize = mp->mp_leaf2_ksize;
- char *const key = page_leaf2key(mp, i, ksize);
- if (unlikely(end_of_page < key + ksize)) {
- rc = bad_page(mp, "leaf2-key beyond (%zu) page-end\n",
- key + ksize - end_of_page);
- continue;
- }
+static __inline MDBX_cmp_func *get_default_keycmp(unsigned flags) {
+ return (flags & MDBX_REVERSEKEY) ? cmp_reverse
+ : (flags & MDBX_INTEGERKEY) ? cmp_int_align2
+ : cmp_lexical;
+}
- if ((options & C_COPYING) == 0) {
- if (unlikely(ksize != mc->mc_dbx->md_klen_min)) {
- if (unlikely(ksize < mc->mc_dbx->md_klen_min ||
- ksize > mc->mc_dbx->md_klen_max))
- rc = bad_page(
- mp, "leaf2-key size (%zu) <> min/max key-length (%zu/%zu)\n",
- ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max);
- else
- mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = ksize;
- }
- if ((options & C_SKIPORD) == 0) {
- here.iov_len = ksize;
- here.iov_base = key;
- if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0))
- rc = bad_page(mp, "leaf2-key #%u wrong order (%s >= %s)\n", i,
- DKEY(&prev), DVAL(&here));
- prev = here;
- }
- }
- } else {
- const MDBX_node *const node = page_node(mp, i);
- const char *node_end = (char *)node + NODESIZE;
- if (unlikely(node_end > end_of_page)) {
- rc = bad_page(mp, "node[%u] (%zu) beyond page-end\n", i,
- node_end - end_of_page);
- continue;
- }
- size_t ksize = node_ks(node);
- char *key = node_key(node);
- if (unlikely(end_of_page < key + ksize)) {
- rc = bad_page(mp, "node[%u] key (%zu) beyond page-end\n", i,
- key + ksize - end_of_page);
- continue;
- }
- if ((IS_LEAF(mp) || i > 0) && (options & C_COPYING) == 0) {
- if (unlikely(ksize < mc->mc_dbx->md_klen_min ||
- ksize > mc->mc_dbx->md_klen_max))
- rc = bad_page(
- mp, "node[%u] key size (%zu) <> min/max key-length (%zu/%zu)\n",
- i, ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max);
- if ((options & C_SKIPORD) == 0) {
- here.iov_base = key;
- here.iov_len = ksize;
- if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0))
- rc = bad_page(mp, "node[%u] key wrong order (%s >= %s)\n", i,
- DKEY(&prev), DVAL(&here));
- prev = here;
- }
- }
- if (IS_BRANCH(mp)) {
- if ((options & C_UPDATING) == 0 && i == 0 && unlikely(ksize != 0))
- rc = bad_page(mp, "branch-node[%u] wrong 0-node key-length (%zu)\n",
- i, ksize);
- if ((options & C_RETIRING) == 0) {
- const pgno_t ref = node_pgno(node);
- if (unlikely(ref < MIN_PAGENO || ref >= mc->mc_txn->mt_next_pgno))
- rc = bad_page(mp, "branch-node[%u] wrong pgno (%u)\n", i, ref);
- }
- if (unlikely(node_flags(node)))
- rc = bad_page(mp, "branch-node[%u] wrong flags (%u)\n", i,
- node_flags(node));
- continue;
- }
+static __inline MDBX_cmp_func *get_default_datacmp(unsigned flags) {
+ return !(flags & MDBX_DUPSORT)
+ ? cmp_lenfast
+ : ((flags & MDBX_INTEGERDUP)
+ ? cmp_int_unaligned
+ : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical));
+}
+
+static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags,
+ MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
+ /* LY: so, accepting only three cases for the table's flags:
+ * 1) user_flags and both comparators are zero
+ * = assume that a by-default mode/flags is requested for reading;
+ * 2) user_flags exactly the same
+ * = assume that the target mode/flags are requested properly;
+ * 3) user_flags differs, but table is empty and MDBX_CREATE is provided
+ * = assume that a properly create request with custom flags;
+ */
+ if ((user_flags ^ txn->mt_dbs[dbi].md_flags) & DB_PERSISTENT_FLAGS) {
+ /* flags are differs, check other conditions */
+ if ((!user_flags && (!keycmp || keycmp == txn->mt_dbxs[dbi].md_cmp) &&
+ (!datacmp || datacmp == txn->mt_dbxs[dbi].md_dcmp)) ||
+ user_flags == MDBX_ACCEDE) {
+ /* no comparators were provided and flags are zero,
+ * seems that is case #1 above */
+ user_flags = txn->mt_dbs[dbi].md_flags;
+ } else if ((user_flags & MDBX_CREATE) && txn->mt_dbs[dbi].md_entries == 0) {
+ if (txn->mt_flags & MDBX_TXN_RDONLY)
+ return /* FIXME: return extended info */ MDBX_EACCESS;
+ /* make sure flags changes get committed */
+ txn->mt_dbs[dbi].md_flags = user_flags & DB_PERSISTENT_FLAGS;
+ txn->mt_flags |= MDBX_TXN_DIRTY;
+ } else {
+ return /* FIXME: return extended info */ MDBX_INCOMPATIBLE;
+ }
+ }
- switch (node_flags(node)) {
- default:
- rc = bad_page(mp, "invalid node[%u] flags (%u)\n", i, node_flags(node));
- break;
- case F_BIGDATA /* data on large-page */:
- case 0 /* usual */:
- case F_SUBDATA /* sub-db */:
- case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
- case F_DUPDATA /* short sub-page */:
- break;
- }
+ if (!keycmp)
+ keycmp = txn->mt_dbxs[dbi].md_cmp ? txn->mt_dbxs[dbi].md_cmp
+ : get_default_keycmp(user_flags);
+ if (txn->mt_dbxs[dbi].md_cmp != keycmp) {
+ if (txn->mt_dbxs[dbi].md_cmp)
+ return MDBX_EINVAL;
+ txn->mt_dbxs[dbi].md_cmp = keycmp;
+ }
- const size_t dsize = node_ds(node);
- const char *const data = node_data(node);
- if (node_flags(node) & F_BIGDATA) {
- if (unlikely(end_of_page < data + sizeof(pgno_t))) {
- rc = bad_page(
- mp, "node-%s(%u of %u, %zu bytes) beyond (%zu) page-end\n",
- "bigdata-pgno", i, nkeys, dsize, data + dsize - end_of_page);
- continue;
- }
- if ((options & C_COPYING) == 0) {
- if (unlikely(dsize <= mc->mc_dbx->md_vlen_min ||
- dsize > mc->mc_dbx->md_vlen_max))
- rc = bad_page(
- mp,
- "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n",
- dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max);
- }
- if ((options & C_RETIRING) == 0) {
- MDBX_page *lp;
- int err = mdbx_page_get(mc, node_largedata_pgno(node), &lp,
- pp_txnid4chk(mp, mc->mc_txn));
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- if (unlikely(!IS_OVERFLOW(lp))) {
- rc = bad_page(mp, "big-node refs to non-overflow page (%u)\n",
- lp->mp_pgno);
- continue;
- }
- if (unlikely(number_of_ovpages(env, dsize) > lp->mp_pages))
- rc =
- bad_page(mp, "big-node size (%zu) mismatch n-pages size (%u)\n",
- dsize, lp->mp_pages);
- }
- continue;
- }
+ if (!datacmp)
+ datacmp = txn->mt_dbxs[dbi].md_dcmp ? txn->mt_dbxs[dbi].md_dcmp
+ : get_default_datacmp(user_flags);
+ if (txn->mt_dbxs[dbi].md_dcmp != datacmp) {
+ if (txn->mt_dbxs[dbi].md_dcmp)
+ return MDBX_EINVAL;
+ txn->mt_dbxs[dbi].md_dcmp = datacmp;
+ }
- if (unlikely(end_of_page < data + dsize)) {
- rc =
- bad_page(mp, "node-%s(%u of %u, %zu bytes) beyond (%zu) page-end\n",
- "data", i, nkeys, dsize, data + dsize - end_of_page);
- continue;
- }
+ return MDBX_SUCCESS;
+}
- switch (node_flags(node)) {
- default:
- /* wrong, but already handled */
- continue;
- case 0 /* usual */:
- if ((options & C_COPYING) == 0) {
- if (unlikely(dsize < mc->mc_dbx->md_vlen_min ||
- dsize > mc->mc_dbx->md_vlen_max)) {
- rc = bad_page(
- mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n",
- dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max);
- continue;
- }
- }
- break;
- case F_SUBDATA /* sub-db */:
- if (unlikely(dsize != sizeof(MDBX_db))) {
- rc = bad_page(mp, "invalid sub-db record size (%zu)\n", dsize);
- continue;
- }
- break;
- case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
- if (unlikely(dsize != sizeof(MDBX_db))) {
- rc = bad_page(mp, "invalid nested-db record size (%zu)\n", dsize);
- continue;
- }
- break;
- case F_DUPDATA /* short sub-page */:
- if (unlikely(dsize <= PAGEHDRSZ)) {
- rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n",
- dsize);
- continue;
- } else {
- const MDBX_page *const sp = (MDBX_page *)data;
- const char *const end_of_subpage = data + dsize;
- const int nsubkeys = page_numkeys(sp);
- switch (sp->mp_flags & /* ignore legacy P_DIRTY flag */ ~0x10) {
- case P_LEAF | P_SUBP:
- case P_LEAF | P_LEAF2 | P_SUBP:
- break;
- default:
- rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n",
- sp->mp_flags);
- continue;
- }
+static int dbi_open(MDBX_txn *txn, const char *table_name, unsigned user_flags,
+ MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
+ MDBX_cmp_func *datacmp) {
+ int rc = MDBX_EINVAL;
+ if (unlikely(!dbi))
+ return rc;
- MDBX_val sub_here, sub_prev = {0, 0};
- for (int j = 0; j < nsubkeys; j++) {
- if (IS_LEAF2(sp)) {
- /* LEAF2 pages have no mp_ptrs[] or node headers */
- size_t sub_ksize = sp->mp_leaf2_ksize;
- char *sub_key = page_leaf2key(sp, j, sub_ksize);
- if (unlikely(end_of_subpage < sub_key + sub_ksize)) {
- rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n",
- sub_key + sub_ksize - end_of_subpage);
- continue;
- }
+ if (unlikely((user_flags & ~DB_USABLE_FLAGS) != 0)) {
+ early_bailout:
+ *dbi = 0;
+ return rc;
+ }
- if ((options & C_COPYING) == 0) {
- if (unlikely(sub_ksize != mc->mc_dbx->md_vlen_min)) {
- if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min ||
- sub_ksize > mc->mc_dbx->md_vlen_max)) {
- rc = bad_page(mp,
- "nested-leaf2-key size (%zu) <> min/max "
- "value-length (%zu/%zu)\n",
- sub_ksize, mc->mc_dbx->md_vlen_min,
- mc->mc_dbx->md_vlen_max);
- continue;
- }
- mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = sub_ksize;
- }
- if ((options & C_SKIPORD) == 0) {
- sub_here.iov_len = sub_ksize;
- sub_here.iov_base = sub_key;
- if (sub_prev.iov_base &&
- unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0))
- rc = bad_page(
- mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", j,
- DKEY(&sub_prev), DVAL(&sub_here));
- sub_prev = sub_here;
- }
- }
- } else {
- const MDBX_node *const sub_node = page_node(sp, j);
- const char *sub_node_end = (char *)sub_node + NODESIZE;
- if (unlikely(sub_node_end > end_of_subpage)) {
- rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n",
- end_of_subpage - sub_node_end);
- continue;
- }
- if (unlikely(node_flags(sub_node) != 0))
- rc = bad_page(mp, "nested-node invalid flags (%u)\n",
- node_flags(sub_node));
+ rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto early_bailout;
- size_t sub_ksize = node_ks(sub_node);
- char *sub_key = node_key(sub_node);
- size_t sub_dsize = node_ds(sub_node);
- /* char *sub_data = node_data(sub_node); */
+ switch (user_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT |
+ MDBX_REVERSEDUP | MDBX_ACCEDE)) {
+ case MDBX_ACCEDE:
+ if ((user_flags & MDBX_CREATE) == 0)
+ break;
+ __fallthrough /* fall through */;
+ default:
+ rc = MDBX_EINVAL;
+ goto early_bailout;
- if ((options & C_COPYING) == 0) {
- if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min ||
- sub_ksize > mc->mc_dbx->md_vlen_max))
- rc = bad_page(mp,
- "nested-node-key size (%zu) <> min/max "
- "value-length (%zu/%zu)\n",
- sub_ksize, mc->mc_dbx->md_vlen_min,
- mc->mc_dbx->md_vlen_max);
+ case MDBX_DUPSORT:
+ case MDBX_DUPSORT | MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
+ case 0:
+ break;
+ }
- if ((options & C_SKIPORD) == 0) {
- sub_here.iov_len = sub_ksize;
- sub_here.iov_base = sub_key;
- if (sub_prev.iov_base &&
- unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0))
- rc = bad_page(
- mp, "nested-node-key #%u wrong order (%s >= %s)\n", j,
- DKEY(&sub_prev), DVAL(&sub_here));
- sub_prev = sub_here;
- }
- }
- if (unlikely(sub_dsize != 0))
- rc = bad_page(mp, "nested-node non-empty data size (%zu)\n",
- sub_dsize);
- if (unlikely(end_of_subpage < sub_key + sub_ksize))
- rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n",
- sub_key + sub_ksize - end_of_subpage);
- }
- }
- }
- break;
- }
+ /* main table? */
+ if (!table_name) {
+ rc = mdbx_dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto early_bailout;
+ *dbi = MAIN_DBI;
+ return rc;
+ }
+
+ MDBX_env *env = txn->mt_env;
+ size_t len = strlen(table_name);
+ if (len > env->me_leaf_nodemax - NODESIZE - sizeof(MDBX_db))
+ return MDBX_EINVAL;
+
+ if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) {
+ txn->mt_dbxs[MAIN_DBI].md_cmp =
+ get_default_keycmp(txn->mt_dbs[MAIN_DBI].md_flags);
+ txn->mt_dbxs[MAIN_DBI].md_dcmp =
+ get_default_datacmp(txn->mt_dbs[MAIN_DBI].md_flags);
+ }
+
+ /* Is the DB already open? */
+ MDBX_dbi scan, slot;
+ for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) {
+ if (!txn->mt_dbxs[scan].md_name.iov_len) {
+ /* Remember this free slot */
+ slot = scan;
+ continue;
+ }
+ if (len == txn->mt_dbxs[scan].md_name.iov_len &&
+ !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) {
+ rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto early_bailout;
+ *dbi = scan;
+ return rc;
}
}
- return rc;
-}
-__cold static int mdbx_cursor_check(MDBX_cursor *mc, unsigned options) {
- mdbx_cassert(mc,
- mc->mc_txn->tw.dirtyroom + mc->mc_txn->tw.dirtylist->length ==
- (mc->mc_txn->mt_parent
- ? mc->mc_txn->mt_parent->tw.dirtyroom
- : mc->mc_txn->mt_env->me_options.dp_limit));
- mdbx_cassert(mc, mc->mc_top == mc->mc_snum - 1 || (options & C_UPDATING));
- if (unlikely(mc->mc_top != mc->mc_snum - 1) && (options & C_UPDATING) == 0)
- return MDBX_CURSOR_FULL;
- mdbx_cassert(mc, (options & C_UPDATING) ? mc->mc_snum <= mc->mc_db->md_depth
- : mc->mc_snum == mc->mc_db->md_depth);
- if (unlikely((options & C_UPDATING) ? mc->mc_snum > mc->mc_db->md_depth
- : mc->mc_snum != mc->mc_db->md_depth))
- return MDBX_CURSOR_FULL;
+ /* Fail, if no free slot and max hit */
+ if (unlikely(slot >= env->me_maxdbs)) {
+ rc = MDBX_DBS_FULL;
+ goto early_bailout;
+ }
- for (int n = 0; n < (int)mc->mc_snum; ++n) {
- MDBX_page *mp = mc->mc_pg[n];
- const unsigned nkeys = page_numkeys(mp);
- const bool expect_branch = (n < mc->mc_db->md_depth - 1) ? true : false;
- const bool expect_nested_leaf =
- (n + 1 == mc->mc_db->md_depth - 1) ? true : false;
- const bool branch = IS_BRANCH(mp) ? true : false;
- mdbx_cassert(mc, branch == expect_branch);
- if (unlikely(branch != expect_branch))
- return MDBX_CURSOR_FULL;
- if ((options & C_UPDATING) == 0) {
- mdbx_cassert(mc,
- nkeys > mc->mc_ki[n] || (!branch && nkeys == mc->mc_ki[n] &&
- (mc->mc_flags & C_EOF) != 0));
- if (unlikely(nkeys <= mc->mc_ki[n] &&
- !(!branch && nkeys == mc->mc_ki[n] &&
- (mc->mc_flags & C_EOF) != 0)))
- return MDBX_CURSOR_FULL;
- } else {
- mdbx_cassert(mc, nkeys + 1 >= mc->mc_ki[n]);
- if (unlikely(nkeys + 1 < mc->mc_ki[n]))
- return MDBX_CURSOR_FULL;
+ /* Cannot mix named table with some main-table flags */
+ if (unlikely(txn->mt_dbs[MAIN_DBI].md_flags &
+ (MDBX_DUPSORT | MDBX_INTEGERKEY))) {
+ rc = (user_flags & MDBX_CREATE) ? MDBX_INCOMPATIBLE : MDBX_NOTFOUND;
+ goto early_bailout;
+ }
+
+ /* Find the DB info */
+ MDBX_val key, data;
+ key.iov_len = len;
+ key.iov_base = (void *)table_name;
+ MDBX_cursor_couple couple;
+ rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto early_bailout;
+ rc = mdbx_cursor_set(&couple.outer, &key, &data, MDBX_SET).err;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE))
+ goto early_bailout;
+ } else {
+ /* make sure this is actually a table */
+ MDBX_node *node = page_node(couple.outer.mc_pg[couple.outer.mc_top],
+ couple.outer.mc_ki[couple.outer.mc_top]);
+ if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) {
+ rc = MDBX_INCOMPATIBLE;
+ goto early_bailout;
+ }
+ if (!MDBX_DISABLE_PAGECHECKS && unlikely(data.iov_len != sizeof(MDBX_db))) {
+ rc = MDBX_CORRUPTED;
+ goto early_bailout;
}
+ }
- int err = mdbx_page_check(mc, mp, options);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+ if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) {
+ rc = MDBX_EACCESS;
+ goto early_bailout;
+ }
- for (unsigned i = 0; i < nkeys; ++i) {
- if (branch) {
- MDBX_node *node = page_node(mp, i);
- mdbx_cassert(mc, node_flags(node) == 0);
- if (unlikely(node_flags(node) != 0))
- return MDBX_CURSOR_FULL;
- pgno_t pgno = node_pgno(node);
- MDBX_page *np;
- int rc = mdbx_page_get(mc, pgno, &np, pp_txnid4chk(mp, mc->mc_txn));
- mdbx_cassert(mc, rc == MDBX_SUCCESS);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- const bool nested_leaf = IS_LEAF(np) ? true : false;
- mdbx_cassert(mc, nested_leaf == expect_nested_leaf);
- if (unlikely(nested_leaf != expect_nested_leaf))
- return MDBX_CURSOR_FULL;
- err = mdbx_page_check(mc, np, options);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- }
+ /* Done here so we cannot fail after creating a new DB */
+ char *namedup = mdbx_strdup(table_name);
+ if (unlikely(!namedup)) {
+ rc = MDBX_ENOMEM;
+ goto early_bailout;
}
- return MDBX_SUCCESS;
-}
-/* Complete a delete operation started by mdbx_cursor_del(). */
-static int mdbx_cursor_del0(MDBX_cursor *mc) {
- int rc;
- MDBX_page *mp;
- indx_t ki;
- unsigned nkeys;
- MDBX_dbi dbi = mc->mc_dbi;
+ int err = mdbx_fastmutex_acquire(&env->me_dbi_lock);
+ if (unlikely(err != MDBX_SUCCESS)) {
+ rc = err;
+ mdbx_free(namedup);
+ goto early_bailout;
+ }
- mdbx_cassert(mc, cursor_is_tracked(mc));
- mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
- ki = mc->mc_ki[mc->mc_top];
- mp = mc->mc_pg[mc->mc_top];
- mdbx_node_del(mc, mc->mc_db->md_xsize);
- mc->mc_db->md_entries--;
+ /* Import handles from env */
+ dbi_import_locked(txn);
- /* Adjust other cursors pointing to mp */
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
- MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == mc || !(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
- continue;
- if (m3->mc_snum < mc->mc_snum)
+ /* Rescan after mutex acquisition & import handles */
+ for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) {
+ if (!txn->mt_dbxs[scan].md_name.iov_len) {
+ /* Remember this free slot */
+ slot = scan;
continue;
- if (m3->mc_pg[mc->mc_top] == mp) {
- if (m3->mc_ki[mc->mc_top] == ki) {
- m3->mc_flags |= C_DEL;
- if (mc->mc_db->md_flags & MDBX_DUPSORT) {
- /* Sub-cursor referred into dataset which is gone */
- m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF);
- }
- continue;
- } else if (m3->mc_ki[mc->mc_top] > ki) {
- m3->mc_ki[mc->mc_top]--;
- }
- if (XCURSOR_INITED(m3))
- XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+ }
+ if (len == txn->mt_dbxs[scan].md_name.iov_len &&
+ !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) {
+ rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto later_bailout;
+ *dbi = scan;
+ goto later_exit;
}
}
- rc = mdbx_rebalance(mc);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
-
- if (unlikely(!mc->mc_snum)) {
- /* DB is totally empty now, just bail out.
- * Other cursors adjustments were already done
- * by mdbx_rebalance and aren't needed here. */
- mdbx_cassert(mc, mc->mc_db->md_entries == 0 && mc->mc_db->md_depth == 0 &&
- mc->mc_db->md_root == P_INVALID);
- mc->mc_flags |= C_EOF;
- return MDBX_SUCCESS;
+ if (unlikely(slot >= env->me_maxdbs)) {
+ rc = MDBX_DBS_FULL;
+ goto later_bailout;
}
- ki = mc->mc_ki[mc->mc_top];
- mp = mc->mc_pg[mc->mc_top];
- mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
- nkeys = page_numkeys(mp);
- mdbx_cassert(mc, (mc->mc_db->md_entries > 0 && nkeys > 0) ||
- ((mc->mc_flags & C_SUB) && mc->mc_db->md_entries == 0 &&
- nkeys == 0));
-
- /* Adjust this and other cursors pointing to mp */
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
- MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
- continue;
- if (m3->mc_snum < mc->mc_snum)
- continue;
- if (m3->mc_pg[mc->mc_top] == mp) {
- /* if m3 points past last node in page, find next sibling */
- if (m3->mc_ki[mc->mc_top] >= nkeys) {
- rc = mdbx_cursor_sibling(m3, SIBLING_RIGHT);
- if (rc == MDBX_NOTFOUND) {
- m3->mc_flags |= C_EOF;
- rc = MDBX_SUCCESS;
- continue;
- }
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- if (m3->mc_ki[mc->mc_top] >= ki ||
- /* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) {
- if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
- MDBX_node *node =
- page_node(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
- /* If this node has dupdata, it may need to be reinited
- * because its data has moved.
- * If the xcursor was not inited it must be reinited.
- * Else if node points to a subDB, nothing is needed. */
- if (node_flags(node) & F_DUPDATA) {
- if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
- if (!(node_flags(node) & F_SUBDATA))
- m3->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node);
- } else {
- rc = mdbx_xcursor_init1(m3, node, m3->mc_pg[m3->mc_top]);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- rc = mdbx_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- }
- m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
- }
- m3->mc_flags |= C_DEL;
- }
+ unsigned dbiflags = DBI_FRESH | DBI_VALID | DBI_USRVALID;
+ MDBX_db db_dummy;
+ if (unlikely(rc)) {
+ /* MDBX_NOTFOUND and MDBX_CREATE: Create new DB */
+ mdbx_tassert(txn, rc == MDBX_NOTFOUND);
+ memset(&db_dummy, 0, sizeof(db_dummy));
+ db_dummy.md_root = P_INVALID;
+ db_dummy.md_mod_txnid = txn->mt_txnid;
+ db_dummy.md_flags = user_flags & DB_PERSISTENT_FLAGS;
+ data.iov_len = sizeof(db_dummy);
+ data.iov_base = &db_dummy;
+ WITH_CURSOR_TRACKING(couple.outer,
+ rc = mdbx_cursor_put(&couple.outer, &key, &data,
+ F_SUBDATA | MDBX_NOOVERWRITE));
+
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto later_bailout;
+
+ dbiflags |= DBI_DIRTY | DBI_CREAT;
+ txn->mt_flags |= MDBX_TXN_DIRTY;
+ mdbx_tassert(txn, (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY) != 0);
+ }
+
+ /* Got info, register DBI in this txn */
+ memset(txn->mt_dbxs + slot, 0, sizeof(MDBX_dbx));
+ memcpy(&txn->mt_dbs[slot], data.iov_base, sizeof(MDBX_db));
+ env->me_dbflags[slot] = 0;
+ rc = mdbx_dbi_bind(txn, slot, user_flags, keycmp, datacmp);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ mdbx_tassert(txn, (dbiflags & DBI_CREAT) == 0);
+ later_bailout:
+ *dbi = 0;
+ later_exit:
+ mdbx_free(namedup);
+ } else {
+ txn->mt_dbistate[slot] = (uint8_t)dbiflags;
+ txn->mt_dbxs[slot].md_name.iov_base = namedup;
+ txn->mt_dbxs[slot].md_name.iov_len = len;
+ txn->mt_dbiseqs[slot] = ++env->me_dbiseqs[slot];
+ if (!(dbiflags & DBI_CREAT))
+ env->me_dbflags[slot] = txn->mt_dbs[slot].md_flags | DB_VALID;
+ if (txn->mt_numdbs == slot) {
+ mdbx_compiler_barrier();
+ txn->mt_numdbs = env->me_numdbs = slot + 1;
+ txn->mt_cursors[slot] = NULL;
}
+ mdbx_assert(env, env->me_numdbs > slot);
+ *dbi = slot;
}
- mdbx_cassert(mc, rc == MDBX_SUCCESS);
- if (mdbx_audit_enabled())
- rc = mdbx_cursor_check(mc, 0);
+ mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
return rc;
+}
-bailout:
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- return rc;
+int mdbx_dbi_open(MDBX_txn *txn, const char *table_name,
+ MDBX_db_flags_t table_flags, MDBX_dbi *dbi) {
+ return dbi_open(txn, table_name, table_flags, dbi, nullptr, nullptr);
}
-int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
- const MDBX_val *data) {
- int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
+int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name,
+ MDBX_db_flags_t table_flags, MDBX_dbi *dbi,
+ MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
+ return dbi_open(txn, table_name, table_flags, dbi, keycmp, datacmp);
+}
+
+__cold int mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest,
+ size_t bytes) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!key))
+ if (unlikely(!dest))
return MDBX_EINVAL;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
return MDBX_BAD_DBI;
- if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
- return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
+ const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
+ if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
+ return MDBX_EINVAL;
- return mdbx_del0(txn, dbi, key, data, 0);
+ if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED))
+ return MDBX_BAD_TXN;
+
+ if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) {
+ rc = mdbx_fetch_sdb(txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ }
+
+ dest->ms_psize = txn->mt_env->me_psize;
+ stat_get(&txn->mt_dbs[dbi], dest, bytes);
+ return MDBX_SUCCESS;
}
-static int mdbx_del0(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
- const MDBX_val *data, unsigned flags) {
- MDBX_cursor_couple cx;
- MDBX_cursor_op op;
- MDBX_val rdata;
- int rc;
- DKBUF_DEBUG;
+static int mdbx_dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) {
+ mdbx_assert(env, dbi >= CORE_DBS);
+ if (unlikely(dbi >= env->me_numdbs))
+ return MDBX_BAD_DBI;
- mdbx_debug("====> delete db %u key [%s], data [%s]", dbi, DKEY_DEBUG(key),
- DVAL_DEBUG(data));
+ char *ptr = env->me_dbxs[dbi].md_name.iov_base;
+ /* If there was no name, this was already closed */
+ if (unlikely(!ptr))
+ return MDBX_BAD_DBI;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ env->me_dbflags[dbi] = 0;
+ env->me_dbiseqs[dbi]++;
+ env->me_dbxs[dbi].md_name.iov_len = 0;
+ mdbx_memory_fence(mo_AcquireRelease, true);
+ env->me_dbxs[dbi].md_name.iov_base = NULL;
+ mdbx_free(ptr);
+
+ if (env->me_numdbs == dbi + 1) {
+ unsigned i = env->me_numdbs;
+ do
+ --i;
+ while (i > CORE_DBS && !env->me_dbxs[i - 1].md_name.iov_base);
+ env->me_numdbs = i;
+ }
+
+ return MDBX_SUCCESS;
+}
+
+int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
+ int rc = check_env(env, true);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (data) {
- op = MDBX_GET_BOTH;
- rdata = *data;
- data = &rdata;
- } else {
- op = MDBX_SET;
- flags |= MDBX_ALLDUPS;
- }
- rc = mdbx_cursor_set(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op).err;
+ if (unlikely(dbi < CORE_DBS || dbi >= env->me_maxdbs))
+ return MDBX_BAD_DBI;
+
+ rc = mdbx_fastmutex_acquire(&env->me_dbi_lock);
if (likely(rc == MDBX_SUCCESS)) {
- /* let mdbx_page_split know about this cursor if needed:
- * delete will trigger a rebalance; if it needs to move
- * a node from one page to another, it will have to
- * update the parent's separator key(s). If the new sepkey
- * is larger than the current one, the parent page may
- * run out of space, triggering a split. We need this
- * cursor to be consistent until the end of the rebalance. */
- cx.outer.mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = &cx.outer;
- rc = mdbx_cursor_del(&cx.outer, flags);
- txn->mt_cursors[dbi] = cx.outer.mc_next;
+ rc = (dbi < env->me_maxdbs && (env->me_dbflags[dbi] & DB_VALID))
+ ? mdbx_dbi_close_locked(env, dbi)
+ : MDBX_BAD_DBI;
+ mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
}
return rc;
}
-/* Split a page and insert a new node.
- * Set MDBX_TXN_ERROR on failure.
- * [in,out] mc Cursor pointing to the page and desired insertion index.
- * The cursor will be updated to point to the actual page and index where
- * the node got inserted after the split.
- * [in] newkey The key for the newly inserted node.
- * [in] newdata The data for the newly inserted node.
- * [in] newpgno The page number, if the new node is a branch node.
- * [in] nflags The NODE_ADD_FLAGS for the new node.
- * Returns 0 on success, non-zero on failure. */
-static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
- MDBX_val *const newdata, pgno_t newpgno,
- unsigned nflags) {
- unsigned flags;
- int rc = MDBX_SUCCESS, foliage = 0;
- unsigned i, ptop;
- MDBX_env *const env = mc->mc_txn->mt_env;
- MDBX_val sepkey, rkey, xdata;
- MDBX_page *tmp_ki_copy = NULL;
- DKBUF;
-
- MDBX_page *const mp = mc->mc_pg[mc->mc_top];
- const unsigned newindx = mc->mc_ki[mc->mc_top];
- unsigned nkeys = page_numkeys(mp);
- if (mdbx_audit_enabled()) {
- rc = mdbx_cursor_check(mc, C_UPDATING);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
- STATIC_ASSERT(P_BRANCH == 1);
- const unsigned minkeys = (mp->mp_flags & P_BRANCH) + 1;
-
- mdbx_debug(">> splitting %s-page %" PRIaPGNO
- " and adding %zu+%zu [%s] at %i, nkeys %i",
- IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, newkey->iov_len,
- newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey),
- mc->mc_ki[mc->mc_top], nkeys);
- mdbx_cassert(mc, nkeys + 1 >= minkeys * 2);
-
- /* Create a new sibling page. */
- struct page_result npr = mdbx_page_new(mc, mp->mp_flags, 1);
- if (unlikely(npr.err != MDBX_SUCCESS))
- return npr.err;
- MDBX_page *const sister = npr.page;
- sister->mp_leaf2_ksize = mp->mp_leaf2_ksize;
- mdbx_debug("new sibling: page %" PRIaPGNO, sister->mp_pgno);
+int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags,
+ unsigned *state) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* Usually when splitting the root page, the cursor
- * height is 1. But when called from mdbx_update_key,
- * the cursor height may be greater because it walks
- * up the stack while finding the branch slot to update. */
- if (mc->mc_top < 1) {
- npr = mdbx_page_new(mc, P_BRANCH, 1);
- rc = npr.err;
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- MDBX_page *const pp = npr.page;
- /* shift current top to make room for new parent */
- mdbx_cassert(mc, mc->mc_snum < 2 && mc->mc_db->md_depth > 0);
-#if MDBX_DEBUG
- memset(mc->mc_pg + 3, 0, sizeof(mc->mc_pg) - sizeof(mc->mc_pg[0]) * 3);
- memset(mc->mc_ki + 3, -1, sizeof(mc->mc_ki) - sizeof(mc->mc_ki[0]) * 3);
-#endif
- mc->mc_pg[2] = mc->mc_pg[1];
- mc->mc_ki[2] = mc->mc_ki[1];
- mc->mc_pg[1] = mc->mc_pg[0];
- mc->mc_ki[1] = mc->mc_ki[0];
- mc->mc_pg[0] = pp;
- mc->mc_ki[0] = 0;
- mc->mc_db->md_root = pp->mp_pgno;
- mdbx_debug("root split! new root = %" PRIaPGNO, pp->mp_pgno);
- foliage = mc->mc_db->md_depth++;
+ if (unlikely(!flags || !state))
+ return MDBX_EINVAL;
- /* Add left (implicit) pointer. */
- rc = mdbx_node_add_branch(mc, 0, NULL, mp->mp_pgno);
- if (unlikely(rc != MDBX_SUCCESS)) {
- /* undo the pre-push */
- mc->mc_pg[0] = mc->mc_pg[1];
- mc->mc_ki[0] = mc->mc_ki[1];
- mc->mc_db->md_root = mp->mp_pgno;
- mc->mc_db->md_depth--;
- goto done;
- }
- mc->mc_snum++;
- mc->mc_top++;
- ptop = 0;
- if (mdbx_audit_enabled()) {
- rc = mdbx_cursor_check(mc, C_UPDATING);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- }
- } else {
- ptop = mc->mc_top - 1;
- mdbx_debug("parent branch page is %" PRIaPGNO, mc->mc_pg[ptop]->mp_pgno);
- }
+ if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
+ return MDBX_BAD_DBI;
- MDBX_cursor mn;
- cursor_copy(mc, &mn);
- mn.mc_pg[mn.mc_top] = sister;
- mn.mc_ki[mn.mc_top] = 0;
- mn.mc_ki[ptop] = mc->mc_ki[ptop] + 1;
+ *flags = txn->mt_dbs[dbi].md_flags & DB_PERSISTENT_FLAGS;
+ *state =
+ txn->mt_dbistate[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE);
- unsigned split_indx =
- (newindx < nkeys)
- ? /* split at the middle */ (nkeys + 1) / 2
- : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1;
+ return MDBX_SUCCESS;
+}
- mdbx_cassert(mc, !IS_BRANCH(mp) || newindx > 0);
- /* It is reasonable and possible to split the page at the begin */
- if (unlikely(newindx < minkeys)) {
- split_indx = minkeys;
- if (newindx == 0 && foliage == 0 && !(nflags & MDBX_SPLIT_REPLACE)) {
- split_indx = 0;
- /* Checking for ability of splitting by the left-side insertion
- * of a pure page with the new key */
- for (i = 0; i < mc->mc_top; ++i)
- if (mc->mc_ki[i]) {
- get_key(page_node(mc->mc_pg[i], mc->mc_ki[i]), &sepkey);
- if (mc->mc_dbx->md_cmp(newkey, &sepkey) >= 0)
- split_indx = minkeys;
- break;
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+int mdbx_dbi_flags(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) {
+ return __inline_mdbx_dbi_flags(txn, dbi, flags);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+
+static int mdbx_drop_tree(MDBX_cursor *mc, const bool may_have_subDBs) {
+ int rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
+ if (likely(rc == MDBX_SUCCESS)) {
+ MDBX_txn *txn = mc->mc_txn;
+
+ /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
+ * This also avoids any P_LEAF2 pages, which have no nodes.
+ * Also if the DB doesn't have sub-DBs and has no overflow
+ * pages, omit scanning leaves. */
+ if (!(may_have_subDBs | mc->mc_db->md_overflow_pages))
+ mdbx_cursor_pop(mc);
+
+ rc = mdbx_pnl_need(&txn->tw.retired_pages,
+ mc->mc_db->md_branch_pages + mc->mc_db->md_leaf_pages +
+ mc->mc_db->md_overflow_pages);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+
+ MDBX_cursor mx;
+ cursor_copy(mc, &mx);
+ while (mc->mc_snum > 0) {
+ MDBX_page *const mp = mc->mc_pg[mc->mc_top];
+ const unsigned nkeys = page_numkeys(mp);
+ if (IS_LEAF(mp)) {
+ mdbx_cassert(mc, mc->mc_snum == mc->mc_db->md_depth);
+ for (unsigned i = 0; i < nkeys; i++) {
+ MDBX_node *node = page_node(mp, i);
+ if (node_flags(node) & F_BIGDATA) {
+ rc = mdbx_page_retire_ex(mc, node_largedata_pgno(node), NULL, 0);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ if (!(may_have_subDBs | mc->mc_db->md_overflow_pages))
+ goto pop;
+ } else if (node_flags(node) & F_SUBDATA) {
+ if (unlikely((node_flags(node) & F_DUPDATA) == 0)) {
+ rc = /* disallowing implicit subDB deletion */ MDBX_INCOMPATIBLE;
+ goto bailout;
+ }
+ rc = mdbx_xcursor_init1(mc, node, mp);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ rc = mdbx_drop_tree(&mc->mc_xcursor->mx_cursor, false);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ }
+ } else {
+ mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth);
+ if (mdbx_audit_enabled())
+ mc->mc_flags |= C_RETIRING;
+ const int pagetype =
+ (IS_FROZEN(txn, mp) ? P_FROZEN : 0) +
+ ((mc->mc_snum + 1 == mc->mc_db->md_depth) ? P_LEAF : P_BRANCH);
+ for (unsigned i = 0; i < nkeys; i++) {
+ MDBX_node *node = page_node(mp, i);
+ mdbx_tassert(txn, (node_flags(node) &
+ (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
+ const pgno_t pgno = node_pgno(node);
+ rc = mdbx_page_retire_ex(mc, pgno, NULL, pagetype);
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ }
+ if (mdbx_audit_enabled())
+ mc->mc_flags -= C_RETIRING;
+ }
+ if (!mc->mc_top)
+ break;
+ mdbx_cassert(mc, nkeys > 0);
+ mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
+ rc = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (unlikely(rc != MDBX_NOTFOUND))
+ goto bailout;
+ /* no more siblings, go back to beginning
+ * of previous level. */
+ pop:
+ mdbx_cursor_pop(mc);
+ mc->mc_ki[0] = 0;
+ for (unsigned i = 1; i < mc->mc_snum; i++) {
+ mc->mc_ki[i] = 0;
+ mc->mc_pg[i] = mx.mc_pg[i];
}
- if (split_indx == 0) {
- /* Save the current first key which was omitted on the parent branch
- * page and should be updated if the new first entry will be added */
- if (IS_LEAF2(mp)) {
- sepkey.iov_len = mp->mp_leaf2_ksize;
- sepkey.iov_base = page_leaf2key(mp, 0, sepkey.iov_len);
- } else
- get_key(page_node(mp, 0), &sepkey);
- mdbx_cassert(mc, mc->mc_dbx->md_cmp(newkey, &sepkey) < 0);
- /* Avoiding rare complex cases of split the parent page */
- if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey))
- split_indx = minkeys;
}
}
+ rc = mdbx_page_retire(mc, mc->mc_pg[0]);
+ bailout:
+ if (unlikely(rc != MDBX_SUCCESS))
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ } else if (rc == MDBX_NOTFOUND) {
+ rc = MDBX_SUCCESS;
}
+ mc->mc_flags &= ~C_INITIALIZED;
+ return rc;
+}
- const bool pure_right = split_indx == nkeys;
- const bool pure_left = split_indx == 0;
- if (unlikely(pure_right)) {
- /* newindx == split_indx == nkeys */
- mdbx_trace("no-split, but add new pure page at the %s", "right/after");
- mdbx_cassert(mc, newindx == nkeys && split_indx == nkeys && minkeys == 1);
- sepkey = *newkey;
- } else if (unlikely(pure_left)) {
- /* newindx == split_indx == 0 */
- mdbx_trace("no-split, but add new pure page at the %s", "left/before");
- mdbx_cassert(mc, newindx == 0 && split_indx == 0 && minkeys == 1);
- mdbx_trace("old-first-key is %s", DKEY_DEBUG(&sepkey));
- } else {
- if (IS_LEAF2(sister)) {
- char *split, *ins;
- unsigned lsize, rsize, ksize;
- /* Move half of the keys to the right sibling */
- const int x = mc->mc_ki[mc->mc_top] - split_indx;
- ksize = mc->mc_db->md_xsize;
- split = page_leaf2key(mp, split_indx, ksize);
- rsize = (nkeys - split_indx) * ksize;
- lsize = (nkeys - split_indx) * sizeof(indx_t);
- mdbx_cassert(mc, mp->mp_lower >= lsize);
- mp->mp_lower -= (indx_t)lsize;
- mdbx_cassert(mc, sister->mp_lower + lsize <= UINT16_MAX);
- sister->mp_lower += (indx_t)lsize;
- mdbx_cassert(mc, mp->mp_upper + rsize - lsize <= UINT16_MAX);
- mp->mp_upper += (indx_t)(rsize - lsize);
- mdbx_cassert(mc, sister->mp_upper >= rsize - lsize);
- sister->mp_upper -= (indx_t)(rsize - lsize);
- sepkey.iov_len = ksize;
- sepkey.iov_base = (newindx != split_indx) ? split : newkey->iov_base;
- if (x < 0) {
- mdbx_cassert(mc, ksize >= sizeof(indx_t));
- ins = page_leaf2key(mp, mc->mc_ki[mc->mc_top], ksize);
- memcpy(sister->mp_ptrs, split, rsize);
- sepkey.iov_base = sister->mp_ptrs;
- memmove(ins + ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize);
- memcpy(ins, newkey->iov_base, ksize);
- mdbx_cassert(mc, UINT16_MAX - mp->mp_lower >= (int)sizeof(indx_t));
- mp->mp_lower += sizeof(indx_t);
- mdbx_cassert(mc, mp->mp_upper >= ksize - sizeof(indx_t));
- mp->mp_upper -= (indx_t)(ksize - sizeof(indx_t));
- } else {
- memcpy(sister->mp_ptrs, split, x * ksize);
- ins = page_leaf2key(sister, x, ksize);
- memcpy(ins, newkey->iov_base, ksize);
- memcpy(ins + ksize, split + x * ksize, rsize - x * ksize);
- mdbx_cassert(mc, UINT16_MAX - sister->mp_lower >= (int)sizeof(indx_t));
- sister->mp_lower += sizeof(indx_t);
- mdbx_cassert(mc, sister->mp_upper >= ksize - sizeof(indx_t));
- sister->mp_upper -= (indx_t)(ksize - sizeof(indx_t));
- mdbx_cassert(mc, x <= (int)UINT16_MAX);
- mc->mc_ki[mc->mc_top] = (indx_t)x;
- }
+int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) {
+ int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- if (mdbx_audit_enabled()) {
- rc = mdbx_cursor_check(mc, C_UPDATING);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- rc = mdbx_cursor_check(&mn, C_UPDATING);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
+ MDBX_cursor *mc;
+ rc = mdbx_cursor_open(txn, dbi, &mc);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ rc = mdbx_drop_tree(mc, dbi == MAIN_DBI ||
+ (mc->mc_db->md_flags & MDBX_DUPSORT) != 0);
+ /* Invalidate the dropped DB's cursors */
+ for (MDBX_cursor *m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next)
+ m2->mc_flags &= ~(C_INITIALIZED | C_EOF);
+ if (unlikely(rc))
+ goto bailout;
+
+ /* Can't delete the main DB */
+ if (del && dbi >= CORE_DBS) {
+ rc = mdbx_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA);
+ if (likely(rc == MDBX_SUCCESS)) {
+ mdbx_tassert(txn, txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY);
+ mdbx_tassert(txn, txn->mt_flags & MDBX_TXN_DIRTY);
+ txn->mt_dbistate[dbi] = DBI_STALE;
+ MDBX_env *env = txn->mt_env;
+ rc = mdbx_fastmutex_acquire(&env->me_dbi_lock);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ goto bailout;
}
+ mdbx_dbi_close_locked(env, dbi);
+ mdbx_ensure(env,
+ mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
} else {
- /* Maximum free space in an empty page */
- const unsigned max_space = page_space(env);
- const size_t new_size = IS_LEAF(mp) ? leaf_size(env, newkey, newdata)
- : branch_size(env, newkey);
+ txn->mt_flags |= MDBX_TXN_ERROR;
+ }
+ } else {
+ /* reset the DB record, mark it dirty */
+ txn->mt_dbistate[dbi] |= DBI_DIRTY;
+ txn->mt_dbs[dbi].md_depth = 0;
+ txn->mt_dbs[dbi].md_branch_pages = 0;
+ txn->mt_dbs[dbi].md_leaf_pages = 0;
+ txn->mt_dbs[dbi].md_overflow_pages = 0;
+ txn->mt_dbs[dbi].md_entries = 0;
+ txn->mt_dbs[dbi].md_root = P_INVALID;
+ txn->mt_dbs[dbi].md_seq = 0;
+ /* txn->mt_dbs[dbi].md_mod_txnid = txn->mt_txnid; */
+ txn->mt_flags |= MDBX_TXN_DIRTY;
+ }
+
+bailout:
+ mdbx_cursor_close(mc);
+ return rc;
+}
- /* grab a page to hold a temporary copy */
- tmp_ki_copy = mdbx_page_malloc(mc->mc_txn, 1);
- if (unlikely(tmp_ki_copy == NULL)) {
- rc = MDBX_ENOMEM;
- goto done;
- }
+int mdbx_set_compare(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* prepare to insert */
- for (unsigned j = i = 0; i < nkeys; ++i, ++j) {
- tmp_ki_copy->mp_ptrs[j] = 0;
- j += (i == newindx);
- tmp_ki_copy->mp_ptrs[j] = mp->mp_ptrs[i];
- }
- tmp_ki_copy->mp_pgno = mp->mp_pgno;
- tmp_ki_copy->mp_flags = mp->mp_flags;
- tmp_ki_copy->mp_txnid = INVALID_TXNID;
- tmp_ki_copy->mp_lower = 0;
- tmp_ki_copy->mp_upper = (indx_t)max_space;
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
- /* When items are relatively large the split point needs
- * to be checked, because being off-by-one will make the
- * difference between success or failure in mdbx_node_add.
- *
- * It's also relevant if a page happens to be laid out
- * such that one half of its nodes are all "small" and
- * the other half of its nodes are "large". If the new
- * item is also "large" and falls on the half with
- * "large" nodes, it also may not fit.
- *
- * As a final tweak, if the new item goes on the last
- * spot on the page (and thus, onto the new page), bias
- * the split so the new page is emptier than the old page.
- * This yields better packing during sequential inserts. */
+ txn->mt_dbxs[dbi].md_cmp = cmp;
+ return MDBX_SUCCESS;
+}
- if (nkeys < 32 || new_size > max_space / 16) {
- /* Find split point */
- int dir;
- if (newindx <= split_indx) {
- i = 0;
- dir = 1;
- } else {
- i = nkeys;
- dir = -1;
- }
- size_t before = 0, after = new_size + page_used(env, mp);
- int best = split_indx;
- int best_offset = nkeys + 1;
+int mdbx_set_dupsort(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- mdbx_trace("seek separator from %u, step %i, default %u, new-idx %u, "
- "new-size %zu",
- i, dir, split_indx, newindx, new_size);
- do {
- mdbx_cassert(mc, i <= nkeys);
- size_t size = new_size;
- if (i != newindx) {
- MDBX_node *node =
- (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ);
- size = NODESIZE + node_ks(node) + sizeof(indx_t);
- if (IS_LEAF(mp))
- size += F_ISSET(node_flags(node), F_BIGDATA) ? sizeof(pgno_t)
- : node_ds(node);
- size = EVEN(size);
- }
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
- before += size;
- after -= size;
- mdbx_trace("step %u, size %zu, before %zu, after %zu, max %u", i,
- size, before, after, max_space);
+ txn->mt_dbxs[dbi].md_dcmp = cmp;
+ return MDBX_SUCCESS;
+}
- if (before <= max_space && after <= max_space) {
- int offset = branchless_abs(split_indx - i);
- if (offset >= best_offset)
- break;
- best_offset = offset;
- best = i;
- }
- i += dir;
- } while (i < nkeys);
+__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
+ void *ctx) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- split_indx = best + (dir > 0);
- split_indx = (split_indx <= nkeys - minkeys + 1) ? split_indx
- : nkeys - minkeys + 1;
- split_indx = (split_indx >= minkeys) ? split_indx : minkeys;
- mdbx_trace("chosen %u", split_indx);
- }
+ if (unlikely(!func))
+ return MDBX_EINVAL;
- sepkey.iov_len = newkey->iov_len;
- sepkey.iov_base = newkey->iov_base;
- if (split_indx != newindx) {
- MDBX_node *node =
- (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[split_indx] +
- PAGEHDRSZ);
- sepkey.iov_len = node_ks(node);
- sepkey.iov_base = node_key(node);
+ rc = MDBX_RESULT_TRUE;
+ int serial = 0;
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (likely(lck)) {
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; i++) {
+ const MDBX_reader *r = lck->mti_readers + i;
+ retry_reader:;
+ const uint32_t pid = atomic_load32(&r->mr_pid, mo_AcquireRelease);
+ if (!pid)
+ continue;
+ txnid_t txnid = safe64_read(&r->mr_txnid);
+ const uint64_t tid = atomic_load64(&r->mr_tid, mo_Relaxed);
+ const pgno_t pages_used =
+ atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed);
+ const uint64_t reader_pages_retired =
+ atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed);
+ if (unlikely(
+ txnid != safe64_read(&r->mr_txnid) ||
+ pid != atomic_load32(&r->mr_pid, mo_AcquireRelease) ||
+ tid != atomic_load64(&r->mr_tid, mo_Relaxed) ||
+ pages_used !=
+ atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed) ||
+ reader_pages_retired !=
+ atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed)))
+ goto retry_reader;
+
+ mdbx_assert(env, txnid > 0);
+ if (txnid >= SAFE64_INVALID_THRESHOLD)
+ txnid = 0;
+
+ size_t bytes_used = 0;
+ size_t bytes_retained = 0;
+ uint64_t lag = 0;
+ if (txnid) {
+ retry_header:;
+ volatile const MDBX_meta *const recent_meta = meta_prefer_last(env);
+ const uint64_t head_pages_retired =
+ unaligned_peek_u64_volatile(4, recent_meta->mm_pages_retired);
+ const txnid_t head_txnid = meta_txnid(env, recent_meta);
+ mdbx_compiler_barrier();
+ if (unlikely(recent_meta != meta_prefer_last(env) ||
+ head_pages_retired !=
+ unaligned_peek_u64_volatile(
+ 4, recent_meta->mm_pages_retired)) ||
+ head_txnid != meta_txnid(env, recent_meta))
+ goto retry_header;
+
+ lag = (head_txnid - txnid) / xMDBX_TXNID_STEP;
+ bytes_used = pgno2bytes(env, pages_used);
+ bytes_retained = (head_pages_retired > reader_pages_retired)
+ ? pgno2bytes(env, (pgno_t)(head_pages_retired -
+ reader_pages_retired))
+ : 0;
}
+ rc = func(ctx, ++serial, i, pid, (mdbx_tid_t)tid, txnid, lag, bytes_used,
+ bytes_retained);
+ if (unlikely(rc != MDBX_SUCCESS))
+ break;
}
}
- mdbx_debug("separator is %d [%s]", split_indx, DKEY_DEBUG(&sepkey));
- bool did_split_parent = false;
- /* Copy separator key to the parent. */
- if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey)) {
- mdbx_trace("need split parent branch-page for key %s", DKEY_DEBUG(&sepkey));
- mdbx_cassert(mc, page_numkeys(mn.mc_pg[ptop]) > 2);
- mdbx_cassert(mc, !pure_left);
- const int snum = mc->mc_snum;
- const int depth = mc->mc_db->md_depth;
- mn.mc_snum--;
- mn.mc_top--;
- did_split_parent = true;
- /* We want other splits to find mn when doing fixups */
- WITH_CURSOR_TRACKING(
- mn, rc = mdbx_page_split(&mn, &sepkey, NULL, sister->mp_pgno, 0));
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- mdbx_cassert(mc, (int)mc->mc_snum - snum == mc->mc_db->md_depth - depth);
- if (mdbx_audit_enabled()) {
- rc = mdbx_cursor_check(mc, C_UPDATING);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- }
+ return rc;
+}
- /* root split? */
- ptop += mc->mc_snum - snum;
+/* Insert pid into list if not already present.
+ * return -1 if already present. */
+__cold static bool mdbx_pid_insert(uint32_t *ids, uint32_t pid) {
+ /* binary search of pid in list */
+ unsigned base = 0;
+ unsigned cursor = 1;
+ int val = 0;
+ unsigned n = ids[0];
- /* Right page might now have changed parent.
- * Check if left page also changed parent. */
- if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
- mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
- for (i = 0; i < ptop; i++) {
- mc->mc_pg[i] = mn.mc_pg[i];
- mc->mc_ki[i] = mn.mc_ki[i];
- }
- mc->mc_pg[ptop] = mn.mc_pg[ptop];
- if (mn.mc_ki[ptop]) {
- mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1;
- } else {
- /* find right page's left sibling */
- mc->mc_ki[ptop] = mn.mc_ki[ptop];
- rc = mdbx_cursor_sibling(mc, SIBLING_LEFT);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_NOTFOUND) /* improper mdbx_cursor_sibling() result */ {
- mdbx_error("unexpected %i error going left sibling", rc);
- rc = MDBX_PROBLEM;
- }
- goto done;
- }
- }
- }
- } else if (unlikely(pure_left)) {
- MDBX_page *ptop_page = mc->mc_pg[ptop];
- mdbx_debug("adding to parent page %u node[%u] left-leaf page #%u key %s",
- ptop_page->mp_pgno, mc->mc_ki[ptop], sister->mp_pgno,
- DKEY(mc->mc_ki[ptop] ? newkey : NULL));
- mc->mc_top--;
- rc = mdbx_node_add_branch(mc, mc->mc_ki[ptop],
- mc->mc_ki[ptop] ? newkey : NULL, sister->mp_pgno);
- mdbx_cassert(mc, mp == mc->mc_pg[ptop + 1] &&
- newindx == mc->mc_ki[ptop + 1] && ptop == mc->mc_top);
+ while (n > 0) {
+ unsigned pivot = n >> 1;
+ cursor = base + pivot + 1;
+ val = pid - ids[cursor];
- if (likely(rc == MDBX_SUCCESS) && mc->mc_ki[ptop] == 0) {
- mdbx_debug("update prev-first key on parent %s", DKEY(&sepkey));
- MDBX_node *node = page_node(mc->mc_pg[ptop], 1);
- mdbx_cassert(mc, node_ks(node) == 0 && node_pgno(node) == mp->mp_pgno);
- mdbx_cassert(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 0);
- mc->mc_ki[ptop] = 1;
- rc = mdbx_update_key(mc, &sepkey);
- mdbx_cassert(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 1);
- mdbx_cassert(mc,
- mp == mc->mc_pg[ptop + 1] && newindx == mc->mc_ki[ptop + 1]);
- mc->mc_ki[ptop] = 0;
+ if (val < 0) {
+ n = pivot;
+ } else if (val > 0) {
+ base = cursor;
+ n -= pivot + 1;
+ } else {
+ /* found, so it's a duplicate */
+ return false;
}
+ }
- mc->mc_top++;
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
+ if (val > 0)
+ ++cursor;
- MDBX_node *node = page_node(mc->mc_pg[ptop], mc->mc_ki[ptop] + 1);
- mdbx_cassert(mc, node_pgno(node) == mp->mp_pgno &&
- mc->mc_pg[ptop] == ptop_page);
- } else {
- mn.mc_top--;
- mdbx_trace("add-to-parent the right-entry[%u] for new sibling-page",
- mn.mc_ki[ptop]);
- rc = mdbx_node_add_branch(&mn, mn.mc_ki[ptop], &sepkey, sister->mp_pgno);
- mn.mc_top++;
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- }
+ ids[0]++;
+ for (n = ids[0]; n > cursor; n--)
+ ids[n] = ids[n - 1];
+ ids[n] = pid;
+ return true;
+}
- if (unlikely(pure_left | pure_right)) {
- mc->mc_pg[mc->mc_top] = sister;
- mc->mc_ki[mc->mc_top] = 0;
- switch (PAGETYPE(sister)) {
- case P_LEAF: {
- mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID);
- rc = mdbx_node_add_leaf(mc, 0, newkey, newdata, nflags);
- } break;
- case P_LEAF | P_LEAF2: {
- mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
- mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID);
- rc = mdbx_node_add_leaf2(mc, 0, newkey);
- } break;
- default:
- rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE(sister));
- }
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
+__cold int mdbx_reader_check(MDBX_env *env, int *dead) {
+ if (dead)
+ *dead = 0;
+ return mdbx_cleanup_dead_readers(env, false, dead);
+}
- if (pure_right) {
- for (i = 0; i < mc->mc_top; i++)
- mc->mc_ki[i] = mn.mc_ki[i];
- } else if (mc->mc_ki[mc->mc_top - 1] == 0) {
- for (i = 2; i <= mc->mc_top; ++i)
- if (mc->mc_ki[mc->mc_top - i]) {
- get_key(
- page_node(mc->mc_pg[mc->mc_top - i], mc->mc_ki[mc->mc_top - i]),
- &sepkey);
- if (mc->mc_dbx->md_cmp(newkey, &sepkey) < 0) {
- mc->mc_top -= i;
- mdbx_debug("update new-first on parent [%i] page %u key %s",
- mc->mc_ki[mc->mc_top], mc->mc_pg[mc->mc_top]->mp_pgno,
- DKEY(newkey));
- rc = mdbx_update_key(mc, newkey);
- mc->mc_top += i;
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- }
- break;
- }
- }
- } else if (!IS_LEAF2(mp)) {
- /* Move nodes */
- mc->mc_pg[mc->mc_top] = sister;
- i = split_indx;
- unsigned n = 0;
- pgno_t pgno = 0;
- do {
- mdbx_trace("i %u, nkeys %u => n %u, rp #%u", i, nkeys, n,
- sister->mp_pgno);
- MDBX_val *rdata = NULL;
- if (i == newindx) {
- rkey.iov_base = newkey->iov_base;
- rkey.iov_len = newkey->iov_len;
- if (IS_LEAF(mp))
- rdata = newdata;
- else
- pgno = newpgno;
- flags = nflags;
- /* Update index for the new key. */
- mc->mc_ki[mc->mc_top] = (indx_t)n;
- } else {
- MDBX_node *node =
- (MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ);
- rkey.iov_base = node_key(node);
- rkey.iov_len = node_ks(node);
- if (IS_LEAF(mp)) {
- xdata.iov_base = node_data(node);
- xdata.iov_len = node_ds(node);
- rdata = &xdata;
- } else
- pgno = node_pgno(node);
- flags = node_flags(node);
- }
+/* Return:
+ * MDBX_RESULT_TRUE - done and mutex recovered
+ * MDBX_SUCCESS - done
+ * Otherwise errcode. */
+__cold MDBX_INTERNAL_FUNC int
+mdbx_cleanup_dead_readers(MDBX_env *env, int rdt_locked, int *dead) {
+ int rc = check_env(env, true);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- switch (PAGETYPE(sister)) {
- case P_BRANCH: {
- mdbx_cassert(mc, 0 == (uint16_t)flags);
- /* First branch index doesn't need key data. */
- rc = mdbx_node_add_branch(mc, n, n ? &rkey : NULL, pgno);
- } break;
- case P_LEAF: {
- mdbx_cassert(mc, pgno == 0);
- mdbx_cassert(mc, rdata != NULL);
- rc = mdbx_node_add_leaf(mc, n, &rkey, rdata, flags);
- } break;
- /* case P_LEAF | P_LEAF2: {
- mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
- mdbx_cassert(mc, gno == 0);
- rc = mdbx_node_add_leaf2(mc, n, &rkey);
- } break; */
- default:
- rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE(sister));
- }
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
+ mdbx_assert(env, rdt_locked >= 0);
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (unlikely(lck == NULL)) {
+ /* exclusive mode */
+ if (dead)
+ *dead = 0;
+ return MDBX_SUCCESS;
+ }
- ++n;
- if (++i > nkeys) {
- i = 0;
- n = 0;
- mc->mc_pg[mc->mc_top] = tmp_ki_copy;
- mdbx_trace("switch to mp #%u", tmp_ki_copy->mp_pgno);
- }
- } while (i != split_indx);
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ uint32_t pidsbuf_onstask[142];
+ uint32_t *const pids =
+ (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
+ ? pidsbuf_onstask
+ : mdbx_malloc((snap_nreaders + 1) * sizeof(uint32_t));
+ if (unlikely(!pids))
+ return MDBX_ENOMEM;
- mdbx_trace("i %u, nkeys %u, n %u, pgno #%u", i, nkeys, n,
- mc->mc_pg[mc->mc_top]->mp_pgno);
+ pids[0] = 0;
+ int count = 0;
+ for (unsigned i = 0; i < snap_nreaders; i++) {
+ const uint32_t pid =
+ atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease);
+ if (pid == 0)
+ continue /* skip empty */;
+ if (pid == env->me_pid)
+ continue /* skip self */;
+ if (!mdbx_pid_insert(pids, pid))
+ continue /* such pid already processed */;
- nkeys = page_numkeys(tmp_ki_copy);
- for (i = 0; i < nkeys; i++)
- mp->mp_ptrs[i] = tmp_ki_copy->mp_ptrs[i];
- mp->mp_lower = tmp_ki_copy->mp_lower;
- mp->mp_upper = tmp_ki_copy->mp_upper;
- memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1),
- env->me_psize - tmp_ki_copy->mp_upper - PAGEHDRSZ);
+ int err = mdbx_rpid_check(env, pid);
+ if (err == MDBX_RESULT_TRUE)
+ continue /* reader is live */;
- /* reset back to original page */
- if (newindx < split_indx) {
- mc->mc_pg[mc->mc_top] = mp;
- } else {
- mc->mc_pg[mc->mc_top] = sister;
- mc->mc_ki[ptop]++;
- /* Make sure mc_ki is still valid. */
- if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
- mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
- for (i = 0; i <= ptop; i++) {
- mc->mc_pg[i] = mn.mc_pg[i];
- mc->mc_ki[i] = mn.mc_ki[i];
- }
- }
+ if (err != MDBX_SUCCESS) {
+ rc = err;
+ break /* mdbx_rpid_check() failed */;
}
- } else if (newindx >= split_indx) {
- mc->mc_pg[mc->mc_top] = sister;
- mc->mc_ki[ptop]++;
- /* Make sure mc_ki is still valid. */
- if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
- mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) {
- for (i = 0; i <= ptop; i++) {
- mc->mc_pg[i] = mn.mc_pg[i];
- mc->mc_ki[i] = mn.mc_ki[i];
+
+ /* stale reader found */
+ if (!rdt_locked) {
+ err = mdbx_rdt_lock(env);
+ if (MDBX_IS_ERROR(err)) {
+ rc = err;
+ break;
}
- }
- }
- /* Adjust other cursors pointing to mp and/or to parent page */
- nkeys = page_numkeys(mp);
- for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2;
- m2 = m2->mc_next) {
- MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
- if (m3 == mc)
- continue;
- if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
- continue;
- if (foliage) {
- /* sub cursors may be on different DB */
- if (m3->mc_pg[0] != mp)
- continue;
- /* root split */
- for (int k = foliage; k >= 0; k--) {
- m3->mc_ki[k + 1] = m3->mc_ki[k];
- m3->mc_pg[k + 1] = m3->mc_pg[k];
+ rdt_locked = -1;
+ if (err == MDBX_RESULT_TRUE) {
+ /* mutex recovered, the mdbx_ipclock_failed() checked all readers */
+ rc = MDBX_RESULT_TRUE;
+ break;
}
- m3->mc_ki[0] = (m3->mc_ki[0] >= nkeys) ? 1 : 0;
- m3->mc_pg[0] = mc->mc_pg[0];
- m3->mc_snum++;
- m3->mc_top++;
+
+ /* a other process may have clean and reused slot, recheck */
+ if (lck->mti_readers[i].mr_pid.weak != pid)
+ continue;
+
+ err = mdbx_rpid_check(env, pid);
+ if (MDBX_IS_ERROR(err)) {
+ rc = err;
+ break;
+ }
+
+ if (err != MDBX_SUCCESS)
+ continue /* the race with other process, slot reused */;
}
- if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp && !pure_left) {
- if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDBX_SPLIT_REPLACE))
- m3->mc_ki[mc->mc_top]++;
- if (m3->mc_ki[mc->mc_top] >= nkeys) {
- m3->mc_pg[mc->mc_top] = sister;
- mdbx_cassert(mc, m3->mc_ki[mc->mc_top] >= nkeys);
- m3->mc_ki[mc->mc_top] -= (indx_t)nkeys;
- for (i = 0; i < mc->mc_top; i++) {
- m3->mc_ki[i] = mn.mc_ki[i];
- m3->mc_pg[i] = mn.mc_pg[i];
- }
+ /* clean it */
+ for (unsigned j = i; j < snap_nreaders; j++) {
+ if (lck->mti_readers[j].mr_pid.weak == pid) {
+ mdbx_debug("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN,
+ (size_t)pid, lck->mti_readers[j].mr_txnid.weak);
+ atomic_store32(&lck->mti_readers[j].mr_pid, 0, mo_Relaxed);
+ atomic_store32(&lck->mti_readers_refresh_flag, true, mo_AcquireRelease);
+ count++;
}
- } else if (!did_split_parent && m3->mc_top >= ptop &&
- m3->mc_pg[ptop] == mc->mc_pg[ptop] &&
- m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
- m3->mc_ki[ptop]++; /* also for the `pure-left` case */
}
- if (XCURSOR_INITED(m3) && IS_LEAF(mp))
- XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
}
- mdbx_trace("mp #%u left: %d, sister #%u left: %d", mp->mp_pgno, page_room(mp),
- sister->mp_pgno, page_room(sister));
-done:
- if (tmp_ki_copy)
- mdbx_dpage_free(env, tmp_ki_copy, 1);
+ if (likely(!MDBX_IS_ERROR(rc)))
+ atomic_store64(&lck->mti_reader_check_timestamp, mdbx_osal_monotime(),
+ mo_Relaxed);
- if (unlikely(rc != MDBX_SUCCESS))
- mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
- else {
- if (mdbx_audit_enabled())
- rc = mdbx_cursor_check(mc, C_UPDATING);
- if (unlikely(nflags & MDBX_RESERVE)) {
- MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
- if (!(node_flags(node) & F_BIGDATA))
- newdata->iov_base = node_data(node);
- }
-#if MDBX_ENABLE_PGOP_STAT
- env->me_lck->mti_pgop_stat.split.weak += 1;
-#endif /* MDBX_ENABLE_PGOP_STAT */
- }
+ if (rdt_locked < 0)
+ mdbx_rdt_unlock(env);
- mdbx_debug("<< mp #%u, rc %d", mp->mp_pgno, rc);
+ if (pids != pidsbuf_onstask)
+ mdbx_free(pids);
+
+ if (dead)
+ *dead = count;
return rc;
}
-int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
- unsigned flags) {
- int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
-
- if (unlikely(!key || !data))
- return MDBX_EINVAL;
-
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
-
- if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
- MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
- MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE)))
- return MDBX_EINVAL;
-
- if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
- return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
+__cold int mdbx_setup_debug(int loglevel, int flags, MDBX_debug_func *logger) {
+ const int rc = mdbx_runtime_flags | (mdbx_loglevel << 16);
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- cx.outer.mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = &cx.outer;
+ if (loglevel != MDBX_LOG_DONTCHANGE)
+ mdbx_loglevel = (uint8_t)loglevel;
- /* LY: support for update (explicit overwrite) */
- if (flags & MDBX_CURRENT) {
- rc = mdbx_cursor_get(&cx.outer, (MDBX_val *)key, NULL, MDBX_SET);
- if (likely(rc == MDBX_SUCCESS) &&
- (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) &&
- (flags & MDBX_ALLDUPS) == 0) {
- /* LY: allows update (explicit overwrite) only for unique keys */
- MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
- cx.outer.mc_ki[cx.outer.mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) &&
- cx.outer.mc_xcursor->mx_db.md_entries > 1);
- rc = MDBX_EMULTIVAL;
- }
- }
+ if (flags != MDBX_DBG_DONTCHANGE) {
+ flags &=
+#if MDBX_DEBUG
+ MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER |
+#endif
+ MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP |
+ MDBX_DBG_DONT_UPGRADE;
+ mdbx_runtime_flags = (uint8_t)flags;
}
- if (likely(rc == MDBX_SUCCESS))
- rc = mdbx_cursor_put(&cx.outer, key, data, flags);
- txn->mt_cursors[dbi] = cx.outer.mc_next;
-
+ if (logger != MDBX_LOGGER_DONTCHANGE)
+ mdbx_debug_logger = logger;
return rc;
}
-/**** COPYING *****************************************************************/
-
-/* State needed for a double-buffering compacting copy. */
-typedef struct mdbx_copy {
- MDBX_env *mc_env;
- MDBX_txn *mc_txn;
- mdbx_condpair_t mc_condpair;
- uint8_t *mc_wbuf[2];
- uint8_t *mc_over[2];
- size_t mc_wlen[2];
- size_t mc_olen[2];
- mdbx_filehandle_t mc_fd;
- /* Error code. Never cleared if set. Both threads can set nonzero
- * to fail the copy. Not mutex-protected, MDBX expects atomic int. */
- volatile int mc_error;
- pgno_t mc_next_pgno;
- volatile unsigned mc_head;
- volatile unsigned mc_tail;
-} mdbx_copy;
+__cold static txnid_t mdbx_kick_longlived_readers(MDBX_env *env,
+ const txnid_t laggard) {
+ mdbx_debug("DB size maxed out by reading #%" PRIaTXN, laggard);
-/* Dedicated writer thread for compacting copy. */
-__cold static THREAD_RESULT THREAD_CALL mdbx_env_copythr(void *arg) {
- mdbx_copy *my = arg;
+ int retry;
+ for (retry = 0; retry < INT_MAX; ++retry) {
+ txnid_t oldest = mdbx_recent_steady_txnid(env);
+ mdbx_assert(env, oldest < env->me_txn0->mt_txnid);
+ mdbx_assert(env, oldest >= laggard);
+ mdbx_assert(env, oldest >= env->me_lck->mti_oldest_reader.weak);
+ MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
+ if (oldest == laggard || unlikely(!lck /* without-LCK mode */))
+ return oldest;
-#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
- sigset_t sigset;
- sigemptyset(&sigset);
- sigaddset(&sigset, SIGPIPE);
- my->mc_error = pthread_sigmask(SIG_BLOCK, &sigset, NULL);
-#endif /* EPIPE */
+ if (MDBX_IS_ERROR(mdbx_cleanup_dead_readers(env, false, NULL)))
+ break;
- mdbx_condpair_lock(&my->mc_condpair);
- while (!my->mc_error) {
- while (my->mc_tail == my->mc_head && !my->mc_error) {
- int err = mdbx_condpair_wait(&my->mc_condpair, true);
- if (err != MDBX_SUCCESS) {
- my->mc_error = err;
- goto bailout;
+ MDBX_reader *asleep = nullptr;
+ uint64_t oldest_retired = UINT64_MAX;
+ const unsigned snap_nreaders =
+ atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
+ for (unsigned i = 0; i < snap_nreaders; ++i) {
+ retry:
+ if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
+ /* mdbx_jitter4testing(true); */
+ const uint64_t snap_retired = atomic_load64(
+ &lck->mti_readers[i].mr_snapshot_pages_retired, mo_Relaxed);
+ const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
+ if (unlikely(snap_retired !=
+ atomic_load64(
+ &lck->mti_readers[i].mr_snapshot_pages_retired,
+ mo_AcquireRelease) ||
+ snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
+ goto retry;
+ if (oldest > snap_txnid &&
+ laggard <= /* ignore pending updates */ snap_txnid) {
+ oldest = snap_txnid;
+ oldest_retired = snap_retired;
+ asleep = &lck->mti_readers[i];
+ }
}
}
- const unsigned toggle = my->mc_tail & 1;
- size_t wsize = my->mc_wlen[toggle];
- if (wsize == 0) {
- my->mc_tail += 1;
- break /* EOF */;
- }
- my->mc_wlen[toggle] = 0;
- uint8_t *ptr = my->mc_wbuf[toggle];
- again:
- if (!my->mc_error) {
- int err = mdbx_write(my->mc_fd, ptr, wsize);
- if (err != MDBX_SUCCESS) {
-#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
- if (err == EPIPE) {
- /* Collect the pending SIGPIPE,
- * otherwise at least OS X gives it to the process on thread-exit. */
- int unused;
- sigwait(&sigset, &unused);
- }
-#endif /* EPIPE */
- my->mc_error = err;
- goto bailout;
+
+ if (laggard < oldest || !asleep) {
+ if (retry && env->me_hsr_callback) {
+ /* LY: notify end of hsr-loop */
+ const txnid_t gap = oldest - laggard;
+ env->me_hsr_callback(env, env->me_txn, 0, 0, laggard,
+ (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, 0,
+ -retry);
}
+ mdbx_notice("hsr-kick: update oldest %" PRIaTXN " -> %" PRIaTXN,
+ lck->mti_oldest_reader.weak, oldest);
+ mdbx_assert(env, lck->mti_oldest_reader.weak <= oldest);
+ return atomic_store64(&lck->mti_oldest_reader, oldest, mo_Relaxed);
}
- /* If there's an overflow page tail, write it too */
- wsize = my->mc_olen[toggle];
- if (wsize) {
- my->mc_olen[toggle] = 0;
- ptr = my->mc_over[toggle];
- goto again;
+ if (!env->me_hsr_callback)
+ break;
+
+ uint32_t pid = atomic_load32(&asleep->mr_pid, mo_AcquireRelease);
+ uint64_t tid = asleep->mr_tid.weak;
+ if (safe64_read(&asleep->mr_txnid) != laggard || pid <= 0)
+ continue;
+
+ const MDBX_meta *head_meta = constmeta_prefer_last(env);
+ const txnid_t gap =
+ (constmeta_txnid(env, head_meta) - laggard) / xMDBX_TXNID_STEP;
+ const uint64_t head_retired =
+ unaligned_peek_u64(4, head_meta->mm_pages_retired);
+ const size_t space =
+ (head_retired > oldest_retired)
+ ? pgno2bytes(env, (pgno_t)(head_retired - oldest_retired))
+ : 0;
+ int rc = env->me_hsr_callback(
+ env, env->me_txn, pid, (mdbx_tid_t)tid, laggard,
+ (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
+ if (rc < 0)
+ break;
+
+ if (rc > 0) {
+ if (rc == 1) {
+ safe64_reset_compare(&asleep->mr_txnid, laggard);
+ } else {
+ safe64_reset(&asleep->mr_txnid, true);
+ atomic_store64(&asleep->mr_tid, 0, mo_Relaxed);
+ atomic_store32(&asleep->mr_pid, 0, mo_Relaxed);
+ }
+ atomic_store32(&lck->mti_readers_refresh_flag, true, mo_Relaxed);
}
- my->mc_tail += 1;
- mdbx_condpair_signal(&my->mc_condpair, false);
}
-bailout:
- mdbx_condpair_unlock(&my->mc_condpair);
- return (THREAD_RESULT)0;
-}
-/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. */
-__cold static int mdbx_env_cthr_toggle(mdbx_copy *my) {
- mdbx_condpair_lock(&my->mc_condpair);
- mdbx_assert(my->mc_env, my->mc_head - my->mc_tail < 2 || my->mc_error);
- my->mc_head += 1;
- mdbx_condpair_signal(&my->mc_condpair, true);
- while (!my->mc_error &&
- my->mc_head - my->mc_tail == 2 /* both buffers in use */) {
- int err = mdbx_condpair_wait(&my->mc_condpair, false);
- if (err != MDBX_SUCCESS)
- my->mc_error = err;
+ if (retry && env->me_hsr_callback) {
+ /* LY: notify end of hsr-loop */
+ env->me_hsr_callback(env, env->me_txn, 0, 0, laggard, 0, 0, -retry);
}
- mdbx_condpair_unlock(&my->mc_condpair);
- return my->mc_error;
+ return mdbx_find_oldest(env->me_txn);
}
-/* Depth-first tree traversal for compacting copy.
- * [in] my control structure.
- * [in,out] pg database root.
- * [in] flags includes F_DUPDATA if it is a sorted-duplicate sub-DB. */
-__cold static int mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) {
- MDBX_cursor_couple couple;
- MDBX_page *mo, *mp, *leaf;
- char *buf, *ptr;
- int rc;
- unsigned i;
-
- /* Empty DB, nothing to do */
- if (*pg == P_INVALID)
- return MDBX_SUCCESS;
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) {
+ return __inline_mdbx_env_set_syncbytes(env, threshold);
+}
- memset(&couple, 0, sizeof(couple));
- couple.outer.mc_snum = 1;
- couple.outer.mc_txn = my->mc_txn;
- couple.outer.mc_flags = couple.inner.mx_cursor.mc_flags =
- C_COPYING | C_SKIPORD;
+__cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) {
+ return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- rc = mdbx_page_get(&couple.outer, *pg, &couple.outer.mc_pg[0],
- my->mc_txn->mt_txnid);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_page_search_root(&couple.outer, NULL, MDBX_PS_FIRST);
+__cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) {
+ int rc = check_env(env, false);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- /* Make cursor pages writable */
- buf = ptr = mdbx_malloc(pgno2bytes(my->mc_env, couple.outer.mc_snum));
- if (buf == NULL)
- return MDBX_ENOMEM;
+ env->me_hsr_callback = hsr;
+ return MDBX_SUCCESS;
+}
- for (i = 0; i < couple.outer.mc_top; i++) {
- mdbx_page_copy((MDBX_page *)ptr, couple.outer.mc_pg[i],
- my->mc_env->me_psize);
- couple.outer.mc_pg[i] = (MDBX_page *)ptr;
- ptr += my->mc_env->me_psize;
- }
+__cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) {
+ return likely(env && env->me_signature.weak == MDBX_ME_SIGNATURE)
+ ? env->me_hsr_callback
+ : NULL;
+}
- /* This is writable space for a leaf page. Usually not needed. */
- leaf = (MDBX_page *)ptr;
+#ifdef __SANITIZE_THREAD__
+/* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */
+__attribute__((__no_sanitize_thread__, __noinline__))
+#endif
+int mdbx_txn_straggler(const MDBX_txn *txn, int *percent)
+{
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return (rc > 0) ? -rc : rc;
- while (couple.outer.mc_snum > 0) {
- mp = couple.outer.mc_pg[couple.outer.mc_top];
- unsigned n = page_numkeys(mp);
+ MDBX_env *env = txn->mt_env;
+ if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) {
+ if (percent)
+ *percent =
+ (int)((txn->mt_next_pgno * UINT64_C(100) + txn->mt_end_pgno / 2) /
+ txn->mt_end_pgno);
+ return 0;
+ }
- if (IS_LEAF(mp)) {
- if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) {
- for (i = 0; i < n; i++) {
- MDBX_node *node = page_node(mp, i);
- if (node_flags(node) & F_BIGDATA) {
- MDBX_page *omp;
+ txnid_t recent = 0;
+ volatile const MDBX_meta *meta = nullptr;
+ do {
+ meta = meta_prefer_last(env);
+ recent = meta_txnid(env, meta);
+ if (percent) {
+ const pgno_t maxpg = meta->mm_geo.now;
+ *percent = (int)((meta->mm_geo.next * UINT64_C(100) + maxpg / 2) / maxpg);
+ }
+ } while (unlikely(recent != meta_txnid(env, meta)));
- /* Need writable leaf */
- if (mp != leaf) {
- couple.outer.mc_pg[couple.outer.mc_top] = leaf;
- mdbx_page_copy(leaf, mp, my->mc_env->me_psize);
- mp = leaf;
- node = page_node(mp, i);
- }
+ txnid_t lag = (recent - txn->mt_txnid) / xMDBX_TXNID_STEP;
+ return (lag > INT_MAX) ? INT_MAX : (int)lag;
+}
- const pgno_t pgno = node_largedata_pgno(node);
- poke_pgno(node_data(node), my->mc_next_pgno);
- rc = mdbx_page_get(&couple.outer, pgno, &omp,
- pp_txnid4chk(mp, my->mc_txn));
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- unsigned toggle = my->mc_head & 1;
- if (my->mc_wlen[toggle] + my->mc_env->me_psize >
- ((size_t)(MDBX_ENVCOPY_WRITEBUF))) {
- rc = mdbx_env_cthr_toggle(my);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- toggle = my->mc_head & 1;
- }
- mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
- memcpy(mo, omp, my->mc_env->me_psize);
- mo->mp_pgno = my->mc_next_pgno;
- my->mc_next_pgno += omp->mp_pages;
- my->mc_wlen[toggle] += my->mc_env->me_psize;
- if (omp->mp_pages > 1) {
- my->mc_olen[toggle] = pgno2bytes(my->mc_env, omp->mp_pages - 1);
- my->mc_over[toggle] = (uint8_t *)omp + my->mc_env->me_psize;
- rc = mdbx_env_cthr_toggle(my);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- toggle = my->mc_head & 1;
- }
- } else if (node_flags(node) & F_SUBDATA) {
- if (!MDBX_DISABLE_PAGECHECKS &&
- unlikely(node_ds(node) != sizeof(MDBX_db))) {
- rc = MDBX_CORRUPTED;
- goto done;
- }
+typedef struct mdbx_walk_ctx {
+ void *mw_user;
+ MDBX_pgvisitor_func *mw_visitor;
+ MDBX_txn *mw_txn;
+ MDBX_cursor *mw_cursor;
+ bool mw_dont_check_keys_ordering;
+} mdbx_walk_ctx_t;
- /* Need writable leaf */
- if (mp != leaf) {
- couple.outer.mc_pg[couple.outer.mc_top] = leaf;
- mdbx_page_copy(leaf, mp, my->mc_env->me_psize);
- mp = leaf;
- node = page_node(mp, i);
- }
+__cold static int mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db,
+ const char *name, int deep);
- MDBX_db db;
- memcpy(&db, node_data(node), sizeof(MDBX_db));
- rc = mdbx_env_cwalk(my, &db.md_root, node_flags(node) & F_DUPDATA);
- if (rc)
- goto done;
- memcpy(node_data(node), &db, sizeof(MDBX_db));
- }
- }
- }
- } else {
- couple.outer.mc_ki[couple.outer.mc_top]++;
- if (couple.outer.mc_ki[couple.outer.mc_top] < n) {
- again:
- rc = mdbx_page_get(
- &couple.outer,
- node_pgno(page_node(mp, couple.outer.mc_ki[couple.outer.mc_top])),
- &mp, pp_txnid4chk(mp, my->mc_txn));
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- couple.outer.mc_top++;
- couple.outer.mc_snum++;
- couple.outer.mc_ki[couple.outer.mc_top] = 0;
- if (IS_BRANCH(mp)) {
- /* Whenever we advance to a sibling branch page,
- * we must proceed all the way down to its first leaf. */
- mdbx_page_copy(couple.outer.mc_pg[couple.outer.mc_top], mp,
- my->mc_env->me_psize);
- goto again;
- } else
- couple.outer.mc_pg[couple.outer.mc_top] = mp;
- continue;
- }
- }
- unsigned toggle = my->mc_head & 1;
- if (my->mc_wlen[toggle] + my->mc_wlen[toggle] >
- ((size_t)(MDBX_ENVCOPY_WRITEBUF))) {
- rc = mdbx_env_cthr_toggle(my);
- if (unlikely(rc != MDBX_SUCCESS))
- goto done;
- toggle = my->mc_head & 1;
+static MDBX_page_type_t walk_page_type(const MDBX_page *mp) {
+ if (mp)
+ switch (mp->mp_flags) {
+ case P_BRANCH:
+ return MDBX_page_branch;
+ case P_LEAF:
+ return MDBX_page_leaf;
+ case P_LEAF | P_LEAF2:
+ return MDBX_page_dupfixed_leaf;
+ case P_OVERFLOW:
+ return MDBX_page_large;
+ case P_META:
+ return MDBX_page_meta;
}
- mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
- mdbx_page_copy(mo, mp, my->mc_env->me_psize);
- mo->mp_pgno = my->mc_next_pgno++;
- my->mc_wlen[toggle] += my->mc_env->me_psize;
- if (couple.outer.mc_top) {
- /* Update parent if there is one */
- node_set_pgno(page_node(couple.outer.mc_pg[couple.outer.mc_top - 1],
- couple.outer.mc_ki[couple.outer.mc_top - 1]),
- mo->mp_pgno);
- mdbx_cursor_pop(&couple.outer);
- } else {
- /* Otherwise we're done */
- *pg = mo->mp_pgno;
+ return MDBX_page_broken;
+}
+
+/* Depth-first tree traversal. */
+__cold static int mdbx_walk_tree(mdbx_walk_ctx_t *ctx, const pgno_t pgno,
+ const char *name, int deep,
+ txnid_t parent_txnid) {
+ assert(pgno != P_INVALID);
+ MDBX_page *mp = nullptr;
+ int rc, err = mdbx_page_get(ctx->mw_cursor, pgno, &mp, parent_txnid);
+ if (err == MDBX_SUCCESS)
+ err = mdbx_page_check(ctx->mw_cursor, mp, 0);
+
+ MDBX_page_type_t type = walk_page_type(mp);
+ const int nentries = (mp && !IS_OVERFLOW(mp)) ? page_numkeys(mp) : 1;
+ unsigned npages = (mp && IS_OVERFLOW(mp)) ? mp->mp_pages : 1;
+ size_t pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages);
+ size_t header_size = (mp && !IS_LEAF2(mp) && !IS_OVERFLOW(mp))
+ ? PAGEHDRSZ + mp->mp_lower
+ : PAGEHDRSZ;
+ size_t payload_size = 0;
+ size_t unused_size =
+ (mp && !IS_OVERFLOW(mp) ? page_room(mp) : pagesize - header_size) -
+ payload_size;
+ size_t align_bytes = 0;
+
+ if (err == MDBX_SUCCESS) {
+ /* LY: Don't use mask here, e.g bitwise
+ * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
+ * Pages should not me marked dirty/loose or otherwise. */
+ switch (mp->mp_flags) {
+ default:
+ err = MDBX_CORRUPTED;
+ break;
+ case P_BRANCH:
+ if (unlikely(nentries < 2))
+ err = MDBX_CORRUPTED;
+ case P_LEAF:
+ case P_LEAF | P_LEAF2:
break;
}
}
-done:
- mdbx_free(buf);
- return rc;
-}
-__cold static void compact_fixup_meta(MDBX_env *env, MDBX_meta *meta) {
- /* Calculate filesize taking in account shrink/growing thresholds */
- if (meta->mm_geo.next != meta->mm_geo.now) {
- meta->mm_geo.now = meta->mm_geo.next;
- const pgno_t aligner = pv2pages(
- meta->mm_geo.grow_pv ? meta->mm_geo.grow_pv : meta->mm_geo.shrink_pv);
- if (aligner) {
- const pgno_t aligned = pgno_align2os_pgno(
- env, meta->mm_geo.next + aligner - meta->mm_geo.next % aligner);
- meta->mm_geo.now = aligned;
+ for (int i = 0; err == MDBX_SUCCESS && i < nentries;
+ align_bytes += ((payload_size + align_bytes) & 1), i++) {
+ if (type == MDBX_page_dupfixed_leaf) {
+ /* LEAF2 pages have no mp_ptrs[] or node headers */
+ payload_size += mp->mp_leaf2_ksize;
+ continue;
}
- }
- if (meta->mm_geo.now < meta->mm_geo.lower)
- meta->mm_geo.now = meta->mm_geo.lower;
- if (meta->mm_geo.now > meta->mm_geo.upper)
- meta->mm_geo.now = meta->mm_geo.upper;
+ MDBX_node *node = page_node(mp, i);
+ payload_size += NODESIZE + node_ks(node);
- /* Update signature */
- assert(meta->mm_geo.now >= meta->mm_geo.next);
- unaligned_poke_u64(4, meta->mm_datasync_sign, meta_sign(meta));
-}
+ if (type == MDBX_page_branch) {
+ assert(i > 0 || node_ks(node) == 0);
+ continue;
+ }
-/* Make resizeable */
-__cold static void make_sizeable(MDBX_meta *meta) {
- meta->mm_geo.lower = MIN_PAGENO;
- if (meta->mm_geo.grow_pv == 0) {
- const pgno_t step = 1 + (meta->mm_geo.upper - meta->mm_geo.lower) / 42;
- meta->mm_geo.grow_pv = pages2pv(step);
- }
- if (meta->mm_geo.shrink_pv == 0) {
- const pgno_t step = pv2pages(meta->mm_geo.grow_pv) << 1;
- meta->mm_geo.shrink_pv = pages2pv(step);
- }
-}
+ assert(type == MDBX_page_leaf);
+ switch (node_flags(node)) {
+ case 0 /* usual node */:
+ payload_size += node_ds(node);
+ break;
-/* Copy environment with compaction. */
-__cold static int mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn,
- mdbx_filehandle_t fd, uint8_t *buffer,
- const bool dest_is_pipe, const int flags) {
- const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
- uint8_t *const data_buffer =
- buffer + ceil_powerof2(meta_bytes, env->me_os_psize);
- MDBX_meta *const meta = mdbx_init_metas(env, buffer);
- meta_set_txnid(env, meta, read_txn->mt_txnid);
+ case F_BIGDATA /* long data on the large/overflow page */: {
+ payload_size += sizeof(pgno_t);
+ const pgno_t large_pgno = node_largedata_pgno(node);
+ const size_t over_payload = node_ds(node);
+ const size_t over_header = PAGEHDRSZ;
+ npages = 1;
- if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
- make_sizeable(meta);
+ MDBX_page *op;
+ err = mdbx_page_get(ctx->mw_cursor, large_pgno, &op,
+ pp_txnid4chk(mp, ctx->mw_txn));
+ if (err == MDBX_SUCCESS)
+ err = mdbx_page_check(ctx->mw_cursor, op, 0);
+ if (err == MDBX_SUCCESS) {
+ /* LY: Don't use mask here, e.g bitwise
+ * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
+ * Pages should not me marked dirty/loose or otherwise. */
+ if (unlikely(P_OVERFLOW != op->mp_flags))
+ err = bad_page(mp, "wrong page type %d for large data", op->mp_flags);
+ else
+ npages = op->mp_pages;
+ }
- /* copy canary sequences if present */
- if (read_txn->mt_canary.v) {
- meta->mm_canary = read_txn->mt_canary;
- meta->mm_canary.v = constmeta_txnid(env, meta);
- }
+ pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages);
+ const size_t over_unused = pagesize - over_payload - over_header;
+ rc = ctx->mw_visitor(large_pgno, npages, ctx->mw_user, deep, name,
+ pagesize, MDBX_page_large, err, 1, over_payload,
+ over_header, over_unused);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
+ } break;
- /* Set metapage 1 with current main DB */
- pgno_t new_root, root = read_txn->mt_dbs[MAIN_DBI].md_root;
- if ((new_root = root) == P_INVALID) {
- /* When the DB is empty, handle it specially to
- * fix any breakage like page leaks from ITS#8174. */
- meta->mm_dbs[MAIN_DBI].md_flags = read_txn->mt_dbs[MAIN_DBI].md_flags;
- compact_fixup_meta(env, meta);
- if (dest_is_pipe) {
- int rc = mdbx_write(fd, buffer, meta_bytes);
- if (rc != MDBX_SUCCESS)
- return rc;
- }
- } else {
- /* Count free pages + GC pages. Subtract from last_pg
- * to find the new last_pg, which also becomes the new root. */
- pgno_t freecount = 0;
- MDBX_cursor_couple couple;
- MDBX_val key, data;
+ case F_SUBDATA /* sub-db */: {
+ const size_t namelen = node_ks(node);
+ payload_size += node_ds(node);
+ if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db)))
+ err = MDBX_CORRUPTED;
+ } break;
- int rc = mdbx_cursor_init(&couple.outer, read_txn, FREE_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- while ((rc = mdbx_cursor_get(&couple.outer, &key, &data, MDBX_NEXT)) == 0)
- freecount += *(pgno_t *)data.iov_base;
- if (unlikely(rc != MDBX_NOTFOUND))
- return rc;
+ case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
+ payload_size += sizeof(MDBX_db);
+ if (unlikely(node_ds(node) != sizeof(MDBX_db)))
+ err = MDBX_CORRUPTED;
+ break;
- freecount += read_txn->mt_dbs[FREE_DBI].md_branch_pages +
- read_txn->mt_dbs[FREE_DBI].md_leaf_pages +
- read_txn->mt_dbs[FREE_DBI].md_overflow_pages;
+ case F_DUPDATA /* short sub-page */: {
+ if (unlikely(node_ds(node) <= PAGEHDRSZ)) {
+ err = MDBX_CORRUPTED;
+ break;
+ }
- new_root = read_txn->mt_next_pgno - 1 - freecount;
- meta->mm_geo.next = new_root + 1;
- meta->mm_dbs[MAIN_DBI] = read_txn->mt_dbs[MAIN_DBI];
- meta->mm_dbs[MAIN_DBI].md_root = new_root;
+ MDBX_page *sp = node_data(node);
+ const int nsubkeys = page_numkeys(sp);
+ size_t subheader_size =
+ IS_LEAF2(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->mp_lower;
+ size_t subunused_size = page_room(sp);
+ size_t subpayload_size = 0;
+ size_t subalign_bytes = 0;
+ MDBX_page_type_t subtype;
- mdbx_copy ctx;
- memset(&ctx, 0, sizeof(ctx));
- rc = mdbx_condpair_init(&ctx.mc_condpair);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ switch (sp->mp_flags & /* ignore legacy P_DIRTY flag */ ~0x10) {
+ case P_LEAF | P_SUBP:
+ subtype = MDBX_subpage_leaf;
+ break;
+ case P_LEAF | P_LEAF2 | P_SUBP:
+ subtype = MDBX_subpage_dupfixed_leaf;
+ break;
+ default:
+ subtype = MDBX_subpage_broken;
+ err = MDBX_CORRUPTED;
+ }
- memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)) * 2);
- ctx.mc_wbuf[0] = data_buffer;
- ctx.mc_wbuf[1] = data_buffer + ((size_t)(MDBX_ENVCOPY_WRITEBUF));
- ctx.mc_next_pgno = NUM_METAS;
- ctx.mc_env = env;
- ctx.mc_fd = fd;
- ctx.mc_txn = read_txn;
+ for (int j = 0; err == MDBX_SUCCESS && j < nsubkeys;
+ subalign_bytes += ((subpayload_size + subalign_bytes) & 1), j++) {
- mdbx_thread_t thread;
- int thread_err = mdbx_thread_create(&thread, mdbx_env_copythr, &ctx);
- if (likely(thread_err == MDBX_SUCCESS)) {
- if (dest_is_pipe) {
- compact_fixup_meta(env, meta);
- rc = mdbx_write(fd, buffer, meta_bytes);
+ if (subtype == MDBX_subpage_dupfixed_leaf) {
+ /* LEAF2 pages have no mp_ptrs[] or node headers */
+ subpayload_size += sp->mp_leaf2_ksize;
+ } else {
+ assert(subtype == MDBX_subpage_leaf);
+ MDBX_node *subnode = page_node(sp, j);
+ subpayload_size += NODESIZE + node_ks(subnode) + node_ds(subnode);
+ if (unlikely(node_flags(subnode) != 0))
+ err = MDBX_CORRUPTED;
+ }
}
- if (rc == MDBX_SUCCESS)
- rc = mdbx_env_cwalk(&ctx, &root, 0);
- mdbx_env_cthr_toggle(&ctx);
- mdbx_env_cthr_toggle(&ctx);
- thread_err = mdbx_thread_join(thread);
- mdbx_assert(env, (ctx.mc_tail == ctx.mc_head &&
- ctx.mc_wlen[ctx.mc_head & 1] == 0) ||
- ctx.mc_error);
- mdbx_condpair_destroy(&ctx.mc_condpair);
+
+ rc = ctx->mw_visitor(pgno, 0, ctx->mw_user, deep + 1, name, node_ds(node),
+ subtype, err, nsubkeys, subpayload_size,
+ subheader_size, subunused_size + subalign_bytes);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
+ header_size += subheader_size;
+ unused_size += subunused_size;
+ payload_size += subpayload_size;
+ align_bytes += subalign_bytes;
+ } break;
+
+ default:
+ err = MDBX_CORRUPTED;
+ }
+ }
+
+ rc = ctx->mw_visitor(pgno, 1, ctx->mw_user, deep, name,
+ ctx->mw_txn->mt_env->me_psize, type, err, nentries,
+ payload_size, header_size, unused_size + align_bytes);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
+
+ for (int i = 0; err == MDBX_SUCCESS && i < nentries; i++) {
+ if (type == MDBX_page_dupfixed_leaf)
+ continue;
+
+ MDBX_node *node = page_node(mp, i);
+ if (type == MDBX_page_branch) {
+ err = mdbx_walk_tree(ctx, node_pgno(node), name, deep + 1,
+ pp_txnid4chk(mp, ctx->mw_txn));
+ if (unlikely(err != MDBX_SUCCESS)) {
+ if (err == MDBX_RESULT_TRUE)
+ break;
+ return err;
+ }
+ continue;
}
- if (unlikely(thread_err != MDBX_SUCCESS))
- return thread_err;
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- if (unlikely(ctx.mc_error != MDBX_SUCCESS))
- return ctx.mc_error;
- if (dest_is_pipe) {
- if (unlikely(root != new_root)) {
- mdbx_error("post-compactification root %" PRIaPGNO
- " NE expected %" PRIaPGNO
- " (source DB corrupted or has a page leak(s))",
- root, new_root);
- return MDBX_CORRUPTED; /* page leak or corrupt DB */
+ assert(type == MDBX_page_leaf);
+ MDBX_db db;
+ switch (node_flags(node)) {
+ default:
+ continue;
+
+ case F_SUBDATA /* sub-db */: {
+ const size_t namelen = node_ks(node);
+ if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) {
+ err = MDBX_CORRUPTED;
+ break;
}
- } else {
- if (unlikely(root > new_root)) {
- mdbx_error("post-compactification root %" PRIaPGNO
- " GT expected %" PRIaPGNO " (source DB corrupted)",
- root, new_root);
- return MDBX_CORRUPTED; /* page leak or corrupt DB */
+
+ char namebuf_onstask[64];
+ char *const sub_name = (namelen < sizeof(namebuf_onstask))
+ ? namebuf_onstask
+ : mdbx_malloc(namelen + 1);
+ if (sub_name) {
+ memcpy(sub_name, node_key(node), namelen);
+ sub_name[namelen] = 0;
+ memcpy(&db, node_data(node), sizeof(db));
+ err = mdbx_walk_sdb(ctx, &db, sub_name, deep + 1);
+ if (sub_name != namebuf_onstask)
+ mdbx_free(sub_name);
+ } else {
+ err = MDBX_ENOMEM;
}
- if (unlikely(root < new_root)) {
- mdbx_warning("post-compactification root %" PRIaPGNO
- " LT expected %" PRIaPGNO " (page leak(s) in source DB)",
- root, new_root);
- /* fixup meta */
- meta->mm_dbs[MAIN_DBI].md_root = root;
- meta->mm_geo.next = root + 1;
+ } break;
+
+ case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
+ if (unlikely(node_ds(node) != sizeof(MDBX_db) ||
+ ctx->mw_cursor->mc_xcursor == NULL))
+ err = MDBX_CORRUPTED;
+ else {
+ memcpy(&db, node_data(node), sizeof(db));
+ assert(ctx->mw_cursor->mc_xcursor ==
+ &container_of(ctx->mw_cursor, MDBX_cursor_couple, outer)->inner);
+ ctx->mw_cursor = &ctx->mw_cursor->mc_xcursor->mx_cursor;
+ err = mdbx_walk_tree(ctx, db.md_root, name, deep + 1,
+ pp_txnid4chk(mp, ctx->mw_txn));
+ MDBX_xcursor *inner_xcursor =
+ container_of(ctx->mw_cursor, MDBX_xcursor, mx_cursor);
+ MDBX_cursor_couple *couple =
+ container_of(inner_xcursor, MDBX_cursor_couple, inner);
+ ctx->mw_cursor = &couple->outer;
}
- compact_fixup_meta(env, meta);
+ break;
}
}
- /* Extend file if required */
- if (meta->mm_geo.now != meta->mm_geo.next) {
- const size_t whole_size = pgno2bytes(env, meta->mm_geo.now);
- if (!dest_is_pipe)
- return mdbx_ftruncate(fd, whole_size);
-
- const size_t used_size = pgno2bytes(env, meta->mm_geo.next);
- memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)));
- for (size_t offset = used_size; offset < whole_size;) {
- const size_t chunk =
- (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < whole_size - offset)
- ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
- : whole_size - offset;
- /* copy to avoid EFAULT in case swapped-out */
- int rc = mdbx_write(fd, data_buffer, chunk);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- offset += chunk;
- }
- }
return MDBX_SUCCESS;
}
-/* Copy environment as-is. */
-__cold static int mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn,
- mdbx_filehandle_t fd, uint8_t *buffer,
- const bool dest_is_pipe, const int flags) {
- /* We must start the actual read txn after blocking writers */
- int rc = mdbx_txn_end(read_txn, MDBX_END_RESET_TMP);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+__cold static int mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db,
+ const char *name, int deep) {
+ if (unlikely(db->md_root == P_INVALID))
+ return MDBX_SUCCESS; /* empty db */
- /* Temporarily block writers until we snapshot the meta pages */
- rc = mdbx_txn_lock(env, false);
+ MDBX_cursor_couple couple;
+ MDBX_dbx dbx = {.md_klen_min = INT_MAX};
+ uint8_t dbistate = DBI_VALID | DBI_AUDITED;
+ int rc = mdbx_couple_init(&couple, ~0u, ctx->mw_txn, db, &dbx, &dbistate);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- rc = mdbx_txn_renew0(read_txn, MDBX_TXN_RDONLY);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_txn_unlock(env);
- return rc;
+ if (ctx->mw_dont_check_keys_ordering) {
+ couple.outer.mc_flags |= C_SKIPORD;
+ couple.inner.mx_cursor.mc_flags |= C_SKIPORD;
}
+ couple.outer.mc_next = ctx->mw_cursor;
+ ctx->mw_cursor = &couple.outer;
+ rc = mdbx_walk_tree(ctx, db->md_root, name, deep, ctx->mw_txn->mt_txnid);
+ ctx->mw_cursor = couple.outer.mc_next;
+ return rc;
+}
- mdbx_jitter4testing(false);
- const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
- /* Make a snapshot of meta-pages,
- * but writing ones after the data was flushed */
- memcpy(buffer, env->me_map, meta_bytes);
- MDBX_meta *const headcopy = /* LY: get pointer to the snapshot copy */
- (MDBX_meta *)(buffer + ((uint8_t *)meta_prefer_last(env) - env->me_map));
- mdbx_txn_unlock(env);
-
- if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
- make_sizeable(headcopy);
- /* Update signature to steady */
- unaligned_poke_u64(4, headcopy->mm_datasync_sign, meta_sign(headcopy));
-
- /* Copy the data */
- const size_t whole_size = pgno_align2os_bytes(env, read_txn->mt_end_pgno);
- const size_t used_size = pgno2bytes(env, read_txn->mt_next_pgno);
- mdbx_jitter4testing(false);
-
- if (dest_is_pipe)
- rc = mdbx_write(fd, buffer, meta_bytes);
+__cold int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor,
+ void *user, bool dont_check_keys_ordering) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- uint8_t *const data_buffer =
- buffer + ceil_powerof2(meta_bytes, env->me_os_psize);
-#if MDBX_USE_COPYFILERANGE
- static bool copyfilerange_unavailable;
- bool not_the_same_filesystem = false;
-#endif /* MDBX_USE_COPYFILERANGE */
- for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) {
-#if MDBX_USE_SENDFILE
- static bool sendfile_unavailable;
- if (dest_is_pipe && likely(!sendfile_unavailable)) {
- off_t in_offset = offset;
- const ssize_t written =
- sendfile(fd, env->me_lazy_fd, &in_offset, used_size - offset);
- if (likely(written > 0)) {
- offset = in_offset;
- continue;
- }
- rc = MDBX_ENODATA;
- if (written == 0 || ignore_enosys(rc = errno) != MDBX_RESULT_TRUE)
- break;
- sendfile_unavailable = true;
- }
-#endif /* MDBX_USE_SENDFILE */
+ mdbx_walk_ctx_t ctx;
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.mw_txn = txn;
+ ctx.mw_user = user;
+ ctx.mw_visitor = visitor;
+ ctx.mw_dont_check_keys_ordering = dont_check_keys_ordering;
-#if MDBX_USE_COPYFILERANGE
- if (!dest_is_pipe && !not_the_same_filesystem &&
- likely(!copyfilerange_unavailable)) {
- off_t in_offset = offset, out_offset = offset;
- ssize_t bytes_copied = copy_file_range(
- env->me_lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0);
- if (likely(bytes_copied > 0)) {
- offset = in_offset;
- continue;
- }
- rc = MDBX_ENODATA;
- if (bytes_copied == 0)
- break;
- rc = errno;
- if (rc == EXDEV)
- not_the_same_filesystem = true;
- else if (ignore_enosys(rc) == MDBX_RESULT_TRUE)
- copyfilerange_unavailable = true;
- else
- break;
- }
-#endif /* MDBX_USE_COPYFILERANGE */
+ rc = visitor(0, NUM_METAS, user, 0, MDBX_PGWALK_META,
+ pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, MDBX_SUCCESS,
+ NUM_METAS, sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS,
+ (txn->mt_env->me_psize - sizeof(MDBX_meta) - PAGEHDRSZ) *
+ NUM_METAS);
+ if (!MDBX_IS_ERROR(rc))
+ rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[FREE_DBI], MDBX_PGWALK_GC, 0);
+ if (!MDBX_IS_ERROR(rc))
+ rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[MAIN_DBI], MDBX_PGWALK_MAIN, 0);
+ return rc;
+}
- /* fallback to portable */
- const size_t chunk =
- (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < used_size - offset)
- ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
- : used_size - offset;
- /* copy to avoid EFAULT in case swapped-out */
- memcpy(data_buffer, env->me_map + offset, chunk);
- rc = mdbx_write(fd, data_buffer, chunk);
- offset += chunk;
- }
+int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) {
+ int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- /* Extend file if required */
- if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) {
- if (!dest_is_pipe)
- rc = mdbx_ftruncate(fd, whole_size);
- else {
- memset(data_buffer, 0, ((size_t)(MDBX_ENVCOPY_WRITEBUF)));
- for (size_t offset = used_size;
- rc == MDBX_SUCCESS && offset < whole_size;) {
- const size_t chunk =
- (((size_t)(MDBX_ENVCOPY_WRITEBUF)) < whole_size - offset)
- ? ((size_t)(MDBX_ENVCOPY_WRITEBUF))
- : whole_size - offset;
- /* copy to avoid EFAULT in case swapped-out */
- rc = mdbx_write(fd, data_buffer, chunk);
- offset += chunk;
- }
- }
+ if (likely(canary)) {
+ if (txn->mt_canary.x == canary->x && txn->mt_canary.y == canary->y &&
+ txn->mt_canary.z == canary->z)
+ return MDBX_SUCCESS;
+ txn->mt_canary.x = canary->x;
+ txn->mt_canary.y = canary->y;
+ txn->mt_canary.z = canary->z;
}
+ txn->mt_canary.v = txn->mt_txnid;
+ txn->mt_flags |= MDBX_TXN_DIRTY;
- return rc;
+ return MDBX_SUCCESS;
}
-__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd,
- unsigned flags) {
- int rc = check_env(env, true);
+int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- const int dest_is_pipe = mdbx_is_pipe(fd);
- if (MDBX_IS_ERROR(dest_is_pipe))
- return dest_is_pipe;
+ if (unlikely(canary == NULL))
+ return MDBX_EINVAL;
- if (!dest_is_pipe) {
- rc = mdbx_fseek(fd, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
+ *canary = txn->mt_canary;
+ return MDBX_SUCCESS;
+}
- const size_t buffer_size =
- pgno_align2os_bytes(env, NUM_METAS) +
- ceil_powerof2(((flags & MDBX_CP_COMPACT)
- ? ((size_t)(MDBX_ENVCOPY_WRITEBUF)) * 2
- : ((size_t)(MDBX_ENVCOPY_WRITEBUF))),
- env->me_os_psize);
+int mdbx_cursor_on_first(const MDBX_cursor *mc) {
+ if (unlikely(mc == NULL))
+ return MDBX_EINVAL;
- uint8_t *buffer = NULL;
- rc = mdbx_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- MDBX_txn *read_txn = NULL;
- /* Do the lock/unlock of the reader mutex before starting the
- * write txn. Otherwise other read txns could block writers. */
- rc = mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &read_txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_memalign_free(buffer);
- return rc;
- }
+ if (!(mc->mc_flags & C_INITIALIZED))
+ return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
- if (!dest_is_pipe) {
- /* Firstly write a stub to meta-pages.
- * Now we sure to incomplete copy will not be used. */
- memset(buffer, -1, pgno2bytes(env, NUM_METAS));
- rc = mdbx_write(fd, buffer, pgno2bytes(env, NUM_METAS));
+ for (unsigned i = 0; i < mc->mc_snum; ++i) {
+ if (mc->mc_ki[i])
+ return MDBX_RESULT_FALSE;
}
- if (likely(rc == MDBX_SUCCESS)) {
- memset(buffer, 0, pgno2bytes(env, NUM_METAS));
- rc = ((flags & MDBX_CP_COMPACT) ? mdbx_env_compact : mdbx_env_copy_asis)(
- env, read_txn, fd, buffer, dest_is_pipe, flags);
- }
- mdbx_txn_abort(read_txn);
+ return MDBX_RESULT_TRUE;
+}
- if (!dest_is_pipe) {
- if (likely(rc == MDBX_SUCCESS))
- rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE);
+int mdbx_cursor_on_last(const MDBX_cursor *mc) {
+ if (unlikely(mc == NULL))
+ return MDBX_EINVAL;
- /* Write actual meta */
- if (likely(rc == MDBX_SUCCESS))
- rc = mdbx_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0);
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
- if (likely(rc == MDBX_SUCCESS))
- rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
+ if (!(mc->mc_flags & C_INITIALIZED))
+ return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
+
+ for (unsigned i = 0; i < mc->mc_snum; ++i) {
+ unsigned nkeys = page_numkeys(mc->mc_pg[i]);
+ if (mc->mc_ki[i] < nkeys - 1)
+ return MDBX_RESULT_FALSE;
}
- mdbx_memalign_free(buffer);
- return rc;
+ return MDBX_RESULT_TRUE;
}
-__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path,
- MDBX_copy_flags_t flags) {
- int rc = check_env(env, true);
+int mdbx_cursor_eof(const MDBX_cursor *mc) {
+ if (unlikely(mc == NULL))
+ return MDBX_EINVAL;
+
+ if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+ return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
+
+ return ((mc->mc_flags & (C_INITIALIZED | C_EOF)) == C_INITIALIZED &&
+ mc->mc_snum &&
+ mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]))
+ ? MDBX_RESULT_FALSE
+ : MDBX_RESULT_TRUE;
+}
+
+//------------------------------------------------------------------------------
+
+struct diff_result {
+ ptrdiff_t diff;
+ unsigned level;
+ int root_nkeys;
+};
+
+/* calculates: r = x - y */
+__hot static int cursor_diff(const MDBX_cursor *const __restrict x,
+ const MDBX_cursor *const __restrict y,
+ struct diff_result *const __restrict r) {
+ r->diff = 0;
+ r->level = 0;
+ r->root_nkeys = 0;
+
+ if (unlikely(x->mc_signature != MDBX_MC_LIVE))
+ return (x->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
+
+ if (unlikely(y->mc_signature != MDBX_MC_LIVE))
+ return (y->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
+
+ int rc = check_txn(x->mc_txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!dest_path))
+ if (unlikely(x->mc_txn != y->mc_txn))
+ return MDBX_BAD_TXN;
+
+ if (unlikely(y->mc_dbi != x->mc_dbi))
+ return MDBX_EINVAL;
+
+ if (unlikely(!(y->mc_flags & x->mc_flags & C_INITIALIZED)))
+ return MDBX_ENODATA;
+
+ while (likely(r->level < y->mc_snum && r->level < x->mc_snum)) {
+ if (unlikely(y->mc_pg[r->level] != x->mc_pg[r->level])) {
+ mdbx_error("Mismatch cursors's pages at %u level", r->level);
+ return MDBX_PROBLEM;
+ }
+
+ int nkeys = page_numkeys(y->mc_pg[r->level]);
+ assert(nkeys > 0);
+ if (r->level == 0)
+ r->root_nkeys = nkeys;
+
+ const int limit_ki = nkeys - 1;
+ const int x_ki = x->mc_ki[r->level];
+ const int y_ki = y->mc_ki[r->level];
+ r->diff = ((x_ki < limit_ki) ? x_ki : limit_ki) -
+ ((y_ki < limit_ki) ? y_ki : limit_ki);
+ if (r->diff == 0) {
+ r->level += 1;
+ continue;
+ }
+
+ while (unlikely(r->diff == 1) &&
+ likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) {
+ r->level += 1;
+ /* DB'PAGEs: 0------------------>MAX
+ *
+ * CURSORs: y < x
+ * STACK[i ]: |
+ * STACK[+1]: ...y++N|0++x...
+ */
+ nkeys = page_numkeys(y->mc_pg[r->level]);
+ r->diff = (nkeys - y->mc_ki[r->level]) + x->mc_ki[r->level];
+ assert(r->diff > 0);
+ }
+
+ while (unlikely(r->diff == -1) &&
+ likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) {
+ r->level += 1;
+ /* DB'PAGEs: 0------------------>MAX
+ *
+ * CURSORs: x < y
+ * STACK[i ]: |
+ * STACK[+1]: ...x--N|0--y...
+ */
+ nkeys = page_numkeys(x->mc_pg[r->level]);
+ r->diff = -(nkeys - x->mc_ki[r->level]) - y->mc_ki[r->level];
+ assert(r->diff < 0);
+ }
+
+ return MDBX_SUCCESS;
+ }
+
+ r->diff = CMP2INT(x->mc_flags & C_EOF, y->mc_flags & C_EOF);
+ return MDBX_SUCCESS;
+}
+
+__hot static ptrdiff_t estimate(const MDBX_db *db,
+ struct diff_result *const __restrict dr) {
+ /* root: branch-page => scale = leaf-factor * branch-factor^(N-1)
+ * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2
+ * level-2: branch-page(s) => scale = leaf-factor * branch-factor
+ * level-N: branch-page(s) => scale = leaf-factor
+ * leaf-level: leaf-page(s) => scale = 1
+ */
+ ptrdiff_t btree_power = (ptrdiff_t)db->md_depth - 2 - (ptrdiff_t)dr->level;
+ if (btree_power < 0)
+ return dr->diff;
+
+ ptrdiff_t estimated =
+ (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)db->md_leaf_pages;
+ if (btree_power == 0)
+ return estimated;
+
+ if (db->md_depth < 4) {
+ assert(dr->level == 0 && btree_power == 1);
+ return (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)dr->root_nkeys;
+ }
+
+ /* average_branchpage_fillfactor = total(branch_entries) / branch_pages
+ total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */
+ const size_t log2_fixedpoint = sizeof(size_t) - 1;
+ const size_t half = UINT64_C(1) << (log2_fixedpoint - 1);
+ const size_t factor =
+ ((db->md_leaf_pages + db->md_branch_pages - 1) << log2_fixedpoint) /
+ db->md_branch_pages;
+ while (1) {
+ switch ((size_t)btree_power) {
+ default: {
+ const size_t square = (factor * factor + half) >> log2_fixedpoint;
+ const size_t quad = (square * square + half) >> log2_fixedpoint;
+ do {
+ estimated = estimated * quad + half;
+ estimated >>= log2_fixedpoint;
+ btree_power -= 4;
+ } while (btree_power >= 4);
+ continue;
+ }
+ case 3:
+ estimated = estimated * factor + half;
+ estimated >>= log2_fixedpoint;
+ __fallthrough /* fall through */;
+ case 2:
+ estimated = estimated * factor + half;
+ estimated >>= log2_fixedpoint;
+ __fallthrough /* fall through */;
+ case 1:
+ estimated = estimated * factor + half;
+ estimated >>= log2_fixedpoint;
+ __fallthrough /* fall through */;
+ case 0:
+ if (unlikely(estimated > (ptrdiff_t)db->md_entries))
+ return (ptrdiff_t)db->md_entries;
+ if (unlikely(estimated < -(ptrdiff_t)db->md_entries))
+ return -(ptrdiff_t)db->md_entries;
+ return estimated;
+ }
+ }
+}
+
+int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last,
+ ptrdiff_t *distance_items) {
+ if (unlikely(first == NULL || last == NULL || distance_items == NULL))
return MDBX_EINVAL;
- /* The destination path must exist, but the destination file must not.
- * We don't want the OS to cache the writes, since the source data is
- * already in the OS cache. */
- mdbx_filehandle_t newfd;
- rc = mdbx_openfile(MDBX_OPEN_COPY, env, dest_path, &newfd,
-#if defined(_WIN32) || defined(_WIN64)
- (mdbx_mode_t)-1
-#else
- S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
-#endif
- );
+ *distance_items = 0;
+ struct diff_result dr;
+ int rc = cursor_diff(last, first, &dr);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- if (rc == MDBX_SUCCESS) {
-#if defined(_WIN32) || defined(_WIN64)
- OVERLAPPED ov;
- memset(&ov, 0, sizeof(ov));
- if (!LockFileEx(newfd, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
- 0, 0, INT32_MAX, &ov))
- rc = GetLastError();
-#else
- struct flock lock_op;
- memset(&lock_op, 0, sizeof(lock_op));
- lock_op.l_type = F_WRLCK;
- lock_op.l_whence = SEEK_SET;
- lock_op.l_start = 0;
- lock_op.l_len =
- (sizeof(lock_op.l_len) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff;
- if (fcntl(newfd, F_SETLK, &lock_op)
-#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \
- (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24)
- || flock(newfd, LOCK_EX | LOCK_NB)
-#endif /* Linux */
- )
- rc = errno;
-#endif /* Windows / POSIX */
+ if (unlikely(dr.diff == 0) &&
+ F_ISSET(first->mc_db->md_flags & last->mc_db->md_flags,
+ MDBX_DUPSORT | C_INITIALIZED)) {
+ first = &first->mc_xcursor->mx_cursor;
+ last = &last->mc_xcursor->mx_cursor;
+ rc = cursor_diff(first, last, &dr);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
}
- if (rc == MDBX_SUCCESS)
- rc = mdbx_env_copy2fd(env, newfd, flags);
-
- if (newfd != INVALID_HANDLE_VALUE) {
- int err = mdbx_closefile(newfd);
- if (rc == MDBX_SUCCESS && err != rc)
- rc = err;
- if (rc != MDBX_SUCCESS)
- (void)mdbx_removefile(dest_path);
- }
+ if (likely(dr.diff != 0))
+ *distance_items = estimate(first->mc_db, &dr);
- return rc;
+ return MDBX_SUCCESS;
}
-/******************************************************************************/
+int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data,
+ MDBX_cursor_op move_op, ptrdiff_t *distance_items) {
+ if (unlikely(cursor == NULL || distance_items == NULL ||
+ move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE))
+ return MDBX_EINVAL;
-__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags,
- bool onoff) {
- int rc = check_env(env, false);
+ if (unlikely(cursor->mc_signature != MDBX_MC_LIVE))
+ return (cursor->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+ : MDBX_EBADSIGN;
+
+ int rc = check_txn(cursor->mc_txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(flags &
- ((env->me_flags & MDBX_ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS
- : ~ENV_USABLE_FLAGS)))
- return MDBX_EPERM;
+ if (!(cursor->mc_flags & C_INITIALIZED))
+ return MDBX_ENODATA;
- if (unlikely(env->me_flags & MDBX_RDONLY))
- return MDBX_EACCESS;
+ MDBX_cursor_couple next;
+ cursor_copy(cursor, &next.outer);
+ if (cursor->mc_db->md_flags & MDBX_DUPSORT) {
+ next.outer.mc_xcursor = &next.inner;
+ rc = mdbx_xcursor_init0(&next.outer);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ MDBX_xcursor *mx = &container_of(cursor, MDBX_cursor_couple, outer)->inner;
+ cursor_copy(&mx->mx_cursor, &next.inner.mx_cursor);
+ }
- if ((env->me_flags & MDBX_ENV_ACTIVE) &&
- unlikely(env->me_txn0->mt_owner == mdbx_thread_self()))
- return MDBX_BUSY;
+ MDBX_val stub = {0, 0};
+ if (data == NULL) {
+ const unsigned mask =
+ 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY;
+ if (unlikely(mask & (1 << move_op)))
+ return MDBX_EINVAL;
+ data = &stub;
+ }
- const bool lock_needed = (env->me_flags & MDBX_ENV_ACTIVE) &&
- env->me_txn0->mt_owner != mdbx_thread_self();
- bool should_unlock = false;
- if (lock_needed) {
- rc = mdbx_txn_lock(env, false);
- if (unlikely(rc))
- return rc;
- should_unlock = true;
+ if (key == NULL) {
+ const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE |
+ 1 << MDBX_SET_KEY | 1 << MDBX_SET |
+ 1 << MDBX_SET_RANGE;
+ if (unlikely(mask & (1 << move_op)))
+ return MDBX_EINVAL;
+ key = &stub;
}
- if (onoff)
- env->me_flags = merge_sync_flags(env->me_flags, flags);
- else
- env->me_flags &= ~flags;
+ next.outer.mc_signature = MDBX_MC_LIVE;
+ rc = mdbx_cursor_get(&next.outer, key, data, move_op);
+ if (unlikely(rc != MDBX_SUCCESS &&
+ (rc != MDBX_NOTFOUND || !(next.outer.mc_flags & C_INITIALIZED))))
+ return rc;
- if (should_unlock)
- mdbx_txn_unlock(env);
- return MDBX_SUCCESS;
+ return mdbx_estimate_distance(cursor, &next.outer, distance_items);
}
-__cold int mdbx_env_get_flags(const MDBX_env *env, unsigned *arg) {
- int rc = check_env(env, false);
+int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key,
+ MDBX_val *begin_data, MDBX_val *end_key,
+ MDBX_val *end_data, ptrdiff_t *size_items) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!arg))
+ if (unlikely(!size_items))
return MDBX_EINVAL;
- *arg = env->me_flags & ENV_USABLE_FLAGS;
- return MDBX_SUCCESS;
-}
+ if (unlikely(begin_data && (begin_key == NULL || begin_key == MDBX_EPSILON)))
+ return MDBX_EINVAL;
-__cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) {
- int rc = check_env(env, false);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (unlikely(end_data && (end_key == NULL || end_key == MDBX_EPSILON)))
+ return MDBX_EINVAL;
- env->me_userctx = ctx;
- return MDBX_SUCCESS;
-}
+ if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON))
+ return MDBX_EINVAL;
-__cold void *mdbx_env_get_userctx(const MDBX_env *env) {
- return env ? env->me_userctx : NULL;
-}
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
-__cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) {
- int rc = check_env(env, false);
+ MDBX_cursor_couple begin;
+ /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */
+ rc = mdbx_cursor_init(&begin.outer, txn, dbi);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
-#if MDBX_DEBUG
- env->me_assert_func = func;
- return MDBX_SUCCESS;
-#else
- (void)func;
- return MDBX_ENOSYS;
-#endif
-}
+ if (unlikely(begin.outer.mc_db->md_entries == 0)) {
+ *size_items = 0;
+ return MDBX_SUCCESS;
+ }
+
+ if (!begin_key) {
+ if (unlikely(!end_key)) {
+ /* LY: FIRST..LAST case */
+ *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries;
+ return MDBX_SUCCESS;
+ }
+ MDBX_val stub = {0, 0};
+ rc = mdbx_cursor_first(&begin.outer, &stub, &stub);
+ if (unlikely(end_key == MDBX_EPSILON)) {
+ /* LY: FIRST..+epsilon case */
+ return (rc == MDBX_SUCCESS)
+ ? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
+ : rc;
+ }
+ } else {
+ if (unlikely(begin_key == MDBX_EPSILON)) {
+ if (end_key == NULL) {
+ /* LY: -epsilon..LAST case */
+ MDBX_val stub = {0, 0};
+ rc = mdbx_cursor_last(&begin.outer, &stub, &stub);
+ return (rc == MDBX_SUCCESS)
+ ? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
+ : rc;
+ }
+ /* LY: -epsilon..value case */
+ assert(end_key != MDBX_EPSILON);
+ begin_key = end_key;
+ } else if (unlikely(end_key == MDBX_EPSILON)) {
+ /* LY: value..+epsilon case */
+ assert(begin_key != MDBX_EPSILON);
+ end_key = begin_key;
+ }
+ if (end_key && !begin_data && !end_data &&
+ (begin_key == end_key ||
+ begin.outer.mc_dbx->md_cmp(begin_key, end_key) == 0)) {
+ /* LY: single key case */
+ rc = mdbx_cursor_set(&begin.outer, begin_key, NULL, MDBX_SET).err;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ *size_items = 0;
+ return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
+ }
+ *size_items = 1;
+ if (begin.outer.mc_xcursor != NULL) {
+ MDBX_node *node = page_node(begin.outer.mc_pg[begin.outer.mc_top],
+ begin.outer.mc_ki[begin.outer.mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ /* LY: return the number of duplicates for given key */
+ mdbx_tassert(txn,
+ begin.outer.mc_xcursor == &begin.inner &&
+ (begin.inner.mx_cursor.mc_flags & C_INITIALIZED));
+ *size_items =
+ (sizeof(*size_items) >= sizeof(begin.inner.mx_db.md_entries) ||
+ begin.inner.mx_db.md_entries <= PTRDIFF_MAX)
+ ? (size_t)begin.inner.mx_db.md_entries
+ : PTRDIFF_MAX;
+ }
+ }
+ return MDBX_SUCCESS;
+ } else {
+ rc = mdbx_cursor_set(&begin.outer, begin_key, begin_data,
+ begin_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE)
+ .err;
+ }
+ }
+
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc != MDBX_NOTFOUND || !(begin.outer.mc_flags & C_INITIALIZED))
+ return rc;
+ }
-__cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
- int rc = check_env(env, true);
+ MDBX_cursor_couple end;
+ rc = mdbx_cursor_init(&end.outer, txn, dbi);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
+ if (!end_key) {
+ MDBX_val stub = {0, 0};
+ rc = mdbx_cursor_last(&end.outer, &stub, &stub);
+ } else {
+ rc = mdbx_cursor_set(&end.outer, end_key, end_data,
+ end_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE)
+ .err;
+ }
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc != MDBX_NOTFOUND || !(end.outer.mc_flags & C_INITIALIZED))
+ return rc;
+ }
- if (unlikely(!arg))
- return MDBX_EINVAL;
-
- *arg = env->me_pathname;
- return MDBX_SUCCESS;
-}
-
-__cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) {
- int rc = check_env(env, true);
+ rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
+ assert(*size_items >= -(ptrdiff_t)begin.outer.mc_db->md_entries &&
+ *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries);
- if (unlikely(!arg))
- return MDBX_EINVAL;
-
- *arg = env->me_lazy_fd;
- return MDBX_SUCCESS;
-}
-
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) {
- return __inline_mdbx_env_stat(env, stat, bytes);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
-
-static void stat_get(const MDBX_db *db, MDBX_stat *st, size_t bytes) {
- st->ms_depth = db->md_depth;
- st->ms_branch_pages = db->md_branch_pages;
- st->ms_leaf_pages = db->md_leaf_pages;
- st->ms_overflow_pages = db->md_overflow_pages;
- st->ms_entries = db->md_entries;
- if (likely(bytes >=
- offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
- st->ms_mod_txnid = db->md_mod_txnid;
-}
-
-static void stat_add(const MDBX_db *db, MDBX_stat *const st,
- const size_t bytes) {
- st->ms_depth += db->md_depth;
- st->ms_branch_pages += db->md_branch_pages;
- st->ms_leaf_pages += db->md_leaf_pages;
- st->ms_overflow_pages += db->md_overflow_pages;
- st->ms_entries += db->md_entries;
- if (likely(bytes >=
- offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
- st->ms_mod_txnid = (st->ms_mod_txnid > db->md_mod_txnid) ? st->ms_mod_txnid
- : db->md_mod_txnid;
-}
-
-__cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
- int err = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- st->ms_psize = txn->mt_env->me_psize;
-#if 1
- /* assuming GC is internal and not subject for accounting */
- stat_get(&txn->mt_dbs[MAIN_DBI], st, bytes);
-#else
- stat_get(&txn->mt_dbs[FREE_DBI], st, bytes);
- stat_add(&txn->mt_dbs[MAIN_DBI], st, bytes);
-#endif
-
- /* account opened named subDBs */
- for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++)
- if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID)
- stat_add(txn->mt_dbs + dbi, st, bytes);
-
- if (!(txn->mt_dbs[MAIN_DBI].md_flags & (MDBX_DUPSORT | MDBX_INTEGERKEY)) &&
- txn->mt_dbs[MAIN_DBI].md_entries /* TODO: use `md_subs` field */) {
- MDBX_cursor_couple cx;
- err = mdbx_cursor_init(&cx.outer, (MDBX_txn *)txn, MAIN_DBI);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- /* scan and account not opened named subDBs */
- err = mdbx_page_search(&cx.outer, NULL, MDBX_PS_FIRST);
- while (err == MDBX_SUCCESS) {
- const MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top];
- for (unsigned i = 0; i < page_numkeys(mp); i++) {
- const MDBX_node *node = page_node(mp, i);
- if (node_flags(node) != F_SUBDATA)
- continue;
- if (unlikely(node_ds(node) != sizeof(MDBX_db)))
- return MDBX_CORRUPTED;
+#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \
+ * results for an inverted ranges. */
- /* skip opened and already accounted */
- for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++)
- if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID &&
- node_ks(node) == txn->mt_dbxs[dbi].md_name.iov_len &&
- memcmp(node_key(node), txn->mt_dbxs[dbi].md_name.iov_base,
- node_ks(node)) == 0) {
- node = NULL;
- break;
- }
+ /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63
+ Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */
- if (node) {
- MDBX_db db;
- memcpy(&db, node_data(node), sizeof(db));
- stat_add(&db, st, bytes);
- }
- }
- err = mdbx_cursor_sibling(&cx.outer, SIBLING_RIGHT);
+ if (*size_items < 0) {
+ /* LY: inverted range case */
+ *size_items += (ptrdiff_t)begin.outer.mc_db->md_entries;
+ } else if (*size_items == 0 && begin_key && end_key) {
+ int cmp = begin.outer.mc_dbx->md_cmp(&origin_begin_key, &origin_end_key);
+ if (cmp == 0 && (begin.inner.mx_cursor.mc_flags & C_INITIALIZED) &&
+ begin_data && end_data)
+ cmp = begin.outer.mc_dbx->md_dcmp(&origin_begin_data, &origin_end_data);
+ if (cmp > 0) {
+ /* LY: inverted range case with empty scope */
+ *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries;
}
- if (unlikely(err != MDBX_NOTFOUND))
- return err;
}
+ assert(*size_items >= 0 &&
+ *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries);
+#endif
return MDBX_SUCCESS;
}
-__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn,
- MDBX_stat *dest, size_t bytes) {
- if (unlikely(!dest))
- return MDBX_EINVAL;
- const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
- if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
- return MDBX_EINVAL;
-
- if (likely(txn)) {
- if (env && unlikely(txn->mt_env != env))
- return MDBX_EINVAL;
- return stat_acc(txn, dest, bytes);
- }
-
- int err = check_env(env, true);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- if (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self())
- /* inside write-txn */
- return stat_acc(env->me_txn, dest, bytes);
-
- MDBX_txn *tmp_txn;
- err = mdbx_txn_begin((MDBX_env *)env, NULL, MDBX_TXN_RDONLY, &tmp_txn);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
+//------------------------------------------------------------------------------
- const int rc = stat_acc(tmp_txn, dest, bytes);
- err = mdbx_txn_abort(tmp_txn);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- return rc;
-}
+/* ПозволÑет обновить или удалить ÑущеÑтвующую запиÑÑŒ Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸ÐµÐ¼
+ * в old_data предыдущего Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ…. При Ñтом еÑли new_data равен
+ * нулю, то выполнÑетÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ, иначе обновление/вÑтавка.
+ *
+ * Текущее значение может находитьÑÑ Ð² уже измененной (грÑзной) Ñтранице.
+ * Ð’ Ñтом Ñлучае Ñтраница будет перезапиÑана при обновлении, а Ñамо Ñтарое
+ * значение утрачено. ПоÑтому иÑходно в old_data должен быть передан
+ * дополнительный буфер Ð´Ð»Ñ ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ñтарого значениÑ.
+ * ЕÑли переданный буфер Ñлишком мал, то Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð²ÐµÑ€Ð½ÐµÑ‚ -1, уÑтановив
+ * old_data->iov_len в ÑоответÑтвующее значение.
+ *
+ * Ð”Ð»Ñ Ð½Ðµ-уникальных ключей также возможен второй Ñценарий иÑпользованиÑ,
+ * когда поÑредÑтвом old_data из запиÑей Ñ Ð¾Ð´Ð¸Ð½Ð°ÐºÐ¾Ð²Ñ‹Ð¼ ключом длÑ
+ * удалениÑ/Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ð²Ñ‹Ð±Ð¸Ñ€Ð°ÐµÑ‚ÑÑ ÐºÐ¾Ð½ÐºÑ€ÐµÑ‚Ð½Ð°Ñ. Ð”Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° Ñтого ÑценариÑ
+ * во flags Ñледует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE.
+ * Именно Ñта ÐºÐ¾Ð¼Ð±Ð¸Ð½Ð°Ñ†Ð¸Ñ Ð²Ñ‹Ð±Ñ€Ð°Ð½Ð°, так как она лишена ÑмыÑла, и Ñтим позволÑет
+ * идентифицировать Ð·Ð°Ð¿Ñ€Ð¾Ñ Ñ‚Ð°ÐºÐ¾Ð³Ð¾ ÑценариÑ.
+ *
+ * Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¼Ð¾Ð¶ÐµÑ‚ быть замещена ÑоответÑтвующими операциÑми Ñ ÐºÑƒÑ€Ñорами
+ * поÑле двух доработок (TODO):
+ * - внешнÑÑ Ð°Ð»Ð»Ð¾ÐºÐ°Ñ†Ð¸Ñ ÐºÑƒÑ€Ñоров, в том чиÑле на Ñтеке (без malloc).
+ * - Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ dirty-ÑтатуÑа Ñтраницы по адреÑу (знать о MUTABLE/WRITEABLE).
+ */
-__cold int mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi,
- uint32_t *mask) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
+ MDBX_val *new_data, MDBX_val *old_data,
+ MDBX_put_flags_t flags, MDBX_preserve_func preserver,
+ void *preserver_context) {
+ int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!mask))
+ if (unlikely(!key || !old_data || old_data == new_data))
return MDBX_EINVAL;
- if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
- return MDBX_BAD_DBI;
-
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- if ((cx.outer.mc_db->md_flags & MDBX_DUPSORT) == 0)
- return MDBX_RESULT_TRUE;
-
- MDBX_val key, data;
- rc = mdbx_cursor_first(&cx.outer, &key, &data);
- *mask = 0;
- while (rc == MDBX_SUCCESS) {
- const MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top],
- cx.outer.mc_ki[cx.outer.mc_top]);
- const MDBX_db *db = node_data(node);
- const unsigned flags = node_flags(node);
- switch (flags) {
- case F_BIGDATA:
- case 0:
- /* single-value entry, deep = 0 */
- *mask |= 1 << 0;
- break;
- case F_DUPDATA:
- /* single sub-page, deep = 1 */
- *mask |= 1 << 1;
- break;
- case F_DUPDATA | F_SUBDATA:
- /* sub-tree */
- *mask |= 1 << UNALIGNED_PEEK_16(db, MDBX_db, md_depth);
- break;
- default:
- mdbx_error("wrong node-flags %u", flags);
- return MDBX_CORRUPTED;
- }
- rc = mdbx_cursor_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP);
- }
+ if (unlikely(old_data->iov_base == NULL && old_data->iov_len))
+ return MDBX_EINVAL;
- return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
-}
+ if (unlikely(new_data == NULL &&
+ (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT))
+ return MDBX_EINVAL;
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info,
- size_t bytes) {
- return __inline_mdbx_env_info(env, info, bytes);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
-__cold static int fetch_envinfo_ex(const MDBX_env *env, const MDBX_txn *txn,
- MDBX_envinfo *arg, const size_t bytes) {
+ if (unlikely(flags &
+ ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
+ MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT)))
+ return MDBX_EINVAL;
- const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
- const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
+ MDBX_cursor_couple cx;
+ rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ cx.outer.mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = &cx.outer;
- /* is the environment open?
- * (todo4recovery://erased_by_github/libmdbx/issues/171) */
- if (unlikely(!env->me_map)) {
- /* environment not yet opened */
-#if 1
- /* default behavior: returns the available info but zeroed the rest */
- memset(arg, 0, bytes);
- arg->mi_geo.lower = env->me_dbgeo.lower;
- arg->mi_geo.upper = env->me_dbgeo.upper;
- arg->mi_geo.shrink = env->me_dbgeo.shrink;
- arg->mi_geo.grow = env->me_dbgeo.grow;
- arg->mi_geo.current = env->me_dbgeo.now;
- arg->mi_maxreaders = env->me_maxreaders;
- arg->mi_dxb_pagesize = env->me_psize;
- arg->mi_sys_pagesize = env->me_os_psize;
- if (likely(bytes > size_before_bootid)) {
- arg->mi_bootid.current.x = bootid.x;
- arg->mi_bootid.current.y = bootid.y;
+ MDBX_val present_key = *key;
+ if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) {
+ /* в old_data значение Ð´Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° конкретного дубликата */
+ if (unlikely(!(txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT))) {
+ rc = MDBX_EINVAL;
+ goto bailout;
}
- return MDBX_SUCCESS;
-#else
- /* some users may prefer this behavior: return appropriate error */
- return MDBX_EPERM;
-#endif
- }
-
- const MDBX_meta *const meta0 = METAPAGE(env, 0);
- const MDBX_meta *const meta1 = METAPAGE(env, 1);
- const MDBX_meta *const meta2 = METAPAGE(env, 2);
- if (unlikely(env->me_flags & MDBX_FATAL_ERROR))
- return MDBX_PANIC;
- volatile const MDBX_meta *const recent_meta = meta_prefer_last(env);
- arg->mi_recent_txnid = meta_txnid(env, recent_meta);
- arg->mi_meta0_txnid = meta_txnid(env, meta0);
- arg->mi_meta0_sign = unaligned_peek_u64(4, meta0->mm_datasync_sign);
- arg->mi_meta1_txnid = meta_txnid(env, meta1);
- arg->mi_meta1_sign = unaligned_peek_u64(4, meta1->mm_datasync_sign);
- arg->mi_meta2_txnid = meta_txnid(env, meta2);
- arg->mi_meta2_sign = unaligned_peek_u64(4, meta2->mm_datasync_sign);
- if (likely(bytes > size_before_bootid)) {
- memcpy(&arg->mi_bootid.meta0, &meta0->mm_bootid, 16);
- memcpy(&arg->mi_bootid.meta1, &meta1->mm_bootid, 16);
- memcpy(&arg->mi_bootid.meta2, &meta2->mm_bootid, 16);
- }
+ /* убираем лишний бит, он был признаком запрошенного режима */
+ flags -= MDBX_NOOVERWRITE;
- volatile const MDBX_meta *txn_meta = recent_meta;
- arg->mi_last_pgno = txn_meta->mm_geo.next - 1;
- arg->mi_geo.current = pgno2bytes(env, txn_meta->mm_geo.now);
- if (txn) {
- arg->mi_last_pgno = txn->mt_next_pgno - 1;
- arg->mi_geo.current = pgno2bytes(env, txn->mt_end_pgno);
+ rc = mdbx_cursor_get(&cx.outer, &present_key, old_data, MDBX_GET_BOTH);
+ if (rc != MDBX_SUCCESS)
+ goto bailout;
+ } else {
+ /* в old_data буфер Ð´Ð»Ñ ÑÐ¾Ñ…Ñ€Ð°Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€ÐµÐ´Ñ‹Ð´ÑƒÑ‰ÐµÐ³Ð¾ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ */
+ if (unlikely(new_data && old_data->iov_base == new_data->iov_base))
+ return MDBX_EINVAL;
+ MDBX_val present_data;
+ rc = mdbx_cursor_get(&cx.outer, &present_key, &present_data, MDBX_SET_KEY);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ old_data->iov_base = NULL;
+ old_data->iov_len = 0;
+ if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT))
+ goto bailout;
+ } else if (flags & MDBX_NOOVERWRITE) {
+ rc = MDBX_KEYEXIST;
+ *old_data = present_data;
+ goto bailout;
+ } else {
+ MDBX_page *page = cx.outer.mc_pg[cx.outer.mc_top];
+ if (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) {
+ if (flags & MDBX_CURRENT) {
+ /* disallow update/delete for multi-values */
+ MDBX_node *node = page_node(page, cx.outer.mc_ki[cx.outer.mc_top]);
+ if (F_ISSET(node_flags(node), F_DUPDATA)) {
+ mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) &&
+ cx.outer.mc_xcursor->mx_db.md_entries > 1);
+ if (cx.outer.mc_xcursor->mx_db.md_entries > 1) {
+ rc = MDBX_EMULTIVAL;
+ goto bailout;
+ }
+ }
+ /* Ð’ оригинальной LMDB флажок MDBX_CURRENT здеÑÑŒ приведет
+ * к замене данных без учета MDBX_DUPSORT Ñортировки,
+ * но здеÑÑŒ Ñто в любом Ñлучае допуÑтимо, так как мы
+ * проверили что Ð´Ð»Ñ ÐºÐ»ÑŽÑ‡Ð° еÑÑ‚ÑŒ только одно значение. */
+ }
+ }
- const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_TXN_RDONLY)
- ? txn->mt_txnid
- : txn->mt_txnid - xMDBX_TXNID_STEP;
- txn_meta = (arg->mi_meta0_txnid == wanna_meta_txnid) ? meta0 : txn_meta;
- txn_meta = (arg->mi_meta1_txnid == wanna_meta_txnid) ? meta1 : txn_meta;
- txn_meta = (arg->mi_meta2_txnid == wanna_meta_txnid) ? meta2 : txn_meta;
+ if (IS_MODIFIABLE(txn, page)) {
+ if (new_data && cmp_lenfast(&present_data, new_data) == 0) {
+ /* еÑли данные Ñовпадают, то ничего делать не надо */
+ *old_data = *new_data;
+ goto bailout;
+ }
+ rc = preserver ? preserver(preserver_context, old_data,
+ present_data.iov_base, present_data.iov_len)
+ : MDBX_SUCCESS;
+ if (unlikely(rc != MDBX_SUCCESS))
+ goto bailout;
+ } else {
+ *old_data = present_data;
+ }
+ flags |= MDBX_CURRENT;
+ }
}
- arg->mi_geo.lower = pgno2bytes(env, txn_meta->mm_geo.lower);
- arg->mi_geo.upper = pgno2bytes(env, txn_meta->mm_geo.upper);
- arg->mi_geo.shrink = pgno2bytes(env, pv2pages(txn_meta->mm_geo.shrink_pv));
- arg->mi_geo.grow = pgno2bytes(env, pv2pages(txn_meta->mm_geo.grow_pv));
- const pgno_t unsynced_pages =
- atomic_load32(&env->me_lck->mti_unsynced_pages, mo_Relaxed) +
- (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) !=
- (uint32_t)arg->mi_recent_txnid);
- arg->mi_mapsize = env->me_dxb_mmap.limit;
+ if (likely(new_data))
+ rc = mdbx_cursor_put(&cx.outer, key, new_data, flags);
+ else
+ rc = mdbx_cursor_del(&cx.outer, flags & MDBX_ALLDUPS);
- const MDBX_lockinfo *const lck = env->me_lck;
- arg->mi_maxreaders = env->me_maxreaders;
- arg->mi_numreaders = env->me_lck_mmap.lck
- ? atomic_load32(&lck->mti_numreaders, mo_Relaxed)
- : INT32_MAX;
- arg->mi_dxb_pagesize = env->me_psize;
- arg->mi_sys_pagesize = env->me_os_psize;
+bailout:
+ txn->mt_cursors[dbi] = cx.outer.mc_next;
+ return rc;
+}
- if (likely(bytes > size_before_bootid)) {
- arg->mi_unsync_volume = pgno2bytes(env, unsynced_pages);
- const uint64_t monotime_now = mdbx_osal_monotime();
- uint64_t ts = atomic_load64(&lck->mti_sync_timestamp, mo_Relaxed);
- arg->mi_since_sync_seconds16dot16 =
- ts ? mdbx_osal_monotime_to_16dot16(monotime_now - ts) : 0;
- ts = atomic_load64(&lck->mti_reader_check_timestamp, mo_Relaxed);
- arg->mi_since_reader_check_seconds16dot16 =
- ts ? mdbx_osal_monotime_to_16dot16(monotime_now - ts) : 0;
- arg->mi_autosync_threshold = pgno2bytes(
- env, atomic_load32(&lck->mti_autosync_threshold, mo_Relaxed));
- arg->mi_autosync_period_seconds16dot16 = mdbx_osal_monotime_to_16dot16(
- atomic_load64(&lck->mti_autosync_period, mo_Relaxed));
- arg->mi_bootid.current.x = bootid.x;
- arg->mi_bootid.current.y = bootid.y;
- arg->mi_mode = env->me_lck_mmap.lck ? lck->mti_envmode.weak : env->me_flags;
+static int default_value_preserver(void *context, MDBX_val *target,
+ const void *src, size_t bytes) {
+ (void)context;
+ if (unlikely(target->iov_len < bytes)) {
+ target->iov_base = nullptr;
+ target->iov_len = bytes;
+ return MDBX_RESULT_TRUE;
}
+ memcpy(target->iov_base, src, target->iov_len = bytes);
+ return MDBX_SUCCESS;
+}
- if (likely(bytes > size_before_pgop_stat)) {
-#if MDBX_ENABLE_PGOP_STAT
- arg->mi_pgop_stat.newly =
- atomic_load64(&lck->mti_pgop_stat.newly, mo_Relaxed);
- arg->mi_pgop_stat.cow = atomic_load64(&lck->mti_pgop_stat.cow, mo_Relaxed);
- arg->mi_pgop_stat.clone =
- atomic_load64(&lck->mti_pgop_stat.clone, mo_Relaxed);
- arg->mi_pgop_stat.split =
- atomic_load64(&lck->mti_pgop_stat.split, mo_Relaxed);
- arg->mi_pgop_stat.merge =
- atomic_load64(&lck->mti_pgop_stat.merge, mo_Relaxed);
- arg->mi_pgop_stat.spill =
- atomic_load64(&lck->mti_pgop_stat.spill, mo_Relaxed);
- arg->mi_pgop_stat.unspill =
- atomic_load64(&lck->mti_pgop_stat.unspill, mo_Relaxed);
- arg->mi_pgop_stat.wops =
- atomic_load64(&lck->mti_pgop_stat.wops, mo_Relaxed);
-#else
- memset(&arg->mi_pgop_stat, 0, sizeof(arg->mi_pgop_stat));
-#endif /* MDBX_ENABLE_PGOP_STAT*/
- }
+int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
+ MDBX_val *new_data, MDBX_val *old_data,
+ MDBX_put_flags_t flags) {
+ return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags,
+ default_value_preserver, nullptr);
+}
- arg->mi_self_latter_reader_txnid = arg->mi_latter_reader_txnid =
- arg->mi_recent_txnid;
- for (unsigned i = 0; i < arg->mi_numreaders; ++i) {
- const uint32_t pid =
- atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease);
- if (pid) {
- const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (arg->mi_latter_reader_txnid > txnid)
- arg->mi_latter_reader_txnid = txnid;
- if (pid == env->me_pid && arg->mi_self_latter_reader_txnid > txnid)
- arg->mi_self_latter_reader_txnid = txnid;
+/* Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ñообщает находитÑÑ Ð»Ð¸ указанный Ð°Ð´Ñ€ÐµÑ Ð² "грÑзной" Ñтранице у
+ * заданной пишущей транзакции. Ð’ конечном Ñчете Ñто позволÑет избавитьÑÑ Ð¾Ñ‚
+ * лишнего ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ… из ÐЕ-грÑзных Ñтраниц.
+ *
+ * "ГрÑзные" Ñтраницы - Ñто те, которые уже были изменены в ходе пишущей
+ * транзакции. СоответÑтвенно, какие-либо дальнейшие Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð¼Ð¾Ð³ÑƒÑ‚ привеÑти
+ * к перезапиÑи таких Ñтраниц. ПоÑтому вÑе функции, выполнÑющие изменениÑ, в
+ * качеÑтве аргументов ÐЕ должны получать указатели на данные в таких
+ * Ñтраницах. Ð’ Ñвою очередь "ÐЕ грÑзные" Ñтраницы перед модификацией будут
+ * Ñкопированы.
+ *
+ * Другими Ñловами, данные из "грÑзных" Ñтраниц должны быть либо Ñкопированы
+ * перед передачей в качеÑтве аргументов Ð´Ð»Ñ Ð´Ð°Ð»ÑŒÐ½ÐµÐ¹ÑˆÐ¸Ñ… модификаций, либо
+ * отвергнуты на Ñтадии проверки корректноÑти аргументов.
+ *
+ * Таким образом, Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¿Ð¾Ð·Ð²Ð¾Ð»Ñет как избавитÑÑ Ð¾Ñ‚ лишнего копированиÑ,
+ * так и выполнить более полную проверку аргументов.
+ *
+ * Ð’ÐЖÐО: Передаваемый указатель должен указывать на начало данных. Только
+ * так гарантируетÑÑ Ñ‡Ñ‚Ð¾ актуальный заголовок Ñтраницы будет физичеÑки
+ * раÑположен в той-же Ñтранице памÑти, в том чиÑле Ð´Ð»Ñ Ð¼Ð½Ð¾Ð³Ð¾Ñтраничных
+ * P_OVERFLOW Ñтраниц Ñ Ð´Ð»Ð¸Ð½Ð½Ñ‹Ð¼Ð¸ данными. */
+int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+
+ const MDBX_env *env = txn->mt_env;
+ const ptrdiff_t offset = (uint8_t *)ptr - env->me_map;
+ if (offset >= 0) {
+ const pgno_t pgno = bytes2pgno(env, offset);
+ if (likely(pgno < txn->mt_next_pgno)) {
+ const MDBX_page *page = pgno2page(env, pgno);
+ if (unlikely(page->mp_pgno != pgno ||
+ (page->mp_flags & P_ILL_BITS) != 0)) {
+ /* The ptr pointed into middle of a large page,
+ * not to the beginning of a data. */
+ return MDBX_EINVAL;
+ }
+ return ((txn->mt_flags & MDBX_TXN_RDONLY) || !IS_MODIFIABLE(txn, page))
+ ? MDBX_RESULT_FALSE
+ : MDBX_RESULT_TRUE;
+ }
+ if ((size_t)offset < env->me_dxb_mmap.limit) {
+ /* Указатель адреÑует что-то в пределах mmap, но за границей
+ * раÑпределенных Ñтраниц. Такое может ÑлучитÑÑ ÐµÑли mdbx_is_dirty()
+ * вызываетÑÑ Ð¿Ð¾Ñле операции, в ходе которой грÑÐ·Ð½Ð°Ñ Ñтраница была
+ * возвращена в нераÑпределенное проÑтранÑтво. */
+ return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EINVAL : MDBX_RESULT_TRUE;
}
}
- mdbx_compiler_barrier();
- return MDBX_SUCCESS;
+ /* Страница вне иÑпользуемого mmap-диапазона, Ñ‚.е. либо в функцию был
+ * передан некорректный адреÑ, либо Ð°Ð´Ñ€ÐµÑ Ð² теневой Ñтранице, ÐºÐ¾Ñ‚Ð¾Ñ€Ð°Ñ Ð±Ñ‹Ð»Ð°
+ * выделена поÑредÑтвом malloc().
+ *
+ * Ð”Ð»Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð° MDBX_WRITE_MAP режима Ñтраница однозначно "не грÑзнаÑ",
+ * а Ð´Ð»Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ð² без MDBX_WRITE_MAP однозначно "не чиÑтаÑ". */
+ return (txn->mt_flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? MDBX_EINVAL
+ : MDBX_RESULT_TRUE;
}
-__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn,
- MDBX_envinfo *arg, size_t bytes) {
- if (unlikely((env == NULL && txn == NULL) || arg == NULL))
- return MDBX_EINVAL;
-
- if (txn) {
- int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- }
- if (env) {
- int err = check_env(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- if (txn && unlikely(txn->mt_env != env))
- return MDBX_EINVAL;
- } else {
- env = txn->mt_env;
- }
-
- const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
- const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
- if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid &&
- bytes != size_before_pgop_stat)
- return MDBX_EINVAL;
-
- MDBX_envinfo snap;
- int rc = fetch_envinfo_ex(env, txn, &snap, sizeof(snap));
+int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result,
+ uint64_t increment) {
+ int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- while (1) {
- rc = fetch_envinfo_ex(env, txn, arg, bytes);
+ if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
+ return MDBX_BAD_DBI;
+
+ if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) {
+ rc = mdbx_fetch_sdb(txn, dbi);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- snap.mi_since_sync_seconds16dot16 = arg->mi_since_sync_seconds16dot16;
- snap.mi_since_reader_check_seconds16dot16 =
- arg->mi_since_reader_check_seconds16dot16;
- if (likely(memcmp(&snap, arg, bytes) == 0))
- return MDBX_SUCCESS;
- memcpy(&snap, arg, bytes);
}
+
+ MDBX_db *dbs = &txn->mt_dbs[dbi];
+ if (likely(result))
+ *result = dbs->md_seq;
+
+ if (likely(increment > 0)) {
+ if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY))
+ return MDBX_EACCESS;
+
+ uint64_t new = dbs->md_seq + increment;
+ if (unlikely(new < increment))
+ return MDBX_RESULT_TRUE;
+
+ mdbx_tassert(txn, new > dbs->md_seq);
+ dbs->md_seq = new;
+ txn->mt_flags |= MDBX_TXN_DIRTY;
+ txn->mt_dbistate[dbi] |= DBI_DIRTY;
+ }
+
+ return MDBX_SUCCESS;
}
-static __inline MDBX_cmp_func *get_default_keycmp(unsigned flags) {
- return (flags & MDBX_REVERSEKEY) ? cmp_reverse
- : (flags & MDBX_INTEGERKEY) ? cmp_int_align2
- : cmp_lexical;
+/*----------------------------------------------------------------------------*/
+
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+__cold MDBX_NOTHROW_CONST_FUNCTION intptr_t mdbx_limits_pgsize_min(void) {
+ return __inline_mdbx_limits_pgsize_min();
}
-static __inline MDBX_cmp_func *get_default_datacmp(unsigned flags) {
- return !(flags & MDBX_DUPSORT)
- ? cmp_lenfast
- : ((flags & MDBX_INTEGERDUP)
- ? cmp_int_unaligned
- : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical));
+__cold MDBX_NOTHROW_CONST_FUNCTION intptr_t mdbx_limits_pgsize_max(void) {
+ return __inline_mdbx_limits_pgsize_max();
}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
-static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags,
- MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
- /* LY: so, accepting only three cases for the table's flags:
- * 1) user_flags and both comparators are zero
- * = assume that a by-default mode/flags is requested for reading;
- * 2) user_flags exactly the same
- * = assume that the target mode/flags are requested properly;
- * 3) user_flags differs, but table is empty and MDBX_CREATE is provided
- * = assume that a properly create request with custom flags;
- */
- if ((user_flags ^ txn->mt_dbs[dbi].md_flags) & DB_PERSISTENT_FLAGS) {
- /* flags are differs, check other conditions */
- if ((!user_flags && (!keycmp || keycmp == txn->mt_dbxs[dbi].md_cmp) &&
- (!datacmp || datacmp == txn->mt_dbxs[dbi].md_dcmp)) ||
- user_flags == MDBX_ACCEDE) {
- /* no comparators were provided and flags are zero,
- * seems that is case #1 above */
- user_flags = txn->mt_dbs[dbi].md_flags;
- } else if ((user_flags & MDBX_CREATE) && txn->mt_dbs[dbi].md_entries == 0) {
- if (txn->mt_flags & MDBX_TXN_RDONLY)
- return /* FIXME: return extended info */ MDBX_EACCESS;
- /* make sure flags changes get committed */
- txn->mt_dbs[dbi].md_flags = user_flags & DB_PERSISTENT_FLAGS;
- txn->mt_flags |= MDBX_TXN_DIRTY;
- } else {
- return /* FIXME: return extended info */ MDBX_INCOMPATIBLE;
- }
- }
+__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) {
+ if (pagesize < 1)
+ pagesize = (intptr_t)mdbx_default_pagesize();
+ else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
+ pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2((size_t)pagesize)))
+ return -1;
- if (!keycmp)
- keycmp = txn->mt_dbxs[dbi].md_cmp ? txn->mt_dbxs[dbi].md_cmp
- : get_default_keycmp(user_flags);
- if (txn->mt_dbxs[dbi].md_cmp != keycmp) {
- if (txn->mt_dbxs[dbi].md_cmp)
- return MDBX_EINVAL;
- txn->mt_dbxs[dbi].md_cmp = keycmp;
- }
+ return MIN_PAGENO * pagesize;
+}
- if (!datacmp)
- datacmp = txn->mt_dbxs[dbi].md_dcmp ? txn->mt_dbxs[dbi].md_dcmp
- : get_default_datacmp(user_flags);
- if (txn->mt_dbxs[dbi].md_dcmp != datacmp) {
- if (txn->mt_dbxs[dbi].md_dcmp)
- return MDBX_EINVAL;
- txn->mt_dbxs[dbi].md_dcmp = datacmp;
- }
+__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) {
+ if (pagesize < 1)
+ pagesize = (intptr_t)mdbx_default_pagesize();
+ else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
+ pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2((size_t)pagesize)))
+ return -1;
- return MDBX_SUCCESS;
+ STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
+ const uint64_t limit = (1 + (uint64_t)MAX_PAGENO) * pagesize;
+ return (limit < MAX_MAPSIZE) ? (intptr_t)limit : (intptr_t)MAX_MAPSIZE;
}
-static int dbi_open(MDBX_txn *txn, const char *table_name, unsigned user_flags,
- MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
- MDBX_cmp_func *datacmp) {
- int rc = MDBX_EINVAL;
- if (unlikely(!dbi))
- return rc;
+__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) {
+ if (pagesize < 1)
+ pagesize = (intptr_t)mdbx_default_pagesize();
+ else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
+ pagesize > (intptr_t)MAX_PAGESIZE ||
+ !is_powerof2((size_t)pagesize)))
+ return -1;
- if (unlikely((user_flags & ~DB_USABLE_FLAGS) != 0)) {
- early_bailout:
- *dbi = 0;
- return rc;
- }
+ STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
+ const uint64_t pgl_limit =
+ pagesize * (uint64_t)(MDBX_PGL_LIMIT / 1.6180339887498948482);
+ const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / 1.6180339887498948482);
+ return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit;
+}
- rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- goto early_bailout;
+/*** Key-making functions to avoid custom comparators *************************/
- switch (user_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT |
- MDBX_REVERSEDUP | MDBX_ACCEDE)) {
- case MDBX_ACCEDE:
- if ((user_flags & MDBX_CREATE) == 0)
- break;
- __fallthrough /* fall through */;
- default:
- rc = MDBX_EINVAL;
- goto early_bailout;
+static __always_inline double key2double(const int64_t key) {
+ union {
+ uint64_t u;
+ double f;
+ } casting;
- case MDBX_DUPSORT:
- case MDBX_DUPSORT | MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
- case 0:
- break;
+ casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000)
+ : UINT64_C(0xffffFFFFffffFFFF) - key;
+ return casting.f;
+}
+
+static __always_inline uint64_t double2key(const double *const ptr) {
+ STATIC_ASSERT(sizeof(double) == sizeof(int64_t));
+ const int64_t i = *(const int64_t *)ptr;
+ const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i
+ : i + UINT64_C(0x8000000000000000);
+ if (mdbx_assert_enabled()) {
+ const double f = key2double(u);
+ assert(memcmp(&f, ptr, 8) == 0);
}
+ return u;
+}
- /* main table? */
- if (!table_name) {
- rc = mdbx_dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp);
- if (unlikely(rc != MDBX_SUCCESS))
- goto early_bailout;
- *dbi = MAIN_DBI;
- return rc;
+static __always_inline float key2float(const int32_t key) {
+ union {
+ uint32_t u;
+ float f;
+ } casting;
+
+ casting.u =
+ (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key;
+ return casting.f;
+}
+
+static __always_inline uint32_t float2key(const float *const ptr) {
+ STATIC_ASSERT(sizeof(float) == sizeof(int32_t));
+ const int32_t i = *(const int32_t *)ptr;
+ const uint32_t u =
+ (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000);
+ if (mdbx_assert_enabled()) {
+ const float f = key2float(u);
+ assert(memcmp(&f, ptr, 4) == 0);
}
+ return u;
+}
- MDBX_env *env = txn->mt_env;
- size_t len = strlen(table_name);
- if (len > env->me_leaf_nodemax - NODESIZE - sizeof(MDBX_db))
- return MDBX_EINVAL;
+uint64_t mdbx_key_from_double(const double ieee754_64bit) {
+ return double2key(&ieee754_64bit);
+}
- if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) {
- txn->mt_dbxs[MAIN_DBI].md_cmp =
- get_default_keycmp(txn->mt_dbs[MAIN_DBI].md_flags);
- txn->mt_dbxs[MAIN_DBI].md_dcmp =
- get_default_datacmp(txn->mt_dbs[MAIN_DBI].md_flags);
- }
+uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) {
+ return double2key(ieee754_64bit);
+}
- /* Is the DB already open? */
- MDBX_dbi scan, slot;
- for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) {
- if (!txn->mt_dbxs[scan].md_name.iov_len) {
- /* Remember this free slot */
- slot = scan;
- continue;
- }
- if (len == txn->mt_dbxs[scan].md_name.iov_len &&
- !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) {
- rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp);
- if (unlikely(rc != MDBX_SUCCESS))
- goto early_bailout;
- *dbi = scan;
- return rc;
- }
- }
+uint32_t mdbx_key_from_float(const float ieee754_32bit) {
+ return float2key(&ieee754_32bit);
+}
- /* Fail, if no free slot and max hit */
- if (unlikely(slot >= env->me_maxdbs)) {
- rc = MDBX_DBS_FULL;
- goto early_bailout;
- }
+uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) {
+ return float2key(ieee754_32bit);
+}
- /* Cannot mix named table with some main-table flags */
- if (unlikely(txn->mt_dbs[MAIN_DBI].md_flags &
- (MDBX_DUPSORT | MDBX_INTEGERKEY))) {
- rc = (user_flags & MDBX_CREATE) ? MDBX_INCOMPATIBLE : MDBX_NOTFOUND;
- goto early_bailout;
- }
+#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
+MDBX_NOTHROW_CONST_FUNCTION uint64_t mdbx_key_from_int64(const int64_t i64) {
+ return __inline_mdbx_key_from_int64(i64);
+}
- /* Find the DB info */
- MDBX_val key, data;
- key.iov_len = len;
- key.iov_base = (void *)table_name;
- MDBX_cursor_couple couple;
- rc = mdbx_cursor_init(&couple.outer, txn, MAIN_DBI);
- if (unlikely(rc != MDBX_SUCCESS))
- goto early_bailout;
- rc = mdbx_cursor_set(&couple.outer, &key, &data, MDBX_SET).err;
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE))
- goto early_bailout;
- } else {
- /* make sure this is actually a table */
- MDBX_node *node = page_node(couple.outer.mc_pg[couple.outer.mc_top],
- couple.outer.mc_ki[couple.outer.mc_top]);
- if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) {
- rc = MDBX_INCOMPATIBLE;
- goto early_bailout;
- }
- if (!MDBX_DISABLE_PAGECHECKS && unlikely(data.iov_len != sizeof(MDBX_db))) {
- rc = MDBX_CORRUPTED;
- goto early_bailout;
- }
- }
+MDBX_NOTHROW_CONST_FUNCTION uint32_t mdbx_key_from_int32(const int32_t i32) {
+ return __inline_mdbx_key_from_int32(i32);
+}
+#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
- if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) {
- rc = MDBX_EACCESS;
- goto early_bailout;
- }
+#define IEEE754_DOUBLE_MANTISSA_SIZE 52
+#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF
+#define IEEE754_DOUBLE_EXPONENTA_MAX 0x7FF
+#define IEEE754_DOUBLE_IMPLICIT_LEAD UINT64_C(0x0010000000000000)
+#define IEEE754_DOUBLE_MANTISSA_MASK UINT64_C(0x000FFFFFFFFFFFFF)
+#define IEEE754_DOUBLE_MANTISSA_AMAX UINT64_C(0x001FFFFFFFFFFFFF)
- /* Done here so we cannot fail after creating a new DB */
- char *namedup = mdbx_strdup(table_name);
- if (unlikely(!namedup)) {
- rc = MDBX_ENOMEM;
- goto early_bailout;
- }
+static __inline int clz64(uint64_t value) {
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_clzl)
+ if (sizeof(value) == sizeof(int))
+ return __builtin_clz(value);
+ if (sizeof(value) == sizeof(long))
+ return __builtin_clzl(value);
+#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \
+ __has_builtin(__builtin_clzll)
+ return __builtin_clzll(value);
+#endif /* have(long long) && long long == uint64_t */
+#endif /* GNU C */
- int err = mdbx_fastmutex_acquire(&env->me_dbi_lock);
- if (unlikely(err != MDBX_SUCCESS)) {
- rc = err;
- mdbx_free(namedup);
- goto early_bailout;
+#if defined(_MSC_VER)
+ unsigned long index;
+#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64)
+ _BitScanReverse64(&index, value);
+ return 63 - index;
+#else
+ if (value > UINT32_MAX) {
+ _BitScanReverse(&index, (uint32_t)(value >> 32));
+ return 31 - index;
}
+ _BitScanReverse(&index, (uint32_t)value);
+ return 63 - index;
+#endif
+#endif /* MSVC */
- /* Import handles from env */
- dbi_import_locked(txn);
+ value |= value >> 1;
+ value |= value >> 2;
+ value |= value >> 4;
+ value |= value >> 8;
+ value |= value >> 16;
+ value |= value >> 32;
+ static const uint8_t debruijn_clz64[64] = {
+ 63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2,
+ 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1,
+ 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18,
+ 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0};
+ return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58];
+}
- /* Rescan after mutex acquisition & import handles */
- for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) {
- if (!txn->mt_dbxs[scan].md_name.iov_len) {
- /* Remember this free slot */
- slot = scan;
- continue;
- }
- if (len == txn->mt_dbxs[scan].md_name.iov_len &&
- !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) {
- rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp);
- if (unlikely(rc != MDBX_SUCCESS))
- goto later_bailout;
- *dbi = scan;
- goto later_exit;
+static __inline uint64_t round_mantissa(const uint64_t u64, int shift) {
+ assert(shift < 0 && u64 > 0);
+ shift = -shift;
+ const unsigned half = 1 << (shift - 1);
+ const unsigned lsb = 1 & (unsigned)(u64 >> shift);
+ const unsigned tie2even = 1 ^ lsb;
+ return (u64 + half - tie2even) >> shift;
+}
+
+uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) {
+ const uint64_t bias = UINT64_C(0x8000000000000000);
+ if (json_integer > 0) {
+ const uint64_t u64 = json_integer;
+ int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
+ uint64_t mantissa = u64 << shift;
+ if (unlikely(shift < 0)) {
+ mantissa = round_mantissa(u64, shift);
+ if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
+ mantissa = round_mantissa(u64, --shift);
}
+
+ assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
+ mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
+ const uint64_t exponent =
+ IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
+ assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
+ const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) +
+ (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
+#if !defined(_MSC_VER) || \
+ defined( \
+ _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
+ symbol __except1 referenced in function __ftol3_except */
+ assert(key == mdbx_key_from_double((double)json_integer));
+#endif /* Workaround for MSVC */
+ return key;
}
- if (unlikely(slot >= env->me_maxdbs)) {
- rc = MDBX_DBS_FULL;
- goto later_bailout;
+ if (json_integer < 0) {
+ const uint64_t u64 = -json_integer;
+ int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
+ uint64_t mantissa = u64 << shift;
+ if (unlikely(shift < 0)) {
+ mantissa = round_mantissa(u64, shift);
+ if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
+ mantissa = round_mantissa(u64, --shift);
+ }
+
+ assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
+ mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
+ const uint64_t exponent =
+ IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
+ assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
+ const uint64_t key = bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) -
+ (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
+#if !defined(_MSC_VER) || \
+ defined( \
+ _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
+ symbol __except1 referenced in function __ftol3_except */
+ assert(key == mdbx_key_from_double((double)json_integer));
+#endif /* Workaround for MSVC */
+ return key;
}
- unsigned dbiflags = DBI_FRESH | DBI_VALID | DBI_USRVALID;
- MDBX_db db_dummy;
- if (unlikely(rc)) {
- /* MDBX_NOTFOUND and MDBX_CREATE: Create new DB */
- mdbx_tassert(txn, rc == MDBX_NOTFOUND);
- memset(&db_dummy, 0, sizeof(db_dummy));
- db_dummy.md_root = P_INVALID;
- db_dummy.md_mod_txnid = txn->mt_txnid;
- db_dummy.md_flags = user_flags & DB_PERSISTENT_FLAGS;
- data.iov_len = sizeof(db_dummy);
- data.iov_base = &db_dummy;
- WITH_CURSOR_TRACKING(couple.outer,
- rc = mdbx_cursor_put(&couple.outer, &key, &data,
- F_SUBDATA | MDBX_NOOVERWRITE));
+ return bias;
+}
+
+int64_t mdbx_jsonInteger_from_key(const MDBX_val v) {
+ assert(v.iov_len == 8);
+ const uint64_t key = unaligned_peek_u64(2, v.iov_base);
+ const uint64_t bias = UINT64_C(0x8000000000000000);
+ const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1;
+ const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 -
+ (IEEE754_DOUBLE_EXPONENTA_MAX &
+ (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE));
+ if (unlikely(shift < 1))
+ return (key < bias) ? INT64_MIN : INT64_MAX;
+ if (unlikely(shift > 63))
+ return 0;
+
+ const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK)
+ << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) +
+ bias;
+ const int64_t absolute = unscaled >> shift;
+ const int64_t value = (key < bias) ? -absolute : absolute;
+ assert(key == mdbx_key_from_jsonInteger(value) ||
+ (mdbx_key_from_jsonInteger(value - 1) < key &&
+ key < mdbx_key_from_jsonInteger(value + 1)));
+ return value;
+}
- if (unlikely(rc != MDBX_SUCCESS))
- goto later_bailout;
+double mdbx_double_from_key(const MDBX_val v) {
+ assert(v.iov_len == 8);
+ return key2double(unaligned_peek_u64(2, v.iov_base));
+}
- dbiflags |= DBI_DIRTY | DBI_CREAT;
- txn->mt_flags |= MDBX_TXN_DIRTY;
- mdbx_tassert(txn, (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY) != 0);
- }
+float mdbx_float_from_key(const MDBX_val v) {
+ assert(v.iov_len == 4);
+ return key2float(unaligned_peek_u32(2, v.iov_base));
+}
- /* Got info, register DBI in this txn */
- memset(txn->mt_dbxs + slot, 0, sizeof(MDBX_dbx));
- memcpy(&txn->mt_dbs[slot], data.iov_base, sizeof(MDBX_db));
- env->me_dbflags[slot] = 0;
- rc = mdbx_dbi_bind(txn, slot, user_flags, keycmp, datacmp);
- if (unlikely(rc != MDBX_SUCCESS)) {
- mdbx_tassert(txn, (dbiflags & DBI_CREAT) == 0);
- later_bailout:
- *dbi = 0;
- later_exit:
- mdbx_free(namedup);
- } else {
- txn->mt_dbistate[slot] = (uint8_t)dbiflags;
- txn->mt_dbxs[slot].md_name.iov_base = namedup;
- txn->mt_dbxs[slot].md_name.iov_len = len;
- txn->mt_dbiseqs[slot] = ++env->me_dbiseqs[slot];
- if (!(dbiflags & DBI_CREAT))
- env->me_dbflags[slot] = txn->mt_dbs[slot].md_flags | DB_VALID;
- if (txn->mt_numdbs == slot) {
- mdbx_compiler_barrier();
- txn->mt_numdbs = env->me_numdbs = slot + 1;
- txn->mt_cursors[slot] = NULL;
- }
- mdbx_assert(env, env->me_numdbs > slot);
- *dbi = slot;
- }
+int32_t mdbx_int32_from_key(const MDBX_val v) {
+ assert(v.iov_len == 4);
+ return (int32_t)(unaligned_peek_u32(2, v.iov_base) - UINT32_C(0x80000000));
+}
- mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
- return rc;
+int64_t mdbx_int64_from_key(const MDBX_val v) {
+ assert(v.iov_len == 8);
+ return (int64_t)(unaligned_peek_u64(2, v.iov_base) -
+ UINT64_C(0x8000000000000000));
}
-int mdbx_dbi_open(MDBX_txn *txn, const char *table_name,
- MDBX_db_flags_t table_flags, MDBX_dbi *dbi) {
- return dbi_open(txn, table_name, table_flags, dbi, nullptr, nullptr);
+__cold MDBX_cmp_func *mdbx_get_keycmp(unsigned flags) {
+ return get_default_keycmp(flags);
}
-int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name,
- MDBX_db_flags_t table_flags, MDBX_dbi *dbi,
- MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
- return dbi_open(txn, table_name, table_flags, dbi, keycmp, datacmp);
+__cold MDBX_cmp_func *mdbx_get_datacmp(unsigned flags) {
+ return get_default_datacmp(flags);
}
-__cold int mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest,
- size_t bytes) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
+ uint64_t value) {
+ int err = check_env(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- if (unlikely(!dest))
- return MDBX_EINVAL;
+ const bool lock_needed = ((env->me_flags & MDBX_ENV_ACTIVE) && env->me_txn0 &&
+ env->me_txn0->mt_owner != mdbx_thread_self());
+ bool should_unlock = false;
+ switch (option) {
+ case MDBX_opt_sync_bytes:
+ if (value == UINT64_MAX)
+ value = SIZE_MAX - 65536;
+ if (unlikely(env->me_flags & MDBX_RDONLY))
+ return MDBX_EACCESS;
+ if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
+ return MDBX_EPERM;
+ if (unlikely(value > SIZE_MAX - 65536))
+ return MDBX_TOO_LARGE;
+ if (atomic_store32(&env->me_lck->mti_autosync_threshold,
+ bytes2pgno(env, (size_t)value + env->me_psize - 1),
+ mo_Relaxed) != 0 &&
+ (env->me_flags & MDBX_ENV_ACTIVE)) {
+ err = mdbx_env_sync_poll(env);
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+ err = MDBX_SUCCESS;
+ }
+ break;
- if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
- return MDBX_BAD_DBI;
+ case MDBX_opt_sync_period:
+ if (value == UINT64_MAX)
+ value = UINT32_MAX;
+ if (unlikely(env->me_flags & MDBX_RDONLY))
+ return MDBX_EACCESS;
+ if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
+ return MDBX_EPERM;
+ if (unlikely(value > UINT32_MAX))
+ return MDBX_TOO_LARGE;
+ if (atomic_store64(&env->me_lck->mti_autosync_period,
+ mdbx_osal_16dot16_to_monotime((uint32_t)value),
+ mo_Relaxed) != 0 &&
+ (env->me_flags & MDBX_ENV_ACTIVE)) {
+ err = mdbx_env_sync_poll(env);
+ if (unlikely(MDBX_IS_ERROR(err)))
+ return err;
+ err = MDBX_SUCCESS;
+ }
+ break;
- const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
- if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
- return MDBX_EINVAL;
+ case MDBX_opt_max_db:
+ if (value == UINT64_MAX)
+ value = MDBX_MAX_DBI;
+ if (unlikely(value > MDBX_MAX_DBI))
+ return MDBX_EINVAL;
+ if (unlikely(env->me_map))
+ return MDBX_EPERM;
+ env->me_maxdbs = (unsigned)value + CORE_DBS;
+ break;
- if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED))
- return MDBX_BAD_TXN;
+ case MDBX_opt_max_readers:
+ if (value == UINT64_MAX)
+ value = MDBX_READERS_LIMIT;
+ if (unlikely(value < 1 || value > MDBX_READERS_LIMIT))
+ return MDBX_EINVAL;
+ if (unlikely(env->me_map))
+ return MDBX_EPERM;
+ env->me_maxreaders = (unsigned)value;
+ break;
- if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) {
- rc = mdbx_fetch_sdb(txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- }
+ case MDBX_opt_dp_reserve_limit:
+ if (value == UINT64_MAX)
+ value = INT_MAX;
+ if (unlikely(value > INT_MAX))
+ return MDBX_EINVAL;
+ if (env->me_options.dp_reserve_limit != (unsigned)value) {
+ if (lock_needed) {
+ err = mdbx_txn_lock(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ should_unlock = true;
+ }
+ env->me_options.dp_reserve_limit = (unsigned)value;
+ while (env->me_dp_reserve_len > env->me_options.dp_reserve_limit) {
+ mdbx_assert(env, env->me_dp_reserve != NULL);
+ MDBX_page *dp = env->me_dp_reserve;
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize);
+ VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
+ env->me_dp_reserve = dp->mp_next;
+ VALGRIND_MEMPOOL_FREE(env, dp);
+ mdbx_free(dp);
+ env->me_dp_reserve_len -= 1;
+ }
+ }
+ break;
- dest->ms_psize = txn->mt_env->me_psize;
- stat_get(&txn->mt_dbs[dbi], dest, bytes);
- return MDBX_SUCCESS;
-}
+ case MDBX_opt_rp_augment_limit:
+ if (value == UINT64_MAX)
+ value = MDBX_PGL_LIMIT;
+ if (unlikely(value > MDBX_PGL_LIMIT))
+ return MDBX_EINVAL;
+ env->me_options.rp_augment_limit = (unsigned)value;
+ break;
-static int mdbx_dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) {
- mdbx_assert(env, dbi >= CORE_DBS);
- if (unlikely(dbi >= env->me_numdbs))
- return MDBX_BAD_DBI;
+ case MDBX_opt_txn_dp_limit:
+ case MDBX_opt_txn_dp_initial:
+ if (value == UINT64_MAX)
+ value = MDBX_PGL_LIMIT;
+ if (unlikely(value > MDBX_PGL_LIMIT || value < CURSOR_STACK * 4))
+ return MDBX_EINVAL;
+ if (unlikely(env->me_flags & MDBX_RDONLY))
+ return MDBX_EACCESS;
+ if (lock_needed) {
+ err = mdbx_txn_lock(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ should_unlock = true;
+ }
+ if (env->me_txn)
+ err = MDBX_EPERM /* unable change during transaction */;
+ else {
+ const pgno_t value32 = (pgno_t)value;
+ if (option == MDBX_opt_txn_dp_initial &&
+ env->me_options.dp_initial != value32) {
+ env->me_options.dp_initial = value32;
+ if (env->me_options.dp_limit < value32) {
+ env->me_options.dp_limit = value32;
+ env->me_options.flags.non_auto.dp_limit = 1;
+ }
+ }
+ if (option == MDBX_opt_txn_dp_limit &&
+ env->me_options.dp_limit != value32) {
+ env->me_options.dp_limit = value32;
+ env->me_options.flags.non_auto.dp_limit = 1;
+ if (env->me_options.dp_initial > value32)
+ env->me_options.dp_initial = value32;
+ }
+ }
+ break;
- char *ptr = env->me_dbxs[dbi].md_name.iov_base;
- /* If there was no name, this was already closed */
- if (unlikely(!ptr))
- return MDBX_BAD_DBI;
+ case MDBX_opt_spill_max_denominator:
+ if (value == UINT64_MAX)
+ value = 255;
+ if (unlikely(value > 255))
+ return MDBX_EINVAL;
+ env->me_options.spill_max_denominator = (uint8_t)value;
+ break;
+ case MDBX_opt_spill_min_denominator:
+ if (unlikely(value > 255))
+ return MDBX_EINVAL;
+ env->me_options.spill_min_denominator = (uint8_t)value;
+ break;
+ case MDBX_opt_spill_parent4child_denominator:
+ if (unlikely(value > 255))
+ return MDBX_EINVAL;
+ env->me_options.spill_parent4child_denominator = (uint8_t)value;
+ break;
+
+ case MDBX_opt_loose_limit:
+ if (value == UINT64_MAX)
+ value = 255;
+ if (unlikely(value > 255))
+ return MDBX_EINVAL;
+ env->me_options.dp_loose_limit = (uint8_t)value;
+ break;
- env->me_dbflags[dbi] = 0;
- env->me_dbiseqs[dbi]++;
- env->me_dbxs[dbi].md_name.iov_len = 0;
- mdbx_memory_fence(mo_AcquireRelease, true);
- env->me_dbxs[dbi].md_name.iov_base = NULL;
- mdbx_free(ptr);
+ case MDBX_opt_merge_threshold_16dot16_percent:
+ if (value == UINT64_MAX)
+ value = 32768;
+ if (unlikely(value < 8192 || value > 32768))
+ return MDBX_EINVAL;
+ env->me_options.merge_threshold_16dot16_percent = (unsigned)value;
+ recalculate_merge_threshold(env);
+ break;
- if (env->me_numdbs == dbi + 1) {
- unsigned i = env->me_numdbs;
- do
- --i;
- while (i > CORE_DBS && !env->me_dbxs[i - 1].md_name.iov_base);
- env->me_numdbs = i;
+ default:
+ return MDBX_EINVAL;
}
- return MDBX_SUCCESS;
+ if (should_unlock)
+ mdbx_txn_unlock(env);
+ return err;
}
-int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
- int rc = check_env(env, true);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
-
- if (unlikely(dbi < CORE_DBS || dbi >= env->me_maxdbs))
- return MDBX_BAD_DBI;
+__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option,
+ uint64_t *pvalue) {
+ int err = check_env(env, false);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+ if (unlikely(!pvalue))
+ return MDBX_EINVAL;
- rc = mdbx_fastmutex_acquire(&env->me_dbi_lock);
- if (likely(rc == MDBX_SUCCESS)) {
- rc = (dbi < env->me_maxdbs && (env->me_dbflags[dbi] & DB_VALID))
- ? mdbx_dbi_close_locked(env, dbi)
- : MDBX_BAD_DBI;
- mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
- }
- return rc;
-}
+ switch (option) {
+ case MDBX_opt_sync_bytes:
+ if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
+ return MDBX_EPERM;
+ *pvalue = pgno2bytes(
+ env, atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed));
+ break;
-int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags,
- unsigned *state) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ case MDBX_opt_sync_period:
+ if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
+ return MDBX_EPERM;
+ *pvalue = mdbx_osal_monotime_to_16dot16(
+ atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed));
+ break;
- if (unlikely(!flags || !state))
- return MDBX_EINVAL;
+ case MDBX_opt_max_db:
+ *pvalue = env->me_maxdbs - CORE_DBS;
+ break;
- if (unlikely(!check_dbi(txn, dbi, DBI_VALID)))
- return MDBX_BAD_DBI;
+ case MDBX_opt_max_readers:
+ *pvalue = env->me_maxreaders;
+ break;
- *flags = txn->mt_dbs[dbi].md_flags & DB_PERSISTENT_FLAGS;
- *state =
- txn->mt_dbistate[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE);
+ case MDBX_opt_dp_reserve_limit:
+ *pvalue = env->me_options.dp_reserve_limit;
+ break;
- return MDBX_SUCCESS;
-}
+ case MDBX_opt_rp_augment_limit:
+ *pvalue = env->me_options.rp_augment_limit;
+ break;
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-int mdbx_dbi_flags(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) {
- return __inline_mdbx_dbi_flags(txn, dbi, flags);
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+ case MDBX_opt_txn_dp_limit:
+ *pvalue = env->me_options.dp_limit;
+ break;
+ case MDBX_opt_txn_dp_initial:
+ *pvalue = env->me_options.dp_initial;
+ break;
-static int mdbx_drop_tree(MDBX_cursor *mc, const bool may_have_subDBs) {
- int rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
- if (likely(rc == MDBX_SUCCESS)) {
- MDBX_txn *txn = mc->mc_txn;
+ case MDBX_opt_spill_max_denominator:
+ *pvalue = env->me_options.spill_max_denominator;
+ break;
+ case MDBX_opt_spill_min_denominator:
+ *pvalue = env->me_options.spill_min_denominator;
+ break;
+ case MDBX_opt_spill_parent4child_denominator:
+ *pvalue = env->me_options.spill_parent4child_denominator;
+ break;
- /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
- * This also avoids any P_LEAF2 pages, which have no nodes.
- * Also if the DB doesn't have sub-DBs and has no overflow
- * pages, omit scanning leaves. */
- if (!(may_have_subDBs | mc->mc_db->md_overflow_pages))
- mdbx_cursor_pop(mc);
+ case MDBX_opt_loose_limit:
+ *pvalue = env->me_options.dp_loose_limit;
+ break;
- rc = mdbx_pnl_need(&txn->tw.retired_pages,
- mc->mc_db->md_branch_pages + mc->mc_db->md_leaf_pages +
- mc->mc_db->md_overflow_pages);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
+ case MDBX_opt_merge_threshold_16dot16_percent:
+ *pvalue = env->me_options.merge_threshold_16dot16_percent;
+ break;
- MDBX_cursor mx;
- cursor_copy(mc, &mx);
- while (mc->mc_snum > 0) {
- MDBX_page *const mp = mc->mc_pg[mc->mc_top];
- const unsigned nkeys = page_numkeys(mp);
- if (IS_LEAF(mp)) {
- mdbx_cassert(mc, mc->mc_snum == mc->mc_db->md_depth);
- for (unsigned i = 0; i < nkeys; i++) {
- MDBX_node *node = page_node(mp, i);
- if (node_flags(node) & F_BIGDATA) {
- rc = mdbx_page_retire_ex(mc, node_largedata_pgno(node), NULL, 0);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- if (!(may_have_subDBs | mc->mc_db->md_overflow_pages))
- goto pop;
- } else if (node_flags(node) & F_SUBDATA) {
- if (unlikely((node_flags(node) & F_DUPDATA) == 0)) {
- rc = /* disallowing implicit subDB deletion */ MDBX_INCOMPATIBLE;
- goto bailout;
- }
- rc = mdbx_xcursor_init1(mc, node, mp);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- rc = mdbx_drop_tree(&mc->mc_xcursor->mx_cursor, false);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- }
- } else {
- mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth);
- if (mdbx_audit_enabled())
- mc->mc_flags |= C_RETIRING;
- const int pagetype =
- (IS_FROZEN(txn, mp) ? P_FROZEN : 0) +
- ((mc->mc_snum + 1 == mc->mc_db->md_depth) ? P_LEAF : P_BRANCH);
- for (unsigned i = 0; i < nkeys; i++) {
- MDBX_node *node = page_node(mp, i);
- mdbx_tassert(txn, (node_flags(node) &
- (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0);
- const pgno_t pgno = node_pgno(node);
- rc = mdbx_page_retire_ex(mc, pgno, NULL, pagetype);
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- }
- if (mdbx_audit_enabled())
- mc->mc_flags -= C_RETIRING;
- }
- if (!mc->mc_top)
- break;
- mdbx_cassert(mc, nkeys > 0);
- mc->mc_ki[mc->mc_top] = (indx_t)nkeys;
- rc = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (unlikely(rc != MDBX_NOTFOUND))
- goto bailout;
- /* no more siblings, go back to beginning
- * of previous level. */
- pop:
- mdbx_cursor_pop(mc);
- mc->mc_ki[0] = 0;
- for (unsigned i = 1; i < mc->mc_snum; i++) {
- mc->mc_ki[i] = 0;
- mc->mc_pg[i] = mx.mc_pg[i];
- }
- }
- }
- rc = mdbx_page_retire(mc, mc->mc_pg[0]);
- bailout:
- if (unlikely(rc != MDBX_SUCCESS))
- txn->mt_flags |= MDBX_TXN_ERROR;
- } else if (rc == MDBX_NOTFOUND) {
- rc = MDBX_SUCCESS;
+ default:
+ return MDBX_EINVAL;
}
- mc->mc_flags &= ~C_INITIALIZED;
- return rc;
+
+ return MDBX_SUCCESS;
}
-int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) {
- int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/*** Attribute support functions for Nexenta **********************************/
+#ifdef MDBX_NEXENTA_ATTRS
- MDBX_cursor *mc;
- rc = mdbx_cursor_open(txn, dbi, &mc);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+static __inline int mdbx_attr_peek(MDBX_val *data, mdbx_attr_t *attrptr) {
+ if (unlikely(data->iov_len < sizeof(mdbx_attr_t)))
+ return MDBX_INCOMPATIBLE;
- rc = mdbx_drop_tree(mc, dbi == MAIN_DBI ||
- (mc->mc_db->md_flags & MDBX_DUPSORT) != 0);
- /* Invalidate the dropped DB's cursors */
- for (MDBX_cursor *m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next)
- m2->mc_flags &= ~(C_INITIALIZED | C_EOF);
- if (unlikely(rc))
- goto bailout;
+ if (likely(attrptr != NULL))
+ *attrptr = *(mdbx_attr_t *)data->iov_base;
+ data->iov_len -= sizeof(mdbx_attr_t);
+ data->iov_base =
+ likely(data->iov_len > 0) ? ((mdbx_attr_t *)data->iov_base) + 1 : NULL;
- /* Can't delete the main DB */
- if (del && dbi >= CORE_DBS) {
- rc = mdbx_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA);
- if (likely(rc == MDBX_SUCCESS)) {
- mdbx_tassert(txn, txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY);
- mdbx_tassert(txn, txn->mt_flags & MDBX_TXN_DIRTY);
- txn->mt_dbistate[dbi] = DBI_STALE;
- MDBX_env *env = txn->mt_env;
- rc = mdbx_fastmutex_acquire(&env->me_dbi_lock);
- if (unlikely(rc != MDBX_SUCCESS)) {
- txn->mt_flags |= MDBX_TXN_ERROR;
- goto bailout;
- }
- mdbx_dbi_close_locked(env, dbi);
- mdbx_ensure(env,
- mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
- } else {
- txn->mt_flags |= MDBX_TXN_ERROR;
+ return MDBX_SUCCESS;
+}
+
+static __inline int mdbx_attr_poke(MDBX_val *reserved, MDBX_val *data,
+ mdbx_attr_t attr, MDBX_put_flags_t flags) {
+ mdbx_attr_t *space = reserved->iov_base;
+ if (flags & MDBX_RESERVE) {
+ if (likely(data != NULL)) {
+ data->iov_base = data->iov_len ? space + 1 : NULL;
}
} else {
- /* reset the DB record, mark it dirty */
- txn->mt_dbistate[dbi] |= DBI_DIRTY;
- txn->mt_dbs[dbi].md_depth = 0;
- txn->mt_dbs[dbi].md_branch_pages = 0;
- txn->mt_dbs[dbi].md_leaf_pages = 0;
- txn->mt_dbs[dbi].md_overflow_pages = 0;
- txn->mt_dbs[dbi].md_entries = 0;
- txn->mt_dbs[dbi].md_root = P_INVALID;
- txn->mt_dbs[dbi].md_seq = 0;
- /* txn->mt_dbs[dbi].md_mod_txnid = txn->mt_txnid; */
- txn->mt_flags |= MDBX_TXN_DIRTY;
+ *space = attr;
+ if (likely(data != NULL)) {
+ memcpy(space + 1, data->iov_base, data->iov_len);
+ }
}
-bailout:
- mdbx_cursor_close(mc);
- return rc;
+ return MDBX_SUCCESS;
}
-int mdbx_set_compare(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
+ mdbx_attr_t *attrptr, MDBX_cursor_op op) {
+ int rc = mdbx_cursor_get(mc, key, data, op);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
-
- txn->mt_dbxs[dbi].md_cmp = cmp;
- return MDBX_SUCCESS;
+ return mdbx_attr_peek(data, attrptr);
}
-int mdbx_set_dupsort(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
+int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
+ uint64_t *attrptr) {
+ int rc = mdbx_get(txn, dbi, key, data);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
-
- txn->mt_dbxs[dbi].md_dcmp = cmp;
- return MDBX_SUCCESS;
+ return mdbx_attr_peek(data, attrptr);
}
-__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
- void *ctx) {
- int rc = check_env(env, true);
+int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
+ mdbx_attr_t attr, MDBX_put_flags_t flags) {
+ MDBX_val reserve;
+ reserve.iov_base = NULL;
+ reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t);
+
+ int rc = mdbx_put(txn, dbi, key, &reserve, flags | MDBX_RESERVE);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
- if (unlikely(!func))
- return MDBX_EINVAL;
+ return mdbx_attr_poke(&reserve, data, attr, flags);
+}
- rc = MDBX_RESULT_TRUE;
- int serial = 0;
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (likely(lck)) {
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; i++) {
- const MDBX_reader *r = lck->mti_readers + i;
- retry_reader:;
- const uint32_t pid = atomic_load32(&r->mr_pid, mo_AcquireRelease);
- if (!pid)
- continue;
- txnid_t txnid = safe64_read(&r->mr_txnid);
- const uint64_t tid = atomic_load64(&r->mr_tid, mo_Relaxed);
- const pgno_t pages_used =
- atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed);
- const uint64_t reader_pages_retired =
- atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed);
- if (unlikely(
- txnid != safe64_read(&r->mr_txnid) ||
- pid != atomic_load32(&r->mr_pid, mo_AcquireRelease) ||
- tid != atomic_load64(&r->mr_tid, mo_Relaxed) ||
- pages_used !=
- atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed) ||
- reader_pages_retired !=
- atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed)))
- goto retry_reader;
+int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data,
+ mdbx_attr_t attr, MDBX_put_flags_t flags) {
+ MDBX_val reserve;
+ reserve.iov_base = NULL;
+ reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t);
- mdbx_assert(env, txnid > 0);
- if (txnid >= SAFE64_INVALID_THRESHOLD)
- txnid = 0;
+ int rc = mdbx_cursor_put(cursor, key, &reserve, flags | MDBX_RESERVE);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- size_t bytes_used = 0;
- size_t bytes_retained = 0;
- uint64_t lag = 0;
- if (txnid) {
- retry_header:;
- volatile const MDBX_meta *const recent_meta = meta_prefer_last(env);
- const uint64_t head_pages_retired =
- unaligned_peek_u64_volatile(4, recent_meta->mm_pages_retired);
- const txnid_t head_txnid = meta_txnid(env, recent_meta);
- mdbx_compiler_barrier();
- if (unlikely(recent_meta != meta_prefer_last(env) ||
- head_pages_retired !=
- unaligned_peek_u64_volatile(
- 4, recent_meta->mm_pages_retired)) ||
- head_txnid != meta_txnid(env, recent_meta))
- goto retry_header;
+ return mdbx_attr_poke(&reserve, data, attr, flags);
+}
- lag = (head_txnid - txnid) / xMDBX_TXNID_STEP;
- bytes_used = pgno2bytes(env, pages_used);
- bytes_retained = (head_pages_retired > reader_pages_retired)
- ? pgno2bytes(env, (pgno_t)(head_pages_retired -
- reader_pages_retired))
- : 0;
- }
- rc = func(ctx, ++serial, i, pid, (mdbx_tid_t)tid, txnid, lag, bytes_used,
- bytes_retained);
- if (unlikely(rc != MDBX_SUCCESS))
- break;
- }
- }
+int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
+ mdbx_attr_t attr) {
+ if (unlikely(!key || !txn))
+ return MDBX_EINVAL;
- return rc;
-}
+ if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
+ return MDBX_EBADSIGN;
-/* Insert pid into list if not already present.
- * return -1 if already present. */
-__cold static bool mdbx_pid_insert(uint32_t *ids, uint32_t pid) {
- /* binary search of pid in list */
- unsigned base = 0;
- unsigned cursor = 1;
- int val = 0;
- unsigned n = ids[0];
+ if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
+ return MDBX_BAD_DBI;
- while (n > 0) {
- unsigned pivot = n >> 1;
- cursor = base + pivot + 1;
- val = pid - ids[cursor];
+ if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
+ return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
- if (val < 0) {
- n = pivot;
- } else if (val > 0) {
- base = cursor;
- n -= pivot + 1;
- } else {
- /* found, so it's a duplicate */
- return false;
+ MDBX_cursor_couple cx;
+ MDBX_val old_data;
+ int rc = mdbx_cursor_init(&cx.outer, txn, dbi);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ rc = mdbx_cursor_set(&cx.outer, key, &old_data, MDBX_SET, NULL);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_NOTFOUND && data) {
+ cx.outer.mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = &cx.outer;
+ rc = mdbx_cursor_put_attr(&cx.outer, key, data, attr, 0);
+ txn->mt_cursors[dbi] = cx.outer.mc_next;
}
+ return rc;
}
- if (val > 0)
- ++cursor;
+ mdbx_attr_t old_attr = 0;
+ rc = mdbx_attr_peek(&old_data, &old_attr);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
- ids[0]++;
- for (n = ids[0]; n > cursor; n--)
- ids[n] = ids[n - 1];
- ids[n] = pid;
- return true;
-}
+ if (old_attr == attr && (!data || (data->iov_len == old_data.iov_len &&
+ memcmp(data->iov_base, old_data.iov_base,
+ old_data.iov_len) == 0)))
+ return MDBX_SUCCESS;
-__cold int mdbx_reader_check(MDBX_env *env, int *dead) {
- if (dead)
- *dead = 0;
- return mdbx_cleanup_dead_readers(env, false, dead);
+ cx.outer.mc_next = txn->mt_cursors[dbi];
+ txn->mt_cursors[dbi] = &cx.outer;
+ rc = mdbx_cursor_put_attr(&cx.outer, key, data ? data : &old_data, attr,
+ MDBX_CURRENT);
+ txn->mt_cursors[dbi] = cx.outer.mc_next;
+ return rc;
}
+#endif /* MDBX_NEXENTA_ATTRS */
+
+/******************************************************************************/
+/* *INDENT-OFF* */
+/* clang-format off */
+
+__dll_export
+#ifdef __attribute_used__
+ __attribute_used__
+#elif defined(__GNUC__) || __has_attribute(__used__)
+ __attribute__((__used__))
+#endif
+#ifdef __attribute_externally_visible__
+ __attribute_externally_visible__
+#elif (defined(__GNUC__) && !defined(__clang__)) || \
+ __has_attribute(__externally_visible__)
+ __attribute__((__externally_visible__))
+#endif
+ const struct MDBX_build_info mdbx_build = {
+#ifdef MDBX_BUILD_TIMESTAMP
+ MDBX_BUILD_TIMESTAMP
+#else
+ "\"" __DATE__ " " __TIME__ "\""
+#endif /* MDBX_BUILD_TIMESTAMP */
+
+ ,
+#ifdef MDBX_BUILD_TARGET
+ MDBX_BUILD_TARGET
+#else
+ #if defined(__ANDROID_API__)
+ "Android" MDBX_STRINGIFY(__ANDROID_API__)
+ #elif defined(__linux__) || defined(__gnu_linux__)
+ "Linux"
+ #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__)
+ "webassembly"
+ #elif defined(__CYGWIN__)
+ "CYGWIN"
+ #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \
+ || defined(__WINDOWS__)
+ "Windows"
+ #elif defined(__APPLE__)
+ #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \
+ || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR)
+ "iOS"
+ #else
+ "MacOS"
+ #endif
+ #elif defined(__FreeBSD__)
+ "FreeBSD"
+ #elif defined(__DragonFly__)
+ "DragonFlyBSD"
+ #elif defined(__NetBSD__)
+ "NetBSD"
+ #elif defined(__OpenBSD__)
+ "OpenBSD"
+ #elif defined(__bsdi__)
+ "UnixBSDI"
+ #elif defined(__MACH__)
+ "MACH"
+ #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC))
+ "HPUX"
+ #elif defined(_AIX)
+ "AIX"
+ #elif defined(__sun) && defined(__SVR4)
+ "Solaris"
+ #elif defined(__BSD__) || defined(BSD)
+ "UnixBSD"
+ #elif defined(__unix__) || defined(UNIX) || defined(__unix) \
+ || defined(__UNIX) || defined(__UNIX__)
+ "UNIX"
+ #elif defined(_POSIX_VERSION)
+ "POSIX" MDBX_STRINGIFY(_POSIX_VERSION)
+ #else
+ "UnknownOS"
+ #endif /* Target OS */
-/* Return:
- * MDBX_RESULT_TRUE - done and mutex recovered
- * MDBX_SUCCESS - done
- * Otherwise errcode. */
-__cold MDBX_INTERNAL_FUNC int
-mdbx_cleanup_dead_readers(MDBX_env *env, int rdt_locked, int *dead) {
- int rc = check_env(env, true);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ "-"
- mdbx_assert(env, rdt_locked >= 0);
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (unlikely(lck == NULL)) {
- /* exclusive mode */
- if (dead)
- *dead = 0;
- return MDBX_SUCCESS;
- }
+ #if defined(__amd64__)
+ "AMD64"
+ #elif defined(__ia32__)
+ "IA32"
+ #elif defined(__e2k__) || defined(__elbrus__)
+ "Elbrus"
+ #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA)
+ "Alpha"
+ #elif defined(__aarch64__) || defined(_M_ARM64)
+ "ARM64"
+ #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \
+ || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \
+ || defined(_M_ARMT) || defined(__arm)
+ "ARM"
+ #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64))
+ "MIPS64"
+ #elif defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__)
+ "MIPS"
+ #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64)
+ "PARISC64"
+ #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa)
+ "PARISC"
+ #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \
+ || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__)
+ "Itanium"
+ #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \
+ || defined(__powerpc64) || defined(_ARCH_PPC64)
+ "PowerPC64"
+ #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \
+ || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__)
+ "PowerPC"
+ #elif defined(__sparc64__) || defined(__sparc64)
+ "SPARC64"
+ #elif defined(__sparc__) || defined(__sparc)
+ "SPARC"
+ #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch)
+ "S390"
+ #else
+ "UnknownARCH"
+ #endif
+#endif /* MDBX_BUILD_TARGET */
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- uint32_t pidsbuf_onstask[142];
- uint32_t *const pids =
- (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
- ? pidsbuf_onstask
- : mdbx_malloc((snap_nreaders + 1) * sizeof(uint32_t));
- if (unlikely(!pids))
- return MDBX_ENOMEM;
+#ifdef MDBX_BUILD_TYPE
+# if defined(_MSC_VER)
+# pragma message("Configuration-depended MDBX_BUILD_TYPE: " MDBX_BUILD_TYPE)
+# endif
+ "-" MDBX_BUILD_TYPE
+#endif /* MDBX_BUILD_TYPE */
+ ,
+ "MDBX_DEBUG=" MDBX_STRINGIFY(MDBX_DEBUG)
+ " MDBX_WORDBITS=" MDBX_STRINGIFY(MDBX_WORDBITS)
+ " BYTE_ORDER="
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ "LITTLE_ENDIAN"
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ "BIG_ENDIAN"
+#else
+ #error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ " MDBX_ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG
+ " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG
+ " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG
+ " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG
+ " MDBX_TRUST_RTC=" MDBX_TRUST_RTC_CONFIG
+ " MDBX_ENABLE_REFUND=" MDBX_STRINGIFY(MDBX_ENABLE_REFUND)
+ " MDBX_ENABLE_MADVISE=" MDBX_STRINGIFY(MDBX_ENABLE_MADVISE)
+#if MDBX_DISABLE_PAGECHECKS
+ " MDBX_DISABLE_PAGECHECKS=YES"
+#endif /* MDBX_DISABLE_PAGECHECKS */
+#ifdef __SANITIZE_ADDRESS__
+ " SANITIZE_ADDRESS=YES"
+#endif /* __SANITIZE_ADDRESS__ */
+#ifdef MDBX_USE_VALGRIND
+ " MDBX_USE_VALGRIND=YES"
+#endif /* MDBX_USE_VALGRIND */
+#if MDBX_FORCE_ASSERTIONS
+ " MDBX_FORCE_ASSERTIONS=YES"
+#endif /* MDBX_FORCE_ASSERTIONS */
+#ifdef _GNU_SOURCE
+ " _GNU_SOURCE=YES"
+#else
+ " _GNU_SOURCE=NO"
+#endif /* _GNU_SOURCE */
+#ifdef __APPLE__
+ " MDBX_OSX_SPEED_INSTEADOF_DURABILITY=" MDBX_STRINGIFY(MDBX_OSX_SPEED_INSTEADOF_DURABILITY)
+#endif /* MacOS */
+#if defined(_WIN32) || defined(_WIN64)
+ " MDBX_WITHOUT_MSVC_CRT=" MDBX_STRINGIFY(MDBX_WITHOUT_MSVC_CRT)
+ " MDBX_BUILD_SHARED_LIBRARY=" MDBX_STRINGIFY(MDBX_BUILD_SHARED_LIBRARY)
+#if !MDBX_BUILD_SHARED_LIBRARY
+ " MDBX_MANUAL_MODULE_HANDLER=" MDBX_STRINGIFY(MDBX_MANUAL_MODULE_HANDLER)
+#endif
+ " WINVER=" MDBX_STRINGIFY(WINVER)
+#else /* Windows */
+ " MDBX_LOCKING=" MDBX_LOCKING_CONFIG
+ " MDBX_USE_OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG
+#endif /* !Windows */
+ " MDBX_CACHELINE_SIZE=" MDBX_STRINGIFY(MDBX_CACHELINE_SIZE)
+ " MDBX_CPU_WRITEBACK_INCOHERENT=" MDBX_STRINGIFY(MDBX_CPU_WRITEBACK_INCOHERENT)
+ " MDBX_MMAP_INCOHERENT_CPU_CACHE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_CPU_CACHE)
+ " MDBX_MMAP_INCOHERENT_FILE_WRITE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_FILE_WRITE)
+ " MDBX_UNALIGNED_OK=" MDBX_STRINGIFY(MDBX_UNALIGNED_OK)
+ " MDBX_PNL_ASCENDING=" MDBX_STRINGIFY(MDBX_PNL_ASCENDING)
+ ,
+#ifdef MDBX_BUILD_COMPILER
+ MDBX_BUILD_COMPILER
+#else
+ #ifdef __INTEL_COMPILER
+ "Intel C/C++ " MDBX_STRINGIFY(__INTEL_COMPILER)
+ #elif defined(__apple_build_version__)
+ "Apple clang " MDBX_STRINGIFY(__apple_build_version__)
+ #elif defined(__ibmxl__)
+ "IBM clang C " MDBX_STRINGIFY(__ibmxl_version__) "." MDBX_STRINGIFY(__ibmxl_release__)
+ "." MDBX_STRINGIFY(__ibmxl_modification__) "." MDBX_STRINGIFY(__ibmxl_ptf_fix_level__)
+ #elif defined(__clang__)
+ "clang " MDBX_STRINGIFY(__clang_version__)
+ #elif defined(__MINGW64__)
+ "MINGW-64 " MDBX_STRINGIFY(__MINGW64_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW64_MINOR_VERSION)
+ #elif defined(__MINGW32__)
+ "MINGW-32 " MDBX_STRINGIFY(__MINGW32_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW32_MINOR_VERSION)
+ #elif defined(__IBMC__)
+ "IBM C " MDBX_STRINGIFY(__IBMC__)
+ #elif defined(__GNUC__)
+ "GNU C/C++ "
+ #ifdef __VERSION__
+ __VERSION__
+ #else
+ MDBX_STRINGIFY(__GNUC__) "." MDBX_STRINGIFY(__GNUC_MINOR__) "." MDBX_STRINGIFY(__GNUC_PATCHLEVEL__)
+ #endif
+ #elif defined(_MSC_VER)
+ "MSVC " MDBX_STRINGIFY(_MSC_FULL_VER) "-" MDBX_STRINGIFY(_MSC_BUILD)
+ #else
+ "Unknown compiler"
+ #endif
+#endif /* MDBX_BUILD_COMPILER */
+ ,
+#ifdef MDBX_BUILD_FLAGS_CONFIG
+ MDBX_BUILD_FLAGS_CONFIG
+#endif /* MDBX_BUILD_FLAGS_CONFIG */
+#ifdef MDBX_BUILD_FLAGS
+ MDBX_BUILD_FLAGS
+#endif /* MDBX_BUILD_FLAGS */
+#if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS))
+ "undefined (please use correct build script)"
+#ifdef _MSC_VER
+#pragma message("warning: Build flags undefined. Please use correct build script")
+#else
+#warning "Build flags undefined. Please use correct build script"
+#endif // _MSC_VER
+#endif
+};
- pids[0] = 0;
- int count = 0;
- for (unsigned i = 0; i < snap_nreaders; i++) {
- const uint32_t pid =
- atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease);
- if (pid == 0)
- continue /* skip empty */;
- if (pid == env->me_pid)
- continue /* skip self */;
- if (!mdbx_pid_insert(pids, pid))
- continue /* such pid already processed */;
+#ifdef __SANITIZE_ADDRESS__
+LIBMDBX_API __attribute__((__weak__)) const char *__asan_default_options() {
+ return "symbolize=1:allow_addr2line=1:"
+#if MDBX_DEBUG
+ "debug=1:"
+ "verbosity=2:"
+#endif /* MDBX_DEBUG */
+ "log_threads=1:"
+ "report_globals=1:"
+ "replace_str=1:replace_intrin=1:"
+ "malloc_context_size=9:"
+ "detect_leaks=1:"
+ "check_printf=1:"
+ "detect_deadlocks=1:"
+#ifndef LTO_ENABLED
+ "check_initialization_order=1:"
+#endif
+ "detect_stack_use_after_return=1:"
+ "intercept_tls_get_addr=1:"
+ "decorate_proc_maps=1:"
+ "abort_on_error=1";
+}
+#endif /* __SANITIZE_ADDRESS__ */
- int err = mdbx_rpid_check(env, pid);
- if (err == MDBX_RESULT_TRUE)
- continue /* reader is live */;
+/* *INDENT-ON* */
+/* clang-format on */
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
- if (err != MDBX_SUCCESS) {
- rc = err;
- break /* mdbx_rpid_check() failed */;
- }
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
- /* stale reader found */
- if (!rdt_locked) {
- err = mdbx_rdt_lock(env);
- if (MDBX_IS_ERROR(err)) {
- rc = err;
- break;
- }
- rdt_locked = -1;
- if (err == MDBX_RESULT_TRUE) {
- /* mutex recovered, the mdbx_ipclock_failed() checked all readers */
- rc = MDBX_RESULT_TRUE;
- break;
- }
+#if defined(_WIN32) || defined(_WIN64)
- /* a other process may have clean and reused slot, recheck */
- if (lck->mti_readers[i].mr_pid.weak != pid)
- continue;
+#include <winioctl.h>
- err = mdbx_rpid_check(env, pid);
- if (MDBX_IS_ERROR(err)) {
- rc = err;
- break;
- }
+static int waitstatus2errcode(DWORD result) {
+ switch (result) {
+ case WAIT_OBJECT_0:
+ return MDBX_SUCCESS;
+ case WAIT_FAILED:
+ return (int)GetLastError();
+ case WAIT_ABANDONED:
+ return ERROR_ABANDONED_WAIT_0;
+ case WAIT_IO_COMPLETION:
+ return ERROR_USER_APC;
+ case WAIT_TIMEOUT:
+ return ERROR_TIMEOUT;
+ default:
+ return ERROR_UNHANDLED_ERROR;
+ }
+}
+
+/* Map a result from an NTAPI call to WIN32 error code. */
+static int ntstatus2errcode(NTSTATUS status) {
+ DWORD dummy;
+ OVERLAPPED ov;
+ memset(&ov, 0, sizeof(ov));
+ ov.Internal = status;
+ return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS
+ : (int)GetLastError();
+}
+
+/* We use native NT APIs to setup the memory map, so that we can
+ * let the DB file grow incrementally instead of always preallocating
+ * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
+ * but those headers are meant for driver-level development and
+ * conflict with the regular user-level headers, so we explicitly
+ * declare them here. Using these APIs also means we must link to
+ * ntdll.dll, which is not linked by default in user code. */
+
+extern NTSTATUS NTAPI NtCreateSection(
+ OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess,
+ IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes,
+ IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection,
+ IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle);
+
+typedef struct _SECTION_BASIC_INFORMATION {
+ ULONG Unknown;
+ ULONG SectionAttributes;
+ LARGE_INTEGER SectionSize;
+} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
+
+extern NTSTATUS NTAPI NtMapViewOfSection(
+ IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress,
+ IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize,
+ IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize,
+ IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType,
+ IN ULONG Win32Protect);
+
+extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle,
+ IN OPTIONAL PVOID BaseAddress);
+
+extern NTSTATUS NTAPI NtClose(HANDLE Handle);
+
+extern NTSTATUS NTAPI NtAllocateVirtualMemory(
+ IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits,
+ IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect);
+
+extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle,
+ IN PVOID *BaseAddress,
+ IN OUT PSIZE_T RegionSize,
+ IN ULONG FreeType);
+
+#ifndef WOF_CURRENT_VERSION
+typedef struct _WOF_EXTERNAL_INFO {
+ DWORD Version;
+ DWORD Provider;
+} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO;
+#endif /* WOF_CURRENT_VERSION */
+
+#ifndef WIM_PROVIDER_CURRENT_VERSION
+#define WIM_PROVIDER_HASH_SIZE 20
+
+typedef struct _WIM_PROVIDER_EXTERNAL_INFO {
+ DWORD Version;
+ DWORD Flags;
+ LARGE_INTEGER DataSourceId;
+ BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE];
+} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO;
+#endif /* WIM_PROVIDER_CURRENT_VERSION */
+
+#ifndef FILE_PROVIDER_CURRENT_VERSION
+typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 {
+ ULONG Version;
+ ULONG Algorithm;
+ ULONG Flags;
+} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1;
+#endif /* FILE_PROVIDER_CURRENT_VERSION */
+
+#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED
+#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL)
+#endif
+#ifndef STATUS_INVALID_DEVICE_REQUEST
+#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L)
+#endif
+#ifndef STATUS_NOT_SUPPORTED
+#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL)
+#endif
- if (err != MDBX_SUCCESS)
- continue /* the race with other process, slot reused */;
- }
+#ifndef FILE_DEVICE_FILE_SYSTEM
+#define FILE_DEVICE_FILE_SYSTEM 0x00000009
+#endif
- /* clean it */
- for (unsigned j = i; j < snap_nreaders; j++) {
- if (lck->mti_readers[j].mr_pid.weak == pid) {
- mdbx_debug("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN,
- (size_t)pid, lck->mti_readers[j].mr_txnid.weak);
- atomic_store32(&lck->mti_readers[j].mr_pid, 0, mo_Relaxed);
- atomic_store32(&lck->mti_readers_refresh_flag, true, mo_AcquireRelease);
- count++;
- }
- }
- }
+#ifndef FSCTL_GET_EXTERNAL_BACKING
+#define FSCTL_GET_EXTERNAL_BACKING \
+ CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS)
+#endif
- if (likely(!MDBX_IS_ERROR(rc)))
- atomic_store64(&lck->mti_reader_check_timestamp, mdbx_osal_monotime(),
- mo_Relaxed);
+#ifndef ERROR_NOT_CAPABLE
+#define ERROR_NOT_CAPABLE 775L
+#endif
- if (rdt_locked < 0)
- mdbx_rdt_unlock(env);
+#endif /* _WIN32 || _WIN64 */
- if (pids != pidsbuf_onstask)
- mdbx_free(pids);
+/*----------------------------------------------------------------------------*/
- if (dead)
- *dead = count;
- return rc;
-}
+#if defined(__UCLIBC__)
+__extern_C void __assert(const char *, const char *, unsigned int, const char *)
+#ifdef __THROW
+ __THROW
+#else
+ __nothrow
+#endif /* __THROW */
+ MDBX_NORETURN;
+#define __assert_fail(assertion, file, line, function) \
+ __assert(assertion, file, line, function)
+
+#elif _POSIX_C_SOURCE > 200212 && \
+ /* workaround for avoid musl libc wrong prototype */ ( \
+ defined(__GLIBC__) || defined(__GNU_LIBRARY__))
+/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */
+__extern_C void __assert_fail(const char *assertion, const char *file,
+ unsigned line, const char *function)
+#ifdef __THROW
+ __THROW
+#else
+ __nothrow
+#endif /* __THROW */
+ MDBX_NORETURN;
+
+#elif defined(__APPLE__) || defined(__MACH__)
+__extern_C void __assert_rtn(const char *function, const char *file, int line,
+ const char *assertion) /* __nothrow */
+#ifdef __dead2
+ __dead2
+#else
+ MDBX_NORETURN
+#endif /* __dead2 */
+#ifdef __disable_tail_calls
+ __disable_tail_calls
+#endif /* __disable_tail_calls */
+ ;
-__cold int mdbx_setup_debug(int loglevel, int flags, MDBX_debug_func *logger) {
- const int rc = mdbx_runtime_flags | (mdbx_loglevel << 16);
+#define __assert_fail(assertion, file, line, function) \
+ __assert_rtn(function, file, line, assertion)
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+__extern_C void __assert_c99(const char *assection, const char *file, int line,
+ const char *function) MDBX_NORETURN;
+#define __assert_fail(assertion, file, line, function) \
+ __assert_c99(assertion, file, line, function)
+#elif defined(__OpenBSD__)
+__extern_C __dead void __assert2(const char *file, int line,
+ const char *function,
+ const char *assertion) /* __nothrow */;
+#define __assert_fail(assertion, file, line, function) \
+ __assert2(file, line, function, assertion)
+#elif defined(__NetBSD__)
+__extern_C __dead void __assert13(const char *file, int line,
+ const char *function,
+ const char *assertion) /* __nothrow */;
+#define __assert_fail(assertion, file, line, function) \
+ __assert13(file, line, function, assertion)
+#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \
+ defined(__DragonFly__)
+__extern_C void __assert(const char *function, const char *file, int line,
+ const char *assertion) /* __nothrow */
+#ifdef __dead2
+ __dead2
+#else
+ MDBX_NORETURN
+#endif /* __dead2 */
+#ifdef __disable_tail_calls
+ __disable_tail_calls
+#endif /* __disable_tail_calls */
+ ;
+#define __assert_fail(assertion, file, line, function) \
+ __assert(function, file, line, assertion)
- if (loglevel != MDBX_LOG_DONTCHANGE)
- mdbx_loglevel = (uint8_t)loglevel;
+#endif /* __assert_fail */
- if (flags != MDBX_DBG_DONTCHANGE) {
- flags &=
+#if !defined(__ANDROID_API__) || MDBX_DEBUG
+
+__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg,
+ const char *func, int line) {
#if MDBX_DEBUG
- MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER |
+ if (env && env->me_assert_func) {
+ env->me_assert_func(env, msg, func, line);
+ return;
+ }
+#else
+ (void)env;
+#endif /* MDBX_DEBUG */
+
+ if (mdbx_debug_logger)
+ mdbx_debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg);
+ else {
+#if defined(_WIN32) || defined(_WIN64)
+ char *message = nullptr;
+ const int num = mdbx_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u",
+ msg, func ? func : "unknown", line);
+ if (num < 1 || !message)
+ message = "<troubles with assertion-message preparation>";
+ OutputDebugStringA(message);
+ if (IsDebuggerPresent())
+ DebugBreak();
+#elif defined(__ANDROID_API__)
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line);
+#else
+ __assert_fail(msg, "mdbx", line, func);
#endif
- MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP |
- MDBX_DBG_DONT_UPGRADE;
- mdbx_runtime_flags = (uint8_t)flags;
}
- if (logger != MDBX_LOGGER_DONTCHANGE)
- mdbx_debug_logger = logger;
- return rc;
+#if defined(_WIN32) || defined(_WIN64)
+ FatalExit(ERROR_UNHANDLED_ERROR);
+#else
+ abort();
+#endif
}
-__cold static txnid_t mdbx_kick_longlived_readers(MDBX_env *env,
- const txnid_t laggard) {
- mdbx_debug("DB size maxed out by reading #%" PRIaTXN, laggard);
-
- int retry;
- for (retry = 0; retry < INT_MAX; ++retry) {
- txnid_t oldest = mdbx_recent_steady_txnid(env);
- mdbx_assert(env, oldest < env->me_txn0->mt_txnid);
- mdbx_assert(env, oldest >= laggard);
- mdbx_assert(env, oldest >= env->me_lck->mti_oldest_reader.weak);
- MDBX_lockinfo *const lck = env->me_lck_mmap.lck;
- if (oldest == laggard || unlikely(!lck /* without-LCK mode */))
- return oldest;
+#endif /* __ANDROID_API__ || MDBX_DEBUG */
- if (MDBX_IS_ERROR(mdbx_cleanup_dead_readers(env, false, NULL)))
- break;
+__cold void mdbx_panic(const char *fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
- MDBX_reader *asleep = nullptr;
- uint64_t oldest_retired = UINT64_MAX;
- const unsigned snap_nreaders =
- atomic_load32(&lck->mti_numreaders, mo_AcquireRelease);
- for (unsigned i = 0; i < snap_nreaders; ++i) {
- retry:
- if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) {
- /* mdbx_jitter4testing(true); */
- const uint64_t snap_retired = atomic_load64(
- &lck->mti_readers[i].mr_snapshot_pages_retired, mo_Relaxed);
- const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid);
- if (unlikely(snap_retired !=
- atomic_load64(
- &lck->mti_readers[i].mr_snapshot_pages_retired,
- mo_AcquireRelease) ||
- snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)))
- goto retry;
- if (oldest > snap_txnid &&
- laggard <= /* ignore pending updates */ snap_txnid) {
- oldest = snap_txnid;
- oldest_retired = snap_retired;
- asleep = &lck->mti_readers[i];
- }
- }
- }
+ char *message = nullptr;
+ const int num = mdbx_vasprintf(&message, fmt, ap);
+ va_end(ap);
+ const char *const const_message =
+ (num < 1 || !message) ? "<troubles with panic-message preparation>"
+ : message;
- if (laggard < oldest || !asleep) {
- if (retry && env->me_hsr_callback) {
- /* LY: notify end of hsr-loop */
- const txnid_t gap = oldest - laggard;
- env->me_hsr_callback(env, env->me_txn, 0, 0, laggard,
- (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, 0,
- -retry);
- }
- mdbx_notice("hsr-kick: update oldest %" PRIaTXN " -> %" PRIaTXN,
- lck->mti_oldest_reader.weak, oldest);
- mdbx_assert(env, lck->mti_oldest_reader.weak <= oldest);
- return atomic_store64(&lck->mti_oldest_reader, oldest, mo_Relaxed);
- }
+#if defined(_WIN32) || defined(_WIN64)
+ OutputDebugStringA("\r\nMDBX-PANIC: ");
+ OutputDebugStringA(const_message);
+ if (IsDebuggerPresent())
+ DebugBreak();
+ FatalExit(ERROR_UNHANDLED_ERROR);
+#else
+#if defined(__ANDROID_API__)
+ __android_log_assert("panic", "mdbx", "%s", const_message);
+#else
+ __assert_fail(const_message, "mdbx", 0, "panic");
+#endif /* __ANDROID_API__ */
+ abort();
+#endif
+}
- if (!env->me_hsr_callback)
- break;
+/*----------------------------------------------------------------------------*/
- uint32_t pid = atomic_load32(&asleep->mr_pid, mo_AcquireRelease);
- uint64_t tid = asleep->mr_tid.weak;
- if (safe64_read(&asleep->mr_txnid) != laggard || pid <= 0)
- continue;
+#ifndef mdbx_vasprintf
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt,
+ va_list ap) {
+ va_list ones;
+ va_copy(ones, ap);
+ int needed = vsnprintf(nullptr, 0, fmt, ap);
- const MDBX_meta *head_meta = constmeta_prefer_last(env);
- const txnid_t gap =
- (constmeta_txnid(env, head_meta) - laggard) / xMDBX_TXNID_STEP;
- const uint64_t head_retired =
- unaligned_peek_u64(4, head_meta->mm_pages_retired);
- const size_t space =
- (head_retired > oldest_retired)
- ? pgno2bytes(env, (pgno_t)(head_retired - oldest_retired))
- : 0;
- int rc = env->me_hsr_callback(
- env, env->me_txn, pid, (mdbx_tid_t)tid, laggard,
- (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
- if (rc < 0)
- break;
+ if (unlikely(needed < 0 || needed >= INT_MAX)) {
+ *strp = nullptr;
+ va_end(ones);
+ return needed;
+ }
- if (rc > 0) {
- if (rc == 1) {
- safe64_reset_compare(&asleep->mr_txnid, laggard);
- } else {
- safe64_reset(&asleep->mr_txnid, true);
- atomic_store64(&asleep->mr_tid, 0, mo_Relaxed);
- atomic_store32(&asleep->mr_pid, 0, mo_Relaxed);
- }
- atomic_store32(&lck->mti_readers_refresh_flag, true, mo_Relaxed);
- }
+ *strp = mdbx_malloc(needed + 1);
+ if (unlikely(*strp == nullptr)) {
+ va_end(ones);
+#if defined(_WIN32) || defined(_WIN64)
+ SetLastError(MDBX_ENOMEM);
+#else
+ errno = MDBX_ENOMEM;
+#endif
+ return -1;
}
- if (retry && env->me_hsr_callback) {
- /* LY: notify end of hsr-loop */
- env->me_hsr_callback(env, env->me_txn, 0, 0, laggard, 0, 0, -retry);
+ int actual = vsnprintf(*strp, needed + 1, fmt, ones);
+ va_end(ones);
+
+ assert(actual == needed);
+ if (unlikely(actual < 0)) {
+ mdbx_free(*strp);
+ *strp = nullptr;
}
- return mdbx_find_oldest(env->me_txn);
+ return actual;
}
+#endif /* mdbx_vasprintf */
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) {
- return __inline_mdbx_env_set_syncbytes(env, threshold);
+#ifndef mdbx_asprintf
+MDBX_INTERNAL_FUNC int mdbx_asprintf(char **strp, const char *fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ int rc = mdbx_vasprintf(strp, fmt, ap);
+ va_end(ap);
+ return rc;
}
+#endif /* mdbx_asprintf */
-__cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) {
- return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16);
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result) {
+ assert(is_powerof2(alignment) && alignment >= sizeof(void *));
+#if defined(_WIN32) || defined(_WIN64)
+ (void)alignment;
+ *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+ return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */;
+#elif defined(_ISOC11_SOURCE)
+ *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment));
+ return *result ? MDBX_SUCCESS : errno;
+#elif _POSIX_VERSION >= 200112L && \
+ (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17)
+ *result = nullptr;
+ return posix_memalign(result, alignment, bytes);
+#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L
+ *result = memalign(alignment, bytes);
+ return *result ? MDBX_SUCCESS : errno;
+#else
+#error FIXME
+#endif
}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+#endif /* mdbx_memalign_alloc */
-__cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) {
- int rc = check_env(env, false);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr) {
+#if defined(_WIN32) || defined(_WIN64)
+ VirtualFree(ptr, 0, MEM_RELEASE);
+#else
+ mdbx_free(ptr);
+#endif
+}
+#endif /* mdbx_memalign_free */
- env->me_hsr_callback = hsr;
- return MDBX_SUCCESS;
+#ifndef mdbx_strdup
+char *mdbx_strdup(const char *str) {
+ if (!str)
+ return NULL;
+ size_t bytes = strlen(str) + 1;
+ char *dup = mdbx_malloc(bytes);
+ if (dup)
+ memcpy(dup, str, bytes);
+ return dup;
}
+#endif /* mdbx_strdup */
-__cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) {
- return likely(env && env->me_signature.weak == MDBX_ME_SIGNATURE)
- ? env->me_hsr_callback
- : NULL;
+/*----------------------------------------------------------------------------*/
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair) {
+ int rc;
+ memset(condpair, 0, sizeof(mdbx_condpair_t));
+#if defined(_WIN32) || defined(_WIN64)
+ if ((condpair->mutex = CreateMutexW(NULL, FALSE, NULL)) == NULL) {
+ rc = (int)GetLastError();
+ goto bailout_mutex;
+ }
+ if ((condpair->event[0] = CreateEventW(NULL, FALSE, FALSE, NULL)) == NULL) {
+ rc = (int)GetLastError();
+ goto bailout_event;
+ }
+ if ((condpair->event[1] = CreateEventW(NULL, FALSE, FALSE, NULL)) != NULL)
+ return MDBX_SUCCESS;
+
+ rc = (int)GetLastError();
+ (void)CloseHandle(condpair->event[0]);
+bailout_event:
+ (void)CloseHandle(condpair->mutex);
+#else
+ rc = pthread_mutex_init(&condpair->mutex, NULL);
+ if (unlikely(rc != 0))
+ goto bailout_mutex;
+ rc = pthread_cond_init(&condpair->cond[0], NULL);
+ if (unlikely(rc != 0))
+ goto bailout_cond;
+ rc = pthread_cond_init(&condpair->cond[1], NULL);
+ if (likely(rc == 0))
+ return MDBX_SUCCESS;
+
+ (void)pthread_cond_destroy(&condpair->cond[0]);
+bailout_cond:
+ (void)pthread_mutex_destroy(&condpair->mutex);
+#endif
+bailout_mutex:
+ memset(condpair, 0, sizeof(mdbx_condpair_t));
+ return rc;
}
-#ifdef __SANITIZE_THREAD__
-/* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */
-__attribute__((__no_sanitize_thread__, __noinline__))
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair) {
+#if defined(_WIN32) || defined(_WIN64)
+ int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
+ rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError();
+ rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError();
+#else
+ int err, rc = pthread_mutex_destroy(&condpair->mutex);
+ rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc;
+ rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc;
#endif
-int mdbx_txn_straggler(const MDBX_txn *txn, int *percent)
-{
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return (rc > 0) ? -rc : rc;
+ memset(condpair, 0, sizeof(mdbx_condpair_t));
+ return rc;
+}
- MDBX_env *env = txn->mt_env;
- if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) {
- if (percent)
- *percent =
- (int)((txn->mt_next_pgno * UINT64_C(100) + txn->mt_end_pgno / 2) /
- txn->mt_end_pgno);
- return 0;
- }
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD code = WaitForSingleObject(condpair->mutex, INFINITE);
+ return waitstatus2errcode(code);
+#else
+ return mdbx_pthread_mutex_lock(&condpair->mutex);
+#endif
+}
- txnid_t recent = 0;
- volatile const MDBX_meta *meta = nullptr;
- do {
- meta = meta_prefer_last(env);
- recent = meta_txnid(env, meta);
- if (percent) {
- const pgno_t maxpg = meta->mm_geo.now;
- *percent = (int)((meta->mm_geo.next * UINT64_C(100) + maxpg / 2) / maxpg);
- }
- } while (unlikely(recent != meta_txnid(env, meta)));
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair) {
+#if defined(_WIN32) || defined(_WIN64)
+ return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ return pthread_mutex_unlock(&condpair->mutex);
+#endif
+}
- txnid_t lag = (recent - txn->mt_txnid) / xMDBX_TXNID_STEP;
- return (lag > INT_MAX) ? INT_MAX : (int)lag;
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part) {
+#if defined(_WIN32) || defined(_WIN64)
+ return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ return pthread_cond_signal(&condpair->cond[part]);
+#endif
}
-typedef struct mdbx_walk_ctx {
- void *mw_user;
- MDBX_pgvisitor_func *mw_visitor;
- MDBX_txn *mw_txn;
- MDBX_cursor *mw_cursor;
- bool mw_dont_check_keys_ordering;
-} mdbx_walk_ctx_t;
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair,
+ bool part) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part],
+ INFINITE, FALSE);
+ if (code == WAIT_OBJECT_0) {
+ code = WaitForSingleObject(condpair->mutex, INFINITE);
+ if (code == WAIT_OBJECT_0)
+ return MDBX_SUCCESS;
+ }
+ return waitstatus2errcode(code);
+#else
+ return pthread_cond_wait(&condpair->cond[part], &condpair->mutex);
+#endif
+}
-__cold static int mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db,
- const char *name, int deep);
+/*----------------------------------------------------------------------------*/
-static MDBX_page_type_t walk_page_type(const MDBX_page *mp) {
- if (mp)
- switch (mp->mp_flags) {
- case P_BRANCH:
- return MDBX_page_branch;
- case P_LEAF:
- return MDBX_page_leaf;
- case P_LEAF | P_LEAF2:
- return MDBX_page_dupfixed_leaf;
- case P_OVERFLOW:
- return MDBX_page_large;
- case P_META:
- return MDBX_page_meta;
- }
- return MDBX_page_broken;
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex) {
+#if defined(_WIN32) || defined(_WIN64)
+ InitializeCriticalSection(fastmutex);
+ return MDBX_SUCCESS;
+#else
+ return pthread_mutex_init(fastmutex, NULL);
+#endif
}
-/* Depth-first tree traversal. */
-__cold static int mdbx_walk_tree(mdbx_walk_ctx_t *ctx, const pgno_t pgno,
- const char *name, int deep,
- txnid_t parent_txnid) {
- assert(pgno != P_INVALID);
- MDBX_page *mp = nullptr;
- int rc, err = mdbx_page_get(ctx->mw_cursor, pgno, &mp, parent_txnid);
- if (err == MDBX_SUCCESS)
- err = mdbx_page_check(ctx->mw_cursor, mp, 0);
-
- MDBX_page_type_t type = walk_page_type(mp);
- const int nentries = (mp && !IS_OVERFLOW(mp)) ? page_numkeys(mp) : 1;
- unsigned npages = (mp && IS_OVERFLOW(mp)) ? mp->mp_pages : 1;
- size_t pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages);
- size_t header_size = (mp && !IS_LEAF2(mp) && !IS_OVERFLOW(mp))
- ? PAGEHDRSZ + mp->mp_lower
- : PAGEHDRSZ;
- size_t payload_size = 0;
- size_t unused_size =
- (mp && !IS_OVERFLOW(mp) ? page_room(mp) : pagesize - header_size) -
- payload_size;
- size_t align_bytes = 0;
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex) {
+#if defined(_WIN32) || defined(_WIN64)
+ DeleteCriticalSection(fastmutex);
+ return MDBX_SUCCESS;
+#else
+ return pthread_mutex_destroy(fastmutex);
+#endif
+}
- if (err == MDBX_SUCCESS) {
- /* LY: Don't use mask here, e.g bitwise
- * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
- * Pages should not me marked dirty/loose or otherwise. */
- switch (mp->mp_flags) {
- default:
- err = MDBX_CORRUPTED;
- break;
- case P_BRANCH:
- if (unlikely(nentries < 2))
- err = MDBX_CORRUPTED;
- case P_LEAF:
- case P_LEAF | P_LEAF2:
- break;
- }
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex) {
+#if defined(_WIN32) || defined(_WIN64)
+ __try {
+ EnterCriticalSection(fastmutex);
+ } __except (
+ (GetExceptionCode() ==
+ 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
+ ? EXCEPTION_EXECUTE_HANDLER
+ : EXCEPTION_CONTINUE_SEARCH) {
+ return ERROR_POSSIBLE_DEADLOCK;
}
+ return MDBX_SUCCESS;
+#else
+ return mdbx_pthread_mutex_lock(fastmutex);
+#endif
+}
- for (int i = 0; err == MDBX_SUCCESS && i < nentries;
- align_bytes += ((payload_size + align_bytes) & 1), i++) {
- if (type == MDBX_page_dupfixed_leaf) {
- /* LEAF2 pages have no mp_ptrs[] or node headers */
- payload_size += mp->mp_leaf2_ksize;
- continue;
- }
-
- MDBX_node *node = page_node(mp, i);
- payload_size += NODESIZE + node_ks(node);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex) {
+#if defined(_WIN32) || defined(_WIN64)
+ LeaveCriticalSection(fastmutex);
+ return MDBX_SUCCESS;
+#else
+ return pthread_mutex_unlock(fastmutex);
+#endif
+}
- if (type == MDBX_page_branch) {
- assert(i > 0 || node_ks(node) == 0);
- continue;
- }
+/*----------------------------------------------------------------------------*/
- assert(type == MDBX_page_leaf);
- switch (node_flags(node)) {
- case 0 /* usual node */:
- payload_size += node_ds(node);
- break;
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname) {
+#if defined(_WIN32) || defined(_WIN64)
+ const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
+ if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
+ return ERROR_INVALID_NAME;
+ wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
+ if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
+ return ERROR_INVALID_NAME;
+ return DeleteFileW(pathnameW) ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ return unlink(pathname) ? errno : MDBX_SUCCESS;
+#endif
+}
- case F_BIGDATA /* long data on the large/overflow page */: {
- payload_size += sizeof(pgno_t);
- const pgno_t large_pgno = node_largedata_pgno(node);
- const size_t over_payload = node_ds(node);
- const size_t over_header = PAGEHDRSZ;
- npages = 1;
+#if !(defined(_WIN32) || defined(_WIN64))
+static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); }
+#endif /*! Windows */
- MDBX_page *op;
- err = mdbx_page_get(ctx->mw_cursor, large_pgno, &op,
- pp_txnid4chk(mp, ctx->mw_txn));
- if (err == MDBX_SUCCESS)
- err = mdbx_page_check(ctx->mw_cursor, op, 0);
- if (err == MDBX_SUCCESS) {
- /* LY: Don't use mask here, e.g bitwise
- * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
- * Pages should not me marked dirty/loose or otherwise. */
- if (unlikely(P_OVERFLOW != op->mp_flags))
- err = bad_page(mp, "wrong page type %d for large data", op->mp_flags);
- else
- npages = op->mp_pages;
- }
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname) {
+#if defined(_WIN32) || defined(_WIN64)
+ const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
+ if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
+ return ERROR_INVALID_NAME;
+ wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
+ if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
+ return ERROR_INVALID_NAME;
+ return RemoveDirectoryW(pathnameW) ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ return rmdir(pathname) ? errno : MDBX_SUCCESS;
+#endif
+}
- pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages);
- const size_t over_unused = pagesize - over_payload - over_header;
- rc = ctx->mw_visitor(large_pgno, npages, ctx->mw_user, deep, name,
- pagesize, MDBX_page_large, err, 1, over_payload,
- over_header, over_unused);
- if (unlikely(rc != MDBX_SUCCESS))
- return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
- } break;
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits) {
+ *fd = INVALID_HANDLE_VALUE;
- case F_SUBDATA /* sub-db */: {
- const size_t namelen = node_ks(node);
- payload_size += node_ds(node);
- if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db)))
- err = MDBX_CORRUPTED;
- } break;
+#if defined(_WIN32) || defined(_WIN64)
+ const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
+ if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
+ return ERROR_INVALID_NAME;
+ wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
+ if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
+ return ERROR_INVALID_NAME;
- case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
- payload_size += sizeof(MDBX_db);
- if (unlikely(node_ds(node) != sizeof(MDBX_db)))
- err = MDBX_CORRUPTED;
- break;
+ DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING;
+ DWORD FlagsAndAttributes =
+ FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED;
+ DWORD DesiredAccess = FILE_READ_ATTRIBUTES;
+ DWORD ShareMode = (env->me_flags & MDBX_EXCLUSIVE)
+ ? 0
+ : (FILE_SHARE_READ | FILE_SHARE_WRITE);
- case F_DUPDATA /* short sub-page */: {
- if (unlikely(node_ds(node) <= PAGEHDRSZ)) {
- err = MDBX_CORRUPTED;
- break;
- }
+ switch (purpose) {
+ default:
+ return ERROR_INVALID_PARAMETER;
+ case MDBX_OPEN_LCK:
+ CreationDisposition = OPEN_ALWAYS;
+ DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
+ FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY;
+ break;
+ case MDBX_OPEN_DXB_READ:
+ CreationDisposition = OPEN_EXISTING;
+ DesiredAccess |= GENERIC_READ;
+ ShareMode |= FILE_SHARE_READ;
+ break;
+ case MDBX_OPEN_DXB_LAZY:
+ DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
+ break;
+ case MDBX_OPEN_DXB_DSYNC:
+ CreationDisposition = OPEN_EXISTING;
+ DesiredAccess |= GENERIC_WRITE;
+ FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH;
+ break;
+ case MDBX_OPEN_COPY:
+ CreationDisposition = CREATE_NEW;
+ ShareMode = 0;
+ DesiredAccess |= GENERIC_WRITE;
+ FlagsAndAttributes |=
+ (env->me_psize < env->me_os_psize) ? 0 : FILE_FLAG_NO_BUFFERING;
+ break;
+ case MDBX_OPEN_DELETE:
+ CreationDisposition = OPEN_EXISTING;
+ ShareMode |= FILE_SHARE_DELETE;
+ DesiredAccess =
+ FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE;
+ break;
+ }
- MDBX_page *sp = node_data(node);
- const int nsubkeys = page_numkeys(sp);
- size_t subheader_size =
- IS_LEAF2(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->mp_lower;
- size_t subunused_size = page_room(sp);
- size_t subpayload_size = 0;
- size_t subalign_bytes = 0;
- MDBX_page_type_t subtype;
+ *fd = CreateFileW(pathnameW, DesiredAccess, ShareMode, NULL,
+ CreationDisposition, FlagsAndAttributes, NULL);
+ if (*fd == INVALID_HANDLE_VALUE)
+ return (int)GetLastError();
- switch (sp->mp_flags & /* ignore legacy P_DIRTY flag */ ~0x10) {
- case P_LEAF | P_SUBP:
- subtype = MDBX_subpage_leaf;
- break;
- case P_LEAF | P_LEAF2 | P_SUBP:
- subtype = MDBX_subpage_dupfixed_leaf;
- break;
- default:
- subtype = MDBX_subpage_broken;
- err = MDBX_CORRUPTED;
- }
+ BY_HANDLE_FILE_INFORMATION info;
+ if (!GetFileInformationByHandle(*fd, &info)) {
+ int err = (int)GetLastError();
+ CloseHandle(*fd);
+ *fd = INVALID_HANDLE_VALUE;
+ return err;
+ }
+ const DWORD AttributesDiff =
+ (info.dwFileAttributes ^ FlagsAndAttributes) &
+ (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
+ FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED);
+ if (AttributesDiff)
+ (void)SetFileAttributesW(pathnameW, info.dwFileAttributes ^ AttributesDiff);
- for (int j = 0; err == MDBX_SUCCESS && j < nsubkeys;
- subalign_bytes += ((subpayload_size + subalign_bytes) & 1), j++) {
+#else
+ int flags = unix_mode_bits ? O_CREAT : 0;
+ switch (purpose) {
+ default:
+ return EINVAL;
+ case MDBX_OPEN_LCK:
+ flags |= O_RDWR;
+ break;
+ case MDBX_OPEN_DXB_READ:
+ flags = O_RDONLY;
+ break;
+ case MDBX_OPEN_DXB_LAZY:
+ flags |= O_RDWR;
+ break;
+ case MDBX_OPEN_COPY:
+ flags = O_CREAT | O_WRONLY | O_EXCL;
+ break;
+ case MDBX_OPEN_DXB_DSYNC:
+ flags |= O_WRONLY;
+#if defined(O_DSYNC)
+ flags |= O_DSYNC;
+#elif defined(O_SYNC)
+ flags |= O_SYNC;
+#elif defined(O_FSYNC)
+ flags |= O_FSYNC;
+#endif
+ break;
+ case MDBX_OPEN_DELETE:
+ flags = O_RDWR;
+ break;
+ }
- if (subtype == MDBX_subpage_dupfixed_leaf) {
- /* LEAF2 pages have no mp_ptrs[] or node headers */
- subpayload_size += sp->mp_leaf2_ksize;
- } else {
- assert(subtype == MDBX_subpage_leaf);
- MDBX_node *subnode = page_node(sp, j);
- subpayload_size += NODESIZE + node_ks(subnode) + node_ds(subnode);
- if (unlikely(node_flags(subnode) != 0))
- err = MDBX_CORRUPTED;
- }
- }
+ const bool direct_nocache_for_copy =
+ env->me_psize >= env->me_os_psize && purpose == MDBX_OPEN_COPY;
+ if (direct_nocache_for_copy) {
+#if defined(O_DIRECT)
+ flags |= O_DIRECT;
+#endif /* O_DIRECT */
+#if defined(O_NOCACHE)
+ flags |= O_NOCACHE;
+#endif /* O_NOCACHE */
+ }
+
+#ifdef O_CLOEXEC
+ flags |= O_CLOEXEC;
+#endif /* O_CLOEXEC */
+
+ /* Safeguard for todo4recovery://erased_by_github/libmdbx/issues/144 */
+#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
+ int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1;
+ static const char dev_null[] = "/dev/null";
+ if (!is_valid_fd(STDIN_FILENO)) {
+ mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN",
+ STDIN_FILENO, dev_null);
+ stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY);
+ }
+ if (!is_valid_fd(STDOUT_FILENO)) {
+ mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub",
+ "OUT", STDOUT_FILENO, dev_null);
+ stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY);
+ }
+ if (!is_valid_fd(STDERR_FILENO)) {
+ mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub",
+ "ERR", STDERR_FILENO, dev_null);
+ stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY);
+ }
+#else
+#error "Unexpected or unsupported UNIX or POSIX system"
+#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
+
+ *fd = open(pathname, flags, unix_mode_bits);
+#if defined(O_DIRECT)
+ if (*fd < 0 && (flags & O_DIRECT) &&
+ (errno == EINVAL || errno == EAFNOSUPPORT)) {
+ flags &= ~(O_DIRECT | O_EXCL);
+ *fd = open(pathname, flags, unix_mode_bits);
+ }
+#endif /* O_DIRECT */
+
+ /* Safeguard for todo4recovery://erased_by_github/libmdbx/issues/144 */
+#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
+ if (*fd == STDIN_FILENO) {
+ mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN",
+ STDIN_FILENO);
+ assert(stub_fd0 == -1);
+ *fd = dup(stub_fd0 = *fd);
+ }
+ if (*fd == STDOUT_FILENO) {
+ mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT",
+ STDOUT_FILENO);
+ assert(stub_fd1 == -1);
+ *fd = dup(stub_fd1 = *fd);
+ }
+ if (*fd == STDERR_FILENO) {
+ mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR",
+ STDERR_FILENO);
+ assert(stub_fd2 == -1);
+ *fd = dup(stub_fd2 = *fd);
+ }
+ if (stub_fd0 != -1)
+ close(stub_fd0);
+ if (stub_fd1 != -1)
+ close(stub_fd1);
+ if (stub_fd2 != -1)
+ close(stub_fd2);
+ if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) {
+ mdbx_error(
+ "Rejecting the use of a FD in the range "
+ "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption",
+ STDIN_FILENO, STDERR_FILENO);
+ close(*fd);
+ return EBADF;
+ }
+#else
+#error "Unexpected or unsupported UNIX or POSIX system"
+#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
- rc = ctx->mw_visitor(pgno, 0, ctx->mw_user, deep + 1, name, node_ds(node),
- subtype, err, nsubkeys, subpayload_size,
- subheader_size, subunused_size + subalign_bytes);
- if (unlikely(rc != MDBX_SUCCESS))
- return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
- header_size += subheader_size;
- unused_size += subunused_size;
- payload_size += subpayload_size;
- align_bytes += subalign_bytes;
- } break;
+ if (*fd < 0)
+ return errno;
- default:
- err = MDBX_CORRUPTED;
- }
+#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC)
+ const int fd_flags = fcntl(*fd, F_GETFD);
+ if (fd_flags != -1)
+ (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC);
+#endif /* FD_CLOEXEC && !O_CLOEXEC */
+
+ if (direct_nocache_for_copy) {
+#if defined(F_NOCACHE) && !defined(O_NOCACHE)
+ (void)fcntl(*fd, F_NOCACHE, 1);
+#endif /* F_NOCACHE */
}
- rc = ctx->mw_visitor(pgno, 1, ctx->mw_user, deep, name,
- ctx->mw_txn->mt_env->me_psize, type, err, nentries,
- payload_size, header_size, unused_size + align_bytes);
- if (unlikely(rc != MDBX_SUCCESS))
- return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
+#endif
+ return MDBX_SUCCESS;
+}
- for (int i = 0; err == MDBX_SUCCESS && i < nentries; i++) {
- if (type == MDBX_page_dupfixed_leaf)
- continue;
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd) {
+#if defined(_WIN32) || defined(_WIN64)
+ return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ assert(fd > STDERR_FILENO);
+ return (close(fd) == 0) ? MDBX_SUCCESS : errno;
+#endif
+}
- MDBX_node *node = page_node(mp, i);
- if (type == MDBX_page_branch) {
- err = mdbx_walk_tree(ctx, node_pgno(node), name, deep + 1,
- pp_txnid4chk(mp, ctx->mw_txn));
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_RESULT_TRUE)
- break;
- return err;
- }
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t bytes,
+ uint64_t offset) {
+ if (bytes > MAX_WRITE)
+ return MDBX_EINVAL;
+#if defined(_WIN32) || defined(_WIN64)
+ OVERLAPPED ov;
+ ov.hEvent = 0;
+ ov.Offset = (DWORD)offset;
+ ov.OffsetHigh = HIGH_DWORD(offset);
+
+ DWORD read = 0;
+ if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) {
+ int rc = (int)GetLastError();
+ return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc;
+ }
+#else
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ intptr_t read = pread(fd, buf, bytes, offset);
+ if (read < 0) {
+ int rc = errno;
+ return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc;
+ }
+#endif
+ return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA;
+}
+
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t bytes, uint64_t offset) {
+ while (true) {
+#if defined(_WIN32) || defined(_WIN64)
+ OVERLAPPED ov;
+ ov.hEvent = 0;
+ ov.Offset = (DWORD)offset;
+ ov.OffsetHigh = HIGH_DWORD(offset);
+
+ DWORD written;
+ if (unlikely(!WriteFile(
+ fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE,
+ &written, &ov)))
+ return (int)GetLastError();
+ if (likely(bytes == written))
+ return MDBX_SUCCESS;
+#else
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ const intptr_t written =
+ pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset);
+ if (likely(bytes == (size_t)written))
+ return MDBX_SUCCESS;
+ if (written < 0) {
+ const int rc = errno;
+ if (rc != EINTR)
+ return rc;
continue;
}
+#endif
+ bytes -= written;
+ offset += written;
+ buf = (char *)buf + written;
+ }
+}
- assert(type == MDBX_page_leaf);
- MDBX_db db;
- switch (node_flags(node)) {
- default:
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t bytes) {
+ while (true) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD written;
+ if (unlikely(!WriteFile(
+ fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE,
+ &written, nullptr)))
+ return (int)GetLastError();
+ if (likely(bytes == written))
+ return MDBX_SUCCESS;
+#else
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ const intptr_t written =
+ write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE);
+ if (likely(bytes == (size_t)written))
+ return MDBX_SUCCESS;
+ if (written < 0) {
+ const int rc = errno;
+ if (rc != EINTR)
+ return rc;
continue;
-
- case F_SUBDATA /* sub-db */: {
- const size_t namelen = node_ks(node);
- if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) {
- err = MDBX_CORRUPTED;
- break;
- }
-
- char namebuf_onstask[64];
- char *const sub_name = (namelen < sizeof(namebuf_onstask))
- ? namebuf_onstask
- : mdbx_malloc(namelen + 1);
- if (sub_name) {
- memcpy(sub_name, node_key(node), namelen);
- sub_name[namelen] = 0;
- memcpy(&db, node_data(node), sizeof(db));
- err = mdbx_walk_sdb(ctx, &db, sub_name, deep + 1);
- if (sub_name != namebuf_onstask)
- mdbx_free(sub_name);
- } else {
- err = MDBX_ENOMEM;
- }
- } break;
-
- case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */:
- if (unlikely(node_ds(node) != sizeof(MDBX_db) ||
- ctx->mw_cursor->mc_xcursor == NULL))
- err = MDBX_CORRUPTED;
- else {
- memcpy(&db, node_data(node), sizeof(db));
- assert(ctx->mw_cursor->mc_xcursor ==
- &container_of(ctx->mw_cursor, MDBX_cursor_couple, outer)->inner);
- ctx->mw_cursor = &ctx->mw_cursor->mc_xcursor->mx_cursor;
- err = mdbx_walk_tree(ctx, db.md_root, name, deep + 1,
- pp_txnid4chk(mp, ctx->mw_txn));
- MDBX_xcursor *inner_xcursor =
- container_of(ctx->mw_cursor, MDBX_xcursor, mx_cursor);
- MDBX_cursor_couple *couple =
- container_of(inner_xcursor, MDBX_cursor_couple, inner);
- ctx->mw_cursor = &couple->outer;
- }
- break;
}
+#endif
+ bytes -= written;
+ buf = (char *)buf + written;
}
+}
- return MDBX_SUCCESS;
+int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt,
+ uint64_t offset, size_t expected_written) {
+#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ < 24)
+ size_t written = 0;
+ for (int i = 0; i < iovcnt; ++i) {
+ int rc = mdbx_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);
+ if (unlikely(rc != MDBX_SUCCESS))
+ return rc;
+ written += iov[i].iov_len;
+ offset += iov[i].iov_len;
+ }
+ return (expected_written == written) ? MDBX_SUCCESS
+ : MDBX_EIO /* ERROR_WRITE_FAULT */;
+#else
+ int rc;
+ intptr_t written;
+ do {
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ written = pwritev(fd, iov, iovcnt, offset);
+ if (likely(expected_written == (size_t)written))
+ return MDBX_SUCCESS;
+ rc = errno;
+ } while (rc == EINTR);
+ return (written < 0) ? rc : MDBX_EIO /* Use which error code? */;
+#endif
}
-__cold static int mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db,
- const char *name, int deep) {
- if (unlikely(db->md_root == P_INVALID))
- return MDBX_SUCCESS; /* empty db */
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ enum mdbx_syncmode_bits mode_bits) {
+#if defined(_WIN32) || defined(_WIN64)
+ if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd))
+ return (int)GetLastError();
+ return MDBX_SUCCESS;
+#else
- MDBX_cursor_couple couple;
- MDBX_dbx dbx = {.md_klen_min = INT_MAX};
- uint8_t dbistate = DBI_VALID | DBI_AUDITED;
- int rc = mdbx_couple_init(&couple, ~0u, ctx->mw_txn, db, &dbx, &dbistate);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#if defined(__APPLE__) && \
+ MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY
+ if (mode_bits & MDBX_SYNC_IODQ)
+ return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno;
+#endif /* MacOS */
- if (ctx->mw_dont_check_keys_ordering) {
- couple.outer.mc_flags |= C_SKIPORD;
- couple.inner.mx_cursor.mc_flags |= C_SKIPORD;
+ /* LY: This approach is always safe and without appreciable performance
+ * degradation, even on a kernel with fdatasync's bug.
+ *
+ * For more info about of a corresponding fdatasync() bug
+ * see http://www.spinics.net/lists/linux-ext4/msg33714.html */
+ while (1) {
+ switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) {
+ case MDBX_SYNC_NONE:
+ return MDBX_SUCCESS /* nothing to do */;
+#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
+ case MDBX_SYNC_DATA:
+ if (fdatasync(fd) == 0)
+ return MDBX_SUCCESS;
+ break /* error */;
+#if defined(__linux__) || defined(__gnu_linux__)
+ case MDBX_SYNC_SIZE:
+ if (mdbx_linux_kernel_version >= 0x03060000)
+ return MDBX_SUCCESS;
+ __fallthrough /* fall through */;
+#endif /* Linux */
+#endif /* _POSIX_SYNCHRONIZED_IO > 0 */
+ default:
+ if (fsync(fd) == 0)
+ return MDBX_SUCCESS;
+ }
+
+ int rc = errno;
+ if (rc != EINTR)
+ return rc;
}
- couple.outer.mc_next = ctx->mw_cursor;
- ctx->mw_cursor = &couple.outer;
- rc = mdbx_walk_tree(ctx, db->md_root, name, deep, ctx->mw_txn->mt_txnid);
- ctx->mw_cursor = couple.outer.mc_next;
- return rc;
+#endif
}
-__cold int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor,
- void *user, bool dont_check_keys_ordering) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length) {
+#if defined(_WIN32) || defined(_WIN64)
+ BY_HANDLE_FILE_INFORMATION info;
+ if (!GetFileInformationByHandle(fd, &info))
+ return (int)GetLastError();
+ *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
+#else
+ struct stat st;
- mdbx_walk_ctx_t ctx;
- memset(&ctx, 0, sizeof(ctx));
- ctx.mw_txn = txn;
- ctx.mw_user = user;
- ctx.mw_visitor = visitor;
- ctx.mw_dont_check_keys_ordering = dont_check_keys_ordering;
+ STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ if (fstat(fd, &st))
+ return errno;
- rc = visitor(0, NUM_METAS, user, 0, MDBX_PGWALK_META,
- pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, MDBX_SUCCESS,
- NUM_METAS, sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS,
- (txn->mt_env->me_psize - sizeof(MDBX_meta) - PAGEHDRSZ) *
- NUM_METAS);
- if (!MDBX_IS_ERROR(rc))
- rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[FREE_DBI], MDBX_PGWALK_GC, 0);
- if (!MDBX_IS_ERROR(rc))
- rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[MAIN_DBI], MDBX_PGWALK_MAIN, 0);
- return rc;
+ *length = st.st_size;
+#endif
+ return MDBX_SUCCESS;
}
-int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) {
- int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd) {
+#if defined(_WIN32) || defined(_WIN64)
+ switch (GetFileType(fd)) {
+ case FILE_TYPE_DISK:
+ return MDBX_RESULT_FALSE;
+ case FILE_TYPE_CHAR:
+ case FILE_TYPE_PIPE:
+ return MDBX_RESULT_TRUE;
+ default:
+ return (int)GetLastError();
+ }
+#else
+ struct stat info;
+ if (fstat(fd, &info))
+ return errno;
+ switch (info.st_mode & S_IFMT) {
+ case S_IFBLK:
+ case S_IFREG:
+ return MDBX_RESULT_FALSE;
+ case S_IFCHR:
+ case S_IFIFO:
+ case S_IFSOCK:
+ return MDBX_RESULT_TRUE;
+ case S_IFDIR:
+ case S_IFLNK:
+ default:
+ return MDBX_INCOMPATIBLE;
+ }
+#endif
+}
- if (likely(canary)) {
- if (txn->mt_canary.x == canary->x && txn->mt_canary.y == canary->y &&
- txn->mt_canary.z == canary->z)
- return MDBX_SUCCESS;
- txn->mt_canary.x = canary->x;
- txn->mt_canary.y = canary->y;
- txn->mt_canary.z = canary->z;
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) {
+#if defined(_WIN32) || defined(_WIN64)
+ if (mdbx_SetFileInformationByHandle) {
+ FILE_END_OF_FILE_INFO EndOfFileInfo;
+ EndOfFileInfo.EndOfFile.QuadPart = length;
+ return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo,
+ &EndOfFileInfo,
+ sizeof(FILE_END_OF_FILE_INFO))
+ ? MDBX_SUCCESS
+ : (int)GetLastError();
+ } else {
+ LARGE_INTEGER li;
+ li.QuadPart = length;
+ return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd))
+ ? MDBX_SUCCESS
+ : (int)GetLastError();
}
- txn->mt_canary.v = txn->mt_txnid;
- txn->mt_flags |= MDBX_TXN_DIRTY;
+#else
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno;
+#endif
+}
- return MDBX_SUCCESS;
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos) {
+#if defined(_WIN32) || defined(_WIN64)
+ LARGE_INTEGER li;
+ li.QuadPart = pos;
+ return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS
+ : (int)GetLastError();
+#else
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS;
+#endif
}
-int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/*----------------------------------------------------------------------------*/
- if (unlikely(canary == NULL))
- return MDBX_EINVAL;
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg) {
+#if defined(_WIN32) || defined(_WIN64)
+ *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL);
+ return *thread ? MDBX_SUCCESS : (int)GetLastError();
+#else
+ return pthread_create(thread, NULL, start_routine, arg);
+#endif
+}
- *canary = txn->mt_canary;
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD code = WaitForSingleObject(thread, INFINITE);
+ return waitstatus2errcode(code);
+#else
+ void *unused_retval = &unused_retval;
+ return pthread_join(thread, &unused_retval);
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits) {
+ uint8_t *ptr = (uint8_t *)map->address + offset;
+#if defined(_WIN32) || defined(_WIN64)
+ if (!FlushViewOfFile(ptr, length))
+ return (int)GetLastError();
+#else
+#if defined(__linux__) || defined(__gnu_linux__)
+ if (mode_bits == MDBX_SYNC_NONE && mdbx_linux_kernel_version > 0x02061300)
+ /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly
+ * tracks dirty pages and flushes them to storage as necessary. */
+ return MDBX_SUCCESS;
+#endif /* Linux */
+ if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC))
+ return errno;
+ mode_bits &= ~MDBX_SYNC_DATA;
+#endif
+ return mdbx_fsync(map->fd, mode_bits);
+}
+
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err) {
+#if defined(_WIN32) || defined(_WIN64)
+ (void)pathname;
+ (void)err;
+ if (!mdbx_GetVolumeInformationByHandleW)
+ return MDBX_ENOSYS;
+ DWORD unused, flags;
+ if (!mdbx_GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused,
+ &flags, nullptr, 0))
+ return (int)GetLastError();
+ if ((flags & FILE_READ_ONLY_VOLUME) == 0)
+ return MDBX_EACCESS;
+#else
+ struct statvfs info;
+ if (err != MDBX_ENOFILE) {
+ if (statvfs(pathname, &info))
+ return errno;
+ if ((info.f_flag & ST_RDONLY) == 0)
+ return err;
+ }
+ if (fstatvfs(handle, &info))
+ return errno;
+ if ((info.f_flag & ST_RDONLY) == 0)
+ return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err;
+#endif /* !Windows */
return MDBX_SUCCESS;
}
-int mdbx_cursor_on_first(const MDBX_cursor *mc) {
- if (unlikely(mc == NULL))
- return MDBX_EINVAL;
+static int mdbx_check_fs_local(mdbx_filehandle_t handle, int flags) {
+#if defined(_WIN32) || defined(_WIN64)
+ if (mdbx_RunningUnderWine() && !(flags & MDBX_EXCLUSIVE))
+ return ERROR_NOT_CAPABLE /* workaround for Wine */;
+
+ if (GetFileType(handle) != FILE_TYPE_DISK)
+ return ERROR_FILE_OFFLINE;
+
+ if (mdbx_GetFileInformationByHandleEx) {
+ FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo;
+ if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo,
+ &RemoteProtocolInfo,
+ sizeof(RemoteProtocolInfo))) {
+ if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) &&
+ !(flags & MDBX_RDONLY))
+ return ERROR_FILE_OFFLINE;
+ if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) &&
+ !(flags & MDBX_EXCLUSIVE))
+ return ERROR_REMOTE_STORAGE_MEDIA_ERROR;
+ }
+ }
+
+ if (mdbx_NtFsControlFile) {
+ NTSTATUS rc;
+ struct {
+ WOF_EXTERNAL_INFO wof_info;
+ union {
+ WIM_PROVIDER_EXTERNAL_INFO wim_info;
+ FILE_PROVIDER_EXTERNAL_INFO_V1 file_info;
+ };
+ size_t reserved_for_microsoft_madness[42];
+ } GetExternalBacking_OutputBuffer;
+ IO_STATUS_BLOCK StatusBlock;
+ rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock,
+ FSCTL_GET_EXTERNAL_BACKING, NULL, 0,
+ &GetExternalBacking_OutputBuffer,
+ sizeof(GetExternalBacking_OutputBuffer));
+ if (NT_SUCCESS(rc)) {
+ if (!(flags & MDBX_EXCLUSIVE))
+ return ERROR_REMOTE_STORAGE_MEDIA_ERROR;
+ } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED &&
+ rc != STATUS_INVALID_DEVICE_REQUEST &&
+ rc != STATUS_NOT_SUPPORTED)
+ return ntstatus2errcode(rc);
+ }
+
+ if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) {
+ WCHAR *PathBuffer = mdbx_malloc(sizeof(WCHAR) * INT16_MAX);
+ if (!PathBuffer)
+ return MDBX_ENOMEM;
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ int rc = MDBX_SUCCESS;
+ DWORD VolumeSerialNumber, FileSystemFlags;
+ if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX,
+ &VolumeSerialNumber, NULL,
+ &FileSystemFlags, NULL, 0)) {
+ rc = (int)GetLastError();
+ goto bailout;
+ }
+
+ if ((flags & MDBX_RDONLY) == 0) {
+ if (FileSystemFlags &
+ (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME |
+ FILE_VOLUME_IS_COMPRESSED)) {
+ rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
+ goto bailout;
+ }
+ }
- if (!(mc->mc_flags & C_INITIALIZED))
- return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
+ if (!mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX,
+ FILE_NAME_NORMALIZED |
+ VOLUME_NAME_NT)) {
+ rc = (int)GetLastError();
+ goto bailout;
+ }
- for (unsigned i = 0; i < mc->mc_snum; ++i) {
- if (mc->mc_ki[i])
- return MDBX_RESULT_FALSE;
+ if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) {
+ if (!(flags & MDBX_EXCLUSIVE)) {
+ rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
+ goto bailout;
+ }
+ } else if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX,
+ FILE_NAME_NORMALIZED |
+ VOLUME_NAME_DOS)) {
+ UINT DriveType = GetDriveTypeW(PathBuffer);
+ if (DriveType == DRIVE_NO_ROOT_DIR &&
+ _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 &&
+ _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) {
+ PathBuffer[7] = 0;
+ DriveType = GetDriveTypeW(PathBuffer + 4);
+ }
+ switch (DriveType) {
+ case DRIVE_CDROM:
+ if (flags & MDBX_RDONLY)
+ break;
+ // fall through
+ case DRIVE_UNKNOWN:
+ case DRIVE_NO_ROOT_DIR:
+ case DRIVE_REMOTE:
+ default:
+ if (!(flags & MDBX_EXCLUSIVE))
+ rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
+ // fall through
+ case DRIVE_REMOVABLE:
+ case DRIVE_FIXED:
+ case DRIVE_RAMDISK:
+ break;
+ }
+ }
+ bailout:
+ mdbx_free(PathBuffer);
+ return rc;
}
- return MDBX_RESULT_TRUE;
-}
+#else
-int mdbx_cursor_on_last(const MDBX_cursor *mc) {
- if (unlikely(mc == NULL))
- return MDBX_EINVAL;
+ struct statvfs statvfs_info;
+ if (fstatvfs(handle, &statvfs_info))
+ return errno;
+#if defined(ST_LOCAL) || defined(ST_EXPORTED)
+ const unsigned long st_flags = statvfs_info.f_flag;
+#endif /* ST_LOCAL || ST_EXPORTED */
+
+#if defined(__NetBSD__)
+ const unsigned type = 0;
+ const char *const name = statvfs_info.f_fstypename;
+ const size_t name_len = VFS_NAMELEN;
+#elif defined(_AIX) || defined(__OS400__)
+ const char *const name = statvfs_info.f_basetype;
+ const size_t name_len = sizeof(statvfs_info.f_basetype);
+ struct stat st;
+ if (fstat(handle, &st))
+ return errno;
+ const unsigned type = st.st_vfstype;
+ if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE))
+ return MDBX_EREMOTE;
+#elif defined(FSTYPSZ) || defined(_FSTYPSZ)
+ const unsigned type = 0;
+ const char *const name = statvfs_info.f_basetype;
+ const size_t name_len = sizeof(statvfs_info.f_basetype);
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ)
+ const unsigned type = 0;
+ struct stat st;
+ if (fstat(handle, &st))
+ return errno;
+ const char *const name = st.st_fstype;
+ const size_t name_len = strlen(name);
+#else
+ struct statfs statfs_info;
+ if (fstatfs(handle, &statfs_info))
+ return errno;
+#if defined(__OpenBSD__)
+ const unsigned type = 0;
+#else
+ const unsigned type = statfs_info.f_type;
+#endif
+#if defined(MNT_LOCAL) || defined(MNT_EXPORTED)
+ const unsigned long mnt_flags = statfs_info.f_flags;
+#endif /* MNT_LOCAL || MNT_EXPORTED */
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
+ defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
+ const char *const name = statfs_info.f_fstypename;
+ const size_t name_len = sizeof(statfs_info.f_fstypename);
+#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21
+ const char *const name = "";
+ const unsigned name_len = 0;
+#else
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ const char *name = "";
+ unsigned name_len = 0;
- if (!(mc->mc_flags & C_INITIALIZED))
- return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
+ struct stat st;
+ if (fstat(handle, &st))
+ return errno;
- for (unsigned i = 0; i < mc->mc_snum; ++i) {
- unsigned nkeys = page_numkeys(mc->mc_pg[i]);
- if (mc->mc_ki[i] < nkeys - 1)
- return MDBX_RESULT_FALSE;
+ char pathbuf[PATH_MAX];
+ FILE *mounted = nullptr;
+#if defined(__linux__) || defined(__gnu_linux__)
+ mounted = setmntent("/proc/mounts", "r");
+#endif /* Linux */
+ if (!mounted)
+ mounted = setmntent("/etc/mtab", "r");
+ if (mounted) {
+ const struct mntent *ent;
+#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \
+ (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19))
+ struct mntent entbuf;
+ const bool should_copy = false;
+ while (nullptr !=
+ (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf))))
+#else
+ const bool should_copy = true;
+ while (nullptr != (ent = getmntent(mounted)))
+#endif
+ {
+ struct stat mnt;
+ if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) {
+ if (should_copy) {
+ name =
+ strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1);
+ pathbuf[name_len] = 0;
+ } else {
+ name = ent->mnt_fsname;
+ name_len = strlen(name);
+ }
+ break;
+ }
+ }
+ endmntent(mounted);
}
+#endif /* !xBSD && !Android/Bionic */
+#endif
- return MDBX_RESULT_TRUE;
+ if (name_len) {
+ if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) ||
+ strncasecmp("cifs", name, name_len) == 0 ||
+ strncasecmp("ncpfs", name, name_len) == 0 ||
+ strncasecmp("smbfs", name, name_len) == 0 ||
+ strcasecmp("9P" /* WSL2 */, name) == 0 ||
+ ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) &&
+ strncasecmp("fuseblk", name, name_len) != 0)) &&
+ !(flags & MDBX_EXCLUSIVE))
+ return MDBX_EREMOTE;
+ if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 ||
+ strcasecmp("sshfs", name) == 0)
+ return MDBX_EREMOTE;
+ }
+
+#ifdef ST_LOCAL
+ if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
+ return MDBX_EREMOTE;
+#elif defined(MNT_LOCAL)
+ if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
+ return MDBX_EREMOTE;
+#endif /* ST/MNT_LOCAL */
+
+#ifdef ST_EXPORTED
+ if ((st_flags & ST_EXPORTED) != 0 && !(flags & MDBX_RDONLY))
+ return MDBX_EREMOTE;
+#elif defined(MNT_EXPORTED)
+ if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & MDBX_RDONLY))
+ return MDBX_EREMOTE;
+#endif /* ST/MNT_EXPORTED */
+
+ switch (type) {
+ case 0xFF534D42 /* CIFS_MAGIC_NUMBER */:
+ case 0x6969 /* NFS_SUPER_MAGIC */:
+ case 0x564c /* NCP_SUPER_MAGIC */:
+ case 0x517B /* SMB_SUPER_MAGIC */:
+#if defined(__digital__) || defined(__osf__) || defined(__osf)
+ case 0x0E /* Tru64 NFS */:
+#endif
+#ifdef ST_FST_NFS
+ case ST_FST_NFS:
+#endif
+ if ((flags & MDBX_EXCLUSIVE) == 0)
+ return MDBX_EREMOTE;
+ case 0:
+ default:
+ break;
+ }
+#endif /* Unix */
+
+ return MDBX_SUCCESS;
}
-int mdbx_cursor_eof(const MDBX_cursor *mc) {
- if (unlikely(mc == NULL))
- return MDBX_EINVAL;
+static int check_mmap_limit(const size_t limit) {
+ const bool should_check =
+#if defined(__SANITIZE_ADDRESS__)
+ true;
+#else
+ RUNNING_ON_VALGRIND;
+#endif /* __SANITIZE_ADDRESS__ */
- if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
- return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ if (should_check) {
+ intptr_t pagesize, total_ram_pages, avail_ram_pages;
+ int err =
+ mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- return ((mc->mc_flags & (C_INITIALIZED | C_EOF)) == C_INITIALIZED &&
- mc->mc_snum &&
- mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]))
- ? MDBX_RESULT_FALSE
- : MDBX_RESULT_TRUE;
+ const int log2page = log2n_powerof2(pagesize);
+ if ((limit >> (log2page + 7)) > (size_t)total_ram_pages ||
+ (limit >> (log2page + 6)) > (size_t)avail_ram_pages) {
+ mdbx_error(
+ "%s (%zu pages) is too large for available (%zu pages) or total "
+ "(%zu pages) system RAM",
+ "database upper size limit", limit >> log2page, avail_ram_pages,
+ total_ram_pages);
+ return MDBX_TOO_LARGE;
+ }
+ }
+
+ return MDBX_SUCCESS;
}
-//------------------------------------------------------------------------------
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t size, const size_t limit,
+ const unsigned options) {
+ assert(size <= limit);
+ map->limit = 0;
+ map->current = 0;
+ map->address = nullptr;
+ map->filesize = 0;
+#if defined(_WIN32) || defined(_WIN64)
+ map->section = NULL;
+#endif /* Windows */
-struct diff_result {
- ptrdiff_t diff;
- unsigned level;
- int root_nkeys;
-};
+ int err = mdbx_check_fs_local(map->fd, flags);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
-/* calculates: r = x - y */
-__hot static int cursor_diff(const MDBX_cursor *const __restrict x,
- const MDBX_cursor *const __restrict y,
- struct diff_result *const __restrict r) {
- r->diff = 0;
- r->level = 0;
- r->root_nkeys = 0;
+ err = check_mmap_limit(limit);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- if (unlikely(x->mc_signature != MDBX_MC_LIVE))
- return (x->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) {
+ err = mdbx_ftruncate(map->fd, size);
+ if (err != MDBX_SUCCESS)
+ return err;
+ map->filesize = size;
+#if !(defined(_WIN32) || defined(_WIN64))
+ map->current = size;
+#endif /* !Windows */
+ } else {
+ err = mdbx_filesize(map->fd, &map->filesize);
+ if (err != MDBX_SUCCESS)
+ return err;
+#if !(defined(_WIN32) || defined(_WIN64))
+ map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
+#endif /* !Windows */
+ }
- if (unlikely(y->mc_signature != MDBX_MC_LIVE))
- return (y->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+#if defined(_WIN32) || defined(_WIN64)
+ LARGE_INTEGER SectionSize;
+ SectionSize.QuadPart = size;
+ err = NtCreateSection(
+ &map->section,
+ /* DesiredAccess */
+ (flags & MDBX_WRITEMAP)
+ ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
+ SECTION_MAP_WRITE
+ : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
+ /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize,
+ /* SectionPageProtection */
+ (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
+ /* AllocationAttributes */ SEC_RESERVE, map->fd);
+ if (!NT_SUCCESS(err))
+ return ntstatus2errcode(err);
+
+ SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0
+ : mdbx_RunningUnderWine() ? size
+ : limit;
+ err = NtMapViewOfSection(
+ map->section, GetCurrentProcess(), &map->address,
+ /* ZeroBits */ 0,
+ /* CommitSize */ 0,
+ /* SectionOffset */ NULL, &ViewSize,
+ /* InheritDisposition */ ViewUnmap,
+ /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
+ /* Win32Protect */
+ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
+ if (!NT_SUCCESS(err)) {
+ NtClose(map->section);
+ map->section = 0;
+ map->address = nullptr;
+ return ntstatus2errcode(err);
+ }
+ assert(map->address != MAP_FAILED);
+
+ map->current = (size_t)SectionSize.QuadPart;
+ map->limit = ViewSize;
- int rc = check_txn(x->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#else /* Windows */
- if (unlikely(x->mc_txn != y->mc_txn))
- return MDBX_BAD_TXN;
+#ifndef MAP_TRYFIXED
+#define MAP_TRYFIXED 0
+#endif
- if (unlikely(y->mc_dbi != x->mc_dbi))
- return MDBX_EINVAL;
+#ifndef MAP_HASSEMAPHORE
+#define MAP_HASSEMAPHORE 0
+#endif
- if (unlikely(!(y->mc_flags & x->mc_flags & C_INITIALIZED)))
- return MDBX_ENODATA;
+#ifndef MAP_CONCEAL
+#define MAP_CONCEAL 0
+#endif
- while (likely(r->level < y->mc_snum && r->level < x->mc_snum)) {
- if (unlikely(y->mc_pg[r->level] != x->mc_pg[r->level])) {
- mdbx_error("Mismatch cursors's pages at %u level", r->level);
- return MDBX_PROBLEM;
- }
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0
+#endif
- int nkeys = page_numkeys(y->mc_pg[r->level]);
- assert(nkeys > 0);
- if (r->level == 0)
- r->root_nkeys = nkeys;
+#ifndef MAP_FIXED_NOREPLACE
+#define MAP_FIXED_NOREPLACE 0
+#endif
- const int limit_ki = nkeys - 1;
- const int x_ki = x->mc_ki[r->level];
- const int y_ki = y->mc_ki[r->level];
- r->diff = ((x_ki < limit_ki) ? x_ki : limit_ki) -
- ((y_ki < limit_ki) ? y_ki : limit_ki);
- if (r->diff == 0) {
- r->level += 1;
- continue;
- }
+#ifndef MAP_NORESERVE
+#define MAP_NORESERVE 0
+#endif
- while (unlikely(r->diff == 1) &&
- likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) {
- r->level += 1;
- /* DB'PAGEs: 0------------------>MAX
- *
- * CURSORs: y < x
- * STACK[i ]: |
- * STACK[+1]: ...y++N|0++x...
- */
- nkeys = page_numkeys(y->mc_pg[r->level]);
- r->diff = (nkeys - y->mc_ki[r->level]) + x->mc_ki[r->level];
- assert(r->diff > 0);
- }
+ map->address = mmap(
+ NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ,
+ MAP_SHARED | MAP_FILE | MAP_NORESERVE |
+ (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) |
+ ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC
+ : MAP_CONCEAL),
+ map->fd, 0);
+
+ if (unlikely(map->address == MAP_FAILED)) {
+ map->limit = 0;
+ map->current = 0;
+ map->address = nullptr;
+ return errno;
+ }
+ map->limit = limit;
- while (unlikely(r->diff == -1) &&
- likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) {
- r->level += 1;
- /* DB'PAGEs: 0------------------>MAX
- *
- * CURSORs: x < y
- * STACK[i ]: |
- * STACK[+1]: ...x--N|0--y...
- */
- nkeys = page_numkeys(x->mc_pg[r->level]);
- r->diff = -(nkeys - x->mc_ki[r->level]) - y->mc_ki[r->level];
- assert(r->diff < 0);
- }
+#if MDBX_ENABLE_MADVISE
+#ifdef MADV_DONTFORK
+ if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0))
+ return errno;
+#endif /* MADV_DONTFORK */
+#ifdef MADV_NOHUGEPAGE
+ (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE);
+#endif /* MADV_NOHUGEPAGE */
+#endif /* MDBX_ENABLE_MADVISE */
- return MDBX_SUCCESS;
- }
+#endif /* ! Windows */
- r->diff = CMP2INT(x->mc_flags & C_EOF, y->mc_flags & C_EOF);
+ VALGRIND_MAKE_MEM_DEFINED(map->address, map->current);
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address, map->current);
return MDBX_SUCCESS;
}
-__hot static ptrdiff_t estimate(const MDBX_db *db,
- struct diff_result *const __restrict dr) {
- /* root: branch-page => scale = leaf-factor * branch-factor^(N-1)
- * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2
- * level-2: branch-page(s) => scale = leaf-factor * branch-factor
- * level-N: branch-page(s) => scale = leaf-factor
- * leaf-level: leaf-page(s) => scale = 1
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map) {
+ VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
+ * when this memory will re-used by malloc or another mmapping.
+ * See todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
*/
- ptrdiff_t btree_power = (ptrdiff_t)db->md_depth - 2 - (ptrdiff_t)dr->level;
- if (btree_power < 0)
- return dr->diff;
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address,
+ (map->filesize && map->filesize < map->limit)
+ ? map->filesize
+ : map->limit);
+#if defined(_WIN32) || defined(_WIN64)
+ if (map->section)
+ NtClose(map->section);
+ NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->address);
+ if (!NT_SUCCESS(rc))
+ ntstatus2errcode(rc);
+#else
+ if (unlikely(munmap(map->address, map->limit)))
+ return errno;
+#endif /* ! Windows */
- ptrdiff_t estimated =
- (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)db->md_leaf_pages;
- if (btree_power == 0)
- return estimated;
+ map->limit = 0;
+ map->current = 0;
+ map->address = nullptr;
+ return MDBX_SUCCESS;
+}
- if (db->md_depth < 4) {
- assert(dr->level == 0 && btree_power == 1);
- return (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)dr->root_nkeys;
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit) {
+ assert(size <= limit);
+#if defined(_WIN32) || defined(_WIN64)
+ assert(size != map->current || limit != map->limit || size < map->filesize);
+
+ NTSTATUS status;
+ LARGE_INTEGER SectionSize;
+ int err, rc = MDBX_SUCCESS;
+
+ if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current &&
+ /* workaround for Wine */ mdbx_NtExtendSection) {
+ /* growth rw-section */
+ SectionSize.QuadPart = size;
+ status = mdbx_NtExtendSection(map->section, &SectionSize);
+ if (!NT_SUCCESS(status))
+ return ntstatus2errcode(status);
+ map->current = size;
+ if (map->filesize < size)
+ map->filesize = size;
+ return MDBX_SUCCESS;
}
- /* average_branchpage_fillfactor = total(branch_entries) / branch_pages
- total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */
- const size_t log2_fixedpoint = sizeof(size_t) - 1;
- const size_t half = UINT64_C(1) << (log2_fixedpoint - 1);
- const size_t factor =
- ((db->md_leaf_pages + db->md_branch_pages - 1) << log2_fixedpoint) /
- db->md_branch_pages;
- while (1) {
- switch ((size_t)btree_power) {
- default: {
- const size_t square = (factor * factor + half) >> log2_fixedpoint;
- const size_t quad = (square * square + half) >> log2_fixedpoint;
- do {
- estimated = estimated * quad + half;
- estimated >>= log2_fixedpoint;
- btree_power -= 4;
- } while (btree_power >= 4);
- continue;
- }
- case 3:
- estimated = estimated * factor + half;
- estimated >>= log2_fixedpoint;
- __fallthrough /* fall through */;
- case 2:
- estimated = estimated * factor + half;
- estimated >>= log2_fixedpoint;
- __fallthrough /* fall through */;
- case 1:
- estimated = estimated * factor + half;
- estimated >>= log2_fixedpoint;
- __fallthrough /* fall through */;
- case 0:
- if (unlikely(estimated > (ptrdiff_t)db->md_entries))
- return (ptrdiff_t)db->md_entries;
- if (unlikely(estimated < -(ptrdiff_t)db->md_entries))
- return -(ptrdiff_t)db->md_entries;
- return estimated;
+ if (limit > map->limit) {
+ err = check_mmap_limit(limit);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
+
+ /* check ability of address space for growth before unmap */
+ PVOID BaseAddress = (PBYTE)map->address + map->limit;
+ SIZE_T RegionSize = limit - map->limit;
+ status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0,
+ &RegionSize, MEM_RESERVE, PAGE_NOACCESS);
+ if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
+ if (!NT_SUCCESS(status))
+ return ntstatus2errcode(status);
+
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize,
+ MEM_RELEASE);
+ if (!NT_SUCCESS(status))
+ return ntstatus2errcode(status);
+ }
+
+ /* Windows unable:
+ * - shrink a mapped file;
+ * - change size of mapped view;
+ * - extend read-only mapping;
+ * Therefore we should unmap/map entire section. */
+ if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0)
+ return MDBX_EPERM;
+
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
+ * when this memory will re-used by malloc or another mmapping.
+ * See todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
+ */
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit);
+ status = NtUnmapViewOfSection(GetCurrentProcess(), map->address);
+ if (!NT_SUCCESS(status))
+ return ntstatus2errcode(status);
+ status = NtClose(map->section);
+ map->section = NULL;
+ PVOID ReservedAddress = NULL;
+ SIZE_T ReservedSize = limit;
+
+ if (!NT_SUCCESS(status)) {
+ bailout_ntstatus:
+ err = ntstatus2errcode(status);
+ bailout:
+ map->address = NULL;
+ map->current = map->limit = 0;
+ if (ReservedAddress) {
+ ReservedSize = 0;
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress,
+ &ReservedSize, MEM_RELEASE);
+ assert(NT_SUCCESS(status));
+ (void)status;
}
+ return err;
}
-}
-int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last,
- ptrdiff_t *distance_items) {
- if (unlikely(first == NULL || last == NULL || distance_items == NULL))
- return MDBX_EINVAL;
+retry_file_and_section:
+ /* resizing of the file may take a while,
+ * therefore we reserve address space to avoid occupy it by other threads */
+ ReservedAddress = map->address;
+ status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0,
+ &ReservedSize, MEM_RESERVE, PAGE_NOACCESS);
+ if (!NT_SUCCESS(status)) {
+ ReservedAddress = NULL;
+ if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
+ goto bailout_ntstatus /* no way to recovery */;
- *distance_items = 0;
- struct diff_result dr;
- int rc = cursor_diff(last, first, &dr);
- if (unlikely(rc != MDBX_SUCCESS))
+ if (flags & MDBX_MRESIZE_MAY_MOVE)
+ /* the base address could be changed */
+ map->address = NULL;
+ }
+
+ err = mdbx_filesize(map->fd, &map->filesize);
+ if (err != MDBX_SUCCESS)
+ goto bailout;
+
+ if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) {
+ err = mdbx_ftruncate(map->fd, size);
+ if (err == MDBX_SUCCESS)
+ map->filesize = size;
+ /* ignore error, because Windows unable shrink file
+ * that already mapped (by another process) */
+ }
+
+ SectionSize.QuadPart = size;
+ status = NtCreateSection(
+ &map->section,
+ /* DesiredAccess */
+ (flags & MDBX_WRITEMAP)
+ ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
+ SECTION_MAP_WRITE
+ : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
+ /* ObjectAttributes */ NULL,
+ /* MaximumSize (InitialSize) */ &SectionSize,
+ /* SectionPageProtection */
+ (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
+ /* AllocationAttributes */ SEC_RESERVE, map->fd);
+
+ if (!NT_SUCCESS(status))
+ goto bailout_ntstatus;
+
+ if (ReservedAddress) {
+ /* release reserved address space */
+ ReservedSize = 0;
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress,
+ &ReservedSize, MEM_RELEASE);
+ ReservedAddress = NULL;
+ if (!NT_SUCCESS(status))
+ goto bailout_ntstatus;
+ }
+
+retry_mapview:;
+ SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit;
+ status = NtMapViewOfSection(
+ map->section, GetCurrentProcess(), &map->address,
+ /* ZeroBits */ 0,
+ /* CommitSize */ 0,
+ /* SectionOffset */ NULL, &ViewSize,
+ /* InheritDisposition */ ViewUnmap,
+ /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
+ /* Win32Protect */
+ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
+
+ if (!NT_SUCCESS(status)) {
+ if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 &&
+ map->address && (flags & MDBX_MRESIZE_MAY_MOVE) != 0) {
+ /* try remap at another base address */
+ map->address = NULL;
+ goto retry_mapview;
+ }
+ NtClose(map->section);
+ map->section = NULL;
+
+ if (map->address && (size != map->current || limit != map->limit)) {
+ /* try remap with previously size and limit,
+ * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
+ rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
+ size = map->current;
+ ReservedSize = limit = map->limit;
+ goto retry_file_and_section;
+ }
+
+ /* no way to recovery */
+ goto bailout_ntstatus;
+ }
+ assert(map->address != MAP_FAILED);
+
+ map->current = (size_t)SectionSize.QuadPart;
+ map->limit = ViewSize;
+
+#else /* Windows */
+
+ map->filesize = 0;
+ int rc = mdbx_filesize(map->fd, &map->filesize);
+ if (rc != MDBX_SUCCESS)
return rc;
- if (unlikely(dr.diff == 0) &&
- F_ISSET(first->mc_db->md_flags & last->mc_db->md_flags,
- MDBX_DUPSORT | C_INITIALIZED)) {
- first = &first->mc_xcursor->mx_cursor;
- last = &last->mc_xcursor->mx_cursor;
- rc = cursor_diff(first, last, &dr);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ if (flags & MDBX_RDONLY) {
+ map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
+ if (map->current != size)
+ rc = (size > map->current) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
+ } else {
+ if (map->filesize != size) {
+ rc = mdbx_ftruncate(map->fd, size);
+ if (rc != MDBX_SUCCESS)
+ return rc;
+ map->filesize = size;
+ }
+
+ if (map->current > size) {
+ /* Clearing asan's bitmask for the region which released in shrinking,
+ * since:
+ * - after the shrinking we will get an exception when accessing
+ * this region and (therefore) do not need the help of ASAN.
+ * - this allows us to clear the mask only within the file size
+ * when closing the mapping. */
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(
+ (char *)map->address + size,
+ ((map->current < map->limit) ? map->current : map->limit) - size);
+ }
+ map->current = size;
}
- if (likely(dr.diff != 0))
- *distance_items = estimate(first->mc_db, &dr);
+ if (limit == map->limit)
+ return rc;
- return MDBX_SUCCESS;
-}
+ if (limit < map->limit) {
+ /* unmap an excess at end of mapping. */
+ // coverity[offset_free : FALSE]
+ if (unlikely(munmap(map->dxb + limit, map->limit - limit)))
+ return errno;
+ map->limit = limit;
+ return rc;
+ }
-int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data,
- MDBX_cursor_op move_op, ptrdiff_t *distance_items) {
- if (unlikely(cursor == NULL || distance_items == NULL ||
- move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE))
- return MDBX_EINVAL;
+ int err = check_mmap_limit(limit);
+ if (unlikely(err != MDBX_SUCCESS))
+ return err;
- if (unlikely(cursor->mc_signature != MDBX_MC_LIVE))
- return (cursor->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
- : MDBX_EBADSIGN;
+ assert(limit > map->limit);
+ uint8_t *ptr = MAP_FAILED;
- int rc = check_txn(cursor->mc_txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#if defined(MREMAP_MAYMOVE)
+ ptr = mremap(map->address, map->limit, limit,
+ (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE : 0);
+ if (ptr == MAP_FAILED) {
+ err = errno;
+ switch (err) {
+ default:
+ return err;
+ case EAGAIN:
+ case ENOMEM:
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
+ case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
+ break;
+ }
+ }
+#endif /* MREMAP_MAYMOVE */
+
+ const unsigned mmap_flags =
+ MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE |
+ (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0);
+ const unsigned mmap_prot =
+ (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ;
+
+ if (ptr == MAP_FAILED) {
+ /* Try to mmap additional space beyond the end of mapping. */
+ ptr = mmap(map->dxb + map->limit, limit - map->limit, mmap_prot,
+ mmap_flags | MAP_FIXED_NOREPLACE, map->fd, map->limit);
+ if (ptr == map->dxb + map->limit)
+ ptr = map->dxb;
+ else if (ptr != MAP_FAILED) {
+ /* the desired address is busy, unmap unsuitable one */
+ if (unlikely(munmap(ptr, limit - map->limit)))
+ return errno;
+ ptr = MAP_FAILED;
+ } else {
+ err = errno;
+ switch (err) {
+ default:
+ return err;
+ case EAGAIN:
+ case ENOMEM:
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
+ case EEXIST: /* address busy */
+ case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */
+ break;
+ }
+ }
+ }
- if (!(cursor->mc_flags & C_INITIALIZED))
- return MDBX_ENODATA;
+ if (ptr == MAP_FAILED) {
+ /* unmap and map again whole region */
+ if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
+ /* TODO: Perhaps here it is worth to implement suspend/resume threads
+ * and perform unmap/map as like for Windows. */
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
+ }
+
+ if (unlikely(munmap(map->address, map->limit)))
+ return errno;
+
+ // coverity[pass_freed_arg : FALSE]
+ ptr = mmap(map->address, limit, mmap_prot,
+ (flags & MDBX_MRESIZE_MAY_MOVE)
+ ? mmap_flags
+ : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE
+ : MAP_FIXED),
+ map->fd, 0);
+ if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED &&
+ unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) &&
+ errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
+ // coverity[pass_freed_arg : FALSE]
+ ptr = mmap(map->address, limit, mmap_prot, mmap_flags | MAP_FIXED,
+ map->fd, 0);
+
+ if (unlikely(ptr == MAP_FAILED)) {
+ /* try to restore prev mapping */
+ // coverity[pass_freed_arg : FALSE]
+ ptr = mmap(map->address, map->limit, mmap_prot,
+ (flags & MDBX_MRESIZE_MAY_MOVE)
+ ? mmap_flags
+ : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE
+ : MAP_FIXED),
+ map->fd, 0);
+ if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED &&
+ unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) &&
+ errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
+ // coverity[pass_freed_arg : FALSE]
+ ptr = mmap(map->address, map->limit, mmap_prot, mmap_flags | MAP_FIXED,
+ map->fd, 0);
+ if (unlikely(ptr == MAP_FAILED)) {
+ VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
+ * when this memory will re-used by malloc or another mmapping.
+ * See
+ * todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
+ */
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(
+ map->address,
+ (map->current < map->limit) ? map->current : map->limit);
+ map->limit = 0;
+ map->current = 0;
+ map->address = nullptr;
+ return errno;
+ }
+ rc = MDBX_UNABLE_EXTEND_MAPSIZE;
+ limit = map->limit;
+ }
+ }
+
+ assert(ptr && ptr != MAP_FAILED);
+ if (map->address != ptr) {
+ VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
+ * when this memory will re-used by malloc or another mmapping.
+ * See
+ * todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
+ */
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(
+ map->address, (map->current < map->limit) ? map->current : map->limit);
- MDBX_cursor_couple next;
- cursor_copy(cursor, &next.outer);
- if (cursor->mc_db->md_flags & MDBX_DUPSORT) {
- next.outer.mc_xcursor = &next.inner;
- rc = mdbx_xcursor_init0(&next.outer);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- MDBX_xcursor *mx = &container_of(cursor, MDBX_cursor_couple, outer)->inner;
- cursor_copy(&mx->mx_cursor, &next.inner.mx_cursor);
+ VALGRIND_MAKE_MEM_DEFINED(ptr, map->current);
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current);
+ map->address = ptr;
}
+ map->limit = limit;
- MDBX_val stub = {0, 0};
- if (data == NULL) {
- const unsigned mask =
- 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY;
- if (unlikely(mask & (1 << move_op)))
- return MDBX_EINVAL;
- data = &stub;
+#if MDBX_ENABLE_MADVISE
+#ifdef MADV_DONTFORK
+ if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0))
+ return errno;
+#endif /* MADV_DONTFORK */
+#ifdef MADV_NOHUGEPAGE
+ (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE);
+#endif /* MADV_NOHUGEPAGE */
+#endif /* MDBX_ENABLE_MADVISE */
+
+#endif /* POSIX / Windows */
+
+ return rc;
+}
+
+/*----------------------------------------------------------------------------*/
+
+__cold MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny) {
+ for (;;) {
+#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
+ defined(__x86_64__)
+ const unsigned salt = 277u * (unsigned)__rdtsc();
+#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT
+ static ULONG state;
+ const unsigned salt = (unsigned)RtlRandomEx(&state);
+#else
+ const unsigned salt = rand();
+#endif
+
+ const unsigned coin = salt % (tiny ? 29u : 43u);
+ if (coin < 43 / 3)
+ break;
+#if defined(_WIN32) || defined(_WIN64)
+ SwitchToThread();
+ if (coin > 43 * 2 / 3)
+ Sleep(1);
+#else
+ sched_yield();
+ if (coin > 43 * 2 / 3)
+ usleep(coin);
+#endif
}
+}
- if (key == NULL) {
- const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE |
- 1 << MDBX_SET_KEY | 1 << MDBX_SET |
- 1 << MDBX_SET_RANGE;
- if (unlikely(mask & (1 << move_op)))
- return MDBX_EINVAL;
- key = &stub;
+#if defined(_WIN32) || defined(_WIN64)
+#elif defined(__APPLE__) || defined(__MACH__)
+#include <mach/mach_time.h>
+#elif defined(__linux__) || defined(__gnu_linux__)
+__cold static clockid_t choice_monoclock(void) {
+ struct timespec probe;
+#if defined(CLOCK_BOOTTIME)
+ if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0)
+ return CLOCK_BOOTTIME;
+#elif defined(CLOCK_MONOTONIC_RAW)
+ if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0)
+ return CLOCK_MONOTONIC_RAW;
+#elif defined(CLOCK_MONOTONIC_COARSE)
+ if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0)
+ return CLOCK_MONOTONIC_COARSE;
+#endif
+ return CLOCK_MONOTONIC;
+}
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_WIN32) || defined(_WIN64)
+static LARGE_INTEGER performance_frequency;
+#elif defined(__APPLE__) || defined(__MACH__)
+static uint64_t ratio_16dot16_to_monotine;
+#endif
+
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16) {
+#if defined(_WIN32) || defined(_WIN64)
+ if (unlikely(performance_frequency.QuadPart == 0))
+ QueryPerformanceFrequency(&performance_frequency);
+ const uint64_t ratio = performance_frequency.QuadPart;
+#elif defined(__APPLE__) || defined(__MACH__)
+ if (unlikely(ratio_16dot16_to_monotine == 0)) {
+ mach_timebase_info_data_t ti;
+ mach_timebase_info(&ti);
+ ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer;
+ }
+ const uint64_t ratio = ratio_16dot16_to_monotine;
+#else
+ const uint64_t ratio = UINT64_C(1000000000);
+#endif
+ const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16;
+ return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1;
+}
+
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime) {
+ static uint64_t limit;
+ if (unlikely(monotime > limit)) {
+ if (limit != 0)
+ return UINT32_MAX;
+ limit = mdbx_osal_16dot16_to_monotime(UINT32_MAX - 1);
+ if (monotime > limit)
+ return UINT32_MAX;
}
+ const uint32_t ret =
+#if defined(_WIN32) || defined(_WIN64)
+ (uint32_t)((monotime << 16) / performance_frequency.QuadPart);
+#elif defined(__APPLE__) || defined(__MACH__)
+ (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine);
+#else
+ (uint32_t)(monotime * 128 / 1953125);
+#endif
+ return likely(ret || monotime == 0) ? ret : /* fix underflow */ 1;
+}
- next.outer.mc_signature = MDBX_MC_LIVE;
- rc = mdbx_cursor_get(&next.outer, key, data, move_op);
- if (unlikely(rc != MDBX_SUCCESS &&
- (rc != MDBX_NOTFOUND || !(next.outer.mc_flags & C_INITIALIZED))))
- return rc;
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ LARGE_INTEGER counter;
+ counter.QuadPart = 0;
+ QueryPerformanceCounter(&counter);
+ return counter.QuadPart;
+#elif defined(__APPLE__) || defined(__MACH__)
+ return mach_absolute_time();
+#else
- return mdbx_estimate_distance(cursor, &next.outer, distance_items);
+#if defined(__linux__) || defined(__gnu_linux__)
+ static clockid_t posix_clockid = -1;
+ if (unlikely(posix_clockid < 0))
+ posix_clockid = choice_monoclock();
+#elif defined(CLOCK_MONOTONIC)
+#define posix_clockid CLOCK_MONOTONIC
+#else
+#define posix_clockid CLOCK_REALTIME
+#endif
+
+ struct timespec ts;
+ if (unlikely(clock_gettime(posix_clockid, &ts) != 0)) {
+ ts.tv_nsec = 0;
+ ts.tv_sec = 0;
+ }
+ return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
+#endif
}
-int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key,
- MDBX_val *begin_data, MDBX_val *end_key,
- MDBX_val *end_data, ptrdiff_t *size_items) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+/*----------------------------------------------------------------------------*/
- if (unlikely(!size_items))
- return MDBX_EINVAL;
+static void bootid_shake(bin128_t *p) {
+ /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */
+ const uint32_t e = p->a - (p->b << 23 | p->b >> 9);
+ p->a = p->b ^ (p->c << 16 | p->c >> 16);
+ p->b = p->c + (p->d << 11 | p->d >> 21);
+ p->c = p->d + e;
+ p->d = e + p->a;
+}
+
+static void bootid_collect(bin128_t *p, const void *s, size_t n) {
+ p->y += UINT64_C(64526882297375213);
+ bootid_shake(p);
+ for (size_t i = 0; i < n; ++i) {
+ bootid_shake(p);
+ p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i];
+ bootid_shake(p);
+ p->y += 14621231;
+ }
+ bootid_shake(p);
+
+ /* minor non-linear tomfoolery */
+ const unsigned z = p->x % 61;
+ p->y = p->y << z | p->y >> (64 - z);
+ bootid_shake(p);
+ bootid_shake(p);
+ const unsigned q = p->x % 59;
+ p->y = p->y << q | p->y >> (64 - q);
+ bootid_shake(p);
+ bootid_shake(p);
+ bootid_shake(p);
+}
- if (unlikely(begin_data && (begin_key == NULL || begin_key == MDBX_EPSILON)))
- return MDBX_EINVAL;
+#if defined(_WIN32) || defined(_WIN64)
- if (unlikely(end_data && (end_key == NULL || end_key == MDBX_EPSILON)))
- return MDBX_EINVAL;
+static uint64_t windows_systemtime_ms() {
+ FILETIME ft;
+ GetSystemTimeAsFileTime(&ft);
+ return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul;
+}
+
+static uint64_t windows_bootime(void) {
+ unsigned confirmed = 0;
+ uint64_t boottime = 0;
+ uint64_t up0 = mdbx_GetTickCount64();
+ uint64_t st0 = windows_systemtime_ms();
+ for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) {
+ YieldProcessor();
+ const uint64_t up1 = mdbx_GetTickCount64();
+ const uint64_t st1 = windows_systemtime_ms();
+ if (st1 > fuse && st1 == st0 && up1 == up0) {
+ uint64_t diff = st1 - up1;
+ if (boottime == diff) {
+ if (++confirmed > 4)
+ return boottime;
+ } else {
+ confirmed = 0;
+ boottime = diff;
+ }
+ fuse = st1;
+ Sleep(1);
+ }
+ st0 = st1;
+ up0 = up1;
+ }
+ return 0;
+}
- if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON))
- return MDBX_EINVAL;
+static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue,
+ PVOID pvData, LPDWORD pcbData) {
+ LSTATUS rc;
+ if (!mdbx_RegGetValueA) {
+ /* an old Windows 2000/XP */
+ HKEY hSubKey;
+ rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey);
+ if (rc == ERROR_SUCCESS) {
+ rc = RegQueryValueExA(hSubKey, lpValue, NULL, NULL, pvData, pcbData);
+ RegCloseKey(hSubKey);
+ }
+ return rc;
+ }
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
+ rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, NULL, pvData,
+ pcbData);
+ if (rc != ERROR_FILE_NOT_FOUND)
+ return rc;
- MDBX_cursor_couple begin;
- /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */
- rc = mdbx_cursor_init(&begin.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
+ rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue,
+ RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */,
+ NULL, pvData, pcbData);
+ if (rc != ERROR_FILE_NOT_FOUND)
return rc;
+ return mdbx_RegGetValueA(hKey, lpSubKey, lpValue,
+ RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */,
+ NULL, pvData, pcbData);
+}
+#endif
- if (unlikely(begin.outer.mc_db->md_entries == 0)) {
- *size_items = 0;
- return MDBX_SUCCESS;
+__cold MDBX_MAYBE_UNUSED static bool
+bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) {
+ if (n > 31) {
+ unsigned bits = 0;
+ for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ {
+ uint8_t c = ((const uint8_t *)p)[i];
+ if (c >= '0' && c <= '9')
+ c -= '0';
+ else if (c >= 'a' && c <= 'f')
+ c -= 'a' - 10;
+ else if (c >= 'A' && c <= 'F')
+ c -= 'A' - 10;
+ else
+ continue;
+ assert(c <= 15);
+ c ^= s->y >> 60;
+ s->y = s->y << 4 | s->x >> 60;
+ s->x = s->x << 4 | c;
+ bits += 4;
+ }
+ if (bits > 42 * 3)
+ /* UUID parsed successfully */
+ return true;
}
- if (!begin_key) {
- if (unlikely(!end_key)) {
- /* LY: FIRST..LAST case */
- *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries;
- return MDBX_SUCCESS;
- }
- MDBX_val stub = {0, 0};
- rc = mdbx_cursor_first(&begin.outer, &stub, &stub);
- if (unlikely(end_key == MDBX_EPSILON)) {
- /* LY: FIRST..+epsilon case */
- return (rc == MDBX_SUCCESS)
- ? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
- : rc;
- }
- } else {
- if (unlikely(begin_key == MDBX_EPSILON)) {
- if (end_key == NULL) {
- /* LY: -epsilon..LAST case */
- MDBX_val stub = {0, 0};
- rc = mdbx_cursor_last(&begin.outer, &stub, &stub);
- return (rc == MDBX_SUCCESS)
- ? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
- : rc;
- }
- /* LY: -epsilon..value case */
- assert(end_key != MDBX_EPSILON);
- begin_key = end_key;
- } else if (unlikely(end_key == MDBX_EPSILON)) {
- /* LY: value..+epsilon case */
- assert(begin_key != MDBX_EPSILON);
- end_key = begin_key;
+ if (n > 15) /* is enough handle it as a binary? */ {
+ if (n == sizeof(bin128_t)) {
+ bin128_t aligned;
+ memcpy(&aligned, p, sizeof(bin128_t));
+ s->x += aligned.x;
+ s->y += aligned.y;
+ } else
+ bootid_collect(s, p, n);
+ return true;
+ }
+
+ if (n)
+ bootid_collect(s, p, n);
+ return false;
+}
+
+__cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) {
+ bin128_t bin = {{0, 0}};
+ bool got_machineid = false, got_boottime = false, got_bootseq = false;
+
+#if defined(__linux__) || defined(__gnu_linux__)
+ {
+ const int fd =
+ open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_NOFOLLOW);
+ if (fd != -1) {
+ struct statfs fs;
+ char buf[42];
+ const ssize_t len =
+ (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0)
+ ? read(fd, buf, sizeof(buf))
+ : -1;
+ const int err = close(fd);
+ assert(err == 0);
+ (void)err;
+ if (len > 0 && bootid_parse_uuid(&bin, buf, len))
+ return bin;
}
- if (end_key && !begin_data && !end_data &&
- (begin_key == end_key ||
- begin.outer.mc_dbx->md_cmp(begin_key, end_key) == 0)) {
- /* LY: single key case */
- rc = mdbx_cursor_set(&begin.outer, begin_key, NULL, MDBX_SET).err;
- if (unlikely(rc != MDBX_SUCCESS)) {
- *size_items = 0;
- return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
- }
- *size_items = 1;
- if (begin.outer.mc_xcursor != NULL) {
- MDBX_node *node = page_node(begin.outer.mc_pg[begin.outer.mc_top],
- begin.outer.mc_ki[begin.outer.mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- /* LY: return the number of duplicates for given key */
- mdbx_tassert(txn,
- begin.outer.mc_xcursor == &begin.inner &&
- (begin.inner.mx_cursor.mc_flags & C_INITIALIZED));
- *size_items =
- (sizeof(*size_items) >= sizeof(begin.inner.mx_db.md_entries) ||
- begin.inner.mx_db.md_entries <= PTRDIFF_MAX)
- ? (size_t)begin.inner.mx_db.md_entries
- : PTRDIFF_MAX;
- }
+ }
+#endif /* Linux */
+
+#if defined(__APPLE__) || defined(__MACH__)
+ {
+ char buf[42];
+ size_t len = sizeof(buf);
+ if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) &&
+ bootid_parse_uuid(&bin, buf, len))
+ return bin;
+
+#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
+ __MAC_OS_X_VERSION_MIN_REQUIRED > 1050
+ uuid_t uuid;
+ struct timespec wait = {0, 1000000000u / 42};
+ if (!gethostuuid(uuid, &wait) &&
+ bootid_parse_uuid(&bin, uuid, sizeof(uuid)))
+ got_machineid = true;
+#endif /* > 10.5 */
+
+ struct timeval boottime;
+ len = sizeof(boottime);
+ if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) &&
+ len == sizeof(boottime) && boottime.tv_sec)
+ got_boottime = true;
+ }
+#endif /* Apple/Darwin */
+
+#if defined(_WIN32) || defined(_WIN64)
+ {
+ union buf {
+ DWORD BootId;
+ DWORD BaseTime;
+ SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo;
+ struct {
+ LARGE_INTEGER BootTime;
+ LARGE_INTEGER CurrentTime;
+ LARGE_INTEGER TimeZoneBias;
+ ULONG TimeZoneId;
+ ULONG Reserved;
+ ULONGLONG BootTimeBias;
+ ULONGLONG SleepTimeBias;
+ } SysTimeOfDayInfoHacked;
+ wchar_t MachineGuid[42];
+ char DigitalProductId[248];
+ } buf;
+
+ static const char HKLM_MicrosoftCryptography[] =
+ "SOFTWARE\\Microsoft\\Cryptography";
+ DWORD len = sizeof(buf);
+ /* Windows is madness and must die */
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography,
+ "MachineGuid", &buf.MachineGuid,
+ &len) == ERROR_SUCCESS &&
+ len < sizeof(buf))
+ got_machineid = bootid_parse_uuid(&bin, &buf.MachineGuid, len);
+
+ if (!got_machineid) {
+ /* again, Windows is madness */
+ static const char HKLM_WindowsNT[] =
+ "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
+ static const char HKLM_WindowsNT_DPK[] =
+ "SOFTWARE\\Microsoft\\Windows "
+ "NT\\CurrentVersion\\DefaultProductKey";
+ static const char HKLM_WindowsNT_DPK2[] =
+ "SOFTWARE\\Microsoft\\Windows "
+ "NT\\CurrentVersion\\DefaultProductKey2";
+
+ len = sizeof(buf);
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT,
+ "DigitalProductId", &buf.DigitalProductId,
+ &len) == ERROR_SUCCESS &&
+ len > 42 && len < sizeof(buf)) {
+ bootid_collect(&bin, &buf.DigitalProductId, len);
+ got_machineid = true;
+ }
+ len = sizeof(buf);
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK,
+ "DigitalProductId", &buf.DigitalProductId,
+ &len) == ERROR_SUCCESS &&
+ len > 42 && len < sizeof(buf)) {
+ bootid_collect(&bin, &buf.DigitalProductId, len);
+ got_machineid = true;
+ }
+ len = sizeof(buf);
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2,
+ "DigitalProductId", &buf.DigitalProductId,
+ &len) == ERROR_SUCCESS &&
+ len > 42 && len < sizeof(buf)) {
+ bootid_collect(&bin, &buf.DigitalProductId, len);
+ got_machineid = true;
+ }
+ }
+
+ static const char HKLM_PrefetcherParams[] =
+ "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory "
+ "Management\\PrefetchParameters";
+ len = sizeof(buf);
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId",
+ &buf.BootId, &len) == ERROR_SUCCESS &&
+ len > 1 && len < sizeof(buf)) {
+ bootid_collect(&bin, &buf.BootId, len);
+ got_bootseq = true;
+ }
+
+ len = sizeof(buf);
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime",
+ &buf.BaseTime, &len) == ERROR_SUCCESS &&
+ len >= sizeof(buf.BaseTime) && buf.BaseTime) {
+ bootid_collect(&bin, &buf.BaseTime, len);
+ got_boottime = true;
+ }
+
+ /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */
+ NTSTATUS status = NtQuerySystemInformation(
+ 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo,
+ sizeof(buf.SysTimeOfDayInfo), &len);
+ if (NT_SUCCESS(status) &&
+ len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) +
+ sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) &&
+ buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) {
+ const uint64_t UnbiasedBootTime =
+ buf.SysTimeOfDayInfoHacked.BootTime.QuadPart -
+ buf.SysTimeOfDayInfoHacked.BootTimeBias;
+ if (UnbiasedBootTime) {
+ bootid_collect(&bin, &UnbiasedBootTime, sizeof(UnbiasedBootTime));
+ got_boottime = true;
+ }
+ }
+
+ if (!got_boottime) {
+ uint64_t boottime = windows_bootime();
+ if (boottime) {
+ bootid_collect(&bin, &boottime, sizeof(boottime));
+ got_boottime = true;
}
- return MDBX_SUCCESS;
- } else {
- rc = mdbx_cursor_set(&begin.outer, begin_key, begin_data,
- begin_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE)
- .err;
}
}
+#endif /* Windows */
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc != MDBX_NOTFOUND || !(begin.outer.mc_flags & C_INITIALIZED))
- return rc;
+#if defined(CTL_HW) && defined(HW_UUID)
+ if (!got_machineid) {
+ static const int mib[] = {CTL_HW, HW_UUID};
+ char buf[42];
+ size_t len = sizeof(buf);
+ if (sysctl(
+#ifdef SYSCTL_LEGACY_NONCONST_MIB
+ (int *)
+#endif
+ mib,
+ ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0)
+ got_machineid = bootid_parse_uuid(&bin, buf, len);
+ }
+#endif /* CTL_HW && HW_UUID */
+
+#if defined(CTL_KERN) && defined(KERN_HOSTUUID)
+ if (!got_machineid) {
+ static const int mib[] = {CTL_KERN, KERN_HOSTUUID};
+ char buf[42];
+ size_t len = sizeof(buf);
+ if (sysctl(
+#ifdef SYSCTL_LEGACY_NONCONST_MIB
+ (int *)
+#endif
+ mib,
+ ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0)
+ got_machineid = bootid_parse_uuid(&bin, buf, len);
}
+#endif /* CTL_KERN && KERN_HOSTUUID */
- MDBX_cursor_couple end;
- rc = mdbx_cursor_init(&end.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- if (!end_key) {
- MDBX_val stub = {0, 0};
- rc = mdbx_cursor_last(&end.outer, &stub, &stub);
- } else {
- rc = mdbx_cursor_set(&end.outer, end_key, end_data,
- end_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE)
- .err;
+#if defined(__NetBSD__)
+ if (!got_machineid) {
+ char buf[42];
+ size_t len = sizeof(buf);
+ if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, NULL, 0) == 0)
+ got_machineid = bootid_parse_uuid(&bin, buf, len);
}
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc != MDBX_NOTFOUND || !(end.outer.mc_flags & C_INITIALIZED))
- return rc;
+#endif /* __NetBSD__ */
+
+#if _XOPEN_SOURCE_EXTENDED
+ if (!got_machineid) {
+ const int hostid = gethostid();
+ if (hostid > 0) {
+ bootid_collect(&bin, &hostid, sizeof(hostid));
+ got_machineid = true;
+ }
}
+#endif /* _XOPEN_SOURCE_EXTENDED */
- rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- assert(*size_items >= -(ptrdiff_t)begin.outer.mc_db->md_entries &&
- *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries);
+ if (!got_machineid) {
+ lack:
+ bin.x = bin.y = 0;
+ return bin;
+ }
-#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \
- * results for an inverted ranges. */
+ /*--------------------------------------------------------------------------*/
- /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63
- Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */
+#if defined(CTL_KERN) && defined(KERN_BOOTTIME)
+ if (!got_boottime) {
+ static const int mib[] = {CTL_KERN, KERN_BOOTTIME};
+ struct timeval boottime;
+ size_t len = sizeof(boottime);
+ if (sysctl(
+#ifdef SYSCTL_LEGACY_NONCONST_MIB
+ (int *)
+#endif
+ mib,
+ ARRAY_LENGTH(mib), &boottime, &len, NULL, 0) == 0 &&
+ len == sizeof(boottime) && boottime.tv_sec) {
+ bootid_collect(&bin, &boottime, len);
+ got_boottime = true;
+ }
+ }
+#endif /* CTL_KERN && KERN_BOOTTIME */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+ if (!got_boottime) {
+ kstat_ctl_t *kc = kstat_open();
+ if (kc) {
+ kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc");
+ if (kp && kstat_read(kc, kp, 0) != -1) {
+ kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time");
+ if (kn) {
+ switch (kn->data_type) {
+ case KSTAT_DATA_INT32:
+ case KSTAT_DATA_UINT32:
+ bootid_collect(&bin, &kn->value, sizeof(int32_t));
+ got_boottime = true;
+ case KSTAT_DATA_INT64:
+ case KSTAT_DATA_UINT64:
+ bootid_collect(&bin, &kn->value, sizeof(int64_t));
+ got_boottime = true;
+ }
+ }
+ }
+ kstat_close(kc);
+ }
+ }
+#endif /* SunOS / Solaris */
- if (*size_items < 0) {
- /* LY: inverted range case */
- *size_items += (ptrdiff_t)begin.outer.mc_db->md_entries;
- } else if (*size_items == 0 && begin_key && end_key) {
- int cmp = begin.outer.mc_dbx->md_cmp(&origin_begin_key, &origin_end_key);
- if (cmp == 0 && (begin.inner.mx_cursor.mc_flags & C_INITIALIZED) &&
- begin_data && end_data)
- cmp = begin.outer.mc_dbx->md_dcmp(&origin_begin_data, &origin_end_data);
- if (cmp > 0) {
- /* LY: inverted range case with empty scope */
- *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries;
+#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME)
+ if (!got_boottime) {
+ setutxent();
+ const struct utmpx id = {.ut_type = BOOT_TIME};
+ const struct utmpx *entry = getutxid(&id);
+ if (entry) {
+ bootid_collect(&bin, entry, sizeof(*entry));
+ got_boottime = true;
+ while (unlikely((entry = getutxid(&id)) != nullptr)) {
+ /* have multiple reboot records, assuming we can distinguish next
+ * bootsession even if RTC is wrong or absent */
+ bootid_collect(&bin, entry, sizeof(*entry));
+ got_bootseq = true;
+ }
}
+ endutxent();
}
- assert(*size_items >= 0 &&
- *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries);
+#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */
+
+ if (!got_bootseq) {
+ if (!got_boottime || !MDBX_TRUST_RTC)
+ goto lack;
+
+#if defined(_WIN32) || defined(_WIN64)
+ FILETIME now;
+ GetSystemTimeAsFileTime(&now);
+ if (0x1CCCCCC > now.dwHighDateTime)
+#else
+ struct timespec mono, real;
+ if (clock_gettime(CLOCK_MONOTONIC, &mono) ||
+ clock_gettime(CLOCK_REALTIME, &real) ||
+ /* wrong time, RTC is mad or absent */
+ 1555555555l > real.tv_sec ||
+ /* seems no adjustment by RTC/NTP, i.e. a fake time */
+ real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec ||
+ (real.tv_sec - mono.tv_sec) % 900u == 0)
#endif
+ goto lack;
+ }
+
+ return bin;
+}
+
+__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages,
+ intptr_t *avail_pages) {
+ if (!page_size && !total_pages && !avail_pages)
+ return MDBX_EINVAL;
+ if (total_pages)
+ *total_pages = -1;
+ if (avail_pages)
+ *avail_pages = -1;
+
+ const intptr_t pagesize = mdbx_syspagesize();
+ if (page_size)
+ *page_size = pagesize;
+ if (unlikely(pagesize < MIN_PAGESIZE || !is_powerof2(pagesize)))
+ return MDBX_INCOMPATIBLE;
+
+ MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize);
+ assert(pagesize == (INT64_C(1) << log2page));
+ (void)log2page;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MEMORYSTATUSEX info;
+ memset(&info, 0, sizeof(info));
+ info.dwLength = sizeof(info);
+ if (!GlobalMemoryStatusEx(&info))
+ return (int)GetLastError();
+#endif
+
+ if (total_pages) {
+#if defined(_WIN32) || defined(_WIN64)
+ const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page);
+#elif defined(_SC_PHYS_PAGES)
+ const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES);
+ if (total_ram_pages == -1)
+ return errno;
+#elif defined(_SC_AIX_REALMEM)
+ const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM);
+ if (total_ram_Kb == -1)
+ return errno;
+ const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page;
+#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || \
+ defined(HW_PHYSMEM)
+ size_t ram, len = sizeof(ram);
+ static const int mib[] = {
+ CTL_HW,
+#if defined(HW_USERMEM)
+ HW_USERMEM
+#elif defined(HW_PHYSMEM64)
+ HW_PHYSMEM64
+#elif defined(HW_MEMSIZE)
+ HW_MEMSIZE
+#else
+ HW_PHYSMEM
+#endif
+ };
+ if (sysctl(
+#ifdef SYSCTL_LEGACY_NONCONST_MIB
+ (int *)
+#endif
+ mib,
+ ARRAY_LENGTH(mib), &ram, &len, NULL, 0) != 0)
+ return errno;
+ if (len != sizeof(ram))
+ return MDBX_ENOSYS;
+ const intptr_t total_ram_pages = (intptr_t)(ram >> log2page);
+#else
+#error "FIXME: Get User-accessible or physical RAM"
+#endif
+ *total_pages = total_ram_pages;
+ if (total_ram_pages < 1)
+ return MDBX_ENOSYS;
+ }
+
+ if (avail_pages) {
+#if defined(_WIN32) || defined(_WIN64)
+ const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page);
+#elif defined(_SC_AVPHYS_PAGES)
+ const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES);
+ if (avail_ram_pages == -1)
+ return errno;
+#elif defined(__MACH__)
+ mach_msg_type_number_t count = HOST_VM_INFO_COUNT;
+ vm_statistics_data_t vmstat;
+ mach_port_t mport = mach_host_self();
+ kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO,
+ (host_info_t)&vmstat, &count);
+ mach_port_deallocate(mach_task_self(), mport);
+ if (unlikely(kerr != KERN_SUCCESS))
+ return MDBX_ENOSYS;
+ const intptr_t avail_ram_pages = vmstat.free_count;
+#elif defined(VM_TOTAL) || defined(VM_METER)
+ struct vmtotal info;
+ size_t len = sizeof(info);
+ static const int mib[] = {
+ CTL_VM,
+#if defined(VM_TOTAL)
+ VM_TOTAL
+#elif defined(VM_METER)
+ VM_METER
+#endif
+ };
+ if (sysctl(
+#ifdef SYSCTL_LEGACY_NONCONST_MIB
+ (int *)
+#endif
+ mib,
+ ARRAY_LENGTH(mib), &info, &len, NULL, 0) != 0)
+ return errno;
+ if (len != sizeof(info))
+ return MDBX_ENOSYS;
+ const intptr_t avail_ram_pages = info.t_free;
+#else
+#error "FIXME: Get Available RAM"
+#endif
+ *avail_pages = avail_ram_pages;
+ if (avail_ram_pages < 1)
+ return MDBX_ENOSYS;
+ }
return MDBX_SUCCESS;
}
+/* This is CMake-template for libmdbx's version.c
+ ******************************************************************************/
-//------------------------------------------------------------------------------
-/* ПозволÑет обновить или удалить ÑущеÑтвующую запиÑÑŒ Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸ÐµÐ¼
- * в old_data предыдущего Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ…. При Ñтом еÑли new_data равен
- * нулю, то выполнÑетÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ, иначе обновление/вÑтавка.
- *
- * Текущее значение может находитьÑÑ Ð² уже измененной (грÑзной) Ñтранице.
- * Ð’ Ñтом Ñлучае Ñтраница будет перезапиÑана при обновлении, а Ñамо Ñтарое
- * значение утрачено. ПоÑтому иÑходно в old_data должен быть передан
- * дополнительный буфер Ð´Ð»Ñ ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ñтарого значениÑ.
- * ЕÑли переданный буфер Ñлишком мал, то Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð²ÐµÑ€Ð½ÐµÑ‚ -1, уÑтановив
- * old_data->iov_len в ÑоответÑтвующее значение.
+#if MDBX_VERSION_MAJOR != 0 || \
+ MDBX_VERSION_MINOR != 11
+#error "API version mismatch! Had `git fetch --tags` done?"
+#endif
+
+static const char sourcery[] = MDBX_STRINGIFY(MDBX_BUILD_SOURCERY);
+
+__dll_export
+#ifdef __attribute_used__
+ __attribute_used__
+#elif defined(__GNUC__) || __has_attribute(__used__)
+ __attribute__((__used__))
+#endif
+#ifdef __attribute_externally_visible__
+ __attribute_externally_visible__
+#elif (defined(__GNUC__) && !defined(__clang__)) || \
+ __has_attribute(__externally_visible__)
+ __attribute__((__externally_visible__))
+#endif
+ const struct MDBX_version_info mdbx_version = {
+ 0,
+ 11,
+ 7,
+ 0,
+ {"2022-04-22T23:28:56+03:00", "6fdefd6844bd6d212ea6dec99892ae1fac981521", "40ec559c8c1e0b9794cdc0cd12c155d05e7e8547",
+ "v0.11.7-0-g40ec559c"},
+ sourcery};
+
+__dll_export
+#ifdef __attribute_used__
+ __attribute_used__
+#elif defined(__GNUC__) || __has_attribute(__used__)
+ __attribute__((__used__))
+#endif
+#ifdef __attribute_externally_visible__
+ __attribute_externally_visible__
+#elif (defined(__GNUC__) && !defined(__clang__)) || \
+ __has_attribute(__externally_visible__)
+ __attribute__((__externally_visible__))
+#endif
+ const char *const mdbx_sourcery_anchor = sourcery;
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
*
- * Ð”Ð»Ñ Ð½Ðµ-уникальных ключей также возможен второй Ñценарий иÑпользованиÑ,
- * когда поÑредÑтвом old_data из запиÑей Ñ Ð¾Ð´Ð¸Ð½Ð°ÐºÐ¾Ð²Ñ‹Ð¼ ключом длÑ
- * удалениÑ/Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ð²Ñ‹Ð±Ð¸Ñ€Ð°ÐµÑ‚ÑÑ ÐºÐ¾Ð½ÐºÑ€ÐµÑ‚Ð½Ð°Ñ. Ð”Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° Ñтого ÑценариÑ
- * во flags Ñледует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE.
- * Именно Ñта ÐºÐ¾Ð¼Ð±Ð¸Ð½Ð°Ñ†Ð¸Ñ Ð²Ñ‹Ð±Ñ€Ð°Ð½Ð°, так как она лишена ÑмыÑла, и Ñтим позволÑет
- * идентифицировать Ð·Ð°Ð¿Ñ€Ð¾Ñ Ñ‚Ð°ÐºÐ¾Ð³Ð¾ ÑценариÑ.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
*
- * Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¼Ð¾Ð¶ÐµÑ‚ быть замещена ÑоответÑтвующими операциÑми Ñ ÐºÑƒÑ€Ñорами
- * поÑле двух доработок (TODO):
- * - внешнÑÑ Ð°Ð»Ð»Ð¾ÐºÐ°Ñ†Ð¸Ñ ÐºÑƒÑ€Ñоров, в том чиÑле на Ñтеке (без malloc).
- * - Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ dirty-ÑтатуÑа Ñтраницы по адреÑу (знать о MUTABLE/WRITEABLE).
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
*/
-int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
- MDBX_val *new_data, MDBX_val *old_data,
- MDBX_put_flags_t flags, MDBX_preserve_func preserver,
- void *preserver_context) {
- int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */
- if (unlikely(!key || !old_data || old_data == new_data))
- return MDBX_EINVAL;
+/* PREAMBLE FOR WINDOWS:
+ *
+ * We are not concerned for performance here.
+ * If you are running Windows a performance could NOT be the goal.
+ * Otherwise please use Linux. */
- if (unlikely(old_data->iov_base == NULL && old_data->iov_len))
- return MDBX_EINVAL;
- if (unlikely(new_data == NULL &&
- (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT))
- return MDBX_EINVAL;
+static void mdbx_winnt_import(void);
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
+#if MDBX_BUILD_SHARED_LIBRARY
+#if MDBX_WITHOUT_MSVC_CRT && defined(NDEBUG)
+/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
+ *
+ * Define dll's entry point only for Release build when NDEBUG is defined and
+ * MDBX_WITHOUT_MSVC_CRT=ON. if the entry point isn't defined then MSVC's will
+ * automatically use DllMainCRTStartup() from CRT library, which also
+ * automatically call DllMain() from our mdbx.dll */
+#pragma comment(linker, "/ENTRY:DllMain")
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
+#else
+#if !MDBX_MANUAL_MODULE_HANDLER
+static
+#endif /* !MDBX_MANUAL_MODULE_HANDLER */
+ void NTAPI
+ mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved)
+#endif /* MDBX_BUILD_SHARED_LIBRARY */
+{
+ (void)reserved;
+ switch (reason) {
+ case DLL_PROCESS_ATTACH:
+ mdbx_winnt_import();
+ mdbx_rthc_global_init();
+ break;
+ case DLL_PROCESS_DETACH:
+ mdbx_rthc_global_dtor();
+ break;
- if (unlikely(flags &
- ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
- MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT)))
- return MDBX_EINVAL;
+ case DLL_THREAD_ATTACH:
+ break;
+ case DLL_THREAD_DETACH:
+ mdbx_rthc_thread_dtor(module);
+ break;
+ }
+#if MDBX_BUILD_SHARED_LIBRARY
+ return TRUE;
+#endif
+}
- MDBX_cursor_couple cx;
- rc = mdbx_cursor_init(&cx.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- cx.outer.mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = &cx.outer;
+#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(_MSC_VER)
+# pragma const_seg(push)
+# pragma data_seg(push)
+
+# ifndef _M_IX86
+ /* kick a linker to create the TLS directory if not already done */
+# pragma comment(linker, "/INCLUDE:_tls_used")
+ /* Force some symbol references. */
+# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
+ /* specific const-segment for WIN64 */
+# pragma const_seg(".CRT$XLB")
+ const
+# else
+ /* kick a linker to create the TLS directory if not already done */
+# pragma comment(linker, "/INCLUDE:__tls_used")
+ /* Force some symbol references. */
+# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
+ /* specific data-segment for WIN32 */
+# pragma data_seg(".CRT$XLB")
+# endif
+
+ __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_module_handler;
+# pragma data_seg(pop)
+# pragma const_seg(pop)
+
+#elif defined(__GNUC__)
+# ifndef _M_IX86
+ const
+# endif
+ PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_module_handler;
+#else
+# error FIXME
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER */
- MDBX_val present_key = *key;
- if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) {
- /* в old_data значение Ð´Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° конкретного дубликата */
- if (unlikely(!(txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT))) {
- rc = MDBX_EINVAL;
- goto bailout;
- }
+/*----------------------------------------------------------------------------*/
- /* убираем лишний бит, он был признаком запрошенного режима */
- flags -= MDBX_NOOVERWRITE;
+#define LCK_SHARED 0
+#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
+#define LCK_WAITFOR 0
+#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
- rc = mdbx_cursor_get(&cx.outer, &present_key, old_data, MDBX_GET_BOTH);
- if (rc != MDBX_SUCCESS)
- goto bailout;
- } else {
- /* в old_data буфер Ð´Ð»Ñ ÑÐ¾Ñ…Ñ€Ð°Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€ÐµÐ´Ñ‹Ð´ÑƒÑ‰ÐµÐ³Ð¾ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ */
- if (unlikely(new_data && old_data->iov_base == new_data->iov_base))
- return MDBX_EINVAL;
- MDBX_val present_data;
- rc = mdbx_cursor_get(&cx.outer, &present_key, &present_data, MDBX_SET_KEY);
- if (unlikely(rc != MDBX_SUCCESS)) {
- old_data->iov_base = NULL;
- old_data->iov_len = 0;
- if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT))
- goto bailout;
- } else if (flags & MDBX_NOOVERWRITE) {
- rc = MDBX_KEYEXIST;
- *old_data = present_data;
- goto bailout;
- } else {
- MDBX_page *page = cx.outer.mc_pg[cx.outer.mc_top];
- if (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) {
- if (flags & MDBX_CURRENT) {
- /* disallow update/delete for multi-values */
- MDBX_node *node = page_node(page, cx.outer.mc_ki[cx.outer.mc_top]);
- if (F_ISSET(node_flags(node), F_DUPDATA)) {
- mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) &&
- cx.outer.mc_xcursor->mx_db.md_entries > 1);
- if (cx.outer.mc_xcursor->mx_db.md_entries > 1) {
- rc = MDBX_EMULTIVAL;
- goto bailout;
- }
- }
- /* Ð’ оригинальной LMDB флажок MDBX_CURRENT здеÑÑŒ приведет
- * к замене данных без учета MDBX_DUPSORT Ñортировки,
- * но здеÑÑŒ Ñто в любом Ñлучае допуÑтимо, так как мы
- * проверили что Ð´Ð»Ñ ÐºÐ»ÑŽÑ‡Ð° еÑÑ‚ÑŒ только одно значение. */
- }
- }
+static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
+ size_t bytes) {
+ OVERLAPPED ov;
+ ov.hEvent = 0;
+ ov.Offset = (DWORD)offset;
+ ov.OffsetHigh = HIGH_DWORD(offset);
+ return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
+}
- if (IS_MODIFIABLE(txn, page)) {
- if (new_data && cmp_lenfast(&present_data, new_data) == 0) {
- /* еÑли данные Ñовпадают, то ничего делать не надо */
- *old_data = *new_data;
- goto bailout;
- }
- rc = preserver ? preserver(preserver_context, old_data,
- present_data.iov_base, present_data.iov_len)
- : MDBX_SUCCESS;
- if (unlikely(rc != MDBX_SUCCESS))
- goto bailout;
- } else {
- *old_data = present_data;
- }
- flags |= MDBX_CURRENT;
+static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
+ size_t bytes) {
+ return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
+ HIGH_DWORD(bytes));
+}
+
+/*----------------------------------------------------------------------------*/
+/* global `write` lock for write-txt processing,
+ * exclusive locking both meta-pages) */
+
+#define LCK_MAXLEN (1u + ((~(size_t)0) >> 1))
+#define LCK_META_OFFSET 0
+#define LCK_META_LEN (MAX_PAGESIZE * NUM_METAS)
+#define LCK_BODY_OFFSET LCK_META_LEN
+#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET)
+#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
+#define LCK_WHOLE 0, LCK_MAXLEN
+
+int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
+ if (dontwait) {
+ if (!TryEnterCriticalSection(&env->me_windowsbug_lock))
+ return MDBX_BUSY;
+ } else {
+ __try {
+ EnterCriticalSection(&env->me_windowsbug_lock);
+ }
+ __except ((GetExceptionCode() ==
+ 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
+ ? EXCEPTION_EXECUTE_HANDLER
+ : EXCEPTION_CONTINUE_SEARCH) {
+ return ERROR_POSSIBLE_DEADLOCK;
}
}
- if (likely(new_data))
- rc = mdbx_cursor_put(&cx.outer, key, new_data, flags);
- else
- rc = mdbx_cursor_del(&cx.outer, flags & MDBX_ALLDUPS);
+ if ((env->me_flags & MDBX_EXCLUSIVE) ||
+ flock(env->me_lazy_fd,
+ dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
+ : (LCK_EXCLUSIVE | LCK_WAITFOR),
+ LCK_BODY))
+ return MDBX_SUCCESS;
+ int rc = (int)GetLastError();
+ LeaveCriticalSection(&env->me_windowsbug_lock);
+ return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
+}
-bailout:
- txn->mt_cursors[dbi] = cx.outer.mc_next;
+void mdbx_txn_unlock(MDBX_env *env) {
+ int rc = (env->me_flags & MDBX_EXCLUSIVE)
+ ? TRUE
+ : funlock(env->me_lazy_fd, LCK_BODY);
+ LeaveCriticalSection(&env->me_windowsbug_lock);
+ if (!rc)
+ mdbx_panic("%s failed: err %u", __func__, (int)GetLastError());
+}
+
+/*----------------------------------------------------------------------------*/
+/* global `read` lock for readers registration,
+ * exclusive locking `mti_numreaders` (second) cacheline */
+
+#define LCK_LO_OFFSET 0
+#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
+#define LCK_UP_OFFSET LCK_LO_LEN
+#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET)
+#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
+#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
+
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
+ mdbx_srwlock_AcquireShared(&env->me_remap_guard);
+ if (env->me_lfd == INVALID_HANDLE_VALUE)
+ return MDBX_SUCCESS; /* readonly database in readonly filesystem */
+
+ /* transition from S-? (used) to S-E (locked),
+ * e.g. exclusive lock upper-part */
+ if ((env->me_flags & MDBX_EXCLUSIVE) ||
+ flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
+ return MDBX_SUCCESS;
+
+ int rc = (int)GetLastError();
+ mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
return rc;
}
-static int default_value_preserver(void *context, MDBX_val *target,
- const void *src, size_t bytes) {
- (void)context;
- if (unlikely(target->iov_len < bytes)) {
- target->iov_base = nullptr;
- target->iov_len = bytes;
- return MDBX_RESULT_TRUE;
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
+ if (env->me_lfd != INVALID_HANDLE_VALUE) {
+ /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */
+ if ((env->me_flags & MDBX_EXCLUSIVE) == 0 &&
+ !funlock(env->me_lfd, LCK_UPPER))
+ mdbx_panic("%s failed: err %u", __func__, (int)GetLastError());
}
- memcpy(target->iov_base, src, target->iov_len = bytes);
- return MDBX_SUCCESS;
+ mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
}
-int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
- MDBX_val *new_data, MDBX_val *old_data,
- MDBX_put_flags_t flags) {
- return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags,
- default_value_preserver, nullptr);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait) {
+ return flock(fd,
+ wait ? LCK_EXCLUSIVE | LCK_WAITFOR
+ : LCK_EXCLUSIVE | LCK_DONTWAIT,
+ 0, LCK_MAXLEN)
+ ? MDBX_SUCCESS
+ : (int)GetLastError();
+}
+
+static int suspend_and_append(mdbx_handle_array_t **array,
+ const DWORD ThreadId) {
+ const unsigned limit = (*array)->limit;
+ if ((*array)->count == limit) {
+ void *ptr = mdbx_realloc(
+ (limit > ARRAY_LENGTH((*array)->handles))
+ ? *array
+ : /* don't free initial array on the stack */ NULL,
+ sizeof(mdbx_handle_array_t) +
+ sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles)));
+ if (!ptr)
+ return MDBX_ENOMEM;
+ if (limit == ARRAY_LENGTH((*array)->handles))
+ memcpy(ptr, *array, sizeof(mdbx_handle_array_t));
+ *array = (mdbx_handle_array_t *)ptr;
+ (*array)->limit = limit * 2;
+ }
+
+ HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
+ FALSE, ThreadId);
+ if (hThread == NULL)
+ return (int)GetLastError();
+
+ if (SuspendThread(hThread) == (DWORD)-1) {
+ int err = (int)GetLastError();
+ DWORD ExitCode;
+ if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
+ !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
+ err = MDBX_SUCCESS;
+ CloseHandle(hThread);
+ return err;
+ }
+
+ (*array)->handles[(*array)->count++] = hThread;
+ return MDBX_SUCCESS;
}
-/* Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ñообщает находитÑÑ Ð»Ð¸ указанный Ð°Ð´Ñ€ÐµÑ Ð² "грÑзной" Ñтранице у
- * заданной пишущей транзакции. Ð’ конечном Ñчете Ñто позволÑет избавитьÑÑ Ð¾Ñ‚
- * лишнего ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ… из ÐЕ-грÑзных Ñтраниц.
- *
- * "ГрÑзные" Ñтраницы - Ñто те, которые уже были изменены в ходе пишущей
- * транзакции. СоответÑтвенно, какие-либо дальнейшие Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð¼Ð¾Ð³ÑƒÑ‚ привеÑти
- * к перезапиÑи таких Ñтраниц. ПоÑтому вÑе функции, выполнÑющие изменениÑ, в
- * качеÑтве аргументов ÐЕ должны получать указатели на данные в таких
- * Ñтраницах. Ð’ Ñвою очередь "ÐЕ грÑзные" Ñтраницы перед модификацией будут
- * Ñкопированы.
- *
- * Другими Ñловами, данные из "грÑзных" Ñтраниц должны быть либо Ñкопированы
- * перед передачей в качеÑтве аргументов Ð´Ð»Ñ Ð´Ð°Ð»ÑŒÐ½ÐµÐ¹ÑˆÐ¸Ñ… модификаций, либо
- * отвергнуты на Ñтадии проверки корректноÑти аргументов.
- *
- * Таким образом, Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¿Ð¾Ð·Ð²Ð¾Ð»Ñет как избавитÑÑ Ð¾Ñ‚ лишнего копированиÑ,
- * так и выполнить более полную проверку аргументов.
- *
- * Ð’ÐЖÐО: Передаваемый указатель должен указывать на начало данных. Только
- * так гарантируетÑÑ Ñ‡Ñ‚Ð¾ актуальный заголовок Ñтраницы будет физичеÑки
- * раÑположен в той-же Ñтранице памÑти, в том чиÑле Ð´Ð»Ñ Ð¼Ð½Ð¾Ð³Ð¾Ñтраничных
- * P_OVERFLOW Ñтраниц Ñ Ð´Ð»Ð¸Ð½Ð½Ñ‹Ð¼Ð¸ данными. */
-int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
+ mdbx_assert(env, (env->me_flags & MDBX_NOTLS) == 0);
+ const uintptr_t CurrentTid = GetCurrentThreadId();
+ int rc;
+ if (env->me_lck_mmap.lck) {
+ /* Scan LCK for threads of the current process */
+ const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers;
+ const MDBX_reader *const end =
+ begin +
+ atomic_load32(&env->me_lck_mmap.lck->mti_numreaders, mo_AcquireRelease);
+ const uintptr_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0;
+ for (const MDBX_reader *reader = begin; reader < end; ++reader) {
+ if (reader->mr_pid.weak != env->me_pid || !reader->mr_tid.weak) {
+ skip_lck:
+ continue;
+ }
+ if (reader->mr_tid.weak == CurrentTid ||
+ reader->mr_tid.weak == WriteTxnOwner)
+ goto skip_lck;
- const MDBX_env *env = txn->mt_env;
- const ptrdiff_t offset = (uint8_t *)ptr - env->me_map;
- if (offset >= 0) {
- const pgno_t pgno = bytes2pgno(env, offset);
- if (likely(pgno < txn->mt_next_pgno)) {
- const MDBX_page *page = pgno2page(env, pgno);
- if (unlikely(page->mp_pgno != pgno ||
- (page->mp_flags & P_ILL_BITS) != 0)) {
- /* The ptr pointed into middle of a large page,
- * not to the beginning of a data. */
- return MDBX_EINVAL;
+ rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid.weak);
+ if (rc != MDBX_SUCCESS) {
+ bailout_lck:
+ (void)mdbx_resume_threads_after_remap(*array);
+ return rc;
}
- return ((txn->mt_flags & MDBX_TXN_RDONLY) || !IS_MODIFIABLE(txn, page))
- ? MDBX_RESULT_FALSE
- : MDBX_RESULT_TRUE;
}
- if ((size_t)offset < env->me_dxb_mmap.limit) {
- /* Указатель адреÑует что-то в пределах mmap, но за границей
- * раÑпределенных Ñтраниц. Такое может ÑлучитÑÑ ÐµÑли mdbx_is_dirty()
- * вызываетÑÑ Ð¿Ð¾Ñле операции, в ходе которой грÑÐ·Ð½Ð°Ñ Ñтраница была
- * возвращена в нераÑпределенное проÑтранÑтво. */
- return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EINVAL : MDBX_RESULT_TRUE;
+ if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
+ rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner);
+ if (rc != MDBX_SUCCESS)
+ goto bailout_lck;
+ }
+ } else {
+ /* Without LCK (i.e. read-only mode).
+ * Walk through a snapshot of all running threads */
+ mdbx_assert(env, env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY));
+ const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
+ if (hSnapshot == INVALID_HANDLE_VALUE)
+ return (int)GetLastError();
+
+ THREADENTRY32 entry;
+ entry.dwSize = sizeof(THREADENTRY32);
+
+ if (!Thread32First(hSnapshot, &entry)) {
+ rc = (int)GetLastError();
+ bailout_toolhelp:
+ CloseHandle(hSnapshot);
+ (void)mdbx_resume_threads_after_remap(*array);
+ return rc;
}
- }
- /* Страница вне иÑпользуемого mmap-диапазона, Ñ‚.е. либо в функцию был
- * передан некорректный адреÑ, либо Ð°Ð´Ñ€ÐµÑ Ð² теневой Ñтранице, ÐºÐ¾Ñ‚Ð¾Ñ€Ð°Ñ Ð±Ñ‹Ð»Ð°
- * выделена поÑредÑтвом malloc().
- *
- * Ð”Ð»Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð° MDBX_WRITE_MAP режима Ñтраница однозначно "не грÑзнаÑ",
- * а Ð´Ð»Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ð² без MDBX_WRITE_MAP однозначно "не чиÑтаÑ". */
- return (txn->mt_flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? MDBX_EINVAL
- : MDBX_RESULT_TRUE;
-}
+ do {
+ if (entry.th32OwnerProcessID != env->me_pid ||
+ entry.th32ThreadID == CurrentTid)
+ continue;
-int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result,
- uint64_t increment) {
- int rc = check_txn(txn, MDBX_TXN_BLOCKED);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ rc = suspend_and_append(array, entry.th32ThreadID);
+ if (rc != MDBX_SUCCESS)
+ goto bailout_toolhelp;
- if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID)))
- return MDBX_BAD_DBI;
+ } while (Thread32Next(hSnapshot, &entry));
- if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) {
- rc = mdbx_fetch_sdb(txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ rc = (int)GetLastError();
+ if (rc != ERROR_NO_MORE_FILES)
+ goto bailout_toolhelp;
+ CloseHandle(hSnapshot);
}
- MDBX_db *dbs = &txn->mt_dbs[dbi];
- if (likely(result))
- *result = dbs->md_seq;
+ return MDBX_SUCCESS;
+}
- if (likely(increment > 0)) {
- if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY))
- return MDBX_EACCESS;
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
+ int rc = MDBX_SUCCESS;
+ for (unsigned i = 0; i < array->count; ++i) {
+ const HANDLE hThread = array->handles[i];
+ if (ResumeThread(hThread) == (DWORD)-1) {
+ const int err = (int)GetLastError();
+ DWORD ExitCode;
+ if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
+ GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
+ rc = err;
+ }
+ CloseHandle(hThread);
+ }
+ return rc;
+}
- uint64_t new = dbs->md_seq + increment;
- if (unlikely(new < increment))
- return MDBX_RESULT_TRUE;
+/*----------------------------------------------------------------------------*/
+/* global `initial` lock for lockfile initialization,
+ * exclusive/shared locking first cacheline */
+
+/* Briefly description of locking schema/algorithm:
+ * - Windows does not support upgrading or downgrading for file locking.
+ * - Therefore upgrading/downgrading is emulated by shared and exclusive
+ * locking of upper and lower halves.
+ * - In other words, we have FSM with possible 9 states,
+ * i.e. free/shared/exclusive x free/shared/exclusive == 9.
+ * Only 6 states of FSM are used, which 2 of ones are transitive.
+ *
+ * States:
+ * ?-? = free, i.e. unlocked
+ * S-? = used, i.e. shared lock
+ * E-? = exclusive-read, i.e. operational exclusive
+ * ?-S
+ * ?-E = middle (transitive state)
+ * S-S
+ * S-E = locked (transitive state)
+ * E-S
+ * E-E = exclusive-write, i.e. exclusive due (re)initialization
+ *
+ * The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked
+ * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible,
+ * or to the "used" (and returns MDBX_RESULT_FALSE).
+ *
+ * The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write"
+ * state to the "used" (i.e. shared) state.
+ *
+ * The mdbx_lck_upgrade() moves the locking-FSM from "used" (i.e. shared)
+ * state to the "exclusive write" state.
+ */
- mdbx_tassert(txn, new > dbs->md_seq);
- dbs->md_seq = new;
- txn->mt_flags |= MDBX_TXN_DIRTY;
- txn->mt_dbistate[dbi] |= DBI_DIRTY;
+static void lck_unlock(MDBX_env *env) {
+ int err;
+
+ if (env->me_lfd != INVALID_HANDLE_VALUE) {
+ /* double `unlock` for robustly remove overlapped shared/exclusive locks */
+ while (funlock(env->me_lfd, LCK_LOWER))
+ ;
+ err = (int)GetLastError();
+ assert(err == ERROR_NOT_LOCKED ||
+ (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
+ (void)err;
+ SetLastError(ERROR_SUCCESS);
+
+ while (funlock(env->me_lfd, LCK_UPPER))
+ ;
+ err = (int)GetLastError();
+ assert(err == ERROR_NOT_LOCKED ||
+ (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
+ (void)err;
+ SetLastError(ERROR_SUCCESS);
}
- return MDBX_SUCCESS;
-}
+ if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
+ /* explicitly unlock to avoid latency for other processes (windows kernel
+ * releases such locks via deferred queues) */
+ while (funlock(env->me_lazy_fd, LCK_BODY))
+ ;
+ err = (int)GetLastError();
+ assert(err == ERROR_NOT_LOCKED ||
+ (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
+ (void)err;
+ SetLastError(ERROR_SUCCESS);
+
+ while (funlock(env->me_lazy_fd, LCK_WHOLE))
+ ;
+ err = (int)GetLastError();
+ assert(err == ERROR_NOT_LOCKED ||
+ (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
+ (void)err;
+ SetLastError(ERROR_SUCCESS);
+ }
+}
+
+/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
+ * or as 'used' (S-? and returns MDBX_RESULT_FALSE).
+ * Otherwise returns an error. */
+static int internal_seize_lck(HANDLE lfd) {
+ int rc;
+ assert(lfd != INVALID_HANDLE_VALUE);
-/*----------------------------------------------------------------------------*/
+ /* 1) now on ?-? (free), get ?-E (middle) */
+ mdbx_jitter4testing(false);
+ if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
+ rc = (int)GetLastError() /* 2) something went wrong, give up */;
+ mdbx_error("%s, err %u", "?-?(free) >> ?-E(middle)", rc);
+ return rc;
+ }
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-__cold MDBX_NOTHROW_CONST_FUNCTION intptr_t mdbx_limits_pgsize_min(void) {
- return __inline_mdbx_limits_pgsize_min();
-}
+ /* 3) now on ?-E (middle), try E-E (exclusive-write) */
+ mdbx_jitter4testing(false);
+ if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
+ return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
-__cold MDBX_NOTHROW_CONST_FUNCTION intptr_t mdbx_limits_pgsize_max(void) {
- return __inline_mdbx_limits_pgsize_max();
-}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
+ /* 5) still on ?-E (middle) */
+ rc = (int)GetLastError();
+ mdbx_jitter4testing(false);
+ if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
+ /* 6) something went wrong, give up */
+ if (!funlock(lfd, LCK_UPPER))
+ mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)",
+ (int)GetLastError());
+ return rc;
+ }
-__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) {
- if (pagesize < 1)
- pagesize = (intptr_t)mdbx_default_pagesize();
- else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
- pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2((size_t)pagesize)))
- return -1;
+ /* 7) still on ?-E (middle), try S-E (locked) */
+ mdbx_jitter4testing(false);
+ rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
+ : (int)GetLastError();
- return MIN_PAGENO * pagesize;
-}
+ mdbx_jitter4testing(false);
+ if (rc != MDBX_RESULT_FALSE)
+ mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc);
-__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) {
- if (pagesize < 1)
- pagesize = (intptr_t)mdbx_default_pagesize();
- else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
- pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2((size_t)pagesize)))
- return -1;
+ /* 8) now on S-E (locked) or still on ?-E (middle),
+ * transition to S-? (used) or ?-? (free) */
+ if (!funlock(lfd, LCK_UPPER))
+ mdbx_panic("%s(%s) failed: err %u", __func__,
+ "X-E(locked/middle) >> X-?(used/free)", (int)GetLastError());
- STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
- const uint64_t limit = (1 + (uint64_t)MAX_PAGENO) * pagesize;
- return (limit < MAX_MAPSIZE) ? (intptr_t)limit : (intptr_t)MAX_MAPSIZE;
+ /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
+ return rc;
}
-__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) {
- if (pagesize < 1)
- pagesize = (intptr_t)mdbx_default_pagesize();
- else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE ||
- pagesize > (intptr_t)MAX_PAGESIZE ||
- !is_powerof2((size_t)pagesize)))
- return -1;
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
+ int rc;
- STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
- const uint64_t pgl_limit =
- pagesize * (uint64_t)(MDBX_PGL_LIMIT / 1.6180339887498948482);
- const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / 1.6180339887498948482);
- return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit;
-}
+ assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
+ if (env->me_flags & MDBX_EXCLUSIVE)
+ return MDBX_RESULT_TRUE /* nope since files were must be opened
+ non-shareable */
+ ;
-/*** Key-making functions to avoid custom comparators *************************/
+ if (env->me_lfd == INVALID_HANDLE_VALUE) {
+ /* LY: without-lck mode (e.g. on read-only filesystem) */
+ mdbx_jitter4testing(false);
+ if (!flock(env->me_lazy_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
+ rc = (int)GetLastError();
+ mdbx_error("%s, err %u", "without-lck", rc);
+ return rc;
+ }
+ return MDBX_RESULT_FALSE;
+ }
-static __always_inline double key2double(const int64_t key) {
- union {
- uint64_t u;
- double f;
- } casting;
+ rc = internal_seize_lck(env->me_lfd);
+ mdbx_jitter4testing(false);
+ if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
+ /* Check that another process don't operates in without-lck mode.
+ * Doing such check by exclusive locking the body-part of db. Should be
+ * noted:
+ * - we need an exclusive lock for do so;
+ * - we can't lock meta-pages, otherwise other process could get an error
+ * while opening db in valid (non-conflict) mode. */
+ if (!flock(env->me_lazy_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
+ rc = (int)GetLastError();
+ mdbx_error("%s, err %u", "lock-against-without-lck", rc);
+ mdbx_jitter4testing(false);
+ lck_unlock(env);
+ } else {
+ mdbx_jitter4testing(false);
+ if (!funlock(env->me_lazy_fd, LCK_BODY))
+ mdbx_panic("%s(%s) failed: err %u", __func__,
+ "unlock-against-without-lck", (int)GetLastError());
+ }
+ }
- casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000)
- : UINT64_C(0xffffFFFFffffFFFF) - key;
- return casting.f;
+ return rc;
}
-static __always_inline uint64_t double2key(const double *const ptr) {
- STATIC_ASSERT(sizeof(double) == sizeof(int64_t));
- const int64_t i = *(const int64_t *)ptr;
- const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i
- : i + UINT64_C(0x8000000000000000);
- if (mdbx_assert_enabled()) {
- const double f = key2double(u);
- assert(memcmp(&f, ptr, 8) == 0);
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
+ /* Transite from exclusive-write state (E-E) to used (S-?) */
+ assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+
+ if (env->me_flags & MDBX_EXCLUSIVE)
+ return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
+ ;
+ /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */
+ if (!funlock(env->me_lfd, LCK_LOWER))
+ mdbx_panic("%s(%s) failed: err %u", __func__,
+ "E-E(exclusive-write) >> ?-E(middle)", (int)GetLastError());
+
+ /* 2) now at ?-E (middle), transition to S-E (locked) */
+ if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
+ int rc = (int)GetLastError() /* 3) something went wrong, give up */;
+ mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc);
+ return rc;
}
- return u;
-}
-static __always_inline float key2float(const int32_t key) {
- union {
- uint32_t u;
- float f;
- } casting;
+ /* 4) got S-E (locked), continue transition to S-? (used) */
+ if (!funlock(env->me_lfd, LCK_UPPER))
+ mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)",
+ (int)GetLastError());
- casting.u =
- (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key;
- return casting.f;
+ return MDBX_SUCCESS /* 5) now at S-? (used), done */;
}
-static __always_inline uint32_t float2key(const float *const ptr) {
- STATIC_ASSERT(sizeof(float) == sizeof(int32_t));
- const int32_t i = *(const int32_t *)ptr;
- const uint32_t u =
- (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000);
- if (mdbx_assert_enabled()) {
- const float f = key2float(u);
- assert(memcmp(&f, ptr, 4) == 0);
+MDBX_INTERNAL_FUNC int mdbx_lck_upgrade(MDBX_env *env) {
+ /* Transite from used state (S-?) to exclusive-write (E-E) */
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+
+ if (env->me_flags & MDBX_EXCLUSIVE)
+ return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
+ ;
+
+ int rc;
+ /* 1) now on S-? (used), try S-E (locked) */
+ mdbx_jitter4testing(false);
+ if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_UPPER)) {
+ rc = (int)GetLastError() /* 2) something went wrong, give up */;
+ mdbx_verbose("%s, err %u", "S-?(used) >> S-E(locked)", rc);
+ return rc;
+ }
+
+ /* 3) now on S-E (locked), transition to ?-E (middle) */
+ if (!funlock(env->me_lfd, LCK_LOWER))
+ mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)",
+ (int)GetLastError());
+
+ /* 4) now on ?-E (middle), try E-E (exclusive-write) */
+ mdbx_jitter4testing(false);
+ if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) {
+ rc = (int)GetLastError() /* 5) something went wrong, give up */;
+ mdbx_verbose("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc);
+ return rc;
}
- return u;
-}
-uint64_t mdbx_key_from_double(const double ieee754_64bit) {
- return double2key(&ieee754_64bit);
+ return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */;
}
-uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) {
- return double2key(ieee754_64bit);
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag) {
+ (void)env;
+ (void)inprocess_neighbor;
+ (void)global_uniqueness_flag;
+ return MDBX_SUCCESS;
}
-uint32_t mdbx_key_from_float(const float ieee754_32bit) {
- return float2key(&ieee754_32bit);
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor) {
+ /* LY: should unmap before releasing the locks to avoid race condition and
+ * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
+ if (env->me_map)
+ mdbx_munmap(&env->me_dxb_mmap);
+ if (env->me_lck_mmap.lck) {
+ const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages.weak == 0;
+ mdbx_munmap(&env->me_lck_mmap);
+ if (synced && !inprocess_neighbor && env->me_lfd != INVALID_HANDLE_VALUE &&
+ mdbx_lck_upgrade(env) == MDBX_SUCCESS)
+ /* this will fail if LCK is used/mmapped by other process(es) */
+ mdbx_ftruncate(env->me_lfd, 0);
+ }
+ lck_unlock(env);
+ return MDBX_SUCCESS;
}
-uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) {
- return float2key(ieee754_32bit);
-}
+/*----------------------------------------------------------------------------*/
+/* reader checking (by pid) */
-#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
-MDBX_NOTHROW_CONST_FUNCTION uint64_t mdbx_key_from_int64(const int64_t i64) {
- return __inline_mdbx_key_from_int64(i64);
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
+ (void)env;
+ return MDBX_SUCCESS;
}
-MDBX_NOTHROW_CONST_FUNCTION uint32_t mdbx_key_from_int32(const int32_t i32) {
- return __inline_mdbx_key_from_int32(i32);
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
+ (void)env;
+ return MDBX_SUCCESS;
}
-#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */
-
-#define IEEE754_DOUBLE_MANTISSA_SIZE 52
-#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF
-#define IEEE754_DOUBLE_EXPONENTA_MAX 0x7FF
-#define IEEE754_DOUBLE_IMPLICIT_LEAD UINT64_C(0x0010000000000000)
-#define IEEE754_DOUBLE_MANTISSA_MASK UINT64_C(0x000FFFFFFFFFFFFF)
-#define IEEE754_DOUBLE_MANTISSA_AMAX UINT64_C(0x001FFFFFFFFFFFFF)
-
-static __inline int clz64(uint64_t value) {
-#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_clzl)
- if (sizeof(value) == sizeof(int))
- return __builtin_clz(value);
- if (sizeof(value) == sizeof(long))
- return __builtin_clzl(value);
-#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \
- __has_builtin(__builtin_clzll)
- return __builtin_clzll(value);
-#endif /* have(long long) && long long == uint64_t */
-#endif /* GNU C */
-#if defined(_MSC_VER)
- unsigned long index;
-#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64)
- _BitScanReverse64(&index, value);
- return 63 - index;
-#else
- if (value > UINT32_MAX) {
- _BitScanReverse(&index, (uint32_t)(value >> 32));
- return 31 - index;
+/* Checks reader by pid.
+ *
+ * Returns:
+ * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
+ * MDBX_RESULT_FALSE, if pid is dead (lock acquired)
+ * or otherwise the errcode. */
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
+ (void)env;
+ HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
+ int rc;
+ if (likely(hProcess)) {
+ rc = WaitForSingleObject(hProcess, 0);
+ if (unlikely(rc == (int)WAIT_FAILED))
+ rc = (int)GetLastError();
+ CloseHandle(hProcess);
+ } else {
+ rc = (int)GetLastError();
}
- _BitScanReverse(&index, (uint32_t)value);
- return 63 - index;
-#endif
-#endif /* MSVC */
- value |= value >> 1;
- value |= value >> 2;
- value |= value >> 4;
- value |= value >> 8;
- value |= value >> 16;
- value |= value >> 32;
- static const uint8_t debruijn_clz64[64] = {
- 63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2,
- 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1,
- 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18,
- 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0};
- return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58];
+ switch (rc) {
+ case ERROR_INVALID_PARAMETER:
+ /* pid seems invalid */
+ return MDBX_RESULT_FALSE;
+ case WAIT_OBJECT_0:
+ /* process just exited */
+ return MDBX_RESULT_FALSE;
+ case ERROR_ACCESS_DENIED:
+ /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc.
+ * assume pid exists */
+ return MDBX_RESULT_TRUE;
+ case WAIT_TIMEOUT:
+ /* pid running */
+ return MDBX_RESULT_TRUE;
+ default:
+ /* failure */
+ return rc;
+ }
}
-static __inline uint64_t round_mantissa(const uint64_t u64, int shift) {
- assert(shift < 0 && u64 > 0);
- shift = -shift;
- const unsigned half = 1 << (shift - 1);
- const unsigned lsb = 1 & (unsigned)(u64 >> shift);
- const unsigned tie2even = 1 ^ lsb;
- return (u64 + half - tie2even) >> shift;
+//----------------------------------------------------------------------------
+// Stub for slim read-write lock
+// Copyright (C) 1995-2002 Brad Wilson
+
+static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) {
+ srwl->readerCount = srwl->writerCount = 0;
}
-uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) {
- const uint64_t bias = UINT64_C(0x8000000000000000);
- if (json_integer > 0) {
- const uint64_t u64 = json_integer;
- int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
- uint64_t mantissa = u64 << shift;
- if (unlikely(shift < 0)) {
- mantissa = round_mantissa(u64, shift);
- if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
- mantissa = round_mantissa(u64, --shift);
+static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) {
+ while (true) {
+ assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
+
+ // If there's a writer already, spin without unnecessarily
+ // interlocking the CPUs
+ if (srwl->writerCount != 0) {
+ YieldProcessor();
+ continue;
}
- assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
- mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
- const uint64_t exponent =
- IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
- assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
- const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) +
- (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
-#if !defined(_MSC_VER) || \
- defined( \
- _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
- symbol __except1 referenced in function __ftol3_except */
- assert(key == mdbx_key_from_double((double)json_integer));
-#endif /* Workaround for MSVC */
- return key;
- }
+ // Add to the readers list
+ _InterlockedIncrement(&srwl->readerCount);
- if (json_integer < 0) {
- const uint64_t u64 = -json_integer;
- int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
- uint64_t mantissa = u64 << shift;
- if (unlikely(shift < 0)) {
- mantissa = round_mantissa(u64, shift);
- if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
- mantissa = round_mantissa(u64, --shift);
- }
+ // Check for writers again (we may have been preempted). If
+ // there are no writers writing or waiting, then we're done.
+ if (srwl->writerCount == 0)
+ break;
- assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
- mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
- const uint64_t exponent =
- IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
- assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
- const uint64_t key = bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) -
- (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
-#if !defined(_MSC_VER) || \
- defined( \
- _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
- symbol __except1 referenced in function __ftol3_except */
- assert(key == mdbx_key_from_double((double)json_integer));
-#endif /* Workaround for MSVC */
- return key;
+ // Remove from the readers list, spin, try again
+ _InterlockedDecrement(&srwl->readerCount);
+ YieldProcessor();
}
+}
- return bias;
+static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) {
+ assert(srwl->readerCount > 0);
+ _InterlockedDecrement(&srwl->readerCount);
}
-int64_t mdbx_jsonInteger_from_key(const MDBX_val v) {
- assert(v.iov_len == 8);
- const uint64_t key = unaligned_peek_u64(2, v.iov_base);
- const uint64_t bias = UINT64_C(0x8000000000000000);
- const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1;
- const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 -
- (IEEE754_DOUBLE_EXPONENTA_MAX &
- (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE));
- if (unlikely(shift < 1))
- return (key < bias) ? INT64_MIN : INT64_MAX;
- if (unlikely(shift > 63))
- return 0;
+static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) {
+ while (true) {
+ assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
- const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK)
- << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) +
- bias;
- const int64_t absolute = unscaled >> shift;
- const int64_t value = (key < bias) ? -absolute : absolute;
- assert(key == mdbx_key_from_jsonInteger(value) ||
- (mdbx_key_from_jsonInteger(value - 1) < key &&
- key < mdbx_key_from_jsonInteger(value + 1)));
- return value;
-}
+ // If there's a writer already, spin without unnecessarily
+ // interlocking the CPUs
+ if (srwl->writerCount != 0) {
+ YieldProcessor();
+ continue;
+ }
-double mdbx_double_from_key(const MDBX_val v) {
- assert(v.iov_len == 8);
- return key2double(unaligned_peek_u64(2, v.iov_base));
-}
+ // See if we can become the writer (expensive, because it inter-
+ // locks the CPUs, so writing should be an infrequent process)
+ if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
+ break;
+ }
-float mdbx_float_from_key(const MDBX_val v) {
- assert(v.iov_len == 4);
- return key2float(unaligned_peek_u32(2, v.iov_base));
+ // Now we're the writer, but there may be outstanding readers.
+ // Spin until there aren't any more; new readers will wait now
+ // that we're the writer.
+ while (srwl->readerCount != 0) {
+ assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
+ YieldProcessor();
+ }
}
-int32_t mdbx_int32_from_key(const MDBX_val v) {
- assert(v.iov_len == 4);
- return (int32_t)(unaligned_peek_u32(2, v.iov_base) - UINT32_C(0x80000000));
+static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) {
+ assert(srwl->writerCount == 1 && srwl->readerCount >= 0);
+ srwl->writerCount = 0;
}
-int64_t mdbx_int64_from_key(const MDBX_val v) {
- assert(v.iov_len == 8);
- return (int64_t)(unaligned_peek_u64(2, v.iov_base) -
- UINT64_C(0x8000000000000000));
+static uint64_t WINAPI stub_GetTickCount64(void) {
+ LARGE_INTEGER Counter, Frequency;
+ return (QueryPerformanceFrequency(&Frequency) &&
+ QueryPerformanceCounter(&Counter))
+ ? Counter.QuadPart * 1000ul / Frequency.QuadPart
+ : 0;
}
-__cold MDBX_cmp_func *mdbx_get_keycmp(unsigned flags) {
- return get_default_keycmp(flags);
-}
+/*----------------------------------------------------------------------------*/
-__cold MDBX_cmp_func *mdbx_get_datacmp(unsigned flags) {
- return get_default_datacmp(flags);
+#ifndef xMDBX_ALLOY
+MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
+ mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
+ mdbx_srwlock_ReleaseExclusive;
+
+MDBX_NtExtendSection mdbx_NtExtendSection;
+MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
+MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
+MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
+MDBX_NtFsControlFile mdbx_NtFsControlFile;
+MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+MDBX_GetTickCount64 mdbx_GetTickCount64;
+MDBX_RegGetValueA mdbx_RegGetValueA;
+#endif /* xMDBX_ALLOY */
+
+#if __GNUC_PREREQ(8, 0)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-function-type"
+#endif /* GCC/MINGW */
+
+static void mdbx_winnt_import(void) {
+ const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
+
+#define GET_PROC_ADDR(dll, ENTRY) \
+ mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(dll, #ENTRY)
+
+ if (GetProcAddress(hNtdll, "wine_get_version")) {
+ assert(mdbx_RunningUnderWine());
+ } else {
+ GET_PROC_ADDR(hNtdll, NtFsControlFile);
+ GET_PROC_ADDR(hNtdll, NtExtendSection);
+ assert(!mdbx_RunningUnderWine());
+ }
+
+ const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
+ GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx);
+ GET_PROC_ADDR(hKernel32dll, GetTickCount64);
+ if (!mdbx_GetTickCount64)
+ mdbx_GetTickCount64 = stub_GetTickCount64;
+ if (!mdbx_RunningUnderWine()) {
+ GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle);
+ GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW);
+ GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW);
+ GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory);
+ }
+
+ const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll");
+ GET_PROC_ADDR(hAdvapi32dll, RegGetValueA);
+#undef GET_PROC_ADDR
+
+ const MDBX_srwlock_function init =
+ (MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock");
+ if (init != NULL) {
+ mdbx_srwlock_Init = init;
+ mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress(
+ hKernel32dll, "AcquireSRWLockShared");
+ mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress(
+ hKernel32dll, "ReleaseSRWLockShared");
+ mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress(
+ hKernel32dll, "AcquireSRWLockExclusive");
+ mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress(
+ hKernel32dll, "ReleaseSRWLockExclusive");
+ } else {
+ mdbx_srwlock_Init = stub_srwlock_Init;
+ mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared;
+ mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared;
+ mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive;
+ mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
+ }
}
-__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
- uint64_t value) {
- int err = check_env(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- const bool lock_needed = ((env->me_flags & MDBX_ENV_ACTIVE) && env->me_txn0 &&
- env->me_txn0->mt_owner != mdbx_thread_self());
- bool should_unlock = false;
- switch (option) {
- case MDBX_opt_sync_bytes:
- if (value == UINT64_MAX)
- value = SIZE_MAX - 65536;
- if (unlikely(env->me_flags & MDBX_RDONLY))
- return MDBX_EACCESS;
- if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
- return MDBX_EPERM;
- if (unlikely(value > SIZE_MAX - 65536))
- return MDBX_TOO_LARGE;
- if (atomic_store32(&env->me_lck->mti_autosync_threshold,
- bytes2pgno(env, (size_t)value + env->me_psize - 1),
- mo_Relaxed) != 0 &&
- (env->me_flags & MDBX_ENV_ACTIVE)) {
- err = mdbx_env_sync_poll(env);
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
- err = MDBX_SUCCESS;
- }
- break;
+#if __GNUC_PREREQ(8, 0)
+#pragma GCC diagnostic pop
+#endif /* GCC/MINGW */
- case MDBX_opt_sync_period:
- if (value == UINT64_MAX)
- value = UINT32_MAX;
- if (unlikely(env->me_flags & MDBX_RDONLY))
- return MDBX_EACCESS;
- if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
- return MDBX_EPERM;
- if (unlikely(value > UINT32_MAX))
- return MDBX_TOO_LARGE;
- if (atomic_store64(&env->me_lck->mti_autosync_period,
- mdbx_osal_16dot16_to_monotime((uint32_t)value),
- mo_Relaxed) != 0 &&
- (env->me_flags & MDBX_ENV_ACTIVE)) {
- err = mdbx_env_sync_poll(env);
- if (unlikely(MDBX_IS_ERROR(err)))
- return err;
- err = MDBX_SUCCESS;
- }
- break;
+#endif /* Windows LCK-implementation */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
- case MDBX_opt_max_db:
- if (value == UINT64_MAX)
- value = MDBX_MAX_DBI;
- if (unlikely(value > MDBX_MAX_DBI))
- return MDBX_EINVAL;
- if (unlikely(env->me_map))
- return MDBX_EPERM;
- env->me_maxdbs = (unsigned)value + CORE_DBS;
- break;
+#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */
- case MDBX_opt_max_readers:
- if (value == UINT64_MAX)
- value = MDBX_READERS_LIMIT;
- if (unlikely(value < 1 || value > MDBX_READERS_LIMIT))
- return MDBX_EINVAL;
- if (unlikely(env->me_map))
- return MDBX_EPERM;
- env->me_maxreaders = (unsigned)value;
- break;
- case MDBX_opt_dp_reserve_limit:
- if (value == UINT64_MAX)
- value = INT_MAX;
- if (unlikely(value > INT_MAX))
- return MDBX_EINVAL;
- if (env->me_options.dp_reserve_limit != (unsigned)value) {
- if (lock_needed) {
- err = mdbx_txn_lock(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- should_unlock = true;
- }
- env->me_options.dp_reserve_limit = (unsigned)value;
- while (env->me_dp_reserve_len > env->me_options.dp_reserve_limit) {
- mdbx_assert(env, env->me_dp_reserve != NULL);
- MDBX_page *dp = env->me_dp_reserve;
- MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize);
- VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
- env->me_dp_reserve = dp->mp_next;
- VALGRIND_MEMPOOL_FREE(env, dp);
- mdbx_free(dp);
- env->me_dp_reserve_len -= 1;
- }
- }
- break;
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+#include <sys/sem.h>
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
- case MDBX_opt_rp_augment_limit:
- if (value == UINT64_MAX)
- value = MDBX_PGL_LIMIT;
- if (unlikely(value > MDBX_PGL_LIMIT))
- return MDBX_EINVAL;
- env->me_options.rp_augment_limit = (unsigned)value;
- break;
+/*----------------------------------------------------------------------------*/
+/* global constructor/destructor */
- case MDBX_opt_txn_dp_limit:
- case MDBX_opt_txn_dp_initial:
- if (value == UINT64_MAX)
- value = MDBX_PGL_LIMIT;
- if (unlikely(value > MDBX_PGL_LIMIT || value < CURSOR_STACK * 4))
- return MDBX_EINVAL;
- if (unlikely(env->me_flags & MDBX_RDONLY))
- return MDBX_EACCESS;
- if (lock_needed) {
- err = mdbx_txn_lock(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- should_unlock = true;
- }
- if (env->me_txn)
- err = MDBX_EPERM /* unable change during transaction */;
- else {
- const pgno_t value32 = (pgno_t)value;
- if (option == MDBX_opt_txn_dp_initial &&
- env->me_options.dp_initial != value32) {
- env->me_options.dp_initial = value32;
- if (env->me_options.dp_limit < value32) {
- env->me_options.dp_limit = value32;
- env->me_options.flags.non_auto.dp_limit = 1;
+#if defined(__linux__) || defined(__gnu_linux__)
+
+#include <sys/utsname.h>
+
+#ifndef xMDBX_ALLOY
+uint32_t mdbx_linux_kernel_version;
+bool mdbx_RunningOnWSL1;
+#endif /* xMDBX_ALLOY */
+
+__cold static uint8_t probe_for_WSL(const char *tag) {
+ const char *const WSL = strstr(tag, "WSL");
+ if (WSL && WSL[3] >= '2' && WSL[3] <= '9')
+ return WSL[3] - '0';
+ const char *const wsl = strstr(tag, "wsl");
+ if (wsl && wsl[3] >= '2' && wsl[3] <= '9')
+ return wsl[3] - '0';
+ if (WSL || wsl || strcasestr(tag, "Microsoft"))
+ /* Expecting no new kernel within WSL1, either it will explicitly
+ * marked by an appropriate WSL-version hint. */
+ return (mdbx_linux_kernel_version < /* 4.19.x */ 0x04130000) ? 1 : 2;
+ return 0;
+}
+
+#endif /* Linux */
+
+__cold static __attribute__((__constructor__)) void
+mdbx_global_constructor(void) {
+#if defined(__linux__) || defined(__gnu_linux__)
+ struct utsname buffer;
+ if (uname(&buffer) == 0) {
+ int i = 0;
+ char *p = buffer.release;
+ while (*p && i < 4) {
+ if (*p >= '0' && *p <= '9') {
+ long number = strtol(p, &p, 10);
+ if (number > 0) {
+ if (number > 255)
+ number = 255;
+ mdbx_linux_kernel_version += number << (24 - i * 8);
}
- }
- if (option == MDBX_opt_txn_dp_limit &&
- env->me_options.dp_limit != value32) {
- env->me_options.dp_limit = value32;
- env->me_options.flags.non_auto.dp_limit = 1;
- if (env->me_options.dp_initial > value32)
- env->me_options.dp_initial = value32;
+ ++i;
+ } else {
+ ++p;
}
}
- break;
+ /* "Official" way of detecting WSL1 but not WSL2
+ * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
+ *
+ * WARNING: False negative detection of WSL1 will result in DATA LOSS!
+ * So, the REQUIREMENTS for this code:
+ * 1. MUST detect WSL1 without false-negatives.
+ * 2. DESIRABLE detect WSL2 but without the risk of violating the first. */
+ mdbx_RunningOnWSL1 = probe_for_WSL(buffer.version) == 1 ||
+ probe_for_WSL(buffer.sysname) == 1 ||
+ probe_for_WSL(buffer.release) == 1;
+ }
+#endif /* Linux */
- case MDBX_opt_spill_max_denominator:
- if (value == UINT64_MAX)
- value = 255;
- if (unlikely(value > 255))
- return MDBX_EINVAL;
- env->me_options.spill_max_denominator = (uint8_t)value;
- break;
- case MDBX_opt_spill_min_denominator:
- if (unlikely(value > 255))
- return MDBX_EINVAL;
- env->me_options.spill_min_denominator = (uint8_t)value;
- break;
- case MDBX_opt_spill_parent4child_denominator:
- if (unlikely(value > 255))
- return MDBX_EINVAL;
- env->me_options.spill_parent4child_denominator = (uint8_t)value;
- break;
+ mdbx_rthc_global_init();
+}
- case MDBX_opt_loose_limit:
- if (value == UINT64_MAX)
- value = 255;
- if (unlikely(value > 255))
- return MDBX_EINVAL;
- env->me_options.dp_loose_limit = (uint8_t)value;
- break;
+__cold static __attribute__((__destructor__)) void
+mdbx_global_destructor(void) {
+ mdbx_rthc_global_dtor();
+}
- case MDBX_opt_merge_threshold_16dot16_percent:
- if (value == UINT64_MAX)
- value = 32768;
- if (unlikely(value < 8192 || value > 32768))
- return MDBX_EINVAL;
- env->me_options.merge_threshold_16dot16_percent = (unsigned)value;
- recalculate_merge_threshold(env);
- break;
+/*----------------------------------------------------------------------------*/
+/* lck */
- default:
- return MDBX_EINVAL;
- }
+/* ОпиÑание реализации блокировок Ð´Ð»Ñ POSIX & Linux:
+ *
+ * lck-файл отображаетÑÑ Ð² памÑÑ‚ÑŒ, в нём организуетÑÑ Ñ‚Ð°Ð±Ð»Ð¸Ñ†Ð° читателей и
+ * размещаютÑÑ ÑовмеÑтно иÑпользуемые posix-мьютекÑÑ‹ (futex). ПоÑредÑтвом
+ * Ñтих мьютекÑов (Ñм struct MDBX_lockinfo) реализуютÑÑ:
+ * - Блокировка таблицы читателей Ð´Ð»Ñ Ñ€ÐµÐ³Ð¸Ñтрации,
+ * т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
+ * - Блокировка БД Ð´Ð»Ñ Ð¿Ð¸ÑˆÑƒÑ‰Ð¸Ñ… транзакций,
+ * т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
+ *
+ * ОÑтальной функционал реализуетÑÑ Ð¾Ñ‚Ð´ÐµÐ»ÑŒÐ½Ð¾ поÑредÑтвом файловых блокировок:
+ * - Первоначальный захват БД в режиме exclusive/shared и поÑледующий перевод
+ * в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
+ * - Проверка приÑутÑтвие процеÑÑов-читателей,
+ * т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
+ *
+ * Ð”Ð»Ñ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ¸ файлов иÑпользуетÑÑ fcntl(F_SETLK), так как:
+ * - lockf() оперирует только ÑкÑклюзивной блокировкой и требует
+ * Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚Ð¸Ñ Ñ„Ð°Ð¹Ð»Ð° в RW-режиме.
+ * - flock() не гарантирует атомарноÑти при Ñмене блокировок
+ * и оперирует только вÑем файлом целиком.
+ * - Ð”Ð»Ñ ÐºÐ¾Ð½Ñ‚Ñ€Ð¾Ð»Ñ Ð¿Ñ€Ð¾Ñ†ÐµÑÑов-читателей иÑпользуютÑÑ Ð¾Ð´Ð½Ð¾Ð±Ð°Ð¹Ñ‚Ð¾Ð²Ñ‹Ðµ
+ * range-блокировки lck-файла поÑредÑтвом fcntl(F_SETLK). При Ñтом
+ * в качеÑтве позиции иÑпользуетÑÑ pid процеÑÑа-читателÑ.
+ * - Ð”Ð»Ñ Ð¿ÐµÑ€Ð²Ð¾Ð½Ð°Ñ‡Ð°Ð»ÑŒÐ½Ð¾Ð³Ð¾ захвата и shared/exclusive выполнÑетÑÑ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ°
+ * оÑновного файла БД и при уÑпехе lck-файла.
+ *
+ * ----------------------------------------------------------------------------
+ * УДЕРЖИВÐЕМЫЕ БЛОКИРОВКИ Ð’ ЗÐВИСИМОСТИ ОТ РЕЖИМРИ СОСТОЯÐИЯ
+ *
+ * ÐкÑклюзивный режим без lck-файла:
+ * = заблокирован веÑÑŒ dxb-файл поÑредÑтвом F_RDLCK или F_WRLCK,
+ * в завиÑимоÑти от MDBX_RDONLY.
+ *
+ * Ðе-операционный режим на Ð²Ñ€ÐµÐ¼Ñ Ð¿ÐµÑ€Ðµ-инициализации и разрушении lck-файла:
+ * = F_WRLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ ждут её
+ * ÑнÑÑ‚Ð¸Ñ Ð¿Ñ€Ð¸ получении F_RDLCK через F_SETLKW.
+ * - блокировки dxb-файла могут менÑÑ‚ÑŒÑÑ Ð´Ð¾ ÑнÑтие ÑкÑклюзивной блокировки
+ * lck-файла:
+ * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле
+ * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
+ * + Ð´Ð»Ñ ÐКСКЛЮЗИВÐОГО режима блокировка pid-байта вÑего dxb-файла
+ * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
+ *
+ * ОПЕРÐЦИОÐÐЫЙ режим Ñ lck-файлом:
+ * = F_RDLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ не могут
+ * получить F_WRLCK и таким образом видÑÑ‚ что БД иÑпользуетÑÑ.
+ * + F_WRLCK блокировка pid-байта в clk-файле поÑле первой транзакции чтениÑ.
+ * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле
+ * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
+ * + Ð´Ð»Ñ ÐКСКЛЮЗИВÐОГО режима блокировка pid-байта вÑего dxb-файла
+ * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
+ */
- if (should_unlock)
- mdbx_txn_unlock(env);
- return err;
+#if MDBX_USE_OFDLOCKS
+static int op_setlk, op_setlkw, op_getlk;
+__cold static void choice_fcntl() {
+ assert(!op_setlk && !op_setlkw && !op_getlk);
+ if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
+#if defined(__linux__) || defined(__gnu_linux__)
+ && mdbx_linux_kernel_version >
+ 0x030f0000 /* OFD locks are available since 3.15, but engages here
+ only for 3.16 and later kernels (i.e. LTS) because
+ of reliability reasons */
+#endif /* linux */
+ ) {
+ op_setlk = F_OFD_SETLK;
+ op_setlkw = F_OFD_SETLKW;
+ op_getlk = F_OFD_GETLK;
+ return;
+ }
+ op_setlk = F_SETLK;
+ op_setlkw = F_SETLKW;
+ op_getlk = F_GETLK;
}
+#else
+#define op_setlk F_SETLK
+#define op_setlkw F_SETLKW
+#define op_getlk F_GETLK
+#endif /* MDBX_USE_OFDLOCKS */
+
+#ifndef OFF_T_MAX
+#define OFF_T_MAX \
+ (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
+#endif
-__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option,
- uint64_t *pvalue) {
- int err = check_env(env, false);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
- if (unlikely(!pvalue))
- return MDBX_EINVAL;
+static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
+ const off_t offset, off_t len) {
+ STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) &&
+ sizeof(off_t) >= sizeof(size_t));
+#ifdef __ANDROID_API__
+ STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS),
+ "The bitness of system `off_t` type is mismatch. Please "
+ "fix build and/or NDK configuration.");
+#endif /* Android */
+ mdbx_jitter4testing(true);
+ assert(offset >= 0 && len > 0);
+ assert((uint64_t)offset < (uint64_t)INT64_MAX &&
+ (uint64_t)len < (uint64_t)INT64_MAX &&
+ (uint64_t)(offset + len) > (uint64_t)offset);
- switch (option) {
- case MDBX_opt_sync_bytes:
- if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
- return MDBX_EPERM;
- *pvalue = pgno2bytes(
- env, atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed));
- break;
+ assert((uint64_t)offset < (uint64_t)OFF_T_MAX &&
+ (uint64_t)len <= (uint64_t)OFF_T_MAX &&
+ (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX);
- case MDBX_opt_sync_period:
- if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE)))
- return MDBX_EPERM;
- *pvalue = mdbx_osal_monotime_to_16dot16(
- atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed));
- break;
+ assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) ==
+ ((uint64_t)offset + (uint64_t)len));
+ for (;;) {
+ struct flock lock_op;
+ STATIC_ASSERT(sizeof(off_t) <= sizeof(lock_op.l_start) &&
+ sizeof(off_t) <= sizeof(lock_op.l_len) &&
+ OFF_T_MAX == (off_t)OFF_T_MAX);
+ memset(&lock_op, 0, sizeof(lock_op));
+ lock_op.l_type = lck;
+ lock_op.l_whence = SEEK_SET;
+ lock_op.l_start = offset;
+ lock_op.l_len = len;
+ int rc = fcntl(fd, cmd, &lock_op);
+ mdbx_jitter4testing(true);
+ if (rc != -1) {
+ if (cmd == op_getlk) {
+ /* Checks reader by pid. Returns:
+ * MDBX_RESULT_TRUE - if pid is live (reader holds a lock).
+ * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */
+ return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
+ : MDBX_RESULT_TRUE;
+ }
+ return MDBX_SUCCESS;
+ }
+ rc = errno;
+#if MDBX_USE_OFDLOCKS
+ if (rc == EINVAL &&
+ (cmd == F_OFD_SETLK || cmd == F_OFD_SETLKW || cmd == F_OFD_GETLK)) {
+ /* fallback to non-OFD locks */
+ if (cmd == F_OFD_SETLK)
+ cmd = F_SETLK;
+ else if (cmd == F_OFD_SETLKW)
+ cmd = F_SETLKW;
+ else
+ cmd = F_GETLK;
+ op_setlk = F_SETLK;
+ op_setlkw = F_SETLKW;
+ op_getlk = F_GETLK;
+ continue;
+ }
+#endif /* MDBX_USE_OFDLOCKS */
+ if (rc != EINTR || cmd == op_setlkw) {
+ assert(MDBX_IS_ERROR(rc));
+ return rc;
+ }
+ }
+}
- case MDBX_opt_max_db:
- *pvalue = env->me_maxdbs - CORE_DBS;
- break;
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait) {
+#if MDBX_USE_OFDLOCKS
+ if (unlikely(op_setlk == 0))
+ choice_fcntl();
+#endif /* MDBX_USE_OFDLOCKS */
+ return lck_op(fd, wait ? op_setlkw : op_setlk, F_WRLCK, 0, OFF_T_MAX);
+}
- case MDBX_opt_max_readers:
- *pvalue = env->me_maxreaders;
- break;
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+ assert(env->me_pid > 0);
+ if (unlikely(mdbx_getpid() != env->me_pid))
+ return MDBX_PANIC;
+ return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1);
+}
- case MDBX_opt_dp_reserve_limit:
- *pvalue = env->me_options.dp_reserve_limit;
- break;
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+ assert(env->me_pid > 0);
+ return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1);
+}
- case MDBX_opt_rp_augment_limit:
- *pvalue = env->me_options.rp_augment_limit;
- break;
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+ assert(pid > 0);
+ return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1);
+}
- case MDBX_opt_txn_dp_limit:
- *pvalue = env->me_options.dp_limit;
- break;
- case MDBX_opt_txn_dp_initial:
- *pvalue = env->me_options.dp_initial;
- break;
+/*---------------------------------------------------------------------------*/
- case MDBX_opt_spill_max_denominator:
- *pvalue = env->me_options.spill_max_denominator;
- break;
- case MDBX_opt_spill_min_denominator:
- *pvalue = env->me_options.spill_min_denominator;
- break;
- case MDBX_opt_spill_parent4child_denominator:
- *pvalue = env->me_options.spill_parent4child_denominator;
- break;
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc) {
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+ return sem_init(ipc, false, 1) ? errno : 0;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+ return pthread_mutex_init(ipc, nullptr);
+#else
+#error "FIXME"
+#endif
+}
- case MDBX_opt_loose_limit:
- *pvalue = env->me_options.dp_loose_limit;
- break;
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc) {
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+ return sem_destroy(ipc) ? errno : 0;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+ return pthread_mutex_destroy(ipc);
+#else
+#error "FIXME"
+#endif
+}
+#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */
- case MDBX_opt_merge_threshold_16dot16_percent:
- *pvalue = env->me_options.merge_threshold_16dot16_percent;
- break;
+static int check_fstat(MDBX_env *env) {
+ struct stat st;
- default:
- return MDBX_EINVAL;
+ int rc = MDBX_SUCCESS;
+ if (fstat(env->me_lazy_fd, &st)) {
+ rc = errno;
+ mdbx_error("fstat(%s), err %d", "DXB", rc);
+ return rc;
+ }
+
+ if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
+#ifdef EBADFD
+ rc = EBADFD;
+#else
+ rc = EPERM;
+#endif
+ mdbx_error("%s %s, err %d", "DXB",
+ (st.st_nlink < 1) ? "file was removed" : "not a regular file",
+ rc);
+ return rc;
+ }
+
+ if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) {
+ mdbx_verbose("dxb-file is too short (%u), exclusive-lock needed",
+ (unsigned)st.st_size);
+ rc = MDBX_RESULT_TRUE;
}
- return MDBX_SUCCESS;
-}
+ //----------------------------------------------------------------------------
-/*** Attribute support functions for Nexenta **********************************/
-#ifdef MDBX_NEXENTA_ATTRS
+ if (fstat(env->me_lfd, &st)) {
+ rc = errno;
+ mdbx_error("fstat(%s), err %d", "LCK", rc);
+ return rc;
+ }
-static __inline int mdbx_attr_peek(MDBX_val *data, mdbx_attr_t *attrptr) {
- if (unlikely(data->iov_len < sizeof(mdbx_attr_t)))
- return MDBX_INCOMPATIBLE;
+ if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
+#ifdef EBADFD
+ rc = EBADFD;
+#else
+ rc = EPERM;
+#endif
+ mdbx_error("%s %s, err %d", "LCK",
+ (st.st_nlink < 1) ? "file was removed" : "not a regular file",
+ rc);
+ return rc;
+ }
- if (likely(attrptr != NULL))
- *attrptr = *(mdbx_attr_t *)data->iov_base;
- data->iov_len -= sizeof(mdbx_attr_t);
- data->iov_base =
- likely(data->iov_len > 0) ? ((mdbx_attr_t *)data->iov_base) + 1 : NULL;
+ /* Checking file size for detect the situation when we got the shared lock
+ * immediately after mdbx_lck_destroy(). */
+ if (st.st_size < (off_t)(sizeof(MDBX_lockinfo) + sizeof(MDBX_reader))) {
+ mdbx_verbose("lck-file is too short (%u), exclusive-lock needed",
+ (unsigned)st.st_size);
+ rc = MDBX_RESULT_TRUE;
+ }
- return MDBX_SUCCESS;
+ return rc;
}
-static __inline int mdbx_attr_poke(MDBX_val *reserved, MDBX_val *data,
- mdbx_attr_t attr, MDBX_put_flags_t flags) {
- mdbx_attr_t *space = reserved->iov_base;
- if (flags & MDBX_RESERVE) {
- if (likely(data != NULL)) {
- data->iov_base = data->iov_len ? space + 1 : NULL;
+__cold MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
+ assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
+ if (unlikely(mdbx_getpid() != env->me_pid))
+ return MDBX_PANIC;
+#if MDBX_USE_OFDLOCKS
+ if (unlikely(op_setlk == 0))
+ choice_fcntl();
+#endif /* MDBX_USE_OFDLOCKS */
+
+ int rc = MDBX_SUCCESS;
+#if defined(__linux__) || defined(__gnu_linux__)
+ if (unlikely(mdbx_RunningOnWSL1)) {
+ rc = ENOLCK /* No record locks available */;
+ mdbx_error("%s, err %u",
+ "WSL1 (Windows Subsystem for Linux) is mad and trouble-full, "
+ "injecting failure to avoid data loss",
+ rc);
+ return rc;
+ }
+#endif /* Linux */
+
+ if (env->me_lfd == INVALID_HANDLE_VALUE) {
+ /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
+ rc =
+ lck_op(env->me_lazy_fd, op_setlk,
+ (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("%s, err %u", "without-lck", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
+ return rc;
}
- } else {
- *space = attr;
- if (likely(data != NULL)) {
- memcpy(space + 1, data->iov_base, data->iov_len);
+ return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
+ }
+#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0
+ sched_yield();
+#endif
+
+retry:
+ if (rc == MDBX_RESULT_TRUE) {
+ rc = lck_op(env->me_lfd, op_setlk, F_UNLCK, 0, 1);
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("%s, err %u", "unlock-before-retry", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
+ return rc;
}
}
- return MDBX_SUCCESS;
-}
+ /* Firstly try to get exclusive locking. */
+ rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
+ if (rc == MDBX_SUCCESS) {
+ rc = check_fstat(env);
+ if (MDBX_IS_ERROR(rc))
+ return rc;
-int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
- mdbx_attr_t *attrptr, MDBX_cursor_op op) {
- int rc = mdbx_cursor_get(mc, key, data, op);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ continue_dxb_exclusive:
+ rc =
+ lck_op(env->me_lazy_fd, op_setlk,
+ (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
+ if (rc == MDBX_SUCCESS)
+ return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
- return mdbx_attr_peek(data, attrptr);
-}
+ int err = check_fstat(env);
+ if (MDBX_IS_ERROR(err))
+ return err;
-int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
- uint64_t *attrptr) {
- int rc = mdbx_get(txn, dbi, key, data);
- if (unlikely(rc != MDBX_SUCCESS))
+ /* the cause may be a collision with POSIX's file-lock recovery. */
+ if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
+ rc == EDEADLK)) {
+ mdbx_error("%s, err %u", "dxb-exclusive", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
+ return rc;
+ }
+
+ /* Fallback to lck-shared */
+ } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY ||
+ rc == EWOULDBLOCK || rc == EDEADLK)) {
+ mdbx_error("%s, err %u", "try-exclusive", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
+ }
- return mdbx_attr_peek(data, attrptr);
-}
+ /* Here could be one of two:
+ * - mdbx_lck_destroy() from the another process was hold the lock
+ * during a destruction.
+ * - either mdbx_lck_seize() from the another process was got the exclusive
+ * lock and doing initialization.
+ * For distinguish these cases will use size of the lck-file later. */
-int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
- mdbx_attr_t attr, MDBX_put_flags_t flags) {
- MDBX_val reserve;
- reserve.iov_base = NULL;
- reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t);
+ /* Wait for lck-shared now. */
+ /* Here may be await during transient processes, for instance until another
+ * competing process doesn't call lck_downgrade(). */
+ rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1);
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("%s, err %u", "try-shared", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
+ return rc;
+ }
- int rc = mdbx_put(txn, dbi, key, &reserve, flags | MDBX_RESERVE);
- if (unlikely(rc != MDBX_SUCCESS))
+ rc = check_fstat(env);
+ if (rc == MDBX_RESULT_TRUE)
+ goto retry;
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("%s, err %u", "lck_fstat", rc);
return rc;
+ }
- return mdbx_attr_poke(&reserve, data, attr, flags);
-}
+ /* got shared, retry exclusive */
+ rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
+ if (rc == MDBX_SUCCESS)
+ goto continue_dxb_exclusive;
-int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data,
- mdbx_attr_t attr, MDBX_put_flags_t flags) {
- MDBX_val reserve;
- reserve.iov_base = NULL;
- reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t);
+ if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
+ rc == EDEADLK)) {
+ mdbx_error("%s, err %u", "try-exclusive", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
+ return rc;
+ }
- int rc = mdbx_cursor_put(cursor, key, &reserve, flags | MDBX_RESERVE);
- if (unlikely(rc != MDBX_SUCCESS))
+ /* Lock against another process operating in without-lck or exclusive mode. */
+ rc =
+ lck_op(env->me_lazy_fd, op_setlk,
+ (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1);
+ if (rc != MDBX_SUCCESS) {
+ mdbx_error("%s, err %u", "lock-against-without-lck", rc);
+ mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
+ }
- return mdbx_attr_poke(&reserve, data, attr, flags);
+ /* Done: return with shared locking. */
+ return MDBX_RESULT_FALSE;
}
-int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
- mdbx_attr_t attr) {
- if (unlikely(!key || !txn))
- return MDBX_EINVAL;
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
+ assert(env->me_lfd != INVALID_HANDLE_VALUE);
+ if (unlikely(mdbx_getpid() != env->me_pid))
+ return MDBX_PANIC;
- if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
- return MDBX_EBADSIGN;
+ int rc = MDBX_SUCCESS;
+ if ((env->me_flags & MDBX_EXCLUSIVE) == 0) {
+ rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, 0, env->me_pid);
+ if (rc == MDBX_SUCCESS)
+ rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, env->me_pid + 1,
+ OFF_T_MAX - env->me_pid - 1);
+ }
+ if (rc == MDBX_SUCCESS)
+ rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
+ if (unlikely(rc != 0)) {
+ mdbx_error("%s, err %u", "lck", rc);
+ assert(MDBX_IS_ERROR(rc));
+ }
+ return rc;
+}
- if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
- return MDBX_BAD_DBI;
+__cold MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor) {
+ if (unlikely(mdbx_getpid() != env->me_pid))
+ return MDBX_PANIC;
- if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
- return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN;
+ int rc = MDBX_SUCCESS;
+ struct stat lck_info;
+ MDBX_lockinfo *lck = env->me_lck_mmap.lck;
+ if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && lck &&
+ /* try get exclusive access */
+ lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
+ /* if LCK was not removed */
+ fstat(env->me_lfd, &lck_info) == 0 && lck_info.st_nlink > 0 &&
+ lck_op(env->me_lazy_fd, op_setlk,
+ (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
+ OFF_T_MAX) == 0) {
+
+ mdbx_verbose("%p got exclusive, drown locks", (void *)env);
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ if (env->me_sysv_ipc.semid != -1)
+ rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0;
+#else
+ rc = mdbx_ipclock_destroy(&lck->mti_rlock);
+ if (rc == 0)
+ rc = mdbx_ipclock_destroy(&lck->mti_wlock);
+#endif /* MDBX_LOCKING */
- MDBX_cursor_couple cx;
- MDBX_val old_data;
- int rc = mdbx_cursor_init(&cx.outer, txn, dbi);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- rc = mdbx_cursor_set(&cx.outer, key, &old_data, MDBX_SET, NULL);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_NOTFOUND && data) {
- cx.outer.mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = &cx.outer;
- rc = mdbx_cursor_put_attr(&cx.outer, key, data, attr, 0);
- txn->mt_cursors[dbi] = cx.outer.mc_next;
+ mdbx_assert(env, rc == 0);
+ if (rc == 0) {
+ const bool synced = lck->mti_unsynced_pages.weak == 0;
+ mdbx_munmap(&env->me_lck_mmap);
+ if (synced)
+ rc = ftruncate(env->me_lfd, 0) ? errno : 0;
}
- return rc;
+
+ mdbx_jitter4testing(false);
}
- mdbx_attr_t old_attr = 0;
- rc = mdbx_attr_peek(&old_data, &old_attr);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
+ /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored
+ * after file was closed.
+ *
+ * 2) File locks would be released (by kernel) while the file-descriptors will
+ * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel,
+ * locks should be released here explicitly with properly order. */
- if (old_attr == attr && (!data || (data->iov_len == old_data.iov_len &&
- memcmp(data->iov_base, old_data.iov_base,
- old_data.iov_len) == 0)))
- return MDBX_SUCCESS;
+ /* close dxb and restore lock */
+ if (env->me_dsync_fd != INVALID_HANDLE_VALUE) {
+ if (unlikely(close(env->me_dsync_fd) != 0) && rc == MDBX_SUCCESS)
+ rc = errno;
+ env->me_dsync_fd = INVALID_HANDLE_VALUE;
+ }
+ if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
+ if (unlikely(close(env->me_lazy_fd) != 0) && rc == MDBX_SUCCESS)
+ rc = errno;
+ env->me_lazy_fd = INVALID_HANDLE_VALUE;
+ if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
+ /* restore file-lock */
+ rc = lck_op(
+ inprocess_neighbor->me_lazy_fd, F_SETLKW,
+ (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
+ (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE)
+ ? 0
+ : inprocess_neighbor->me_pid,
+ (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
+ }
+ }
- cx.outer.mc_next = txn->mt_cursors[dbi];
- txn->mt_cursors[dbi] = &cx.outer;
- rc = mdbx_cursor_put_attr(&cx.outer, key, data ? data : &old_data, attr,
- MDBX_CURRENT);
- txn->mt_cursors[dbi] = cx.outer.mc_next;
+ /* close clk and restore locks */
+ if (env->me_lfd != INVALID_HANDLE_VALUE) {
+ if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS)
+ rc = errno;
+ env->me_lfd = INVALID_HANDLE_VALUE;
+ if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
+ /* restore file-locks */
+ rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1);
+ if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader)
+ rc = mdbx_rpid_set(inprocess_neighbor);
+ }
+ }
+
+ if (inprocess_neighbor && rc != MDBX_SUCCESS)
+ inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR;
return rc;
}
-#endif /* MDBX_NEXENTA_ATTRS */
-/******************************************************************************/
-/* *INDENT-OFF* */
-/* clang-format off */
+/*---------------------------------------------------------------------------*/
-__dll_export
-#ifdef __attribute_used__
- __attribute_used__
-#elif defined(__GNUC__) || __has_attribute(__used__)
- __attribute__((__used__))
-#endif
-#ifdef __attribute_externally_visible__
- __attribute_externally_visible__
-#elif (defined(__GNUC__) && !defined(__clang__)) || \
- __has_attribute(__externally_visible__)
- __attribute__((__externally_visible__))
-#endif
- const struct MDBX_build_info mdbx_build = {
-#ifdef MDBX_BUILD_TIMESTAMP
- MDBX_BUILD_TIMESTAMP
-#else
- "\"" __DATE__ " " __TIME__ "\""
-#endif /* MDBX_BUILD_TIMESTAMP */
+__cold MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag) {
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ int semid = -1;
+ /* don't initialize semaphores twice */
+ (void)inprocess_neighbor;
+ if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
+ struct stat st;
+ if (fstat(env->me_lazy_fd, &st))
+ return errno;
+ sysv_retry_create:
+ semid = semget(env->me_sysv_ipc.key, 2,
+ IPC_CREAT | IPC_EXCL |
+ (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)));
+ if (unlikely(semid == -1)) {
+ int err = errno;
+ if (err != EEXIST)
+ return err;
- ,
-#ifdef MDBX_BUILD_TARGET
- MDBX_BUILD_TARGET
-#else
- #if defined(__ANDROID_API__)
- "Android" MDBX_STRINGIFY(__ANDROID_API__)
- #elif defined(__linux__) || defined(__gnu_linux__)
- "Linux"
- #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__)
- "webassembly"
- #elif defined(__CYGWIN__)
- "CYGWIN"
- #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \
- || defined(__WINDOWS__)
- "Windows"
- #elif defined(__APPLE__)
- #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \
- || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR)
- "iOS"
- #else
- "MacOS"
- #endif
- #elif defined(__FreeBSD__)
- "FreeBSD"
- #elif defined(__DragonFly__)
- "DragonFlyBSD"
- #elif defined(__NetBSD__)
- "NetBSD"
- #elif defined(__OpenBSD__)
- "OpenBSD"
- #elif defined(__bsdi__)
- "UnixBSDI"
- #elif defined(__MACH__)
- "MACH"
- #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC))
- "HPUX"
- #elif defined(_AIX)
- "AIX"
- #elif defined(__sun) && defined(__SVR4)
- "Solaris"
- #elif defined(__BSD__) || defined(BSD)
- "UnixBSD"
- #elif defined(__unix__) || defined(UNIX) || defined(__unix) \
- || defined(__UNIX) || defined(__UNIX__)
- "UNIX"
- #elif defined(_POSIX_VERSION)
- "POSIX" MDBX_STRINGIFY(_POSIX_VERSION)
- #else
- "UnknownOS"
- #endif /* Target OS */
+ /* remove and re-create semaphore set */
+ semid = semget(env->me_sysv_ipc.key, 2, 0);
+ if (semid == -1) {
+ err = errno;
+ if (err != ENOENT)
+ return err;
+ goto sysv_retry_create;
+ }
+ if (semctl(semid, 2, IPC_RMID)) {
+ err = errno;
+ if (err != EIDRM)
+ return err;
+ }
+ goto sysv_retry_create;
+ }
+
+ unsigned short val_array[2] = {1, 1};
+ if (semctl(semid, 2, SETALL, val_array))
+ return errno;
+ } else {
+ semid = semget(env->me_sysv_ipc.key, 2, 0);
+ if (semid == -1)
+ return errno;
- "-"
+ /* check read & write access */
+ struct semid_ds data[2];
+ if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data))
+ return errno;
+ }
- #if defined(__amd64__)
- "AMD64"
- #elif defined(__ia32__)
- "IA32"
- #elif defined(__e2k__) || defined(__elbrus__)
- "Elbrus"
- #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA)
- "Alpha"
- #elif defined(__aarch64__) || defined(_M_ARM64)
- "ARM64"
- #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \
- || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \
- || defined(_M_ARMT) || defined(__arm)
- "ARM"
- #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64))
- "MIPS64"
- #elif defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__)
- "MIPS"
- #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64)
- "PARISC64"
- #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa)
- "PARISC"
- #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \
- || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__)
- "Itanium"
- #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \
- || defined(__powerpc64) || defined(_ARCH_PPC64)
- "PowerPC64"
- #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \
- || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__)
- "PowerPC"
- #elif defined(__sparc64__) || defined(__sparc64)
- "SPARC64"
- #elif defined(__sparc__) || defined(__sparc)
- "SPARC"
- #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch)
- "S390"
- #else
- "UnknownARCH"
- #endif
-#endif /* MDBX_BUILD_TARGET */
+ env->me_sysv_ipc.semid = semid;
+ return MDBX_SUCCESS;
-#ifdef MDBX_BUILD_TYPE
-# if defined(_MSC_VER)
-# pragma message("Configuration-depended MDBX_BUILD_TYPE: " MDBX_BUILD_TYPE)
-# endif
- "-" MDBX_BUILD_TYPE
-#endif /* MDBX_BUILD_TYPE */
- ,
- "MDBX_DEBUG=" MDBX_STRINGIFY(MDBX_DEBUG)
- " MDBX_WORDBITS=" MDBX_STRINGIFY(MDBX_WORDBITS)
- " BYTE_ORDER="
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- "LITTLE_ENDIAN"
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- "BIG_ENDIAN"
+#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
+ (void)inprocess_neighbor;
+ if (global_uniqueness_flag != MDBX_RESULT_TRUE)
+ return MDBX_SUCCESS;
+#error "FIXME: Not implemented"
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+
+ /* don't initialize semaphores twice */
+ (void)inprocess_neighbor;
+ if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
+ if (sem_init(&env->me_lck_mmap.lck->mti_rlock, true, 1))
+ return errno;
+ if (sem_init(&env->me_lck_mmap.lck->mti_wlock, true, 1))
+ return errno;
+ }
+ return MDBX_SUCCESS;
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+ if (inprocess_neighbor)
+ return MDBX_SUCCESS /* don't need any initialization for mutexes
+ if LCK already opened/used inside current process */
+ ;
+
+ /* FIXME: Unfortunately, there is no other reliable way but to long testing
+ * on each platform. On the other hand, behavior like FreeBSD is incorrect
+ * and we can expect it to be rare. Moreover, even on FreeBSD without
+ * additional in-process initialization, the probability of an problem
+ * occurring is vanishingly small, and the symptom is a return of EINVAL
+ * while locking a mutex. In other words, in the worst case, the problem
+ * results in an EINVAL error at the start of the transaction, but NOT data
+ * loss, nor database corruption, nor other fatal troubles. Thus, the code
+ * below I am inclined to think the workaround for erroneous platforms (like
+ * FreeBSD), rather than a defect of libmdbx. */
+#if defined(__FreeBSD__)
+ /* seems that shared mutexes on FreeBSD required in-process initialization */
+ (void)global_uniqueness_flag;
#else
- #error "FIXME: Unsupported byte order"
-#endif /* __BYTE_ORDER__ */
- " MDBX_ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG
- " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG
- " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG
- " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG
- " MDBX_TRUST_RTC=" MDBX_TRUST_RTC_CONFIG
- " MDBX_ENABLE_REFUND=" MDBX_STRINGIFY(MDBX_ENABLE_REFUND)
- " MDBX_ENABLE_MADVISE=" MDBX_STRINGIFY(MDBX_ENABLE_MADVISE)
-#if MDBX_DISABLE_PAGECHECKS
- " MDBX_DISABLE_PAGECHECKS=YES"
-#endif /* MDBX_DISABLE_PAGECHECKS */
-#ifdef __SANITIZE_ADDRESS__
- " SANITIZE_ADDRESS=YES"
-#endif /* __SANITIZE_ADDRESS__ */
-#ifdef MDBX_USE_VALGRIND
- " MDBX_USE_VALGRIND=YES"
-#endif /* MDBX_USE_VALGRIND */
-#if MDBX_FORCE_ASSERTIONS
- " MDBX_FORCE_ASSERTIONS=YES"
-#endif /* MDBX_FORCE_ASSERTIONS */
-#ifdef _GNU_SOURCE
- " _GNU_SOURCE=YES"
+ /* shared mutexes on many other platforms (including Darwin and Linux's
+ * futexes) doesn't need any addition in-process initialization */
+ if (global_uniqueness_flag != MDBX_RESULT_TRUE)
+ return MDBX_SUCCESS;
+#endif
+
+ pthread_mutexattr_t ma;
+ int rc = pthread_mutexattr_init(&ma);
+ if (rc)
+ return rc;
+
+ rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
+ if (rc)
+ goto bailout;
+
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust)
+ rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
+#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \
+ defined(pthread_mutexattr_setrobust_np)
+ rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
+#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
+ rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#else
- " _GNU_SOURCE=NO"
-#endif /* _GNU_SOURCE */
-#ifdef __APPLE__
- " MDBX_OSX_SPEED_INSTEADOF_DURABILITY=" MDBX_STRINGIFY(MDBX_OSX_SPEED_INSTEADOF_DURABILITY)
-#endif /* MacOS */
-#if defined(_WIN32) || defined(_WIN64)
- " MDBX_WITHOUT_MSVC_CRT=" MDBX_STRINGIFY(MDBX_WITHOUT_MSVC_CRT)
- " MDBX_BUILD_SHARED_LIBRARY=" MDBX_STRINGIFY(MDBX_BUILD_SHARED_LIBRARY)
-#if !MDBX_BUILD_SHARED_LIBRARY
- " MDBX_MANUAL_MODULE_HANDLER=" MDBX_STRINGIFY(MDBX_MANUAL_MODULE_HANDLER)
+ rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#endif
- " WINVER=" MDBX_STRINGIFY(WINVER)
-#else /* Windows */
- " MDBX_LOCKING=" MDBX_LOCKING_CONFIG
- " MDBX_USE_OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG
-#endif /* !Windows */
- " MDBX_CACHELINE_SIZE=" MDBX_STRINGIFY(MDBX_CACHELINE_SIZE)
- " MDBX_CPU_WRITEBACK_INCOHERENT=" MDBX_STRINGIFY(MDBX_CPU_WRITEBACK_INCOHERENT)
- " MDBX_MMAP_INCOHERENT_CPU_CACHE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_CPU_CACHE)
- " MDBX_MMAP_INCOHERENT_FILE_WRITE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_FILE_WRITE)
- " MDBX_UNALIGNED_OK=" MDBX_STRINGIFY(MDBX_UNALIGNED_OK)
- " MDBX_PNL_ASCENDING=" MDBX_STRINGIFY(MDBX_PNL_ASCENDING)
- ,
-#ifdef MDBX_BUILD_COMPILER
- MDBX_BUILD_COMPILER
+ if (rc)
+ goto bailout;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
+
+#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \
+ !defined(MDBX_SAFE4QEMU)
+ rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
+ if (rc == ENOTSUP)
+ rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
+ if (rc && rc != ENOTSUP)
+ goto bailout;
+#endif /* PTHREAD_PRIO_INHERIT */
+
+ rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
+ if (rc && rc != ENOTSUP)
+ goto bailout;
+
+ rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_rlock, &ma);
+ if (rc)
+ goto bailout;
+ rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_wlock, &ma);
+
+bailout:
+ pthread_mutexattr_destroy(&ma);
+ return rc;
#else
- #ifdef __INTEL_COMPILER
- "Intel C/C++ " MDBX_STRINGIFY(__INTEL_COMPILER)
- #elif defined(__apple_build_version__)
- "Apple clang " MDBX_STRINGIFY(__apple_build_version__)
- #elif defined(__ibmxl__)
- "IBM clang C " MDBX_STRINGIFY(__ibmxl_version__) "." MDBX_STRINGIFY(__ibmxl_release__)
- "." MDBX_STRINGIFY(__ibmxl_modification__) "." MDBX_STRINGIFY(__ibmxl_ptf_fix_level__)
- #elif defined(__clang__)
- "clang " MDBX_STRINGIFY(__clang_version__)
- #elif defined(__MINGW64__)
- "MINGW-64 " MDBX_STRINGIFY(__MINGW64_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW64_MINOR_VERSION)
- #elif defined(__MINGW32__)
- "MINGW-32 " MDBX_STRINGIFY(__MINGW32_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW32_MINOR_VERSION)
- #elif defined(__IBMC__)
- "IBM C " MDBX_STRINGIFY(__IBMC__)
- #elif defined(__GNUC__)
- "GNU C/C++ "
- #ifdef __VERSION__
- __VERSION__
- #else
- MDBX_STRINGIFY(__GNUC__) "." MDBX_STRINGIFY(__GNUC_MINOR__) "." MDBX_STRINGIFY(__GNUC_PATCHLEVEL__)
- #endif
- #elif defined(_MSC_VER)
- "MSVC " MDBX_STRINGIFY(_MSC_FULL_VER) "-" MDBX_STRINGIFY(_MSC_BUILD)
- #else
- "Unknown compiler"
- #endif
-#endif /* MDBX_BUILD_COMPILER */
- ,
-#ifdef MDBX_BUILD_FLAGS_CONFIG
- MDBX_BUILD_FLAGS_CONFIG
-#endif /* MDBX_BUILD_FLAGS_CONFIG */
-#ifdef MDBX_BUILD_FLAGS
- MDBX_BUILD_FLAGS
-#endif /* MDBX_BUILD_FLAGS */
-#if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS))
- "undefined (please use correct build script)"
-#ifdef _MSC_VER
-#pragma message("warning: Build flags undefined. Please use correct build script")
+#error "FIXME"
+#endif /* MDBX_LOCKING > 0 */
+}
+
+__cold static int mdbx_ipclock_failed(MDBX_env *env, mdbx_ipclock_t *ipc,
+ const int err) {
+ int rc = err;
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV
+ if (err == EOWNERDEAD) {
+ /* We own the mutex. Clean up after dead previous owner. */
+
+ const bool rlocked = ipc == &env->me_lck->mti_rlock;
+ rc = MDBX_SUCCESS;
+ if (!rlocked) {
+ if (unlikely(env->me_txn)) {
+ /* env is hosed if the dead thread was ours */
+ env->me_flags |= MDBX_FATAL_ERROR;
+ env->me_txn = NULL;
+ rc = MDBX_PANIC;
+ }
+ }
+ mdbx_warning("%clock owner died, %s", (rlocked ? 'r' : 'w'),
+ (rc ? "this process' env is hosed" : "recovering"));
+
+ int check_rc = mdbx_cleanup_dead_readers(env, rlocked, NULL);
+ check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
#else
-#warning "Build flags undefined. Please use correct build script"
-#endif // _MSC_VER
+#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent)
+ int mreco_rc = pthread_mutex_consistent(ipc);
+#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np)
+ int mreco_rc = pthread_mutex_consistent_np(ipc);
+#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
+ int mreco_rc = pthread_mutex_consistent_np(ipc);
+#else
+ int mreco_rc = pthread_mutex_consistent(ipc);
#endif
-};
+ check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
-#ifdef __SANITIZE_ADDRESS__
-LIBMDBX_API __attribute__((__weak__)) const char *__asan_default_options() {
- return "symbolize=1:allow_addr2line=1:"
-#if MDBX_DEBUG
- "debug=1:"
- "verbosity=2:"
-#endif /* MDBX_DEBUG */
- "log_threads=1:"
- "report_globals=1:"
- "replace_str=1:replace_intrin=1:"
- "malloc_context_size=9:"
- "detect_leaks=1:"
- "check_printf=1:"
- "detect_deadlocks=1:"
-#ifndef LTO_ENABLED
- "check_initialization_order=1:"
+ if (unlikely(mreco_rc))
+ mdbx_error("lock recovery failed, %s", mdbx_strerror(mreco_rc));
+
+ rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
+ if (MDBX_IS_ERROR(rc))
+ pthread_mutex_unlock(ipc);
+#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
+ return rc;
+ }
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001
+ (void)ipc;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+ (void)ipc;
+#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
+#ifdef _MSC_VER
+#pragma message("warning: TODO")
+#else
+#warning "TODO"
#endif
- "detect_stack_use_after_return=1:"
- "intercept_tls_get_addr=1:"
- "decorate_proc_maps=1:"
- "abort_on_error=1";
+ (void)ipc;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+ mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
+ if (rc != EDEADLK)
+ env->me_flags |= MDBX_FATAL_ERROR;
+ return rc;
}
-#endif /* __SANITIZE_ADDRESS__ */
-/* *INDENT-ON* */
-/* clang-format on */
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void) {
+ /* avoid 32-bit Bionic bug/hang with 32-pit TID */
+ if (sizeof(pthread_mutex_t) < sizeof(pid_t) + sizeof(unsigned)) {
+ pid_t tid = gettid();
+ if (unlikely(tid > 0xffff)) {
+ mdbx_fatal("Raise the ENOSYS(%d) error to avoid hang due "
+ "the 32-bit Bionic/Android bug with tid/thread_id 0x%08x(%i) "
+ "that don’t fit in 16 bits, see "
+ "https://android.googlesource.com/platform/bionic/+/master/"
+ "docs/32-bit-abi.md#is-too-small-for-large-pids",
+ ENOSYS, tid, tid);
+ return ENOSYS;
+ }
+ }
+ return 0;
+}
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+static int mdbx_ipclock_lock(MDBX_env *env, mdbx_ipclock_t *ipc,
+ const bool dont_wait) {
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+ int rc = mdbx_check_tid4bionic();
+ if (likely(rc == 0))
+ rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc);
+ rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+ int rc = MDBX_SUCCESS;
+ if (dont_wait) {
+ if (sem_trywait(ipc)) {
+ rc = errno;
+ if (rc == EAGAIN)
+ rc = MDBX_BUSY;
+ }
+ } else if (sem_wait(ipc))
+ rc = errno;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+ struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock),
+ .sem_op = -1,
+ .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO};
+ int rc;
+ if (semop(env->me_sysv_ipc.semid, &op, 1)) {
+ rc = errno;
+ if (dont_wait && rc == EAGAIN)
+ rc = MDBX_BUSY;
+ } else {
+ rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS;
+ *ipc = env->me_pid;
+ }
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+ if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY))
+ rc = mdbx_ipclock_failed(env, ipc, rc);
+ return rc;
+}
+
+static int mdbx_ipclock_unlock(MDBX_env *env, mdbx_ipclock_t *ipc) {
+#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+ int rc = pthread_mutex_unlock(ipc);
+ (void)env;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+ int rc = sem_post(ipc) ? errno : MDBX_SUCCESS;
+ (void)env;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+ if (unlikely(*ipc != (pid_t)env->me_pid))
+ return EPERM;
+ *ipc = 0;
+ struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock),
+ .sem_op = 1,
+ .sem_flg = SEM_UNDO};
+ int rc = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+ return rc;
+}
+
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
+ mdbx_trace("%s", ">>");
+ mdbx_jitter4testing(true);
+ int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_rlock, false);
+ mdbx_trace("<< rc %d", rc);
+ return rc;
+}
+
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
+ mdbx_trace("%s", ">>");
+ int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_rlock);
+ mdbx_trace("<< rc %d", rc);
+ if (unlikely(rc != MDBX_SUCCESS))
+ mdbx_panic("%s() failed: err %d\n", __func__, rc);
+ mdbx_jitter4testing(true);
+}
+
+int mdbx_txn_lock(MDBX_env *env, bool dont_wait) {
+ mdbx_trace("%swait %s", dont_wait ? "dont-" : "", ">>");
+ mdbx_jitter4testing(true);
+ int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_wlock, dont_wait);
+ mdbx_trace("<< rc %d", rc);
+ return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
+}
+
+void mdbx_txn_unlock(MDBX_env *env) {
+ mdbx_trace("%s", ">>");
+ int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_wlock);
+ mdbx_trace("<< rc %d", rc);
+ if (unlikely(rc != MDBX_SUCCESS))
+ mdbx_panic("%s() failed: err %d\n", __func__, rc);
+ mdbx_jitter4testing(true);
+}
+
+#else
+#ifdef _MSC_VER
+#pragma warning(disable : 4206) /* nonstandard extension used: translation \
+ unit is empty */
+#endif /* _MSC_VER (warnings) */
+#endif /* !Windows LCK-implementation */
diff --git a/3rdparty/libmdbx/mdbx.c++ b/3rdparty/libmdbx/mdbx.c++
new file mode 100644
index 0000000000000000000000000000000000000000..b890b48b0c5654447382248b69c3c5d62f9239f0
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx.c++
@@ -0,0 +1,5202 @@
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define xMDBX_ALLOY 1
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h++"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+//
+// Copyright (c) 2020-2022, Leonid Yuriev <leo@yuriev.ru>.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Non-inline part of the libmdbx C++ API
+//
+
+#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+
+
+#include <array>
+#include <atomic>
+#include <cctype> // for isxdigit(), etc
+#include <system_error>
+
+namespace {
+
+#if 0 /* Unused for now */
+
+class trouble_location {
+
+#ifndef TROUBLE_PROVIDE_LINENO
+#define TROUBLE_PROVIDE_LINENO 1
+#endif
+
+#ifndef TROUBLE_PROVIDE_CONDITION
+#define TROUBLE_PROVIDE_CONDITION 1
+#endif
+
+#ifndef TROUBLE_PROVIDE_FUNCTION
+#define TROUBLE_PROVIDE_FUNCTION 1
+#endif
+
+#ifndef TROUBLE_PROVIDE_FILENAME
+#define TROUBLE_PROVIDE_FILENAME 1
+#endif
+
+#if TROUBLE_PROVIDE_LINENO
+ const unsigned line_;
+#endif
+#if TROUBLE_PROVIDE_CONDITION
+ const char *const condition_;
+#endif
+#if TROUBLE_PROVIDE_FUNCTION
+ const char *const function_;
+#endif
+#if TROUBLE_PROVIDE_FILENAME
+ const char *const filename_;
+#endif
+
+public:
+ MDBX_CXX11_CONSTEXPR trouble_location(unsigned line, const char *condition,
+ const char *function, const char *filename)
+ :
+#if TROUBLE_PROVIDE_LINENO
+ line_(line)
+#endif
+#if TROUBLE_PROVIDE_CONDITION
+ ,
+ condition_(condition)
+#endif
+#if TROUBLE_PROVIDE_FUNCTION
+ ,
+ function_(function)
+#endif
+#if TROUBLE_PROVIDE_FILENAME
+ ,
+ filename_(filename)
+#endif
+ {
+#if !TROUBLE_PROVIDE_LINENO
+ (void)line;
+#endif
+#if !TROUBLE_PROVIDE_CONDITION
+ (void)condition;
+#endif
+#if !TROUBLE_PROVIDE_FUNCTION
+ (void)function;
+#endif
+#if !TROUBLE_PROVIDE_FILENAME
+ (void)filename;
+#endif
+ }
+
+ trouble_location(const trouble_location &&) = delete;
+
+ unsigned line() const {
+#if TROUBLE_PROVIDE_LINENO
+ return line_;
+#else
+ return 0;
+#endif
+ }
+
+ const char *condition() const {
+#if TROUBLE_PROVIDE_CONDITION
+ return condition_;
+#else
+ return "";
+#endif
+ }
+
+ const char *function() const {
+#if TROUBLE_PROVIDE_FUNCTION
+ return function_;
+#else
+ return "";
+#endif
+ }
+
+ const char *filename() const {
+#if TROUBLE_PROVIDE_FILENAME
+ return filename_;
+#else
+ return "";
+#endif
+ }
+};
+
+//------------------------------------------------------------------------------
+
+__cold std::string format_va(const char *fmt, va_list ap) {
+ va_list ones;
+ va_copy(ones, ap);
+#ifdef _MSC_VER
+ int needed = _vscprintf(fmt, ap);
+#else
+ int needed = vsnprintf(nullptr, 0, fmt, ap);
+#endif
+ assert(needed >= 0);
+ std::string result;
+ result.reserve(size_t(needed + 1));
+ result.resize(size_t(needed), '\0');
+ assert(int(result.capacity()) > needed);
+ int actual = vsnprintf(const_cast<char *>(result.data()), result.capacity(),
+ fmt, ones);
+ assert(actual == needed);
+ (void)actual;
+ va_end(ones);
+ return result;
+}
+
+__cold std::string format(const char *fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ std::string result = format_va(fmt, ap);
+ va_end(ap);
+ return result;
+}
+
+class bug : public std::runtime_error {
+ const trouble_location &location_;
+
+public:
+ bug(const trouble_location &) noexcept;
+ /* temporary workaround for "private field 'FOO' is not used" from CLANG
+ * and for "function 'BAR' was declared but never referenced" from LCC. */
+#ifndef __LCC__
+ const trouble_location &location() const noexcept { return location_; }
+#endif
+ virtual ~bug() noexcept;
+};
+
+__cold bug::bug(const trouble_location &location) noexcept
+ : std::runtime_error(format("mdbx.bug: %s.%s at %s:%u", location.function(),
+ location.condition(), location.filename(),
+ location.line())),
+ location_(location) {}
+
+__cold bug::~bug() noexcept {}
+
+[[noreturn]] __cold void raise_bug(const trouble_location &what_and_where) {
+ throw bug(what_and_where);
+}
+
+#define RAISE_BUG(line, condition, function, file) \
+ do { \
+ static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \
+ function, file); \
+ raise_bug(bug); \
+ } while (0)
+
+#define ENSURE(condition) \
+ do \
+ if (MDBX_UNLIKELY(!(condition))) \
+ MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \
+ while (0)
+
+#define NOT_IMPLEMENTED() \
+ RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__);
+
+#endif /* Unused*/
+
+//------------------------------------------------------------------------------
+
+template <typename PATH> struct path_to_pchar {
+ const std::string str;
+ path_to_pchar(const PATH &path) : str(path.generic_string()) {}
+ operator const char *() const { return str.c_str(); }
+};
+
+template <typename PATH>
+MDBX_MAYBE_UNUSED PATH pchar_to_path(const char *c_str) {
+ return PATH(c_str);
+}
+
+template <> struct path_to_pchar<std::string> {
+ const char *const ptr;
+ path_to_pchar(const std::string &path) : ptr(path.c_str()) {}
+ operator const char *() const { return ptr; }
+};
+
+#if defined(_WIN32) || defined(_WIN64)
+
+#ifndef WC_ERR_INVALID_CHARS
+static const DWORD WC_ERR_INVALID_CHARS =
+ (6 /* Windows Vista */ <= /* MajorVersion */ LOBYTE(LOWORD(GetVersion())))
+ ? 0x00000080
+ : 0;
+#endif /* WC_ERR_INVALID_CHARS */
+
+template <> struct path_to_pchar<std::wstring> {
+ std::string str;
+ path_to_pchar(const std::wstring &path) {
+ if (!path.empty()) {
+ const int chars =
+ WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(),
+ int(path.size()), nullptr, 0, nullptr, nullptr);
+ if (chars == 0)
+ mdbx::error::throw_exception(GetLastError());
+ str.append(chars, '\0');
+ WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(),
+ int(path.size()), const_cast<char *>(str.data()),
+ chars, nullptr, nullptr);
+ }
+ }
+ operator const char *() const { return str.c_str(); }
+};
+
+template <>
+MDBX_MAYBE_UNUSED std::wstring pchar_to_path<std::wstring>(const char *c_str) {
+ std::wstring wstr;
+ if (c_str && *c_str) {
+ const int chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str,
+ int(strlen(c_str)), nullptr, 0);
+ if (chars == 0)
+ mdbx::error::throw_exception(GetLastError());
+ wstr.append(chars, '\0');
+ MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str,
+ int(strlen(c_str)), const_cast<wchar_t *>(wstr.data()),
+ chars);
+ }
+ return wstr;
+}
+
+#endif /* Windows */
+
+} // namespace
+
+//------------------------------------------------------------------------------
+
+namespace mdbx {
+
+[[noreturn]] __cold void throw_max_length_exceeded() {
+ throw std::length_error(
+ "mdbx:: Exceeded the maximal length of data/slice/buffer.");
+}
+
+[[noreturn]] __cold void throw_too_small_target_buffer() {
+ throw std::length_error("mdbx:: The target buffer is too small.");
+}
+
+[[noreturn]] __cold void throw_out_range() {
+ throw std::out_of_range("mdbx:: Slice or buffer method was called with "
+ "an argument that exceeds the length.");
+}
+
+[[noreturn]] __cold void throw_allocators_mismatch() {
+ throw std::logic_error(
+ "mdbx:: An allocators mismatch, so an object could not be transferred "
+ "into an incompatible memory allocation scheme.");
+}
+
+__cold exception::exception(const ::mdbx::error &error) noexcept
+ : base(error.what()), error_(error) {}
+
+__cold exception::~exception() noexcept {}
+
+static std::atomic_int fatal_countdown;
+
+__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) {
+ ++fatal_countdown;
+}
+
+__cold fatal::~fatal() noexcept {
+ if (--fatal_countdown == 0)
+ std::terminate();
+}
+
+#define DEFINE_EXCEPTION(NAME) \
+ __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \
+ __cold NAME::~NAME() noexcept {}
+
+DEFINE_EXCEPTION(bad_map_id)
+DEFINE_EXCEPTION(bad_transaction)
+DEFINE_EXCEPTION(bad_value_size)
+DEFINE_EXCEPTION(db_corrupted)
+DEFINE_EXCEPTION(db_full)
+DEFINE_EXCEPTION(db_invalid)
+DEFINE_EXCEPTION(db_too_large)
+DEFINE_EXCEPTION(db_unable_extend)
+DEFINE_EXCEPTION(db_version_mismatch)
+DEFINE_EXCEPTION(db_wanna_write_for_recovery)
+DEFINE_EXCEPTION(incompatible_operation)
+DEFINE_EXCEPTION(internal_page_full)
+DEFINE_EXCEPTION(internal_problem)
+DEFINE_EXCEPTION(key_exists)
+DEFINE_EXCEPTION(key_mismatch)
+DEFINE_EXCEPTION(max_maps_reached)
+DEFINE_EXCEPTION(max_readers_reached)
+DEFINE_EXCEPTION(multivalue)
+DEFINE_EXCEPTION(no_data)
+DEFINE_EXCEPTION(not_found)
+DEFINE_EXCEPTION(operation_not_permitted)
+DEFINE_EXCEPTION(permission_denied_or_not_writeable)
+DEFINE_EXCEPTION(reader_slot_busy)
+DEFINE_EXCEPTION(remote_media)
+DEFINE_EXCEPTION(something_busy)
+DEFINE_EXCEPTION(thread_mismatch)
+DEFINE_EXCEPTION(transaction_full)
+DEFINE_EXCEPTION(transaction_overlapping)
+
+#undef DEFINE_EXCEPTION
+
+__cold const char *error::what() const noexcept {
+ if (is_mdbx_error())
+ return mdbx_liberr2str(code());
+
+ switch (code()) {
+#define ERROR_CASE(CODE) \
+ case CODE: \
+ return MDBX_STRINGIFY(CODE)
+ ERROR_CASE(MDBX_ENODATA);
+ ERROR_CASE(MDBX_EINVAL);
+ ERROR_CASE(MDBX_EACCESS);
+ ERROR_CASE(MDBX_ENOMEM);
+ ERROR_CASE(MDBX_EROFS);
+ ERROR_CASE(MDBX_ENOSYS);
+ ERROR_CASE(MDBX_EIO);
+ ERROR_CASE(MDBX_EPERM);
+ ERROR_CASE(MDBX_EINTR);
+ ERROR_CASE(MDBX_ENOFILE);
+ ERROR_CASE(MDBX_EREMOTE);
+#undef ERROR_CASE
+ default:
+ return "SYSTEM";
+ }
+}
+
+__cold std::string error::message() const {
+ char buf[1024];
+ const char *msg = ::mdbx_strerror_r(code(), buf, sizeof(buf));
+ return std::string(msg ? msg : "unknown");
+}
+
+[[noreturn]] __cold void error::panic(const char *context,
+ const char *func) const noexcept {
+ assert(code() != MDBX_SUCCESS);
+ ::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code());
+ std::terminate();
+}
+
+__cold void error::throw_exception() const {
+ switch (code()) {
+ case MDBX_EINVAL:
+ throw std::invalid_argument("mdbx");
+ case MDBX_ENOMEM:
+ throw std::bad_alloc();
+ case MDBX_SUCCESS:
+ static_assert(MDBX_SUCCESS == MDBX_RESULT_FALSE, "WTF?");
+ throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)");
+ case MDBX_RESULT_TRUE:
+ throw std::logic_error("MDBX_RESULT_TRUE");
+#define CASE_EXCEPTION(NAME, CODE) \
+ case CODE: \
+ throw NAME(code())
+ CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI);
+ CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN);
+ CASE_EXCEPTION(bad_value_size, MDBX_BAD_VALSIZE);
+ CASE_EXCEPTION(db_corrupted, MDBX_CORRUPTED);
+ CASE_EXCEPTION(db_corrupted, MDBX_CURSOR_FULL); /* branch-pages loop */
+ CASE_EXCEPTION(db_corrupted, MDBX_PAGE_NOTFOUND);
+ CASE_EXCEPTION(db_full, MDBX_MAP_FULL);
+ CASE_EXCEPTION(db_invalid, MDBX_INVALID);
+ CASE_EXCEPTION(db_too_large, MDBX_TOO_LARGE);
+ CASE_EXCEPTION(db_unable_extend, MDBX_UNABLE_EXTEND_MAPSIZE);
+ CASE_EXCEPTION(db_version_mismatch, MDBX_VERSION_MISMATCH);
+ CASE_EXCEPTION(db_wanna_write_for_recovery, MDBX_WANNA_RECOVERY);
+ CASE_EXCEPTION(fatal, MDBX_EBADSIGN);
+ CASE_EXCEPTION(fatal, MDBX_PANIC);
+ CASE_EXCEPTION(incompatible_operation, MDBX_INCOMPATIBLE);
+ CASE_EXCEPTION(internal_page_full, MDBX_PAGE_FULL);
+ CASE_EXCEPTION(internal_problem, MDBX_PROBLEM);
+ CASE_EXCEPTION(key_mismatch, MDBX_EKEYMISMATCH);
+ CASE_EXCEPTION(max_maps_reached, MDBX_DBS_FULL);
+ CASE_EXCEPTION(max_readers_reached, MDBX_READERS_FULL);
+ CASE_EXCEPTION(multivalue, MDBX_EMULTIVAL);
+ CASE_EXCEPTION(no_data, MDBX_ENODATA);
+ CASE_EXCEPTION(not_found, MDBX_NOTFOUND);
+ CASE_EXCEPTION(operation_not_permitted, MDBX_EPERM);
+ CASE_EXCEPTION(permission_denied_or_not_writeable, MDBX_EACCESS);
+ CASE_EXCEPTION(reader_slot_busy, MDBX_BAD_RSLOT);
+ CASE_EXCEPTION(remote_media, MDBX_EREMOTE);
+ CASE_EXCEPTION(something_busy, MDBX_BUSY);
+ CASE_EXCEPTION(thread_mismatch, MDBX_THREAD_MISMATCH);
+ CASE_EXCEPTION(transaction_full, MDBX_TXN_FULL);
+ CASE_EXCEPTION(transaction_overlapping, MDBX_TXN_OVERLAPPING);
+#undef CASE_EXCEPTION
+ default:
+ if (is_mdbx_error())
+ throw exception(*this);
+ throw std::system_error(std::error_code(code(), std::system_category()));
+ }
+}
+
+//------------------------------------------------------------------------------
+
+bool slice::is_printable(bool disable_utf8) const noexcept {
+ enum : byte {
+ LS = 4, // shift for UTF8 sequence length
+ P_ = 1 << LS, // printable ASCII flag
+ N_ = 0, // non-printable ASCII
+ second_range_mask = P_ - 1, // mask for range flag
+ r80_BF = 0, // flag for UTF8 2nd byte range
+ rA0_BF = 1, // flag for UTF8 2nd byte range
+ r80_9F = 2, // flag for UTF8 2nd byte range
+ r90_BF = 3, // flag for UTF8 2nd byte range
+ r80_8F = 4, // flag for UTF8 2nd byte range
+
+ // valid utf-8 byte sequences
+ // http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+ // Code | Bytes | | |
+ // Points | 1st | 2nd | 3rd |4th
+ // --------------------|--------|--------|--------|---
+ C2 = 2 << LS | r80_BF, // U+000080..U+0007FF | C2..DF | 80..BF | |
+ E0 = 3 << LS | rA0_BF, // U+000800..U+000FFF | E0 | A0..BF | 80..BF |
+ E1 = 3 << LS | r80_BF, // U+001000..U+00CFFF | E1..EC | 80..BF | 80..BF |
+ ED = 3 << LS | r80_9F, // U+00D000..U+00D7FF | ED | 80..9F | 80..BF |
+ EE = 3 << LS | r80_BF, // U+00E000..U+00FFFF | EE..EF | 80..BF | 80..BF |
+ F0 = 4 << LS | r90_BF, // U+010000..U+03FFFF | F0 | 90..BF | 80..BF |...
+ F1 = 4 << LS | r80_BF, // U+040000..U+0FFFFF | F1..F3 | 80..BF | 80..BF |...
+ F4 = 4 << LS | r80_BF, // U+100000..U+10FFFF | F4 | 80..8F | 80..BF |...
+ };
+
+ static const byte range_from[] = {0x80, 0xA0, 0x80, 0x90, 0x80};
+ static const byte range_to[] = {0xBF, 0xBF, 0x9F, 0xBF, 0x8F};
+
+ static const byte map[256] = {
+ // 1 2 3 4 5 6 7 8 9 a b c d e f
+ N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 00
+ N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 10
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 20
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 30
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 40
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 50
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 60
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, // 70
+ N_, N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, N_, // 80
+ N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, P_, // 90
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // a0
+ P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // b0
+ P_, P_, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // c0
+ C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // df
+ E0, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, ED, EE, EE, // e0
+ F0, F1, F1, F1, F4, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_ // f0
+ };
+
+ if (MDBX_UNLIKELY(length() < 1))
+ MDBX_CXX20_UNLIKELY return false;
+
+ auto src = byte_ptr();
+ const auto end = src + length();
+ if (MDBX_UNLIKELY(disable_utf8)) {
+ do
+ if (MDBX_UNLIKELY((P_ & map[*src]) == 0))
+ MDBX_CXX20_UNLIKELY return false;
+ while (++src < end);
+ return true;
+ }
+
+ do {
+ const auto bits = map[*src];
+ const auto second_from = range_from[bits & second_range_mask];
+ const auto second_to = range_to[bits & second_range_mask];
+ switch (bits >> LS) {
+ default:
+ MDBX_CXX20_UNLIKELY return false;
+ case 1:
+ src += 1;
+ continue;
+ case 2:
+ if (MDBX_UNLIKELY(src + 1 >= end))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
+ MDBX_CXX20_UNLIKELY return false;
+ src += 2;
+ continue;
+ case 3:
+ if (MDBX_UNLIKELY(src + 3 >= end))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[2] < 0x80 || src[2] > 0xBF))
+ MDBX_CXX20_UNLIKELY return false;
+ src += 3;
+ continue;
+ case 4:
+ if (MDBX_UNLIKELY(src + 4 >= end))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[2] < 0x80 || src[2] > 0xBF))
+ MDBX_CXX20_UNLIKELY return false;
+ if (MDBX_UNLIKELY(src[3] < 0x80 || src[3] > 0xBF))
+ MDBX_CXX20_UNLIKELY return false;
+ src += 4;
+ continue;
+ }
+ } while (src < end);
+
+ return true;
+}
+
+//------------------------------------------------------------------------------
+
+char *to_hex::write_bytes(char *__restrict const dest, size_t dest_size) const {
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ const char alphabase = (uppercase ? 'A' : 'a') - 10;
+ auto line = ptr;
+ for (const auto end = source.end_byte_ptr(); src != end; ++src) {
+ if (wrap_width && size_t(ptr - line) >= wrap_width) {
+ *ptr = '\n';
+ line = ++ptr;
+ }
+ const int8_t hi = *src >> 4;
+ const int8_t lo = *src & 15;
+ ptr[0] = char(alphabase + hi + (((hi - 10) >> 7) & -7));
+ ptr[1] = char(alphabase + lo + (((lo - 10) >> 7) & -7));
+ ptr += 2;
+ assert(ptr <= dest + dest_size);
+ }
+ return ptr;
+}
+
+::std::ostream &to_hex::output(::std::ostream &out) const {
+ if (MDBX_LIKELY(!is_empty()))
+ MDBX_CXX20_LIKELY {
+ ::std::ostream::sentry sentry(out);
+ auto src = source.byte_ptr();
+ const char alphabase = (uppercase ? 'A' : 'a') - 10;
+ unsigned width = 0;
+ for (const auto end = source.end_byte_ptr(); src != end; ++src) {
+ if (wrap_width && width >= wrap_width) {
+ out << ::std::endl;
+ width = 0;
+ }
+ const int8_t hi = *src >> 4;
+ const int8_t lo = *src & 15;
+ out.put(char(alphabase + hi + (((hi - 10) >> 7) & -7)));
+ out.put(char(alphabase + lo + (((lo - 10) >> 7) & -7)));
+ width += 2;
+ }
+ }
+ return out;
+}
+
+char *from_hex::write_bytes(char *__restrict const dest,
+ size_t dest_size) const {
+ if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces))
+ MDBX_CXX20_UNLIKELY throw std::domain_error(
+ "mdbx::from_hex:: odd length of hexadecimal string");
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(*src <= ' ') &&
+ MDBX_LIKELY(ignore_spaces && isspace(*src))) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1])))
+ MDBX_CXX20_UNLIKELY throw std::domain_error(
+ "mdbx::from_hex:: invalid hexadecimal string");
+
+ int8_t hi = src[0];
+ hi = (hi | 0x20) - 'a';
+ hi += 10 + ((hi >> 7) & 7);
+
+ int8_t lo = src[1];
+ lo = (lo | 0x20) - 'a';
+ lo += 10 + ((lo >> 7) & 7);
+
+ *ptr++ = hi << 4 | lo;
+ src += 2;
+ left -= 2;
+ assert(ptr <= dest + dest_size);
+ }
+ return ptr;
+}
+
+bool from_hex::is_erroneous() const noexcept {
+ if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces))
+ MDBX_CXX20_UNLIKELY return true;
+
+ bool got = false;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(*src <= ' ') &&
+ MDBX_LIKELY(ignore_spaces && isspace(*src))) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1])))
+ MDBX_CXX20_UNLIKELY return true;
+
+ got = true;
+ src += 2;
+ left -= 2;
+ }
+ return !got;
+}
+
+//------------------------------------------------------------------------------
+
+enum : signed char {
+ OO /* ASCII NUL */ = -8,
+ EQ /* BASE64 '=' pad */ = -4,
+ SP /* SPACE */ = -2,
+ IL /* invalid */ = -1
+};
+
+static const byte b58_alphabet[58] = {
+ '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
+ 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
+ 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
+
+#ifndef bswap64
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+static inline uint64_t bswap64(uint64_t v) noexcept {
+#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \
+ __has_builtin(__builtin_bswap64)
+ return __builtin_bswap64(v);
+#elif defined(_MSC_VER) && !defined(__clang__)
+ return _byteswap_uint64(v);
+#elif defined(__bswap_64)
+ return __bswap_64(v);
+#elif defined(bswap_64)
+ return bswap_64(v);
+#else
+ return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
+ ((v << 24) & UINT64_C(0x0000ff0000000000)) |
+ ((v << 8) & UINT64_C(0x000000ff00000000)) |
+ ((v >> 8) & UINT64_C(0x00000000ff000000)) |
+ ((v >> 24) & UINT64_C(0x0000000000ff0000)) |
+ ((v >> 40) & UINT64_C(0x000000000000ff00));
+#endif
+}
+#endif /* __BYTE_ORDER__ */
+#endif /* ifndef bswap64 */
+
+static inline char b58_8to11(uint64_t &v) noexcept {
+ const unsigned i = unsigned(v % 58);
+ v /= 58;
+ return b58_alphabet[i];
+}
+
+char *to_base58::write_bytes(char *__restrict const dest,
+ size_t dest_size) const {
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ size_t left = source.length();
+ auto line = ptr;
+ while (MDBX_LIKELY(left > 7)) {
+ uint64_t v;
+ std::memcpy(&v, src, 8);
+ src += 8;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ v = bswap64(v);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ ptr[10] = b58_8to11(v);
+ ptr[9] = b58_8to11(v);
+ ptr[8] = b58_8to11(v);
+ ptr[7] = b58_8to11(v);
+ ptr[6] = b58_8to11(v);
+ ptr[5] = b58_8to11(v);
+ ptr[4] = b58_8to11(v);
+ ptr[3] = b58_8to11(v);
+ ptr[2] = b58_8to11(v);
+ ptr[1] = b58_8to11(v);
+ ptr[0] = b58_8to11(v);
+ assert(v == 0);
+ ptr += 11;
+ left -= 8;
+ if (wrap_width && size_t(ptr - line) >= wrap_width && left) {
+ *ptr = '\n';
+ line = ++ptr;
+ }
+ assert(ptr <= dest + dest_size);
+ }
+
+ if (left) {
+ uint64_t v = 0;
+ unsigned parrots = 31;
+ do {
+ v = (v << 8) + *src++;
+ parrots += 43;
+ } while (--left);
+
+ auto tail = ptr += parrots >> 5;
+ assert(ptr <= dest + dest_size);
+ do {
+ *--tail = b58_8to11(v);
+ parrots -= 32;
+ } while (parrots > 31);
+ assert(v == 0);
+ }
+
+ return ptr;
+}
+
+::std::ostream &to_base58::output(::std::ostream &out) const {
+ if (MDBX_LIKELY(!is_empty()))
+ MDBX_CXX20_LIKELY {
+ ::std::ostream::sentry sentry(out);
+ auto src = source.byte_ptr();
+ size_t left = source.length();
+ unsigned width = 0;
+ std::array<char, 11> buf;
+
+ while (MDBX_LIKELY(left > 7)) {
+ uint64_t v;
+ std::memcpy(&v, src, 8);
+ src += 8;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ v = bswap64(v);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ buf[10] = b58_8to11(v);
+ buf[9] = b58_8to11(v);
+ buf[8] = b58_8to11(v);
+ buf[7] = b58_8to11(v);
+ buf[6] = b58_8to11(v);
+ buf[5] = b58_8to11(v);
+ buf[4] = b58_8to11(v);
+ buf[3] = b58_8to11(v);
+ buf[2] = b58_8to11(v);
+ buf[1] = b58_8to11(v);
+ buf[0] = b58_8to11(v);
+ assert(v == 0);
+ out.write(&buf.front(), 11);
+ left -= 8;
+ if (wrap_width && (width += 11) >= wrap_width && left) {
+ out << ::std::endl;
+ width = 0;
+ }
+ }
+
+ if (left) {
+ uint64_t v = 0;
+ unsigned parrots = 31;
+ do {
+ v = (v << 8) + *src++;
+ parrots += 43;
+ } while (--left);
+
+ auto ptr = buf.end();
+ do {
+ *--ptr = b58_8to11(v);
+ parrots -= 32;
+ } while (parrots > 31);
+ assert(v == 0);
+ out.write(&*ptr, buf.end() - ptr);
+ }
+ }
+ return out;
+}
+
+const signed char b58_map[256] = {
+ // 1 2 3 4 5 6 7 8 9 a b c d e f
+ OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10
+ SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 20
+ IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, IL, IL, IL, IL, IL, IL, // 30
+ IL, 9, 10, 11, 12, 13, 14, 15, 16, IL, 17, 18, 19, 20, 21, IL, // 40
+ 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, IL, IL, IL, IL, IL, // 50
+ IL, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, IL, 44, 45, 46, // 60
+ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, IL, IL, IL, IL, IL, // 70
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
+};
+
+static inline signed char b58_11to8(uint64_t &v, const byte c) noexcept {
+ const signed char m = b58_map[c];
+ v = v * 58 + m;
+ return m;
+}
+
+char *from_base58::write_bytes(char *__restrict const dest,
+ size_t dest_size) const {
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(isspace(*src)) && ignore_spaces) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_LIKELY(left > 10)) {
+ uint64_t v = 0;
+ if (MDBX_UNLIKELY((b58_11to8(v, src[0]) | b58_11to8(v, src[1]) |
+ b58_11to8(v, src[2]) | b58_11to8(v, src[3]) |
+ b58_11to8(v, src[4]) | b58_11to8(v, src[5]) |
+ b58_11to8(v, src[6]) | b58_11to8(v, src[7]) |
+ b58_11to8(v, src[8]) | b58_11to8(v, src[9]) |
+ b58_11to8(v, src[10])) < 0))
+ MDBX_CXX20_UNLIKELY goto bailout;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ v = bswap64(v);
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ std::memcpy(ptr, &v, 8);
+ ptr += 8;
+ src += 11;
+ left -= 11;
+ assert(ptr <= dest + dest_size);
+ continue;
+ }
+
+ constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8;
+ if (MDBX_UNLIKELY(invalid_length_mask & (1 << left)))
+ MDBX_CXX20_UNLIKELY goto bailout;
+
+ uint64_t v = 1;
+ unsigned parrots = 0;
+ do {
+ if (MDBX_UNLIKELY(b58_11to8(v, *src++) < 0))
+ MDBX_CXX20_UNLIKELY goto bailout;
+ parrots += 32;
+ } while (--left);
+
+ auto tail = ptr += parrots / 43;
+ assert(ptr <= dest + dest_size);
+ do {
+ *--tail = byte(v);
+ v >>= 8;
+ } while (v > 255);
+ break;
+ }
+ return ptr;
+
+bailout:
+ throw std::domain_error("mdbx::from_base58:: invalid base58 string");
+}
+
+bool from_base58::is_erroneous() const noexcept {
+ bool got = false;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(*src <= ' ') &&
+ MDBX_LIKELY(ignore_spaces && isspace(*src))) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_LIKELY(left > 10)) {
+ if (MDBX_UNLIKELY((b58_map[src[0]] | b58_map[src[1]] | b58_map[src[2]] |
+ b58_map[src[3]] | b58_map[src[4]] | b58_map[src[5]] |
+ b58_map[src[6]] | b58_map[src[7]] | b58_map[src[8]] |
+ b58_map[src[9]] | b58_map[src[10]]) < 0))
+ MDBX_CXX20_UNLIKELY return true;
+ src += 11;
+ left -= 11;
+ got = true;
+ continue;
+ }
+
+ constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8;
+ if (invalid_length_mask & (1 << left))
+ return false;
+
+ do
+ if (MDBX_UNLIKELY(b58_map[*src++] < 0))
+ MDBX_CXX20_UNLIKELY return true;
+ while (--left);
+ got = true;
+ break;
+ }
+ return !got;
+}
+
+//------------------------------------------------------------------------------
+
+static inline void b64_3to4(const byte x, const byte y, const byte z,
+ char *__restrict dest) noexcept {
+ static const byte alphabet[64] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
+ dest[0] = alphabet[(x & 0xfc) >> 2];
+ dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)];
+ dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)];
+ dest[3] = alphabet[z & 0x3f];
+}
+
+char *to_base64::write_bytes(char *__restrict const dest,
+ size_t dest_size) const {
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ size_t left = source.length();
+ auto line = ptr;
+ while (true) {
+ switch (left) {
+ default:
+ MDBX_CXX20_LIKELY left -= 3;
+ b64_3to4(src[0], src[1], src[2], ptr);
+ ptr += 4;
+ src += 3;
+ if (wrap_width && size_t(ptr - line) >= wrap_width && left) {
+ *ptr = '\n';
+ line = ++ptr;
+ }
+ assert(ptr <= dest + dest_size);
+ continue;
+ case 2:
+ b64_3to4(src[0], src[1], 0, ptr);
+ ptr[3] = '=';
+ assert(ptr + 4 <= dest + dest_size);
+ return ptr + 4;
+ case 1:
+ b64_3to4(src[0], 0, 0, ptr);
+ ptr[2] = ptr[3] = '=';
+ assert(ptr + 4 <= dest + dest_size);
+ return ptr + 4;
+ case 0:
+ return ptr;
+ }
+ }
+}
+
+::std::ostream &to_base64::output(::std::ostream &out) const {
+ if (MDBX_LIKELY(!is_empty()))
+ MDBX_CXX20_LIKELY {
+ ::std::ostream::sentry sentry(out);
+ auto src = source.byte_ptr();
+ size_t left = source.length();
+ unsigned width = 0;
+ std::array<char, 4> buf;
+
+ while (true) {
+ switch (left) {
+ default:
+ MDBX_CXX20_LIKELY left -= 3;
+ b64_3to4(src[0], src[1], src[2], &buf.front());
+ src += 3;
+ out.write(&buf.front(), 4);
+ if (wrap_width && (width += 4) >= wrap_width && left) {
+ out << ::std::endl;
+ width = 0;
+ }
+ continue;
+ case 2:
+ b64_3to4(src[0], src[1], 0, &buf.front());
+ buf[3] = '=';
+ return out.write(&buf.front(), 4);
+ case 1:
+ b64_3to4(src[0], 0, 0, &buf.front());
+ buf[2] = buf[3] = '=';
+ return out.write(&buf.front(), 4);
+ case 0:
+ return out;
+ }
+ }
+ }
+ return out;
+}
+
+static const signed char b64_map[256] = {
+ // 1 2 3 4 5 6 7 8 9 a b c d e f
+ OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10
+ SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, 62, IL, IL, IL, 63, // 20
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, IL, IL, IL, EQ, IL, IL, // 30
+ IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 40
+ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, IL, IL, IL, IL, IL, // 50
+ IL, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 60
+ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, IL, IL, IL, IL, IL, // 70
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0
+ IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
+};
+
+static inline signed char b64_4to3(signed char a, signed char b, signed char c,
+ signed char d,
+ char *__restrict dest) noexcept {
+ dest[0] = byte((a << 2) + ((b & 0x30) >> 4));
+ dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2));
+ dest[2] = byte(((c & 0x3) << 6) + d);
+ return a | b | c | d;
+}
+
+char *from_base64::write_bytes(char *__restrict const dest,
+ size_t dest_size) const {
+ if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces))
+ MDBX_CXX20_UNLIKELY throw std::domain_error(
+ "mdbx::from_base64:: odd length of base64 string");
+ if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
+ MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
+
+ auto ptr = dest;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(*src <= ' ') &&
+ MDBX_LIKELY(ignore_spaces && isspace(*src))) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_UNLIKELY(left < 3))
+ MDBX_CXX20_UNLIKELY {
+ bailout:
+ throw std::domain_error("mdbx::from_base64:: invalid base64 string");
+ }
+ const signed char a = b64_map[src[0]], b = b64_map[src[1]],
+ c = b64_map[src[2]], d = b64_map[src[3]];
+ if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, ptr) < 0)) {
+ if (left == 4 && (a | b) >= 0 && d == EQ) {
+ if (c >= 0) {
+ assert(ptr + 2 <= dest + dest_size);
+ return ptr + 2;
+ }
+ if (c == d) {
+ assert(ptr + 1 <= dest + dest_size);
+ return ptr + 1;
+ }
+ }
+ MDBX_CXX20_UNLIKELY goto bailout;
+ }
+ src += 4;
+ left -= 4;
+ ptr += 3;
+ assert(ptr <= dest + dest_size);
+ }
+ return ptr;
+}
+
+bool from_base64::is_erroneous() const noexcept {
+ if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces))
+ MDBX_CXX20_UNLIKELY return true;
+
+ bool got = false;
+ auto src = source.byte_ptr();
+ for (auto left = source.length(); left > 0;) {
+ if (MDBX_UNLIKELY(*src <= ' ') &&
+ MDBX_LIKELY(ignore_spaces && isspace(*src))) {
+ ++src;
+ --left;
+ continue;
+ }
+
+ if (MDBX_UNLIKELY(left < 3))
+ MDBX_CXX20_UNLIKELY return false;
+ const signed char a = b64_map[src[0]], b = b64_map[src[1]],
+ c = b64_map[src[2]], d = b64_map[src[3]];
+ if (MDBX_UNLIKELY((a | b | c | d) < 0))
+ MDBX_CXX20_UNLIKELY {
+ if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d))
+ return false;
+ return true;
+ }
+ got = true;
+ src += 4;
+ left -= 4;
+ }
+ return !got;
+}
+
+//------------------------------------------------------------------------------
+
+template class LIBMDBX_API_TYPE buffer<legacy_allocator>;
+
+#if defined(__cpp_lib_memory_resource) && \
+ __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI
+template class LIBMDBX_API_TYPE buffer<polymorphic_allocator>;
+#endif /* __cpp_lib_memory_resource >= 201603L */
+
+//------------------------------------------------------------------------------
+
+static inline MDBX_env_flags_t mode2flags(env::mode mode) {
+ switch (mode) {
+ default:
+ MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::mode is invalid");
+ case env::mode::readonly:
+ return MDBX_RDONLY;
+ case env::mode::write_file_io:
+ return MDBX_ENV_DEFAULTS;
+ case env::mode::write_mapped_io:
+ return MDBX_WRITEMAP;
+ }
+}
+
+__cold MDBX_env_flags_t
+env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
+ MDBX_env_flags_t flags = mode2flags(mode);
+ if (accede)
+ flags |= MDBX_ACCEDE;
+ if (!use_subdirectory)
+ flags |= MDBX_NOSUBDIR;
+ if (options.exclusive)
+ flags |= MDBX_EXCLUSIVE;
+ if (options.orphan_read_transactions)
+ flags |= MDBX_NOTLS;
+ if (options.disable_readahead)
+ flags |= MDBX_NORDAHEAD;
+ if (options.disable_clear_memory)
+ flags |= MDBX_NOMEMINIT;
+
+ if (mode != readonly) {
+ if (options.nested_write_transactions)
+ flags &= ~MDBX_WRITEMAP;
+ if (reclaiming.coalesce)
+ flags |= MDBX_COALESCE;
+ if (reclaiming.lifo)
+ flags |= MDBX_LIFORECLAIM;
+ switch (durability) {
+ default:
+ MDBX_CXX20_UNLIKELY throw std::invalid_argument(
+ "db::durability is invalid");
+ case env::durability::robust_synchronous:
+ break;
+ case env::durability::half_synchronous_weak_last:
+ flags |= MDBX_NOMETASYNC;
+ break;
+ case env::durability::lazy_weak_tail:
+ static_assert(MDBX_MAPASYNC == MDBX_SAFE_NOSYNC, "WTF? Obsolete C API?");
+ flags |= MDBX_SAFE_NOSYNC;
+ break;
+ case env::durability::whole_fragile:
+ flags |= MDBX_UTTERLY_NOSYNC;
+ break;
+ }
+ }
+ return flags;
+}
+
+env::mode
+env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept {
+ if (flags & MDBX_RDONLY)
+ return env::mode::readonly;
+ return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io
+ : env::mode::write_file_io;
+}
+
+env::durability env::operate_parameters::durability_from_flags(
+ MDBX_env_flags_t flags) noexcept {
+ if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC)
+ return env::durability::whole_fragile;
+ if (flags & MDBX_SAFE_NOSYNC)
+ return env::durability::lazy_weak_tail;
+ if (flags & MDBX_NOMETASYNC)
+ return env::durability::half_synchronous_weak_last;
+ return env::durability::robust_synchronous;
+}
+
+env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept
+ : lifo((flags & MDBX_LIFORECLAIM) ? true : false),
+ coalesce((flags & MDBX_COALESCE) ? true : false) {}
+
+env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept
+ : orphan_read_transactions(
+ ((flags & (MDBX_NOTLS | MDBX_EXCLUSIVE)) == MDBX_NOTLS) ? true
+ : false),
+ nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false
+ : true),
+ exclusive((flags & MDBX_EXCLUSIVE) ? true : false),
+ disable_readahead((flags & MDBX_NORDAHEAD) ? true : false),
+ disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {}
+
+bool env::is_pristine() const {
+ return get_stat().ms_mod_txnid == 0 &&
+ get_info().mi_recent_txnid == INITIAL_TXNID;
+}
+
+bool env::is_empty() const { return get_stat().ms_leaf_pages == 0; }
+
+#ifdef MDBX_STD_FILESYSTEM_PATH
+env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify,
+ bool force_dynamic_size) {
+ const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(destination);
+ error::success_or_throw(
+ ::mdbx_env_copy(handle_, utf8,
+ (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
+ (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
+ : MDBX_CP_DEFAULTS)));
+ return *this;
+}
+#endif /* MDBX_STD_FILESYSTEM_PATH */
+
+#if defined(_WIN32) || defined(_WIN64)
+env &env::copy(const ::std::wstring &destination, bool compactify,
+ bool force_dynamic_size) {
+ const path_to_pchar<::std::wstring> utf8(destination);
+ error::success_or_throw(
+ ::mdbx_env_copy(handle_, utf8,
+ (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
+ (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
+ : MDBX_CP_DEFAULTS)));
+ return *this;
+}
+#endif /* Windows */
+
+env &env::copy(const ::std::string &destination, bool compactify,
+ bool force_dynamic_size) {
+ const path_to_pchar<::std::string> utf8(destination);
+ error::success_or_throw(
+ ::mdbx_env_copy(handle_, utf8,
+ (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
+ (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
+ : MDBX_CP_DEFAULTS)));
+ return *this;
+}
+
+env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) {
+ error::success_or_throw(
+ ::mdbx_env_copy2fd(handle_, fd,
+ (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
+ (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
+ : MDBX_CP_DEFAULTS)));
+ return *this;
+}
+
+path env::get_path() const {
+ const char *c_str;
+ error::success_or_throw(::mdbx_env_get_path(handle_, &c_str));
+ return pchar_to_path<path>(c_str);
+}
+
+#ifdef MDBX_STD_FILESYSTEM_PATH
+bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname,
+ const remove_mode mode) {
+ const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
+ return error::boolean_or_throw(
+ ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
+}
+#endif /* MDBX_STD_FILESYSTEM_PATH */
+
+#if defined(_WIN32) || defined(_WIN64)
+bool env::remove(const ::std::wstring &pathname, const remove_mode mode) {
+ const path_to_pchar<::std::wstring> utf8(pathname);
+ return error::boolean_or_throw(
+ ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
+}
+#endif /* Windows */
+
+bool env::remove(const ::std::string &pathname, const remove_mode mode) {
+ const path_to_pchar<::std::string> utf8(pathname);
+ return error::boolean_or_throw(
+ ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
+}
+
+//------------------------------------------------------------------------------
+
+static inline MDBX_env *create_env() {
+ MDBX_env *ptr;
+ error::success_or_throw(::mdbx_env_create(&ptr));
+ assert(ptr != nullptr);
+ return ptr;
+}
+
+env_managed::~env_managed() noexcept {
+ if (MDBX_UNLIKELY(handle_))
+ MDBX_CXX20_UNLIKELY error::success_or_panic(
+ ::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close");
+}
+
+void env_managed::close(bool dont_sync) {
+ const error rc =
+ static_cast<MDBX_error_t>(::mdbx_env_close_ex(handle_, dont_sync));
+ switch (rc.code()) {
+ case MDBX_EBADSIGN:
+ MDBX_CXX20_UNLIKELY handle_ = nullptr;
+ __fallthrough /* fall through */;
+ default:
+ MDBX_CXX20_UNLIKELY rc.throw_exception();
+ case MDBX_SUCCESS:
+ MDBX_CXX20_LIKELY handle_ = nullptr;
+ }
+}
+
+__cold void env_managed::setup(unsigned max_maps, unsigned max_readers) {
+ if (max_readers > 0)
+ error::success_or_throw(::mdbx_env_set_maxreaders(handle_, max_readers));
+ if (max_maps > 0)
+ error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps));
+}
+
+#ifdef MDBX_STD_FILESYSTEM_PATH
+__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
+ const operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+
+__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
+ const env_managed::create_parameters &cp,
+ const env::operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
+ set_geometry(cp.geometry);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
+ cp.file_mode_bits));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+#endif /* MDBX_STD_FILESYSTEM_PATH */
+
+#if defined(_WIN32) || defined(_WIN64)
+__cold env_managed::env_managed(const ::std::wstring &pathname,
+ const operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<::std::wstring> utf8(pathname);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+
+__cold env_managed::env_managed(const ::std::wstring &pathname,
+ const env_managed::create_parameters &cp,
+ const env::operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<::std::wstring> utf8(pathname);
+ set_geometry(cp.geometry);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
+ cp.file_mode_bits));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+#endif /* Windows */
+
+__cold env_managed::env_managed(const ::std::string &pathname,
+ const operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<::std::string> utf8(pathname);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+
+__cold env_managed::env_managed(const ::std::string &pathname,
+ const env_managed::create_parameters &cp,
+ const env::operate_parameters &op, bool accede)
+ : env_managed(create_env()) {
+ setup(op.max_maps, op.max_readers);
+ const path_to_pchar<::std::string> utf8(pathname);
+ set_geometry(cp.geometry);
+ error::success_or_throw(
+ ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
+ cp.file_mode_bits));
+
+ if (op.options.nested_write_transactions &&
+ !get_options().nested_write_transactions)
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
+}
+
+//------------------------------------------------------------------------------
+
+txn_managed txn::start_nested() {
+ MDBX_txn *nested;
+ error::throw_on_nullptr(handle_, MDBX_BAD_TXN);
+ error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_,
+ MDBX_TXN_READWRITE, &nested));
+ assert(nested != nullptr);
+ return txn_managed(nested);
+}
+
+txn_managed::~txn_managed() noexcept {
+ if (MDBX_UNLIKELY(handle_))
+ MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_),
+ "mdbx::~txn", "mdbx_txn_abort");
+}
+
+void txn_managed::abort() {
+ const error err = static_cast<MDBX_error_t>(::mdbx_txn_abort(handle_));
+ if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH))
+ MDBX_CXX20_LIKELY handle_ = nullptr;
+ if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS))
+ MDBX_CXX20_UNLIKELY err.throw_exception();
+}
+
+void txn_managed::commit() {
+ const error err = static_cast<MDBX_error_t>(::mdbx_txn_commit(handle_));
+ if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH))
+ MDBX_CXX20_LIKELY handle_ = nullptr;
+ if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS))
+ MDBX_CXX20_UNLIKELY err.throw_exception();
+}
+
+//------------------------------------------------------------------------------
+
+bool txn::drop_map(const char *name, bool throw_if_absent) {
+ map_handle map;
+ const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi);
+ switch (err) {
+ case MDBX_SUCCESS:
+ drop_map(map);
+ return true;
+ case MDBX_NOTFOUND:
+ case MDBX_BAD_DBI:
+ if (!throw_if_absent)
+ return false;
+ MDBX_CXX17_FALLTHROUGH /* fallthrough */;
+ default:
+ MDBX_CXX20_UNLIKELY error::throw_exception(err);
+ }
+}
+
+bool txn::clear_map(const char *name, bool throw_if_absent) {
+ map_handle map;
+ const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi);
+ switch (err) {
+ case MDBX_SUCCESS:
+ clear_map(map);
+ return true;
+ case MDBX_NOTFOUND:
+ case MDBX_BAD_DBI:
+ if (!throw_if_absent)
+ return false;
+ MDBX_CXX17_FALLTHROUGH /* fallthrough */;
+ default:
+ MDBX_CXX20_UNLIKELY error::throw_exception(err);
+ }
+}
+
+//------------------------------------------------------------------------------
+
+void cursor_managed::close() {
+ if (MDBX_UNLIKELY(!handle_))
+ MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL);
+ ::mdbx_cursor_close(handle_);
+ handle_ = nullptr;
+}
+
+//------------------------------------------------------------------------------
+
+__cold ::std::ostream &operator<<(::std::ostream &out, const slice &it) {
+ out << "{";
+ if (!it.is_valid())
+ out << "INVALID." << it.length();
+ else if (it.is_null())
+ out << "NULL";
+ else if (it.empty())
+ out << "EMPTY->" << it.data();
+ else {
+ const slice root(it.head(std::min(it.length(), size_t(64))));
+ out << it.length() << ".";
+ if (root.is_printable())
+ (out << "\"").write(root.char_ptr(), root.length()) << "\"";
+ else
+ out << root.encode_base58();
+ if (root.length() < it.length())
+ out << "...";
+ }
+ return out << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) {
+ return out << "{" << it.key << " => " << it.value << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out, const pair_result &it) {
+ return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => "
+ << it.value << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const ::mdbx::env::geometry::size &it) {
+ switch (it.bytes) {
+ case ::mdbx::env::geometry::default_value:
+ return out << "default";
+ case ::mdbx::env::geometry::minimal_value:
+ return out << "minimal";
+ case ::mdbx::env::geometry::maximal_value:
+ return out << "maximal";
+ }
+
+ const auto bytes = (it.bytes < 0) ? out << "-",
+ size_t(-it.bytes) : size_t(it.bytes);
+ struct {
+ size_t one;
+ const char *suffix;
+ } static const scales[] = {
+#if MDBX_WORDBITS > 32
+ {env_managed::geometry::EiB, "EiB"},
+ {env_managed::geometry::EB, "EB"},
+ {env_managed::geometry::PiB, "PiB"},
+ {env_managed::geometry::PB, "PB"},
+ {env_managed::geometry::TiB, "TiB"},
+ {env_managed::geometry::TB, "TB"},
+#endif
+ {env_managed::geometry::GiB, "GiB"},
+ {env_managed::geometry::GB, "GB"},
+ {env_managed::geometry::MiB, "MiB"},
+ {env_managed::geometry::MB, "MB"},
+ {env_managed::geometry::KiB, "KiB"},
+ {env_managed::geometry::kB, "kB"},
+ {1, " bytes"}
+ };
+
+ for (const auto i : scales)
+ if (bytes % i.one == 0)
+ return out << bytes / i.one << i.suffix;
+
+ assert(false);
+ __unreachable();
+ return out;
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env::geometry &it) {
+ return //
+ out << "\tlower " << env::geometry::size(it.size_lower) //
+ << ",\n\tnow " << env::geometry::size(it.size_now) //
+ << ",\n\tupper " << env::geometry::size(it.size_upper) //
+ << ",\n\tgrowth " << env::geometry::size(it.growth_step) //
+ << ",\n\tshrink " << env::geometry::size(it.shrink_threshold) //
+ << ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env::operate_parameters &it) {
+ return out << "{\n" //
+ << "\tmax_maps " << it.max_maps //
+ << ",\n\tmax_readers " << it.max_readers //
+ << ",\n\tmode " << it.mode //
+ << ",\n\tdurability " << it.durability //
+ << ",\n\treclaiming " << it.reclaiming //
+ << ",\n\toptions " << it.options //
+ << "\n}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) {
+ switch (it) {
+ case env::mode::readonly:
+ return out << "readonly";
+ case env::mode::write_file_io:
+ return out << "write_file_io";
+ case env::mode::write_mapped_io:
+ return out << "write_mapped_io";
+ default:
+ return out << "mdbx::env::mode::invalid";
+ }
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env::durability &it) {
+ switch (it) {
+ case env::durability::robust_synchronous:
+ return out << "robust_synchronous";
+ case env::durability::half_synchronous_weak_last:
+ return out << "half_synchronous_weak_last";
+ case env::durability::lazy_weak_tail:
+ return out << "lazy_weak_tail";
+ case env::durability::whole_fragile:
+ return out << "whole_fragile";
+ default:
+ return out << "mdbx::env::durability::invalid";
+ }
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env::reclaiming_options &it) {
+ return out << "{" //
+ << "lifo: " << (it.lifo ? "yes" : "no") //
+ << ", coalesce: " << (it.coalesce ? "yes" : "no") //
+ << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env::operate_options &it) {
+ static const char comma[] = ", ";
+ const char *delimiter = "";
+ out << "{";
+ if (it.orphan_read_transactions) {
+ out << delimiter << "orphan_read_transactions";
+ delimiter = comma;
+ }
+ if (it.nested_write_transactions) {
+ out << delimiter << "nested_write_transactions";
+ delimiter = comma;
+ }
+ if (it.exclusive) {
+ out << delimiter << "exclusive";
+ delimiter = comma;
+ }
+ if (it.disable_readahead) {
+ out << delimiter << "disable_readahead";
+ delimiter = comma;
+ }
+ if (it.disable_clear_memory) {
+ out << delimiter << "disable_clear_memory";
+ delimiter = comma;
+ }
+ if (delimiter != comma)
+ out << "default";
+ return out << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const env_managed::create_parameters &it) {
+ return out << "{\n" //
+ << "\tfile_mode " << std::oct << it.file_mode_bits << std::dec //
+ << ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") //
+ << ",\n"
+ << it.geometry << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const MDBX_log_level_t &it) {
+ switch (it) {
+ case MDBX_LOG_FATAL:
+ return out << "LOG_FATAL";
+ case MDBX_LOG_ERROR:
+ return out << "LOG_ERROR";
+ case MDBX_LOG_WARN:
+ return out << "LOG_WARN";
+ case MDBX_LOG_NOTICE:
+ return out << "LOG_NOTICE";
+ case MDBX_LOG_VERBOSE:
+ return out << "LOG_VERBOSE";
+ case MDBX_LOG_DEBUG:
+ return out << "LOG_DEBUG";
+ case MDBX_LOG_TRACE:
+ return out << "LOG_TRACE";
+ case MDBX_LOG_EXTRA:
+ return out << "LOG_EXTRA";
+ case MDBX_LOG_DONTCHANGE:
+ return out << "LOG_DONTCHANGE";
+ default:
+ return out << "mdbx::log_level::invalid";
+ }
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const MDBX_debug_flags_t &it) {
+ if (it == MDBX_DBG_DONTCHANGE)
+ return out << "DBG_DONTCHANGE";
+
+ static const char comma[] = "|";
+ const char *delimiter = "";
+ out << "{";
+ if (it & MDBX_DBG_ASSERT) {
+ out << delimiter << "DBG_ASSERT";
+ delimiter = comma;
+ }
+ if (it & MDBX_DBG_AUDIT) {
+ out << delimiter << "DBG_AUDIT";
+ delimiter = comma;
+ }
+ if (it & MDBX_DBG_JITTER) {
+ out << delimiter << "DBG_JITTER";
+ delimiter = comma;
+ }
+ if (it & MDBX_DBG_DUMP) {
+ out << delimiter << "DBG_DUMP";
+ delimiter = comma;
+ }
+ if (it & MDBX_DBG_LEGACY_MULTIOPEN) {
+ out << delimiter << "DBG_LEGACY_MULTIOPEN";
+ delimiter = comma;
+ }
+ if (it & MDBX_DBG_LEGACY_OVERLAP) {
+ out << delimiter << "DBG_LEGACY_OVERLAP";
+ delimiter = comma;
+ }
+ if (delimiter != comma)
+ out << "DBG_NONE";
+ return out << "}";
+}
+
+__cold ::std::ostream &operator<<(::std::ostream &out,
+ const ::mdbx::error &err) {
+ return out << err.what() << " (" << long(err.code()) << ")";
+}
+
+} // namespace mdbx
diff --git a/3rdparty/libmdbx/mdbx.h b/3rdparty/libmdbx/mdbx.h
index 414d6444a785cd3d269b20f3d3d539ee6b0a5069..2fc7e2de0ad17f646eece08e7189951da0a20a9f 100644
--- a/3rdparty/libmdbx/mdbx.h
+++ b/3rdparty/libmdbx/mdbx.h
@@ -209,8 +209,7 @@ typedef mode_t mdbx_mode_t;
#pragma warning(pop)
#endif
-/** end of c_api @}
- *
+/** @} close c_api
* \defgroup api_macros Common Macros
* @{ */
@@ -236,15 +235,12 @@ typedef mode_t mdbx_mode_t;
#define __has_builtin(x) (0)
#endif /* __has_builtin */
-/** \brief The 'pure' function attribute for optimization.
- * \details Many functions have no effects except the return value and their
+/** Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
-#if defined(DOXYGEN)
-#define MDBX_PURE_FUNCTION [[gnu::pure]]
-#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \
+#if (defined(__GNUC__) || __has_attribute(__pure__)) && \
(!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \
|| !defined(__cplusplus) || !__has_feature(cxx_exceptions))
#define MDBX_PURE_FUNCTION __attribute__((__pure__))
@@ -257,12 +253,9 @@ typedef mode_t mdbx_mode_t;
#define MDBX_PURE_FUNCTION
#endif /* MDBX_PURE_FUNCTION */
-/** \brief The 'pure nothrow' function attribute for optimization.
- * \details Like \ref MDBX_PURE_FUNCTION with addition `noexcept` restriction
+/** Like \ref MDBX_PURE_FUNCTION with addition `noexcept` restriction
* that is compatible to CLANG and proposed [[pure]]. */
-#if defined(DOXYGEN)
-#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure, gnu::nothrow]]
-#elif defined(__GNUC__) || \
+#if defined(__GNUC__) || \
(__has_attribute(__pure__) && __has_attribute(__nothrow__))
#define MDBX_NOTHROW_PURE_FUNCTION __attribute__((__pure__, __nothrow__))
#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920
@@ -283,8 +276,7 @@ typedef mode_t mdbx_mode_t;
#define MDBX_NOTHROW_PURE_FUNCTION
#endif /* MDBX_NOTHROW_PURE_FUNCTION */
-/** \brief The 'const' function attribute for optimization.
- * \details Many functions do not examine any values except their arguments,
+/** Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
@@ -293,9 +285,7 @@ typedef mode_t mdbx_mode_t;
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
-#if defined(DOXYGEN)
-#define MDBX_CONST_FUNCTION [[gnu::const]]
-#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \
+#if (defined(__GNUC__) || __has_attribute(__pure__)) && \
(!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \
|| !defined(__cplusplus) || !__has_feature(cxx_exceptions))
#define MDBX_CONST_FUNCTION __attribute__((__const__))
@@ -308,12 +298,9 @@ typedef mode_t mdbx_mode_t;
#define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION
#endif /* MDBX_CONST_FUNCTION */
-/** \brief The 'const nothrow' function attribute for optimization.
- * \details Like \ref MDBX_CONST_FUNCTION with addition `noexcept` restriction
+/** Like \ref MDBX_CONST_FUNCTION with addition `noexcept` restriction
* that is compatible to CLANG and future [[const]]. */
-#if defined(DOXYGEN)
-#define MDBX_NOTHROW_CONST_FUNCTION [[gnu::const, gnu::nothrow]]
-#elif defined(__GNUC__) || \
+#if defined(__GNUC__) || \
(__has_attribute(__const__) && __has_attribute(__nothrow__))
#define MDBX_NOTHROW_CONST_FUNCTION __attribute__((__const__, __nothrow__))
#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920
@@ -330,19 +317,9 @@ typedef mode_t mdbx_mode_t;
#define MDBX_NOTHROW_CONST_FUNCTION MDBX_NOTHROW_PURE_FUNCTION
#endif /* MDBX_NOTHROW_CONST_FUNCTION */
-/** \brief The 'deprecated' attribute to produce warnings when used.
- * \note This macro may be predefined as empty to avoid "deprecated" warnings.
- */
-#ifndef MDBX_DEPRECATED
+#ifndef MDBX_DEPRECATED /* may be predefined to avoid warnings "deprecated" */
#ifdef __deprecated
#define MDBX_DEPRECATED __deprecated
-#elif defined(DOXYGEN) || \
- (defined(__cplusplus) && __cplusplus >= 201603L && \
- __has_cpp_attribute(maybe_unused) && \
- __has_cpp_attribute(maybe_unused) >= 201603L) || \
- (!defined(__cplusplus) && defined(__STDC_VERSION__) && \
- __STDC_VERSION__ > 202005L)
-#define MDBX_DEPRECATED [[deprecated]]
#elif defined(__GNUC__) || __has_attribute(__deprecated__)
#define MDBX_DEPRECATED __attribute__((__deprecated__))
#elif defined(_MSC_VER)
@@ -415,21 +392,15 @@ typedef mode_t mdbx_mode_t;
#endif
#endif /* bool without __cplusplus */
-/** Workaround for old compilers without support for C++17 `noexcept`. */
-#if defined(DOXYGEN)
-#define MDBX_CXX17_NOEXCEPT noexcept
-#elif !defined(__cpp_noexcept_function_type) || \
- __cpp_noexcept_function_type < 201510L
+#if !defined(DOXYGEN) && (!defined(__cpp_noexcept_function_type) || \
+ __cpp_noexcept_function_type < 201510L)
#define MDBX_CXX17_NOEXCEPT
#else
#define MDBX_CXX17_NOEXCEPT noexcept
#endif /* MDBX_CXX17_NOEXCEPT */
-/** Workaround for old compilers without support for any kind of `constexpr`. */
-#if defined(DOXYGEN)
-#define MDBX_CXX01_CONSTEXPR constexpr
-#define MDBX_CXX01_CONSTEXPR_VAR constexpr
-#elif !defined(__cplusplus)
+/* Workaround for old compilers without properly support for constexpr. */
+#if !defined(__cplusplus)
#define MDBX_CXX01_CONSTEXPR __inline
#define MDBX_CXX01_CONSTEXPR_VAR const
#elif !defined(DOXYGEN) && \
@@ -447,12 +418,7 @@ typedef mode_t mdbx_mode_t;
#define MDBX_CXX01_CONSTEXPR_VAR constexpr
#endif /* MDBX_CXX01_CONSTEXPR */
-/** Workaround for old compilers without properly support for C++11 `constexpr`.
- */
-#if defined(DOXYGEN)
-#define MDBX_CXX11_CONSTEXPR constexpr
-#define MDBX_CXX11_CONSTEXPR_VAR constexpr
-#elif !defined(__cplusplus)
+#if !defined(__cplusplus)
#define MDBX_CXX11_CONSTEXPR __inline
#define MDBX_CXX11_CONSTEXPR_VAR const
#elif !defined(DOXYGEN) && \
@@ -469,12 +435,7 @@ typedef mode_t mdbx_mode_t;
#define MDBX_CXX11_CONSTEXPR_VAR constexpr
#endif /* MDBX_CXX11_CONSTEXPR */
-/** Workaround for old compilers without properly support for C++14 `constexpr`.
- */
-#if defined(DOXYGEN)
-#define MDBX_CXX14_CONSTEXPR constexpr
-#define MDBX_CXX14_CONSTEXPR_VAR constexpr
-#elif !defined(__cplusplus)
+#if !defined(__cplusplus)
#define MDBX_CXX14_CONSTEXPR __inline
#define MDBX_CXX14_CONSTEXPR_VAR const
#elif defined(DOXYGEN) || \
@@ -494,10 +455,6 @@ typedef mode_t mdbx_mode_t;
#define MDBX_NORETURN __noreturn
#elif defined(_Noreturn)
#define MDBX_NORETURN _Noreturn
-#elif defined(DOXYGEN) || (defined(__cplusplus) && __cplusplus >= 201103L) || \
- (!defined(__cplusplus) && defined(__STDC_VERSION__) && \
- __STDC_VERSION__ > 202005L)
-#define MDBX_NORETURN [[noreturn]]
#elif defined(__GNUC__) || __has_attribute(__noreturn__)
#define MDBX_NORETURN __attribute__((__noreturn__))
#elif defined(_MSC_VER) && !defined(__clang__)
@@ -507,7 +464,7 @@ typedef mode_t mdbx_mode_t;
#endif /* MDBX_NORETURN */
#ifndef MDBX_PRINTF_ARGS
-#if defined(__GNUC__) || __has_attribute(__format__) || defined(DOXYGEN)
+#if defined(__GNUC__) || __has_attribute(__format__)
#if defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)
#define MDBX_PRINTF_ARGS(format_index, first_arg) \
__attribute__((__format__(__gnu_printf__, format_index, first_arg)))
@@ -521,9 +478,9 @@ typedef mode_t mdbx_mode_t;
#endif /* MDBX_PRINTF_ARGS */
#if defined(DOXYGEN) || \
- (defined(__cplusplus) && __cplusplus >= 201603L && \
+ (defined(__cplusplus) && __cplusplus >= 201603 && \
__has_cpp_attribute(maybe_unused) && \
- __has_cpp_attribute(maybe_unused) >= 201603L) || \
+ __has_cpp_attribute(maybe_unused) >= 201603) || \
(!defined(__cplusplus) && defined(__STDC_VERSION__) && \
__STDC_VERSION__ > 202005L)
#define MDBX_MAYBE_UNUSED [[maybe_unused]]
@@ -533,7 +490,7 @@ typedef mode_t mdbx_mode_t;
#define MDBX_MAYBE_UNUSED
#endif /* MDBX_MAYBE_UNUSED */
-#if __has_attribute(no_sanitize) || defined(DOXYGEN)
+#if __has_attribute(no_sanitize)
#define MDBX_NOSANITIZE_ENUM __attribute((__no_sanitize__("enum")))
#else
#define MDBX_NOSANITIZE_ENUM
@@ -617,7 +574,7 @@ typedef mode_t mdbx_mode_t;
#endif /* DEFINE_ENUM_FLAG_OPERATORS */
-/** end of api_macros @} */
+/** @} end of Common Macros */
/*----------------------------------------------------------------------------*/
@@ -1004,7 +961,7 @@ LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf,
/** \brief Panics with message and causes abnormal process termination. */
LIBMDBX_API void mdbx_panic(const char *fmt, ...) MDBX_PRINTF_ARGS(1, 2);
-/** end of c_debug @} */
+/** @} end of logging & debug */
/** \brief Environment flags
* \ingroup c_opening
@@ -1404,7 +1361,7 @@ enum MDBX_env_flags_t {
* \ref mdbx_txn_begin() for particular write transaction. \see sync_modes */
MDBX_UTTERLY_NOSYNC = MDBX_SAFE_NOSYNC | UINT32_C(0x100000),
- /** end of sync_modes @} */
+ /** @} end of SYNC MODES */
};
#ifndef __cplusplus
/** \ingroup c_opening */
@@ -1446,49 +1403,12 @@ enum MDBX_txn_flags_t {
MDBX_TXN_TRY = UINT32_C(0x10000000),
/** Exactly the same as \ref MDBX_NOMETASYNC,
- * but for this transaction only. */
+ * but for this transaction only */
MDBX_TXN_NOMETASYNC = MDBX_NOMETASYNC,
/** Exactly the same as \ref MDBX_SAFE_NOSYNC,
- * but for this transaction only. */
- MDBX_TXN_NOSYNC = MDBX_SAFE_NOSYNC,
-
- /* Transaction state flags ---------------------------------------------- */
-
- /** Transaction is invalid.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_INVALID = INT32_MIN,
-
- /** Transaction is finished or never began.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_FINISHED = 0x01,
-
- /** Transaction is unusable after an error.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_ERROR = 0x02,
-
- /** Transaction must write, even if dirty list is empty.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_DIRTY = 0x04,
-
- /** Transaction or a parent has spilled pages.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_SPILLS = 0x08,
-
- /** Transaction has a nested child transaction.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_HAS_CHILD = 0x10,
-
- /** Most operations on the transaction are currently illegal.
- * \note Transaction state flag. Returned from \ref mdbx_txn_flags()
- * but can't be used with \ref mdbx_txn_begin(). */
- MDBX_TXN_BLOCKED = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD
+ * but for this transaction only */
+ MDBX_TXN_NOSYNC = MDBX_SAFE_NOSYNC
};
#ifndef __cplusplus
typedef enum MDBX_txn_flags_t MDBX_txn_flags_t;
@@ -3215,7 +3135,7 @@ mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags);
/** \deprecated Please use \ref mdbx_env_get_maxkeysize_ex()
* and/or \ref mdbx_env_get_maxvalsize_ex()
* \ingroup c_statinfo */
-MDBX_DEPRECATED MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int
+MDBX_NOTHROW_PURE_FUNCTION MDBX_DEPRECATED LIBMDBX_API int
mdbx_env_get_maxkeysize(const MDBX_env *env);
/** \brief Sets application information (a context pointer) associated with
@@ -3466,7 +3386,7 @@ mdbx_txn_env(const MDBX_txn *txn);
/** \brief Return the transaction's flags.
* \ingroup c_transactions
*
- * This returns the flags, including internal, associated with this transaction.
+ * This returns the flags associated with this transaction.
*
* \param [in] txn A transaction handle returned by \ref mdbx_txn_begin().
*
@@ -3490,7 +3410,6 @@ mdbx_txn_id(const MDBX_txn *txn);
/** \brief Latency of commit stages in 1/65536 of seconds units.
* \warning This structure may be changed in future releases.
- * \ingroup c_statinfo
* \see mdbx_txn_commit_ex() */
struct MDBX_commit_latency {
/** \brief Duration of preparation (commit child transactions, update
@@ -3519,7 +3438,7 @@ typedef struct MDBX_commit_latency MDBX_commit_latency;
/** \brief Commit all the operations of a transaction into the database and
* collect latency information.
* \see mdbx_txn_commit()
- * \ingroup c_transactions
+ * \ingroup c_statinfo
* \warning This function may be changed in future releases. */
LIBMDBX_API int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency);
@@ -3887,7 +3806,7 @@ MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32,
(const int32_t i32)) {
return UINT32_C(0x80000000) + i32;
}
-/** end of value2key @} */
+/** @} */
/** \defgroup key2value Key-to-Value functions
* \brief Key-to-Value functions to
@@ -3908,7 +3827,7 @@ mdbx_int32_from_key(const MDBX_val);
MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t
mdbx_int64_from_key(const MDBX_val);
-/** end of value2key @} */
+/** @} */
/** \brief Retrieve statistics for a database.
* \ingroup c_statinfo
@@ -5181,7 +5100,7 @@ LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname,
* leg(s). */
LIBMDBX_API int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target_meta);
-/** end of btree_traversal @} */
+/** @} B-tree Traversal */
/**** Attribute support functions for Nexenta (scheduled for removal)
* *****************************************************************/
@@ -5351,10 +5270,10 @@ LIBMDBX_API int mdbx_cursor_get_attr(MDBX_cursor *cursor, MDBX_val *key,
* \retval MDBX_EINVAL An invalid parameter was specified. */
LIBMDBX_API int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
MDBX_val *data, mdbx_attr_t *pattr);
-/** end of nexenta @} */
+/** @} end of Attribute support functions for Nexenta */
#endif /* MDBX_NEXENTA_ATTRS */
-/** end of c_api @} */
+/** @} end of C API */
/*******************************************************************************
* Workaround for mmaped-lookahead-cross-page-boundary bug
diff --git a/3rdparty/libmdbx/mdbx.h++ b/3rdparty/libmdbx/mdbx.h++
index 8d9a28caaf835099fbdbc29d8c70e78e8fc76ae6..11d209e6948984d2a9724e65ec9f5008456c06c1 100644
--- a/3rdparty/libmdbx/mdbx.h++
+++ b/3rdparty/libmdbx/mdbx.h++
@@ -129,35 +129,26 @@
#endif
#endif /* Byte Order */
-/** Workaround for old compilers without properly support for `C++17 constexpr`.
- */
-#if defined(DOXYGEN)
-#define MDBX_CXX17_CONSTEXPR constexpr
-#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \
- ((defined(_MSC_VER) && _MSC_VER >= 1915) || \
- (defined(__clang__) && __clang_major__ > 5) || \
- (defined(__GNUC__) && __GNUC__ > 7) || \
- (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER)))
+#if defined(DOXYGEN) || \
+ defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \
+ ((defined(_MSC_VER) && _MSC_VER >= 1915) || \
+ (defined(__clang__) && __clang_major__ > 5) || \
+ (defined(__GNUC__) && __GNUC__ > 7) || \
+ (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER)))
#define MDBX_CXX17_CONSTEXPR constexpr
#else
#define MDBX_CXX17_CONSTEXPR inline
#endif /* MDBX_CXX17_CONSTEXPR */
-/** Workaround for old compilers without properly support for C++20 `constexpr`.
- */
-#if defined(DOXYGEN)
-#define MDBX_CXX20_CONSTEXPR constexpr
-#elif defined(__cpp_lib_is_constant_evaluated) && \
- __cpp_lib_is_constant_evaluated >= 201811L && \
- defined(__cpp_lib_constexpr_string) && \
- __cpp_lib_constexpr_string >= 201907L
+#if defined(DOXYGEN) || defined(__cpp_lib_is_constant_evaluated) && \
+ __cpp_lib_is_constant_evaluated >= 201811L && \
+ defined(__cpp_lib_constexpr_string) && \
+ __cpp_lib_constexpr_string >= 201907L
#define MDBX_CXX20_CONSTEXPR constexpr
#else
#define MDBX_CXX20_CONSTEXPR inline
#endif /* MDBX_CXX20_CONSTEXPR */
-/** Workaround for old compilers without support assertion inside `constexpr`
- * functions. */
#if defined(CONSTEXPR_ASSERT)
#define MDBX_CONSTEXPR_ASSERT(expr) CONSTEXPR_ASSERT(expr)
#elif defined NDEBUG
@@ -187,11 +178,7 @@
#endif
#endif /* MDBX_UNLIKELY */
-/** Workaround for old compilers without properly support for C++20 `if
- * constexpr`. */
-#if defined(DOXYGEN)
-#define MDBX_IF_CONSTEXPR constexpr
-#elif defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L
+#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L
#define MDBX_IF_CONSTEXPR constexpr
#else
#define MDBX_IF_CONSTEXPR
@@ -269,13 +256,11 @@
#endif /* _MSC_VER (warnings) */
//------------------------------------------------------------------------------
-/// \brief The libmdbx C++ API namespace
-/// \ingroup cxx_api
-namespace mdbx {
-
/// \defgroup cxx_api C++ API
/// @{
+namespace mdbx {
+
// Functions whose signature depends on the `mdbx::byte` type
// must be strictly defined as inline!
#if defined(DOXYGEN) || (defined(__cpp_char8_t) && __cpp_char8_t >= 201811)
@@ -352,12 +337,6 @@ using filehandle = ::mdbx_filehandle_t;
(!defined(__IPHONE_OS_VERSION_MIN_REQUIRED) || \
__IPHONE_OS_VERSION_MIN_REQUIRED >= 130100))
namespace filesystem = ::std::filesystem;
-/// \brief Defined if `mdbx::filesystem::path` is available.
-/// \details If defined, it is always `mdbx::filesystem::path`,
-/// which in turn can be refs to `std::filesystem::path`
-/// or `std::experimental::filesystem::path`.
-/// Nonetheless `MDBX_STD_FILESYSTEM_PATH` not defined if the `::mdbx::path`
-/// is fallbacked to c `std::string` or `std::wstring`.
#define MDBX_STD_FILESYSTEM_PATH ::mdbx::filesystem::path
#elif defined(__cpp_lib_experimental_filesystem) && \
__cpp_lib_experimental_filesystem >= 201406L
@@ -373,9 +352,6 @@ using path = ::std::wstring;
using path = ::std::string;
#endif /* mdbx::path */
-/// \defgroup cxx_exceptions exceptions and errors
-/// @{
-
/// \brief Transfers C++ exceptions thru C callbacks.
/// \details Implements saving exceptions before returning
/// from an C++'s environment to the intermediate C code and re-throwing after
@@ -531,17 +507,11 @@ static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom,
size_t payload);
static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload,
size_t tailroom);
-
-/// end of cxx_exceptions @}
-
static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept;
static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src,
size_t bytes) noexcept;
//------------------------------------------------------------------------------
-/// \defgroup cxx_data slices and buffers
-/// @{
-
#if MDBX_HAVE_CXX20_CONCEPTS
template <typename T>
@@ -2771,8 +2741,6 @@ struct pair_result : public pair {
}
};
-/// end of cxx_data @}
-
//------------------------------------------------------------------------------
/// \brief Loop control constants for readers enumeration functor and other
@@ -3212,7 +3180,7 @@ public:
env ©(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify,
bool force_dynamic_size = false);
#endif /* MDBX_STD_FILESYSTEM_PATH */
-#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
+#if defined(_WIN32) || defined(_WIN64)
env ©(const ::std::wstring &destination, bool compactify,
bool force_dynamic_size = false);
#endif /* Windows */
@@ -3245,7 +3213,7 @@ public:
static bool remove(const MDBX_STD_FILESYSTEM_PATH &,
const remove_mode mode = just_remove);
#endif /* MDBX_STD_FILESYSTEM_PATH */
-#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
+#if defined(_WIN32) || defined(_WIN64)
static bool remove(const ::std::wstring &,
const remove_mode mode = just_remove);
#endif /* Windows */
@@ -3490,7 +3458,7 @@ public:
env_managed(const MDBX_STD_FILESYSTEM_PATH &, const operate_parameters &,
bool accede = true);
#endif /* MDBX_STD_FILESYSTEM_PATH */
-#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
+#if defined(_WIN32) || defined(_WIN64)
env_managed(const ::std::wstring &, const operate_parameters &,
bool accede = true);
#endif /* Windows */
@@ -3511,7 +3479,7 @@ public:
env_managed(const MDBX_STD_FILESYSTEM_PATH &, const create_parameters &,
const operate_parameters &, bool accede = true);
#endif /* MDBX_STD_FILESYSTEM_PATH */
-#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
+#if defined(_WIN32) || defined(_WIN64)
env_managed(const ::std::wstring &, const create_parameters &,
const operate_parameters &, bool accede = true);
#endif /* Windows */
@@ -5909,18 +5877,12 @@ inline bool cursor::erase(const slice &key, const slice &value) {
return data.done && erase();
}
-/// end cxx_api @}
} // namespace mdbx
//------------------------------------------------------------------------------
-/// \brief The `std:: namespace part of libmdbx C++ API
-/// \ingroup cxx_api
namespace std {
-/// \defgroup cxx_api C++ API
-/// @{
-
inline string to_string(const ::mdbx::slice &value) {
ostringstream out;
out << value;
@@ -6012,9 +5974,10 @@ template <> struct hash<::mdbx::slice> {
}
};
-/// end cxx_api @}
} // namespace std
#ifdef _MSC_VER
#pragma warning(pop)
#endif
+
+/// @} end of C++ API
diff --git a/3rdparty/libmdbx/mdbx_chk.c b/3rdparty/libmdbx/mdbx_chk.c
new file mode 100644
index 0000000000000000000000000000000000000000..27506c7386a1a0d0f74098883fc5f3dd06aafc7a
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_chk.c
@@ -0,0 +1,5264 @@
+/* mdbx_chk.c - memory-mapped database check tool */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+typedef struct flagbit {
+ int bit;
+ const char *name;
+} flagbit;
+
+const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"},
+ {MDBX_INTEGERKEY, "integerkey"},
+ {MDBX_REVERSEKEY, "reversekey"},
+ {MDBX_DUPFIXED, "dupfixed"},
+ {MDBX_REVERSEDUP, "reversedup"},
+ {MDBX_INTEGERDUP, "integerdup"},
+ {0, nullptr}};
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = 1;
+ return true;
+}
+
+static uint64_t GetMilliseconds(void) {
+ LARGE_INTEGER Counter, Frequency;
+ return (QueryPerformanceFrequency(&Frequency) &&
+ QueryPerformanceCounter(&Counter))
+ ? Counter.QuadPart * 1000ul / Frequency.QuadPart
+ : 0;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+#define EXIT_INTERRUPTED (EXIT_FAILURE + 4)
+#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3)
+#define EXIT_FAILURE_MDBX (EXIT_FAILURE + 2)
+#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1)
+#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE
+
+typedef struct {
+ const char *name;
+ struct {
+ uint64_t branch, large_count, large_volume, leaf;
+ uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed;
+ uint64_t total, empty, other;
+ } pages;
+ uint64_t payload_bytes;
+ uint64_t lost_bytes;
+} walk_dbi_t;
+
+struct {
+ short *pagemap;
+ uint64_t total_payload_bytes;
+ uint64_t pgcount;
+ walk_dbi_t
+ dbi[MDBX_MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1];
+} walk;
+
+#define dbi_free walk.dbi[FREE_DBI]
+#define dbi_main walk.dbi[MAIN_DBI]
+#define dbi_meta walk.dbi[CORE_DBS]
+
+int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE;
+MDBX_env *env;
+MDBX_txn *txn;
+MDBX_envinfo envinfo;
+size_t userdb_count, skipped_subdb;
+uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages,
+ unused_pages, backed_pages;
+unsigned verbose;
+bool ignore_wrong_order, quiet, dont_traversal;
+const char *only_subdb;
+int stuck_meta = -1;
+
+struct problem {
+ struct problem *pr_next;
+ size_t count;
+ const char *caption;
+};
+
+struct problem *problems_list;
+unsigned total_problems, data_tree_problems, gc_tree_problems;
+
+static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) {
+ if (!quiet) {
+ va_list args;
+
+ fflush(stderr);
+ va_start(args, msg);
+ vfprintf(stdout, msg, args);
+ va_end(args);
+ }
+}
+
+static void va_log(MDBX_log_level_t level, const char *msg, va_list args) {
+ static const char *const prefixes[] = {
+ "!!!fatal: ", " ! " /* error */, " ~ " /* warning */,
+ " " /* notice */, " // " /* verbose */, " //// " /* debug */,
+ " ////// " /* trace */
+ };
+
+ FILE *out = stdout;
+ if (level <= MDBX_LOG_ERROR) {
+ total_problems++;
+ out = stderr;
+ }
+
+ if (!quiet && verbose + 1 >= (unsigned)level &&
+ (unsigned)level < ARRAY_LENGTH(prefixes)) {
+ fflush(nullptr);
+ fputs(prefixes[level], out);
+ vfprintf(out, msg, args);
+ if (msg[strlen(msg) - 1] != '\n')
+ fputc('\n', out);
+ fflush(nullptr);
+ }
+
+ if (level == MDBX_LOG_FATAL) {
+ exit(EXIT_FAILURE_MDBX);
+ abort();
+ }
+}
+
+static void MDBX_PRINTF_ARGS(1, 2) error(const char *msg, ...) {
+ va_list args;
+ va_start(args, msg);
+ va_log(MDBX_LOG_ERROR, msg, args);
+ va_end(args);
+}
+
+static void logger(MDBX_log_level_t level, const char *function, int line,
+ const char *msg, va_list args) {
+ (void)line;
+ (void)function;
+ if (level < MDBX_LOG_EXTRA)
+ va_log(level, msg, args);
+}
+
+static int check_user_break(void) {
+ switch (user_break) {
+ case 0:
+ return MDBX_SUCCESS;
+ case 1:
+ print(" - interrupted by signal\n");
+ fflush(nullptr);
+ user_break = 2;
+ }
+ return MDBX_EINTR;
+}
+
+static void pagemap_cleanup(void) {
+ for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1;
+ i < ARRAY_LENGTH(walk.dbi); ++i) {
+ if (walk.dbi[i].name) {
+ mdbx_free((void *)walk.dbi[i].name);
+ walk.dbi[i].name = nullptr;
+ }
+ }
+
+ mdbx_free(walk.pagemap);
+ walk.pagemap = nullptr;
+}
+
+static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
+ static walk_dbi_t *last;
+
+ if (dbi_name == MDBX_PGWALK_MAIN)
+ return &dbi_main;
+ if (dbi_name == MDBX_PGWALK_GC)
+ return &dbi_free;
+ if (dbi_name == MDBX_PGWALK_META)
+ return &dbi_meta;
+
+ if (last && strcmp(last->name, dbi_name) == 0)
+ return last;
+
+ walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1;
+ for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) {
+ if (strcmp(dbi->name, dbi_name) == 0)
+ return last = dbi;
+ }
+
+ if (verbose > 0 && !silent) {
+ print(" - found '%s' area\n", dbi_name);
+ fflush(nullptr);
+ }
+
+ if (dbi == ARRAY_END(walk.dbi))
+ return nullptr;
+
+ dbi->name = mdbx_strdup(dbi_name);
+ return last = dbi;
+}
+
+static void MDBX_PRINTF_ARGS(4, 5)
+ problem_add(const char *object, uint64_t entry_number, const char *msg,
+ const char *extra, ...) {
+ total_problems++;
+
+ if (!quiet) {
+ int need_fflush = 0;
+ struct problem *p;
+
+ for (p = problems_list; p; p = p->pr_next)
+ if (p->caption == msg)
+ break;
+
+ if (!p) {
+ p = mdbx_calloc(1, sizeof(*p));
+ if (unlikely(!p))
+ return;
+ p->caption = msg;
+ p->pr_next = problems_list;
+ problems_list = p;
+ need_fflush = 1;
+ }
+
+ p->count++;
+ if (verbose > 1) {
+ print(" %s #%" PRIu64 ": %s", object, entry_number, msg);
+ if (extra) {
+ va_list args;
+ printf(" (");
+ va_start(args, extra);
+ vfprintf(stdout, extra, args);
+ va_end(args);
+ printf(")");
+ }
+ printf("\n");
+ if (need_fflush)
+ fflush(nullptr);
+ }
+ }
+}
+
+static struct problem *problems_push(void) {
+ struct problem *p = problems_list;
+ problems_list = nullptr;
+ return p;
+}
+
+static size_t problems_pop(struct problem *list) {
+ size_t count = 0;
+
+ if (problems_list) {
+ int i;
+
+ print(" - problems: ");
+ for (i = 0; problems_list; ++i) {
+ struct problem *p = problems_list->pr_next;
+ count += problems_list->count;
+ print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption,
+ problems_list->count);
+ mdbx_free(problems_list);
+ problems_list = p;
+ }
+ print("\n");
+ fflush(nullptr);
+ }
+
+ problems_list = list;
+ return count;
+}
+
+static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
+ void *const ctx, const int deep,
+ const char *const dbi_name_or_tag, const size_t page_size,
+ const MDBX_page_type_t pagetype, const MDBX_error_t err,
+ const size_t nentries, const size_t payload_bytes,
+ const size_t header_bytes, const size_t unused_bytes) {
+ (void)ctx;
+ const bool is_gc_tree = dbi_name_or_tag == MDBX_PGWALK_GC;
+ if (deep > 42) {
+ problem_add("deep", deep, "too large", nullptr);
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ return MDBX_CORRUPTED /* avoid infinite loop/recursion */;
+ }
+
+ walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false);
+ if (!dbi) {
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ return MDBX_ENOMEM;
+ }
+
+ const size_t page_bytes = payload_bytes + header_bytes + unused_bytes;
+ walk.pgcount += pgnumber;
+
+ const char *pagetype_caption;
+ bool branch = false;
+ switch (pagetype) {
+ default:
+ problem_add("page", pgno, "unknown page-type", "type %u, deep %i",
+ (unsigned)pagetype, deep);
+ pagetype_caption = "unknown";
+ dbi->pages.other += pgnumber;
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ break;
+ case MDBX_page_broken:
+ pagetype_caption = "broken";
+ dbi->pages.other += pgnumber;
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ break;
+ case MDBX_subpage_broken:
+ pagetype_caption = "broken-subpage";
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ break;
+ case MDBX_page_meta:
+ pagetype_caption = "meta";
+ dbi->pages.other += pgnumber;
+ break;
+ case MDBX_page_large:
+ pagetype_caption = "large";
+ dbi->pages.large_volume += pgnumber;
+ dbi->pages.large_count += 1;
+ break;
+ case MDBX_page_branch:
+ pagetype_caption = "branch";
+ dbi->pages.branch += pgnumber;
+ branch = true;
+ break;
+ case MDBX_page_leaf:
+ pagetype_caption = "leaf";
+ dbi->pages.leaf += pgnumber;
+ break;
+ case MDBX_page_dupfixed_leaf:
+ pagetype_caption = "leaf-dupfixed";
+ dbi->pages.leaf_dupfixed += pgnumber;
+ break;
+ case MDBX_subpage_leaf:
+ pagetype_caption = "subleaf-dupsort";
+ dbi->pages.subleaf_dupsort += 1;
+ break;
+ case MDBX_subpage_dupfixed_leaf:
+ pagetype_caption = "subleaf-dupfixed";
+ dbi->pages.subleaf_dupfixed += 1;
+ break;
+ }
+
+ if (pgnumber) {
+ if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) {
+ if (pgnumber == 1)
+ print(" %s-page %" PRIu64, pagetype_caption, pgno);
+ else
+ print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber);
+ print(" of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR
+ ", unused %" PRIiPTR ", deep %i\n",
+ dbi->name, header_bytes,
+ (pagetype == MDBX_page_branch) ? "keys" : "entries", nentries,
+ payload_bytes, unused_bytes, deep);
+ }
+
+ bool already_used = false;
+ for (unsigned n = 0; n < pgnumber; ++n) {
+ uint64_t spanpgno = pgno + n;
+ if (spanpgno >= alloc_pages) {
+ problem_add("page", spanpgno, "wrong page-no",
+ "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i",
+ pagetype_caption, spanpgno, alloc_pages, deep);
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ } else if (walk.pagemap[spanpgno]) {
+ walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1];
+ problem_add("page", spanpgno,
+ (branch && coll_dbi == dbi) ? "loop" : "already used",
+ "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name,
+ deep);
+ already_used = true;
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ } else {
+ walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1);
+ dbi->pages.total += 1;
+ }
+ }
+
+ if (already_used)
+ return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */
+ : MDBX_SUCCESS;
+ }
+
+ if (MDBX_IS_ERROR(err)) {
+ problem_add("page", pgno, "invalid/corrupted", "%s-page", pagetype_caption);
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ } else {
+ if (unused_bytes > page_size) {
+ problem_add("page", pgno, "illegal unused-bytes",
+ "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0,
+ unused_bytes, envinfo.mi_dxb_pagesize);
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ }
+
+ if (header_bytes < (int)sizeof(long) ||
+ (size_t)header_bytes >= envinfo.mi_dxb_pagesize - sizeof(long)) {
+ problem_add("page", pgno, "illegal header-length",
+ "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR,
+ pagetype_caption, sizeof(long), header_bytes,
+ envinfo.mi_dxb_pagesize - sizeof(long));
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ }
+ if (payload_bytes < 1) {
+ if (nentries > 1) {
+ problem_add("page", pgno, "zero size-of-entry",
+ "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
+ pagetype_caption, payload_bytes, nentries);
+ /* if ((size_t)header_bytes + unused_bytes < page_size) {
+ // LY: hush a misuse error
+ page_bytes = page_size;
+ } */
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ } else {
+ problem_add("page", pgno, "empty",
+ "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR
+ " entries, deep %i",
+ pagetype_caption, payload_bytes, nentries, deep);
+ dbi->pages.empty += 1;
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ }
+ }
+
+ if (pgnumber) {
+ if (page_bytes != page_size) {
+ problem_add("page", pgno, "misused",
+ "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR
+ "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i",
+ pagetype_caption, page_size, page_bytes, header_bytes,
+ payload_bytes, unused_bytes, deep);
+ if (page_size > page_bytes)
+ dbi->lost_bytes += page_size - page_bytes;
+ data_tree_problems += !is_gc_tree;
+ gc_tree_problems += is_gc_tree;
+ } else {
+ dbi->payload_bytes += payload_bytes + header_bytes;
+ walk.total_payload_bytes += payload_bytes + header_bytes;
+ }
+ }
+ }
+
+ return check_user_break();
+}
+
+typedef int(visitor)(const uint64_t record_number, const MDBX_val *key,
+ const MDBX_val *data);
+static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
+ bool silent);
+
+static int handle_userdb(const uint64_t record_number, const MDBX_val *key,
+ const MDBX_val *data) {
+ (void)record_number;
+ (void)key;
+ (void)data;
+ return check_user_break();
+}
+
+static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
+ const MDBX_val *data) {
+ char *bad = "";
+ pgno_t *iptr = data->iov_base;
+
+ if (key->iov_len != sizeof(txnid_t))
+ problem_add("entry", record_number, "wrong txn-id size",
+ "key-size %" PRIiPTR, key->iov_len);
+ else {
+ txnid_t txnid;
+ memcpy(&txnid, key->iov_base, sizeof(txnid));
+ if (txnid < 1 || txnid > envinfo.mi_recent_txnid)
+ problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid);
+ else {
+ if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
+ problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR,
+ data->iov_len);
+ size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0;
+ if (number < 1 || number > MDBX_PGL_LIMIT)
+ problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number);
+ else if ((number + 1) * sizeof(pgno_t) > data->iov_len) {
+ problem_add("entry", txnid, "trimmed idl",
+ "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
+ (number + 1) * sizeof(pgno_t), data->iov_len);
+ number = data->iov_len / sizeof(pgno_t) - 1;
+ } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
+ /* LY: allow gap up to one page. it is ok
+ * and better than shink-and-retry inside mdbx_update_gc() */
+ envinfo.mi_dxb_pagesize)
+ problem_add("entry", txnid, "extra idl space",
+ "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)",
+ (number + 1) * sizeof(pgno_t), data->iov_len);
+
+ gc_pages += number;
+ if (envinfo.mi_latter_reader_txnid > txnid)
+ reclaimable_pages += number;
+
+ pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno;
+ pgno_t span = 1;
+ for (unsigned i = 0; i < number; ++i) {
+ if (check_user_break())
+ return MDBX_EINTR;
+ const pgno_t pgno = iptr[i];
+ if (pgno < NUM_METAS)
+ problem_add("entry", txnid, "wrong idl entry",
+ "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS);
+ else if (pgno >= backed_pages)
+ problem_add("entry", txnid, "wrong idl entry",
+ "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno,
+ backed_pages);
+ else if (pgno >= alloc_pages)
+ problem_add("entry", txnid, "wrong idl entry",
+ "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno,
+ alloc_pages - 1);
+ else {
+ if (MDBX_PNL_DISORDERED(prev, pgno)) {
+ bad = " [bad sequence]";
+ problem_add("entry", txnid, "bad sequence",
+ "%" PRIaPGNO " %c [%u].%" PRIaPGNO, prev,
+ (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'),
+ i, pgno);
+ }
+ if (walk.pagemap) {
+ int idx = walk.pagemap[pgno];
+ if (idx == 0)
+ walk.pagemap[pgno] = -1;
+ else if (idx > 0)
+ problem_add("page", pgno, "already used", "by %s",
+ walk.dbi[idx - 1].name);
+ else
+ problem_add("page", pgno, "already listed in GC", nullptr);
+ }
+ }
+ prev = pgno;
+ while (i + span < number &&
+ iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span)
+ : pgno_sub(pgno, span)))
+ ++span;
+ }
+ if (verbose > 3 && !only_subdb) {
+ print(" transaction %" PRIaTXN ", %" PRIuPTR
+ " pages, maxspan %" PRIaPGNO "%s\n",
+ txnid, number, span, bad);
+ if (verbose > 4) {
+ for (unsigned i = 0; i < number; i += span) {
+ const pgno_t pgno = iptr[i];
+ for (span = 1;
+ i + span < number &&
+ iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span)
+ : pgno_sub(pgno, span));
+ ++span)
+ ;
+ if (span > 1) {
+ print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span);
+ } else
+ print(" %9" PRIaPGNO "\n", pgno);
+ }
+ }
+ }
+ }
+ }
+
+ return check_user_break();
+}
+
+static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
+ return (a->iov_len == b->iov_len &&
+ memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
+ ? 0
+ : 1;
+}
+
+static int handle_maindb(const uint64_t record_number, const MDBX_val *key,
+ const MDBX_val *data) {
+ char *name;
+ int rc;
+ size_t i;
+
+ name = key->iov_base;
+ for (i = 0; i < key->iov_len; ++i) {
+ if (name[i] < ' ')
+ return handle_userdb(record_number, key, data);
+ }
+
+ name = mdbx_malloc(key->iov_len + 1);
+ if (unlikely(!name))
+ return MDBX_ENOMEM;
+ memcpy(name, key->iov_base, key->iov_len);
+ name[key->iov_len] = '\0';
+ userdb_count++;
+
+ rc = process_db(~0u, name, handle_userdb, false);
+ mdbx_free(name);
+ if (rc != MDBX_INCOMPATIBLE)
+ return rc;
+
+ return handle_userdb(record_number, key, data);
+}
+
+static const char *db_flags2keymode(unsigned flags) {
+ flags &= (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
+ switch (flags) {
+ case 0:
+ return "usual";
+ case MDBX_REVERSEKEY:
+ return "reserve";
+ case MDBX_INTEGERKEY:
+ return "ordinal";
+ case MDBX_REVERSEKEY | MDBX_INTEGERKEY:
+ return "msgpack";
+ default:
+ assert(false);
+ __unreachable();
+ }
+}
+
+static const char *db_flags2valuemode(unsigned flags) {
+ flags &= (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP);
+ switch (flags) {
+ case 0:
+ return "single";
+ case MDBX_DUPSORT:
+ return "multi";
+ case MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_REVERSEDUP:
+ return "multi-reverse";
+ case MDBX_DUPFIXED:
+ case MDBX_DUPSORT | MDBX_DUPFIXED:
+ return "multi-samelength";
+ case MDBX_DUPFIXED | MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
+ return "multi-reverse-samelength";
+ case MDBX_INTEGERDUP:
+ case MDBX_DUPSORT | MDBX_INTEGERDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
+ case MDBX_DUPFIXED | MDBX_INTEGERDUP:
+ return "multi-ordinal";
+ case MDBX_INTEGERDUP | MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
+ return "multi-msgpack";
+ case MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
+ case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
+ return "reserved";
+ default:
+ assert(false);
+ __unreachable();
+ }
+}
+
+static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
+ bool silent) {
+ MDBX_cursor *mc;
+ MDBX_stat ms;
+ MDBX_val key, data;
+ MDBX_val prev_key, prev_data;
+ unsigned flags;
+ int rc, i;
+ struct problem *saved_list;
+ uint64_t problems_count;
+
+ uint64_t record_count = 0, dups = 0;
+ uint64_t key_bytes = 0, data_bytes = 0;
+
+ if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) {
+ print(" ! abort processing '%s' due to a previous error\n",
+ dbi_name ? dbi_name : "@MAIN");
+ return MDBX_BAD_TXN;
+ }
+
+ if (dbi_handle == ~0u) {
+ rc = mdbx_dbi_open_ex(
+ txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle,
+ (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr,
+ (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr);
+ if (rc) {
+ if (!dbi_name ||
+ rc !=
+ MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ {
+ error("mdbx_dbi_open('%s') failed, error %d %s\n",
+ dbi_name ? dbi_name : "main", rc, mdbx_strerror(rc));
+ }
+ return rc;
+ }
+ }
+
+ if (dbi_handle >= CORE_DBS && dbi_name && only_subdb &&
+ strcmp(only_subdb, dbi_name) != 0) {
+ if (verbose) {
+ print("Skip processing '%s'...\n", dbi_name);
+ fflush(nullptr);
+ }
+ skipped_subdb++;
+ return MDBX_SUCCESS;
+ }
+
+ if (!silent && verbose) {
+ print("Processing '%s'...\n", dbi_name ? dbi_name : "@MAIN");
+ fflush(nullptr);
+ }
+
+ rc = mdbx_dbi_flags(txn, dbi_handle, &flags);
+ if (rc) {
+ error("mdbx_dbi_flags() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return rc;
+ }
+
+ rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms));
+ if (rc) {
+ error("mdbx_dbi_stat() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return rc;
+ }
+
+ if (!silent && verbose) {
+ print(" - key-value kind: %s-key => %s-value", db_flags2keymode(flags),
+ db_flags2valuemode(flags));
+ if (verbose > 1) {
+ print(", flags:");
+ if (!flags)
+ print(" none");
+ else {
+ for (i = 0; dbflags[i].bit; i++)
+ if (flags & dbflags[i].bit)
+ print(" %s", dbflags[i].name);
+ }
+ if (verbose > 2)
+ print(" (0x%02X), dbi-id %d", flags, dbi_handle);
+ }
+ print("\n");
+ if (ms.ms_mod_txnid)
+ print(" - last modification txn#%" PRIu64 "\n", ms.ms_mod_txnid);
+ if (verbose > 1) {
+ print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize,
+ ms.ms_entries);
+ print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64
+ ", overflow %" PRIu64 "\n",
+ ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages,
+ ms.ms_overflow_pages);
+ }
+ }
+
+ walk_dbi_t *dbi = (dbi_handle < CORE_DBS)
+ ? &walk.dbi[dbi_handle]
+ : pagemap_lookup_dbi(dbi_name, true);
+ if (!dbi) {
+ error("too many DBIs or out of memory\n");
+ return MDBX_ENOMEM;
+ }
+ if (!dont_traversal) {
+ const uint64_t subtotal_pages =
+ ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages;
+ if (subtotal_pages != dbi->pages.total)
+ error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
+ "subtotal", subtotal_pages, dbi->pages.total);
+ if (ms.ms_branch_pages != dbi->pages.branch)
+ error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch",
+ ms.ms_branch_pages, dbi->pages.branch);
+ const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed;
+ if (ms.ms_leaf_pages != allleaf_pages)
+ error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
+ "all-leaf", ms.ms_leaf_pages, allleaf_pages);
+ if (ms.ms_overflow_pages != dbi->pages.large_volume)
+ error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
+ "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume);
+ }
+ rc = mdbx_cursor_open(txn, dbi_handle, &mc);
+ if (rc) {
+ error("mdbx_cursor_open() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return rc;
+ }
+
+ if (ignore_wrong_order) { /* for debugging with enabled assertions */
+ mc->mc_flags |= C_SKIPORD;
+ if (mc->mc_xcursor)
+ mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
+ }
+
+ const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, flags);
+ saved_list = problems_push();
+ prev_key.iov_base = nullptr;
+ prev_key.iov_len = 0;
+ prev_data.iov_base = nullptr;
+ prev_data.iov_len = 0;
+ rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST);
+ while (rc == MDBX_SUCCESS) {
+ rc = check_user_break();
+ if (rc)
+ goto bailout;
+
+ bool bad_key = false;
+ if (key.iov_len > maxkeysize) {
+ problem_add("entry", record_count, "key length exceeds max-key-size",
+ "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize);
+ bad_key = true;
+ } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) &&
+ key.iov_len != sizeof(uint32_t)) {
+ problem_add("entry", record_count, "wrong key length",
+ "%" PRIuPTR " != 4or8", key.iov_len);
+ bad_key = true;
+ }
+
+ bool bad_data = false;
+ if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) &&
+ data.iov_len != sizeof(uint32_t)) {
+ problem_add("entry", record_count, "wrong data length",
+ "%" PRIuPTR " != 4or8", data.iov_len);
+ bad_data = true;
+ }
+
+ if (prev_key.iov_base) {
+ if (prev_data.iov_base && !bad_data && (flags & MDBX_DUPFIXED) &&
+ prev_data.iov_len != data.iov_len) {
+ problem_add("entry", record_count, "different data length",
+ "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len,
+ data.iov_len);
+ bad_data = true;
+ }
+
+ if (!bad_key) {
+ int cmp = mdbx_cmp(txn, dbi_handle, &key, &prev_key);
+ if (cmp == 0) {
+ ++dups;
+ if ((flags & MDBX_DUPSORT) == 0) {
+ problem_add("entry", record_count, "duplicated entries", nullptr);
+ if (prev_data.iov_base && data.iov_len == prev_data.iov_len &&
+ memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) {
+ problem_add("entry", record_count, "complete duplicate", nullptr);
+ }
+ } else if (!bad_data && prev_data.iov_base) {
+ cmp = mdbx_dcmp(txn, dbi_handle, &data, &prev_data);
+ if (cmp == 0) {
+ problem_add("entry", record_count, "complete duplicate", nullptr);
+ } else if (cmp < 0 && !ignore_wrong_order) {
+ problem_add("entry", record_count, "wrong order of multi-values",
+ nullptr);
+ }
+ }
+ } else if (cmp < 0 && !ignore_wrong_order) {
+ problem_add("entry", record_count, "wrong order of entries", nullptr);
+ }
+ }
+ }
+
+ if (handler) {
+ rc = handler(record_count, &key, &data);
+ if (MDBX_IS_ERROR(rc))
+ goto bailout;
+ }
+
+ record_count++;
+ key_bytes += key.iov_len;
+ data_bytes += data.iov_len;
+
+ if (!bad_key) {
+ if (verbose && (flags & MDBX_INTEGERKEY) && !prev_key.iov_base)
+ print(" - fixed key-size %" PRIuPTR "\n", key.iov_len);
+ prev_key = key;
+ }
+ if (!bad_data) {
+ if (verbose && (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) &&
+ !prev_data.iov_base)
+ print(" - fixed data-size %" PRIuPTR "\n", data.iov_len);
+ prev_data = data;
+ }
+ rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT);
+ }
+ if (rc != MDBX_NOTFOUND)
+ error("mdbx_cursor_get() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ else
+ rc = 0;
+
+ if (record_count != ms.ms_entries)
+ problem_add("entry", record_count, "different number of entries",
+ "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries);
+bailout:
+ problems_count = problems_pop(saved_list);
+ if (!silent && verbose) {
+ print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64
+ " key's bytes, %" PRIu64 " data's "
+ "bytes, %" PRIu64 " problems\n",
+ record_count, dups, key_bytes, data_bytes, problems_count);
+ fflush(nullptr);
+ }
+
+ mdbx_cursor_close(mc);
+ return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS;
+}
+
+static void usage(char *prog) {
+ fprintf(stderr,
+ "usage: %s [-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s subdb] "
+ "dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -v\t\tmore verbose, could be used multiple times\n"
+ " -q\t\tbe quiet\n"
+ " -c\t\tforce cooperative mode (don't try exclusive)\n"
+ " -w\t\twrite-mode checking\n"
+ " -d\t\tdisable page-by-page traversal of B-tree\n"
+ " -i\t\tignore wrong order errors (for custom comparators case)\n"
+ " -s subdb\tprocess a specific subdatabase only\n"
+ " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n"
+ " -t\t\tturn to a specified meta-page on successful check\n"
+ " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n",
+ prog);
+ exit(EXIT_INTERRUPTED);
+}
+
+static bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b,
+ uint64_t sign_b, const bool wanna_steady) {
+ if (txn_a == txn_b)
+ return SIGN_IS_STEADY(sign_b);
+
+ if (wanna_steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b))
+ return SIGN_IS_STEADY(sign_b);
+
+ return txn_a < txn_b;
+}
+
+static bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b,
+ uint64_t sign_b) {
+ if (!txn_a || txn_a != txn_b)
+ return false;
+
+ if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b))
+ return false;
+
+ return true;
+}
+
+static int meta_recent(const bool wanna_steady) {
+ if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady))
+ return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
+ envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady)
+ ? 1
+ : 2;
+ else
+ return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, wanna_steady)
+ ? 2
+ : 0;
+}
+
+static int meta_tail(int head) {
+ switch (head) {
+ case 0:
+ return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
+ envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true)
+ ? 1
+ : 2;
+ case 1:
+ return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true)
+ ? 0
+ : 2;
+ case 2:
+ return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true)
+ ? 0
+ : 1;
+ default:
+ assert(false);
+ return -1;
+ }
+}
+
+static int meta_head(void) { return meta_recent(false); }
+
+void verbose_meta(int num, txnid_t txnid, uint64_t sign, uint64_t bootid_x,
+ uint64_t bootid_y) {
+ const bool have_bootid = (bootid_x | bootid_y) != 0;
+ const bool bootid_match = bootid_x == envinfo.mi_bootid.current.x &&
+ bootid_y == envinfo.mi_bootid.current.y;
+
+ print(" - meta-%d: ", num);
+ switch (sign) {
+ case MDBX_DATASIGN_NONE:
+ print("no-sync/legacy");
+ break;
+ case MDBX_DATASIGN_WEAK:
+ print("weak-%s", bootid_match ? (have_bootid ? "intact (same boot-id)"
+ : "unknown (no boot-id")
+ : "dead");
+ break;
+ default:
+ print("steady");
+ break;
+ }
+ print(" txn#%" PRIu64, txnid);
+
+ const int head = meta_head();
+ if (num == head)
+ print(", head");
+ else if (num == meta_tail(head))
+ print(", tail");
+ else
+ print(", stay");
+
+ if (stuck_meta >= 0) {
+ if (num == stuck_meta)
+ print(", forced for checking");
+ } else if (txnid > envinfo.mi_recent_txnid &&
+ (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE)
+ print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")",
+ txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid);
+ print("\n");
+}
+
+static uint64_t get_meta_txnid(const unsigned meta_id) {
+ switch (meta_id) {
+ default:
+ assert(false);
+ error("unexpected meta_id %u\n", meta_id);
+ return 0;
+ case 0:
+ return envinfo.mi_meta0_txnid;
+ case 1:
+ return envinfo.mi_meta1_txnid;
+ case 2:
+ return envinfo.mi_meta2_txnid;
+ }
+}
+
+static void print_size(const char *prefix, const uint64_t value,
+ const char *suffix) {
+ const char sf[] =
+ "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */
+ double k = 1024.0;
+ size_t i;
+ for (i = 0; sf[i + 1] && value / k > 1000.0; ++i)
+ k *= 1024;
+ print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix);
+}
+
+int main(int argc, char *argv[]) {
+ int rc;
+ char *prog = argv[0];
+ char *envname;
+ unsigned problems_maindb = 0, problems_freedb = 0, problems_meta = 0;
+ bool write_locked = false;
+ bool turn_meta = false;
+ bool force_turn_meta = false;
+
+ double elapsed;
+#if defined(_WIN32) || defined(_WIN64)
+ uint64_t timestamp_start, timestamp_finish;
+ timestamp_start = GetMilliseconds();
+#else
+ struct timespec timestamp_start, timestamp_finish;
+ if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) {
+ rc = errno;
+ error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return EXIT_FAILURE_SYS;
+ }
+#endif
+
+ dbi_meta.name = "@META";
+ dbi_free.name = "@GC";
+ dbi_main.name = "@MAIN";
+ atexit(pagemap_cleanup);
+
+ if (argc < 2)
+ usage(prog);
+
+ for (int i; (i = getopt(argc, argv,
+ "0"
+ "1"
+ "2"
+ "T"
+ "V"
+ "v"
+ "q"
+ "n"
+ "w"
+ "c"
+ "t"
+ "d"
+ "i"
+ "s:")) != EOF;) {
+ switch (i) {
+ case 'V':
+ printf("mdbx_chk version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ case 'v':
+ verbose++;
+ break;
+ case '0':
+ stuck_meta = 0;
+ break;
+ case '1':
+ stuck_meta = 1;
+ break;
+ case '2':
+ stuck_meta = 2;
+ break;
+ case 't':
+ turn_meta = true;
+ break;
+ case 'T':
+ turn_meta = force_turn_meta = true;
+ quiet = false;
+ if (verbose < 2)
+ verbose = 2;
+ break;
+ case 'q':
+ quiet = true;
+ break;
+ case 'n':
+ break;
+ case 'w':
+ envflags &= ~MDBX_RDONLY;
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ /* Temporary `workaround` for OpenBSD kernel's flaw.
+ * See todo4recovery://erased_by_github/libmdbx/issues/67 */
+ envflags |= MDBX_WRITEMAP;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+ break;
+ case 'c':
+ envflags = (envflags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE;
+ break;
+ case 'd':
+ dont_traversal = true;
+ break;
+ case 's':
+ if (only_subdb && strcmp(only_subdb, optarg))
+ usage(prog);
+ only_subdb = optarg;
+ break;
+ case 'i':
+ ignore_wrong_order = true;
+ break;
+ default:
+ usage(prog);
+ }
+ }
+
+ if (optind != argc - 1)
+ usage(prog);
+
+ rc = MDBX_SUCCESS;
+ if (stuck_meta >= 0 && (envflags & MDBX_EXCLUSIVE) == 0) {
+ error("exclusive mode is required to using specific meta-page(%d) for "
+ "checking.\n",
+ stuck_meta);
+ rc = EXIT_INTERRUPTED;
+ }
+ if (turn_meta) {
+ if (stuck_meta < 0) {
+ error("meta-page must be specified (by -0, -1 or -2 options) to turn to "
+ "it.\n");
+ rc = EXIT_INTERRUPTED;
+ }
+ if (envflags & MDBX_RDONLY) {
+ error("write-mode must be enabled to turn to the specified meta-page.\n");
+ rc = EXIT_INTERRUPTED;
+ }
+ if (only_subdb || dont_traversal) {
+ error("whole database checking with tree-traversal are required to turn "
+ "to the specified meta-page.\n");
+ rc = EXIT_INTERRUPTED;
+ }
+ }
+ if (rc)
+ exit(rc);
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ envname = argv[optind];
+ print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, envname,
+ (envflags & MDBX_RDONLY) ? "only" : "write");
+ fflush(nullptr);
+ mdbx_setup_debug((verbose < MDBX_LOG_TRACE - 1)
+ ? (MDBX_log_level_t)(verbose + 1)
+ : MDBX_LOG_TRACE,
+ MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, logger);
+
+ rc = mdbx_env_create(&env);
+ if (rc) {
+ error("mdbx_env_create() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return rc < 0 ? EXIT_FAILURE_MDBX : EXIT_FAILURE_SYS;
+ }
+
+ rc = mdbx_env_set_maxdbs(env, MDBX_MAX_DBI);
+ if (rc) {
+ error("mdbx_env_set_maxdbs() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ if (stuck_meta >= 0) {
+ rc = mdbx_env_open_for_recovery(env, envname, stuck_meta,
+ (envflags & MDBX_RDONLY) ? false : true);
+ } else {
+ rc = mdbx_env_open(env, envname, envflags, 0);
+ if ((envflags & MDBX_EXCLUSIVE) &&
+ (rc == MDBX_BUSY ||
+#if defined(_WIN32) || defined(_WIN64)
+ rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION
+#else
+ rc == EBUSY || rc == EAGAIN
+#endif
+ )) {
+ envflags &= ~MDBX_EXCLUSIVE;
+ rc = mdbx_env_open(env, envname, envflags | MDBX_ACCEDE, 0);
+ }
+ }
+
+ if (rc) {
+ error("mdbx_env_open() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY))
+ print("Please run %s in the read-write mode (with '-w' option).\n", prog);
+ goto bailout;
+ }
+ if (verbose)
+ print(" - %s mode\n",
+ (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative");
+
+ if ((envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0) {
+ rc = mdbx_txn_lock(env, false);
+ if (rc != MDBX_SUCCESS) {
+ error("mdbx_txn_lock() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+ write_locked = true;
+ }
+
+ rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
+ if (rc) {
+ error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo));
+ if (rc) {
+ error("mdbx_env_info_ex() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+ if (verbose) {
+ print(" - current boot-id ");
+ if (envinfo.mi_bootid.current.x | envinfo.mi_bootid.current.y)
+ print("%016" PRIx64 "-%016" PRIx64 "\n", envinfo.mi_bootid.current.x,
+ envinfo.mi_bootid.current.y);
+ else
+ print("unavailable\n");
+ }
+
+ mdbx_filehandle_t dxb_fd;
+ rc = mdbx_env_get_fd(env, &dxb_fd);
+ if (rc) {
+ error("mdbx_env_get_fd() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ uint64_t dxb_filesize = 0;
+#if defined(_WIN32) || defined(_WIN64)
+ {
+ BY_HANDLE_FILE_INFORMATION info;
+ if (!GetFileInformationByHandle(dxb_fd, &info))
+ rc = GetLastError();
+ else
+ dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
+ }
+#else
+ {
+ struct stat st;
+ STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t),
+ "libmdbx requires 64-bit file I/O on 64-bit systems");
+ if (fstat(dxb_fd, &st))
+ rc = errno;
+ else
+ dxb_filesize = st.st_size;
+ }
+#endif
+ if (rc) {
+ error("mdbx_filesize() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ errno = 0;
+ const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize;
+ alloc_pages = txn->mt_next_pgno;
+ backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize;
+ if (backed_pages > dxbfile_pages) {
+ print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
+ backed_pages, dxbfile_pages);
+ ++problems_meta;
+ }
+ if (dxbfile_pages < NUM_METAS)
+ print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS);
+ if (backed_pages < NUM_METAS)
+ print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS);
+ if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS)
+ goto bailout;
+ if (backed_pages > MAX_PAGENO + 1) {
+ print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n",
+ backed_pages, MAX_PAGENO + 1);
+ ++problems_meta;
+ backed_pages = MAX_PAGENO + 1;
+ }
+
+ if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) {
+ if (backed_pages > dxbfile_pages) {
+ print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
+ backed_pages, dxbfile_pages);
+ ++problems_meta;
+ backed_pages = dxbfile_pages;
+ }
+ if (alloc_pages > backed_pages) {
+ print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n",
+ alloc_pages, backed_pages);
+ ++problems_meta;
+ alloc_pages = backed_pages;
+ }
+ } else {
+ /* LY: DB may be shrinked by writer down to the allocated pages. */
+ if (alloc_pages > backed_pages) {
+ print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n",
+ alloc_pages, backed_pages);
+ ++problems_meta;
+ alloc_pages = backed_pages;
+ }
+ if (alloc_pages > dxbfile_pages) {
+ print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
+ alloc_pages, dxbfile_pages);
+ ++problems_meta;
+ alloc_pages = dxbfile_pages;
+ }
+ if (backed_pages > dxbfile_pages)
+ backed_pages = dxbfile_pages;
+ }
+
+ if (verbose) {
+ print(" - pagesize %u (%u system), max keysize %d..%d"
+ ", max readers %u\n",
+ envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize,
+ mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT),
+ mdbx_env_get_maxkeysize_ex(env, 0), envinfo.mi_maxreaders);
+ print_size(" - mapsize ", envinfo.mi_mapsize, "\n");
+ if (envinfo.mi_geo.lower == envinfo.mi_geo.upper)
+ print_size(" - fixed datafile: ", envinfo.mi_geo.current, "");
+ else {
+ print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, "");
+ print_size(" .. ", envinfo.mi_geo.upper, ", ");
+ print_size("+", envinfo.mi_geo.grow, ", ");
+ print_size("-", envinfo.mi_geo.shrink, "\n");
+ print_size(" - current datafile: ", envinfo.mi_geo.current, "");
+ }
+ printf(", %" PRIu64 " pages\n",
+ envinfo.mi_geo.current / envinfo.mi_dxb_pagesize);
+#if defined(_WIN32) || defined(_WIN64)
+ if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper)
+ print(
+ " WARNING: Due Windows system limitations a "
+ "file couldn't\n be truncated while the database "
+ "is opened. So, the size\n database file "
+ "of may by large than the database itself,\n "
+ "until it will be closed or reopened in read-write mode.\n");
+#endif
+ verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_bootid.meta0.x, envinfo.mi_bootid.meta0.y);
+ verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
+ envinfo.mi_bootid.meta1.x, envinfo.mi_bootid.meta1.y);
+ verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
+ envinfo.mi_bootid.meta2.x, envinfo.mi_bootid.meta2.y);
+ }
+
+ if (stuck_meta >= 0) {
+ if (verbose) {
+ print(" - skip checking meta-pages since the %u"
+ " is selected for verification\n",
+ stuck_meta);
+ print(" - transactions: recent %" PRIu64
+ ", selected for verification %" PRIu64 ", lag %" PRIi64 "\n",
+ envinfo.mi_recent_txnid, get_meta_txnid(stuck_meta),
+ envinfo.mi_recent_txnid - get_meta_txnid(stuck_meta));
+ }
+ } else {
+ if (verbose > 1)
+ print(" - performs check for meta-pages clashes\n");
+ if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
+ envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) {
+ print(" ! meta-%d and meta-%d are clashed\n", 0, 1);
+ ++problems_meta;
+ }
+ if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
+ envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) {
+ print(" ! meta-%d and meta-%d are clashed\n", 1, 2);
+ ++problems_meta;
+ }
+ if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
+ envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) {
+ print(" ! meta-%d and meta-%d are clashed\n", 2, 0);
+ ++problems_meta;
+ }
+
+ const unsigned steady_meta_id = meta_recent(true);
+ const uint64_t steady_meta_txnid = get_meta_txnid(steady_meta_id);
+ const unsigned weak_meta_id = meta_recent(false);
+ const uint64_t weak_meta_txnid = get_meta_txnid(weak_meta_id);
+ if (envflags & MDBX_EXCLUSIVE) {
+ if (verbose > 1)
+ print(" - performs full check recent-txn-id with meta-pages\n");
+ if (steady_meta_txnid != envinfo.mi_recent_txnid) {
+ print(" ! steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64
+ " != %" PRIi64 ")\n",
+ steady_meta_id, steady_meta_txnid, envinfo.mi_recent_txnid);
+ ++problems_meta;
+ }
+ } else if (write_locked) {
+ if (verbose > 1)
+ print(" - performs lite check recent-txn-id with meta-pages (not a "
+ "monopolistic mode)\n");
+ if (weak_meta_txnid != envinfo.mi_recent_txnid) {
+ print(" ! weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64
+ " != %" PRIi64 ")\n",
+ weak_meta_id, weak_meta_txnid, envinfo.mi_recent_txnid);
+ ++problems_meta;
+ }
+ } else if (verbose) {
+ print(" - skip check recent-txn-id with meta-pages (monopolistic or "
+ "read-write mode only)\n");
+ }
+ total_problems += problems_meta;
+
+ if (verbose)
+ print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64
+ ", lag %" PRIi64 "\n",
+ envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid,
+ envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid);
+ }
+
+ if (!dont_traversal) {
+ struct problem *saved_list;
+ size_t traversal_problems;
+ uint64_t empty_pages, lost_bytes;
+
+ print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid);
+ fflush(nullptr);
+ walk.pagemap = mdbx_calloc((size_t)backed_pages, sizeof(*walk.pagemap));
+ if (!walk.pagemap) {
+ rc = errno ? errno : MDBX_ENOMEM;
+ error("calloc() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ saved_list = problems_push();
+ rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr,
+ true /* always skip key ordering checking to avoid
+ MDBX_CORRUPTED when using custom comparators */);
+ traversal_problems = problems_pop(saved_list);
+
+ if (rc) {
+ if (rc != MDBX_EINTR || !check_user_break())
+ error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ goto bailout;
+ }
+
+ for (uint64_t n = 0; n < alloc_pages; ++n)
+ if (!walk.pagemap[n])
+ unused_pages += 1;
+
+ empty_pages = lost_bytes = 0;
+ for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name;
+ ++dbi) {
+ empty_pages += dbi->pages.empty;
+ lost_bytes += dbi->lost_bytes;
+ }
+
+ if (verbose) {
+ uint64_t total_page_bytes = walk.pgcount * envinfo.mi_dxb_pagesize;
+ print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n",
+ walk.pgcount, unused_pages);
+ if (verbose > 1) {
+ for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
+ ++dbi) {
+ print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total);
+ if (dbi->pages.other && dbi->pages.other != dbi->pages.total)
+ print(", other %" PRIu64, dbi->pages.other);
+ if (dbi->pages.branch)
+ print(", branch %" PRIu64, dbi->pages.branch);
+ if (dbi->pages.large_count)
+ print(", large %" PRIu64, dbi->pages.large_count);
+ uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed;
+ if (all_leaf) {
+ print(", leaf %" PRIu64, all_leaf);
+ if (verbose > 2 &&
+ (dbi->pages.subleaf_dupsort | dbi->pages.leaf_dupfixed |
+ dbi->pages.subleaf_dupfixed))
+ print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64
+ ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")",
+ dbi->pages.leaf, dbi->pages.subleaf_dupsort,
+ dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed);
+ }
+ print("\n");
+ }
+ }
+
+ if (verbose > 1)
+ print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64
+ " (%.1f%%), unused "
+ "%" PRIu64 " (%.1f%%)\n",
+ total_page_bytes, walk.total_payload_bytes,
+ walk.total_payload_bytes * 100.0 / total_page_bytes,
+ total_page_bytes - walk.total_payload_bytes,
+ (total_page_bytes - walk.total_payload_bytes) * 100.0 /
+ total_page_bytes);
+ if (verbose > 2) {
+ for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
+ ++dbi)
+ if (dbi->pages.total) {
+ uint64_t dbi_bytes = dbi->pages.total * envinfo.mi_dxb_pagesize;
+ print(" %s: subtotal %" PRIu64 " bytes (%.1f%%),"
+ " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)",
+ dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes,
+ dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes,
+ dbi_bytes - dbi->payload_bytes,
+ (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes);
+ if (dbi->pages.empty)
+ print(", %" PRIu64 " empty pages", dbi->pages.empty);
+ if (dbi->lost_bytes)
+ print(", %" PRIu64 " bytes lost", dbi->lost_bytes);
+ print("\n");
+ } else
+ print(" %s: empty\n", dbi->name);
+ }
+ print(" - summary: average fill %.1f%%",
+ walk.total_payload_bytes * 100.0 / total_page_bytes);
+ if (empty_pages)
+ print(", %" PRIu64 " empty pages", empty_pages);
+ if (lost_bytes)
+ print(", %" PRIu64 " bytes lost", lost_bytes);
+ print(", %" PRIuPTR " problems\n", traversal_problems);
+ }
+ } else if (verbose) {
+ print("Skipping b-tree walk...\n");
+ fflush(nullptr);
+ }
+
+ if (!verbose)
+ print("Iterating DBIs...\n");
+ if (data_tree_problems) {
+ print("Skip processing %s since tree is corrupted (%u problems)\n", "@MAIN",
+ data_tree_problems);
+ problems_maindb = data_tree_problems;
+ } else
+ problems_maindb = process_db(~0u, /* MAIN_DBI */ nullptr, nullptr, false);
+
+ if (gc_tree_problems) {
+ print("Skip processing %s since tree is corrupted (%u problems)\n", "@GC",
+ gc_tree_problems);
+ problems_freedb = gc_tree_problems;
+ } else
+ problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb, false);
+
+ if (verbose) {
+ uint64_t value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize;
+ double percent = value / 100.0;
+ print(" - space: %" PRIu64 " total pages", value);
+ print(", backed %" PRIu64 " (%.1f%%)", backed_pages,
+ backed_pages / percent);
+ print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages,
+ alloc_pages / percent);
+
+ if (verbose > 1) {
+ value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages;
+ print(", remained %" PRIu64 " (%.1f%%)", value, value / percent);
+
+ value = dont_traversal ? alloc_pages - gc_pages : walk.pgcount;
+ print(", used %" PRIu64 " (%.1f%%)", value, value / percent);
+
+ print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent);
+
+ value = gc_pages - reclaimable_pages;
+ print(", detained %" PRIu64 " (%.1f%%)", value, value / percent);
+
+ print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages,
+ reclaimable_pages / percent);
+ }
+
+ value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages +
+ reclaimable_pages;
+ print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent);
+ }
+
+ if (problems_maindb == 0 && problems_freedb == 0) {
+ if (!dont_traversal &&
+ (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) {
+ if (walk.pgcount != alloc_pages - gc_pages) {
+ error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64
+ "(allocated - GC))\n",
+ walk.pgcount, alloc_pages - gc_pages);
+ }
+ if (unused_pages != gc_pages) {
+ error("gc pages mismatch (%" PRIu64 "(expected) != %" PRIu64 "(GC))\n",
+ unused_pages, gc_pages);
+ }
+ } else if (verbose) {
+ print(" - skip check used and gc pages (btree-traversal with "
+ "monopolistic or read-write mode only)\n");
+ }
+
+ if (!process_db(MAIN_DBI, nullptr, handle_maindb, true)) {
+ if (!userdb_count && verbose)
+ print(" - does not contain multiple databases\n");
+ }
+ }
+
+ if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal &&
+ (envflags & MDBX_RDONLY) == 0 && !only_subdb && stuck_meta < 0 &&
+ get_meta_txnid(meta_recent(true)) < envinfo.mi_recent_txnid) {
+ print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64
+ "\n",
+ envinfo.mi_recent_txnid);
+ fflush(nullptr);
+ if (write_locked) {
+ mdbx_txn_unlock(env);
+ write_locked = false;
+ }
+ rc = mdbx_env_sync_ex(env, true, false);
+ if (rc != MDBX_SUCCESS)
+ error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ else {
+ total_problems -= 1;
+ problems_meta -= 1;
+ }
+ }
+
+ if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb &&
+ (envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) {
+ const bool successful_check = (rc | total_problems | problems_meta) == 0;
+ if (successful_check || force_turn_meta) {
+ fflush(nullptr);
+ print(" = Performing turn to the specified meta-page (%d) due to %s!\n",
+ stuck_meta,
+ successful_check ? "successful check" : "the -T option was given");
+ fflush(nullptr);
+ rc = mdbx_env_turn_for_recovery(env, stuck_meta);
+ if (rc != MDBX_SUCCESS)
+ error("mdbx_env_turn_for_recovery() failed, error %d %s\n", rc,
+ mdbx_strerror(rc));
+ } else {
+ print(" = Skipping turn to the specified meta-page (%d) due to "
+ "unsuccessful check!\n",
+ stuck_meta);
+ }
+ }
+
+bailout:
+ if (txn)
+ mdbx_txn_abort(txn);
+ if (write_locked) {
+ mdbx_txn_unlock(env);
+ write_locked = false;
+ }
+ if (env) {
+ const bool dont_sync = rc != 0 || total_problems;
+ mdbx_env_close_ex(env, dont_sync);
+ }
+ fflush(nullptr);
+ if (rc) {
+ if (rc < 0)
+ return user_break ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS;
+ return EXIT_FAILURE_MDBX;
+ }
+
+#if defined(_WIN32) || defined(_WIN64)
+ timestamp_finish = GetMilliseconds();
+ elapsed = (timestamp_finish - timestamp_start) * 1e-3;
+#else
+ if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) {
+ rc = errno;
+ error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc));
+ return EXIT_FAILURE_SYS;
+ }
+ elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec +
+ (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9;
+#endif /* !WINDOWS */
+
+ if (total_problems) {
+ print("Total %u error%s detected, elapsed %.3f seconds.\n", total_problems,
+ (total_problems > 1) ? "s are" : " is", elapsed);
+ if (problems_meta || problems_maindb || problems_freedb)
+ return EXIT_FAILURE_CHECK_MAJOR;
+ return EXIT_FAILURE_CHECK_MINOR;
+ }
+ print("No error is detected, elapsed %.3f seconds\n", elapsed);
+ return EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/mdbx_copy.c b/3rdparty/libmdbx/mdbx_copy.c
new file mode 100644
index 0000000000000000000000000000000000000000..8cfc4f5cfff71f0135b10d9c9df78d9feda35a37
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_copy.c
@@ -0,0 +1,3660 @@
+/* mdbx_copy.c - memory-mapped database backup tool */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = true;
+ return true;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+static void usage(const char *prog) {
+ fprintf(stderr,
+ "usage: %s [-V] [-q] [-c] src_path [dest_path]\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -c\t\tenable compactification (skip unused pages)\n"
+ " src_path\tsource database\n"
+ " dest_path\tdestination (stdout if not specified)\n",
+ prog);
+ exit(EXIT_FAILURE);
+}
+
+int main(int argc, char *argv[]) {
+ int rc;
+ MDBX_env *env = NULL;
+ const char *progname = argv[0], *act;
+ unsigned flags = MDBX_RDONLY;
+ unsigned cpflags = 0;
+ bool quiet = false;
+
+ for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
+ if (argv[1][1] == 'n' && argv[1][2] == '\0')
+ flags |= MDBX_NOSUBDIR;
+ else if (argv[1][1] == 'c' && argv[1][2] == '\0')
+ cpflags |= MDBX_CP_COMPACT;
+ else if (argv[1][1] == 'q' && argv[1][2] == '\0')
+ quiet = true;
+ else if ((argv[1][1] == 'h' && argv[1][2] == '\0') ||
+ strcmp(argv[1], "--help") == 0)
+ usage(progname);
+ else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
+ printf("mdbx_copy version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ } else
+ argc = 0;
+ }
+
+ if (argc < 2 || argc > 3)
+ usage(progname);
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ if (!quiet) {
+ fprintf((argc == 2) ? stderr : stdout,
+ "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]);
+ fflush(NULL);
+ }
+
+ act = "opening environment";
+ rc = mdbx_env_create(&env);
+ if (rc == MDBX_SUCCESS)
+ rc = mdbx_env_open(env, argv[1], flags, 0);
+
+ if (rc == MDBX_SUCCESS) {
+ act = "copying";
+ if (argc == 2) {
+ mdbx_filehandle_t fd;
+#if defined(_WIN32) || defined(_WIN64)
+ fd = GetStdHandle(STD_OUTPUT_HANDLE);
+#else
+ fd = fileno(stdout);
+#endif
+ rc = mdbx_env_copy2fd(env, fd, cpflags);
+ } else
+ rc = mdbx_env_copy(env, argv[2], cpflags);
+ }
+ if (rc)
+ fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
+ mdbx_strerror(rc));
+ mdbx_env_close(env);
+
+ return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/mdbx_drop.c b/3rdparty/libmdbx/mdbx_drop.c
new file mode 100644
index 0000000000000000000000000000000000000000..9b0487161c459e4f5382b73ce67c602d3958bc62
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_drop.c
@@ -0,0 +1,3715 @@
+/* mdbx_drop.c - memory-mapped database delete tool */
+
+/*
+ * Copyright 2021 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ *
+ * Copyright 2016-2022 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#include <ctype.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = true;
+ return true;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+static char *prog;
+bool quiet = false;
+static void usage(void) {
+ fprintf(stderr,
+ "usage: %s [-V] [-q] [-d] [-s name] dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -d\t\tdelete the specified database, don't just empty it\n"
+ " -s name\tdrop the specified named subDB\n"
+ " \t\tby default empty the main DB\n",
+ prog);
+ exit(EXIT_FAILURE);
+}
+
+static void error(const char *func, int rc) {
+ if (!quiet)
+ fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
+ mdbx_strerror(rc));
+}
+
+int main(int argc, char *argv[]) {
+ int i, rc;
+ MDBX_env *env;
+ MDBX_txn *txn;
+ MDBX_dbi dbi;
+ char *envname = nullptr;
+ char *subname = nullptr;
+ int envflags = MDBX_ACCEDE;
+ bool delete = false;
+
+ prog = argv[0];
+ if (argc < 2)
+ usage();
+
+ while ((i = getopt(argc, argv,
+ "d"
+ "s:"
+ "n"
+ "q"
+ "V")) != EOF) {
+ switch (i) {
+ case 'V':
+ printf("mdbx_drop version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ case 'q':
+ quiet = true;
+ break;
+ case 'd':
+ delete = true;
+ break;
+ case 'n':
+ break;
+ case 's':
+ subname = optarg;
+ break;
+ default:
+ usage();
+ }
+ }
+
+ if (optind != argc - 1)
+ usage();
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ envname = argv[optind];
+ if (!quiet) {
+ printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, envname, subname ? subname : "@MAIN");
+ fflush(nullptr);
+ }
+
+ rc = mdbx_env_create(&env);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_create", rc);
+ return EXIT_FAILURE;
+ }
+
+ if (subname) {
+ rc = mdbx_env_set_maxdbs(env, 2);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_set_maxdbs", rc);
+ goto env_close;
+ }
+ }
+
+ rc = mdbx_env_open(env, envname, envflags, 0);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_open", rc);
+ goto env_close;
+ }
+
+ rc = mdbx_txn_begin(env, NULL, 0, &txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_begin", rc);
+ goto env_close;
+ }
+
+ rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_open", rc);
+ goto txn_abort;
+ }
+
+ rc = mdbx_drop(txn, dbi, delete);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_drop", rc);
+ goto txn_abort;
+ }
+
+ rc = mdbx_txn_commit(txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_commit", rc);
+ goto txn_abort;
+ }
+ txn = nullptr;
+
+txn_abort:
+ if (txn)
+ mdbx_txn_abort(txn);
+env_close:
+ mdbx_env_close(env);
+
+ return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/mdbx_dump.c b/3rdparty/libmdbx/mdbx_dump.c
new file mode 100644
index 0000000000000000000000000000000000000000..e806fa8c99ab4922fc7949138e6c921ed7b79cdf
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_dump.c
@@ -0,0 +1,4020 @@
+/* mdbx_dump.c - memory-mapped database dump tool */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#include <ctype.h>
+
+#define PRINT 1
+#define GLOBAL 2
+static int mode = GLOBAL;
+
+typedef struct flagbit {
+ int bit;
+ char *name;
+} flagbit;
+
+flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
+ {MDBX_DUPSORT, "dupsort"},
+ {MDBX_INTEGERKEY, "integerkey"},
+ {MDBX_DUPFIXED, "dupfixed"},
+ {MDBX_INTEGERDUP, "integerdup"},
+ {MDBX_REVERSEDUP, "reversedup"},
+ {0, nullptr}};
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = true;
+ return true;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+static const char hexc[] = "0123456789abcdef";
+
+static void dumpbyte(unsigned char c) {
+ putchar(hexc[c >> 4]);
+ putchar(hexc[c & 0xf]);
+}
+
+static void text(MDBX_val *v) {
+ unsigned char *c, *end;
+
+ putchar(' ');
+ c = v->iov_base;
+ end = c + v->iov_len;
+ while (c < end) {
+ if (isprint(*c) && *c != '\\') {
+ putchar(*c);
+ } else {
+ putchar('\\');
+ dumpbyte(*c);
+ }
+ c++;
+ }
+ putchar('\n');
+}
+
+static void dumpval(MDBX_val *v) {
+ unsigned char *c, *end;
+
+ putchar(' ');
+ c = v->iov_base;
+ end = c + v->iov_len;
+ while (c < end)
+ dumpbyte(*c++);
+ putchar('\n');
+}
+
+bool quiet = false, rescue = false;
+const char *prog;
+static void error(const char *func, int rc) {
+ if (!quiet)
+ fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
+ mdbx_strerror(rc));
+}
+
+/* Dump in BDB-compatible format */
+static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
+ unsigned int flags;
+ int rc = mdbx_dbi_flags(txn, dbi, &flags);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_flags", rc);
+ return rc;
+ }
+
+ MDBX_stat ms;
+ rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_stat", rc);
+ return rc;
+ }
+
+ MDBX_envinfo info;
+ rc = mdbx_env_info_ex(mdbx_txn_env(txn), txn, &info, sizeof(info));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_info_ex", rc);
+ return rc;
+ }
+
+ printf("VERSION=3\n");
+ if (mode & GLOBAL) {
+ mode -= GLOBAL;
+ if (info.mi_geo.upper != info.mi_geo.lower)
+ printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
+ ",g%" PRIu64 "\n",
+ info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper,
+ info.mi_geo.shrink, info.mi_geo.grow);
+ printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper);
+ printf("maxreaders=%u\n", info.mi_maxreaders);
+
+ MDBX_canary canary;
+ rc = mdbx_canary_get(txn, &canary);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_canary_get", rc);
+ return rc;
+ }
+ if (canary.v)
+ printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n",
+ canary.v, canary.x, canary.y, canary.z);
+ }
+ printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
+ if (name)
+ printf("database=%s\n", name);
+ printf("type=btree\n");
+ printf("db_pagesize=%u\n", ms.ms_psize);
+ /* if (ms.ms_mod_txnid)
+ printf("txnid=%" PRIaTXN "\n", ms.ms_mod_txnid);
+ else if (!name)
+ printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */
+
+ printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED |
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP))
+ ? 1
+ : 0);
+ for (int i = 0; dbflags[i].bit; i++)
+ if (flags & dbflags[i].bit)
+ printf("%s=1\n", dbflags[i].name);
+
+ uint64_t sequence;
+ rc = mdbx_dbi_sequence(txn, dbi, &sequence, 0);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_sequence", rc);
+ return rc;
+ }
+ if (sequence)
+ printf("sequence=%" PRIu64 "\n", sequence);
+
+ printf("HEADER=END\n"); /*-------------------------------------------------*/
+
+ MDBX_cursor *cursor;
+ MDBX_val key, data;
+ rc = mdbx_cursor_open(txn, dbi, &cursor);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_open", rc);
+ return rc;
+ }
+ if (MDBX_DEBUG > 0 && rescue) {
+ cursor->mc_flags |= C_SKIPORD;
+ if (cursor->mc_xcursor)
+ cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
+ }
+
+ while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
+ MDBX_SUCCESS) {
+ if (user_break) {
+ rc = MDBX_EINTR;
+ break;
+ }
+ if (mode & PRINT) {
+ text(&key);
+ text(&data);
+ } else {
+ dumpval(&key);
+ dumpval(&data);
+ }
+ }
+ printf("DATA=END\n");
+ if (rc == MDBX_NOTFOUND)
+ rc = MDBX_SUCCESS;
+ if (unlikely(rc != MDBX_SUCCESS))
+ error("mdbx_cursor_get", rc);
+
+ mdbx_cursor_close(cursor);
+ return rc;
+}
+
+static void usage(void) {
+ fprintf(stderr,
+ "usage: %s [-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s subdb] "
+ "dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -f\t\twrite to file instead of stdout\n"
+ " -l\t\tlist subDBs and exit\n"
+ " -p\t\tuse printable characters\n"
+ " -r\t\trescue mode (ignore errors to dump corrupted DB)\n"
+ " -a\t\tdump main DB and all subDBs\n"
+ " -s name\tdump only the specified named subDB\n"
+ " \t\tby default dump only the main DB\n",
+ prog);
+ exit(EXIT_FAILURE);
+}
+
+static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
+ return (a->iov_len == b->iov_len &&
+ memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
+ ? 0
+ : 1;
+}
+
+int main(int argc, char *argv[]) {
+ int i, rc;
+ MDBX_env *env;
+ MDBX_txn *txn;
+ MDBX_dbi dbi;
+ prog = argv[0];
+ char *envname;
+ char *subname = nullptr, *buf4free = nullptr;
+ unsigned envflags = 0;
+ bool alldbs = false, list = false;
+
+ if (argc < 2)
+ usage();
+
+ while ((i = getopt(argc, argv,
+ "a"
+ "f:"
+ "l"
+ "n"
+ "p"
+ "s:"
+ "V"
+ "r"
+ "q")) != EOF) {
+ switch (i) {
+ case 'V':
+ printf("mdbx_dump version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ case 'l':
+ list = true;
+ /*FALLTHROUGH*/;
+ __fallthrough;
+ case 'a':
+ if (subname)
+ usage();
+ alldbs = true;
+ break;
+ case 'f':
+ if (freopen(optarg, "w", stdout) == nullptr) {
+ fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
+ mdbx_strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ break;
+ case 'n':
+ break;
+ case 'p':
+ mode |= PRINT;
+ break;
+ case 's':
+ if (alldbs)
+ usage();
+ subname = optarg;
+ break;
+ case 'q':
+ quiet = true;
+ break;
+ case 'r':
+ rescue = true;
+ break;
+ default:
+ usage();
+ }
+ }
+
+ if (optind != argc - 1)
+ usage();
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ envname = argv[optind];
+ if (!quiet) {
+ fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, envname);
+ fflush(nullptr);
+ }
+
+ rc = mdbx_env_create(&env);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_create", rc);
+ return EXIT_FAILURE;
+ }
+
+ if (alldbs || subname) {
+ rc = mdbx_env_set_maxdbs(env, 2);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_set_maxdbs", rc);
+ goto env_close;
+ }
+ }
+
+ rc = mdbx_env_open(
+ env, envname,
+ envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE : MDBX_RDONLY), 0);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_open", rc);
+ goto env_close;
+ }
+
+ rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_begin", rc);
+ goto env_close;
+ }
+
+ rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_open", rc);
+ goto txn_abort;
+ }
+
+ if (alldbs) {
+ assert(dbi == MAIN_DBI);
+
+ MDBX_cursor *cursor;
+ rc = mdbx_cursor_open(txn, MAIN_DBI, &cursor);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_open", rc);
+ goto txn_abort;
+ }
+ if (MDBX_DEBUG > 0 && rescue) {
+ cursor->mc_flags |= C_SKIPORD;
+ if (cursor->mc_xcursor)
+ cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
+ }
+
+ bool have_raw = false;
+ int count = 0;
+ MDBX_val key;
+ while (MDBX_SUCCESS ==
+ (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
+ if (user_break) {
+ rc = MDBX_EINTR;
+ break;
+ }
+
+ if (memchr(key.iov_base, '\0', key.iov_len))
+ continue;
+ subname = mdbx_realloc(buf4free, key.iov_len + 1);
+ if (!subname) {
+ rc = MDBX_ENOMEM;
+ break;
+ }
+
+ buf4free = subname;
+ memcpy(subname, key.iov_base, key.iov_len);
+ subname[key.iov_len] = '\0';
+
+ MDBX_dbi sub_dbi;
+ rc = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi,
+ rescue ? equal_or_greater : nullptr,
+ rescue ? equal_or_greater : nullptr);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_INCOMPATIBLE) {
+ have_raw = true;
+ continue;
+ }
+ error("mdbx_dbi_open", rc);
+ if (!rescue)
+ break;
+ } else {
+ count++;
+ if (list) {
+ printf("%s\n", subname);
+ } else {
+ rc = dump_sdb(txn, sub_dbi, subname);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (!rescue)
+ break;
+ if (!quiet)
+ fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog,
+ envname, mdbx_strerror(rc), subname);
+ /* Here is a hack for rescue mode, don't do that:
+ * - we should restart transaction in case error due
+ * database corruption;
+ * - but we won't close cursor, reopen and re-positioning it
+ * for new a transaction;
+ * - this is possible since DB is opened in read-only exclusive
+ * mode and transaction is the same, i.e. has the same address
+ * and so on. */
+ rc = mdbx_txn_reset(txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_reset", rc);
+ goto env_close;
+ }
+ rc = mdbx_txn_renew(txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_renew", rc);
+ goto env_close;
+ }
+ }
+ }
+ rc = mdbx_dbi_close(env, sub_dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_close", rc);
+ break;
+ }
+ }
+ }
+ mdbx_cursor_close(cursor);
+ cursor = nullptr;
+
+ if (have_raw && (!count /* || rescue */))
+ rc = dump_sdb(txn, MAIN_DBI, nullptr);
+ else if (!count) {
+ if (!quiet)
+ fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
+ envname);
+ rc = MDBX_NOTFOUND;
+ }
+ } else {
+ rc = dump_sdb(txn, dbi, subname);
+ }
+
+ switch (rc) {
+ case MDBX_NOTFOUND:
+ rc = MDBX_SUCCESS;
+ case MDBX_SUCCESS:
+ break;
+ case MDBX_EINTR:
+ if (!quiet)
+ fprintf(stderr, "Interrupted by signal/user\n");
+ break;
+ default:
+ if (unlikely(rc != MDBX_SUCCESS))
+ error("mdbx_cursor_get", rc);
+ }
+
+ mdbx_dbi_close(env, dbi);
+txn_abort:
+ mdbx_txn_abort(txn);
+env_close:
+ mdbx_env_close(env);
+ free(buf4free);
+
+ return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/mdbx_load.c b/3rdparty/libmdbx/mdbx_load.c
new file mode 100644
index 0000000000000000000000000000000000000000..098ccd536b976d5f1464d49c0cb5d36f22c6f2af
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_load.c
@@ -0,0 +1,4391 @@
+/* mdbx_load.c - memory-mapped database load tool */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#include <ctype.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = true;
+ return true;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+static char *prog;
+static bool quiet = false;
+static size_t lineno;
+static void error(const char *func, int rc) {
+ if (!quiet) {
+ if (lineno)
+ fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n",
+ prog, lineno, func, rc, mdbx_strerror(rc));
+ else
+ fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
+ mdbx_strerror(rc));
+ }
+}
+
+static char *valstr(char *line, const char *item) {
+ const size_t len = strlen(item);
+ if (strncmp(line, item, len) != 0)
+ return nullptr;
+ if (line[len] != '=') {
+ if (line[len] > ' ')
+ return nullptr;
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog,
+ lineno, item);
+ exit(EXIT_FAILURE);
+ }
+ char *ptr = strchr(line, '\n');
+ if (ptr)
+ *ptr = '\0';
+ return line + len + 1;
+}
+
+static bool valnum(char *line, const char *item, uint64_t *value) {
+ char *str = valstr(line, item);
+ if (!str)
+ return false;
+
+ char *end = nullptr;
+ *value = strtoull(str, &end, 0);
+ if (end && *end) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n",
+ prog, lineno, item);
+ exit(EXIT_FAILURE);
+ }
+ return true;
+}
+
+static bool valbool(char *line, const char *item, bool *value) {
+ uint64_t u64;
+ if (!valnum(line, item, &u64))
+ return false;
+
+ if (u64 > 1) {
+ if (!quiet)
+ fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n",
+ prog, lineno, item);
+ exit(EXIT_FAILURE);
+ }
+ *value = u64 != 0;
+ return true;
+}
+
+/*----------------------------------------------------------------------------*/
+
+static char *subname = nullptr;
+static int dbi_flags;
+static txnid_t txnid;
+static uint64_t sequence;
+static MDBX_canary canary;
+static MDBX_envinfo envinfo;
+
+#define PRINT 1
+#define NOHDR 2
+#define GLOBAL 4
+static int mode = GLOBAL;
+
+static MDBX_val kbuf, dbuf;
+
+#define STRLENOF(s) (sizeof(s) - 1)
+
+typedef struct flagbit {
+ unsigned bit;
+ unsigned len;
+ char *name;
+} flagbit;
+
+#define S(s) STRLENOF(s), s
+
+flagbit dbflags[] = {
+ {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")},
+ {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")},
+ {MDBX_DUPFIXED, S("dupfixed")}, {MDBX_INTEGERDUP, S("integerdup")},
+ {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}};
+
+static int readhdr(void) {
+ /* reset parameters */
+ if (subname) {
+ free(subname);
+ subname = nullptr;
+ }
+ dbi_flags = 0;
+ txnid = 0;
+ sequence = 0;
+
+ while (true) {
+ errno = 0;
+ if (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) == nullptr)
+ return errno ? errno : EOF;
+ if (user_break)
+ return MDBX_EINTR;
+
+ lineno++;
+ uint64_t u64;
+
+ if (valnum(dbuf.iov_base, "VERSION", &u64)) {
+ if (u64 != 3) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unsupported value %" PRIu64
+ " for %s\n",
+ prog, lineno, u64, "VERSION");
+ exit(EXIT_FAILURE);
+ }
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "db_pagesize", &u64)) {
+ if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore value %" PRIu64
+ " for '%s' in non-global context\n",
+ prog, lineno, u64, "db_pagesize");
+ } else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64
+ " for %s\n",
+ prog, lineno, u64, "db_pagesize");
+ } else
+ envinfo.mi_dxb_pagesize = (uint32_t)u64;
+ continue;
+ }
+
+ char *str = valstr(dbuf.iov_base, "format");
+ if (str) {
+ if (strcmp(str, "print") == 0) {
+ mode |= PRINT;
+ continue;
+ }
+ if (strcmp(str, "bytevalue") == 0) {
+ mode &= ~PRINT;
+ continue;
+ }
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog,
+ lineno, str, "format");
+ exit(EXIT_FAILURE);
+ }
+
+ str = valstr(dbuf.iov_base, "database");
+ if (str) {
+ if (*str) {
+ free(subname);
+ subname = mdbx_strdup(str);
+ if (!subname) {
+ if (!quiet)
+ perror("strdup()");
+ exit(EXIT_FAILURE);
+ }
+ }
+ continue;
+ }
+
+ str = valstr(dbuf.iov_base, "type");
+ if (str) {
+ if (strcmp(str, "btree") != 0) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n",
+ prog, lineno, str, "type");
+ free(subname);
+ exit(EXIT_FAILURE);
+ }
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "mapaddr", &u64)) {
+ if (u64) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
+ " for %s\n",
+ prog, lineno, u64, "mapaddr");
+ }
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "mapsize", &u64)) {
+ if (!(mode & GLOBAL)) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore value %" PRIu64
+ " for '%s' in non-global context\n",
+ prog, lineno, u64, "mapsize");
+ } else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
+ " for %s\n",
+ prog, lineno, u64, "mapsize");
+ } else
+ envinfo.mi_mapsize = (size_t)u64;
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "maxreaders", &u64)) {
+ if (!(mode & GLOBAL)) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore value %" PRIu64
+ " for '%s' in non-global context\n",
+ prog, lineno, u64, "maxreaders");
+ } else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
+ " for %s\n",
+ prog, lineno, u64, "maxreaders");
+ } else
+ envinfo.mi_maxreaders = (int)u64;
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "txnid", &u64)) {
+ if (u64 < MIN_TXNID || u64 > MAX_TXNID) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
+ " for %s\n",
+ prog, lineno, u64, "txnid");
+ } else
+ txnid = u64;
+ continue;
+ }
+
+ if (valnum(dbuf.iov_base, "sequence", &u64)) {
+ sequence = u64;
+ continue;
+ }
+
+ str = valstr(dbuf.iov_base, "geometry");
+ if (str) {
+ if (!(mode & GLOBAL)) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore values %s"
+ " for '%s' in non-global context\n",
+ prog, lineno, str, "geometry");
+ } else if (sscanf(str,
+ "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
+ ",g%" PRIu64,
+ &envinfo.mi_geo.lower, &envinfo.mi_geo.current,
+ &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink,
+ &envinfo.mi_geo.grow) != 5) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
+ prog, lineno, "geometry");
+ exit(EXIT_FAILURE);
+ }
+ continue;
+ }
+
+ str = valstr(dbuf.iov_base, "canary");
+ if (str) {
+ if (!(mode & GLOBAL)) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": ignore values %s"
+ " for '%s' in non-global context\n",
+ prog, lineno, str, "canary");
+ } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64,
+ &canary.v, &canary.x, &canary.y, &canary.z) != 4) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
+ prog, lineno, "canary");
+ exit(EXIT_FAILURE);
+ }
+ continue;
+ }
+
+ for (int i = 0; dbflags[i].bit; i++) {
+ bool value = false;
+ if (valbool(dbuf.iov_base, dbflags[i].name, &value)) {
+ if (value)
+ dbi_flags |= dbflags[i].bit;
+ else
+ dbi_flags &= ~dbflags[i].bit;
+ goto next;
+ }
+ }
+
+ str = valstr(dbuf.iov_base, "HEADER");
+ if (str) {
+ if (strcmp(str, "END") == 0)
+ return MDBX_SUCCESS;
+ }
+
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
+ prog, lineno, (char *)dbuf.iov_base);
+ next:;
+ }
+ return EOF;
+}
+
+static int badend(void) {
+ if (!quiet)
+ fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
+ lineno);
+ return errno ? errno : MDBX_ENODATA;
+}
+
+static __inline int unhex(unsigned char *c2) {
+ int x, c;
+ x = *c2++ & 0x4f;
+ if (x & 0x40)
+ x -= 55;
+ c = x << 4;
+ x = *c2 & 0x4f;
+ if (x & 0x40)
+ x -= 55;
+ c |= x;
+ return c;
+}
+
+__hot static int readline(MDBX_val *out, MDBX_val *buf) {
+ unsigned char *c1, *c2, *end;
+ size_t len, l2;
+ int c;
+
+ if (user_break)
+ return MDBX_EINTR;
+
+ errno = 0;
+ if (!(mode & NOHDR)) {
+ c = fgetc(stdin);
+ if (c == EOF)
+ return errno ? errno : EOF;
+ if (c != ' ') {
+ lineno++;
+ errno = 0;
+ if (fgets(buf->iov_base, (int)buf->iov_len, stdin)) {
+ if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
+ return EOF;
+ }
+ return badend();
+ }
+ }
+ if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == nullptr)
+ return errno ? errno : EOF;
+ lineno++;
+
+ c1 = buf->iov_base;
+ len = strlen((char *)c1);
+ l2 = len;
+
+ /* Is buffer too short? */
+ while (c1[len - 1] != '\n') {
+ buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
+ if (!buf->iov_base) {
+ if (!quiet)
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog,
+ lineno);
+ return MDBX_ENOMEM;
+ }
+ c1 = buf->iov_base;
+ c1 += l2;
+ errno = 0;
+ if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == nullptr)
+ return errno ? errno : EOF;
+ buf->iov_len *= 2;
+ len = strlen((char *)c1);
+ l2 += len;
+ }
+ c1 = c2 = buf->iov_base;
+ len = l2;
+ c1[--len] = '\0';
+ end = c1 + len;
+
+ if (mode & PRINT) {
+ while (c2 < end) {
+ if (unlikely(*c2 == '\\')) {
+ if (c2[1] == '\\') {
+ *c1++ = '\\';
+ } else {
+ if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2]))
+ return badend();
+ *c1++ = (char)unhex(++c2);
+ }
+ c2 += 2;
+ } else {
+ /* copies are redundant when no escapes were used */
+ *c1++ = *c2++;
+ }
+ }
+ } else {
+ /* odd length not allowed */
+ if (len & 1)
+ return badend();
+ while (c2 < end) {
+ if (!isxdigit(*c2) || !isxdigit(c2[1]))
+ return badend();
+ *c1++ = (char)unhex(c2);
+ c2 += 2;
+ }
+ }
+ c2 = out->iov_base = buf->iov_base;
+ out->iov_len = c1 - c2;
+
+ return MDBX_SUCCESS;
+}
+
+static void usage(void) {
+ fprintf(stderr,
+ "usage: %s "
+ "[-V] [-q] [-a] [-f file] [-s name] [-N] [-p] [-T] [-r] [-n] dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -a\t\tappend records in input order (required for custom "
+ "comparators)\n"
+ " -f file\tread from file instead of stdin\n"
+ " -s name\tload into specified named subDB\n"
+ " -N\t\tdon't overwrite existing records when loading, just skip "
+ "ones\n"
+ " -p\t\tpurge subDB before loading\n"
+ " -T\t\tread plaintext\n"
+ " -r\t\trescue mode (ignore errors to load corrupted DB dump)\n"
+ " -n\t\tdon't use subdirectory for newly created database "
+ "(MDBX_NOSUBDIR)\n",
+ prog);
+ exit(EXIT_FAILURE);
+}
+
+static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
+ return (a->iov_len == b->iov_len &&
+ memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
+ ? 0
+ : 1;
+}
+
+int main(int argc, char *argv[]) {
+ int i, rc;
+ MDBX_env *env = nullptr;
+ MDBX_txn *txn = nullptr;
+ MDBX_cursor *mc = nullptr;
+ MDBX_dbi dbi;
+ char *envname = nullptr;
+ int envflags = MDBX_SAFE_NOSYNC | MDBX_ACCEDE, putflags = MDBX_UPSERT;
+ bool rescue = false;
+ bool purge = false;
+
+ prog = argv[0];
+ if (argc < 2)
+ usage();
+
+ while ((i = getopt(argc, argv,
+ "a"
+ "f:"
+ "n"
+ "s:"
+ "N"
+ "p"
+ "T"
+ "V"
+ "r"
+ "q")) != EOF) {
+ switch (i) {
+ case 'V':
+ printf("mdbx_load version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ case 'a':
+ putflags |= MDBX_APPEND;
+ break;
+ case 'f':
+ if (freopen(optarg, "r", stdin) == nullptr) {
+ if (!quiet)
+ fprintf(stderr, "%s: %s: open: %s\n", prog, optarg,
+ mdbx_strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ break;
+ case 'n':
+ envflags |= MDBX_NOSUBDIR;
+ break;
+ case 's':
+ subname = mdbx_strdup(optarg);
+ break;
+ case 'N':
+ putflags |= MDBX_NOOVERWRITE | MDBX_NODUPDATA;
+ break;
+ case 'p':
+ purge = true;
+ break;
+ case 'T':
+ mode |= NOHDR | PRINT;
+ break;
+ case 'q':
+ quiet = true;
+ break;
+ case 'r':
+ rescue = true;
+ break;
+ default:
+ usage();
+ }
+ }
+
+ if (optind != argc - 1)
+ usage();
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ envname = argv[optind];
+ if (!quiet)
+ printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, envname);
+ fflush(nullptr);
+
+ dbuf.iov_len = 4096;
+ dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
+ if (!dbuf.iov_base) {
+ rc = MDBX_ENOMEM;
+ error("value-buffer", rc);
+ goto env_close;
+ }
+
+ /* read first header for mapsize= */
+ if (!(mode & NOHDR)) {
+ rc = readhdr();
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == EOF)
+ rc = MDBX_ENODATA;
+ error("readheader", rc);
+ goto env_close;
+ }
+ }
+
+ rc = mdbx_env_create(&env);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_create", rc);
+ return EXIT_FAILURE;
+ }
+
+ mdbx_env_set_maxdbs(env, 2);
+ if (envinfo.mi_maxreaders) {
+ rc = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_set_maxreaders", rc);
+ goto env_close;
+ }
+ }
+
+ if (envinfo.mi_geo.current | envinfo.mi_mapsize) {
+ if (envinfo.mi_geo.current) {
+ rc = mdbx_env_set_geometry(
+ env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current,
+ (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink,
+ (intptr_t)envinfo.mi_geo.grow,
+ envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
+ } else {
+ if (envinfo.mi_mapsize > MAX_MAPSIZE) {
+ if (!quiet)
+ fprintf(
+ stderr,
+ "Database size is too large for current system (mapsize=%" PRIu64
+ " is great than system-limit %zu)\n",
+ envinfo.mi_mapsize, (size_t)MAX_MAPSIZE);
+ goto env_close;
+ }
+ rc = mdbx_env_set_geometry(
+ env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize,
+ (intptr_t)envinfo.mi_mapsize, 0, 0,
+ envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
+ }
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_set_geometry", rc);
+ goto env_close;
+ }
+ }
+
+ rc = mdbx_env_open(env, envname, envflags, 0664);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_open", rc);
+ goto env_close;
+ }
+
+ kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, 0) + 1;
+ if (kbuf.iov_len >= INTPTR_MAX / 2) {
+ if (!quiet)
+ fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n",
+ kbuf.iov_len);
+ goto env_close;
+ }
+
+ kbuf.iov_base = malloc(kbuf.iov_len);
+ if (!kbuf.iov_base) {
+ rc = MDBX_ENOMEM;
+ error("key-buffer", rc);
+ goto env_close;
+ }
+
+ while (rc == MDBX_SUCCESS) {
+ if (user_break) {
+ rc = MDBX_EINTR;
+ break;
+ }
+
+ rc = mdbx_txn_begin(env, nullptr, 0, &txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_begin", rc);
+ goto env_close;
+ }
+
+ if (mode & GLOBAL) {
+ mode -= GLOBAL;
+ if (canary.v | canary.x | canary.y | canary.z) {
+ rc = mdbx_canary_put(txn, &canary);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_canary_put", rc);
+ goto txn_abort;
+ }
+ }
+ }
+
+ const char *const dbi_name = subname ? subname : "@MAIN";
+ rc =
+ mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
+ (putflags & MDBX_APPEND) ? equal_or_greater : nullptr,
+ (putflags & MDBX_APPEND) ? equal_or_greater : nullptr);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_open_ex", rc);
+ goto txn_abort;
+ }
+
+ uint64_t present_sequence;
+ rc = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_sequence", rc);
+ goto txn_abort;
+ }
+ if (present_sequence > sequence) {
+ if (!quiet)
+ fprintf(stderr,
+ "present sequence for '%s' value (%" PRIu64
+ ") is greater than loaded (%" PRIu64 ")\n",
+ dbi_name, present_sequence, sequence);
+ rc = MDBX_RESULT_TRUE;
+ goto txn_abort;
+ }
+ if (present_sequence < sequence) {
+ rc = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_sequence", rc);
+ goto txn_abort;
+ }
+ }
+
+ if (purge) {
+ rc = mdbx_drop(txn, dbi, false);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_drop", rc);
+ goto txn_abort;
+ }
+ }
+
+ if (putflags & MDBX_APPEND)
+ putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP
+ : putflags & ~MDBX_APPENDDUP;
+
+ rc = mdbx_cursor_open(txn, dbi, &mc);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_open", rc);
+ goto txn_abort;
+ }
+
+ int batch = 0;
+ while (rc == MDBX_SUCCESS) {
+ MDBX_val key, data;
+ rc = readline(&key, &kbuf);
+ if (rc == EOF)
+ break;
+
+ if (rc == MDBX_SUCCESS)
+ rc = readline(&data, &dbuf);
+ if (rc) {
+ if (!quiet)
+ fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
+ prog, lineno);
+ goto txn_abort;
+ }
+
+ rc = mdbx_cursor_put(mc, &key, &data, putflags);
+ if (rc == MDBX_KEYEXIST && putflags)
+ continue;
+ if (rc == MDBX_BAD_VALSIZE && rescue) {
+ if (!quiet)
+ fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno,
+ mdbx_strerror(rc));
+ continue;
+ }
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_put", rc);
+ goto txn_abort;
+ }
+ batch++;
+
+ MDBX_txn_info txn_info;
+ rc = mdbx_txn_info(txn, &txn_info, false);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_info", rc);
+ goto txn_abort;
+ }
+
+ if (batch == 10000 || txn_info.txn_space_dirty > MEGABYTE * 256) {
+ rc = mdbx_txn_commit(txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_commit", rc);
+ goto env_close;
+ }
+ batch = 0;
+
+ rc = mdbx_txn_begin(env, nullptr, 0, &txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_begin", rc);
+ goto env_close;
+ }
+ rc = mdbx_cursor_bind(txn, mc, dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_bind", rc);
+ goto txn_abort;
+ }
+ }
+ }
+
+ mdbx_cursor_close(mc);
+ mc = nullptr;
+ rc = mdbx_txn_commit(txn);
+ txn = nullptr;
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_commit", rc);
+ goto env_close;
+ }
+ if (subname) {
+ assert(dbi != MAIN_DBI);
+ rc = mdbx_dbi_close(env, dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_close", rc);
+ goto env_close;
+ }
+ } else {
+ assert(dbi == MAIN_DBI);
+ }
+
+ /* try read next header */
+ if (!(mode & NOHDR))
+ rc = readhdr();
+ else if (ferror(stdin) || feof(stdin))
+ break;
+ }
+
+ switch (rc) {
+ case EOF:
+ rc = MDBX_SUCCESS;
+ case MDBX_SUCCESS:
+ break;
+ case MDBX_EINTR:
+ if (!quiet)
+ fprintf(stderr, "Interrupted by signal/user\n");
+ break;
+ default:
+ if (unlikely(rc != MDBX_SUCCESS))
+ error("readline", rc);
+ }
+
+txn_abort:
+ mdbx_cursor_close(mc);
+ mdbx_txn_abort(txn);
+env_close:
+ mdbx_env_close(env);
+ free(kbuf.iov_base);
+ free(dbuf.iov_base);
+
+ return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/mdbx_stat.c b/3rdparty/libmdbx/mdbx_stat.c
new file mode 100644
index 0000000000000000000000000000000000000000..68a46191eb4bc3d52b2e9290eea05253a074bc04
--- /dev/null
+++ b/3rdparty/libmdbx/mdbx_stat.c
@@ -0,0 +1,4041 @@
+/* mdbx_stat.c - memory-mapped database status tool */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#ifdef _MSC_VER
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
+#endif /* _MSC_VER (warnings) */
+
+#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>. */
+
+#define MDBX_BUILD_SOURCERY c510b93d78b5848764f976ea67f06d278a7b99f141b4c98ceb095285f834f31c_v0_11_7_0_g40ec559c
+#ifdef MDBX_CONFIG_H
+#include MDBX_CONFIG_H
+#endif
+
+#define LIBMDBX_INTERNALS
+#ifdef xMDBX_TOOLS
+#define MDBX_DEPRECATED
+#endif /* xMDBX_TOOLS */
+
+#ifdef xMDBX_ALLOY
+/* Amalgamated build */
+#define MDBX_INTERNAL_FUNC static
+#define MDBX_INTERNAL_VAR static
+#else
+/* Non-amalgamated build */
+#define MDBX_INTERNAL_FUNC
+#define MDBX_INTERNAL_VAR extern
+#endif /* xMDBX_ALLOY */
+
+/** Disables using GNU/Linux libc extensions.
+ * \ingroup build_option
+ * \note This option couldn't be moved to the options.h since dependant
+ * control macros/defined should be prepared before include the options.h */
+#ifndef MDBX_DISABLE_GNU_SOURCE
+#define MDBX_DISABLE_GNU_SOURCE 0
+#endif
+#if MDBX_DISABLE_GNU_SOURCE
+#undef _GNU_SOURCE
+#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif /* MDBX_DISABLE_GNU_SOURCE */
+
+/*----------------------------------------------------------------------------*/
+
+/* Should be defined before any includes */
+#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
+ !defined(ANDROID)
+#define _FILE_OFFSET_BITS 64
+#endif
+
+#ifdef __APPLE__
+#define _DARWIN_C_SOURCE
+#endif
+
+#ifdef _MSC_VER
+#if _MSC_FULL_VER < 190024234
+/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
+ * Studio 2015 Update 3). But you could remove this #error and try to continue
+ * at your own risk. In such case please don't rise up an issues related ONLY to
+ * old compilers.
+ *
+ * NOTE:
+ * Unfortunately, there are several different builds of "Visual Studio" that
+ * are called "Visual Studio 2015 Update 3".
+ *
+ * The 190024234 is used here because it is minimal version of Visual Studio
+ * that was used for build and testing libmdbx in recent years. Soon this
+ * value will be increased to 19.0.24241.7, since build and testing using
+ * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
+ *
+ * Please ask Microsoft (but not us) for information about version differences
+ * and how to and where you can obtain the latest "Visual Studio 2015" build
+ * with all fixes.
+ */
+#error \
+ "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
+#endif
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif
+#if _MSC_VER > 1913
+#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
+ */
+#endif
+#if _MSC_VER > 1914
+#pragma warning( \
+ disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
+ producing 'defined' has undefined behavior */
+#endif
+#pragma warning(disable : 4710) /* 'xyz': function not inlined */
+#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
+ inline expansion */
+#pragma warning( \
+ disable : 4201) /* nonstandard extension used : nameless struct / union */
+#pragma warning(disable : 4702) /* unreachable code */
+#pragma warning(disable : 4706) /* assignment within conditional expression */
+#pragma warning(disable : 4127) /* conditional expression is constant */
+#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
+ alignment specifier */
+#pragma warning(disable : 4310) /* cast truncates constant value */
+#pragma warning( \
+ disable : 4820) /* bytes padding added after data member for alignment */
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
+ unaligned */
+#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
+ array in struct/union */
+#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
+ aggregate initializer */
+#pragma warning( \
+ disable : 4505) /* unreferenced local function has been removed */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(__GNUC__) && __GNUC__ < 9
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif /* GCC < 9 */
+
+#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* __USE_MINGW_ANSI_STDIO */
+
+#include "mdbx.h"
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#ifndef __GNUC_PREREQ
+# if defined(__GNUC__) && defined(__GNUC_MINOR__)
+# define __GNUC_PREREQ(maj, min) \
+ ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GNUC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+# ifdef __clang__
+# define __CLANG_PREREQ(maj,min) \
+ ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+# else
+# define __CLANG_PREREQ(maj,min) (0)
+# endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __GLIBC_PREREQ
+# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
+# define __GLIBC_PREREQ(maj, min) \
+ ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
+# else
+# define __GLIBC_PREREQ(maj, min) (0)
+# endif
+#endif /* __GLIBC_PREREQ */
+
+#ifndef __has_warning
+# define __has_warning(x) (0)
+#endif
+
+#ifndef __has_include
+# define __has_include(x) (0)
+#endif
+
+#if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__ 1
+#endif
+
+#if __has_feature(address_sanitizer)
+# define __SANITIZE_ADDRESS__ 1
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __extern_C
+# ifdef __cplusplus
+# define __extern_C extern "C"
+# else
+# define __extern_C
+# endif
+#endif /* __extern_C */
+
+#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
+# define nullptr NULL
+#endif
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef __always_inline
+# if defined(__GNUC__) || __has_attribute(__always_inline__)
+# define __always_inline __inline __attribute__((__always_inline__))
+# elif defined(_MSC_VER)
+# define __always_inline __forceinline
+# else
+# define __always_inline
+# endif
+#endif /* __always_inline */
+
+#ifndef __noinline
+# if defined(__GNUC__) || __has_attribute(__noinline__)
+# define __noinline __attribute__((__noinline__))
+# elif defined(_MSC_VER)
+# define __noinline __declspec(noinline)
+# else
+# define __noinline
+# endif
+#endif /* __noinline */
+
+#ifndef __must_check_result
+# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
+# define __must_check_result __attribute__((__warn_unused_result__))
+# else
+# define __must_check_result
+# endif
+#endif /* __must_check_result */
+
+#if !defined(__noop) && !defined(_MSC_VER)
+# define __noop(...) do {} while(0)
+#endif /* __noop */
+
+#if defined(__fallthrough) && \
+ (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
+#undef __fallthrough
+#endif /* __fallthrough workaround for MinGW */
+
+#ifndef __fallthrough
+# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
+ (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
+# define __fallthrough [[fallthrough]]
+# elif __GNUC_PREREQ(7, 0) && \
+ (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
+ (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
+# define __fallthrough __attribute__((__fallthrough__))
+# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
+ __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
+# define __fallthrough [[clang::fallthrough]]
+# else
+# define __fallthrough
+# endif
+#endif /* __fallthrough */
+
+#ifndef __unreachable
+# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
+# define __unreachable() __builtin_unreachable()
+# elif defined(_MSC_VER)
+# define __unreachable() __assume(0)
+# else
+# define __unreachable() __noop()
+# endif
+#endif /* __unreachable */
+
+#ifndef __prefetch
+# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
+# define __prefetch(ptr) __builtin_prefetch(ptr)
+# else
+# define __prefetch(ptr) __noop(ptr)
+# endif
+#endif /* __prefetch */
+
+#ifndef __nothrow
+# if defined(__cplusplus)
+# if __cplusplus < 201703L
+# define __nothrow throw()
+# else
+# define __nothrow noexcept(true)
+# endif /* __cplusplus */
+# elif defined(__GNUC__) || __has_attribute(__nothrow__)
+# define __nothrow __attribute__((__nothrow__))
+# elif defined(_MSC_VER) && defined(__cplusplus)
+# define __nothrow __declspec(nothrow)
+# else
+# define __nothrow
+# endif
+#endif /* __nothrow */
+
+#ifndef __hidden
+# if defined(__GNUC__) || __has_attribute(__visibility__)
+# define __hidden __attribute__((__visibility__("hidden")))
+# else
+# define __hidden
+# endif
+#endif /* __hidden */
+
+#ifndef __optimize
+# if defined(__OPTIMIZE__)
+# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
+# define __optimize(ops) __attribute__((__optimize__(ops)))
+# else
+# define __optimize(ops)
+# endif
+# else
+# define __optimize(ops)
+# endif
+#endif /* __optimize */
+
+#ifndef __hot
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __hot __attribute__((__hot__)) __optimize(3)
+# elif defined(__clang__) && !__has_attribute(__hot_) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put frequently used functions in separate section */
+# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
+# elif defined(__GNUC__) || __has_attribute(__hot__)
+# define __hot __attribute__((__hot__)) __optimize("O3")
+# else
+# define __hot __optimize("O3")
+# endif
+# else
+# define __hot
+# endif
+#endif /* __hot */
+
+#ifndef __cold
+# if defined(__OPTIMIZE__)
+# if defined(__e2k__)
+# define __cold __attribute__((__cold__)) __optimize(1)
+# elif defined(__clang__) && !__has_attribute(cold) \
+ && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
+ /* just put infrequently used functions in separate section */
+# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
+# elif defined(__GNUC__) || __has_attribute(cold)
+# define __cold __attribute__((__cold__)) __optimize("Os")
+# else
+# define __cold __optimize("Os")
+# endif
+# else
+# define __cold
+# endif
+#endif /* __cold */
+
+#ifndef __flatten
+# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
+# define __flatten __attribute__((__flatten__))
+# else
+# define __flatten
+# endif
+#endif /* __flatten */
+
+#ifndef likely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define likely(cond) __builtin_expect(!!(cond), 1)
+# else
+# define likely(x) (!!(x))
+# endif
+#endif /* likely */
+
+#ifndef unlikely
+# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
+# define unlikely(cond) __builtin_expect(!!(cond), 0)
+# else
+# define unlikely(x) (!!(x))
+# endif
+#endif /* unlikely */
+
+#ifndef __anonymous_struct_extension__
+# if defined(__GNUC__)
+# define __anonymous_struct_extension__ __extension__
+# else
+# define __anonymous_struct_extension__
+# endif
+#endif /* __anonymous_struct_extension__ */
+
+#ifndef __Wpedantic_format_voidptr
+ MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
+ __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
+# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
+#endif /* __Wpedantic_format_voidptr */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(MDBX_USE_VALGRIND)
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
+ /* LY: available since Valgrind 3.10 */
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# endif
+#elif !defined(RUNNING_ON_VALGRIND)
+# define VALGRIND_CREATE_MEMPOOL(h,r,z)
+# define VALGRIND_DESTROY_MEMPOOL(h)
+# define VALGRIND_MEMPOOL_TRIM(h,a,s)
+# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
+# define VALGRIND_MEMPOOL_FREE(h,a)
+# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
+# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
+# define VALGRIND_MAKE_MEM_DEFINED(a,s)
+# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
+# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
+# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
+# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
+# define RUNNING_ON_VALGRIND (0)
+#endif /* MDBX_USE_VALGRIND */
+
+#ifdef __SANITIZE_ADDRESS__
+# include <sanitizer/asan_interface.h>
+#elif !defined(ASAN_POISON_MEMORY_REGION)
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ ((void)(addr), (void)(size))
+#endif /* __SANITIZE_ADDRESS__ */
+
+/*----------------------------------------------------------------------------*/
+
+#ifndef ARRAY_LENGTH
+# ifdef __cplusplus
+ template <typename T, size_t N>
+ char (&__ArraySizeHelper(T (&array)[N]))[N];
+# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
+# else
+# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
+# endif
+#endif /* ARRAY_LENGTH */
+
+#ifndef ARRAY_END
+# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
+#endif /* ARRAY_END */
+
+#define CONCAT(a,b) a##b
+#define XCONCAT(a,b) CONCAT(a,b)
+
+#ifndef offsetof
+# define offsetof(type, member) __builtin_offsetof(type, member)
+#endif /* offsetof */
+
+#ifndef container_of
+# define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - offsetof(type, member)))
+#endif /* container_of */
+
+#define MDBX_TETRAD(a, b, c, d) \
+ ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
+
+#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
+
+#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
+
+#ifndef STATIC_ASSERT_MSG
+# if defined(static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
+# elif defined(_STATIC_ASSERT)
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif defined(_MSC_VER)
+# include <crtdbg.h>
+# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
+# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
+ || __has_feature(c_static_assert)
+# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
+# else
+# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
+# endif
+#endif /* STATIC_ASSERT */
+
+#ifndef STATIC_ASSERT
+# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
+#endif
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
+/* Actually libmdbx was not tested with compilers older than GCC 4.2.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required GCC >= 4.2"
+#endif
+
+#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
+/* Actually libmdbx was not tested with CLANG older than 3.8.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old compilers.
+ */
+#warning "libmdbx required CLANG >= 3.8"
+#endif
+
+#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
+/* Actually libmdbx was not tested with something older than glibc 2.12.
+ * But you could ignore this warning at your own risk.
+ * In such case please don't rise up an issues related ONLY to old systems.
+ */
+#warning "libmdbx was only tested with GLIBC >= 2.12."
+#endif
+
+#ifdef __SANITIZE_THREAD__
+#warning \
+ "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
+#endif /* __SANITIZE_THREAD__ */
+
+#if __has_warning("-Wnested-anon-types")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wnested-anon-types"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wnested-anon-types"
+#else
+#pragma warning disable "nested-anon-types"
+#endif
+#endif /* -Wnested-anon-types */
+
+#if __has_warning("-Wconstant-logical-operand")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wconstant-logical-operand"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
+#else
+#pragma warning disable "constant-logical-operand"
+#endif
+#endif /* -Wconstant-logical-operand */
+
+#if defined(__LCC__) && (__LCC__ <= 121)
+/* bug #2798 */
+#pragma diag_suppress alignment_reduction_ignored
+#elif defined(__ICC)
+#pragma warning(disable : 3453 1366)
+#elif __has_warning("-Walignment-reduction-ignored")
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
+#else
+#pragma warning disable "alignment-reduction-ignored"
+#endif
+#endif /* -Walignment-reduction-ignored */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+/*
+ * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#endif /* _MSC_VER (warnings) */
+
+#if defined(_WIN32) || defined(_WIN64)
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
+#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
+ !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
+#define _NO_CRT_STDIO_INLINE
+#endif
+#elif !defined(_POSIX_C_SOURCE)
+#define _POSIX_C_SOURCE 200809L
+#endif /* Windows */
+
+/*----------------------------------------------------------------------------*/
+/* C99 includes */
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+/* C11' alignas() */
+#if __has_include(<stdalign.h>)
+#include <stdalign.h>
+#endif
+#if defined(alignas) || defined(__cplusplus)
+#define MDBX_ALIGNAS(N) alignas(N)
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define MDBX_ALIGNAS(N) _Alignas(N)
+#elif defined(_MSC_VER)
+#define MDBX_ALIGNAS(N) __declspec(align(N))
+#elif __has_attribute(__aligned__) || defined(__GNUC__)
+#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
+#else
+#error "FIXME: Required alignas() or equivalent."
+#endif /* MDBX_ALIGNAS */
+
+/*----------------------------------------------------------------------------*/
+/* Systems includes */
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif /* Apple OSX & iOS */
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+ defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
+ defined(__APPLE__) || defined(__MACH__)
+#include <sys/cdefs.h>
+#include <sys/mount.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+#include <vm/vm_param.h>
+#elif defined(__OpenBSD__) || defined(__NetBSD__)
+#include <uvm/uvm_param.h>
+#else
+#define SYSCTL_LEGACY_NONCONST_MIB
+#endif
+#ifndef __MACH__
+#include <sys/vmmeter.h>
+#endif
+#else
+#include <malloc.h>
+#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
+ defined(_WIN32) || defined(_WIN64))
+#include <mntent.h>
+#endif /* !Solaris */
+#endif /* !xBSD */
+
+#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
+#include <malloc_np.h>
+#endif
+
+#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
+#include <malloc/malloc.h>
+#endif /* MacOS */
+
+#if defined(__MACH__)
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+#include <mach/mach_port.h>
+#include <uuid/uuid.h>
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+#include <sched.h>
+#include <sys/sendfile.h>
+#include <sys/statfs.h>
+#endif /* Linux */
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 0
+#endif
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+#define _XOPEN_SOURCE_EXTENDED 0
+#else
+#include <utmpx.h>
+#endif /* _XOPEN_SOURCE_EXTENDED */
+
+#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#include <kstat.h>
+#include <sys/mnttab.h>
+/* On Solaris, it's easier to add a missing prototype rather than find a
+ * combination of #defines that break nothing. */
+__extern_C key_t ftok(const char *, int);
+#endif /* SunOS/Solaris */
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0601 /* Windows 7 */
+#elif _WIN32_WINNT < 0x0500
+#error At least 'Windows 2000' API is required for libmdbx.
+#endif /* _WIN32_WINNT */
+#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
+ !defined(__USE_MINGW_ANSI_STDIO)
+#define __USE_MINGW_ANSI_STDIO 1
+#endif /* MinGW */
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif /* WIN32_LEAN_AND_MEAN */
+#include <excpt.h>
+#include <tlhelp32.h>
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#define HAVE_SYS_STAT_H
+#define HAVE_SYS_TYPES_H
+typedef HANDLE mdbx_thread_t;
+typedef unsigned mdbx_thread_key_t;
+#define MAP_FAILED NULL
+#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
+#define THREAD_CALL WINAPI
+#define THREAD_RESULT DWORD
+typedef struct {
+ HANDLE mutex;
+ HANDLE event[2];
+} mdbx_condpair_t;
+typedef CRITICAL_SECTION mdbx_fastmutex_t;
+
+#if !defined(_MSC_VER) && !defined(__try)
+/* *INDENT-OFF* */
+/* clang-format off */
+#define __try
+#define __except(COND) if(false)
+/* *INDENT-ON* */
+/* clang-format on */
+#endif /* stub for MSVC's __try/__except */
+
+#if MDBX_WITHOUT_MSVC_CRT
+
+#ifndef mdbx_malloc
+static inline void *mdbx_malloc(size_t bytes) {
+ return HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_malloc */
+
+#ifndef mdbx_calloc
+static inline void *mdbx_calloc(size_t nelem, size_t size) {
+ return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
+}
+#endif /* mdbx_calloc */
+
+#ifndef mdbx_realloc
+static inline void *mdbx_realloc(void *ptr, size_t bytes) {
+ return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
+ : HeapAlloc(GetProcessHeap(), 0, bytes);
+}
+#endif /* mdbx_realloc */
+
+#ifndef mdbx_free
+static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
+#endif /* mdbx_free */
+
+#else /* MDBX_WITHOUT_MSVC_CRT */
+
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup _strdup
+
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+#ifndef snprintf
+#define snprintf _snprintf /* ntdll */
+#endif
+
+#ifndef vsnprintf
+#define vsnprintf _vsnprintf /* ntdll */
+#endif
+
+#else /*----------------------------------------------------------------------*/
+
+#include <unistd.h>
+#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
+#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
+#endif /* _POSIX_MAPPED_FILES */
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/uio.h>
+typedef pthread_t mdbx_thread_t;
+typedef pthread_key_t mdbx_thread_key_t;
+#define INVALID_HANDLE_VALUE (-1)
+#define THREAD_CALL
+#define THREAD_RESULT void *
+typedef struct {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond[2];
+} mdbx_condpair_t;
+typedef pthread_mutex_t mdbx_fastmutex_t;
+#define mdbx_malloc malloc
+#define mdbx_calloc calloc
+#define mdbx_realloc realloc
+#define mdbx_free free
+#define mdbx_strdup strdup
+#endif /* Platform */
+
+#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
+/* malloc_usable_size() already provided */
+#elif defined(__APPLE__)
+#define malloc_usable_size(ptr) malloc_size(ptr)
+#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
+#define malloc_usable_size(ptr) _msize(ptr)
+#endif /* malloc_usable_size */
+
+#ifdef __ANDROID_API__
+#include <android/log.h>
+#if __ANDROID_API__ >= 21
+#include <sys/sendfile.h>
+#endif
+#endif /* Android */
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
+#include <sys/stat.h>
+#endif
+#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
+#include <sys/types.h>
+#endif
+#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
+#include <sys/file.h>
+#endif
+/* *INDENT-ON* */
+/* clang-format on */
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX INTPTR_MAX
+#endif
+
+#if !defined(MADV_DODUMP) && defined(MADV_CORE)
+#define MADV_DODUMP MADV_CORE
+#endif /* MADV_CORE -> MADV_DODUMP */
+
+#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
+#define MADV_DONTDUMP MADV_NOCORE
+#endif /* MADV_NOCORE -> MADV_DONTDUMP */
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+ defined(i486) || defined(__i486) || defined(__i486__) || \
+ defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
+ defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
+ defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
+ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
+ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
+ defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#error \
+ "Sanity checking failed: Two's complement, reasonably sized integer types"
+#endif
+
+#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
+#define MDBX_WORDBITS 64
+#else
+#define MDBX_WORDBITS 32
+#endif /* MDBX_WORDBITS */
+
+#if defined(__ANDROID_API__) || defined(ANDROID)
+#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
+#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
+#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
+#endif
+#endif /* Android */
+
+/*----------------------------------------------------------------------------*/
+/* Compiler's includes for builtins/intrinsics */
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
+#if defined(__ia32__) || defined(__e2k__)
+#include <x86intrin.h>
+#endif /* __ia32__ */
+#if defined(__ia32__)
+#include <cpuid.h>
+#endif /* __ia32__ */
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+#include <mbarrier.h>
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+#include <machine/sys/inline.h>
+#elif defined(__IBMC__) && defined(__powerpc)
+#include <atomic.h>
+#elif defined(_AIX)
+#include <builtins.h>
+#include <sys/atomic_op.h>
+#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
+#include <c_asm.h>
+#include <machine/builtins.h>
+#elif defined(__MWERKS__)
+/* CodeWarrior - troubles ? */
+#pragma gcc_extensions
+#elif defined(__SNC__)
+/* Sony PS3 - troubles ? */
+#elif defined(__hppa__) || defined(__hppa)
+#include <machine/inline.h>
+#else
+#error Unsupported C compiler, please use GNU C 4.4 or newer
+#endif /* Compiler */
+
+/*----------------------------------------------------------------------------*/
+/* Byteorder */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
+ !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
+ defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
+ defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
+ (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
+ defined(__NetBSD__) || \
+ defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) || \
+ (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
+ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
+ defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
+ defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
+ defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
+ defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
+ defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
+ defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
+ defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
+ defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) || \
+ (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
+ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
+ defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
+ defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
+ defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
+ defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
+ defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/* Get the size of a memory page for the system.
+ * This is the basic size that the platform's memory manager uses, and is
+ * fundamental to the use of memory-mapped files. */
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
+mdbx_syspagesize(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGE_SIZE);
+#endif
+}
+
+typedef struct mdbx_mmap_param {
+ union {
+ void *address;
+ uint8_t *dxb;
+ struct MDBX_lockinfo *lck;
+ };
+ mdbx_filehandle_t fd;
+ size_t limit; /* mapping length, but NOT a size of file nor DB */
+ size_t current; /* mapped region size, i.e. the size of file and DB */
+ uint64_t filesize /* in-process cache of a file size */;
+#if defined(_WIN32) || defined(_WIN64)
+ HANDLE section; /* memory-mapped section handle */
+#endif
+} mdbx_mmap_t;
+
+typedef union bin128 {
+ __anonymous_struct_extension__ struct { uint64_t x, y; };
+ __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
+} bin128_t;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef union MDBX_srwlock {
+ __anonymous_struct_extension__ struct {
+ long volatile readerCount;
+ long volatile writerCount;
+ };
+ RTL_SRWLOCK native;
+} MDBX_srwlock;
+#endif /* Windows */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
+#include <cstdatomic>
+#define MDBX_HAVE_C11ATOMICS
+#elif !defined(__cplusplus) && \
+ (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
+ !defined(__STDC_NO_ATOMICS__) && \
+ (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
+ !(defined(__GNUC__) || defined(__clang__)))
+#include <stdatomic.h>
+#define MDBX_HAVE_C11ATOMICS
+#elif defined(__GNUC__) || defined(__clang__)
+#elif defined(_MSC_VER)
+#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
+#pragma warning(disable : 4133) /* 'function': incompatible types - from \
+ 'size_t' to 'LONGLONG' */
+#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
+ 'std::size_t', possible loss of data */
+#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
+ 'long', possible loss of data */
+#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
+#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
+#elif defined(__APPLE__)
+#include <libkern/OSAtomic.h>
+#else
+#error FIXME atomic-ops
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Memory/Compiler barriers, cache coherence */
+
+#if __has_include(<sys/cachectl.h>)
+#include <sys/cachectl.h>
+#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS should have explicit cache control */
+#include <sys/cachectl.h>
+#endif
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
+#if defined(__clang__) || defined(__GNUC__)
+ __asm__ __volatile__("" ::: "memory");
+#elif defined(_MSC_VER)
+ _ReadWriteBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+ __memory_barrier();
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __compiler_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __fence();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(memory_order_seq_cst);
+#elif defined(__ATOMIC_SEQ_CST)
+#ifdef __clang__
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+#elif defined(__clang__) || defined(__GNUC__)
+ __sync_synchronize();
+#elif defined(_WIN32) || defined(_WIN64)
+ MemoryBarrier();
+#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
+#if defined(__ia32__)
+ _mm_mfence();
+#else
+ __mf();
+#endif
+#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
+ __machine_rw_barrier();
+#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
+ (defined(HP_IA64) || defined(__ia64))
+ _Asm_mf();
+#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
+ defined(__ppc64__) || defined(__powerpc64__)
+ __lwsync();
+#else
+#error "Could not guess the kind of compiler, please report to us."
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* libc compatibility stuff */
+
+#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
+ (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
+#define mdbx_asprintf asprintf
+#define mdbx_vasprintf vasprintf
+#else
+MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
+ MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
+MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* OS abstraction layer stuff */
+
+/* max bytes to write in one call */
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_WRITE UINT32_C(0x01000000)
+#else
+#define MAX_WRITE UINT32_C(0x3fff0000)
+#endif
+
+#if defined(__linux__) || defined(__gnu_linux__)
+MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
+MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
+#endif /* Linux */
+
+#ifndef mdbx_strdup
+LIBMDBX_API char *mdbx_strdup(const char *str);
+#endif
+
+MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
+#if defined(_WIN32) || defined(_WIN64)
+ DWORD rc = GetLastError();
+#else
+ int rc = errno;
+#endif
+ return rc;
+}
+
+#ifndef mdbx_memalign_alloc
+MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
+ void **result);
+#endif
+#ifndef mdbx_memalign_free
+MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
+#endif
+
+MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
+MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
+ bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
+MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
+
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
+MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
+
+MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
+ int iovcnt, uint64_t offset,
+ size_t expected_written);
+MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
+ uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
+ size_t count, uint64_t offset);
+MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
+ size_t count);
+
+MDBX_INTERNAL_FUNC int
+mdbx_thread_create(mdbx_thread_t *thread,
+ THREAD_RESULT(THREAD_CALL *start_routine)(void *),
+ void *arg);
+MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
+
+enum mdbx_syncmode_bits {
+ MDBX_SYNC_NONE = 0,
+ MDBX_SYNC_DATA = 1,
+ MDBX_SYNC_SIZE = 2,
+ MDBX_SYNC_IODQ = 4
+};
+
+MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
+ const enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
+MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
+MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
+
+enum mdbx_openfile_purpose {
+ MDBX_OPEN_DXB_READ = 0,
+ MDBX_OPEN_DXB_LAZY = 1,
+ MDBX_OPEN_DXB_DSYNC = 2,
+ MDBX_OPEN_LCK = 3,
+ MDBX_OPEN_COPY = 4,
+ MDBX_OPEN_DELETE = 5
+};
+
+MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
+ const MDBX_env *env, const char *pathname,
+ mdbx_filehandle_t *fd,
+ mdbx_mode_t unix_mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
+MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
+MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
+
+#define MMAP_OPTION_TRUNCATE 1
+#define MMAP_OPTION_SEMAPHORE 2
+MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
+ const size_t must, const size_t limit,
+ const unsigned options);
+MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
+#define MDBX_MRESIZE_MAY_MOVE 0x00000100
+#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
+MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
+ size_t size, size_t limit);
+#if defined(_WIN32) || defined(_WIN64)
+typedef struct {
+ unsigned limit, count;
+ HANDLE handles[31];
+} mdbx_handle_array_t;
+MDBX_INTERNAL_FUNC int
+mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
+MDBX_INTERNAL_FUNC int
+mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
+#endif /* Windows */
+MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
+ size_t length,
+ enum mdbx_syncmode_bits mode_bits);
+MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
+ const char *pathname, int err);
+
+MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
+ STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
+#if defined(_WIN32) || defined(_WIN64)
+ return GetCurrentProcessId();
+#else
+ STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
+ return getpid();
+#endif
+}
+
+MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
+ mdbx_tid_t thunk;
+ STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
+#if defined(_WIN32) || defined(_WIN64)
+ thunk = GetCurrentThreadId();
+#else
+ thunk = pthread_self();
+#endif
+ return (uintptr_t)thunk;
+}
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
+MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
+#else
+static __inline int mdbx_check_tid4bionic(void) { return 0; }
+#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
+
+MDBX_MAYBE_UNUSED static __inline int
+mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
+ int err = mdbx_check_tid4bionic();
+ return unlikely(err) ? err : pthread_mutex_lock(mutex);
+}
+#endif /* !Windows */
+
+MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
+MDBX_INTERNAL_FUNC uint64_t
+mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
+MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
+
+MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
+/*----------------------------------------------------------------------------*/
+/* lck stuff */
+
+/// \brief Initialization of synchronization primitives linked with MDBX_env
+/// instance both in LCK-file and within the current process.
+/// \param
+/// global_uniqueness_flag = true - denotes that there are no other processes
+/// working with DB and LCK-file. Thus the function MUST initialize
+/// shared synchronization objects in memory-mapped LCK-file.
+/// global_uniqueness_flag = false - denotes that at least one process is
+/// already working with DB and LCK-file, including the case when DB
+/// has already been opened in the current process. Thus the function
+/// MUST NOT initialize shared synchronization objects in memory-mapped
+/// LCK-file that are already in use.
+/// \return Error code or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
+ MDBX_env *inprocess_neighbor,
+ int global_uniqueness_flag);
+
+/// \brief Disconnects from shared interprocess objects and destructs
+/// synchronization objects linked with MDBX_env instance
+/// within the current process.
+/// \param
+/// inprocess_neighbor = NULL - if the current process does not have other
+/// instances of MDBX_env linked with the DB being closed.
+/// Thus the function MUST check for other processes working with DB or
+/// LCK-file, and keep or destroy shared synchronization objects in
+/// memory-mapped LCK-file depending on the result.
+/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
+/// (anyone of there is several) working with DB or LCK-file within the
+/// current process. Thus the function MUST NOT try to acquire exclusive
+/// lock and/or try to destruct shared synchronization objects linked with
+/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
+/// of other instances of MDBX_env within the current process, e.g.
+/// restore POSIX-fcntl locks after the closing of file descriptors.
+/// \return Error code (MDBX_PANIC) or zero on success.
+MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
+ MDBX_env *inprocess_neighbor);
+
+/// \brief Connects to shared interprocess locking objects and tries to acquire
+/// the maximum lock level (shared if exclusive is not available)
+/// Depending on implementation or/and platform (Windows) this function may
+/// acquire the non-OS super-level lock (e.g. for shared synchronization
+/// objects initialization), which will be downgraded to OS-exclusive or
+/// shared via explicit calling of mdbx_lck_downgrade().
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
+/// the current process is the first and only after the last use of DB.
+/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
+/// DB has already been opened and now is used by other processes.
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
+
+/// \brief Downgrades the level of initially acquired lock to
+/// operational level specified by argument. The reson for such downgrade:
+/// - unblocking of other processes that are waiting for access, i.e.
+/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
+/// should be made aware that access is unavailable rather than
+/// wait for it.
+/// - freeing locks that interfere file operation (especially for Windows)
+/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
+/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
+/// operational lock.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
+
+/// \brief Locks LCK-file or/and table of readers for (de)registering.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
+
+/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
+MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
+
+/// \brief Acquires lock for DB change (on writing transaction start)
+/// Reading transactions will not be blocked.
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+/// \return Error code or zero on success
+LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
+
+/// \brief Releases lock once DB changes is made (after writing transaction
+/// has finished).
+/// Declared as LIBMDBX_API because it is used in mdbx_chk.
+LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
+
+/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
+/// the current process. The function does no more than needed for
+/// the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
+
+/// \brief Resets alive-flag of reader presence (indicative lock)
+/// for PID of the current process. The function does no more than needed
+/// for the correct working of mdbx_rpid_check() in other processes.
+/// \return Error code or zero on success
+MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
+
+/// \brief Checks for reading process status with the given pid with help of
+/// alive-flag of presence (indicative lock) or using another way.
+/// \return
+/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
+/// and working with DB (indicative lock is present).
+/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
+/// or not working with DB (indicative lock is not present).
+/// Otherwise (not 0 and not -1) - error code.
+MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
+
+#if defined(_WIN32) || defined(_WIN64)
+
+typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
+MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
+ mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
+ mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
+
+#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
+typedef enum _FILE_INFO_BY_HANDLE_CLASS {
+ FileBasicInfo,
+ FileStandardInfo,
+ FileNameInfo,
+ FileRenameInfo,
+ FileDispositionInfo,
+ FileAllocationInfo,
+ FileEndOfFileInfo,
+ FileStreamInfo,
+ FileCompressionInfo,
+ FileAttributeTagInfo,
+ FileIdBothDirectoryInfo,
+ FileIdBothDirectoryRestartInfo,
+ FileIoPriorityHintInfo,
+ FileRemoteProtocolInfo,
+ MaximumFileInfoByHandleClass
+} FILE_INFO_BY_HANDLE_CLASS,
+ *PFILE_INFO_BY_HANDLE_CLASS;
+
+typedef struct _FILE_END_OF_FILE_INFO {
+ LARGE_INTEGER EndOfFile;
+} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
+
+#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
+#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
+
+typedef struct _FILE_REMOTE_PROTOCOL_INFO {
+ USHORT StructureVersion;
+ USHORT StructureSize;
+ DWORD Protocol;
+ USHORT ProtocolMajorVersion;
+ USHORT ProtocolMinorVersion;
+ USHORT ProtocolRevision;
+ USHORT Reserved;
+ DWORD Flags;
+ struct {
+ DWORD Reserved[8];
+ } GenericReserved;
+ struct {
+ DWORD Reserved[16];
+ } ProtocolSpecificReserved;
+} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
+
+#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
+
+typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
+ mdbx_GetFileInformationByHandleEx;
+
+typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
+ _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
+ _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
+ _Out_opt_ LPDWORD lpMaximumComponentLength,
+ _Out_opt_ LPDWORD lpFileSystemFlags,
+ _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
+MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
+ mdbx_GetVolumeInformationByHandleW;
+
+typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
+ _Out_ LPWSTR lpszFilePath,
+ _In_ DWORD cchFilePath,
+ _In_ DWORD dwFlags);
+MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
+
+typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
+ _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+ _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
+MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
+ mdbx_SetFileInformationByHandle;
+
+typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
+ IN HANDLE FileHandle, IN OUT HANDLE Event,
+ IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
+ OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
+ IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
+ OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
+MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
+
+typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
+MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
+
+#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
+typedef struct _WIN32_MEMORY_RANGE_ENTRY {
+ PVOID VirtualAddress;
+ SIZE_T NumberOfBytes;
+} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
+#endif /* Windows 8.x */
+
+typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
+ HANDLE hProcess, ULONG_PTR NumberOfEntries,
+ PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
+MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
+
+typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
+
+typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
+ IN PLARGE_INTEGER NewSectionSize);
+MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
+
+static __inline bool mdbx_RunningUnderWine(void) {
+ return !mdbx_NtExtendSection;
+}
+
+typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
+ LPCSTR lpValue, DWORD dwFlags,
+ LPDWORD pdwType, PVOID pvData,
+ LPDWORD pcbData);
+MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
+
+NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
+
+#endif /* Windows */
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+
+#if defined(_MSC_VER) && _MSC_VER >= 1900
+/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
+ * for internal format-args checker. */
+#undef PRIuPTR
+#undef PRIiPTR
+#undef PRIdPTR
+#undef PRIxPTR
+#define PRIuPTR "Iu"
+#define PRIiPTR "Ii"
+#define PRIdPTR "Id"
+#define PRIxPTR "Ix"
+#define PRIuSIZE "zu"
+#define PRIiSIZE "zi"
+#define PRIdSIZE "zd"
+#define PRIxSIZE "zx"
+#endif /* fix PRI*PTR for _MSC_VER */
+
+#ifndef PRIuSIZE
+#define PRIuSIZE PRIuPTR
+#define PRIiSIZE PRIiPTR
+#define PRIdSIZE PRIdPTR
+#define PRIxSIZE PRIxPTR
+#endif /* PRI*SIZE macros for MSVC */
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
+#if defined(xMDBX_TOOLS)
+extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
+#endif
+
+/*******************************************************************************
+ *******************************************************************************
+ *******************************************************************************
+ *
+ *
+ * #### ##### ##### # #### # # ####
+ * # # # # # # # # ## # #
+ * # # # # # # # # # # # ####
+ * # # ##### # # # # # # # #
+ * # # # # # # # # ## # #
+ * #### # # # #### # # ####
+ *
+ *
+ */
+
+/** \defgroup build_option Build options
+ * The libmdbx build options.
+ @{ */
+
+/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
+#define MDBX_OSX_WANNA_DURABILITY 0
+/** Using fsync() with chance of data lost on power failure */
+#define MDBX_OSX_WANNA_SPEED 1
+
+#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
+/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
+ * for OSX & iOS */
+#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
+#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
+
+/** Controls checking PID against reuse DB environment after the fork() */
+#ifndef MDBX_ENV_CHECKPID
+#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
+/* PID check could be omitted:
+ * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
+ * mapped pages will not be available for child process.
+ * - in Windows where fork() not available. */
+#define MDBX_ENV_CHECKPID 0
+#else
+#define MDBX_ENV_CHECKPID 1
+#endif
+#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#else
+#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
+#endif /* MDBX_ENV_CHECKPID */
+
+/** Controls checking transaction owner thread against misuse transactions from
+ * other threads. */
+#ifndef MDBX_TXN_CHECKOWNER
+#define MDBX_TXN_CHECKOWNER 1
+#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#else
+#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
+#endif /* MDBX_TXN_CHECKOWNER */
+
+/** Does a system have battery-backed Real-Time Clock or just a fake. */
+#ifndef MDBX_TRUST_RTC
+#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
+ defined(__OpenBSD__)
+#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
+#else
+#define MDBX_TRUST_RTC 1
+#endif
+#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#else
+#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
+#endif /* MDBX_TRUST_RTC */
+
+/** Controls online database auto-compactification during write-transactions. */
+#ifndef MDBX_ENABLE_REFUND
+#define MDBX_ENABLE_REFUND 1
+#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
+#error MDBX_ENABLE_REFUND must be defined as 0 or 1
+#endif /* MDBX_ENABLE_REFUND */
+
+/** Controls gathering statistics for page operations. */
+#ifndef MDBX_ENABLE_PGOP_STAT
+#define MDBX_ENABLE_PGOP_STAT 1
+#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
+#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+/** Controls use of POSIX madvise() hints and friends. */
+#ifndef MDBX_ENABLE_MADVISE
+#define MDBX_ENABLE_MADVISE 1
+#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
+#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
+#endif /* MDBX_ENABLE_MADVISE */
+
+/** Disable some checks to reduce an overhead and detection probability of
+ * database corruption to a values closer to the LMDB. */
+#ifndef MDBX_DISABLE_PAGECHECKS
+#define MDBX_DISABLE_PAGECHECKS 0
+#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
+#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
+#endif /* MDBX_DISABLE_PAGECHECKS */
+
+#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
+#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
+
+#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
+#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
+#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
+ MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
+#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
+#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
+
+/* Basically, this build-option is for TODO. Guess it should be replaced
+ * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
+ * 0/OFF = Don't track dirty pages at all and don't spilling ones.
+ * This should be by-default on Linux and may-be other systems
+ * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
+ * properly LRU tracking and async writing on-demand.
+ * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
+ * spilling with msync(MS_ASYNC). */
+#ifndef MDBX_FAKE_SPILL_WRITEMAP
+#if defined(__linux__) || defined(__gnu_linux__)
+#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
+#else
+#define MDBX_FAKE_SPILL_WRITEMAP 0
+#endif
+#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
+#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
+#endif /* MDBX_FAKE_SPILL_WRITEMAP */
+
+/** Controls sort order of internal page number lists.
+ * This mostly experimental/advanced option with not for regular MDBX users.
+ * \warning The database format depend on this option and libmdbx builded with
+ * different option value are incompatible. */
+#ifndef MDBX_PNL_ASCENDING
+#define MDBX_PNL_ASCENDING 0
+#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
+#error MDBX_PNL_ASCENDING must be defined as 0 or 1
+#endif /* MDBX_PNL_ASCENDING */
+
+/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
+#ifndef MDBX_WITHOUT_MSVC_CRT
+#define MDBX_WITHOUT_MSVC_CRT 1
+#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
+#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
+#endif /* MDBX_WITHOUT_MSVC_CRT */
+
+/** Size of buffer used during copying a environment/database file. */
+#ifndef MDBX_ENVCOPY_WRITEBUF
+#define MDBX_ENVCOPY_WRITEBUF 1048576u
+#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
+ MDBX_ENVCOPY_WRITEBUF % 65536u
+#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
+#endif /* MDBX_ENVCOPY_WRITEBUF */
+
+/** Forces assertion checking */
+#ifndef MDBX_FORCE_ASSERTIONS
+#define MDBX_FORCE_ASSERTIONS 0
+#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
+#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
+#endif /* MDBX_FORCE_ASSERTIONS */
+
+/** Presumed malloc size overhead for each allocation
+ * to adjust allocations to be more aligned. */
+#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
+#ifdef __SIZEOF_POINTER__
+#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
+#else
+#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
+#endif
+#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
+ MDBX_ASSUME_MALLOC_OVERHEAD % 4
+#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
+#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
+
+/** In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
+#ifndef MDBX_DEBUG
+#ifdef NDEBUG
+#define MDBX_DEBUG 0
+#else
+#define MDBX_DEBUG 1
+#endif
+#endif /* MDBX_DEBUG */
+
+/** If defined then enables integration with Valgrind,
+ * a memory analyzing tool. */
+#ifndef MDBX_USE_VALGRIND
+#endif /* MDBX_USE_VALGRIND */
+
+/** If defined then enables use C11 atomics,
+ * otherwise detects ones availability automatically. */
+#ifndef MDBX_HAVE_C11ATOMICS
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+//------------------------------------------------------------------------------
+
+/** Win32 File Locking API for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_WIN32FILES -1
+
+/** SystemV IPC semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_SYSV 5
+
+/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX1988 1988
+
+/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2001 2001
+
+/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_POSIX2008 2008
+
+/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
+#define MDBX_LOCKING_BENAPHORE 1995
+
+/** Advanced: Choices the locking implementation (autodetection by default). */
+#if defined(_WIN32) || defined(_WIN64)
+#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
+#else
+#ifndef MDBX_LOCKING
+#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
+ _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
+
+/* Some platforms define the EOWNERDEAD error code even though they
+ * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
+#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
+ ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
+ _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
+ (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
+ _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
+ defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
+ (!defined(__GLIBC__) || \
+ __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
+#else
+#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
+#endif
+#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
+#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
+#else
+#define MDBX_LOCKING MDBX_LOCKING_SYSV
+#endif
+#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
+#else
+#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
+#endif /* MDBX_LOCKING */
+#endif /* !Windows */
+
+/** Advanced: Using POSIX OFD-locks (autodetection by default). */
+#ifndef MDBX_USE_OFDLOCKS
+#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
+ !defined(MDBX_SAFE4QEMU) && \
+ !defined(__sun) /* OFD-lock are broken on Solaris */
+#define MDBX_USE_OFDLOCKS 1
+#else
+#define MDBX_USE_OFDLOCKS 0
+#endif
+#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#else
+#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
+#endif /* MDBX_USE_OFDLOCKS */
+
+/** Advanced: Using sendfile() syscall (autodetection by default). */
+#ifndef MDBX_USE_SENDFILE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
+#define MDBX_USE_SENDFILE 1
+#else
+#define MDBX_USE_SENDFILE 0
+#endif
+#endif /* MDBX_USE_SENDFILE */
+
+/** Advanced: Using copy_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_COPYFILERANGE
+#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
+#define MDBX_USE_COPYFILERANGE 1
+#else
+#define MDBX_USE_COPYFILERANGE 0
+#endif
+#endif /* MDBX_USE_COPYFILERANGE */
+
+/** Advanced: Using sync_file_range() syscall (autodetection by default). */
+#ifndef MDBX_USE_SYNCFILERANGE
+#if ((defined(__linux__) || defined(__gnu_linux__)) && \
+ defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
+ (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
+#define MDBX_USE_SYNCFILERANGE 1
+#else
+#define MDBX_USE_SYNCFILERANGE 0
+#endif
+#endif /* MDBX_USE_SYNCFILERANGE */
+
+//------------------------------------------------------------------------------
+
+#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
+#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
+ defined(__hppa__) || defined(DOXYGEN)
+#define MDBX_CPU_WRITEBACK_INCOHERENT 0
+#else
+#define MDBX_CPU_WRITEBACK_INCOHERENT 1
+#endif
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
+#ifdef __OpenBSD__
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
+#else
+#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
+#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
+ defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
+ defined(__MWERKS__) || defined(__sgi)
+/* MIPS has cache coherency issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
+#else
+/* LY: assume no relevant mmap/dcache issues. */
+#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
+#endif
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#ifndef MDBX_64BIT_ATOMIC
+#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
+#define MDBX_64BIT_ATOMIC 1
+#else
+#define MDBX_64BIT_ATOMIC 0
+#endif
+#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#else
+#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
+#endif /* MDBX_64BIT_ATOMIC */
+
+#ifndef MDBX_64BIT_CAS
+#if defined(ATOMIC_LLONG_LOCK_FREE)
+#if ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
+#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
+#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS 0
+#endif
+#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
+#define MDBX_64BIT_CAS 1
+#else
+#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
+#endif
+#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#else
+#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
+#endif /* MDBX_64BIT_CAS */
+
+#ifndef MDBX_UNALIGNED_OK
+#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#elif defined(__ARM_FEATURE_UNALIGNED)
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
+/* expecting an optimization will well done, also this
+ * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
+#define MDBX_UNALIGNED_OK 0
+#elif defined(__e2k__) || defined(__elbrus__)
+#if __iset__ > 4
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
+#endif
+#elif defined(__ia32__)
+#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
+#else
+#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
+#endif
+#elif MDBX_UNALIGNED_OK == 1
+#undef MDBX_UNALIGNED_OK
+#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
+#endif /* MDBX_UNALIGNED_OK */
+
+#ifndef MDBX_CACHELINE_SIZE
+#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
+#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
+#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
+#define MDBX_CACHELINE_SIZE 128
+#else
+#define MDBX_CACHELINE_SIZE 64
+#endif
+#endif /* MDBX_CACHELINE_SIZE */
+
+/** @} end of build options */
+/*******************************************************************************
+ *******************************************************************************
+ ******************************************************************************/
+
+#ifdef DOXYGEN
+/* !!! Actually this is a fake definitions !!!
+ * !!! for documentation generation by Doxygen !!! */
+
+/** Controls enabling of debugging features.
+ *
+ * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
+ * including logging and assertion controls.
+ * Logging level and corresponding debug flags changing
+ * by \ref mdbx_setup_debug() will not have effect.
+ * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
+ * assertions checking and internal audit).
+ * Simultaneously sets the default logging level
+ * to the `MDBX_DEBUG` value.
+ * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
+ *
+ * \ingroup build_option */
+#define MDBX_DEBUG 0...7
+
+/** Disables using of GNU libc extensions. */
+#define MDBX_DISABLE_GNU_SOURCE 0 or 1
+
+#endif /* DOXYGEN */
+
+/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
+#if MDBX_DEBUG
+#undef NDEBUG
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Atomics */
+
+enum MDBX_memory_order {
+ mo_Relaxed,
+ mo_AcquireRelease,
+ mo_SequentialConsistency
+};
+
+typedef union {
+ volatile uint32_t weak;
+#ifdef MDBX_HAVE_C11ATOMICS
+ volatile _Atomic uint32_t c11a;
+#endif /* MDBX_HAVE_C11ATOMICS */
+} MDBX_atomic_uint32_t;
+
+typedef union {
+ volatile uint64_t weak;
+#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
+ volatile _Atomic uint64_t c11a;
+#endif
+#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
+ __anonymous_struct_extension__ struct {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ MDBX_atomic_uint32_t low, high;
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ MDBX_atomic_uint32_t high, low;
+#else
+#error "FIXME: Unsupported byte order"
+#endif /* __BYTE_ORDER__ */
+ };
+#endif
+} MDBX_atomic_uint64_t;
+
+#ifdef MDBX_HAVE_C11ATOMICS
+
+/* Crutches for C11 atomic compiler's bugs */
+#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
+#elif defined(__clang__) && __clang__ < 8
+#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#else
+#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
+#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
+#endif /* Crutches for C11 atomic compiler's bugs */
+
+static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_release;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+
+static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
+ switch (fence) {
+ default:
+ assert(false);
+ __unreachable();
+ case mo_Relaxed:
+ return memory_order_relaxed;
+ case mo_AcquireRelease:
+ return memory_order_acquire;
+ case mo_SequentialConsistency:
+ return memory_order_seq_cst;
+ }
+}
+#endif /* MDBX_HAVE_C11ATOMICS */
+
+#ifndef __cplusplus
+
+MDBX_MAYBE_UNUSED static __always_inline void
+mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
+#ifdef MDBX_HAVE_C11ATOMICS
+ atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_compiler_barrier();
+ if (write &&
+ order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
+ mdbx_memory_barrier();
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
+ enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
+ atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ p->weak = value;
+ mdbx_memory_fence(order, true);
+#endif /* MDBX_HAVE_C11ATOMICS */
+ return value;
+}
+
+MDBX_MAYBE_UNUSED static __always_inline uint32_t
+atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
+ STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
+#ifdef MDBX_HAVE_C11ATOMICS
+ assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
+ return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
+#else /* MDBX_HAVE_C11ATOMICS */
+ mdbx_memory_fence(order, false);
+ const uint32_t value = p->weak;
+ if (order != mo_Relaxed)
+ mdbx_compiler_barrier();
+ return value;
+#endif /* MDBX_HAVE_C11ATOMICS */
+}
+
+#endif /* !__cplusplus */
+
+/*----------------------------------------------------------------------------*/
+/* Basic constants and types */
+
+/* A stamp that identifies a file as an MDBX file.
+ * There's nothing special about this value other than that it is easily
+ * recognizable, and it will reflect any byte order mismatches. */
+#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
+
+/* FROZEN: The version number for a database's datafile format. */
+#define MDBX_DATA_VERSION 3
+/* The version number for a database's lockfile format. */
+#define MDBX_LOCK_VERSION 4
+
+/* handle for the DB used to track free pages. */
+#define FREE_DBI 0
+/* handle for the default DB. */
+#define MAIN_DBI 1
+/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS 2
+
+/* Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS 3
+
+/* A page number in the database.
+ *
+ * MDBX uses 32 bit for page numbers. This limits database
+ * size up to 2^44 bytes, in case of 4K pages. */
+typedef uint32_t pgno_t;
+typedef MDBX_atomic_uint32_t atomic_pgno_t;
+#define PRIaPGNO PRIu32
+#define MAX_PAGENO UINT32_C(0x7FFFffff)
+#define MIN_PAGENO NUM_METAS
+
+#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
+
+/* A transaction ID. */
+typedef uint64_t txnid_t;
+typedef MDBX_atomic_uint64_t atomic_txnid_t;
+#define PRIaTXN PRIi64
+#define MIN_TXNID UINT64_C(1)
+#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
+#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
+#define INVALID_TXNID UINT64_MAX
+/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
+ * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
+#ifndef xMDBX_TXNID_STEP
+#if MDBX_64BIT_CAS
+#define xMDBX_TXNID_STEP 1u
+#else
+#define xMDBX_TXNID_STEP 2u
+#endif
+#endif /* xMDBX_TXNID_STEP */
+
+/* Used for offsets within a single page.
+ * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
+ * this is plenty. */
+typedef uint16_t indx_t;
+
+#define MEGABYTE ((size_t)1 << 20)
+
+/*----------------------------------------------------------------------------*/
+/* Core structures for database and shared memory (i.e. format definition) */
+#pragma pack(push, 4)
+
+/* Information about a single database in the environment. */
+typedef struct MDBX_db {
+ uint16_t md_flags; /* see mdbx_dbi_open */
+ uint16_t md_depth; /* depth of this tree */
+ uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
+ pgno_t md_root; /* the root page of this tree */
+ pgno_t md_branch_pages; /* number of internal pages */
+ pgno_t md_leaf_pages; /* number of leaf pages */
+ pgno_t md_overflow_pages; /* number of overflow pages */
+ uint64_t md_seq; /* table sequence counter */
+ uint64_t md_entries; /* number of data items */
+ uint64_t md_mod_txnid; /* txnid of last committed modification */
+} MDBX_db;
+
+/* database size-related parameters */
+typedef struct MDBX_geo {
+ uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
+ quantized) value */
+ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
+ (exponential quantized) value */
+ pgno_t lower; /* minimal size of datafile in pages */
+ pgno_t upper; /* maximal size of datafile in pages */
+ pgno_t now; /* current size of datafile in pages */
+ pgno_t next; /* first unused page in the datafile,
+ but actually the file may be shorter. */
+} MDBX_geo;
+
+/* Meta page content.
+ * A meta page is the start point for accessing a database snapshot.
+ * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
+typedef struct MDBX_meta {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
+ uint32_t mm_magic_and_version[2];
+
+ /* txnid that committed this page, the first of a two-phase-update pair */
+ uint32_t mm_txnid_a[2];
+
+ uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
+ uint8_t mm_validator_id; /* ID of checksum and page validation method,
+ * zero (nothing) for now */
+ uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
+ * zero (nothing) for now */
+
+ MDBX_geo mm_geo; /* database size-related parameters */
+
+ MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
+ /* The size of pages used in this DB */
+#define mm_psize mm_dbs[FREE_DBI].md_xsize
+ MDBX_canary mm_canary;
+
+#define MDBX_DATASIGN_NONE 0u
+#define MDBX_DATASIGN_WEAK 1u
+#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
+#define META_IS_STEADY(meta) \
+ SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
+ uint32_t mm_datasync_sign[2];
+
+ /* txnid that committed this page, the second of a two-phase-update pair */
+ uint32_t mm_txnid_b[2];
+
+ /* Number of non-meta pages which were put in GC after COW. May be 0 in case
+ * DB was previously handled by libmdbx without corresponding feature.
+ * This value in couple with mr_snapshot_pages_retired allows fast estimation
+ * of "how much reader is restraining GC recycling". */
+ uint32_t mm_pages_retired[2];
+
+ /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
+ * whether the system was rebooted after the last use of the database files.
+ * If there was no reboot, but there is no need to rollback to the last
+ * steady sync point. Zeros mean that no relevant information is available
+ * from the system. */
+ bin128_t mm_bootid;
+
+} MDBX_meta;
+
+#pragma pack(1)
+
+/* Common header for all page types. The page type depends on mp_flags.
+ *
+ * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
+ * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
+ * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
+ *
+ * P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of F_BIGDATA nodes.
+ *
+ * P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
+ *
+ * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a GC record. */
+typedef struct MDBX_page {
+ union {
+#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
+#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
+#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
+#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
+ uint64_t mp_txnid;
+ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
+ };
+ uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
+#define P_BRANCH 0x01 /* branch page */
+#define P_LEAF 0x02 /* leaf page */
+#define P_OVERFLOW 0x04 /* overflow page */
+#define P_META 0x08 /* meta page */
+#define P_BAD 0x10 /* explicit flag for invalid/bad page */
+#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
+#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
+#define P_SPILLED 0x2000 /* spilled in parent txn */
+#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
+#define P_FROZEN 0x8000 /* used for retire page with known status */
+#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
+ uint16_t mp_flags;
+ union {
+ uint32_t mp_pages; /* number of overflow pages */
+ __anonymous_struct_extension__ struct {
+ indx_t mp_lower; /* lower bound of free space */
+ indx_t mp_upper; /* upper bound of free space */
+ };
+ };
+ pgno_t mp_pgno; /* page number */
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ indx_t mp_ptrs[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_page;
+
+/* Size of the page header, excluding dynamic data at the end */
+#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
+
+#pragma pack(pop)
+
+#if MDBX_ENABLE_PGOP_STAT
+/* Statistics of page operations overall of all (running, completed and aborted)
+ * transactions */
+typedef struct {
+ MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
+ MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
+ MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
+ for nested transactions */
+ MDBX_atomic_uint64_t split; /* Page splits */
+ MDBX_atomic_uint64_t merge; /* Page merges */
+ MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
+ MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
+ MDBX_atomic_uint64_t
+ wops; /* Number of explicit write operations (not a pages) to a disk */
+} MDBX_pgop_stat_t;
+#endif /* MDBX_ENABLE_PGOP_STAT */
+
+#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
+#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
+typedef void mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
+
+#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
+typedef mdbx_pid_t mdbx_ipclock_t;
+#ifndef EOWNERDEAD
+#define EOWNERDEAD MDBX_RESULT_TRUE
+#endif
+
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
+ MDBX_LOCKING == MDBX_LOCKING_POSIX2008
+#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
+typedef pthread_mutex_t mdbx_ipclock_t;
+#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
+#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
+typedef sem_t mdbx_ipclock_t;
+#else
+#error "FIXME"
+#endif /* MDBX_LOCKING */
+
+#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
+MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
+MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
+#endif /* MDBX_LOCKING */
+
+/* Reader Lock Table
+ *
+ * Readers don't acquire any locks for their data access. Instead, they
+ * simply record their transaction ID in the reader table. The reader
+ * mutex is needed just to find an empty slot in the reader table. The
+ * slot's address is saved in thread-specific data so that subsequent
+ * read transactions started by the same thread need no further locking to
+ * proceed.
+ *
+ * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
+ * No reader table is used if the database is on a read-only filesystem.
+ *
+ * Since the database uses multi-version concurrency control, readers don't
+ * actually need any locking. This table is used to keep track of which
+ * readers are using data from which old transactions, so that we'll know
+ * when a particular old transaction is no longer in use. Old transactions
+ * that have discarded any data pages can then have those pages reclaimed
+ * for use by a later write transaction.
+ *
+ * The lock table is constructed such that reader slots are aligned with the
+ * processor's cache line size. Any slot is only ever used by one thread.
+ * This alignment guarantees that there will be no contention or cache
+ * thrashing as threads update their own slot info, and also eliminates
+ * any need for locking when accessing a slot.
+ *
+ * A writer thread will scan every slot in the table to determine the oldest
+ * outstanding reader transaction. Any freed pages older than this will be
+ * reclaimed by the writer. The writer doesn't use any locks when scanning
+ * this table. This means that there's no guarantee that the writer will
+ * see the most up-to-date reader info, but that's not required for correct
+ * operation - all we need is to know the upper bound on the oldest reader,
+ * we don't care at all about the newest reader. So the only consequence of
+ * reading stale information here is that old pages might hang around a
+ * while longer before being reclaimed. That's actually good anyway, because
+ * the longer we delay reclaiming old pages, the more likely it is that a
+ * string of contiguous pages can be found after coalescing old pages from
+ * many old transactions together. */
+
+/* The actual reader record, with cacheline padding. */
+typedef struct MDBX_reader {
+ /* Current Transaction ID when this transaction began, or (txnid_t)-1.
+ * Multiple readers that start at the same time will probably have the
+ * same ID here. Again, it's not important to exclude them from
+ * anything; all we need to know is which version of the DB they
+ * started from so we can avoid overwriting any data used in that
+ * particular version. */
+ MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
+
+ /* The information we store in a single slot of the reader table.
+ * In addition to a transaction ID, we also record the process and
+ * thread ID that owns a slot, so that we can detect stale information,
+ * e.g. threads or processes that went away without cleaning up.
+ *
+ * NOTE: We currently don't check for stale records.
+ * We simply re-init the table when we know that we're the only process
+ * opening the lock file. */
+
+ /* The thread ID of the thread owning this txn. */
+ MDBX_atomic_uint64_t mr_tid;
+
+ /* The process ID of the process owning this reader txn. */
+ MDBX_atomic_uint32_t mr_pid;
+
+ /* The number of pages used in the reader's MVCC snapshot,
+ * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
+ atomic_pgno_t mr_snapshot_pages_used;
+ /* Number of retired pages at the time this reader starts transaction. So,
+ * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
+ * will give the number of pages which this reader restraining from reuse. */
+ MDBX_atomic_uint64_t mr_snapshot_pages_retired;
+} MDBX_reader;
+
+/* The header for the reader table (a memory-mapped lock file). */
+typedef struct MDBX_lockinfo {
+ /* Stamp identifying this as an MDBX file.
+ * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
+ uint64_t mti_magic_and_version;
+
+ /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
+ uint32_t mti_os_and_format;
+
+ /* Flags which environment was opened. */
+ MDBX_atomic_uint32_t mti_envmode;
+
+ /* Threshold of un-synced-with-disk pages for auto-sync feature,
+ * zero means no-threshold, i.e. auto-sync is disabled. */
+ atomic_pgno_t mti_autosync_threshold;
+
+ /* Low 32-bit of txnid with which meta-pages was synced,
+ * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
+ MDBX_atomic_uint32_t mti_meta_sync_txnid;
+
+ /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
+ * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
+ * The time value is represented in a suitable system-dependent form, for
+ * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
+ * Zero means timed auto-sync is disabled. */
+ MDBX_atomic_uint64_t mti_autosync_period;
+
+ /* Marker to distinguish uniqueness of DB/CLK. */
+ MDBX_atomic_uint64_t mti_bait_uniqueness;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+#if MDBX_ENABLE_PGOP_STAT
+ /* Statistics of costly ops of all (running, completed and aborted)
+ * transactions */
+ MDBX_pgop_stat_t mti_pgop_stat;
+#endif /* MDBX_ENABLE_PGOP_STAT*/
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Write transaction lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_wlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ atomic_txnid_t mti_oldest_reader;
+
+ /* Timestamp of the last steady sync. Value is represented in a suitable
+ * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
+ * clock_gettime(CLOCK_MONOTONIC). */
+ MDBX_atomic_uint64_t mti_sync_timestamp;
+
+ /* Number un-synced-with-disk pages for auto-sync feature. */
+ atomic_pgno_t mti_unsynced_pages;
+
+ /* Number of page which was discarded last time by madvise(MADV_FREE). */
+ atomic_pgno_t mti_discarded_tail;
+
+ /* Timestamp of the last readers check. */
+ MDBX_atomic_uint64_t mti_reader_check_timestamp;
+
+ /* Shared anchor for tracking readahead edge and enabled/disabled status. */
+ pgno_t mti_readahead_anchor;
+
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+
+ /* Readeaders registration lock. */
+#if MDBX_LOCKING > 0
+ mdbx_ipclock_t mti_rlock;
+#endif /* MDBX_LOCKING > 0 */
+
+ /* The number of slots that have been used in the reader table.
+ * This always records the maximum count, it is not decremented
+ * when readers release their slots. */
+ MDBX_atomic_uint32_t mti_numreaders;
+ MDBX_atomic_uint32_t mti_readers_refresh_flag;
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
+ MDBX_reader mti_readers[] /* dynamic size */;
+#endif /* C99 */
+} MDBX_lockinfo;
+
+/* Lockfile format signature: version, features and field layout */
+#define MDBX_LOCK_FORMAT \
+ (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
+ (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
+ (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
+
+#define MDBX_DATA_MAGIC \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
+ ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
+
+/* The maximum size of a database page.
+ *
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ *
+ * MDBX will use database pages < OS pages if needed.
+ * That causes more I/O in write transactions: The OS must
+ * know (read) the whole page before writing a partial page.
+ *
+ * Note that we don't currently support Huge pages. On Linux,
+ * regular data files cannot use Huge pages, and in general
+ * Huge pages aren't actually pageable. We rely on the OS
+ * demand-pager to read our data and page it out when memory
+ * pressure from other processes is high. So until OSs have
+ * actual paging support for Huge pages, they're not viable. */
+#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
+#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
+
+#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
+#if defined(_WIN32) || defined(_WIN64)
+#define MAX_MAPSIZE32 UINT32_C(0x38000000)
+#else
+#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
+#endif
+#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
+
+#if MDBX_WORDBITS >= 64
+#define MAX_MAPSIZE MAX_MAPSIZE64
+#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
+#else
+#define MAX_MAPSIZE MAX_MAPSIZE32
+#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
+#endif /* MDBX_WORDBITS */
+
+#define MDBX_READERS_LIMIT 32767
+#define MDBX_RADIXSORT_THRESHOLD 333
+
+/*----------------------------------------------------------------------------*/
+
+/* An PNL is an Page Number List, a sorted array of IDs.
+ * The first element of the array is a counter for how many actual page-numbers
+ * are in the list. By default PNLs are sorted in descending order, this allow
+ * cut off a page with lowest pgno (at the tail) just truncating the list. The
+ * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
+typedef pgno_t *MDBX_PNL;
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
+#else
+#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
+#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
+#endif
+
+/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
+typedef txnid_t *MDBX_TXL;
+
+/* An Dirty-Page list item is an pgno/pointer pair. */
+typedef struct MDBX_dp {
+ MDBX_page *ptr;
+ pgno_t pgno;
+ union {
+ unsigned extra;
+ __anonymous_struct_extension__ struct {
+ unsigned multi : 1;
+ unsigned lru : 31;
+ };
+ };
+} MDBX_dp;
+
+/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
+typedef struct MDBX_dpl {
+ unsigned sorted;
+ unsigned length;
+ unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
+#endif
+} MDBX_dpl;
+
+/* PNL sizes */
+#define MDBX_PNL_GRANULATE 1024
+#define MDBX_PNL_INITIAL \
+ (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
+
+#define MDBX_TXL_GRANULATE 32
+#define MDBX_TXL_INITIAL \
+ (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+#define MDBX_TXL_MAX \
+ ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
+
+#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
+#define MDBX_PNL_SIZE(pl) ((pl)[0])
+#define MDBX_PNL_FIRST(pl) ((pl)[1])
+#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
+#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
+#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
+
+#if MDBX_PNL_ASCENDING
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
+#else
+#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
+#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
+#endif
+
+#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
+#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
+
+/*----------------------------------------------------------------------------*/
+/* Internal structures */
+
+/* Auxiliary DB info.
+ * The information here is mostly static/read-only. There is
+ * only a single copy of this record in the environment. */
+typedef struct MDBX_dbx {
+ MDBX_val md_name; /* name of the database */
+ MDBX_cmp_func *md_cmp; /* function for comparing keys */
+ MDBX_cmp_func *md_dcmp; /* function for comparing data items */
+ size_t md_klen_min, md_klen_max; /* min/max key length for the database */
+ size_t md_vlen_min,
+ md_vlen_max; /* min/max value/data length for the database */
+} MDBX_dbx;
+
+/* A database transaction.
+ * Every operation requires a transaction handle. */
+struct MDBX_txn {
+#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
+ uint32_t mt_signature;
+
+ /* Transaction Flags */
+ /* mdbx_txn_begin() flags */
+#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
+#define MDBX_TXN_RW_BEGIN_FLAGS \
+ (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
+ /* Additional flag for mdbx_sync_locked() */
+#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
+
+ /* internal txn flags */
+#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */
+#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */
+#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */
+#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */
+#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */
+ /* most operations on the txn are currently illegal */
+#define MDBX_TXN_BLOCKED \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD)
+
+#define TXN_FLAGS \
+ (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
+ MDBX_TXN_HAS_CHILD)
+
+#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
+ ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
+ MDBX_SHRINK_ALLOWED)
+#error "Oops, some flags overlapped or wrong"
+#endif
+ uint32_t mt_flags;
+
+ MDBX_txn *mt_parent; /* parent of a nested txn */
+ /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
+ MDBX_txn *mt_child;
+ MDBX_geo mt_geo;
+ /* next unallocated page */
+#define mt_next_pgno mt_geo.next
+ /* corresponding to the current size of datafile */
+#define mt_end_pgno mt_geo.now
+
+ /* The ID of this transaction. IDs are integers incrementing from 1.
+ * Only committed write transactions increment the ID. If a transaction
+ * aborts, the ID may be re-used by the next writer. */
+ txnid_t mt_txnid;
+ txnid_t mt_front;
+
+ MDBX_env *mt_env; /* the DB environment */
+ /* Array of records for each DB known in the environment. */
+ MDBX_dbx *mt_dbxs;
+ /* Array of MDBX_db records for each known DB */
+ MDBX_db *mt_dbs;
+ /* Array of sequence numbers for each DB handle */
+ unsigned *mt_dbiseqs;
+
+ /* Transaction DBI Flags */
+#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
+#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
+#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
+#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
+#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
+#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
+#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
+ /* Array of flags for each DB */
+ uint8_t *mt_dbistate;
+ /* Number of DB records in use, or 0 when the txn is finished.
+ * This number only ever increments until the txn finishes; we
+ * don't decrement it when individual DB handles are closed. */
+ MDBX_dbi mt_numdbs;
+ size_t mt_owner; /* thread ID that owns this transaction */
+ MDBX_canary mt_canary;
+ void *mt_userctx; /* User-settable context */
+ MDBX_cursor **mt_cursors;
+
+ union {
+ struct {
+ /* For read txns: This thread/txn's reader table slot, or NULL. */
+ MDBX_reader *reader;
+ } to;
+ struct {
+ /* In write txns, array of cursors for each DB */
+ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
+ txnid_t last_reclaimed; /* ID of last used record */
+#if MDBX_ENABLE_REFUND
+ pgno_t loose_refund_wl /* FIXME: describe */;
+#endif /* MDBX_ENABLE_REFUND */
+ /* dirtylist room: Dirty array size - dirty pages visible to this txn.
+ * Includes ancestor txns' dirty pages not hidden by other txns'
+ * dirty/spilled pages. Thus commit(nested txn) has room to merge
+ * dirtylist into mt_parent after freeing hidden mt_parent pages. */
+ unsigned dirtyroom;
+ /* a sequence to spilling dirty page with LRU policy */
+ unsigned dirtylru;
+ /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
+ MDBX_dpl *dirtylist;
+ /* The list of reclaimed txns from GC */
+ MDBX_TXL lifo_reclaimed;
+ /* The list of pages that became unused during this transaction. */
+ MDBX_PNL retired_pages;
+ /* The list of loose pages that became unused and may be reused
+ * in this transaction, linked through `mp_next`. */
+ MDBX_page *loose_pages;
+ /* Number of loose pages (tw.loose_pages) */
+ unsigned loose_count;
+ /* The sorted list of dirty pages we temporarily wrote to disk
+ * because the dirty list was full. page numbers in here are
+ * shifted left by 1, deleted slots have the LSB set. */
+ MDBX_PNL spill_pages;
+ unsigned spill_least_removed;
+ } tw;
+ };
+};
+
+#if MDBX_WORDBITS >= 64
+#define CURSOR_STACK 32
+#else
+#define CURSOR_STACK 24
+#endif
+
+struct MDBX_xcursor;
+
+/* Cursors are used for all DB operations.
+ * A cursor holds a path of (page pointer, key index) from the DB
+ * root to a position in the DB, plus other state. MDBX_DUPSORT
+ * cursors include an xcursor to the current data item. Write txns
+ * track their cursors and keep them up to date when data moves.
+ * Exception: An xcursor's pointer to a P_SUBP page can be stale.
+ * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
+struct MDBX_cursor {
+#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
+#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
+#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
+ uint32_t mc_signature;
+ /* The database handle this cursor operates on */
+ MDBX_dbi mc_dbi;
+ /* Next cursor on this DB in this txn */
+ MDBX_cursor *mc_next;
+ /* Backup of the original cursor if this cursor is a shadow */
+ MDBX_cursor *mc_backup;
+ /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
+ struct MDBX_xcursor *mc_xcursor;
+ /* The transaction that owns this cursor */
+ MDBX_txn *mc_txn;
+ /* The database record for this cursor */
+ MDBX_db *mc_db;
+ /* The database auxiliary record for this cursor */
+ MDBX_dbx *mc_dbx;
+ /* The mt_dbistate for this database */
+ uint8_t *mc_dbistate;
+ unsigned mc_snum; /* number of pushed pages */
+ unsigned mc_top; /* index of top page, normally mc_snum-1 */
+
+ /* Cursor state flags. */
+#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
+#define C_EOF 0x02 /* No more data */
+#define C_SUB 0x04 /* Cursor is a sub-cursor */
+#define C_DEL 0x08 /* last op was a cursor_del */
+#define C_UNTRACK 0x10 /* Un-track cursor when closing */
+#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
+#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
+
+ /* Cursor checking flags. */
+#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
+#define C_UPDATING 0x200 /* update/rebalance pending */
+#define C_RETIRING 0x400 /* refs to child pages may be invalid */
+#define C_SKIPORD 0x800 /* don't check keys ordering */
+
+ unsigned mc_flags; /* see mdbx_cursor */
+ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
+ indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
+};
+
+/* Context for sorted-dup records.
+ * We could have gone to a fully recursive design, with arbitrarily
+ * deep nesting of sub-databases. But for now we only handle these
+ * levels - main DB, optional sub-DB, sorted-duplicate DB. */
+typedef struct MDBX_xcursor {
+ /* A sub-cursor for traversing the Dup DB */
+ MDBX_cursor mx_cursor;
+ /* The database record for this Dup DB */
+ MDBX_db mx_db;
+ /* The auxiliary DB record for this Dup DB */
+ MDBX_dbx mx_dbx;
+} MDBX_xcursor;
+
+typedef struct MDBX_cursor_couple {
+ MDBX_cursor outer;
+ void *mc_userctx; /* User-settable context */
+ MDBX_xcursor inner;
+} MDBX_cursor_couple;
+
+/* The database environment. */
+struct MDBX_env {
+ /* ----------------------------------------------------- mostly static part */
+#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
+ MDBX_atomic_uint32_t me_signature;
+ /* Failed to update the meta page. Probably an I/O error. */
+#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
+ /* Some fields are initialized. */
+#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
+ /* me_txkey is set */
+#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
+ /* Legacy MDBX_MAPASYNC (prior v0.9) */
+#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
+#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
+ uint32_t me_flags;
+ mdbx_mmap_t me_dxb_mmap; /* The main data file */
+#define me_map me_dxb_mmap.dxb
+#define me_lazy_fd me_dxb_mmap.fd
+ mdbx_filehandle_t me_dsync_fd;
+ mdbx_mmap_t me_lck_mmap; /* The lock file */
+#define me_lfd me_lck_mmap.fd
+ struct MDBX_lockinfo *me_lck;
+
+ unsigned me_psize; /* DB page size, initialized from me_os_psize */
+ unsigned me_leaf_nodemax; /* max size of a leaf-node */
+ uint8_t me_psize2log; /* log2 of DB page size */
+ int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
+ uint16_t me_merge_threshold,
+ me_merge_threshold_gc; /* pages emptier than this are candidates for
+ merging */
+ unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
+ unsigned me_maxreaders; /* size of the reader table */
+ MDBX_dbi me_maxdbs; /* size of the DB table */
+ uint32_t me_pid; /* process ID of this env */
+ mdbx_thread_key_t me_txkey; /* thread-key for readers */
+ char *me_pathname; /* path to the DB files */
+ void *me_pbuf; /* scratch area for DUPSORT put() */
+ MDBX_txn *me_txn0; /* preallocated write transaction */
+
+ MDBX_dbx *me_dbxs; /* array of static DB info */
+ uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
+ unsigned *me_dbiseqs; /* array of dbi sequence numbers */
+ unsigned
+ me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
+ uint32_t me_live_reader; /* have liveness lock in reader table */
+ void *me_userctx; /* User-settable context */
+ MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
+
+ struct {
+ unsigned dp_reserve_limit;
+ unsigned rp_augment_limit;
+ unsigned dp_limit;
+ unsigned dp_initial;
+ uint8_t dp_loose_limit;
+ uint8_t spill_max_denominator;
+ uint8_t spill_min_denominator;
+ uint8_t spill_parent4child_denominator;
+ unsigned merge_threshold_16dot16_percent;
+ union {
+ unsigned all;
+ /* tracks options with non-auto values but tuned by user */
+ struct {
+ unsigned dp_limit : 1;
+ } non_auto;
+ } flags;
+ } me_options;
+
+ /* struct me_dbgeo used for accepting db-geo params from user for the new
+ * database creation, i.e. when mdbx_env_set_geometry() was called before
+ * mdbx_env_open(). */
+ struct {
+ size_t lower; /* minimal size of datafile */
+ size_t upper; /* maximal size of datafile */
+ size_t now; /* current size of datafile */
+ size_t grow; /* step to grow datafile */
+ size_t shrink; /* threshold to shrink datafile */
+ } me_dbgeo;
+
+#if MDBX_LOCKING == MDBX_LOCKING_SYSV
+ union {
+ key_t key;
+ int semid;
+ } me_sysv_ipc;
+#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
+
+ MDBX_env *me_lcklist_next;
+
+ /* --------------------------------------------------- mostly volatile part */
+
+ MDBX_txn *me_txn; /* current write transaction */
+ mdbx_fastmutex_t me_dbi_lock;
+ MDBX_dbi me_numdbs; /* number of DBs opened */
+
+ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
+ unsigned me_dp_reserve_len;
+ /* PNL of pages that became unused in a write txn */
+ MDBX_PNL me_retired_pages;
+
+#if defined(_WIN32) || defined(_WIN64)
+ MDBX_srwlock me_remap_guard;
+ /* Workaround for LockFileEx and WriteFile multithread bug */
+ CRITICAL_SECTION me_windowsbug_lock;
+#else
+ mdbx_fastmutex_t me_remap_guard;
+#endif
+
+ /* -------------------------------------------------------------- debugging */
+
+#if MDBX_DEBUG
+ MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
+#endif
+#ifdef MDBX_USE_VALGRIND
+ int me_valgrind_handle;
+#endif
+#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
+ pgno_t me_poison_edge;
+#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
+
+#ifndef xMDBX_DEBUG_SPILLING
+#define xMDBX_DEBUG_SPILLING 0
+#endif
+#if xMDBX_DEBUG_SPILLING == 2
+ unsigned debug_dirtied_est, debug_dirtied_act;
+#endif /* xMDBX_DEBUG_SPILLING */
+
+ /* ------------------------------------------------- stub for lck-less mode */
+ MDBX_atomic_uint64_t
+ x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
+ sizeof(MDBX_atomic_uint64_t)];
+};
+
+#ifndef __cplusplus
+/*----------------------------------------------------------------------------*/
+/* Debug and Logging stuff */
+
+#define MDBX_RUNTIME_FLAGS_INIT \
+ ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
+
+extern uint8_t mdbx_runtime_flags;
+extern uint8_t mdbx_loglevel;
+extern MDBX_debug_func *mdbx_debug_logger;
+
+MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
+#if MDBX_DEBUG
+ if (MDBX_DBG_JITTER & mdbx_runtime_flags)
+ mdbx_osal_jitter(tiny);
+#else
+ (void)tiny;
+#endif
+}
+
+MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
+ mdbx_debug_log(int level, const char *function, int line, const char *fmt,
+ ...) MDBX_PRINTF_ARGS(4, 5);
+MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
+ int line, const char *fmt,
+ va_list args);
+
+#if MDBX_DEBUG
+#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
+#else /* MDBX_DEBUG */
+#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
+#define mdbx_audit_enabled() (0)
+#endif /* MDBX_DEBUG */
+
+#if MDBX_FORCE_ASSERTIONS
+#define mdbx_assert_enabled() (1)
+#elif MDBX_DEBUG
+#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
+#else
+#define mdbx_assert_enabled() (0)
+#endif /* assertions */
+
+#if !MDBX_DEBUG && defined(__ANDROID_API__)
+#define mdbx_assert_fail(env, msg, func, line) \
+ __android_log_assert(msg, "mdbx", "%s:%u", func, line)
+#else
+void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
+ int line);
+#endif
+
+#define mdbx_debug_extra(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug_extra_print(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
+ mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_trace(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
+ mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_debug(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
+ mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_verbose(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
+ mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_notice(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
+ mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_warning(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_WARN)) \
+ mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_error(fmt, ...) \
+ do { \
+ if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
+ mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
+ __VA_ARGS__); \
+ } while (0)
+
+#define mdbx_fatal(fmt, ...) \
+ mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
+
+#define mdbx_ensure_msg(env, expr, msg) \
+ do { \
+ if (unlikely(!(expr))) \
+ mdbx_assert_fail(env, msg, __func__, __LINE__); \
+ } while (0)
+
+#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
+
+/* assert(3) variant in environment context */
+#define mdbx_assert(env, expr) \
+ do { \
+ if (mdbx_assert_enabled()) \
+ mdbx_ensure(env, expr); \
+ } while (0)
+
+/* assert(3) variant in cursor context */
+#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
+
+/* assert(3) variant in transaction context */
+#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
+
+#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
+#undef assert
+#define assert(expr) mdbx_assert(NULL, expr)
+#endif
+
+/*----------------------------------------------------------------------------*/
+/* Cache coherence and mmap invalidation */
+
+#if MDBX_CPU_WRITEBACK_INCOHERENT
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
+#else
+#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
+#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
+
+MDBX_MAYBE_UNUSED static __inline void
+mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
+#if MDBX_MMAP_INCOHERENT_FILE_WRITE
+ char *const begin = (char *)(-pagesize & (intptr_t)addr);
+ char *const end =
+ (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
+ int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
+ mdbx_assert(nullptr, err == 0);
+ (void)err;
+#else
+ (void)pagesize;
+#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
+
+#if MDBX_MMAP_INCOHERENT_CPU_CACHE
+#ifdef DCACHE
+ /* MIPS has cache coherency issues.
+ * Note: for any nbytes >= on-chip cache size, entire is flushed. */
+ cacheflush(addr, nbytes, DCACHE);
+#else
+#error "Oops, cacheflush() not available"
+#endif /* DCACHE */
+#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
+
+#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
+ (void)addr;
+ (void)nbytes;
+#endif
+}
+
+/*----------------------------------------------------------------------------*/
+/* Internal prototypes */
+
+MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
+ int *dead);
+MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
+ MDBX_reader *begin, MDBX_reader *end);
+MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
+
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
+MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
+
+#endif /* !__cplusplus */
+
+#define MDBX_IS_ERROR(rc) \
+ ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
+
+/* Internal error codes, not exposed outside libmdbx */
+#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
+
+/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
+#define DDBI(mc) \
+ (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
+
+/* Key size which fits in a DKBUF (debug key buffer). */
+#define DKBUF_MAX 511
+#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
+#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
+#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
+
+#if MDBX_DEBUG
+#define DKBUF_DEBUG DKBUF
+#define DKEY_DEBUG(x) DKEY(x)
+#define DVAL_DEBUG(x) DVAL(x)
+#else
+#define DKBUF_DEBUG ((void)(0))
+#define DKEY_DEBUG(x) ("-")
+#define DVAL_DEBUG(x) ("-")
+#endif
+
+/* An invalid page number.
+ * Mainly used to denote an empty tree. */
+#define P_INVALID (~(pgno_t)0)
+
+/* Test if the flags f are set in a flag word w. */
+#define F_ISSET(w, f) (((w) & (f)) == (f))
+
+/* Round n up to an even number. */
+#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
+
+/* Default size of memory map.
+ * This is certainly too small for any actual applications. Apps should
+ * always set the size explicitly using mdbx_env_set_geometry(). */
+#define DEFAULT_MAPSIZE MEGABYTE
+
+/* Number of slots in the reader table.
+ * This value was chosen somewhat arbitrarily. The 61 is a prime number,
+ * and such readers plus a couple mutexes fit into single 4KB page.
+ * Applications should set the table size using mdbx_env_set_maxreaders(). */
+#define DEFAULT_READERS 61
+
+/* Test if a page is a leaf page */
+#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
+/* Test if a page is a LEAF2 page */
+#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
+/* Test if a page is a branch page */
+#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
+/* Test if a page is an overflow page */
+#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
+/* Test if a page is a sub page */
+#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
+
+#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
+
+/* Header for a single key/data pair within a page.
+ * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
+ * We guarantee 2-byte alignment for 'MDBX_node's.
+ *
+ * Leaf node flags describe node contents. F_BIGDATA says the node's
+ * data part is the page number of an overflow page with actual data.
+ * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
+ * a sub-page/sub-database, and named databases (just F_SUBDATA). */
+typedef struct MDBX_node {
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union {
+ uint32_t mn_dsize;
+ uint32_t mn_pgno32;
+ };
+ uint8_t mn_flags; /* see mdbx_node flags */
+ uint8_t mn_extra;
+ uint16_t mn_ksize; /* key size */
+#else
+ uint16_t mn_ksize; /* key size */
+ uint8_t mn_extra;
+ uint8_t mn_flags; /* see mdbx_node flags */
+ union {
+ uint32_t mn_pgno32;
+ uint32_t mn_dsize;
+ };
+#endif /* __BYTE_ORDER__ */
+
+ /* mdbx_node Flags */
+#define F_BIGDATA 0x01 /* data put on overflow page */
+#define F_SUBDATA 0x02 /* data is a sub-database */
+#define F_DUPDATA 0x04 /* data has duplicates */
+
+ /* valid flags for mdbx_node_add() */
+#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
+
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
+ (!defined(__cplusplus) && defined(_MSC_VER))
+ uint8_t mn_data[] /* key and data are appended here */;
+#endif /* C99 */
+} MDBX_node;
+
+#define DB_PERSISTENT_FLAGS \
+ (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
+ MDBX_INTEGERDUP | MDBX_REVERSEDUP)
+
+/* mdbx_dbi_open() flags */
+#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
+
+#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
+#define DB_INTERNAL_FLAGS DB_VALID
+
+#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
+#error "Oops, some flags overlapped or wrong"
+#endif
+
+/* max number of pages to commit in one writev() call */
+#define MDBX_COMMIT_PAGES 64
+#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
+#undef MDBX_COMMIT_PAGES
+#define MDBX_COMMIT_PAGES IOV_MAX
+#endif
+
+/*
+ * /
+ * | -1, a < b
+ * CMP2INT(a,b) = < 0, a == b
+ * | 1, a > b
+ * \
+ */
+#ifndef __e2k__
+/* LY: fast enough on most systems */
+#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
+#else
+/* LY: more parallelable on VLIW Elbrus */
+#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
+#endif
+
+/* Do not spill pages to disk if txn is getting full, may fail instead */
+#define MDBX_NOSPILL 0x8000
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+int64pgno(int64_t i64) {
+ if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
+ return (pgno_t)i64;
+ return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_add(size_t base, size_t augend) {
+ assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
+ return int64pgno(base + augend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
+pgno_sub(size_t base, size_t subtrahend) {
+ assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
+ subtrahend < MAX_PAGENO);
+ return int64pgno(base - subtrahend);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
+is_powerof2(size_t x) {
+ return (x & (x - 1)) == 0;
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+floor_powerof2(size_t value, size_t granularity) {
+ assert(is_powerof2(granularity));
+ return value & ~(granularity - 1);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
+ceil_powerof2(size_t value, size_t granularity) {
+ return floor_powerof2(value + granularity - 1, granularity);
+}
+
+MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
+log2n_powerof2(size_t value) {
+ assert(value > 0 && value < INT32_MAX && is_powerof2(value));
+ assert((value & -(int32_t)value) == value);
+#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
+ return __builtin_ctzl(value);
+#elif defined(_MSC_VER)
+ unsigned long index;
+ _BitScanForward(&index, (unsigned long)value);
+ return index;
+#else
+ static const uint8_t debruijn_ctz32[32] = {
+ 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+ 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
+ return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
+#endif
+}
+
+/* Only a subset of the mdbx_env flags can be changed
+ * at runtime. Changing other flags requires closing the
+ * environment and re-opening it with the new flags. */
+#define ENV_CHANGEABLE_FLAGS \
+ (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
+ MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
+#define ENV_CHANGELESS_FLAGS \
+ (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
+ MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
+#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
+
+#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
+MDBX_MAYBE_UNUSED static void static_checks(void) {
+ STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
+ "Oops, MDBX_MAX_DBI or CORE_DBS?");
+ STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
+ ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
+ (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
+ "Oops, some flags overlapped or wrong");
+ STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
+ "Oops, some flags overlapped or wrong");
+}
+#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_POISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+ do { \
+ mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
+ (size_t)(size), __LINE__); \
+ ASAN_UNPOISON_MEMORY_REGION(addr, size); \
+ } while (0)
+
+#if defined(_WIN32) || defined(_WIN64)
+/*
+ * POSIX getopt for Windows
+ *
+ * AT&T Public License
+ *
+ * Code given out at the 1985 UNIFORUM conference in Dallas.
+ */
+
+/*----------------------------------------------------------------------------*/
+/* Microsoft compiler generates a lot of warning for self includes... */
+
+#ifdef _MSC_VER
+#pragma warning(push, 1)
+#pragma warning(disable : 4548) /* expression before comma has no effect; \
+ expected expression with side - effect */
+#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
+ * semantics are not enabled. Specify /EHsc */
+#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
+ * mode specified; termination on exception is \
+ * not guaranteed. Specify /EHsc */
+#if !defined(_CRT_SECURE_NO_WARNINGS)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#endif /* _MSC_VER (warnings) */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+/*----------------------------------------------------------------------------*/
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#ifndef EOF
+#define EOF (-1)
+#endif
+
+int optind = 1;
+int optopt;
+char *optarg;
+
+int getopt(int argc, char *const argv[], const char *opts) {
+ static int sp = 1;
+ int c;
+ const char *cp;
+
+ if (sp == 1) {
+ if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
+ return EOF;
+ else if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ return EOF;
+ }
+ }
+ optopt = c = argv[optind][sp];
+ if (c == ':' || (cp = strchr(opts, c)) == NULL) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
+ if (argv[optind][++sp] == '\0') {
+ optind++;
+ sp = 1;
+ }
+ return '?';
+ }
+ if (*++cp == ':') {
+ if (argv[optind][sp + 1] != '\0')
+ optarg = &argv[optind++][sp + 1];
+ else if (++optind >= argc) {
+ fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
+ c);
+ sp = 1;
+ return '?';
+ } else
+ optarg = argv[optind++];
+ sp = 1;
+ } else {
+ if (argv[optind][++sp] == '\0') {
+ sp = 1;
+ optind++;
+ }
+ optarg = NULL;
+ }
+ return c;
+}
+
+static volatile BOOL user_break;
+static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
+ (void)dwCtrlType;
+ user_break = true;
+ return true;
+}
+
+#else /* WINDOWS */
+
+static volatile sig_atomic_t user_break;
+static void signal_handler(int sig) {
+ (void)sig;
+ user_break = 1;
+}
+
+#endif /* !WINDOWS */
+
+static void print_stat(MDBX_stat *ms) {
+ printf(" Pagesize: %u\n", ms->ms_psize);
+ printf(" Tree depth: %u\n", ms->ms_depth);
+ printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
+ printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
+ printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
+ printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
+}
+
+static void usage(const char *prog) {
+ fprintf(stderr,
+ "usage: %s [-V] [-q] [-e] [-f[f[f]]] [-r[r]] [-a|-s name] dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -p\t\tshow statistics of page operations for current session\n"
+ " -e\t\tshow whole DB info\n"
+ " -f\t\tshow GC info\n"
+ " -r\t\tshow readers\n"
+ " -a\t\tprint stat of main DB and all subDBs\n"
+ " -s name\tprint stat of only the specified named subDB\n"
+ " \t\tby default print stat of only the main DB\n",
+ prog);
+ exit(EXIT_FAILURE);
+}
+
+static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid,
+ mdbx_tid_t thread, uint64_t txnid, uint64_t lag,
+ size_t bytes_used, size_t bytes_retained) {
+ (void)ctx;
+ if (num == 1)
+ printf("Reader Table\n"
+ " #\tslot\t%6s %*s %20s %10s %13s %13s\n",
+ "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used",
+ "retained");
+
+ printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid,
+ (int)sizeof(size_t) * 2, (uintptr_t)thread);
+ if (txnid)
+ printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag,
+ bytes_used / 1048576.0, bytes_retained / 1048576.0);
+ else
+ printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0");
+
+ return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
+}
+
+const char *prog;
+bool quiet = false;
+static void error(const char *func, int rc) {
+ if (!quiet)
+ fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
+ mdbx_strerror(rc));
+}
+
+int main(int argc, char *argv[]) {
+ int opt, rc;
+ MDBX_env *env;
+ MDBX_txn *txn;
+ MDBX_dbi dbi;
+ MDBX_envinfo mei;
+ prog = argv[0];
+ char *envname;
+ char *subname = nullptr;
+ bool alldbs = false, envinfo = false, pgop = false;
+ int freinfo = 0, rdrinfo = 0;
+
+ if (argc < 2)
+ usage(prog);
+
+ while ((opt = getopt(argc, argv,
+ "V"
+ "q"
+ "p"
+ "a"
+ "e"
+ "f"
+ "n"
+ "r"
+ "s:")) != EOF) {
+ switch (opt) {
+ case 'V':
+ printf("mdbx_stat version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ mdbx_version.major, mdbx_version.minor, mdbx_version.release,
+ mdbx_version.revision, mdbx_version.git.describe,
+ mdbx_version.git.datetime, mdbx_version.git.commit,
+ mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
+ mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
+ mdbx_build.options);
+ return EXIT_SUCCESS;
+ case 'q':
+ quiet = true;
+ break;
+ case 'p':
+ pgop = true;
+ break;
+ case 'a':
+ if (subname)
+ usage(prog);
+ alldbs = true;
+ break;
+ case 'e':
+ envinfo = true;
+ break;
+ case 'f':
+ freinfo += 1;
+ break;
+ case 'n':
+ break;
+ case 'r':
+ rdrinfo += 1;
+ break;
+ case 's':
+ if (alldbs)
+ usage(prog);
+ subname = optarg;
+ break;
+ default:
+ usage(prog);
+ }
+ }
+
+ if (optind != argc - 1)
+ usage(prog);
+
+#if defined(_WIN32) || defined(_WIN64)
+ SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
+#else
+#ifdef SIGPIPE
+ signal(SIGPIPE, signal_handler);
+#endif
+#ifdef SIGHUP
+ signal(SIGHUP, signal_handler);
+#endif
+ signal(SIGINT, signal_handler);
+ signal(SIGTERM, signal_handler);
+#endif /* !WINDOWS */
+
+ envname = argv[optind];
+ envname = argv[optind];
+ if (!quiet) {
+ printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n",
+ mdbx_version.git.describe, mdbx_version.git.datetime,
+ mdbx_version.git.tree, envname);
+ fflush(nullptr);
+ }
+
+ rc = mdbx_env_create(&env);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_create", rc);
+ return EXIT_FAILURE;
+ }
+
+ if (alldbs || subname) {
+ rc = mdbx_env_set_maxdbs(env, 2);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_set_maxdbs", rc);
+ goto env_close;
+ }
+ }
+
+ rc = mdbx_env_open(env, envname, MDBX_RDONLY, 0);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_open", rc);
+ goto env_close;
+ }
+
+ rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_txn_begin", rc);
+ goto txn_abort;
+ }
+
+ if (envinfo || freinfo || pgop) {
+ rc = mdbx_env_info_ex(env, txn, &mei, sizeof(mei));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_env_info_ex", rc);
+ goto txn_abort;
+ }
+ } else {
+ /* LY: zap warnings from gcc */
+ memset(&mei, 0, sizeof(mei));
+ }
+
+ if (pgop) {
+ printf("Page Operations (for current session):\n");
+ printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n",
+ mei.mi_pgop_stat.newly);
+ printf(" CoW: %8" PRIu64
+ "\t// quantity of pages copied for altering\n",
+ mei.mi_pgop_stat.cow);
+ printf(" Clone: %8" PRIu64 "\t// quantity of parent's dirty pages "
+ "clones for nested transactions\n",
+ mei.mi_pgop_stat.clone);
+ printf(" Split: %8" PRIu64
+ "\t// page splits during insertions or updates\n",
+ mei.mi_pgop_stat.split);
+ printf(" Merge: %8" PRIu64
+ "\t// page merges during deletions or updates\n",
+ mei.mi_pgop_stat.merge);
+ printf(" Spill: %8" PRIu64 "\t// quantity of spilled/ousted `dirty` "
+ "pages during large transactions\n",
+ mei.mi_pgop_stat.spill);
+ printf(" Unspill: %8" PRIu64 "\t// quantity of unspilled/redone `dirty` "
+ "pages during large transactions\n",
+ mei.mi_pgop_stat.unspill);
+ printf(" WOP: %8" PRIu64
+ "\t// number of explicit write operations (not a pages) to a disk\n",
+ mei.mi_pgop_stat.wops);
+ }
+
+ if (envinfo) {
+ printf("Environment Info\n");
+ printf(" Pagesize: %u\n", mei.mi_dxb_pagesize);
+ if (mei.mi_geo.lower != mei.mi_geo.upper) {
+ printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
+ "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
+ "/-%" PRIu64 ")\n",
+ mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
+ mei.mi_geo.shrink, mei.mi_geo.lower / mei.mi_dxb_pagesize,
+ mei.mi_geo.upper / mei.mi_dxb_pagesize,
+ mei.mi_geo.grow / mei.mi_dxb_pagesize,
+ mei.mi_geo.shrink / mei.mi_dxb_pagesize);
+ printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
+ mei.mi_mapsize, mei.mi_mapsize / mei.mi_dxb_pagesize);
+ printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
+ mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
+#if defined(_WIN32) || defined(_WIN64)
+ if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper)
+ printf(" WARNING: Due Windows system limitations a "
+ "file couldn't\n be truncated while database "
+ "is opened. So, the size of\n database file "
+ "may by large than the database itself,\n "
+ "until it will be closed or reopened in read-write mode.\n");
+#endif
+ } else {
+ printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
+ mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
+ }
+ printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
+ printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
+ mei.mi_latter_reader_txnid,
+ mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
+ printf(" Max readers: %u\n", mei.mi_maxreaders);
+ printf(" Number of reader slots uses: %u\n", mei.mi_numreaders);
+ }
+
+ if (rdrinfo) {
+ rc = mdbx_reader_list(env, reader_list_func, nullptr);
+ if (MDBX_IS_ERROR(rc)) {
+ error("mdbx_reader_list", rc);
+ goto txn_abort;
+ }
+ if (rc == MDBX_RESULT_TRUE)
+ printf("Reader Table is empty\n");
+ else if (rc == MDBX_SUCCESS && rdrinfo > 1) {
+ int dead;
+ rc = mdbx_reader_check(env, &dead);
+ if (MDBX_IS_ERROR(rc)) {
+ error("mdbx_reader_check", rc);
+ goto txn_abort;
+ }
+ if (rc == MDBX_RESULT_TRUE) {
+ printf(" %d stale readers cleared.\n", dead);
+ rc = mdbx_reader_list(env, reader_list_func, nullptr);
+ if (rc == MDBX_RESULT_TRUE)
+ printf(" Now Reader Table is empty\n");
+ } else
+ printf(" No stale readers.\n");
+ }
+ if (!(subname || alldbs || freinfo))
+ goto txn_abort;
+ }
+
+ if (freinfo) {
+ printf("Garbage Collection\n");
+ dbi = 0;
+ MDBX_cursor *cursor;
+ rc = mdbx_cursor_open(txn, dbi, &cursor);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_open", rc);
+ goto txn_abort;
+ }
+
+ MDBX_stat mst;
+ rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_stat", rc);
+ goto txn_abort;
+ }
+ print_stat(&mst);
+
+ pgno_t pages = 0, *iptr;
+ pgno_t reclaimable = 0;
+ MDBX_val key, data;
+ while (MDBX_SUCCESS ==
+ (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) {
+ if (user_break) {
+ rc = MDBX_EINTR;
+ break;
+ }
+ iptr = data.iov_base;
+ const pgno_t number = *iptr++;
+
+ pages += number;
+ if (envinfo && mei.mi_latter_reader_txnid > *(txnid_t *)key.iov_base)
+ reclaimable += number;
+
+ if (freinfo > 1) {
+ char *bad = "";
+ pgno_t prev =
+ MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
+ pgno_t span = 1;
+ for (unsigned i = 0; i < number; ++i) {
+ pgno_t pg = iptr[i];
+ if (MDBX_PNL_DISORDERED(prev, pg))
+ bad = " [bad sequence]";
+ prev = pg;
+ while (i + span < number &&
+ iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
+ : pgno_sub(pg, span)))
+ ++span;
+ }
+ printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
+ " pages, maxspan %" PRIaPGNO "%s\n",
+ *(txnid_t *)key.iov_base, number, span, bad);
+ if (freinfo > 2) {
+ for (unsigned i = 0; i < number; i += span) {
+ const pgno_t pg = iptr[i];
+ for (span = 1;
+ i + span < number &&
+ iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
+ : pgno_sub(pg, span));
+ ++span)
+ ;
+ if (span > 1)
+ printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span);
+ else
+ printf(" %9" PRIaPGNO "\n", pg);
+ }
+ }
+ }
+ }
+ mdbx_cursor_close(cursor);
+ cursor = nullptr;
+
+ switch (rc) {
+ case MDBX_SUCCESS:
+ case MDBX_NOTFOUND:
+ break;
+ case MDBX_EINTR:
+ if (!quiet)
+ fprintf(stderr, "Interrupted by signal/user\n");
+ goto txn_abort;
+ default:
+ error("mdbx_cursor_get", rc);
+ goto txn_abort;
+ }
+
+ if (envinfo) {
+ uint64_t value = mei.mi_mapsize / mei.mi_dxb_pagesize;
+ double percent = value / 100.0;
+ printf("Page Usage\n");
+ printf(" Total: %" PRIu64 " 100%%\n", value);
+
+ value = mei.mi_geo.current / mei.mi_dxb_pagesize;
+ printf(" Backed: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = mei.mi_last_pgno + 1;
+ printf(" Allocated: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1);
+ printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = mei.mi_last_pgno + 1 - pages;
+ printf(" Used: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = pages;
+ printf(" GC: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = pages - reclaimable;
+ printf(" Retained: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = reclaimable;
+ printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
+
+ value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) +
+ reclaimable;
+ printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
+ } else
+ printf(" GC: %" PRIaPGNO " pages\n", pages);
+ }
+
+ rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_open", rc);
+ goto txn_abort;
+ }
+
+ MDBX_stat mst;
+ rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_stat", rc);
+ goto txn_abort;
+ }
+ printf("Status of %s\n", subname ? subname : "Main DB");
+ print_stat(&mst);
+
+ if (alldbs) {
+ MDBX_cursor *cursor;
+ rc = mdbx_cursor_open(txn, dbi, &cursor);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_cursor_open", rc);
+ goto txn_abort;
+ }
+
+ MDBX_val key;
+ while (MDBX_SUCCESS ==
+ (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
+ MDBX_dbi subdbi;
+ if (memchr(key.iov_base, '\0', key.iov_len))
+ continue;
+ subname = mdbx_malloc(key.iov_len + 1);
+ memcpy(subname, key.iov_base, key.iov_len);
+ subname[key.iov_len] = '\0';
+ rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &subdbi);
+ if (rc == MDBX_SUCCESS)
+ printf("Status of %s\n", subname);
+ mdbx_free(subname);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ if (rc == MDBX_INCOMPATIBLE)
+ continue;
+ error("mdbx_dbi_open", rc);
+ goto txn_abort;
+ }
+
+ rc = mdbx_dbi_stat(txn, subdbi, &mst, sizeof(mst));
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_stat", rc);
+ goto txn_abort;
+ }
+ print_stat(&mst);
+
+ rc = mdbx_dbi_close(env, subdbi);
+ if (unlikely(rc != MDBX_SUCCESS)) {
+ error("mdbx_dbi_close", rc);
+ goto txn_abort;
+ }
+ }
+ mdbx_cursor_close(cursor);
+ cursor = nullptr;
+ }
+
+ switch (rc) {
+ case MDBX_SUCCESS:
+ case MDBX_NOTFOUND:
+ break;
+ case MDBX_EINTR:
+ if (!quiet)
+ fprintf(stderr, "Interrupted by signal/user\n");
+ break;
+ default:
+ if (unlikely(rc != MDBX_SUCCESS))
+ error("mdbx_cursor_get", rc);
+ }
+
+ mdbx_dbi_close(env, dbi);
+txn_abort:
+ mdbx_txn_abort(txn);
+env_close:
+ mdbx_env_close(env);
+
+ return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/3rdparty/libmdbx/src/ntdll.def b/3rdparty/libmdbx/ntdll.def
similarity index 100%
rename from 3rdparty/libmdbx/src/ntdll.def
rename to 3rdparty/libmdbx/ntdll.def
diff --git a/3rdparty/libmdbx/packages/buildroot/0001-package-libmdbx-new-package-library-database.patch b/3rdparty/libmdbx/packages/buildroot/0001-package-libmdbx-new-package-library-database.patch
deleted file mode 100644
index 8b34d40c2b3f18848ed36171e9e722e46c2a2d24..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/packages/buildroot/0001-package-libmdbx-new-package-library-database.patch
+++ /dev/null
@@ -1,165 +0,0 @@
-From 4c309b57bca0526b0c110990c315682f027ccfad Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?=
- =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= <leo@yuriev.ru>
-Date: Thu, 28 Apr 2022 11:02:59 +0300
-Subject: [PATCH] package/libmdbx: new package (library/database).
-
-This patch adds libmdbx v0.11.7:
- - libmdbx is one of the fastest compact embeddable key-value ACID database.
- - libmdbx has a specific set of properties and capabilities,
- focused on creating unique lightweight solutions.
- - libmdbx surpasses the legendary LMDB (Lightning Memory-Mapped Database)
- in terms of reliability, features and performance.
- - https://libmdbx.dqdkfa.ru/
-
-Signed-off-by: Leonid Yuriev <leo@yuriev.ru>
----
- DEVELOPERS | 3 +++
- package/Config.in | 1 +
- package/libmdbx/Config.in | 45 ++++++++++++++++++++++++++++++++++++
- package/libmdbx/libmdbx.hash | 5 ++++
- package/libmdbx/libmdbx.mk | 42 +++++++++++++++++++++++++++++++++
- 5 files changed, 96 insertions(+)
- create mode 100644 package/libmdbx/Config.in
- create mode 100644 package/libmdbx/libmdbx.hash
- create mode 100644 package/libmdbx/libmdbx.mk
-
-diff --git a/DEVELOPERS b/DEVELOPERS
-index 9ab1e125f4..758ff6a2d5 100644
---- a/DEVELOPERS
-+++ b/DEVELOPERS
-@@ -1482,6 +1482,9 @@ N: Leon Anavi <leon.anavi@konsulko.com>
- F: board/olimex/a10_olinuxino
- F: configs/olimex_a10_olinuxino_lime_defconfig
-
-+N: Leonid Yuriev <leo@yuriev.ru>
-+F: package/libmdbx/
-+
- N: Lionel Flandrin <lionel@svkt.org>
- F: package/python-babel/
- F: package/python-daemonize/
-diff --git a/package/Config.in b/package/Config.in
-index 016a99ed1a..a6f95bfaa9 100644
---- a/package/Config.in
-+++ b/package/Config.in
-@@ -1372,6 +1372,7 @@ menu "Database"
- source "package/kompexsqlite/Config.in"
- source "package/leveldb/Config.in"
- source "package/libgit2/Config.in"
-+ source "package/libmdbx/Config.in"
- source "package/libodb/Config.in"
- source "package/libodb-boost/Config.in"
- source "package/libodb-mysql/Config.in"
-diff --git a/package/libmdbx/Config.in b/package/libmdbx/Config.in
-new file mode 100644
-index 0000000000..ef44a955fe
---- /dev/null
-+++ b/package/libmdbx/Config.in
-@@ -0,0 +1,45 @@
-+config BR2_PACKAGE_LIBMDBX
-+ bool "libmdbx"
-+ depends on BR2_USE_MMU
-+ depends on BR2_TOOLCHAIN_HAS_SYNC_4
-+ depends on BR2_TOOLCHAIN_HAS_THREADS
-+ depends on BR2_TOOLCHAIN_GCC_AT_LEAST_4_4
-+ help
-+ One of the fastest compact key-value ACID database
-+ without WAL. libmdbx has a specific set of properties
-+ and capabilities, focused on creating unique lightweight
-+ solutions.
-+
-+ libmdbx surpasses the legendary LMDB in terms of
-+ reliability, features and performance.
-+
-+ https://libmdbx.dqdkfa.ru/
-+
-+if BR2_PACKAGE_LIBMDBX
-+
-+config BR2_PACKAGE_LIBMDBX_TOOLS
-+ bool "install tools"
-+ help
-+ Install libmdbx tools for checking, dump, restore
-+ and show statistics of databases.
-+
-+config BR2_PACKAGE_LIBMDBX_CXX
-+ bool "C++ API"
-+ depends on BR2_INSTALL_LIBSTDCPP
-+ depends on BR2_TOOLCHAIN_GCC_AT_LEAST_4_8
-+ depends on !BR2_TOOLCHAIN_HAS_GCC_BUG_64735
-+ help
-+ Enable modern C++11/14/17/20 API for libmdbx.
-+
-+comment "libmdbx C++ support needs a toolchain w/ C++11, gcc >= 4.8 w/o bug#64735"
-+ depends on !BR2_INSTALL_LIBSTDCPP || \
-+ !BR2_TOOLCHAIN_GCC_AT_LEAST_4_8 || \
-+ BR2_TOOLCHAIN_HAS_GCC_BUG_64735
-+
-+endif
-+
-+comment "libmdbx needs MMU, a toolchain w/ threads, gcc >= 4.4 w/ 4-byte atomics"
-+ depends on BR2_USE_MMU
-+ depends on !BR2_TOOLCHAIN_HAS_THREADS || \
-+ !BR2_TOOLCHAIN_HAS_SYNC_4 || \
-+ !BR2_TOOLCHAIN_GCC_AT_LEAST_4_4
-diff --git a/package/libmdbx/libmdbx.hash b/package/libmdbx/libmdbx.hash
-new file mode 100644
-index 0000000000..61f800d433
---- /dev/null
-+++ b/package/libmdbx/libmdbx.hash
-@@ -0,0 +1,5 @@
-+# Hashes from https://libmdbx.dqdkfa.ru//release/SHA256SUMS
-+sha256 3a9fb6a4cd941e646597235518714373fda1ca6d4c5e23669afe70ea87c20940 libmdbx-amalgamated-0.11.7.tar.xz
-+
-+# Locally calculated
-+sha256 310fe25c858a9515fc8c8d7d1f24a67c9496f84a91e0a0e41ea9975b1371e569 LICENSE
-diff --git a/package/libmdbx/libmdbx.mk b/package/libmdbx/libmdbx.mk
-new file mode 100644
-index 0000000000..ab279614ab
---- /dev/null
-+++ b/package/libmdbx/libmdbx.mk
-@@ -0,0 +1,42 @@
-+################################################################################
-+#
-+# libmdbx
-+#
-+################################################################################
-+
-+LIBMDBX_VERSION = 0.11.7
-+LIBMDBX_SOURCE = libmdbx-amalgamated-$(LIBMDBX_VERSION).tar.xz
-+LIBMDBX_SITE = https://libmdbx.dqdkfa.ru/release
-+LIBMDBX_SUPPORTS_IN_SOURCE_BUILD = NO
-+LIBMDBX_LICENSE = OLDAP-2.8
-+LIBMDBX_LICENSE_FILES = LICENSE
-+LIBMDBX_REDISTRIBUTE = YES
-+LIBMDBX_STRIP_COMPONENTS = 0
-+LIBMDBX_INSTALL_STAGING = YES
-+
-+# Set CMAKE_BUILD_TYPE to Release to remove -Werror and avoid a build failure
-+# with glibc < 2.12
-+LIBMDBX_CONF_OPTS = \
-+ -DCMAKE_BUILD_TYPE=Release \
-+ -DMDBX_INSTALL_MANPAGES=OFF \
-+ -DBUILD_FOR_NATIVE_CPU=OFF \
-+ -DMDBX_BUILD_CXX=$(if $(BR2_PACKAGE_LIBMDBX_CXX),ON,OFF) \
-+ -DMDBX_BUILD_TOOLS=$(if $(BR2_PACKAGE_LIBMDBX_TOOLS),ON,OFF)
-+
-+ifeq ($(BR2_STATIC_LIBS)$(BR2_SHARED_STATIC_LIBS),y)
-+LIBMDBX_CONF_OPTS += -DMDBX_INSTALL_STATIC=ON
-+else
-+LIBMDBX_CONF_OPTS += -DMDBX_INSTALL_STATIC=OFF
-+endif
-+
-+ifeq ($(BR2_SHARED_LIBS)$(BR2_SHARED_STATIC_LIBS),y)
-+LIBMDBX_CONF_OPTS += \
-+ -DMDBX_BUILD_SHARED_LIBRARY=ON \
-+ -DMDBX_LINK_TOOLS_NONSTATIC=ON
-+else
-+LIBMDBX_CONF_OPTS += \
-+ -DMDBX_BUILD_SHARED_LIBRARY=OFF \
-+ -DMDBX_LINK_TOOLS_NONSTATIC=OFF
-+endif
-+
-+$(eval $(cmake-package))
---
-2.32.0
-
diff --git a/3rdparty/libmdbx/packages/rpm/CMakeLists.txt b/3rdparty/libmdbx/packages/rpm/CMakeLists.txt
deleted file mode 100644
index 5949e9f0acfa834e66507116e077e487a2b0e631..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/packages/rpm/CMakeLists.txt
+++ /dev/null
@@ -1,184 +0,0 @@
-cmake_minimum_required(VERSION 2.8.7)
-set(TARGET mdbx)
-project(${TARGET})
-
-set(MDBX_VERSION_MAJOR 0)
-set(MDBX_VERSION_MINOR 3)
-set(MDBX_VERSION_RELEASE 1)
-set(MDBX_VERSION_REVISION 0)
-
-set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE})
-
-enable_language(C)
-enable_language(CXX)
-
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_CXX_STANDARD_REQUIRED on)
-
-add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1)
-
-find_package(Threads REQUIRED)
-
-get_directory_property(hasParent PARENT_DIRECTORY)
-if(hasParent)
- set(STANDALONE_BUILD 0)
-else()
- set(STANDALONE_BUILD 1)
- enable_testing()
-
- if (CMAKE_C_COMPILER_ID MATCHES GNU)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread")
- endif()
-
- if (CMAKE_CXX_COMPILER_ID MATCHES GNU)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
-
- set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3")
- set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3")
- endif()
-
- if (COVERAGE)
- if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
- message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}")
- endif()
-
- message(STATUS "Setting coverage compiler flags")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
- add_definitions(-DCOVERAGE_TEST)
- endif()
-
- if (NOT TRAVIS)
- set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan")
- endif()
-endif()
-
-set(${TARGET}_SRC
- mdbx.h
- src/bits.h
- src/defs.h
- src/lck-linux.c
- src/mdbx.c
- src/osal.c
- src/osal.h
- src/version.c
- )
-
-add_library(${TARGET}_STATIC STATIC
- ${${TARGET}_SRC}
- )
-
-add_library(${TARGET} ALIAS ${TARGET}_STATIC)
-
-add_library(${TARGET}_SHARED SHARED
- ${${TARGET}_SRC}
- )
-
-set_target_properties(${TARGET}_SHARED PROPERTIES
- VERSION ${MDBX_VERSION_STRING}
- SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
- OUTPUT_NAME ${TARGET}
- CLEAN_DIRECT_OUTPUT 1
- )
-
-set_target_properties(${TARGET}_STATIC PROPERTIES
- VERSION ${MDBX_VERSION_STRING}
- SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
- OUTPUT_NAME ${TARGET}
- CLEAN_DIRECT_OUTPUT 1
- )
-
-target_include_directories(${TARGET}_STATIC PUBLIC
- ${CMAKE_CURRENT_SOURCE_DIR})
-target_include_directories(${TARGET}_SHARED PUBLIC
- ${CMAKE_CURRENT_SOURCE_DIR})
-
-target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT})
-target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT})
-if(UNIX AND NOT APPLE)
- target_link_libraries(${TARGET}_STATIC rt)
- target_link_libraries(${TARGET}_SHARED rt)
-endif()
-
-install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
-install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
-install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel)
-
-add_subdirectory(src/tools)
-add_subdirectory(test)
-add_subdirectory(test/pcrf)
-add_subdirectory(tutorial)
-
-##############################################################################
-
-set(CPACK_GENERATOR "RPM")
-set(CPACK_RPM_COMPONENT_INSTALL ON)
-
-# Version
-if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "")
- set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER})
-else()
- if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "")
- set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID})
- else()
- set(CPACK_PACKAGE_RELEASE 1)
- endif()
-endif()
-set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
-
-set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING})
-set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE})
-
-set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}")
-
-set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true")
-set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx)
-set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel)
-set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB")
-
-set(CPACK_PACKAGE_VENDOR "???")
-set(CPACK_PACKAGE_CONTACT "Vladimir Romanov")
-set(CPACK_PACKAGE_RELOCATABLE false)
-set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
-set(CPACK_RPM_PACKAGE_REQUIRES "")
-set(CPACK_RPM_PACKAGE_GROUP "Applications/Database")
-
-set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
-set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
-
-set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
- /usr/local
- /usr/local/bin
- /usr/local/lib64
- /usr/local/include
- /usr/local/man
- /usr/local/man/man1
- )
-
-include(CPack)
diff --git a/3rdparty/libmdbx/packages/rpm/build.sh b/3rdparty/libmdbx/packages/rpm/build.sh
deleted file mode 100755
index 5170882265d758ecd93bb6481bc8ddabda437bac..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/packages/rpm/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-CONFIG=$1
-
-if [[ -z "${CONFIG}" ]]; then
- CONFIG=Debug
-fi
-if [[ -r /opt/rh/devtoolset-6/enable ]]; then
- source /opt/rh/devtoolset-6/enable
-fi
-#rm -f -r build || true
-mkdir -p cmake-build-${CONFIG}
-pushd cmake-build-${CONFIG} &> /dev/null
-if [[ ! -r Makefile ]]; then
- cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
-fi
-make -j8 || exit 1
-popd &> /dev/null
diff --git a/3rdparty/libmdbx/packages/rpm/package.sh b/3rdparty/libmdbx/packages/rpm/package.sh
deleted file mode 100755
index d7f9ab297ab09ce976e4e08677288ae5c9e5a335..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/packages/rpm/package.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-
-CONFIG=$1
-
-if [[ -z "${CONFIG}" ]]; then
- CONFIG=Debug
-fi
-
-DIRNAME=`dirname ${BASH_SOURCE[0]}`
-DIRNAME=`readlink --canonicalize ${DIRNAME}`
-
-if [[ -r /opt/rh/devtoolset-6/enable ]]; then
- source /opt/rh/devtoolset-6/enable
-fi
-
-mkdir -p cmake-build-${CONFIG}
-pushd cmake-build-${CONFIG} &> /dev/null
-if [[ ! -r Makefile ]]; then
- cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
-fi
-rm -f *.rpm
-make -j8 package || exit 1
-rm -f *-Unspecified.rpm
-popd &> /dev/null
diff --git a/3rdparty/libmdbx/src/alloy.c b/3rdparty/libmdbx/src/alloy.c
deleted file mode 100644
index 1e770f235fe9cf405cee80e6a986903d7211d363..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/alloy.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#define xMDBX_ALLOY 1 /* alloyed build */
-#include "internals.h" /* must be included first */
-
-#include "core.c"
-#include "osal.c"
-#include "version.c"
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "lck-windows.c"
-#else
-#include "lck-posix.c"
-#endif
diff --git a/3rdparty/libmdbx/src/bits.md b/3rdparty/libmdbx/src/bits.md
deleted file mode 100644
index 99cef8e875278911071284318cd1f8f396b42dc1..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/bits.md
+++ /dev/null
@@ -1,34 +0,0 @@
-N | MASK | ENV | TXN | DB | PUT | DBI | NODE | PAGE | MRESIZE |
---|---------|-----------|--------------|----------|-----------|------------|---------|----------|---------|
-0 |0000 0001|ALLOC_CACHE|TXN_FINISHED | | |DBI_DIRTY |F_BIGDATA|P_BRANCH | |
-1 |0000 0002|ALLOC_GC |TXN_ERROR |REVERSEKEY|F_SUBDATA |DBI_STALE |F_SUBDATA|P_LEAF | |
-2 |0000 0004|ALLOC_NEW |TXN_DIRTY |DUPSORT | |DBI_FRESH |F_DUPDATA|P_OVERFLOW| |
-3 |0000 0008|ALLOC_SLOT |TXN_SPILLS |INTEGERKEY| |DBI_CREAT | |P_META | |
-4 |0000 0010|ALLOC_FAKE |TXN_HAS_CHILD |DUPFIXED |NOOVERWRITE|DBI_VALID | |P_BAD | |
-5 |0000 0020| | |INTEGERDUP|NODUPDATA |DBI_USRVALID| |P_LEAF2 | |
-6 |0000 0040| | |REVERSEDUP|CURRENT |DBI_DUPDATA | |P_SUBP | |
-7 |0000 0080| | | |ALLDUPS |DBI_AUDITED | | | |
-8 |0000 0100| _MAY_MOVE | | | | | | | <= |
-9 |0000 0200| _MAY_UNMAP| | | | | | | <= |
-10|0000 0400| | | | | | | | |
-11|0000 0800| | | | | | | | |
-12|0000 1000| | | | | | | | |
-13|0000 2000| | | | | | |P_SPILLED | |
-14|0000 4000|NOSUBDIR | | | | | |P_LOOSE | |
-15|0000 8000| | |DB_VALID |NOSPILL | | |P_FROZEN | |
-16|0001 0000|SAFE_NOSYNC|TXN_NOSYNC | |RESERVE | |RESERVE | | |
-17|0002 0000|RDONLY |TXN_RDONLY | |APPEND | |APPEND | | <= |
-18|0004 0000|NOMETASYNC |TXN_NOMETASYNC|CREATE |APPENDDUP | | | | |
-19|0008 0000|WRITEMAP |<= | |MULTIPLE | | | | <= |
-20|0010 0000|UTTERLY | | | | | | | <= |
-21|0020 0000|NOTLS |<= | | | | | | |
-22|0040 0000|EXCLUSIVE | | | | | | | |
-23|0080 0000|NORDAHEAD | | | | | | | |
-24|0100 0000|NOMEMINIT |TXN_PREPARE | | | | | | |
-25|0200 0000|COALESCE | | | | | | | |
-26|0400 0000|LIFORECLAIM| | | | | | | |
-27|0800 0000|PAGEPERTURB| | | | | | | |
-28|1000 0000|ENV_TXKEY |TXN_TRY | | | | | | |
-29|2000 0000|ENV_ACTIVE | | | | | | | |
-30|4000 0000|ACCEDE |SHRINK_ALLOWED|DB_ACCEDE | | | | | |
-31|8000 0000|FATAL_ERROR| | | | | | | |
diff --git a/3rdparty/libmdbx/src/debug_begin.h b/3rdparty/libmdbx/src/debug_begin.h
deleted file mode 100644
index 9a90409583e2b2097712bd80974ce13a9450b82b..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/debug_begin.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#if defined(__GNUC__) && !defined(__LCC__)
-
-#pragma push_macro("mdbx_trace")
-#pragma push_macro("mdbx_debug")
-#pragma push_macro("mdbx_verbose")
-#pragma push_macro("mdbx_notice")
-#pragma push_macro("mdbx_warning")
-#pragma push_macro("mdbx_error")
-#pragma push_macro("mdbx_assert")
-
-#undef mdbx_trace
-#define mdbx_trace(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_debug
-#define mdbx_debug(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_verbose
-#define mdbx_verbose(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_notice
-#define mdbx_notice(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_warning
-#define mdbx_warning(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_error
-#define mdbx_error(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__)
-
-#undef mdbx_assert
-#define mdbx_assert(env, expr) mdbx_ensure(env, expr)
-
-#if !defined(__clang__)
-#pragma GCC optimize("-O0")
-#endif
-
-#endif /* GCC only */
diff --git a/3rdparty/libmdbx/src/debug_end.h b/3rdparty/libmdbx/src/debug_end.h
deleted file mode 100644
index a854f7159609e9f45ad41457a6c0407d21457ac7..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/debug_end.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#if defined(__GNUC__) && !defined(__LCC__)
-
-#pragma pop_macro("mdbx_trace")
-#pragma pop_macro("mdbx_debug")
-#pragma pop_macro("mdbx_verbose")
-#pragma pop_macro("mdbx_notice")
-#pragma pop_macro("mdbx_warning")
-#pragma pop_macro("mdbx_error")
-#pragma pop_macro("mdbx_assert")
-
-#if !defined(__clang__)
-#pragma GCC reset_options
-#endif
-
-#endif /* GCC only */
diff --git a/3rdparty/libmdbx/src/defs.h b/3rdparty/libmdbx/src/defs.h
deleted file mode 100644
index 92c16ed1c7076348a79c013f15fd49e5fb66966f..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/defs.h
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-/* *INDENT-OFF* */
-/* clang-format off */
-
-#ifndef __GNUC_PREREQ
-# if defined(__GNUC__) && defined(__GNUC_MINOR__)
-# define __GNUC_PREREQ(maj, min) \
- ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
-# else
-# define __GNUC_PREREQ(maj, min) (0)
-# endif
-#endif /* __GNUC_PREREQ */
-
-#ifndef __CLANG_PREREQ
-# ifdef __clang__
-# define __CLANG_PREREQ(maj,min) \
- ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
-# else
-# define __CLANG_PREREQ(maj,min) (0)
-# endif
-#endif /* __CLANG_PREREQ */
-
-#ifndef __GLIBC_PREREQ
-# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
-# define __GLIBC_PREREQ(maj, min) \
- ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
-# else
-# define __GLIBC_PREREQ(maj, min) (0)
-# endif
-#endif /* __GLIBC_PREREQ */
-
-#ifndef __has_warning
-# define __has_warning(x) (0)
-#endif
-
-#ifndef __has_include
-# define __has_include(x) (0)
-#endif
-
-#if __has_feature(thread_sanitizer)
-# define __SANITIZE_THREAD__ 1
-#endif
-
-#if __has_feature(address_sanitizer)
-# define __SANITIZE_ADDRESS__ 1
-#endif
-
-/*----------------------------------------------------------------------------*/
-
-#ifndef __extern_C
-# ifdef __cplusplus
-# define __extern_C extern "C"
-# else
-# define __extern_C
-# endif
-#endif /* __extern_C */
-
-#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
-# define nullptr NULL
-#endif
-
-/*----------------------------------------------------------------------------*/
-
-#ifndef __always_inline
-# if defined(__GNUC__) || __has_attribute(__always_inline__)
-# define __always_inline __inline __attribute__((__always_inline__))
-# elif defined(_MSC_VER)
-# define __always_inline __forceinline
-# else
-# define __always_inline
-# endif
-#endif /* __always_inline */
-
-#ifndef __noinline
-# if defined(__GNUC__) || __has_attribute(__noinline__)
-# define __noinline __attribute__((__noinline__))
-# elif defined(_MSC_VER)
-# define __noinline __declspec(noinline)
-# else
-# define __noinline
-# endif
-#endif /* __noinline */
-
-#ifndef __must_check_result
-# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
-# define __must_check_result __attribute__((__warn_unused_result__))
-# else
-# define __must_check_result
-# endif
-#endif /* __must_check_result */
-
-#if !defined(__noop) && !defined(_MSC_VER)
-# define __noop(...) do {} while(0)
-#endif /* __noop */
-
-#if defined(__fallthrough) && \
- (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
-#undef __fallthrough
-#endif /* __fallthrough workaround for MinGW */
-
-#ifndef __fallthrough
-# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
- (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L
-# define __fallthrough [[fallthrough]]
-# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
-# define __fallthrough [[fallthrough]]
-# elif __GNUC_PREREQ(7, 0) && \
- (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
- (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
-# define __fallthrough __attribute__((__fallthrough__))
-# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\
- __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
-# define __fallthrough [[clang::fallthrough]]
-# else
-# define __fallthrough
-# endif
-#endif /* __fallthrough */
-
-#ifndef __unreachable
-# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable)
-# define __unreachable() __builtin_unreachable()
-# elif defined(_MSC_VER)
-# define __unreachable() __assume(0)
-# else
-# define __unreachable() __noop()
-# endif
-#endif /* __unreachable */
-
-#ifndef __prefetch
-# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
-# define __prefetch(ptr) __builtin_prefetch(ptr)
-# else
-# define __prefetch(ptr) __noop(ptr)
-# endif
-#endif /* __prefetch */
-
-#ifndef __nothrow
-# if defined(__cplusplus)
-# if __cplusplus < 201703L
-# define __nothrow throw()
-# else
-# define __nothrow noexcept(true)
-# endif /* __cplusplus */
-# elif defined(__GNUC__) || __has_attribute(__nothrow__)
-# define __nothrow __attribute__((__nothrow__))
-# elif defined(_MSC_VER) && defined(__cplusplus)
-# define __nothrow __declspec(nothrow)
-# else
-# define __nothrow
-# endif
-#endif /* __nothrow */
-
-#ifndef __hidden
-# if defined(__GNUC__) || __has_attribute(__visibility__)
-# define __hidden __attribute__((__visibility__("hidden")))
-# else
-# define __hidden
-# endif
-#endif /* __hidden */
-
-#ifndef __optimize
-# if defined(__OPTIMIZE__)
-# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__)
-# define __optimize(ops) __attribute__((__optimize__(ops)))
-# else
-# define __optimize(ops)
-# endif
-# else
-# define __optimize(ops)
-# endif
-#endif /* __optimize */
-
-#ifndef __hot
-# if defined(__OPTIMIZE__)
-# if defined(__e2k__)
-# define __hot __attribute__((__hot__)) __optimize(3)
-# elif defined(__clang__) && !__has_attribute(__hot_) \
- && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
- /* just put frequently used functions in separate section */
-# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
-# elif defined(__GNUC__) || __has_attribute(__hot__)
-# define __hot __attribute__((__hot__)) __optimize("O3")
-# else
-# define __hot __optimize("O3")
-# endif
-# else
-# define __hot
-# endif
-#endif /* __hot */
-
-#ifndef __cold
-# if defined(__OPTIMIZE__)
-# if defined(__e2k__)
-# define __cold __attribute__((__cold__)) __optimize(1)
-# elif defined(__clang__) && !__has_attribute(cold) \
- && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
- /* just put infrequently used functions in separate section */
-# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
-# elif defined(__GNUC__) || __has_attribute(cold)
-# define __cold __attribute__((__cold__)) __optimize("Os")
-# else
-# define __cold __optimize("Os")
-# endif
-# else
-# define __cold
-# endif
-#endif /* __cold */
-
-#ifndef __flatten
-# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
-# define __flatten __attribute__((__flatten__))
-# else
-# define __flatten
-# endif
-#endif /* __flatten */
-
-#ifndef likely
-# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
-# define likely(cond) __builtin_expect(!!(cond), 1)
-# else
-# define likely(x) (!!(x))
-# endif
-#endif /* likely */
-
-#ifndef unlikely
-# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
-# define unlikely(cond) __builtin_expect(!!(cond), 0)
-# else
-# define unlikely(x) (!!(x))
-# endif
-#endif /* unlikely */
-
-#ifndef __anonymous_struct_extension__
-# if defined(__GNUC__)
-# define __anonymous_struct_extension__ __extension__
-# else
-# define __anonymous_struct_extension__
-# endif
-#endif /* __anonymous_struct_extension__ */
-
-#ifndef __Wpedantic_format_voidptr
- MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void*
- __Wpedantic_format_voidptr(const void* ptr) {return ptr;}
-# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
-#endif /* __Wpedantic_format_voidptr */
-
-/*----------------------------------------------------------------------------*/
-
-#if defined(MDBX_USE_VALGRIND)
-# include <valgrind/memcheck.h>
-# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
- /* LY: available since Valgrind 3.10 */
-# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
-# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
-# endif
-#elif !defined(RUNNING_ON_VALGRIND)
-# define VALGRIND_CREATE_MEMPOOL(h,r,z)
-# define VALGRIND_DESTROY_MEMPOOL(h)
-# define VALGRIND_MEMPOOL_TRIM(h,a,s)
-# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
-# define VALGRIND_MEMPOOL_FREE(h,a)
-# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
-# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
-# define VALGRIND_MAKE_MEM_DEFINED(a,s)
-# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
-# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
-# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
-# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
-# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
-# define RUNNING_ON_VALGRIND (0)
-#endif /* MDBX_USE_VALGRIND */
-
-#ifdef __SANITIZE_ADDRESS__
-# include <sanitizer/asan_interface.h>
-#elif !defined(ASAN_POISON_MEMORY_REGION)
-# define ASAN_POISON_MEMORY_REGION(addr, size) \
- ((void)(addr), (void)(size))
-# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
- ((void)(addr), (void)(size))
-#endif /* __SANITIZE_ADDRESS__ */
-
-/*----------------------------------------------------------------------------*/
-
-#ifndef ARRAY_LENGTH
-# ifdef __cplusplus
- template <typename T, size_t N>
- char (&__ArraySizeHelper(T (&array)[N]))[N];
-# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
-# else
-# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
-# endif
-#endif /* ARRAY_LENGTH */
-
-#ifndef ARRAY_END
-# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
-#endif /* ARRAY_END */
-
-#define CONCAT(a,b) a##b
-#define XCONCAT(a,b) CONCAT(a,b)
-
-#ifndef offsetof
-# define offsetof(type, member) __builtin_offsetof(type, member)
-#endif /* offsetof */
-
-#ifndef container_of
-# define container_of(ptr, type, member) \
- ((type *)((char *)(ptr) - offsetof(type, member)))
-#endif /* container_of */
-
-#define MDBX_TETRAD(a, b, c, d) \
- ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
-
-#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
-
-#define FIXME "FIXME: " __FILE__ ", " MDBX_STRINGIFY(__LINE__)
-
-#ifndef STATIC_ASSERT_MSG
-# if defined(static_assert)
-# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
-# elif defined(_STATIC_ASSERT)
-# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
-# elif defined(_MSC_VER)
-# include <crtdbg.h>
-# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
-# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
- || __has_feature(c_static_assert)
-# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
-# else
-# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
-# endif
-#endif /* STATIC_ASSERT */
-
-#ifndef STATIC_ASSERT
-# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
-#endif
-
-/* *INDENT-ON* */
-/* clang-format on */
diff --git a/3rdparty/libmdbx/src/internals.h b/3rdparty/libmdbx/src/internals.h
deleted file mode 100644
index 13203afe5dee3d21dca778104c856a59c29e698c..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/internals.h
+++ /dev/null
@@ -1,1623 +0,0 @@
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#pragma once
-#ifdef MDBX_CONFIG_H
-#include MDBX_CONFIG_H
-#endif
-
-#define LIBMDBX_INTERNALS
-#ifdef xMDBX_TOOLS
-#define MDBX_DEPRECATED
-#endif /* xMDBX_TOOLS */
-
-#ifdef xMDBX_ALLOY
-/* Amalgamated build */
-#define MDBX_INTERNAL_FUNC static
-#define MDBX_INTERNAL_VAR static
-#else
-/* Non-amalgamated build */
-#define MDBX_INTERNAL_FUNC
-#define MDBX_INTERNAL_VAR extern
-#endif /* xMDBX_ALLOY */
-
-/** Disables using GNU/Linux libc extensions.
- * \ingroup build_option
- * \note This option couldn't be moved to the options.h since dependant
- * control macros/defined should be prepared before include the options.h */
-#ifndef MDBX_DISABLE_GNU_SOURCE
-#define MDBX_DISABLE_GNU_SOURCE 0
-#endif
-#if MDBX_DISABLE_GNU_SOURCE
-#undef _GNU_SOURCE
-#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE)
-#define _GNU_SOURCE
-#endif /* MDBX_DISABLE_GNU_SOURCE */
-
-/*----------------------------------------------------------------------------*/
-
-/* Should be defined before any includes */
-#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
- !defined(ANDROID)
-#define _FILE_OFFSET_BITS 64
-#endif
-
-#ifdef __APPLE__
-#define _DARWIN_C_SOURCE
-#endif
-
-#ifdef _MSC_VER
-#if _MSC_FULL_VER < 190024234
-/* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual
- * Studio 2015 Update 3). But you could remove this #error and try to continue
- * at your own risk. In such case please don't rise up an issues related ONLY to
- * old compilers.
- *
- * NOTE:
- * Unfortunately, there are several different builds of "Visual Studio" that
- * are called "Visual Studio 2015 Update 3".
- *
- * The 190024234 is used here because it is minimal version of Visual Studio
- * that was used for build and testing libmdbx in recent years. Soon this
- * value will be increased to 19.0.24241.7, since build and testing using
- * "Visual Studio 2015" will be performed only at https://ci.appveyor.com.
- *
- * Please ask Microsoft (but not us) for information about version differences
- * and how to and where you can obtain the latest "Visual Studio 2015" build
- * with all fixes.
- */
-#error \
- "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
-#endif
-#ifndef _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_WARNINGS
-#endif /* _CRT_SECURE_NO_WARNINGS */
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#if _MSC_VER > 1913
-#pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... \
- */
-#endif
-#if _MSC_VER > 1914
-#pragma warning( \
- disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
- producing 'defined' has undefined behavior */
-#endif
-#pragma warning(disable : 4710) /* 'xyz': function not inlined */
-#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
- inline expansion */
-#pragma warning( \
- disable : 4201) /* nonstandard extension used : nameless struct / union */
-#pragma warning(disable : 4702) /* unreachable code */
-#pragma warning(disable : 4706) /* assignment within conditional expression */
-#pragma warning(disable : 4127) /* conditional expression is constant */
-#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
- alignment specifier */
-#pragma warning(disable : 4310) /* cast truncates constant value */
-#pragma warning( \
- disable : 4820) /* bytes padding added after data member for alignment */
-#pragma warning(disable : 4548) /* expression before comma has no effect; \
- expected expression with side - effect */
-#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
- unaligned */
-#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
- array in struct/union */
-#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
- aggregate initializer */
-#pragma warning( \
- disable : 4505) /* unreferenced local function has been removed */
-#endif /* _MSC_VER (warnings) */
-
-#if defined(__GNUC__) && __GNUC__ < 9
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif /* GCC < 9 */
-
-#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
- !defined(__USE_MINGW_ANSI_STDIO)
-#define __USE_MINGW_ANSI_STDIO 1
-#endif /* __USE_MINGW_ANSI_STDIO */
-
-#include "../mdbx.h"
-#include "defs.h"
-
-#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2)
-/* Actually libmdbx was not tested with compilers older than GCC 4.2.
- * But you could ignore this warning at your own risk.
- * In such case please don't rise up an issues related ONLY to old compilers.
- */
-#warning "libmdbx required GCC >= 4.2"
-#endif
-
-#if defined(__clang__) && !__CLANG_PREREQ(3, 8)
-/* Actually libmdbx was not tested with CLANG older than 3.8.
- * But you could ignore this warning at your own risk.
- * In such case please don't rise up an issues related ONLY to old compilers.
- */
-#warning "libmdbx required CLANG >= 3.8"
-#endif
-
-#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12)
-/* Actually libmdbx was not tested with something older than glibc 2.12.
- * But you could ignore this warning at your own risk.
- * In such case please don't rise up an issues related ONLY to old systems.
- */
-#warning "libmdbx was only tested with GLIBC >= 2.12."
-#endif
-
-#ifdef __SANITIZE_THREAD__
-#warning \
- "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
-#endif /* __SANITIZE_THREAD__ */
-
-#if __has_warning("-Wnested-anon-types")
-#if defined(__clang__)
-#pragma clang diagnostic ignored "-Wnested-anon-types"
-#elif defined(__GNUC__)
-#pragma GCC diagnostic ignored "-Wnested-anon-types"
-#else
-#pragma warning disable "nested-anon-types"
-#endif
-#endif /* -Wnested-anon-types */
-
-#if __has_warning("-Wconstant-logical-operand")
-#if defined(__clang__)
-#pragma clang diagnostic ignored "-Wconstant-logical-operand"
-#elif defined(__GNUC__)
-#pragma GCC diagnostic ignored "-Wconstant-logical-operand"
-#else
-#pragma warning disable "constant-logical-operand"
-#endif
-#endif /* -Wconstant-logical-operand */
-
-#if defined(__LCC__) && (__LCC__ <= 121)
-/* bug #2798 */
-#pragma diag_suppress alignment_reduction_ignored
-#elif defined(__ICC)
-#pragma warning(disable : 3453 1366)
-#elif __has_warning("-Walignment-reduction-ignored")
-#if defined(__clang__)
-#pragma clang diagnostic ignored "-Walignment-reduction-ignored"
-#elif defined(__GNUC__)
-#pragma GCC diagnostic ignored "-Walignment-reduction-ignored"
-#else
-#pragma warning disable "alignment-reduction-ignored"
-#endif
-#endif /* -Walignment-reduction-ignored */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "osal.h"
-
-#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY)
-#if defined(xMDBX_TOOLS)
-extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
-#endif
-
-#include "options.h"
-
-/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
-#if MDBX_DEBUG
-#undef NDEBUG
-#endif
-
-/*----------------------------------------------------------------------------*/
-/* Atomics */
-
-enum MDBX_memory_order {
- mo_Relaxed,
- mo_AcquireRelease,
- mo_SequentialConsistency
-};
-
-typedef union {
- volatile uint32_t weak;
-#ifdef MDBX_HAVE_C11ATOMICS
- volatile _Atomic uint32_t c11a;
-#endif /* MDBX_HAVE_C11ATOMICS */
-} MDBX_atomic_uint32_t;
-
-typedef union {
- volatile uint64_t weak;
-#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC)
- volatile _Atomic uint64_t c11a;
-#endif
-#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC
- __anonymous_struct_extension__ struct {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- MDBX_atomic_uint32_t low, high;
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
- MDBX_atomic_uint32_t high, low;
-#else
-#error "FIXME: Unsupported byte order"
-#endif /* __BYTE_ORDER__ */
- };
-#endif
-} MDBX_atomic_uint64_t;
-
-#ifdef MDBX_HAVE_C11ATOMICS
-
-/* Crutches for C11 atomic compiler's bugs */
-#if defined(__e2k__) && defined(__LCC__) && __LCC__ < /* FIXME */ 127
-#define MDBX_c11a_ro(type, ptr) (&(ptr)->weak)
-#define MDBX_c11a_rw(type, ptr) (&(ptr)->weak)
-#elif defined(__clang__) && __clang__ < 8
-#define MDBX_c11a_ro(type, ptr) ((volatile _Atomic(type) *)&(ptr)->c11a)
-#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
-#else
-#define MDBX_c11a_ro(type, ptr) (&(ptr)->c11a)
-#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
-#endif /* Crutches for C11 atomic compiler's bugs */
-
-static __always_inline memory_order mo_c11_store(enum MDBX_memory_order fence) {
- switch (fence) {
- default:
- assert(false);
- __unreachable();
- case mo_Relaxed:
- return memory_order_relaxed;
- case mo_AcquireRelease:
- return memory_order_release;
- case mo_SequentialConsistency:
- return memory_order_seq_cst;
- }
-}
-
-static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
- switch (fence) {
- default:
- assert(false);
- __unreachable();
- case mo_Relaxed:
- return memory_order_relaxed;
- case mo_AcquireRelease:
- return memory_order_acquire;
- case mo_SequentialConsistency:
- return memory_order_seq_cst;
- }
-}
-#endif /* MDBX_HAVE_C11ATOMICS */
-
-#ifndef __cplusplus
-
-MDBX_MAYBE_UNUSED static __always_inline void
-mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
-#ifdef MDBX_HAVE_C11ATOMICS
- atomic_thread_fence(write ? mo_c11_store(order) : mo_c11_load(order));
-#else /* MDBX_HAVE_C11ATOMICS */
- mdbx_compiler_barrier();
- if (write &&
- order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))
- mdbx_memory_barrier();
-#endif /* MDBX_HAVE_C11ATOMICS */
-}
-
-MDBX_MAYBE_UNUSED static __always_inline uint32_t
-atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value,
- enum MDBX_memory_order order) {
- STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
-#ifdef MDBX_HAVE_C11ATOMICS
- assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
- atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
-#else /* MDBX_HAVE_C11ATOMICS */
- if (order != mo_Relaxed)
- mdbx_compiler_barrier();
- p->weak = value;
- mdbx_memory_fence(order, true);
-#endif /* MDBX_HAVE_C11ATOMICS */
- return value;
-}
-
-MDBX_MAYBE_UNUSED static __always_inline uint32_t
-atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
- STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4);
-#ifdef MDBX_HAVE_C11ATOMICS
- assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
- return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
-#else /* MDBX_HAVE_C11ATOMICS */
- mdbx_memory_fence(order, false);
- const uint32_t value = p->weak;
- if (order != mo_Relaxed)
- mdbx_compiler_barrier();
- return value;
-#endif /* MDBX_HAVE_C11ATOMICS */
-}
-
-#endif /* !__cplusplus */
-
-/*----------------------------------------------------------------------------*/
-/* Basic constants and types */
-
-/* A stamp that identifies a file as an MDBX file.
- * There's nothing special about this value other than that it is easily
- * recognizable, and it will reflect any byte order mismatches. */
-#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
-
-/* FROZEN: The version number for a database's datafile format. */
-#define MDBX_DATA_VERSION 3
-/* The version number for a database's lockfile format. */
-#define MDBX_LOCK_VERSION 4
-
-/* handle for the DB used to track free pages. */
-#define FREE_DBI 0
-/* handle for the default DB. */
-#define MAIN_DBI 1
-/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
-#define CORE_DBS 2
-
-/* Number of meta pages - also hardcoded elsewhere */
-#define NUM_METAS 3
-
-/* A page number in the database.
- *
- * MDBX uses 32 bit for page numbers. This limits database
- * size up to 2^44 bytes, in case of 4K pages. */
-typedef uint32_t pgno_t;
-typedef MDBX_atomic_uint32_t atomic_pgno_t;
-#define PRIaPGNO PRIu32
-#define MAX_PAGENO UINT32_C(0x7FFFffff)
-#define MIN_PAGENO NUM_METAS
-
-#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000)
-
-/* A transaction ID. */
-typedef uint64_t txnid_t;
-typedef MDBX_atomic_uint64_t atomic_txnid_t;
-#define PRIaTXN PRIi64
-#define MIN_TXNID UINT64_C(1)
-#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1)
-#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1)
-#define INVALID_TXNID UINT64_MAX
-/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
- * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
-#ifndef xMDBX_TXNID_STEP
-#if MDBX_64BIT_CAS
-#define xMDBX_TXNID_STEP 1u
-#else
-#define xMDBX_TXNID_STEP 2u
-#endif
-#endif /* xMDBX_TXNID_STEP */
-
-/* Used for offsets within a single page.
- * Since memory pages are typically 4 or 8KB in size, 12-13 bits,
- * this is plenty. */
-typedef uint16_t indx_t;
-
-#define MEGABYTE ((size_t)1 << 20)
-
-/*----------------------------------------------------------------------------*/
-/* Core structures for database and shared memory (i.e. format definition) */
-#pragma pack(push, 4)
-
-/* Information about a single database in the environment. */
-typedef struct MDBX_db {
- uint16_t md_flags; /* see mdbx_dbi_open */
- uint16_t md_depth; /* depth of this tree */
- uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */
- pgno_t md_root; /* the root page of this tree */
- pgno_t md_branch_pages; /* number of internal pages */
- pgno_t md_leaf_pages; /* number of leaf pages */
- pgno_t md_overflow_pages; /* number of overflow pages */
- uint64_t md_seq; /* table sequence counter */
- uint64_t md_entries; /* number of data items */
- uint64_t md_mod_txnid; /* txnid of last committed modification */
-} MDBX_db;
-
-/* database size-related parameters */
-typedef struct MDBX_geo {
- uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential
- quantized) value */
- uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed
- (exponential quantized) value */
- pgno_t lower; /* minimal size of datafile in pages */
- pgno_t upper; /* maximal size of datafile in pages */
- pgno_t now; /* current size of datafile in pages */
- pgno_t next; /* first unused page in the datafile,
- but actually the file may be shorter. */
-} MDBX_geo;
-
-/* Meta page content.
- * A meta page is the start point for accessing a database snapshot.
- * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */
-typedef struct MDBX_meta {
- /* Stamp identifying this as an MDBX file.
- * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */
- uint32_t mm_magic_and_version[2];
-
- /* txnid that committed this page, the first of a two-phase-update pair */
- uint32_t mm_txnid_a[2];
-
- uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */
- uint8_t mm_validator_id; /* ID of checksum and page validation method,
- * zero (nothing) for now */
- uint8_t mm_extra_pagehdr; /* extra bytes in the page header,
- * zero (nothing) for now */
-
- MDBX_geo mm_geo; /* database size-related parameters */
-
- MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */
- /* The size of pages used in this DB */
-#define mm_psize mm_dbs[FREE_DBI].md_xsize
- MDBX_canary mm_canary;
-
-#define MDBX_DATASIGN_NONE 0u
-#define MDBX_DATASIGN_WEAK 1u
-#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK)
-#define META_IS_STEADY(meta) \
- SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_datasync_sign))
- uint32_t mm_datasync_sign[2];
-
- /* txnid that committed this page, the second of a two-phase-update pair */
- uint32_t mm_txnid_b[2];
-
- /* Number of non-meta pages which were put in GC after COW. May be 0 in case
- * DB was previously handled by libmdbx without corresponding feature.
- * This value in couple with mr_snapshot_pages_retired allows fast estimation
- * of "how much reader is restraining GC recycling". */
- uint32_t mm_pages_retired[2];
-
- /* The analogue /proc/sys/kernel/random/boot_id or similar to determine
- * whether the system was rebooted after the last use of the database files.
- * If there was no reboot, but there is no need to rollback to the last
- * steady sync point. Zeros mean that no relevant information is available
- * from the system. */
- bin128_t mm_bootid;
-
-} MDBX_meta;
-
-#pragma pack(1)
-
-/* Common header for all page types. The page type depends on mp_flags.
- *
- * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with
- * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages
- * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header.
- *
- * P_OVERFLOW records occupy one or more contiguous pages where only the
- * first has a page header. They hold the real data of F_BIGDATA nodes.
- *
- * P_SUBP sub-pages are small leaf "pages" with duplicate data.
- * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page.
- * (Duplicate data can also go in sub-databases, which use normal pages.)
- *
- * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
- *
- * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
- * in the snapshot: Either used by a database or listed in a GC record. */
-typedef struct MDBX_page {
- union {
-#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid)
-#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid)
-#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid)
-#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front)
-#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front)
- uint64_t mp_txnid;
- struct MDBX_page *mp_next; /* for in-memory list of freed pages */
- };
- uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */
-#define P_BRANCH 0x01 /* branch page */
-#define P_LEAF 0x02 /* leaf page */
-#define P_OVERFLOW 0x04 /* overflow page */
-#define P_META 0x08 /* meta page */
-#define P_BAD 0x10 /* explicit flag for invalid/bad page */
-#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */
-#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */
-#define P_SPILLED 0x2000 /* spilled in parent txn */
-#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */
-#define P_FROZEN 0x8000 /* used for retire page with known status */
-#define P_ILL_BITS (~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED))
- uint16_t mp_flags;
- union {
- uint32_t mp_pages; /* number of overflow pages */
- __anonymous_struct_extension__ struct {
- indx_t mp_lower; /* lower bound of free space */
- indx_t mp_upper; /* upper bound of free space */
- };
- };
- pgno_t mp_pgno; /* page number */
-
-#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
- (!defined(__cplusplus) && defined(_MSC_VER))
- indx_t mp_ptrs[] /* dynamic size */;
-#endif /* C99 */
-} MDBX_page;
-
-/* Size of the page header, excluding dynamic data at the end */
-#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs))
-
-#pragma pack(pop)
-
-#if MDBX_ENABLE_PGOP_STAT
-/* Statistics of page operations overall of all (running, completed and aborted)
- * transactions */
-typedef struct {
- MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */
- MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */
- MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones
- for nested transactions */
- MDBX_atomic_uint64_t split; /* Page splits */
- MDBX_atomic_uint64_t merge; /* Page merges */
- MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
- MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
- MDBX_atomic_uint64_t
- wops; /* Number of explicit write operations (not a pages) to a disk */
-} MDBX_pgop_stat_t;
-#endif /* MDBX_ENABLE_PGOP_STAT */
-
-#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES
-#define MDBX_CLOCK_SIGN UINT32_C(0xF10C)
-typedef void mdbx_ipclock_t;
-#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
-
-#define MDBX_CLOCK_SIGN UINT32_C(0xF18D)
-typedef mdbx_pid_t mdbx_ipclock_t;
-#ifndef EOWNERDEAD
-#define EOWNERDEAD MDBX_RESULT_TRUE
-#endif
-
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
-#define MDBX_CLOCK_SIGN UINT32_C(0x8017)
-typedef pthread_mutex_t mdbx_ipclock_t;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
-#define MDBX_CLOCK_SIGN UINT32_C(0xFC29)
-typedef sem_t mdbx_ipclock_t;
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-
-#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus)
-MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc);
-MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc);
-#endif /* MDBX_LOCKING */
-
-/* Reader Lock Table
- *
- * Readers don't acquire any locks for their data access. Instead, they
- * simply record their transaction ID in the reader table. The reader
- * mutex is needed just to find an empty slot in the reader table. The
- * slot's address is saved in thread-specific data so that subsequent
- * read transactions started by the same thread need no further locking to
- * proceed.
- *
- * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data.
- * No reader table is used if the database is on a read-only filesystem.
- *
- * Since the database uses multi-version concurrency control, readers don't
- * actually need any locking. This table is used to keep track of which
- * readers are using data from which old transactions, so that we'll know
- * when a particular old transaction is no longer in use. Old transactions
- * that have discarded any data pages can then have those pages reclaimed
- * for use by a later write transaction.
- *
- * The lock table is constructed such that reader slots are aligned with the
- * processor's cache line size. Any slot is only ever used by one thread.
- * This alignment guarantees that there will be no contention or cache
- * thrashing as threads update their own slot info, and also eliminates
- * any need for locking when accessing a slot.
- *
- * A writer thread will scan every slot in the table to determine the oldest
- * outstanding reader transaction. Any freed pages older than this will be
- * reclaimed by the writer. The writer doesn't use any locks when scanning
- * this table. This means that there's no guarantee that the writer will
- * see the most up-to-date reader info, but that's not required for correct
- * operation - all we need is to know the upper bound on the oldest reader,
- * we don't care at all about the newest reader. So the only consequence of
- * reading stale information here is that old pages might hang around a
- * while longer before being reclaimed. That's actually good anyway, because
- * the longer we delay reclaiming old pages, the more likely it is that a
- * string of contiguous pages can be found after coalescing old pages from
- * many old transactions together. */
-
-/* The actual reader record, with cacheline padding. */
-typedef struct MDBX_reader {
- /* Current Transaction ID when this transaction began, or (txnid_t)-1.
- * Multiple readers that start at the same time will probably have the
- * same ID here. Again, it's not important to exclude them from
- * anything; all we need to know is which version of the DB they
- * started from so we can avoid overwriting any data used in that
- * particular version. */
- MDBX_atomic_uint64_t /* txnid_t */ mr_txnid;
-
- /* The information we store in a single slot of the reader table.
- * In addition to a transaction ID, we also record the process and
- * thread ID that owns a slot, so that we can detect stale information,
- * e.g. threads or processes that went away without cleaning up.
- *
- * NOTE: We currently don't check for stale records.
- * We simply re-init the table when we know that we're the only process
- * opening the lock file. */
-
- /* The thread ID of the thread owning this txn. */
- MDBX_atomic_uint64_t mr_tid;
-
- /* The process ID of the process owning this reader txn. */
- MDBX_atomic_uint32_t mr_pid;
-
- /* The number of pages used in the reader's MVCC snapshot,
- * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */
- atomic_pgno_t mr_snapshot_pages_used;
- /* Number of retired pages at the time this reader starts transaction. So,
- * at any time the difference mm_pages_retired - mr_snapshot_pages_retired
- * will give the number of pages which this reader restraining from reuse. */
- MDBX_atomic_uint64_t mr_snapshot_pages_retired;
-} MDBX_reader;
-
-/* The header for the reader table (a memory-mapped lock file). */
-typedef struct MDBX_lockinfo {
- /* Stamp identifying this as an MDBX file.
- * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */
- uint64_t mti_magic_and_version;
-
- /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */
- uint32_t mti_os_and_format;
-
- /* Flags which environment was opened. */
- MDBX_atomic_uint32_t mti_envmode;
-
- /* Threshold of un-synced-with-disk pages for auto-sync feature,
- * zero means no-threshold, i.e. auto-sync is disabled. */
- atomic_pgno_t mti_autosync_threshold;
-
- /* Low 32-bit of txnid with which meta-pages was synced,
- * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
- MDBX_atomic_uint32_t mti_meta_sync_txnid;
-
- /* Period for timed auto-sync feature, i.e. at the every steady checkpoint
- * the mti_unsynced_timeout sets to the current_time + mti_autosync_period.
- * The time value is represented in a suitable system-dependent form, for
- * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC).
- * Zero means timed auto-sync is disabled. */
- MDBX_atomic_uint64_t mti_autosync_period;
-
- /* Marker to distinguish uniqueness of DB/CLK. */
- MDBX_atomic_uint64_t mti_bait_uniqueness;
-
- MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
-
-#if MDBX_ENABLE_PGOP_STAT
- /* Statistics of costly ops of all (running, completed and aborted)
- * transactions */
- MDBX_pgop_stat_t mti_pgop_stat;
-#endif /* MDBX_ENABLE_PGOP_STAT*/
-
- MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
-
- /* Write transaction lock. */
-#if MDBX_LOCKING > 0
- mdbx_ipclock_t mti_wlock;
-#endif /* MDBX_LOCKING > 0 */
-
- atomic_txnid_t mti_oldest_reader;
-
- /* Timestamp of the last steady sync. Value is represented in a suitable
- * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or
- * clock_gettime(CLOCK_MONOTONIC). */
- MDBX_atomic_uint64_t mti_sync_timestamp;
-
- /* Number un-synced-with-disk pages for auto-sync feature. */
- atomic_pgno_t mti_unsynced_pages;
-
- /* Number of page which was discarded last time by madvise(MADV_FREE). */
- atomic_pgno_t mti_discarded_tail;
-
- /* Timestamp of the last readers check. */
- MDBX_atomic_uint64_t mti_reader_check_timestamp;
-
- /* Shared anchor for tracking readahead edge and enabled/disabled status. */
- pgno_t mti_readahead_anchor;
-
- MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
-
- /* Readeaders registration lock. */
-#if MDBX_LOCKING > 0
- mdbx_ipclock_t mti_rlock;
-#endif /* MDBX_LOCKING > 0 */
-
- /* The number of slots that have been used in the reader table.
- * This always records the maximum count, it is not decremented
- * when readers release their slots. */
- MDBX_atomic_uint32_t mti_numreaders;
- MDBX_atomic_uint32_t mti_readers_refresh_flag;
-
-#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
- (!defined(__cplusplus) && defined(_MSC_VER))
- MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/
- MDBX_reader mti_readers[] /* dynamic size */;
-#endif /* C99 */
-} MDBX_lockinfo;
-
-/* Lockfile format signature: version, features and field layout */
-#define MDBX_LOCK_FORMAT \
- (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \
- (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \
- (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \
- (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \
- (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29)
-
-#define MDBX_DATA_MAGIC \
- ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
-
-#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
- ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
-
-#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
-
-#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
-
-/* The maximum size of a database page.
- *
- * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
- *
- * MDBX will use database pages < OS pages if needed.
- * That causes more I/O in write transactions: The OS must
- * know (read) the whole page before writing a partial page.
- *
- * Note that we don't currently support Huge pages. On Linux,
- * regular data files cannot use Huge pages, and in general
- * Huge pages aren't actually pageable. We rely on the OS
- * demand-pager to read our data and page it out when memory
- * pressure from other processes is high. So until OSs have
- * actual paging support for Huge pages, they're not viable. */
-#define MAX_PAGESIZE MDBX_MAX_PAGESIZE
-#define MIN_PAGESIZE MDBX_MIN_PAGESIZE
-
-#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO)
-#if defined(_WIN32) || defined(_WIN64)
-#define MAX_MAPSIZE32 UINT32_C(0x38000000)
-#else
-#define MAX_MAPSIZE32 UINT32_C(0x7f000000)
-#endif
-#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE)
-
-#if MDBX_WORDBITS >= 64
-#define MAX_MAPSIZE MAX_MAPSIZE64
-#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO)
-#else
-#define MAX_MAPSIZE MAX_MAPSIZE32
-#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE)
-#endif /* MDBX_WORDBITS */
-
-#define MDBX_READERS_LIMIT 32767
-#define MDBX_RADIXSORT_THRESHOLD 333
-
-/*----------------------------------------------------------------------------*/
-
-/* An PNL is an Page Number List, a sorted array of IDs.
- * The first element of the array is a counter for how many actual page-numbers
- * are in the list. By default PNLs are sorted in descending order, this allow
- * cut off a page with lowest pgno (at the tail) just truncating the list. The
- * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */
-typedef pgno_t *MDBX_PNL;
-
-#if MDBX_PNL_ASCENDING
-#define MDBX_PNL_ORDERED(first, last) ((first) < (last))
-#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last))
-#else
-#define MDBX_PNL_ORDERED(first, last) ((first) > (last))
-#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last))
-#endif
-
-/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */
-typedef txnid_t *MDBX_TXL;
-
-/* An Dirty-Page list item is an pgno/pointer pair. */
-typedef struct MDBX_dp {
- MDBX_page *ptr;
- pgno_t pgno;
- union {
- unsigned extra;
- __anonymous_struct_extension__ struct {
- unsigned multi : 1;
- unsigned lru : 31;
- };
- };
-} MDBX_dp;
-
-/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */
-typedef struct MDBX_dpl {
- unsigned sorted;
- unsigned length;
- unsigned detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */
-#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
- (!defined(__cplusplus) && defined(_MSC_VER))
- MDBX_dp items[] /* dynamic size with holes at zero and after the last */;
-#endif
-} MDBX_dpl;
-
-/* PNL sizes */
-#define MDBX_PNL_GRANULATE 1024
-#define MDBX_PNL_INITIAL \
- (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
-
-#define MDBX_TXL_GRANULATE 32
-#define MDBX_TXL_INITIAL \
- (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
-#define MDBX_TXL_MAX \
- ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t))
-
-#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
-#define MDBX_PNL_SIZE(pl) ((pl)[0])
-#define MDBX_PNL_FIRST(pl) ((pl)[1])
-#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)])
-#define MDBX_PNL_BEGIN(pl) (&(pl)[1])
-#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1])
-
-#if MDBX_PNL_ASCENDING
-#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl)
-#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl)
-#else
-#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl)
-#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl)
-#endif
-
-#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t))
-#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0)
-
-/*----------------------------------------------------------------------------*/
-/* Internal structures */
-
-/* Auxiliary DB info.
- * The information here is mostly static/read-only. There is
- * only a single copy of this record in the environment. */
-typedef struct MDBX_dbx {
- MDBX_val md_name; /* name of the database */
- MDBX_cmp_func *md_cmp; /* function for comparing keys */
- MDBX_cmp_func *md_dcmp; /* function for comparing data items */
- size_t md_klen_min, md_klen_max; /* min/max key length for the database */
- size_t md_vlen_min,
- md_vlen_max; /* min/max value/data length for the database */
-} MDBX_dbx;
-
-/* A database transaction.
- * Every operation requires a transaction handle. */
-struct MDBX_txn {
-#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31)
- uint32_t mt_signature;
-
- /* Transaction Flags */
- /* mdbx_txn_begin() flags */
-#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE)
-#define MDBX_TXN_RW_BEGIN_FLAGS \
- (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY)
- /* Additional flag for mdbx_sync_locked() */
-#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000)
-
-#define TXN_FLAGS \
- (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \
- MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID)
-
-#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \
- ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \
- MDBX_SHRINK_ALLOWED)
-#error "Oops, some txn flags overlapped or wrong"
-#endif
- uint32_t mt_flags;
-
- MDBX_txn *mt_parent; /* parent of a nested txn */
- /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */
- MDBX_txn *mt_child;
- MDBX_geo mt_geo;
- /* next unallocated page */
-#define mt_next_pgno mt_geo.next
- /* corresponding to the current size of datafile */
-#define mt_end_pgno mt_geo.now
-
- /* The ID of this transaction. IDs are integers incrementing from 1.
- * Only committed write transactions increment the ID. If a transaction
- * aborts, the ID may be re-used by the next writer. */
- txnid_t mt_txnid;
- txnid_t mt_front;
-
- MDBX_env *mt_env; /* the DB environment */
- /* Array of records for each DB known in the environment. */
- MDBX_dbx *mt_dbxs;
- /* Array of MDBX_db records for each known DB */
- MDBX_db *mt_dbs;
- /* Array of sequence numbers for each DB handle */
- unsigned *mt_dbiseqs;
-
- /* Transaction DBI Flags */
-#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */
-#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */
-#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */
-#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */
-#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */
-#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */
-#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */
- /* Array of flags for each DB */
- uint8_t *mt_dbistate;
- /* Number of DB records in use, or 0 when the txn is finished.
- * This number only ever increments until the txn finishes; we
- * don't decrement it when individual DB handles are closed. */
- MDBX_dbi mt_numdbs;
- size_t mt_owner; /* thread ID that owns this transaction */
- MDBX_canary mt_canary;
- void *mt_userctx; /* User-settable context */
- MDBX_cursor **mt_cursors;
-
- union {
- struct {
- /* For read txns: This thread/txn's reader table slot, or NULL. */
- MDBX_reader *reader;
- } to;
- struct {
- /* In write txns, array of cursors for each DB */
- pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
- txnid_t last_reclaimed; /* ID of last used record */
-#if MDBX_ENABLE_REFUND
- pgno_t loose_refund_wl /* FIXME: describe */;
-#endif /* MDBX_ENABLE_REFUND */
- /* dirtylist room: Dirty array size - dirty pages visible to this txn.
- * Includes ancestor txns' dirty pages not hidden by other txns'
- * dirty/spilled pages. Thus commit(nested txn) has room to merge
- * dirtylist into mt_parent after freeing hidden mt_parent pages. */
- unsigned dirtyroom;
- /* a sequence to spilling dirty page with LRU policy */
- unsigned dirtylru;
- /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
- MDBX_dpl *dirtylist;
- /* The list of reclaimed txns from GC */
- MDBX_TXL lifo_reclaimed;
- /* The list of pages that became unused during this transaction. */
- MDBX_PNL retired_pages;
- /* The list of loose pages that became unused and may be reused
- * in this transaction, linked through `mp_next`. */
- MDBX_page *loose_pages;
- /* Number of loose pages (tw.loose_pages) */
- unsigned loose_count;
- /* The sorted list of dirty pages we temporarily wrote to disk
- * because the dirty list was full. page numbers in here are
- * shifted left by 1, deleted slots have the LSB set. */
- MDBX_PNL spill_pages;
- unsigned spill_least_removed;
- } tw;
- };
-};
-
-#if MDBX_WORDBITS >= 64
-#define CURSOR_STACK 32
-#else
-#define CURSOR_STACK 24
-#endif
-
-struct MDBX_xcursor;
-
-/* Cursors are used for all DB operations.
- * A cursor holds a path of (page pointer, key index) from the DB
- * root to a position in the DB, plus other state. MDBX_DUPSORT
- * cursors include an xcursor to the current data item. Write txns
- * track their cursors and keep them up to date when data moves.
- * Exception: An xcursor's pointer to a P_SUBP page can be stale.
- * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */
-struct MDBX_cursor {
-#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1)
-#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047)
-#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7)
- uint32_t mc_signature;
- /* The database handle this cursor operates on */
- MDBX_dbi mc_dbi;
- /* Next cursor on this DB in this txn */
- MDBX_cursor *mc_next;
- /* Backup of the original cursor if this cursor is a shadow */
- MDBX_cursor *mc_backup;
- /* Context used for databases with MDBX_DUPSORT, otherwise NULL */
- struct MDBX_xcursor *mc_xcursor;
- /* The transaction that owns this cursor */
- MDBX_txn *mc_txn;
- /* The database record for this cursor */
- MDBX_db *mc_db;
- /* The database auxiliary record for this cursor */
- MDBX_dbx *mc_dbx;
- /* The mt_dbistate for this database */
- uint8_t *mc_dbistate;
- unsigned mc_snum; /* number of pushed pages */
- unsigned mc_top; /* index of top page, normally mc_snum-1 */
-
- /* Cursor state flags. */
-#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */
-#define C_EOF 0x02 /* No more data */
-#define C_SUB 0x04 /* Cursor is a sub-cursor */
-#define C_DEL 0x08 /* last op was a cursor_del */
-#define C_UNTRACK 0x10 /* Un-track cursor when closing */
-#define C_RECLAIMING 0x20 /* GC lookup is prohibited */
-#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
-
- /* Cursor checking flags. */
-#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */
-#define C_UPDATING 0x200 /* update/rebalance pending */
-#define C_RETIRING 0x400 /* refs to child pages may be invalid */
-#define C_SKIPORD 0x800 /* don't check keys ordering */
-
- unsigned mc_flags; /* see mdbx_cursor */
- MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */
- indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */
-};
-
-/* Context for sorted-dup records.
- * We could have gone to a fully recursive design, with arbitrarily
- * deep nesting of sub-databases. But for now we only handle these
- * levels - main DB, optional sub-DB, sorted-duplicate DB. */
-typedef struct MDBX_xcursor {
- /* A sub-cursor for traversing the Dup DB */
- MDBX_cursor mx_cursor;
- /* The database record for this Dup DB */
- MDBX_db mx_db;
- /* The auxiliary DB record for this Dup DB */
- MDBX_dbx mx_dbx;
-} MDBX_xcursor;
-
-typedef struct MDBX_cursor_couple {
- MDBX_cursor outer;
- void *mc_userctx; /* User-settable context */
- MDBX_xcursor inner;
-} MDBX_cursor_couple;
-
-/* The database environment. */
-struct MDBX_env {
- /* ----------------------------------------------------- mostly static part */
-#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641)
- MDBX_atomic_uint32_t me_signature;
- /* Failed to update the meta page. Probably an I/O error. */
-#define MDBX_FATAL_ERROR UINT32_C(0x80000000)
- /* Some fields are initialized. */
-#define MDBX_ENV_ACTIVE UINT32_C(0x20000000)
- /* me_txkey is set */
-#define MDBX_ENV_TXKEY UINT32_C(0x10000000)
- /* Legacy MDBX_MAPASYNC (prior v0.9) */
-#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000)
-#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY)
- uint32_t me_flags;
- mdbx_mmap_t me_dxb_mmap; /* The main data file */
-#define me_map me_dxb_mmap.dxb
-#define me_lazy_fd me_dxb_mmap.fd
- mdbx_filehandle_t me_dsync_fd;
- mdbx_mmap_t me_lck_mmap; /* The lock file */
-#define me_lfd me_lck_mmap.fd
- struct MDBX_lockinfo *me_lck;
-
- unsigned me_psize; /* DB page size, initialized from me_os_psize */
- unsigned me_leaf_nodemax; /* max size of a leaf-node */
- uint8_t me_psize2log; /* log2 of DB page size */
- int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */
- uint16_t me_merge_threshold,
- me_merge_threshold_gc; /* pages emptier than this are candidates for
- merging */
- unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */
- unsigned me_maxreaders; /* size of the reader table */
- MDBX_dbi me_maxdbs; /* size of the DB table */
- uint32_t me_pid; /* process ID of this env */
- mdbx_thread_key_t me_txkey; /* thread-key for readers */
- char *me_pathname; /* path to the DB files */
- void *me_pbuf; /* scratch area for DUPSORT put() */
- MDBX_txn *me_txn0; /* preallocated write transaction */
-
- MDBX_dbx *me_dbxs; /* array of static DB info */
- uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */
- unsigned *me_dbiseqs; /* array of dbi sequence numbers */
- unsigned
- me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */
- uint32_t me_live_reader; /* have liveness lock in reader table */
- void *me_userctx; /* User-settable context */
- MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */
-
- struct {
- unsigned dp_reserve_limit;
- unsigned rp_augment_limit;
- unsigned dp_limit;
- unsigned dp_initial;
- uint8_t dp_loose_limit;
- uint8_t spill_max_denominator;
- uint8_t spill_min_denominator;
- uint8_t spill_parent4child_denominator;
- unsigned merge_threshold_16dot16_percent;
- union {
- unsigned all;
- /* tracks options with non-auto values but tuned by user */
- struct {
- unsigned dp_limit : 1;
- } non_auto;
- } flags;
- } me_options;
-
- /* struct me_dbgeo used for accepting db-geo params from user for the new
- * database creation, i.e. when mdbx_env_set_geometry() was called before
- * mdbx_env_open(). */
- struct {
- size_t lower; /* minimal size of datafile */
- size_t upper; /* maximal size of datafile */
- size_t now; /* current size of datafile */
- size_t grow; /* step to grow datafile */
- size_t shrink; /* threshold to shrink datafile */
- } me_dbgeo;
-
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- union {
- key_t key;
- int semid;
- } me_sysv_ipc;
-#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
-
- MDBX_env *me_lcklist_next;
-
- /* --------------------------------------------------- mostly volatile part */
-
- MDBX_txn *me_txn; /* current write transaction */
- mdbx_fastmutex_t me_dbi_lock;
- MDBX_dbi me_numdbs; /* number of DBs opened */
-
- MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */
- unsigned me_dp_reserve_len;
- /* PNL of pages that became unused in a write txn */
- MDBX_PNL me_retired_pages;
-
-#if defined(_WIN32) || defined(_WIN64)
- MDBX_srwlock me_remap_guard;
- /* Workaround for LockFileEx and WriteFile multithread bug */
- CRITICAL_SECTION me_windowsbug_lock;
-#else
- mdbx_fastmutex_t me_remap_guard;
-#endif
-
- /* -------------------------------------------------------------- debugging */
-
-#if MDBX_DEBUG
- MDBX_assert_func *me_assert_func; /* Callback for assertion failures */
-#endif
-#ifdef MDBX_USE_VALGRIND
- int me_valgrind_handle;
-#endif
-#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
- pgno_t me_poison_edge;
-#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */
-
-#ifndef xMDBX_DEBUG_SPILLING
-#define xMDBX_DEBUG_SPILLING 0
-#endif
-#if xMDBX_DEBUG_SPILLING == 2
- unsigned debug_dirtied_est, debug_dirtied_act;
-#endif /* xMDBX_DEBUG_SPILLING */
-
- /* ------------------------------------------------- stub for lck-less mode */
- MDBX_atomic_uint64_t
- x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) /
- sizeof(MDBX_atomic_uint64_t)];
-};
-
-#ifndef __cplusplus
-/*----------------------------------------------------------------------------*/
-/* Debug and Logging stuff */
-
-#define MDBX_RUNTIME_FLAGS_INIT \
- ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT
-
-extern uint8_t mdbx_runtime_flags;
-extern uint8_t mdbx_loglevel;
-extern MDBX_debug_func *mdbx_debug_logger;
-
-MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
-#if MDBX_DEBUG
- if (MDBX_DBG_JITTER & mdbx_runtime_flags)
- mdbx_osal_jitter(tiny);
-#else
- (void)tiny;
-#endif
-}
-
-MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5)
- mdbx_debug_log(int level, const char *function, int line, const char *fmt,
- ...) MDBX_PRINTF_ARGS(4, 5);
-MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function,
- int line, const char *fmt,
- va_list args);
-
-#if MDBX_DEBUG
-#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel)
-#define mdbx_audit_enabled() unlikely((mdbx_runtime_flags & MDBX_DBG_AUDIT))
-#else /* MDBX_DEBUG */
-#define mdbx_log_enabled(msg) (msg < MDBX_LOG_VERBOSE && msg <= mdbx_loglevel)
-#define mdbx_audit_enabled() (0)
-#endif /* MDBX_DEBUG */
-
-#if MDBX_FORCE_ASSERTIONS
-#define mdbx_assert_enabled() (1)
-#elif MDBX_DEBUG
-#define mdbx_assert_enabled() likely((mdbx_runtime_flags & MDBX_DBG_ASSERT))
-#else
-#define mdbx_assert_enabled() (0)
-#endif /* assertions */
-
-#if !MDBX_DEBUG && defined(__ANDROID_API__)
-#define mdbx_assert_fail(env, msg, func, line) \
- __android_log_assert(msg, "mdbx", "%s:%u", func, line)
-#else
-void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
- int line);
-#endif
-
-#define mdbx_debug_extra(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
- mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
- } while (0)
-
-#define mdbx_debug_extra_print(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \
- mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \
- } while (0)
-
-#define mdbx_trace(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_TRACE)) \
- mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_debug(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \
- mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_verbose(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \
- mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_notice(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \
- mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_warning(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_WARN)) \
- mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_error(fmt, ...) \
- do { \
- if (mdbx_log_enabled(MDBX_LOG_ERROR)) \
- mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \
- __VA_ARGS__); \
- } while (0)
-
-#define mdbx_fatal(fmt, ...) \
- mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
-
-#define mdbx_ensure_msg(env, expr, msg) \
- do { \
- if (unlikely(!(expr))) \
- mdbx_assert_fail(env, msg, __func__, __LINE__); \
- } while (0)
-
-#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr)
-
-/* assert(3) variant in environment context */
-#define mdbx_assert(env, expr) \
- do { \
- if (mdbx_assert_enabled()) \
- mdbx_ensure(env, expr); \
- } while (0)
-
-/* assert(3) variant in cursor context */
-#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr)
-
-/* assert(3) variant in transaction context */
-#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr)
-
-#ifndef xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#undef assert
-#define assert(expr) mdbx_assert(NULL, expr)
-#endif
-
-/*----------------------------------------------------------------------------*/
-/* Cache coherence and mmap invalidation */
-
-#if MDBX_CPU_WRITEBACK_INCOHERENT
-#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier()
-#else
-#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier()
-#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
-
-MDBX_MAYBE_UNUSED static __inline void
-mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) {
-#if MDBX_MMAP_INCOHERENT_FILE_WRITE
- char *const begin = (char *)(-pagesize & (intptr_t)addr);
- char *const end =
- (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
- int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
- mdbx_assert(nullptr, err == 0);
- (void)err;
-#else
- (void)pagesize;
-#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
-
-#if MDBX_MMAP_INCOHERENT_CPU_CACHE
-#ifdef DCACHE
- /* MIPS has cache coherency issues.
- * Note: for any nbytes >= on-chip cache size, entire is flushed. */
- cacheflush(addr, nbytes, DCACHE);
-#else
-#error "Oops, cacheflush() not available"
-#endif /* DCACHE */
-#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
-
-#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
- (void)addr;
- (void)nbytes;
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-/* Internal prototypes */
-
-MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked,
- int *dead);
-MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key,
- MDBX_reader *begin, MDBX_reader *end);
-MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key);
-
-MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
-MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
-MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
-
-#endif /* !__cplusplus */
-
-#define MDBX_IS_ERROR(rc) \
- ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
-
-/* Internal error codes, not exposed outside libmdbx */
-#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10)
-
-/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */
-#define DDBI(mc) \
- (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
-
-/* Key size which fits in a DKBUF (debug key buffer). */
-#define DKBUF_MAX 511
-#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2]
-#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1)
-#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
-
-#if MDBX_DEBUG
-#define DKBUF_DEBUG DKBUF
-#define DKEY_DEBUG(x) DKEY(x)
-#define DVAL_DEBUG(x) DVAL(x)
-#else
-#define DKBUF_DEBUG ((void)(0))
-#define DKEY_DEBUG(x) ("-")
-#define DVAL_DEBUG(x) ("-")
-#endif
-
-/* An invalid page number.
- * Mainly used to denote an empty tree. */
-#define P_INVALID (~(pgno_t)0)
-
-/* Test if the flags f are set in a flag word w. */
-#define F_ISSET(w, f) (((w) & (f)) == (f))
-
-/* Round n up to an even number. */
-#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
-
-/* Default size of memory map.
- * This is certainly too small for any actual applications. Apps should
- * always set the size explicitly using mdbx_env_set_geometry(). */
-#define DEFAULT_MAPSIZE MEGABYTE
-
-/* Number of slots in the reader table.
- * This value was chosen somewhat arbitrarily. The 61 is a prime number,
- * and such readers plus a couple mutexes fit into single 4KB page.
- * Applications should set the table size using mdbx_env_set_maxreaders(). */
-#define DEFAULT_READERS 61
-
-/* Test if a page is a leaf page */
-#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0)
-/* Test if a page is a LEAF2 page */
-#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0)
-/* Test if a page is a branch page */
-#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0)
-/* Test if a page is an overflow page */
-#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0)
-/* Test if a page is a sub page */
-#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0)
-
-#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))
-
-/* Header for a single key/data pair within a page.
- * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2.
- * We guarantee 2-byte alignment for 'MDBX_node's.
- *
- * Leaf node flags describe node contents. F_BIGDATA says the node's
- * data part is the page number of an overflow page with actual data.
- * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in
- * a sub-page/sub-database, and named databases (just F_SUBDATA). */
-typedef struct MDBX_node {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- union {
- uint32_t mn_dsize;
- uint32_t mn_pgno32;
- };
- uint8_t mn_flags; /* see mdbx_node flags */
- uint8_t mn_extra;
- uint16_t mn_ksize; /* key size */
-#else
- uint16_t mn_ksize; /* key size */
- uint8_t mn_extra;
- uint8_t mn_flags; /* see mdbx_node flags */
- union {
- uint32_t mn_pgno32;
- uint32_t mn_dsize;
- };
-#endif /* __BYTE_ORDER__ */
-
- /* mdbx_node Flags */
-#define F_BIGDATA 0x01 /* data put on overflow page */
-#define F_SUBDATA 0x02 /* data is a sub-database */
-#define F_DUPDATA 0x04 /* data has duplicates */
-
- /* valid flags for mdbx_node_add() */
-#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND)
-
-#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
- (!defined(__cplusplus) && defined(_MSC_VER))
- uint8_t mn_data[] /* key and data are appended here */;
-#endif /* C99 */
-} MDBX_node;
-
-#define DB_PERSISTENT_FLAGS \
- (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \
- MDBX_INTEGERDUP | MDBX_REVERSEDUP)
-
-/* mdbx_dbi_open() flags */
-#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE)
-
-#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */
-#define DB_INTERNAL_FLAGS DB_VALID
-
-#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
-#error "Oops, some flags overlapped or wrong"
-#endif
-#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
-#error "Oops, some flags overlapped or wrong"
-#endif
-
-/* max number of pages to commit in one writev() call */
-#define MDBX_COMMIT_PAGES 64
-#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */
-#undef MDBX_COMMIT_PAGES
-#define MDBX_COMMIT_PAGES IOV_MAX
-#endif
-
-/*
- * /
- * | -1, a < b
- * CMP2INT(a,b) = < 0, a == b
- * | 1, a > b
- * \
- */
-#ifndef __e2k__
-/* LY: fast enough on most systems */
-#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b))
-#else
-/* LY: more parallelable on VLIW Elbrus */
-#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a)))
-#endif
-
-/* Do not spill pages to disk if txn is getting full, may fail instead */
-#define MDBX_NOSPILL 0x8000
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
-int64pgno(int64_t i64) {
- if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
- return (pgno_t)i64;
- return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
-pgno_add(size_t base, size_t augend) {
- assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
- return int64pgno(base + augend);
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t
-pgno_sub(size_t base, size_t subtrahend) {
- assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
- subtrahend < MAX_PAGENO);
- return int64pgno(base - subtrahend);
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool
-is_powerof2(size_t x) {
- return (x & (x - 1)) == 0;
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
-floor_powerof2(size_t value, size_t granularity) {
- assert(is_powerof2(granularity));
- return value & ~(granularity - 1);
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t
-ceil_powerof2(size_t value, size_t granularity) {
- return floor_powerof2(value + granularity - 1, granularity);
-}
-
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned
-log2n_powerof2(size_t value) {
- assert(value > 0 && value < INT32_MAX && is_powerof2(value));
- assert((value & -(int32_t)value) == value);
-#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
- return __builtin_ctzl(value);
-#elif defined(_MSC_VER)
- unsigned long index;
- _BitScanForward(&index, (unsigned long)value);
- return index;
-#else
- static const uint8_t debruijn_ctz32[32] = {
- 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
- 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
- return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27];
-#endif
-}
-
-/* Only a subset of the mdbx_env flags can be changed
- * at runtime. Changing other flags requires closing the
- * environment and re-opening it with the new flags. */
-#define ENV_CHANGEABLE_FLAGS \
- (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \
- MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
-#define ENV_CHANGELESS_FLAGS \
- (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
- MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
-#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
-
-#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
-MDBX_MAYBE_UNUSED static void static_checks(void) {
- STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
- "Oops, MDBX_MAX_DBI or CORE_DBS?");
- STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
- ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
- (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
- "Oops, some flags overlapped or wrong");
- STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
- "Oops, some flags overlapped or wrong");
-}
-#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */
-
-#ifdef __cplusplus
-}
-#endif
-
-#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
- do { \
- mdbx_trace("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
- (size_t)(size), __LINE__); \
- ASAN_POISON_MEMORY_REGION(addr, size); \
- } while (0)
-
-#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
- do { \
- mdbx_trace("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
- (size_t)(size), __LINE__); \
- ASAN_UNPOISON_MEMORY_REGION(addr, size); \
- } while (0)
diff --git a/3rdparty/libmdbx/src/lck-posix.c b/3rdparty/libmdbx/src/lck-posix.c
deleted file mode 100644
index 5753fdf19e496ae49c39906500f05082b52ade47..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/lck-posix.c
+++ /dev/null
@@ -1,935 +0,0 @@
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */
-
-#include "internals.h"
-
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
-#include <sys/sem.h>
-#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
-
-/*----------------------------------------------------------------------------*/
-/* global constructor/destructor */
-
-#if defined(__linux__) || defined(__gnu_linux__)
-
-#include <sys/utsname.h>
-
-#ifndef xMDBX_ALLOY
-uint32_t mdbx_linux_kernel_version;
-bool mdbx_RunningOnWSL1;
-#endif /* xMDBX_ALLOY */
-
-__cold static uint8_t probe_for_WSL(const char *tag) {
- const char *const WSL = strstr(tag, "WSL");
- if (WSL && WSL[3] >= '2' && WSL[3] <= '9')
- return WSL[3] - '0';
- const char *const wsl = strstr(tag, "wsl");
- if (wsl && wsl[3] >= '2' && wsl[3] <= '9')
- return wsl[3] - '0';
- if (WSL || wsl || strcasestr(tag, "Microsoft"))
- /* Expecting no new kernel within WSL1, either it will explicitly
- * marked by an appropriate WSL-version hint. */
- return (mdbx_linux_kernel_version < /* 4.19.x */ 0x04130000) ? 1 : 2;
- return 0;
-}
-
-#endif /* Linux */
-
-__cold static __attribute__((__constructor__)) void
-mdbx_global_constructor(void) {
-#if defined(__linux__) || defined(__gnu_linux__)
- struct utsname buffer;
- if (uname(&buffer) == 0) {
- int i = 0;
- char *p = buffer.release;
- while (*p && i < 4) {
- if (*p >= '0' && *p <= '9') {
- long number = strtol(p, &p, 10);
- if (number > 0) {
- if (number > 255)
- number = 255;
- mdbx_linux_kernel_version += number << (24 - i * 8);
- }
- ++i;
- } else {
- ++p;
- }
- }
- /* "Official" way of detecting WSL1 but not WSL2
- * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
- *
- * WARNING: False negative detection of WSL1 will result in DATA LOSS!
- * So, the REQUIREMENTS for this code:
- * 1. MUST detect WSL1 without false-negatives.
- * 2. DESIRABLE detect WSL2 but without the risk of violating the first. */
- mdbx_RunningOnWSL1 = probe_for_WSL(buffer.version) == 1 ||
- probe_for_WSL(buffer.sysname) == 1 ||
- probe_for_WSL(buffer.release) == 1;
- }
-#endif /* Linux */
-
- mdbx_rthc_global_init();
-}
-
-__cold static __attribute__((__destructor__)) void
-mdbx_global_destructor(void) {
- mdbx_rthc_global_dtor();
-}
-
-/*----------------------------------------------------------------------------*/
-/* lck */
-
-/* ОпиÑание реализации блокировок Ð´Ð»Ñ POSIX & Linux:
- *
- * lck-файл отображаетÑÑ Ð² памÑÑ‚ÑŒ, в нём организуетÑÑ Ñ‚Ð°Ð±Ð»Ð¸Ñ†Ð° читателей и
- * размещаютÑÑ ÑовмеÑтно иÑпользуемые posix-мьютекÑÑ‹ (futex). ПоÑредÑтвом
- * Ñтих мьютекÑов (Ñм struct MDBX_lockinfo) реализуютÑÑ:
- * - Блокировка таблицы читателей Ð´Ð»Ñ Ñ€ÐµÐ³Ð¸Ñтрации,
- * т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
- * - Блокировка БД Ð´Ð»Ñ Ð¿Ð¸ÑˆÑƒÑ‰Ð¸Ñ… транзакций,
- * т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
- *
- * ОÑтальной функционал реализуетÑÑ Ð¾Ñ‚Ð´ÐµÐ»ÑŒÐ½Ð¾ поÑредÑтвом файловых блокировок:
- * - Первоначальный захват БД в режиме exclusive/shared и поÑледующий перевод
- * в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
- * - Проверка приÑутÑтвие процеÑÑов-читателей,
- * т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
- *
- * Ð”Ð»Ñ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ¸ файлов иÑпользуетÑÑ fcntl(F_SETLK), так как:
- * - lockf() оперирует только ÑкÑклюзивной блокировкой и требует
- * Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚Ð¸Ñ Ñ„Ð°Ð¹Ð»Ð° в RW-режиме.
- * - flock() не гарантирует атомарноÑти при Ñмене блокировок
- * и оперирует только вÑем файлом целиком.
- * - Ð”Ð»Ñ ÐºÐ¾Ð½Ñ‚Ñ€Ð¾Ð»Ñ Ð¿Ñ€Ð¾Ñ†ÐµÑÑов-читателей иÑпользуютÑÑ Ð¾Ð´Ð½Ð¾Ð±Ð°Ð¹Ñ‚Ð¾Ð²Ñ‹Ðµ
- * range-блокировки lck-файла поÑредÑтвом fcntl(F_SETLK). При Ñтом
- * в качеÑтве позиции иÑпользуетÑÑ pid процеÑÑа-читателÑ.
- * - Ð”Ð»Ñ Ð¿ÐµÑ€Ð²Ð¾Ð½Ð°Ñ‡Ð°Ð»ÑŒÐ½Ð¾Ð³Ð¾ захвата и shared/exclusive выполнÑетÑÑ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ°
- * оÑновного файла БД и при уÑпехе lck-файла.
- *
- * ----------------------------------------------------------------------------
- * УДЕРЖИВÐЕМЫЕ БЛОКИРОВКИ Ð’ ЗÐВИСИМОСТИ ОТ РЕЖИМРИ СОСТОЯÐИЯ
- *
- * ÐкÑклюзивный режим без lck-файла:
- * = заблокирован веÑÑŒ dxb-файл поÑредÑтвом F_RDLCK или F_WRLCK,
- * в завиÑимоÑти от MDBX_RDONLY.
- *
- * Ðе-операционный режим на Ð²Ñ€ÐµÐ¼Ñ Ð¿ÐµÑ€Ðµ-инициализации и разрушении lck-файла:
- * = F_WRLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ ждут её
- * ÑнÑÑ‚Ð¸Ñ Ð¿Ñ€Ð¸ получении F_RDLCK через F_SETLKW.
- * - блокировки dxb-файла могут менÑÑ‚ÑŒÑÑ Ð´Ð¾ ÑнÑтие ÑкÑклюзивной блокировки
- * lck-файла:
- * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле
- * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
- * + Ð´Ð»Ñ ÐКСКЛЮЗИВÐОГО режима блокировка pid-байта вÑего dxb-файла
- * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
- *
- * ОПЕРÐЦИОÐÐЫЙ режим Ñ lck-файлом:
- * = F_RDLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ не могут
- * получить F_WRLCK и таким образом видÑÑ‚ что БД иÑпользуетÑÑ.
- * + F_WRLCK блокировка pid-байта в clk-файле поÑле первой транзакции чтениÑ.
- * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле
- * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
- * + Ð´Ð»Ñ ÐКСКЛЮЗИВÐОГО режима блокировка pid-байта вÑего dxb-файла
- * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY.
- */
-
-#if MDBX_USE_OFDLOCKS
-static int op_setlk, op_setlkw, op_getlk;
-__cold static void choice_fcntl() {
- assert(!op_setlk && !op_setlkw && !op_getlk);
- if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
-#if defined(__linux__) || defined(__gnu_linux__)
- && mdbx_linux_kernel_version >
- 0x030f0000 /* OFD locks are available since 3.15, but engages here
- only for 3.16 and later kernels (i.e. LTS) because
- of reliability reasons */
-#endif /* linux */
- ) {
- op_setlk = F_OFD_SETLK;
- op_setlkw = F_OFD_SETLKW;
- op_getlk = F_OFD_GETLK;
- return;
- }
- op_setlk = F_SETLK;
- op_setlkw = F_SETLKW;
- op_getlk = F_GETLK;
-}
-#else
-#define op_setlk F_SETLK
-#define op_setlkw F_SETLKW
-#define op_getlk F_GETLK
-#endif /* MDBX_USE_OFDLOCKS */
-
-#ifndef OFF_T_MAX
-#define OFF_T_MAX \
- (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
-#endif
-
-static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
- const off_t offset, off_t len) {
- STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) &&
- sizeof(off_t) >= sizeof(size_t));
-#ifdef __ANDROID_API__
- STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS),
- "The bitness of system `off_t` type is mismatch. Please "
- "fix build and/or NDK configuration.");
-#endif /* Android */
- mdbx_jitter4testing(true);
- assert(offset >= 0 && len > 0);
- assert((uint64_t)offset < (uint64_t)INT64_MAX &&
- (uint64_t)len < (uint64_t)INT64_MAX &&
- (uint64_t)(offset + len) > (uint64_t)offset);
-
- assert((uint64_t)offset < (uint64_t)OFF_T_MAX &&
- (uint64_t)len <= (uint64_t)OFF_T_MAX &&
- (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX);
-
- assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) ==
- ((uint64_t)offset + (uint64_t)len));
- for (;;) {
- struct flock lock_op;
- STATIC_ASSERT(sizeof(off_t) <= sizeof(lock_op.l_start) &&
- sizeof(off_t) <= sizeof(lock_op.l_len) &&
- OFF_T_MAX == (off_t)OFF_T_MAX);
- memset(&lock_op, 0, sizeof(lock_op));
- lock_op.l_type = lck;
- lock_op.l_whence = SEEK_SET;
- lock_op.l_start = offset;
- lock_op.l_len = len;
- int rc = fcntl(fd, cmd, &lock_op);
- mdbx_jitter4testing(true);
- if (rc != -1) {
- if (cmd == op_getlk) {
- /* Checks reader by pid. Returns:
- * MDBX_RESULT_TRUE - if pid is live (reader holds a lock).
- * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */
- return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
- : MDBX_RESULT_TRUE;
- }
- return MDBX_SUCCESS;
- }
- rc = errno;
-#if MDBX_USE_OFDLOCKS
- if (rc == EINVAL &&
- (cmd == F_OFD_SETLK || cmd == F_OFD_SETLKW || cmd == F_OFD_GETLK)) {
- /* fallback to non-OFD locks */
- if (cmd == F_OFD_SETLK)
- cmd = F_SETLK;
- else if (cmd == F_OFD_SETLKW)
- cmd = F_SETLKW;
- else
- cmd = F_GETLK;
- op_setlk = F_SETLK;
- op_setlkw = F_SETLKW;
- op_getlk = F_GETLK;
- continue;
- }
-#endif /* MDBX_USE_OFDLOCKS */
- if (rc != EINTR || cmd == op_setlkw) {
- assert(MDBX_IS_ERROR(rc));
- return rc;
- }
- }
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait) {
-#if MDBX_USE_OFDLOCKS
- if (unlikely(op_setlk == 0))
- choice_fcntl();
-#endif /* MDBX_USE_OFDLOCKS */
- return lck_op(fd, wait ? op_setlkw : op_setlk, F_WRLCK, 0, OFF_T_MAX);
-}
-
-MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
- assert(env->me_pid > 0);
- if (unlikely(mdbx_getpid() != env->me_pid))
- return MDBX_PANIC;
- return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1);
-}
-
-MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
- assert(env->me_pid > 0);
- return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1);
-}
-
-MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
- assert(pid > 0);
- return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1);
-}
-
-/*---------------------------------------------------------------------------*/
-
-#if MDBX_LOCKING > MDBX_LOCKING_SYSV
-MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc) {
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- return sem_init(ipc, false, 1) ? errno : 0;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- return pthread_mutex_init(ipc, nullptr);
-#else
-#error "FIXME"
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc) {
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- return sem_destroy(ipc) ? errno : 0;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- return pthread_mutex_destroy(ipc);
-#else
-#error "FIXME"
-#endif
-}
-#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */
-
-static int check_fstat(MDBX_env *env) {
- struct stat st;
-
- int rc = MDBX_SUCCESS;
- if (fstat(env->me_lazy_fd, &st)) {
- rc = errno;
- mdbx_error("fstat(%s), err %d", "DXB", rc);
- return rc;
- }
-
- if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
-#ifdef EBADFD
- rc = EBADFD;
-#else
- rc = EPERM;
-#endif
- mdbx_error("%s %s, err %d", "DXB",
- (st.st_nlink < 1) ? "file was removed" : "not a regular file",
- rc);
- return rc;
- }
-
- if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) {
- mdbx_verbose("dxb-file is too short (%u), exclusive-lock needed",
- (unsigned)st.st_size);
- rc = MDBX_RESULT_TRUE;
- }
-
- //----------------------------------------------------------------------------
-
- if (fstat(env->me_lfd, &st)) {
- rc = errno;
- mdbx_error("fstat(%s), err %d", "LCK", rc);
- return rc;
- }
-
- if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
-#ifdef EBADFD
- rc = EBADFD;
-#else
- rc = EPERM;
-#endif
- mdbx_error("%s %s, err %d", "LCK",
- (st.st_nlink < 1) ? "file was removed" : "not a regular file",
- rc);
- return rc;
- }
-
- /* Checking file size for detect the situation when we got the shared lock
- * immediately after mdbx_lck_destroy(). */
- if (st.st_size < (off_t)(sizeof(MDBX_lockinfo) + sizeof(MDBX_reader))) {
- mdbx_verbose("lck-file is too short (%u), exclusive-lock needed",
- (unsigned)st.st_size);
- rc = MDBX_RESULT_TRUE;
- }
-
- return rc;
-}
-
-__cold MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
- assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
- if (unlikely(mdbx_getpid() != env->me_pid))
- return MDBX_PANIC;
-#if MDBX_USE_OFDLOCKS
- if (unlikely(op_setlk == 0))
- choice_fcntl();
-#endif /* MDBX_USE_OFDLOCKS */
-
- int rc = MDBX_SUCCESS;
-#if defined(__linux__) || defined(__gnu_linux__)
- if (unlikely(mdbx_RunningOnWSL1)) {
- rc = ENOLCK /* No record locks available */;
- mdbx_error("%s, err %u",
- "WSL1 (Windows Subsystem for Linux) is mad and trouble-full, "
- "injecting failure to avoid data loss",
- rc);
- return rc;
- }
-#endif /* Linux */
-
- if (env->me_lfd == INVALID_HANDLE_VALUE) {
- /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
- rc =
- lck_op(env->me_lazy_fd, op_setlk,
- (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
- if (rc != MDBX_SUCCESS) {
- mdbx_error("%s, err %u", "without-lck", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
- return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
- }
-#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0
- sched_yield();
-#endif
-
-retry:
- if (rc == MDBX_RESULT_TRUE) {
- rc = lck_op(env->me_lfd, op_setlk, F_UNLCK, 0, 1);
- if (rc != MDBX_SUCCESS) {
- mdbx_error("%s, err %u", "unlock-before-retry", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
- }
-
- /* Firstly try to get exclusive locking. */
- rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
- if (rc == MDBX_SUCCESS) {
- rc = check_fstat(env);
- if (MDBX_IS_ERROR(rc))
- return rc;
-
- continue_dxb_exclusive:
- rc =
- lck_op(env->me_lazy_fd, op_setlk,
- (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
- if (rc == MDBX_SUCCESS)
- return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
-
- int err = check_fstat(env);
- if (MDBX_IS_ERROR(err))
- return err;
-
- /* the cause may be a collision with POSIX's file-lock recovery. */
- if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
- rc == EDEADLK)) {
- mdbx_error("%s, err %u", "dxb-exclusive", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
-
- /* Fallback to lck-shared */
- } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY ||
- rc == EWOULDBLOCK || rc == EDEADLK)) {
- mdbx_error("%s, err %u", "try-exclusive", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
-
- /* Here could be one of two:
- * - mdbx_lck_destroy() from the another process was hold the lock
- * during a destruction.
- * - either mdbx_lck_seize() from the another process was got the exclusive
- * lock and doing initialization.
- * For distinguish these cases will use size of the lck-file later. */
-
- /* Wait for lck-shared now. */
- /* Here may be await during transient processes, for instance until another
- * competing process doesn't call lck_downgrade(). */
- rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1);
- if (rc != MDBX_SUCCESS) {
- mdbx_error("%s, err %u", "try-shared", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
-
- rc = check_fstat(env);
- if (rc == MDBX_RESULT_TRUE)
- goto retry;
- if (rc != MDBX_SUCCESS) {
- mdbx_error("%s, err %u", "lck_fstat", rc);
- return rc;
- }
-
- /* got shared, retry exclusive */
- rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
- if (rc == MDBX_SUCCESS)
- goto continue_dxb_exclusive;
-
- if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
- rc == EDEADLK)) {
- mdbx_error("%s, err %u", "try-exclusive", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
-
- /* Lock against another process operating in without-lck or exclusive mode. */
- rc =
- lck_op(env->me_lazy_fd, op_setlk,
- (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1);
- if (rc != MDBX_SUCCESS) {
- mdbx_error("%s, err %u", "lock-against-without-lck", rc);
- mdbx_assert(env, MDBX_IS_ERROR(rc));
- return rc;
- }
-
- /* Done: return with shared locking. */
- return MDBX_RESULT_FALSE;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
- if (unlikely(mdbx_getpid() != env->me_pid))
- return MDBX_PANIC;
-
- int rc = MDBX_SUCCESS;
- if ((env->me_flags & MDBX_EXCLUSIVE) == 0) {
- rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, 0, env->me_pid);
- if (rc == MDBX_SUCCESS)
- rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, env->me_pid + 1,
- OFF_T_MAX - env->me_pid - 1);
- }
- if (rc == MDBX_SUCCESS)
- rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
- if (unlikely(rc != 0)) {
- mdbx_error("%s, err %u", "lck", rc);
- assert(MDBX_IS_ERROR(rc));
- }
- return rc;
-}
-
-__cold MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
- MDBX_env *inprocess_neighbor) {
- if (unlikely(mdbx_getpid() != env->me_pid))
- return MDBX_PANIC;
-
- int rc = MDBX_SUCCESS;
- struct stat lck_info;
- MDBX_lockinfo *lck = env->me_lck_mmap.lck;
- if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && lck &&
- /* try get exclusive access */
- lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
- /* if LCK was not removed */
- fstat(env->me_lfd, &lck_info) == 0 && lck_info.st_nlink > 0 &&
- lck_op(env->me_lazy_fd, op_setlk,
- (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
- OFF_T_MAX) == 0) {
-
- mdbx_verbose("%p got exclusive, drown locks", (void *)env);
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- if (env->me_sysv_ipc.semid != -1)
- rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0;
-#else
- rc = mdbx_ipclock_destroy(&lck->mti_rlock);
- if (rc == 0)
- rc = mdbx_ipclock_destroy(&lck->mti_wlock);
-#endif /* MDBX_LOCKING */
-
- mdbx_assert(env, rc == 0);
- if (rc == 0) {
- const bool synced = lck->mti_unsynced_pages.weak == 0;
- mdbx_munmap(&env->me_lck_mmap);
- if (synced)
- rc = ftruncate(env->me_lfd, 0) ? errno : 0;
- }
-
- mdbx_jitter4testing(false);
- }
-
- /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored
- * after file was closed.
- *
- * 2) File locks would be released (by kernel) while the file-descriptors will
- * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel,
- * locks should be released here explicitly with properly order. */
-
- /* close dxb and restore lock */
- if (env->me_dsync_fd != INVALID_HANDLE_VALUE) {
- if (unlikely(close(env->me_dsync_fd) != 0) && rc == MDBX_SUCCESS)
- rc = errno;
- env->me_dsync_fd = INVALID_HANDLE_VALUE;
- }
- if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
- if (unlikely(close(env->me_lazy_fd) != 0) && rc == MDBX_SUCCESS)
- rc = errno;
- env->me_lazy_fd = INVALID_HANDLE_VALUE;
- if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
- /* restore file-lock */
- rc = lck_op(
- inprocess_neighbor->me_lazy_fd, F_SETLKW,
- (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
- (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE)
- ? 0
- : inprocess_neighbor->me_pid,
- (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
- }
- }
-
- /* close clk and restore locks */
- if (env->me_lfd != INVALID_HANDLE_VALUE) {
- if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS)
- rc = errno;
- env->me_lfd = INVALID_HANDLE_VALUE;
- if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
- /* restore file-locks */
- rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1);
- if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader)
- rc = mdbx_rpid_set(inprocess_neighbor);
- }
- }
-
- if (inprocess_neighbor && rc != MDBX_SUCCESS)
- inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR;
- return rc;
-}
-
-/*---------------------------------------------------------------------------*/
-
-__cold MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
- MDBX_env *inprocess_neighbor,
- int global_uniqueness_flag) {
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- int semid = -1;
- /* don't initialize semaphores twice */
- (void)inprocess_neighbor;
- if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
- struct stat st;
- if (fstat(env->me_lazy_fd, &st))
- return errno;
- sysv_retry_create:
- semid = semget(env->me_sysv_ipc.key, 2,
- IPC_CREAT | IPC_EXCL |
- (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)));
- if (unlikely(semid == -1)) {
- int err = errno;
- if (err != EEXIST)
- return err;
-
- /* remove and re-create semaphore set */
- semid = semget(env->me_sysv_ipc.key, 2, 0);
- if (semid == -1) {
- err = errno;
- if (err != ENOENT)
- return err;
- goto sysv_retry_create;
- }
- if (semctl(semid, 2, IPC_RMID)) {
- err = errno;
- if (err != EIDRM)
- return err;
- }
- goto sysv_retry_create;
- }
-
- unsigned short val_array[2] = {1, 1};
- if (semctl(semid, 2, SETALL, val_array))
- return errno;
- } else {
- semid = semget(env->me_sysv_ipc.key, 2, 0);
- if (semid == -1)
- return errno;
-
- /* check read & write access */
- struct semid_ds data[2];
- if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data))
- return errno;
- }
-
- env->me_sysv_ipc.semid = semid;
- return MDBX_SUCCESS;
-
-#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
- (void)inprocess_neighbor;
- if (global_uniqueness_flag != MDBX_RESULT_TRUE)
- return MDBX_SUCCESS;
-#error "FIXME: Not implemented"
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
-
- /* don't initialize semaphores twice */
- (void)inprocess_neighbor;
- if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
- if (sem_init(&env->me_lck_mmap.lck->mti_rlock, true, 1))
- return errno;
- if (sem_init(&env->me_lck_mmap.lck->mti_wlock, true, 1))
- return errno;
- }
- return MDBX_SUCCESS;
-
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- if (inprocess_neighbor)
- return MDBX_SUCCESS /* don't need any initialization for mutexes
- if LCK already opened/used inside current process */
- ;
-
- /* FIXME: Unfortunately, there is no other reliable way but to long testing
- * on each platform. On the other hand, behavior like FreeBSD is incorrect
- * and we can expect it to be rare. Moreover, even on FreeBSD without
- * additional in-process initialization, the probability of an problem
- * occurring is vanishingly small, and the symptom is a return of EINVAL
- * while locking a mutex. In other words, in the worst case, the problem
- * results in an EINVAL error at the start of the transaction, but NOT data
- * loss, nor database corruption, nor other fatal troubles. Thus, the code
- * below I am inclined to think the workaround for erroneous platforms (like
- * FreeBSD), rather than a defect of libmdbx. */
-#if defined(__FreeBSD__)
- /* seems that shared mutexes on FreeBSD required in-process initialization */
- (void)global_uniqueness_flag;
-#else
- /* shared mutexes on many other platforms (including Darwin and Linux's
- * futexes) doesn't need any addition in-process initialization */
- if (global_uniqueness_flag != MDBX_RESULT_TRUE)
- return MDBX_SUCCESS;
-#endif
-
- pthread_mutexattr_t ma;
- int rc = pthread_mutexattr_init(&ma);
- if (rc)
- return rc;
-
- rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
- if (rc)
- goto bailout;
-
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008
-#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust)
- rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
-#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \
- defined(pthread_mutexattr_setrobust_np)
- rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
-#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
- rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
-#else
- rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
-#endif
- if (rc)
- goto bailout;
-#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
-
-#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \
- !defined(MDBX_SAFE4QEMU)
- rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
- if (rc == ENOTSUP)
- rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
- if (rc && rc != ENOTSUP)
- goto bailout;
-#endif /* PTHREAD_PRIO_INHERIT */
-
- rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
- if (rc && rc != ENOTSUP)
- goto bailout;
-
- rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_rlock, &ma);
- if (rc)
- goto bailout;
- rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_wlock, &ma);
-
-bailout:
- pthread_mutexattr_destroy(&ma);
- return rc;
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING > 0 */
-}
-
-__cold static int mdbx_ipclock_failed(MDBX_env *env, mdbx_ipclock_t *ipc,
- const int err) {
- int rc = err;
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV
- if (err == EOWNERDEAD) {
- /* We own the mutex. Clean up after dead previous owner. */
-
- const bool rlocked = ipc == &env->me_lck->mti_rlock;
- rc = MDBX_SUCCESS;
- if (!rlocked) {
- if (unlikely(env->me_txn)) {
- /* env is hosed if the dead thread was ours */
- env->me_flags |= MDBX_FATAL_ERROR;
- env->me_txn = NULL;
- rc = MDBX_PANIC;
- }
- }
- mdbx_warning("%clock owner died, %s", (rlocked ? 'r' : 'w'),
- (rc ? "this process' env is hosed" : "recovering"));
-
- int check_rc = mdbx_cleanup_dead_readers(env, rlocked, NULL);
- check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
-
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
-#else
-#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent)
- int mreco_rc = pthread_mutex_consistent(ipc);
-#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np)
- int mreco_rc = pthread_mutex_consistent_np(ipc);
-#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
- int mreco_rc = pthread_mutex_consistent_np(ipc);
-#else
- int mreco_rc = pthread_mutex_consistent(ipc);
-#endif
- check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
-
- if (unlikely(mreco_rc))
- mdbx_error("lock recovery failed, %s", mdbx_strerror(mreco_rc));
-
- rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
- if (MDBX_IS_ERROR(rc))
- pthread_mutex_unlock(ipc);
-#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
- return rc;
- }
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001
- (void)ipc;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- (void)ipc;
-#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
-#ifdef _MSC_VER
-#pragma message("warning: TODO")
-#else
-#warning "TODO"
-#endif
- (void)ipc;
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-
- mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
- if (rc != EDEADLK)
- env->me_flags |= MDBX_FATAL_ERROR;
- return rc;
-}
-
-#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
-MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void) {
- /* avoid 32-bit Bionic bug/hang with 32-pit TID */
- if (sizeof(pthread_mutex_t) < sizeof(pid_t) + sizeof(unsigned)) {
- pid_t tid = gettid();
- if (unlikely(tid > 0xffff)) {
- mdbx_fatal("Raise the ENOSYS(%d) error to avoid hang due "
- "the 32-bit Bionic/Android bug with tid/thread_id 0x%08x(%i) "
- "that don’t fit in 16 bits, see "
- "https://android.googlesource.com/platform/bionic/+/master/"
- "docs/32-bit-abi.md#is-too-small-for-large-pids",
- ENOSYS, tid, tid);
- return ENOSYS;
- }
- }
- return 0;
-}
-#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
-
-static int mdbx_ipclock_lock(MDBX_env *env, mdbx_ipclock_t *ipc,
- const bool dont_wait) {
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- int rc = mdbx_check_tid4bionic();
- if (likely(rc == 0))
- rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc);
- rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- int rc = MDBX_SUCCESS;
- if (dont_wait) {
- if (sem_trywait(ipc)) {
- rc = errno;
- if (rc == EAGAIN)
- rc = MDBX_BUSY;
- }
- } else if (sem_wait(ipc))
- rc = errno;
-#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
- struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock),
- .sem_op = -1,
- .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO};
- int rc;
- if (semop(env->me_sysv_ipc.semid, &op, 1)) {
- rc = errno;
- if (dont_wait && rc == EAGAIN)
- rc = MDBX_BUSY;
- } else {
- rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS;
- *ipc = env->me_pid;
- }
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-
- if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY))
- rc = mdbx_ipclock_failed(env, ipc, rc);
- return rc;
-}
-
-static int mdbx_ipclock_unlock(MDBX_env *env, mdbx_ipclock_t *ipc) {
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- int rc = pthread_mutex_unlock(ipc);
- (void)env;
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- int rc = sem_post(ipc) ? errno : MDBX_SUCCESS;
- (void)env;
-#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
- if (unlikely(*ipc != (pid_t)env->me_pid))
- return EPERM;
- *ipc = 0;
- struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock),
- .sem_op = 1,
- .sem_flg = SEM_UNDO};
- int rc = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS;
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
- return rc;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
- mdbx_trace("%s", ">>");
- mdbx_jitter4testing(true);
- int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_rlock, false);
- mdbx_trace("<< rc %d", rc);
- return rc;
-}
-
-MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
- mdbx_trace("%s", ">>");
- int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_rlock);
- mdbx_trace("<< rc %d", rc);
- if (unlikely(rc != MDBX_SUCCESS))
- mdbx_panic("%s() failed: err %d\n", __func__, rc);
- mdbx_jitter4testing(true);
-}
-
-int mdbx_txn_lock(MDBX_env *env, bool dont_wait) {
- mdbx_trace("%swait %s", dont_wait ? "dont-" : "", ">>");
- mdbx_jitter4testing(true);
- int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_wlock, dont_wait);
- mdbx_trace("<< rc %d", rc);
- return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
-}
-
-void mdbx_txn_unlock(MDBX_env *env) {
- mdbx_trace("%s", ">>");
- int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_wlock);
- mdbx_trace("<< rc %d", rc);
- if (unlikely(rc != MDBX_SUCCESS))
- mdbx_panic("%s() failed: err %d\n", __func__, rc);
- mdbx_jitter4testing(true);
-}
-
-#else
-#ifdef _MSC_VER
-#pragma warning(disable : 4206) /* nonstandard extension used: translation \
- unit is empty */
-#endif /* _MSC_VER (warnings) */
-#endif /* !Windows LCK-implementation */
diff --git a/3rdparty/libmdbx/src/lck-windows.c b/3rdparty/libmdbx/src/lck-windows.c
deleted file mode 100644
index 6ba3f3a9b53302514abe4031b9b8ba9cae3ceb9a..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/lck-windows.c
+++ /dev/null
@@ -1,817 +0,0 @@
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */
-
-/* PREAMBLE FOR WINDOWS:
- *
- * We are not concerned for performance here.
- * If you are running Windows a performance could NOT be the goal.
- * Otherwise please use Linux. */
-
-#include "internals.h"
-
-static void mdbx_winnt_import(void);
-
-#if MDBX_BUILD_SHARED_LIBRARY
-#if MDBX_WITHOUT_MSVC_CRT && defined(NDEBUG)
-/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
- *
- * Define dll's entry point only for Release build when NDEBUG is defined and
- * MDBX_WITHOUT_MSVC_CRT=ON. if the entry point isn't defined then MSVC's will
- * automatically use DllMainCRTStartup() from CRT library, which also
- * automatically call DllMain() from our mdbx.dll */
-#pragma comment(linker, "/ENTRY:DllMain")
-#endif /* MDBX_WITHOUT_MSVC_CRT */
-
-BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
-#else
-#if !MDBX_MANUAL_MODULE_HANDLER
-static
-#endif /* !MDBX_MANUAL_MODULE_HANDLER */
- void NTAPI
- mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved)
-#endif /* MDBX_BUILD_SHARED_LIBRARY */
-{
- (void)reserved;
- switch (reason) {
- case DLL_PROCESS_ATTACH:
- mdbx_winnt_import();
- mdbx_rthc_global_init();
- break;
- case DLL_PROCESS_DETACH:
- mdbx_rthc_global_dtor();
- break;
-
- case DLL_THREAD_ATTACH:
- break;
- case DLL_THREAD_DETACH:
- mdbx_rthc_thread_dtor(module);
- break;
- }
-#if MDBX_BUILD_SHARED_LIBRARY
- return TRUE;
-#endif
-}
-
-#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER
-/* *INDENT-OFF* */
-/* clang-format off */
-#if defined(_MSC_VER)
-# pragma const_seg(push)
-# pragma data_seg(push)
-
-# ifndef _M_IX86
- /* kick a linker to create the TLS directory if not already done */
-# pragma comment(linker, "/INCLUDE:_tls_used")
- /* Force some symbol references. */
-# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
- /* specific const-segment for WIN64 */
-# pragma const_seg(".CRT$XLB")
- const
-# else
- /* kick a linker to create the TLS directory if not already done */
-# pragma comment(linker, "/INCLUDE:__tls_used")
- /* Force some symbol references. */
-# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
- /* specific data-segment for WIN32 */
-# pragma data_seg(".CRT$XLB")
-# endif
-
- __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_module_handler;
-# pragma data_seg(pop)
-# pragma const_seg(pop)
-
-#elif defined(__GNUC__)
-# ifndef _M_IX86
- const
-# endif
- PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_module_handler;
-#else
-# error FIXME
-#endif
-/* *INDENT-ON* */
-/* clang-format on */
-#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER */
-
-/*----------------------------------------------------------------------------*/
-
-#define LCK_SHARED 0
-#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
-#define LCK_WAITFOR 0
-#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
-
-static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
- size_t bytes) {
- OVERLAPPED ov;
- ov.hEvent = 0;
- ov.Offset = (DWORD)offset;
- ov.OffsetHigh = HIGH_DWORD(offset);
- return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
-}
-
-static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
- size_t bytes) {
- return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
- HIGH_DWORD(bytes));
-}
-
-/*----------------------------------------------------------------------------*/
-/* global `write` lock for write-txt processing,
- * exclusive locking both meta-pages) */
-
-#define LCK_MAXLEN (1u + ((~(size_t)0) >> 1))
-#define LCK_META_OFFSET 0
-#define LCK_META_LEN (MAX_PAGESIZE * NUM_METAS)
-#define LCK_BODY_OFFSET LCK_META_LEN
-#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET)
-#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
-#define LCK_WHOLE 0, LCK_MAXLEN
-
-int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
- if (dontwait) {
- if (!TryEnterCriticalSection(&env->me_windowsbug_lock))
- return MDBX_BUSY;
- } else {
- __try {
- EnterCriticalSection(&env->me_windowsbug_lock);
- }
- __except ((GetExceptionCode() ==
- 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
- ? EXCEPTION_EXECUTE_HANDLER
- : EXCEPTION_CONTINUE_SEARCH) {
- return ERROR_POSSIBLE_DEADLOCK;
- }
- }
-
- if ((env->me_flags & MDBX_EXCLUSIVE) ||
- flock(env->me_lazy_fd,
- dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
- : (LCK_EXCLUSIVE | LCK_WAITFOR),
- LCK_BODY))
- return MDBX_SUCCESS;
- int rc = (int)GetLastError();
- LeaveCriticalSection(&env->me_windowsbug_lock);
- return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
-}
-
-void mdbx_txn_unlock(MDBX_env *env) {
- int rc = (env->me_flags & MDBX_EXCLUSIVE)
- ? TRUE
- : funlock(env->me_lazy_fd, LCK_BODY);
- LeaveCriticalSection(&env->me_windowsbug_lock);
- if (!rc)
- mdbx_panic("%s failed: err %u", __func__, (int)GetLastError());
-}
-
-/*----------------------------------------------------------------------------*/
-/* global `read` lock for readers registration,
- * exclusive locking `mti_numreaders` (second) cacheline */
-
-#define LCK_LO_OFFSET 0
-#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
-#define LCK_UP_OFFSET LCK_LO_LEN
-#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET)
-#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
-#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
-
-MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
- mdbx_srwlock_AcquireShared(&env->me_remap_guard);
- if (env->me_lfd == INVALID_HANDLE_VALUE)
- return MDBX_SUCCESS; /* readonly database in readonly filesystem */
-
- /* transition from S-? (used) to S-E (locked),
- * e.g. exclusive lock upper-part */
- if ((env->me_flags & MDBX_EXCLUSIVE) ||
- flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
- return MDBX_SUCCESS;
-
- int rc = (int)GetLastError();
- mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
- return rc;
-}
-
-MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
- if (env->me_lfd != INVALID_HANDLE_VALUE) {
- /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */
- if ((env->me_flags & MDBX_EXCLUSIVE) == 0 &&
- !funlock(env->me_lfd, LCK_UPPER))
- mdbx_panic("%s failed: err %u", __func__, (int)GetLastError());
- }
- mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait) {
- return flock(fd,
- wait ? LCK_EXCLUSIVE | LCK_WAITFOR
- : LCK_EXCLUSIVE | LCK_DONTWAIT,
- 0, LCK_MAXLEN)
- ? MDBX_SUCCESS
- : (int)GetLastError();
-}
-
-static int suspend_and_append(mdbx_handle_array_t **array,
- const DWORD ThreadId) {
- const unsigned limit = (*array)->limit;
- if ((*array)->count == limit) {
- void *ptr = mdbx_realloc(
- (limit > ARRAY_LENGTH((*array)->handles))
- ? *array
- : /* don't free initial array on the stack */ NULL,
- sizeof(mdbx_handle_array_t) +
- sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles)));
- if (!ptr)
- return MDBX_ENOMEM;
- if (limit == ARRAY_LENGTH((*array)->handles))
- memcpy(ptr, *array, sizeof(mdbx_handle_array_t));
- *array = (mdbx_handle_array_t *)ptr;
- (*array)->limit = limit * 2;
- }
-
- HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
- FALSE, ThreadId);
- if (hThread == NULL)
- return (int)GetLastError();
-
- if (SuspendThread(hThread) == (DWORD)-1) {
- int err = (int)GetLastError();
- DWORD ExitCode;
- if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
- !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
- err = MDBX_SUCCESS;
- CloseHandle(hThread);
- return err;
- }
-
- (*array)->handles[(*array)->count++] = hThread;
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int
-mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
- mdbx_assert(env, (env->me_flags & MDBX_NOTLS) == 0);
- const uintptr_t CurrentTid = GetCurrentThreadId();
- int rc;
- if (env->me_lck_mmap.lck) {
- /* Scan LCK for threads of the current process */
- const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers;
- const MDBX_reader *const end =
- begin +
- atomic_load32(&env->me_lck_mmap.lck->mti_numreaders, mo_AcquireRelease);
- const uintptr_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0;
- for (const MDBX_reader *reader = begin; reader < end; ++reader) {
- if (reader->mr_pid.weak != env->me_pid || !reader->mr_tid.weak) {
- skip_lck:
- continue;
- }
- if (reader->mr_tid.weak == CurrentTid ||
- reader->mr_tid.weak == WriteTxnOwner)
- goto skip_lck;
-
- rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid.weak);
- if (rc != MDBX_SUCCESS) {
- bailout_lck:
- (void)mdbx_resume_threads_after_remap(*array);
- return rc;
- }
- }
- if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
- rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner);
- if (rc != MDBX_SUCCESS)
- goto bailout_lck;
- }
- } else {
- /* Without LCK (i.e. read-only mode).
- * Walk through a snapshot of all running threads */
- mdbx_assert(env, env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY));
- const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
- if (hSnapshot == INVALID_HANDLE_VALUE)
- return (int)GetLastError();
-
- THREADENTRY32 entry;
- entry.dwSize = sizeof(THREADENTRY32);
-
- if (!Thread32First(hSnapshot, &entry)) {
- rc = (int)GetLastError();
- bailout_toolhelp:
- CloseHandle(hSnapshot);
- (void)mdbx_resume_threads_after_remap(*array);
- return rc;
- }
-
- do {
- if (entry.th32OwnerProcessID != env->me_pid ||
- entry.th32ThreadID == CurrentTid)
- continue;
-
- rc = suspend_and_append(array, entry.th32ThreadID);
- if (rc != MDBX_SUCCESS)
- goto bailout_toolhelp;
-
- } while (Thread32Next(hSnapshot, &entry));
-
- rc = (int)GetLastError();
- if (rc != ERROR_NO_MORE_FILES)
- goto bailout_toolhelp;
- CloseHandle(hSnapshot);
- }
-
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int
-mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
- int rc = MDBX_SUCCESS;
- for (unsigned i = 0; i < array->count; ++i) {
- const HANDLE hThread = array->handles[i];
- if (ResumeThread(hThread) == (DWORD)-1) {
- const int err = (int)GetLastError();
- DWORD ExitCode;
- if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
- GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
- rc = err;
- }
- CloseHandle(hThread);
- }
- return rc;
-}
-
-/*----------------------------------------------------------------------------*/
-/* global `initial` lock for lockfile initialization,
- * exclusive/shared locking first cacheline */
-
-/* Briefly description of locking schema/algorithm:
- * - Windows does not support upgrading or downgrading for file locking.
- * - Therefore upgrading/downgrading is emulated by shared and exclusive
- * locking of upper and lower halves.
- * - In other words, we have FSM with possible 9 states,
- * i.e. free/shared/exclusive x free/shared/exclusive == 9.
- * Only 6 states of FSM are used, which 2 of ones are transitive.
- *
- * States:
- * ?-? = free, i.e. unlocked
- * S-? = used, i.e. shared lock
- * E-? = exclusive-read, i.e. operational exclusive
- * ?-S
- * ?-E = middle (transitive state)
- * S-S
- * S-E = locked (transitive state)
- * E-S
- * E-E = exclusive-write, i.e. exclusive due (re)initialization
- *
- * The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked
- * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible,
- * or to the "used" (and returns MDBX_RESULT_FALSE).
- *
- * The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write"
- * state to the "used" (i.e. shared) state.
- *
- * The mdbx_lck_upgrade() moves the locking-FSM from "used" (i.e. shared)
- * state to the "exclusive write" state.
- */
-
-static void lck_unlock(MDBX_env *env) {
- int err;
-
- if (env->me_lfd != INVALID_HANDLE_VALUE) {
- /* double `unlock` for robustly remove overlapped shared/exclusive locks */
- while (funlock(env->me_lfd, LCK_LOWER))
- ;
- err = (int)GetLastError();
- assert(err == ERROR_NOT_LOCKED ||
- (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
- (void)err;
- SetLastError(ERROR_SUCCESS);
-
- while (funlock(env->me_lfd, LCK_UPPER))
- ;
- err = (int)GetLastError();
- assert(err == ERROR_NOT_LOCKED ||
- (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
- (void)err;
- SetLastError(ERROR_SUCCESS);
- }
-
- if (env->me_lazy_fd != INVALID_HANDLE_VALUE) {
- /* explicitly unlock to avoid latency for other processes (windows kernel
- * releases such locks via deferred queues) */
- while (funlock(env->me_lazy_fd, LCK_BODY))
- ;
- err = (int)GetLastError();
- assert(err == ERROR_NOT_LOCKED ||
- (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
- (void)err;
- SetLastError(ERROR_SUCCESS);
-
- while (funlock(env->me_lazy_fd, LCK_WHOLE))
- ;
- err = (int)GetLastError();
- assert(err == ERROR_NOT_LOCKED ||
- (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
- (void)err;
- SetLastError(ERROR_SUCCESS);
- }
-}
-
-/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
- * or as 'used' (S-? and returns MDBX_RESULT_FALSE).
- * Otherwise returns an error. */
-static int internal_seize_lck(HANDLE lfd) {
- int rc;
- assert(lfd != INVALID_HANDLE_VALUE);
-
- /* 1) now on ?-? (free), get ?-E (middle) */
- mdbx_jitter4testing(false);
- if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
- rc = (int)GetLastError() /* 2) something went wrong, give up */;
- mdbx_error("%s, err %u", "?-?(free) >> ?-E(middle)", rc);
- return rc;
- }
-
- /* 3) now on ?-E (middle), try E-E (exclusive-write) */
- mdbx_jitter4testing(false);
- if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
- return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
-
- /* 5) still on ?-E (middle) */
- rc = (int)GetLastError();
- mdbx_jitter4testing(false);
- if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
- /* 6) something went wrong, give up */
- if (!funlock(lfd, LCK_UPPER))
- mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)",
- (int)GetLastError());
- return rc;
- }
-
- /* 7) still on ?-E (middle), try S-E (locked) */
- mdbx_jitter4testing(false);
- rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
- : (int)GetLastError();
-
- mdbx_jitter4testing(false);
- if (rc != MDBX_RESULT_FALSE)
- mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc);
-
- /* 8) now on S-E (locked) or still on ?-E (middle),
- * transition to S-? (used) or ?-? (free) */
- if (!funlock(lfd, LCK_UPPER))
- mdbx_panic("%s(%s) failed: err %u", __func__,
- "X-E(locked/middle) >> X-?(used/free)", (int)GetLastError());
-
- /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
- return rc;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
- int rc;
-
- assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
- if (env->me_flags & MDBX_EXCLUSIVE)
- return MDBX_RESULT_TRUE /* nope since files were must be opened
- non-shareable */
- ;
-
- if (env->me_lfd == INVALID_HANDLE_VALUE) {
- /* LY: without-lck mode (e.g. on read-only filesystem) */
- mdbx_jitter4testing(false);
- if (!flock(env->me_lazy_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
- rc = (int)GetLastError();
- mdbx_error("%s, err %u", "without-lck", rc);
- return rc;
- }
- return MDBX_RESULT_FALSE;
- }
-
- rc = internal_seize_lck(env->me_lfd);
- mdbx_jitter4testing(false);
- if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
- /* Check that another process don't operates in without-lck mode.
- * Doing such check by exclusive locking the body-part of db. Should be
- * noted:
- * - we need an exclusive lock for do so;
- * - we can't lock meta-pages, otherwise other process could get an error
- * while opening db in valid (non-conflict) mode. */
- if (!flock(env->me_lazy_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
- rc = (int)GetLastError();
- mdbx_error("%s, err %u", "lock-against-without-lck", rc);
- mdbx_jitter4testing(false);
- lck_unlock(env);
- } else {
- mdbx_jitter4testing(false);
- if (!funlock(env->me_lazy_fd, LCK_BODY))
- mdbx_panic("%s(%s) failed: err %u", __func__,
- "unlock-against-without-lck", (int)GetLastError());
- }
- }
-
- return rc;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
- /* Transite from exclusive-write state (E-E) to used (S-?) */
- assert(env->me_lazy_fd != INVALID_HANDLE_VALUE);
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
-
- if (env->me_flags & MDBX_EXCLUSIVE)
- return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
- ;
- /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */
- if (!funlock(env->me_lfd, LCK_LOWER))
- mdbx_panic("%s(%s) failed: err %u", __func__,
- "E-E(exclusive-write) >> ?-E(middle)", (int)GetLastError());
-
- /* 2) now at ?-E (middle), transition to S-E (locked) */
- if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
- int rc = (int)GetLastError() /* 3) something went wrong, give up */;
- mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc);
- return rc;
- }
-
- /* 4) got S-E (locked), continue transition to S-? (used) */
- if (!funlock(env->me_lfd, LCK_UPPER))
- mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)",
- (int)GetLastError());
-
- return MDBX_SUCCESS /* 5) now at S-? (used), done */;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_upgrade(MDBX_env *env) {
- /* Transite from used state (S-?) to exclusive-write (E-E) */
- assert(env->me_lfd != INVALID_HANDLE_VALUE);
-
- if (env->me_flags & MDBX_EXCLUSIVE)
- return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
- ;
-
- int rc;
- /* 1) now on S-? (used), try S-E (locked) */
- mdbx_jitter4testing(false);
- if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_UPPER)) {
- rc = (int)GetLastError() /* 2) something went wrong, give up */;
- mdbx_verbose("%s, err %u", "S-?(used) >> S-E(locked)", rc);
- return rc;
- }
-
- /* 3) now on S-E (locked), transition to ?-E (middle) */
- if (!funlock(env->me_lfd, LCK_LOWER))
- mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)",
- (int)GetLastError());
-
- /* 4) now on ?-E (middle), try E-E (exclusive-write) */
- mdbx_jitter4testing(false);
- if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) {
- rc = (int)GetLastError() /* 5) something went wrong, give up */;
- mdbx_verbose("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc);
- return rc;
- }
-
- return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
- MDBX_env *inprocess_neighbor,
- int global_uniqueness_flag) {
- (void)env;
- (void)inprocess_neighbor;
- (void)global_uniqueness_flag;
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
- MDBX_env *inprocess_neighbor) {
- /* LY: should unmap before releasing the locks to avoid race condition and
- * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
- if (env->me_map)
- mdbx_munmap(&env->me_dxb_mmap);
- if (env->me_lck_mmap.lck) {
- const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages.weak == 0;
- mdbx_munmap(&env->me_lck_mmap);
- if (synced && !inprocess_neighbor && env->me_lfd != INVALID_HANDLE_VALUE &&
- mdbx_lck_upgrade(env) == MDBX_SUCCESS)
- /* this will fail if LCK is used/mmapped by other process(es) */
- mdbx_ftruncate(env->me_lfd, 0);
- }
- lck_unlock(env);
- return MDBX_SUCCESS;
-}
-
-/*----------------------------------------------------------------------------*/
-/* reader checking (by pid) */
-
-MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
- (void)env;
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
- (void)env;
- return MDBX_SUCCESS;
-}
-
-/* Checks reader by pid.
- *
- * Returns:
- * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
- * MDBX_RESULT_FALSE, if pid is dead (lock acquired)
- * or otherwise the errcode. */
-MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
- (void)env;
- HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
- int rc;
- if (likely(hProcess)) {
- rc = WaitForSingleObject(hProcess, 0);
- if (unlikely(rc == (int)WAIT_FAILED))
- rc = (int)GetLastError();
- CloseHandle(hProcess);
- } else {
- rc = (int)GetLastError();
- }
-
- switch (rc) {
- case ERROR_INVALID_PARAMETER:
- /* pid seems invalid */
- return MDBX_RESULT_FALSE;
- case WAIT_OBJECT_0:
- /* process just exited */
- return MDBX_RESULT_FALSE;
- case ERROR_ACCESS_DENIED:
- /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc.
- * assume pid exists */
- return MDBX_RESULT_TRUE;
- case WAIT_TIMEOUT:
- /* pid running */
- return MDBX_RESULT_TRUE;
- default:
- /* failure */
- return rc;
- }
-}
-
-//----------------------------------------------------------------------------
-// Stub for slim read-write lock
-// Copyright (C) 1995-2002 Brad Wilson
-
-static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) {
- srwl->readerCount = srwl->writerCount = 0;
-}
-
-static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) {
- while (true) {
- assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
-
- // If there's a writer already, spin without unnecessarily
- // interlocking the CPUs
- if (srwl->writerCount != 0) {
- YieldProcessor();
- continue;
- }
-
- // Add to the readers list
- _InterlockedIncrement(&srwl->readerCount);
-
- // Check for writers again (we may have been preempted). If
- // there are no writers writing or waiting, then we're done.
- if (srwl->writerCount == 0)
- break;
-
- // Remove from the readers list, spin, try again
- _InterlockedDecrement(&srwl->readerCount);
- YieldProcessor();
- }
-}
-
-static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) {
- assert(srwl->readerCount > 0);
- _InterlockedDecrement(&srwl->readerCount);
-}
-
-static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) {
- while (true) {
- assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
-
- // If there's a writer already, spin without unnecessarily
- // interlocking the CPUs
- if (srwl->writerCount != 0) {
- YieldProcessor();
- continue;
- }
-
- // See if we can become the writer (expensive, because it inter-
- // locks the CPUs, so writing should be an infrequent process)
- if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
- break;
- }
-
- // Now we're the writer, but there may be outstanding readers.
- // Spin until there aren't any more; new readers will wait now
- // that we're the writer.
- while (srwl->readerCount != 0) {
- assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
- YieldProcessor();
- }
-}
-
-static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) {
- assert(srwl->writerCount == 1 && srwl->readerCount >= 0);
- srwl->writerCount = 0;
-}
-
-static uint64_t WINAPI stub_GetTickCount64(void) {
- LARGE_INTEGER Counter, Frequency;
- return (QueryPerformanceFrequency(&Frequency) &&
- QueryPerformanceCounter(&Counter))
- ? Counter.QuadPart * 1000ul / Frequency.QuadPart
- : 0;
-}
-
-/*----------------------------------------------------------------------------*/
-
-#ifndef xMDBX_ALLOY
-MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
- mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
- mdbx_srwlock_ReleaseExclusive;
-
-MDBX_NtExtendSection mdbx_NtExtendSection;
-MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
-MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
-MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
-MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
-MDBX_NtFsControlFile mdbx_NtFsControlFile;
-MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
-MDBX_GetTickCount64 mdbx_GetTickCount64;
-MDBX_RegGetValueA mdbx_RegGetValueA;
-#endif /* xMDBX_ALLOY */
-
-#if __GNUC_PREREQ(8, 0)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wcast-function-type"
-#endif /* GCC/MINGW */
-
-static void mdbx_winnt_import(void) {
- const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
-
-#define GET_PROC_ADDR(dll, ENTRY) \
- mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(dll, #ENTRY)
-
- if (GetProcAddress(hNtdll, "wine_get_version")) {
- assert(mdbx_RunningUnderWine());
- } else {
- GET_PROC_ADDR(hNtdll, NtFsControlFile);
- GET_PROC_ADDR(hNtdll, NtExtendSection);
- assert(!mdbx_RunningUnderWine());
- }
-
- const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
- GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx);
- GET_PROC_ADDR(hKernel32dll, GetTickCount64);
- if (!mdbx_GetTickCount64)
- mdbx_GetTickCount64 = stub_GetTickCount64;
- if (!mdbx_RunningUnderWine()) {
- GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle);
- GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW);
- GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW);
- GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory);
- }
-
- const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll");
- GET_PROC_ADDR(hAdvapi32dll, RegGetValueA);
-#undef GET_PROC_ADDR
-
- const MDBX_srwlock_function init =
- (MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock");
- if (init != NULL) {
- mdbx_srwlock_Init = init;
- mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress(
- hKernel32dll, "AcquireSRWLockShared");
- mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress(
- hKernel32dll, "ReleaseSRWLockShared");
- mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress(
- hKernel32dll, "AcquireSRWLockExclusive");
- mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress(
- hKernel32dll, "ReleaseSRWLockExclusive");
- } else {
- mdbx_srwlock_Init = stub_srwlock_Init;
- mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared;
- mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared;
- mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive;
- mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
- }
-}
-
-#if __GNUC_PREREQ(8, 0)
-#pragma GCC diagnostic pop
-#endif /* GCC/MINGW */
-
-#endif /* Windows LCK-implementation */
diff --git a/3rdparty/libmdbx/src/mdbx.c++ b/3rdparty/libmdbx/src/mdbx.c++
deleted file mode 100644
index 373f5282a71c8db861305edd5d7f1179ad6252fd..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx.c++
+++ /dev/null
@@ -1,1771 +0,0 @@
-//
-// Copyright (c) 2020-2022, Leonid Yuriev <leo@yuriev.ru>.
-// SPDX-License-Identifier: Apache-2.0
-//
-// Non-inline part of the libmdbx C++ API
-//
-
-#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS)
-#define _CRT_SECURE_NO_WARNINGS
-#endif /* _CRT_SECURE_NO_WARNINGS */
-
-#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
- !defined(__USE_MINGW_ANSI_STDIO)
-#define __USE_MINGW_ANSI_STDIO 1
-#endif /* __USE_MINGW_ANSI_STDIO */
-
-#include "../mdbx.h++"
-
-#include "defs.h"
-#include "internals.h"
-
-#include <array>
-#include <atomic>
-#include <cctype> // for isxdigit(), etc
-#include <system_error>
-
-namespace {
-
-#if 0 /* Unused for now */
-
-class trouble_location {
-
-#ifndef TROUBLE_PROVIDE_LINENO
-#define TROUBLE_PROVIDE_LINENO 1
-#endif
-
-#ifndef TROUBLE_PROVIDE_CONDITION
-#define TROUBLE_PROVIDE_CONDITION 1
-#endif
-
-#ifndef TROUBLE_PROVIDE_FUNCTION
-#define TROUBLE_PROVIDE_FUNCTION 1
-#endif
-
-#ifndef TROUBLE_PROVIDE_FILENAME
-#define TROUBLE_PROVIDE_FILENAME 1
-#endif
-
-#if TROUBLE_PROVIDE_LINENO
- const unsigned line_;
-#endif
-#if TROUBLE_PROVIDE_CONDITION
- const char *const condition_;
-#endif
-#if TROUBLE_PROVIDE_FUNCTION
- const char *const function_;
-#endif
-#if TROUBLE_PROVIDE_FILENAME
- const char *const filename_;
-#endif
-
-public:
- MDBX_CXX11_CONSTEXPR trouble_location(unsigned line, const char *condition,
- const char *function, const char *filename)
- :
-#if TROUBLE_PROVIDE_LINENO
- line_(line)
-#endif
-#if TROUBLE_PROVIDE_CONDITION
- ,
- condition_(condition)
-#endif
-#if TROUBLE_PROVIDE_FUNCTION
- ,
- function_(function)
-#endif
-#if TROUBLE_PROVIDE_FILENAME
- ,
- filename_(filename)
-#endif
- {
-#if !TROUBLE_PROVIDE_LINENO
- (void)line;
-#endif
-#if !TROUBLE_PROVIDE_CONDITION
- (void)condition;
-#endif
-#if !TROUBLE_PROVIDE_FUNCTION
- (void)function;
-#endif
-#if !TROUBLE_PROVIDE_FILENAME
- (void)filename;
-#endif
- }
-
- trouble_location(const trouble_location &&) = delete;
-
- unsigned line() const {
-#if TROUBLE_PROVIDE_LINENO
- return line_;
-#else
- return 0;
-#endif
- }
-
- const char *condition() const {
-#if TROUBLE_PROVIDE_CONDITION
- return condition_;
-#else
- return "";
-#endif
- }
-
- const char *function() const {
-#if TROUBLE_PROVIDE_FUNCTION
- return function_;
-#else
- return "";
-#endif
- }
-
- const char *filename() const {
-#if TROUBLE_PROVIDE_FILENAME
- return filename_;
-#else
- return "";
-#endif
- }
-};
-
-//------------------------------------------------------------------------------
-
-__cold std::string format_va(const char *fmt, va_list ap) {
- va_list ones;
- va_copy(ones, ap);
-#ifdef _MSC_VER
- int needed = _vscprintf(fmt, ap);
-#else
- int needed = vsnprintf(nullptr, 0, fmt, ap);
-#endif
- assert(needed >= 0);
- std::string result;
- result.reserve(size_t(needed + 1));
- result.resize(size_t(needed), '\0');
- assert(int(result.capacity()) > needed);
- int actual = vsnprintf(const_cast<char *>(result.data()), result.capacity(),
- fmt, ones);
- assert(actual == needed);
- (void)actual;
- va_end(ones);
- return result;
-}
-
-__cold std::string format(const char *fmt, ...) {
- va_list ap;
- va_start(ap, fmt);
- std::string result = format_va(fmt, ap);
- va_end(ap);
- return result;
-}
-
-class bug : public std::runtime_error {
- const trouble_location &location_;
-
-public:
- bug(const trouble_location &) noexcept;
- /* temporary workaround for "private field 'FOO' is not used" from CLANG
- * and for "function 'BAR' was declared but never referenced" from LCC. */
-#ifndef __LCC__
- const trouble_location &location() const noexcept { return location_; }
-#endif
- virtual ~bug() noexcept;
-};
-
-__cold bug::bug(const trouble_location &location) noexcept
- : std::runtime_error(format("mdbx.bug: %s.%s at %s:%u", location.function(),
- location.condition(), location.filename(),
- location.line())),
- location_(location) {}
-
-__cold bug::~bug() noexcept {}
-
-[[noreturn]] __cold void raise_bug(const trouble_location &what_and_where) {
- throw bug(what_and_where);
-}
-
-#define RAISE_BUG(line, condition, function, file) \
- do { \
- static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \
- function, file); \
- raise_bug(bug); \
- } while (0)
-
-#define ENSURE(condition) \
- do \
- if (MDBX_UNLIKELY(!(condition))) \
- MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \
- while (0)
-
-#define NOT_IMPLEMENTED() \
- RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__);
-
-#endif /* Unused*/
-
-//------------------------------------------------------------------------------
-
-template <typename PATH> struct path_to_pchar {
- const std::string str;
- path_to_pchar(const PATH &path) : str(path.generic_string()) {}
- operator const char *() const { return str.c_str(); }
-};
-
-template <typename PATH>
-MDBX_MAYBE_UNUSED PATH pchar_to_path(const char *c_str) {
- return PATH(c_str);
-}
-
-template <> struct path_to_pchar<std::string> {
- const char *const ptr;
- path_to_pchar(const std::string &path) : ptr(path.c_str()) {}
- operator const char *() const { return ptr; }
-};
-
-#if defined(_WIN32) || defined(_WIN64)
-
-#ifndef WC_ERR_INVALID_CHARS
-static const DWORD WC_ERR_INVALID_CHARS =
- (6 /* Windows Vista */ <= /* MajorVersion */ LOBYTE(LOWORD(GetVersion())))
- ? 0x00000080
- : 0;
-#endif /* WC_ERR_INVALID_CHARS */
-
-template <> struct path_to_pchar<std::wstring> {
- std::string str;
- path_to_pchar(const std::wstring &path) {
- if (!path.empty()) {
- const int chars =
- WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(),
- int(path.size()), nullptr, 0, nullptr, nullptr);
- if (chars == 0)
- mdbx::error::throw_exception(GetLastError());
- str.append(chars, '\0');
- WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(),
- int(path.size()), const_cast<char *>(str.data()),
- chars, nullptr, nullptr);
- }
- }
- operator const char *() const { return str.c_str(); }
-};
-
-template <>
-MDBX_MAYBE_UNUSED std::wstring pchar_to_path<std::wstring>(const char *c_str) {
- std::wstring wstr;
- if (c_str && *c_str) {
- const int chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str,
- int(strlen(c_str)), nullptr, 0);
- if (chars == 0)
- mdbx::error::throw_exception(GetLastError());
- wstr.append(chars, '\0');
- MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str,
- int(strlen(c_str)), const_cast<wchar_t *>(wstr.data()),
- chars);
- }
- return wstr;
-}
-
-#endif /* Windows */
-
-} // namespace
-
-//------------------------------------------------------------------------------
-
-namespace mdbx {
-
-[[noreturn]] __cold void throw_max_length_exceeded() {
- throw std::length_error(
- "mdbx:: Exceeded the maximal length of data/slice/buffer.");
-}
-
-[[noreturn]] __cold void throw_too_small_target_buffer() {
- throw std::length_error("mdbx:: The target buffer is too small.");
-}
-
-[[noreturn]] __cold void throw_out_range() {
- throw std::out_of_range("mdbx:: Slice or buffer method was called with "
- "an argument that exceeds the length.");
-}
-
-[[noreturn]] __cold void throw_allocators_mismatch() {
- throw std::logic_error(
- "mdbx:: An allocators mismatch, so an object could not be transferred "
- "into an incompatible memory allocation scheme.");
-}
-
-__cold exception::exception(const ::mdbx::error &error) noexcept
- : base(error.what()), error_(error) {}
-
-__cold exception::~exception() noexcept {}
-
-static std::atomic_int fatal_countdown;
-
-__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) {
- ++fatal_countdown;
-}
-
-__cold fatal::~fatal() noexcept {
- if (--fatal_countdown == 0)
- std::terminate();
-}
-
-#define DEFINE_EXCEPTION(NAME) \
- __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \
- __cold NAME::~NAME() noexcept {}
-
-DEFINE_EXCEPTION(bad_map_id)
-DEFINE_EXCEPTION(bad_transaction)
-DEFINE_EXCEPTION(bad_value_size)
-DEFINE_EXCEPTION(db_corrupted)
-DEFINE_EXCEPTION(db_full)
-DEFINE_EXCEPTION(db_invalid)
-DEFINE_EXCEPTION(db_too_large)
-DEFINE_EXCEPTION(db_unable_extend)
-DEFINE_EXCEPTION(db_version_mismatch)
-DEFINE_EXCEPTION(db_wanna_write_for_recovery)
-DEFINE_EXCEPTION(incompatible_operation)
-DEFINE_EXCEPTION(internal_page_full)
-DEFINE_EXCEPTION(internal_problem)
-DEFINE_EXCEPTION(key_exists)
-DEFINE_EXCEPTION(key_mismatch)
-DEFINE_EXCEPTION(max_maps_reached)
-DEFINE_EXCEPTION(max_readers_reached)
-DEFINE_EXCEPTION(multivalue)
-DEFINE_EXCEPTION(no_data)
-DEFINE_EXCEPTION(not_found)
-DEFINE_EXCEPTION(operation_not_permitted)
-DEFINE_EXCEPTION(permission_denied_or_not_writeable)
-DEFINE_EXCEPTION(reader_slot_busy)
-DEFINE_EXCEPTION(remote_media)
-DEFINE_EXCEPTION(something_busy)
-DEFINE_EXCEPTION(thread_mismatch)
-DEFINE_EXCEPTION(transaction_full)
-DEFINE_EXCEPTION(transaction_overlapping)
-
-#undef DEFINE_EXCEPTION
-
-__cold const char *error::what() const noexcept {
- if (is_mdbx_error())
- return mdbx_liberr2str(code());
-
- switch (code()) {
-#define ERROR_CASE(CODE) \
- case CODE: \
- return MDBX_STRINGIFY(CODE)
- ERROR_CASE(MDBX_ENODATA);
- ERROR_CASE(MDBX_EINVAL);
- ERROR_CASE(MDBX_EACCESS);
- ERROR_CASE(MDBX_ENOMEM);
- ERROR_CASE(MDBX_EROFS);
- ERROR_CASE(MDBX_ENOSYS);
- ERROR_CASE(MDBX_EIO);
- ERROR_CASE(MDBX_EPERM);
- ERROR_CASE(MDBX_EINTR);
- ERROR_CASE(MDBX_ENOFILE);
- ERROR_CASE(MDBX_EREMOTE);
-#undef ERROR_CASE
- default:
- return "SYSTEM";
- }
-}
-
-__cold std::string error::message() const {
- char buf[1024];
- const char *msg = ::mdbx_strerror_r(code(), buf, sizeof(buf));
- return std::string(msg ? msg : "unknown");
-}
-
-[[noreturn]] __cold void error::panic(const char *context,
- const char *func) const noexcept {
- assert(code() != MDBX_SUCCESS);
- ::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code());
- std::terminate();
-}
-
-__cold void error::throw_exception() const {
- switch (code()) {
- case MDBX_EINVAL:
- throw std::invalid_argument("mdbx");
- case MDBX_ENOMEM:
- throw std::bad_alloc();
- case MDBX_SUCCESS:
- static_assert(MDBX_SUCCESS == MDBX_RESULT_FALSE, "WTF?");
- throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)");
- case MDBX_RESULT_TRUE:
- throw std::logic_error("MDBX_RESULT_TRUE");
-#define CASE_EXCEPTION(NAME, CODE) \
- case CODE: \
- throw NAME(code())
- CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI);
- CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN);
- CASE_EXCEPTION(bad_value_size, MDBX_BAD_VALSIZE);
- CASE_EXCEPTION(db_corrupted, MDBX_CORRUPTED);
- CASE_EXCEPTION(db_corrupted, MDBX_CURSOR_FULL); /* branch-pages loop */
- CASE_EXCEPTION(db_corrupted, MDBX_PAGE_NOTFOUND);
- CASE_EXCEPTION(db_full, MDBX_MAP_FULL);
- CASE_EXCEPTION(db_invalid, MDBX_INVALID);
- CASE_EXCEPTION(db_too_large, MDBX_TOO_LARGE);
- CASE_EXCEPTION(db_unable_extend, MDBX_UNABLE_EXTEND_MAPSIZE);
- CASE_EXCEPTION(db_version_mismatch, MDBX_VERSION_MISMATCH);
- CASE_EXCEPTION(db_wanna_write_for_recovery, MDBX_WANNA_RECOVERY);
- CASE_EXCEPTION(fatal, MDBX_EBADSIGN);
- CASE_EXCEPTION(fatal, MDBX_PANIC);
- CASE_EXCEPTION(incompatible_operation, MDBX_INCOMPATIBLE);
- CASE_EXCEPTION(internal_page_full, MDBX_PAGE_FULL);
- CASE_EXCEPTION(internal_problem, MDBX_PROBLEM);
- CASE_EXCEPTION(key_mismatch, MDBX_EKEYMISMATCH);
- CASE_EXCEPTION(max_maps_reached, MDBX_DBS_FULL);
- CASE_EXCEPTION(max_readers_reached, MDBX_READERS_FULL);
- CASE_EXCEPTION(multivalue, MDBX_EMULTIVAL);
- CASE_EXCEPTION(no_data, MDBX_ENODATA);
- CASE_EXCEPTION(not_found, MDBX_NOTFOUND);
- CASE_EXCEPTION(operation_not_permitted, MDBX_EPERM);
- CASE_EXCEPTION(permission_denied_or_not_writeable, MDBX_EACCESS);
- CASE_EXCEPTION(reader_slot_busy, MDBX_BAD_RSLOT);
- CASE_EXCEPTION(remote_media, MDBX_EREMOTE);
- CASE_EXCEPTION(something_busy, MDBX_BUSY);
- CASE_EXCEPTION(thread_mismatch, MDBX_THREAD_MISMATCH);
- CASE_EXCEPTION(transaction_full, MDBX_TXN_FULL);
- CASE_EXCEPTION(transaction_overlapping, MDBX_TXN_OVERLAPPING);
-#undef CASE_EXCEPTION
- default:
- if (is_mdbx_error())
- throw exception(*this);
- throw std::system_error(std::error_code(code(), std::system_category()));
- }
-}
-
-//------------------------------------------------------------------------------
-
-bool slice::is_printable(bool disable_utf8) const noexcept {
- enum : byte {
- LS = 4, // shift for UTF8 sequence length
- P_ = 1 << LS, // printable ASCII flag
- N_ = 0, // non-printable ASCII
- second_range_mask = P_ - 1, // mask for range flag
- r80_BF = 0, // flag for UTF8 2nd byte range
- rA0_BF = 1, // flag for UTF8 2nd byte range
- r80_9F = 2, // flag for UTF8 2nd byte range
- r90_BF = 3, // flag for UTF8 2nd byte range
- r80_8F = 4, // flag for UTF8 2nd byte range
-
- // valid utf-8 byte sequences
- // http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
- // Code | Bytes | | |
- // Points | 1st | 2nd | 3rd |4th
- // --------------------|--------|--------|--------|---
- C2 = 2 << LS | r80_BF, // U+000080..U+0007FF | C2..DF | 80..BF | |
- E0 = 3 << LS | rA0_BF, // U+000800..U+000FFF | E0 | A0..BF | 80..BF |
- E1 = 3 << LS | r80_BF, // U+001000..U+00CFFF | E1..EC | 80..BF | 80..BF |
- ED = 3 << LS | r80_9F, // U+00D000..U+00D7FF | ED | 80..9F | 80..BF |
- EE = 3 << LS | r80_BF, // U+00E000..U+00FFFF | EE..EF | 80..BF | 80..BF |
- F0 = 4 << LS | r90_BF, // U+010000..U+03FFFF | F0 | 90..BF | 80..BF |...
- F1 = 4 << LS | r80_BF, // U+040000..U+0FFFFF | F1..F3 | 80..BF | 80..BF |...
- F4 = 4 << LS | r80_BF, // U+100000..U+10FFFF | F4 | 80..8F | 80..BF |...
- };
-
- static const byte range_from[] = {0x80, 0xA0, 0x80, 0x90, 0x80};
- static const byte range_to[] = {0xBF, 0xBF, 0x9F, 0xBF, 0x8F};
-
- static const byte map[256] = {
- // 1 2 3 4 5 6 7 8 9 a b c d e f
- N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 00
- N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 10
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 20
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 30
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 40
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 50
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 60
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, // 70
- N_, N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, N_, // 80
- N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, P_, // 90
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // a0
- P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // b0
- P_, P_, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // c0
- C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // df
- E0, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, ED, EE, EE, // e0
- F0, F1, F1, F1, F4, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_ // f0
- };
-
- if (MDBX_UNLIKELY(length() < 1))
- MDBX_CXX20_UNLIKELY return false;
-
- auto src = byte_ptr();
- const auto end = src + length();
- if (MDBX_UNLIKELY(disable_utf8)) {
- do
- if (MDBX_UNLIKELY((P_ & map[*src]) == 0))
- MDBX_CXX20_UNLIKELY return false;
- while (++src < end);
- return true;
- }
-
- do {
- const auto bits = map[*src];
- const auto second_from = range_from[bits & second_range_mask];
- const auto second_to = range_to[bits & second_range_mask];
- switch (bits >> LS) {
- default:
- MDBX_CXX20_UNLIKELY return false;
- case 1:
- src += 1;
- continue;
- case 2:
- if (MDBX_UNLIKELY(src + 1 >= end))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
- MDBX_CXX20_UNLIKELY return false;
- src += 2;
- continue;
- case 3:
- if (MDBX_UNLIKELY(src + 3 >= end))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[2] < 0x80 || src[2] > 0xBF))
- MDBX_CXX20_UNLIKELY return false;
- src += 3;
- continue;
- case 4:
- if (MDBX_UNLIKELY(src + 4 >= end))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[1] < second_from || src[1] > second_to))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[2] < 0x80 || src[2] > 0xBF))
- MDBX_CXX20_UNLIKELY return false;
- if (MDBX_UNLIKELY(src[3] < 0x80 || src[3] > 0xBF))
- MDBX_CXX20_UNLIKELY return false;
- src += 4;
- continue;
- }
- } while (src < end);
-
- return true;
-}
-
-//------------------------------------------------------------------------------
-
-char *to_hex::write_bytes(char *__restrict const dest, size_t dest_size) const {
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- const char alphabase = (uppercase ? 'A' : 'a') - 10;
- auto line = ptr;
- for (const auto end = source.end_byte_ptr(); src != end; ++src) {
- if (wrap_width && size_t(ptr - line) >= wrap_width) {
- *ptr = '\n';
- line = ++ptr;
- }
- const int8_t hi = *src >> 4;
- const int8_t lo = *src & 15;
- ptr[0] = char(alphabase + hi + (((hi - 10) >> 7) & -7));
- ptr[1] = char(alphabase + lo + (((lo - 10) >> 7) & -7));
- ptr += 2;
- assert(ptr <= dest + dest_size);
- }
- return ptr;
-}
-
-::std::ostream &to_hex::output(::std::ostream &out) const {
- if (MDBX_LIKELY(!is_empty()))
- MDBX_CXX20_LIKELY {
- ::std::ostream::sentry sentry(out);
- auto src = source.byte_ptr();
- const char alphabase = (uppercase ? 'A' : 'a') - 10;
- unsigned width = 0;
- for (const auto end = source.end_byte_ptr(); src != end; ++src) {
- if (wrap_width && width >= wrap_width) {
- out << ::std::endl;
- width = 0;
- }
- const int8_t hi = *src >> 4;
- const int8_t lo = *src & 15;
- out.put(char(alphabase + hi + (((hi - 10) >> 7) & -7)));
- out.put(char(alphabase + lo + (((lo - 10) >> 7) & -7)));
- width += 2;
- }
- }
- return out;
-}
-
-char *from_hex::write_bytes(char *__restrict const dest,
- size_t dest_size) const {
- if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces))
- MDBX_CXX20_UNLIKELY throw std::domain_error(
- "mdbx::from_hex:: odd length of hexadecimal string");
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(*src <= ' ') &&
- MDBX_LIKELY(ignore_spaces && isspace(*src))) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1])))
- MDBX_CXX20_UNLIKELY throw std::domain_error(
- "mdbx::from_hex:: invalid hexadecimal string");
-
- int8_t hi = src[0];
- hi = (hi | 0x20) - 'a';
- hi += 10 + ((hi >> 7) & 7);
-
- int8_t lo = src[1];
- lo = (lo | 0x20) - 'a';
- lo += 10 + ((lo >> 7) & 7);
-
- *ptr++ = hi << 4 | lo;
- src += 2;
- left -= 2;
- assert(ptr <= dest + dest_size);
- }
- return ptr;
-}
-
-bool from_hex::is_erroneous() const noexcept {
- if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces))
- MDBX_CXX20_UNLIKELY return true;
-
- bool got = false;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(*src <= ' ') &&
- MDBX_LIKELY(ignore_spaces && isspace(*src))) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1])))
- MDBX_CXX20_UNLIKELY return true;
-
- got = true;
- src += 2;
- left -= 2;
- }
- return !got;
-}
-
-//------------------------------------------------------------------------------
-
-enum : signed char {
- OO /* ASCII NUL */ = -8,
- EQ /* BASE64 '=' pad */ = -4,
- SP /* SPACE */ = -2,
- IL /* invalid */ = -1
-};
-
-static const byte b58_alphabet[58] = {
- '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
- 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
- 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
-
-#ifndef bswap64
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-static inline uint64_t bswap64(uint64_t v) noexcept {
-#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \
- __has_builtin(__builtin_bswap64)
- return __builtin_bswap64(v);
-#elif defined(_MSC_VER) && !defined(__clang__)
- return _byteswap_uint64(v);
-#elif defined(__bswap_64)
- return __bswap_64(v);
-#elif defined(bswap_64)
- return bswap_64(v);
-#else
- return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
- ((v << 24) & UINT64_C(0x0000ff0000000000)) |
- ((v << 8) & UINT64_C(0x000000ff00000000)) |
- ((v >> 8) & UINT64_C(0x00000000ff000000)) |
- ((v >> 24) & UINT64_C(0x0000000000ff0000)) |
- ((v >> 40) & UINT64_C(0x000000000000ff00));
-#endif
-}
-#endif /* __BYTE_ORDER__ */
-#endif /* ifndef bswap64 */
-
-static inline char b58_8to11(uint64_t &v) noexcept {
- const unsigned i = unsigned(v % 58);
- v /= 58;
- return b58_alphabet[i];
-}
-
-char *to_base58::write_bytes(char *__restrict const dest,
- size_t dest_size) const {
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- size_t left = source.length();
- auto line = ptr;
- while (MDBX_LIKELY(left > 7)) {
- uint64_t v;
- std::memcpy(&v, src, 8);
- src += 8;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- v = bswap64(v);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#else
-#error "FIXME: Unsupported byte order"
-#endif /* __BYTE_ORDER__ */
- ptr[10] = b58_8to11(v);
- ptr[9] = b58_8to11(v);
- ptr[8] = b58_8to11(v);
- ptr[7] = b58_8to11(v);
- ptr[6] = b58_8to11(v);
- ptr[5] = b58_8to11(v);
- ptr[4] = b58_8to11(v);
- ptr[3] = b58_8to11(v);
- ptr[2] = b58_8to11(v);
- ptr[1] = b58_8to11(v);
- ptr[0] = b58_8to11(v);
- assert(v == 0);
- ptr += 11;
- left -= 8;
- if (wrap_width && size_t(ptr - line) >= wrap_width && left) {
- *ptr = '\n';
- line = ++ptr;
- }
- assert(ptr <= dest + dest_size);
- }
-
- if (left) {
- uint64_t v = 0;
- unsigned parrots = 31;
- do {
- v = (v << 8) + *src++;
- parrots += 43;
- } while (--left);
-
- auto tail = ptr += parrots >> 5;
- assert(ptr <= dest + dest_size);
- do {
- *--tail = b58_8to11(v);
- parrots -= 32;
- } while (parrots > 31);
- assert(v == 0);
- }
-
- return ptr;
-}
-
-::std::ostream &to_base58::output(::std::ostream &out) const {
- if (MDBX_LIKELY(!is_empty()))
- MDBX_CXX20_LIKELY {
- ::std::ostream::sentry sentry(out);
- auto src = source.byte_ptr();
- size_t left = source.length();
- unsigned width = 0;
- std::array<char, 11> buf;
-
- while (MDBX_LIKELY(left > 7)) {
- uint64_t v;
- std::memcpy(&v, src, 8);
- src += 8;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- v = bswap64(v);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#else
-#error "FIXME: Unsupported byte order"
-#endif /* __BYTE_ORDER__ */
- buf[10] = b58_8to11(v);
- buf[9] = b58_8to11(v);
- buf[8] = b58_8to11(v);
- buf[7] = b58_8to11(v);
- buf[6] = b58_8to11(v);
- buf[5] = b58_8to11(v);
- buf[4] = b58_8to11(v);
- buf[3] = b58_8to11(v);
- buf[2] = b58_8to11(v);
- buf[1] = b58_8to11(v);
- buf[0] = b58_8to11(v);
- assert(v == 0);
- out.write(&buf.front(), 11);
- left -= 8;
- if (wrap_width && (width += 11) >= wrap_width && left) {
- out << ::std::endl;
- width = 0;
- }
- }
-
- if (left) {
- uint64_t v = 0;
- unsigned parrots = 31;
- do {
- v = (v << 8) + *src++;
- parrots += 43;
- } while (--left);
-
- auto ptr = buf.end();
- do {
- *--ptr = b58_8to11(v);
- parrots -= 32;
- } while (parrots > 31);
- assert(v == 0);
- out.write(&*ptr, buf.end() - ptr);
- }
- }
- return out;
-}
-
-const signed char b58_map[256] = {
- // 1 2 3 4 5 6 7 8 9 a b c d e f
- OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10
- SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 20
- IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, IL, IL, IL, IL, IL, IL, // 30
- IL, 9, 10, 11, 12, 13, 14, 15, 16, IL, 17, 18, 19, 20, 21, IL, // 40
- 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, IL, IL, IL, IL, IL, // 50
- IL, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, IL, 44, 45, 46, // 60
- 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, IL, IL, IL, IL, IL, // 70
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
-};
-
-static inline signed char b58_11to8(uint64_t &v, const byte c) noexcept {
- const signed char m = b58_map[c];
- v = v * 58 + m;
- return m;
-}
-
-char *from_base58::write_bytes(char *__restrict const dest,
- size_t dest_size) const {
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(isspace(*src)) && ignore_spaces) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_LIKELY(left > 10)) {
- uint64_t v = 0;
- if (MDBX_UNLIKELY((b58_11to8(v, src[0]) | b58_11to8(v, src[1]) |
- b58_11to8(v, src[2]) | b58_11to8(v, src[3]) |
- b58_11to8(v, src[4]) | b58_11to8(v, src[5]) |
- b58_11to8(v, src[6]) | b58_11to8(v, src[7]) |
- b58_11to8(v, src[8]) | b58_11to8(v, src[9]) |
- b58_11to8(v, src[10])) < 0))
- MDBX_CXX20_UNLIKELY goto bailout;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- v = bswap64(v);
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#else
-#error "FIXME: Unsupported byte order"
-#endif /* __BYTE_ORDER__ */
- std::memcpy(ptr, &v, 8);
- ptr += 8;
- src += 11;
- left -= 11;
- assert(ptr <= dest + dest_size);
- continue;
- }
-
- constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8;
- if (MDBX_UNLIKELY(invalid_length_mask & (1 << left)))
- MDBX_CXX20_UNLIKELY goto bailout;
-
- uint64_t v = 1;
- unsigned parrots = 0;
- do {
- if (MDBX_UNLIKELY(b58_11to8(v, *src++) < 0))
- MDBX_CXX20_UNLIKELY goto bailout;
- parrots += 32;
- } while (--left);
-
- auto tail = ptr += parrots / 43;
- assert(ptr <= dest + dest_size);
- do {
- *--tail = byte(v);
- v >>= 8;
- } while (v > 255);
- break;
- }
- return ptr;
-
-bailout:
- throw std::domain_error("mdbx::from_base58:: invalid base58 string");
-}
-
-bool from_base58::is_erroneous() const noexcept {
- bool got = false;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(*src <= ' ') &&
- MDBX_LIKELY(ignore_spaces && isspace(*src))) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_LIKELY(left > 10)) {
- if (MDBX_UNLIKELY((b58_map[src[0]] | b58_map[src[1]] | b58_map[src[2]] |
- b58_map[src[3]] | b58_map[src[4]] | b58_map[src[5]] |
- b58_map[src[6]] | b58_map[src[7]] | b58_map[src[8]] |
- b58_map[src[9]] | b58_map[src[10]]) < 0))
- MDBX_CXX20_UNLIKELY return true;
- src += 11;
- left -= 11;
- got = true;
- continue;
- }
-
- constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8;
- if (invalid_length_mask & (1 << left))
- return false;
-
- do
- if (MDBX_UNLIKELY(b58_map[*src++] < 0))
- MDBX_CXX20_UNLIKELY return true;
- while (--left);
- got = true;
- break;
- }
- return !got;
-}
-
-//------------------------------------------------------------------------------
-
-static inline void b64_3to4(const byte x, const byte y, const byte z,
- char *__restrict dest) noexcept {
- static const byte alphabet[64] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
- dest[0] = alphabet[(x & 0xfc) >> 2];
- dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)];
- dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)];
- dest[3] = alphabet[z & 0x3f];
-}
-
-char *to_base64::write_bytes(char *__restrict const dest,
- size_t dest_size) const {
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- size_t left = source.length();
- auto line = ptr;
- while (true) {
- switch (left) {
- default:
- MDBX_CXX20_LIKELY left -= 3;
- b64_3to4(src[0], src[1], src[2], ptr);
- ptr += 4;
- src += 3;
- if (wrap_width && size_t(ptr - line) >= wrap_width && left) {
- *ptr = '\n';
- line = ++ptr;
- }
- assert(ptr <= dest + dest_size);
- continue;
- case 2:
- b64_3to4(src[0], src[1], 0, ptr);
- ptr[3] = '=';
- assert(ptr + 4 <= dest + dest_size);
- return ptr + 4;
- case 1:
- b64_3to4(src[0], 0, 0, ptr);
- ptr[2] = ptr[3] = '=';
- assert(ptr + 4 <= dest + dest_size);
- return ptr + 4;
- case 0:
- return ptr;
- }
- }
-}
-
-::std::ostream &to_base64::output(::std::ostream &out) const {
- if (MDBX_LIKELY(!is_empty()))
- MDBX_CXX20_LIKELY {
- ::std::ostream::sentry sentry(out);
- auto src = source.byte_ptr();
- size_t left = source.length();
- unsigned width = 0;
- std::array<char, 4> buf;
-
- while (true) {
- switch (left) {
- default:
- MDBX_CXX20_LIKELY left -= 3;
- b64_3to4(src[0], src[1], src[2], &buf.front());
- src += 3;
- out.write(&buf.front(), 4);
- if (wrap_width && (width += 4) >= wrap_width && left) {
- out << ::std::endl;
- width = 0;
- }
- continue;
- case 2:
- b64_3to4(src[0], src[1], 0, &buf.front());
- buf[3] = '=';
- return out.write(&buf.front(), 4);
- case 1:
- b64_3to4(src[0], 0, 0, &buf.front());
- buf[2] = buf[3] = '=';
- return out.write(&buf.front(), 4);
- case 0:
- return out;
- }
- }
- }
- return out;
-}
-
-static const signed char b64_map[256] = {
- // 1 2 3 4 5 6 7 8 9 a b c d e f
- OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10
- SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, 62, IL, IL, IL, 63, // 20
- 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, IL, IL, IL, EQ, IL, IL, // 30
- IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 40
- 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, IL, IL, IL, IL, IL, // 50
- IL, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 60
- 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, IL, IL, IL, IL, IL, // 70
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0
- IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
-};
-
-static inline signed char b64_4to3(signed char a, signed char b, signed char c,
- signed char d,
- char *__restrict dest) noexcept {
- dest[0] = byte((a << 2) + ((b & 0x30) >> 4));
- dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2));
- dest[2] = byte(((c & 0x3) << 6) + d);
- return a | b | c | d;
-}
-
-char *from_base64::write_bytes(char *__restrict const dest,
- size_t dest_size) const {
- if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces))
- MDBX_CXX20_UNLIKELY throw std::domain_error(
- "mdbx::from_base64:: odd length of base64 string");
- if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
- MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
-
- auto ptr = dest;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(*src <= ' ') &&
- MDBX_LIKELY(ignore_spaces && isspace(*src))) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_UNLIKELY(left < 3))
- MDBX_CXX20_UNLIKELY {
- bailout:
- throw std::domain_error("mdbx::from_base64:: invalid base64 string");
- }
- const signed char a = b64_map[src[0]], b = b64_map[src[1]],
- c = b64_map[src[2]], d = b64_map[src[3]];
- if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, ptr) < 0)) {
- if (left == 4 && (a | b) >= 0 && d == EQ) {
- if (c >= 0) {
- assert(ptr + 2 <= dest + dest_size);
- return ptr + 2;
- }
- if (c == d) {
- assert(ptr + 1 <= dest + dest_size);
- return ptr + 1;
- }
- }
- MDBX_CXX20_UNLIKELY goto bailout;
- }
- src += 4;
- left -= 4;
- ptr += 3;
- assert(ptr <= dest + dest_size);
- }
- return ptr;
-}
-
-bool from_base64::is_erroneous() const noexcept {
- if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces))
- MDBX_CXX20_UNLIKELY return true;
-
- bool got = false;
- auto src = source.byte_ptr();
- for (auto left = source.length(); left > 0;) {
- if (MDBX_UNLIKELY(*src <= ' ') &&
- MDBX_LIKELY(ignore_spaces && isspace(*src))) {
- ++src;
- --left;
- continue;
- }
-
- if (MDBX_UNLIKELY(left < 3))
- MDBX_CXX20_UNLIKELY return false;
- const signed char a = b64_map[src[0]], b = b64_map[src[1]],
- c = b64_map[src[2]], d = b64_map[src[3]];
- if (MDBX_UNLIKELY((a | b | c | d) < 0))
- MDBX_CXX20_UNLIKELY {
- if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d))
- return false;
- return true;
- }
- got = true;
- src += 4;
- left -= 4;
- }
- return !got;
-}
-
-//------------------------------------------------------------------------------
-
-template class LIBMDBX_API_TYPE buffer<legacy_allocator>;
-
-#if defined(__cpp_lib_memory_resource) && \
- __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI
-template class LIBMDBX_API_TYPE buffer<polymorphic_allocator>;
-#endif /* __cpp_lib_memory_resource >= 201603L */
-
-//------------------------------------------------------------------------------
-
-static inline MDBX_env_flags_t mode2flags(env::mode mode) {
- switch (mode) {
- default:
- MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::mode is invalid");
- case env::mode::readonly:
- return MDBX_RDONLY;
- case env::mode::write_file_io:
- return MDBX_ENV_DEFAULTS;
- case env::mode::write_mapped_io:
- return MDBX_WRITEMAP;
- }
-}
-
-__cold MDBX_env_flags_t
-env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
- MDBX_env_flags_t flags = mode2flags(mode);
- if (accede)
- flags |= MDBX_ACCEDE;
- if (!use_subdirectory)
- flags |= MDBX_NOSUBDIR;
- if (options.exclusive)
- flags |= MDBX_EXCLUSIVE;
- if (options.orphan_read_transactions)
- flags |= MDBX_NOTLS;
- if (options.disable_readahead)
- flags |= MDBX_NORDAHEAD;
- if (options.disable_clear_memory)
- flags |= MDBX_NOMEMINIT;
-
- if (mode != readonly) {
- if (options.nested_write_transactions)
- flags &= ~MDBX_WRITEMAP;
- if (reclaiming.coalesce)
- flags |= MDBX_COALESCE;
- if (reclaiming.lifo)
- flags |= MDBX_LIFORECLAIM;
- switch (durability) {
- default:
- MDBX_CXX20_UNLIKELY throw std::invalid_argument(
- "db::durability is invalid");
- case env::durability::robust_synchronous:
- break;
- case env::durability::half_synchronous_weak_last:
- flags |= MDBX_NOMETASYNC;
- break;
- case env::durability::lazy_weak_tail:
- static_assert(MDBX_MAPASYNC == MDBX_SAFE_NOSYNC, "WTF? Obsolete C API?");
- flags |= MDBX_SAFE_NOSYNC;
- break;
- case env::durability::whole_fragile:
- flags |= MDBX_UTTERLY_NOSYNC;
- break;
- }
- }
- return flags;
-}
-
-env::mode
-env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept {
- if (flags & MDBX_RDONLY)
- return env::mode::readonly;
- return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io
- : env::mode::write_file_io;
-}
-
-env::durability env::operate_parameters::durability_from_flags(
- MDBX_env_flags_t flags) noexcept {
- if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC)
- return env::durability::whole_fragile;
- if (flags & MDBX_SAFE_NOSYNC)
- return env::durability::lazy_weak_tail;
- if (flags & MDBX_NOMETASYNC)
- return env::durability::half_synchronous_weak_last;
- return env::durability::robust_synchronous;
-}
-
-env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept
- : lifo((flags & MDBX_LIFORECLAIM) ? true : false),
- coalesce((flags & MDBX_COALESCE) ? true : false) {}
-
-env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept
- : orphan_read_transactions(
- ((flags & (MDBX_NOTLS | MDBX_EXCLUSIVE)) == MDBX_NOTLS) ? true
- : false),
- nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false
- : true),
- exclusive((flags & MDBX_EXCLUSIVE) ? true : false),
- disable_readahead((flags & MDBX_NORDAHEAD) ? true : false),
- disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {}
-
-bool env::is_pristine() const {
- return get_stat().ms_mod_txnid == 0 &&
- get_info().mi_recent_txnid == INITIAL_TXNID;
-}
-
-bool env::is_empty() const { return get_stat().ms_leaf_pages == 0; }
-
-#ifdef MDBX_STD_FILESYSTEM_PATH
-env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify,
- bool force_dynamic_size) {
- const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(destination);
- error::success_or_throw(
- ::mdbx_env_copy(handle_, utf8,
- (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
- (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
- : MDBX_CP_DEFAULTS)));
- return *this;
-}
-#endif /* MDBX_STD_FILESYSTEM_PATH */
-
-#if defined(_WIN32) || defined(_WIN64)
-env &env::copy(const ::std::wstring &destination, bool compactify,
- bool force_dynamic_size) {
- const path_to_pchar<::std::wstring> utf8(destination);
- error::success_or_throw(
- ::mdbx_env_copy(handle_, utf8,
- (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
- (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
- : MDBX_CP_DEFAULTS)));
- return *this;
-}
-#endif /* Windows */
-
-env &env::copy(const ::std::string &destination, bool compactify,
- bool force_dynamic_size) {
- const path_to_pchar<::std::string> utf8(destination);
- error::success_or_throw(
- ::mdbx_env_copy(handle_, utf8,
- (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
- (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
- : MDBX_CP_DEFAULTS)));
- return *this;
-}
-
-env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) {
- error::success_or_throw(
- ::mdbx_env_copy2fd(handle_, fd,
- (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
- (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
- : MDBX_CP_DEFAULTS)));
- return *this;
-}
-
-path env::get_path() const {
- const char *c_str;
- error::success_or_throw(::mdbx_env_get_path(handle_, &c_str));
- return pchar_to_path<path>(c_str);
-}
-
-#ifdef MDBX_STD_FILESYSTEM_PATH
-bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname,
- const remove_mode mode) {
- const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
- return error::boolean_or_throw(
- ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
-}
-#endif /* MDBX_STD_FILESYSTEM_PATH */
-
-#if defined(_WIN32) || defined(_WIN64)
-bool env::remove(const ::std::wstring &pathname, const remove_mode mode) {
- const path_to_pchar<::std::wstring> utf8(pathname);
- return error::boolean_or_throw(
- ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
-}
-#endif /* Windows */
-
-bool env::remove(const ::std::string &pathname, const remove_mode mode) {
- const path_to_pchar<::std::string> utf8(pathname);
- return error::boolean_or_throw(
- ::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
-}
-
-//------------------------------------------------------------------------------
-
-static inline MDBX_env *create_env() {
- MDBX_env *ptr;
- error::success_or_throw(::mdbx_env_create(&ptr));
- assert(ptr != nullptr);
- return ptr;
-}
-
-env_managed::~env_managed() noexcept {
- if (MDBX_UNLIKELY(handle_))
- MDBX_CXX20_UNLIKELY error::success_or_panic(
- ::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close");
-}
-
-void env_managed::close(bool dont_sync) {
- const error rc =
- static_cast<MDBX_error_t>(::mdbx_env_close_ex(handle_, dont_sync));
- switch (rc.code()) {
- case MDBX_EBADSIGN:
- MDBX_CXX20_UNLIKELY handle_ = nullptr;
- __fallthrough /* fall through */;
- default:
- MDBX_CXX20_UNLIKELY rc.throw_exception();
- case MDBX_SUCCESS:
- MDBX_CXX20_LIKELY handle_ = nullptr;
- }
-}
-
-__cold void env_managed::setup(unsigned max_maps, unsigned max_readers) {
- if (max_readers > 0)
- error::success_or_throw(::mdbx_env_set_maxreaders(handle_, max_readers));
- if (max_maps > 0)
- error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps));
-}
-
-#ifdef MDBX_STD_FILESYSTEM_PATH
-__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
- const operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-
-__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
- const env_managed::create_parameters &cp,
- const env::operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<MDBX_STD_FILESYSTEM_PATH> utf8(pathname);
- set_geometry(cp.geometry);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
- cp.file_mode_bits));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-#endif /* MDBX_STD_FILESYSTEM_PATH */
-
-#if defined(_WIN32) || defined(_WIN64)
-__cold env_managed::env_managed(const ::std::wstring &pathname,
- const operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<::std::wstring> utf8(pathname);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-
-__cold env_managed::env_managed(const ::std::wstring &pathname,
- const env_managed::create_parameters &cp,
- const env::operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<::std::wstring> utf8(pathname);
- set_geometry(cp.geometry);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
- cp.file_mode_bits));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-#endif /* Windows */
-
-__cold env_managed::env_managed(const ::std::string &pathname,
- const operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<::std::string> utf8(pathname);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-
-__cold env_managed::env_managed(const ::std::string &pathname,
- const env_managed::create_parameters &cp,
- const env::operate_parameters &op, bool accede)
- : env_managed(create_env()) {
- setup(op.max_maps, op.max_readers);
- const path_to_pchar<::std::string> utf8(pathname);
- set_geometry(cp.geometry);
- error::success_or_throw(
- ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
- cp.file_mode_bits));
-
- if (op.options.nested_write_transactions &&
- !get_options().nested_write_transactions)
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
-}
-
-//------------------------------------------------------------------------------
-
-txn_managed txn::start_nested() {
- MDBX_txn *nested;
- error::throw_on_nullptr(handle_, MDBX_BAD_TXN);
- error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_,
- MDBX_TXN_READWRITE, &nested));
- assert(nested != nullptr);
- return txn_managed(nested);
-}
-
-txn_managed::~txn_managed() noexcept {
- if (MDBX_UNLIKELY(handle_))
- MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_),
- "mdbx::~txn", "mdbx_txn_abort");
-}
-
-void txn_managed::abort() {
- const error err = static_cast<MDBX_error_t>(::mdbx_txn_abort(handle_));
- if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH))
- MDBX_CXX20_LIKELY handle_ = nullptr;
- if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS))
- MDBX_CXX20_UNLIKELY err.throw_exception();
-}
-
-void txn_managed::commit() {
- const error err = static_cast<MDBX_error_t>(::mdbx_txn_commit(handle_));
- if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH))
- MDBX_CXX20_LIKELY handle_ = nullptr;
- if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS))
- MDBX_CXX20_UNLIKELY err.throw_exception();
-}
-
-//------------------------------------------------------------------------------
-
-bool txn::drop_map(const char *name, bool throw_if_absent) {
- map_handle map;
- const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi);
- switch (err) {
- case MDBX_SUCCESS:
- drop_map(map);
- return true;
- case MDBX_NOTFOUND:
- case MDBX_BAD_DBI:
- if (!throw_if_absent)
- return false;
- MDBX_CXX17_FALLTHROUGH /* fallthrough */;
- default:
- MDBX_CXX20_UNLIKELY error::throw_exception(err);
- }
-}
-
-bool txn::clear_map(const char *name, bool throw_if_absent) {
- map_handle map;
- const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi);
- switch (err) {
- case MDBX_SUCCESS:
- clear_map(map);
- return true;
- case MDBX_NOTFOUND:
- case MDBX_BAD_DBI:
- if (!throw_if_absent)
- return false;
- MDBX_CXX17_FALLTHROUGH /* fallthrough */;
- default:
- MDBX_CXX20_UNLIKELY error::throw_exception(err);
- }
-}
-
-//------------------------------------------------------------------------------
-
-void cursor_managed::close() {
- if (MDBX_UNLIKELY(!handle_))
- MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL);
- ::mdbx_cursor_close(handle_);
- handle_ = nullptr;
-}
-
-//------------------------------------------------------------------------------
-
-__cold ::std::ostream &operator<<(::std::ostream &out, const slice &it) {
- out << "{";
- if (!it.is_valid())
- out << "INVALID." << it.length();
- else if (it.is_null())
- out << "NULL";
- else if (it.empty())
- out << "EMPTY->" << it.data();
- else {
- const slice root(it.head(std::min(it.length(), size_t(64))));
- out << it.length() << ".";
- if (root.is_printable())
- (out << "\"").write(root.char_ptr(), root.length()) << "\"";
- else
- out << root.encode_base58();
- if (root.length() < it.length())
- out << "...";
- }
- return out << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) {
- return out << "{" << it.key << " => " << it.value << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out, const pair_result &it) {
- return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => "
- << it.value << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const ::mdbx::env::geometry::size &it) {
- switch (it.bytes) {
- case ::mdbx::env::geometry::default_value:
- return out << "default";
- case ::mdbx::env::geometry::minimal_value:
- return out << "minimal";
- case ::mdbx::env::geometry::maximal_value:
- return out << "maximal";
- }
-
- const auto bytes = (it.bytes < 0) ? out << "-",
- size_t(-it.bytes) : size_t(it.bytes);
- struct {
- size_t one;
- const char *suffix;
- } static const scales[] = {
-#if MDBX_WORDBITS > 32
- {env_managed::geometry::EiB, "EiB"},
- {env_managed::geometry::EB, "EB"},
- {env_managed::geometry::PiB, "PiB"},
- {env_managed::geometry::PB, "PB"},
- {env_managed::geometry::TiB, "TiB"},
- {env_managed::geometry::TB, "TB"},
-#endif
- {env_managed::geometry::GiB, "GiB"},
- {env_managed::geometry::GB, "GB"},
- {env_managed::geometry::MiB, "MiB"},
- {env_managed::geometry::MB, "MB"},
- {env_managed::geometry::KiB, "KiB"},
- {env_managed::geometry::kB, "kB"},
- {1, " bytes"}
- };
-
- for (const auto i : scales)
- if (bytes % i.one == 0)
- return out << bytes / i.one << i.suffix;
-
- assert(false);
- __unreachable();
- return out;
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env::geometry &it) {
- return //
- out << "\tlower " << env::geometry::size(it.size_lower) //
- << ",\n\tnow " << env::geometry::size(it.size_now) //
- << ",\n\tupper " << env::geometry::size(it.size_upper) //
- << ",\n\tgrowth " << env::geometry::size(it.growth_step) //
- << ",\n\tshrink " << env::geometry::size(it.shrink_threshold) //
- << ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env::operate_parameters &it) {
- return out << "{\n" //
- << "\tmax_maps " << it.max_maps //
- << ",\n\tmax_readers " << it.max_readers //
- << ",\n\tmode " << it.mode //
- << ",\n\tdurability " << it.durability //
- << ",\n\treclaiming " << it.reclaiming //
- << ",\n\toptions " << it.options //
- << "\n}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) {
- switch (it) {
- case env::mode::readonly:
- return out << "readonly";
- case env::mode::write_file_io:
- return out << "write_file_io";
- case env::mode::write_mapped_io:
- return out << "write_mapped_io";
- default:
- return out << "mdbx::env::mode::invalid";
- }
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env::durability &it) {
- switch (it) {
- case env::durability::robust_synchronous:
- return out << "robust_synchronous";
- case env::durability::half_synchronous_weak_last:
- return out << "half_synchronous_weak_last";
- case env::durability::lazy_weak_tail:
- return out << "lazy_weak_tail";
- case env::durability::whole_fragile:
- return out << "whole_fragile";
- default:
- return out << "mdbx::env::durability::invalid";
- }
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env::reclaiming_options &it) {
- return out << "{" //
- << "lifo: " << (it.lifo ? "yes" : "no") //
- << ", coalesce: " << (it.coalesce ? "yes" : "no") //
- << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env::operate_options &it) {
- static const char comma[] = ", ";
- const char *delimiter = "";
- out << "{";
- if (it.orphan_read_transactions) {
- out << delimiter << "orphan_read_transactions";
- delimiter = comma;
- }
- if (it.nested_write_transactions) {
- out << delimiter << "nested_write_transactions";
- delimiter = comma;
- }
- if (it.exclusive) {
- out << delimiter << "exclusive";
- delimiter = comma;
- }
- if (it.disable_readahead) {
- out << delimiter << "disable_readahead";
- delimiter = comma;
- }
- if (it.disable_clear_memory) {
- out << delimiter << "disable_clear_memory";
- delimiter = comma;
- }
- if (delimiter != comma)
- out << "default";
- return out << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const env_managed::create_parameters &it) {
- return out << "{\n" //
- << "\tfile_mode " << std::oct << it.file_mode_bits << std::dec //
- << ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") //
- << ",\n"
- << it.geometry << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const MDBX_log_level_t &it) {
- switch (it) {
- case MDBX_LOG_FATAL:
- return out << "LOG_FATAL";
- case MDBX_LOG_ERROR:
- return out << "LOG_ERROR";
- case MDBX_LOG_WARN:
- return out << "LOG_WARN";
- case MDBX_LOG_NOTICE:
- return out << "LOG_NOTICE";
- case MDBX_LOG_VERBOSE:
- return out << "LOG_VERBOSE";
- case MDBX_LOG_DEBUG:
- return out << "LOG_DEBUG";
- case MDBX_LOG_TRACE:
- return out << "LOG_TRACE";
- case MDBX_LOG_EXTRA:
- return out << "LOG_EXTRA";
- case MDBX_LOG_DONTCHANGE:
- return out << "LOG_DONTCHANGE";
- default:
- return out << "mdbx::log_level::invalid";
- }
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const MDBX_debug_flags_t &it) {
- if (it == MDBX_DBG_DONTCHANGE)
- return out << "DBG_DONTCHANGE";
-
- static const char comma[] = "|";
- const char *delimiter = "";
- out << "{";
- if (it & MDBX_DBG_ASSERT) {
- out << delimiter << "DBG_ASSERT";
- delimiter = comma;
- }
- if (it & MDBX_DBG_AUDIT) {
- out << delimiter << "DBG_AUDIT";
- delimiter = comma;
- }
- if (it & MDBX_DBG_JITTER) {
- out << delimiter << "DBG_JITTER";
- delimiter = comma;
- }
- if (it & MDBX_DBG_DUMP) {
- out << delimiter << "DBG_DUMP";
- delimiter = comma;
- }
- if (it & MDBX_DBG_LEGACY_MULTIOPEN) {
- out << delimiter << "DBG_LEGACY_MULTIOPEN";
- delimiter = comma;
- }
- if (it & MDBX_DBG_LEGACY_OVERLAP) {
- out << delimiter << "DBG_LEGACY_OVERLAP";
- delimiter = comma;
- }
- if (delimiter != comma)
- out << "DBG_NONE";
- return out << "}";
-}
-
-__cold ::std::ostream &operator<<(::std::ostream &out,
- const ::mdbx::error &err) {
- return out << err.what() << " (" << long(err.code()) << ")";
-}
-
-} // namespace mdbx
diff --git a/3rdparty/libmdbx/src/mdbx_chk.c b/3rdparty/libmdbx/src/mdbx_chk.c
deleted file mode 100644
index 576b275c6ced38a075a6299187ecf109a409311a..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_chk.c
+++ /dev/null
@@ -1,1746 +0,0 @@
-/* mdbx_chk.c - memory-mapped database check tool */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-typedef struct flagbit {
- int bit;
- const char *name;
-} flagbit;
-
-const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"},
- {MDBX_INTEGERKEY, "integerkey"},
- {MDBX_REVERSEKEY, "reversekey"},
- {MDBX_DUPFIXED, "dupfixed"},
- {MDBX_REVERSEDUP, "reversedup"},
- {MDBX_INTEGERDUP, "integerdup"},
- {0, nullptr}};
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = 1;
- return true;
-}
-
-static uint64_t GetMilliseconds(void) {
- LARGE_INTEGER Counter, Frequency;
- return (QueryPerformanceFrequency(&Frequency) &&
- QueryPerformanceCounter(&Counter))
- ? Counter.QuadPart * 1000ul / Frequency.QuadPart
- : 0;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-#define EXIT_INTERRUPTED (EXIT_FAILURE + 4)
-#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3)
-#define EXIT_FAILURE_MDBX (EXIT_FAILURE + 2)
-#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1)
-#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE
-
-typedef struct {
- const char *name;
- struct {
- uint64_t branch, large_count, large_volume, leaf;
- uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed;
- uint64_t total, empty, other;
- } pages;
- uint64_t payload_bytes;
- uint64_t lost_bytes;
-} walk_dbi_t;
-
-struct {
- short *pagemap;
- uint64_t total_payload_bytes;
- uint64_t pgcount;
- walk_dbi_t
- dbi[MDBX_MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1];
-} walk;
-
-#define dbi_free walk.dbi[FREE_DBI]
-#define dbi_main walk.dbi[MAIN_DBI]
-#define dbi_meta walk.dbi[CORE_DBS]
-
-int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE;
-MDBX_env *env;
-MDBX_txn *txn;
-MDBX_envinfo envinfo;
-size_t userdb_count, skipped_subdb;
-uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages,
- unused_pages, backed_pages;
-unsigned verbose;
-bool ignore_wrong_order, quiet, dont_traversal;
-const char *only_subdb;
-int stuck_meta = -1;
-
-struct problem {
- struct problem *pr_next;
- size_t count;
- const char *caption;
-};
-
-struct problem *problems_list;
-unsigned total_problems, data_tree_problems, gc_tree_problems;
-
-static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) {
- if (!quiet) {
- va_list args;
-
- fflush(stderr);
- va_start(args, msg);
- vfprintf(stdout, msg, args);
- va_end(args);
- }
-}
-
-static void va_log(MDBX_log_level_t level, const char *msg, va_list args) {
- static const char *const prefixes[] = {
- "!!!fatal: ", " ! " /* error */, " ~ " /* warning */,
- " " /* notice */, " // " /* verbose */, " //// " /* debug */,
- " ////// " /* trace */
- };
-
- FILE *out = stdout;
- if (level <= MDBX_LOG_ERROR) {
- total_problems++;
- out = stderr;
- }
-
- if (!quiet && verbose + 1 >= (unsigned)level &&
- (unsigned)level < ARRAY_LENGTH(prefixes)) {
- fflush(nullptr);
- fputs(prefixes[level], out);
- vfprintf(out, msg, args);
- if (msg[strlen(msg) - 1] != '\n')
- fputc('\n', out);
- fflush(nullptr);
- }
-
- if (level == MDBX_LOG_FATAL) {
- exit(EXIT_FAILURE_MDBX);
- abort();
- }
-}
-
-static void MDBX_PRINTF_ARGS(1, 2) error(const char *msg, ...) {
- va_list args;
- va_start(args, msg);
- va_log(MDBX_LOG_ERROR, msg, args);
- va_end(args);
-}
-
-static void logger(MDBX_log_level_t level, const char *function, int line,
- const char *msg, va_list args) {
- (void)line;
- (void)function;
- if (level < MDBX_LOG_EXTRA)
- va_log(level, msg, args);
-}
-
-static int check_user_break(void) {
- switch (user_break) {
- case 0:
- return MDBX_SUCCESS;
- case 1:
- print(" - interrupted by signal\n");
- fflush(nullptr);
- user_break = 2;
- }
- return MDBX_EINTR;
-}
-
-static void pagemap_cleanup(void) {
- for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1;
- i < ARRAY_LENGTH(walk.dbi); ++i) {
- if (walk.dbi[i].name) {
- mdbx_free((void *)walk.dbi[i].name);
- walk.dbi[i].name = nullptr;
- }
- }
-
- mdbx_free(walk.pagemap);
- walk.pagemap = nullptr;
-}
-
-static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
- static walk_dbi_t *last;
-
- if (dbi_name == MDBX_PGWALK_MAIN)
- return &dbi_main;
- if (dbi_name == MDBX_PGWALK_GC)
- return &dbi_free;
- if (dbi_name == MDBX_PGWALK_META)
- return &dbi_meta;
-
- if (last && strcmp(last->name, dbi_name) == 0)
- return last;
-
- walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1;
- for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) {
- if (strcmp(dbi->name, dbi_name) == 0)
- return last = dbi;
- }
-
- if (verbose > 0 && !silent) {
- print(" - found '%s' area\n", dbi_name);
- fflush(nullptr);
- }
-
- if (dbi == ARRAY_END(walk.dbi))
- return nullptr;
-
- dbi->name = mdbx_strdup(dbi_name);
- return last = dbi;
-}
-
-static void MDBX_PRINTF_ARGS(4, 5)
- problem_add(const char *object, uint64_t entry_number, const char *msg,
- const char *extra, ...) {
- total_problems++;
-
- if (!quiet) {
- int need_fflush = 0;
- struct problem *p;
-
- for (p = problems_list; p; p = p->pr_next)
- if (p->caption == msg)
- break;
-
- if (!p) {
- p = mdbx_calloc(1, sizeof(*p));
- if (unlikely(!p))
- return;
- p->caption = msg;
- p->pr_next = problems_list;
- problems_list = p;
- need_fflush = 1;
- }
-
- p->count++;
- if (verbose > 1) {
- print(" %s #%" PRIu64 ": %s", object, entry_number, msg);
- if (extra) {
- va_list args;
- printf(" (");
- va_start(args, extra);
- vfprintf(stdout, extra, args);
- va_end(args);
- printf(")");
- }
- printf("\n");
- if (need_fflush)
- fflush(nullptr);
- }
- }
-}
-
-static struct problem *problems_push(void) {
- struct problem *p = problems_list;
- problems_list = nullptr;
- return p;
-}
-
-static size_t problems_pop(struct problem *list) {
- size_t count = 0;
-
- if (problems_list) {
- int i;
-
- print(" - problems: ");
- for (i = 0; problems_list; ++i) {
- struct problem *p = problems_list->pr_next;
- count += problems_list->count;
- print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption,
- problems_list->count);
- mdbx_free(problems_list);
- problems_list = p;
- }
- print("\n");
- fflush(nullptr);
- }
-
- problems_list = list;
- return count;
-}
-
-static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
- void *const ctx, const int deep,
- const char *const dbi_name_or_tag, const size_t page_size,
- const MDBX_page_type_t pagetype, const MDBX_error_t err,
- const size_t nentries, const size_t payload_bytes,
- const size_t header_bytes, const size_t unused_bytes) {
- (void)ctx;
- const bool is_gc_tree = dbi_name_or_tag == MDBX_PGWALK_GC;
- if (deep > 42) {
- problem_add("deep", deep, "too large", nullptr);
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- return MDBX_CORRUPTED /* avoid infinite loop/recursion */;
- }
-
- walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false);
- if (!dbi) {
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- return MDBX_ENOMEM;
- }
-
- const size_t page_bytes = payload_bytes + header_bytes + unused_bytes;
- walk.pgcount += pgnumber;
-
- const char *pagetype_caption;
- bool branch = false;
- switch (pagetype) {
- default:
- problem_add("page", pgno, "unknown page-type", "type %u, deep %i",
- (unsigned)pagetype, deep);
- pagetype_caption = "unknown";
- dbi->pages.other += pgnumber;
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- break;
- case MDBX_page_broken:
- pagetype_caption = "broken";
- dbi->pages.other += pgnumber;
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- break;
- case MDBX_subpage_broken:
- pagetype_caption = "broken-subpage";
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- break;
- case MDBX_page_meta:
- pagetype_caption = "meta";
- dbi->pages.other += pgnumber;
- break;
- case MDBX_page_large:
- pagetype_caption = "large";
- dbi->pages.large_volume += pgnumber;
- dbi->pages.large_count += 1;
- break;
- case MDBX_page_branch:
- pagetype_caption = "branch";
- dbi->pages.branch += pgnumber;
- branch = true;
- break;
- case MDBX_page_leaf:
- pagetype_caption = "leaf";
- dbi->pages.leaf += pgnumber;
- break;
- case MDBX_page_dupfixed_leaf:
- pagetype_caption = "leaf-dupfixed";
- dbi->pages.leaf_dupfixed += pgnumber;
- break;
- case MDBX_subpage_leaf:
- pagetype_caption = "subleaf-dupsort";
- dbi->pages.subleaf_dupsort += 1;
- break;
- case MDBX_subpage_dupfixed_leaf:
- pagetype_caption = "subleaf-dupfixed";
- dbi->pages.subleaf_dupfixed += 1;
- break;
- }
-
- if (pgnumber) {
- if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) {
- if (pgnumber == 1)
- print(" %s-page %" PRIu64, pagetype_caption, pgno);
- else
- print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber);
- print(" of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR
- ", unused %" PRIiPTR ", deep %i\n",
- dbi->name, header_bytes,
- (pagetype == MDBX_page_branch) ? "keys" : "entries", nentries,
- payload_bytes, unused_bytes, deep);
- }
-
- bool already_used = false;
- for (unsigned n = 0; n < pgnumber; ++n) {
- uint64_t spanpgno = pgno + n;
- if (spanpgno >= alloc_pages) {
- problem_add("page", spanpgno, "wrong page-no",
- "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i",
- pagetype_caption, spanpgno, alloc_pages, deep);
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- } else if (walk.pagemap[spanpgno]) {
- walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1];
- problem_add("page", spanpgno,
- (branch && coll_dbi == dbi) ? "loop" : "already used",
- "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name,
- deep);
- already_used = true;
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- } else {
- walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1);
- dbi->pages.total += 1;
- }
- }
-
- if (already_used)
- return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */
- : MDBX_SUCCESS;
- }
-
- if (MDBX_IS_ERROR(err)) {
- problem_add("page", pgno, "invalid/corrupted", "%s-page", pagetype_caption);
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- } else {
- if (unused_bytes > page_size) {
- problem_add("page", pgno, "illegal unused-bytes",
- "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0,
- unused_bytes, envinfo.mi_dxb_pagesize);
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- }
-
- if (header_bytes < (int)sizeof(long) ||
- (size_t)header_bytes >= envinfo.mi_dxb_pagesize - sizeof(long)) {
- problem_add("page", pgno, "illegal header-length",
- "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR,
- pagetype_caption, sizeof(long), header_bytes,
- envinfo.mi_dxb_pagesize - sizeof(long));
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- }
- if (payload_bytes < 1) {
- if (nentries > 1) {
- problem_add("page", pgno, "zero size-of-entry",
- "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
- pagetype_caption, payload_bytes, nentries);
- /* if ((size_t)header_bytes + unused_bytes < page_size) {
- // LY: hush a misuse error
- page_bytes = page_size;
- } */
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- } else {
- problem_add("page", pgno, "empty",
- "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR
- " entries, deep %i",
- pagetype_caption, payload_bytes, nentries, deep);
- dbi->pages.empty += 1;
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- }
- }
-
- if (pgnumber) {
- if (page_bytes != page_size) {
- problem_add("page", pgno, "misused",
- "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR
- "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i",
- pagetype_caption, page_size, page_bytes, header_bytes,
- payload_bytes, unused_bytes, deep);
- if (page_size > page_bytes)
- dbi->lost_bytes += page_size - page_bytes;
- data_tree_problems += !is_gc_tree;
- gc_tree_problems += is_gc_tree;
- } else {
- dbi->payload_bytes += payload_bytes + header_bytes;
- walk.total_payload_bytes += payload_bytes + header_bytes;
- }
- }
- }
-
- return check_user_break();
-}
-
-typedef int(visitor)(const uint64_t record_number, const MDBX_val *key,
- const MDBX_val *data);
-static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
- bool silent);
-
-static int handle_userdb(const uint64_t record_number, const MDBX_val *key,
- const MDBX_val *data) {
- (void)record_number;
- (void)key;
- (void)data;
- return check_user_break();
-}
-
-static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
- const MDBX_val *data) {
- char *bad = "";
- pgno_t *iptr = data->iov_base;
-
- if (key->iov_len != sizeof(txnid_t))
- problem_add("entry", record_number, "wrong txn-id size",
- "key-size %" PRIiPTR, key->iov_len);
- else {
- txnid_t txnid;
- memcpy(&txnid, key->iov_base, sizeof(txnid));
- if (txnid < 1 || txnid > envinfo.mi_recent_txnid)
- problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid);
- else {
- if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
- problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR,
- data->iov_len);
- size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0;
- if (number < 1 || number > MDBX_PGL_LIMIT)
- problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number);
- else if ((number + 1) * sizeof(pgno_t) > data->iov_len) {
- problem_add("entry", txnid, "trimmed idl",
- "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
- (number + 1) * sizeof(pgno_t), data->iov_len);
- number = data->iov_len / sizeof(pgno_t) - 1;
- } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
- /* LY: allow gap up to one page. it is ok
- * and better than shink-and-retry inside mdbx_update_gc() */
- envinfo.mi_dxb_pagesize)
- problem_add("entry", txnid, "extra idl space",
- "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)",
- (number + 1) * sizeof(pgno_t), data->iov_len);
-
- gc_pages += number;
- if (envinfo.mi_latter_reader_txnid > txnid)
- reclaimable_pages += number;
-
- pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno;
- pgno_t span = 1;
- for (unsigned i = 0; i < number; ++i) {
- if (check_user_break())
- return MDBX_EINTR;
- const pgno_t pgno = iptr[i];
- if (pgno < NUM_METAS)
- problem_add("entry", txnid, "wrong idl entry",
- "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS);
- else if (pgno >= backed_pages)
- problem_add("entry", txnid, "wrong idl entry",
- "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno,
- backed_pages);
- else if (pgno >= alloc_pages)
- problem_add("entry", txnid, "wrong idl entry",
- "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno,
- alloc_pages - 1);
- else {
- if (MDBX_PNL_DISORDERED(prev, pgno)) {
- bad = " [bad sequence]";
- problem_add("entry", txnid, "bad sequence",
- "%" PRIaPGNO " %c [%u].%" PRIaPGNO, prev,
- (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'),
- i, pgno);
- }
- if (walk.pagemap) {
- int idx = walk.pagemap[pgno];
- if (idx == 0)
- walk.pagemap[pgno] = -1;
- else if (idx > 0)
- problem_add("page", pgno, "already used", "by %s",
- walk.dbi[idx - 1].name);
- else
- problem_add("page", pgno, "already listed in GC", nullptr);
- }
- }
- prev = pgno;
- while (i + span < number &&
- iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span)
- : pgno_sub(pgno, span)))
- ++span;
- }
- if (verbose > 3 && !only_subdb) {
- print(" transaction %" PRIaTXN ", %" PRIuPTR
- " pages, maxspan %" PRIaPGNO "%s\n",
- txnid, number, span, bad);
- if (verbose > 4) {
- for (unsigned i = 0; i < number; i += span) {
- const pgno_t pgno = iptr[i];
- for (span = 1;
- i + span < number &&
- iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span)
- : pgno_sub(pgno, span));
- ++span)
- ;
- if (span > 1) {
- print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span);
- } else
- print(" %9" PRIaPGNO "\n", pgno);
- }
- }
- }
- }
- }
-
- return check_user_break();
-}
-
-static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
- return (a->iov_len == b->iov_len &&
- memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
- ? 0
- : 1;
-}
-
-static int handle_maindb(const uint64_t record_number, const MDBX_val *key,
- const MDBX_val *data) {
- char *name;
- int rc;
- size_t i;
-
- name = key->iov_base;
- for (i = 0; i < key->iov_len; ++i) {
- if (name[i] < ' ')
- return handle_userdb(record_number, key, data);
- }
-
- name = mdbx_malloc(key->iov_len + 1);
- if (unlikely(!name))
- return MDBX_ENOMEM;
- memcpy(name, key->iov_base, key->iov_len);
- name[key->iov_len] = '\0';
- userdb_count++;
-
- rc = process_db(~0u, name, handle_userdb, false);
- mdbx_free(name);
- if (rc != MDBX_INCOMPATIBLE)
- return rc;
-
- return handle_userdb(record_number, key, data);
-}
-
-static const char *db_flags2keymode(unsigned flags) {
- flags &= (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
- switch (flags) {
- case 0:
- return "usual";
- case MDBX_REVERSEKEY:
- return "reserve";
- case MDBX_INTEGERKEY:
- return "ordinal";
- case MDBX_REVERSEKEY | MDBX_INTEGERKEY:
- return "msgpack";
- default:
- assert(false);
- __unreachable();
- }
-}
-
-static const char *db_flags2valuemode(unsigned flags) {
- flags &= (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP);
- switch (flags) {
- case 0:
- return "single";
- case MDBX_DUPSORT:
- return "multi";
- case MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_REVERSEDUP:
- return "multi-reverse";
- case MDBX_DUPFIXED:
- case MDBX_DUPSORT | MDBX_DUPFIXED:
- return "multi-samelength";
- case MDBX_DUPFIXED | MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
- return "multi-reverse-samelength";
- case MDBX_INTEGERDUP:
- case MDBX_DUPSORT | MDBX_INTEGERDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
- case MDBX_DUPFIXED | MDBX_INTEGERDUP:
- return "multi-ordinal";
- case MDBX_INTEGERDUP | MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
- return "multi-msgpack";
- case MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
- case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
- return "reserved";
- default:
- assert(false);
- __unreachable();
- }
-}
-
-static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
- bool silent) {
- MDBX_cursor *mc;
- MDBX_stat ms;
- MDBX_val key, data;
- MDBX_val prev_key, prev_data;
- unsigned flags;
- int rc, i;
- struct problem *saved_list;
- uint64_t problems_count;
-
- uint64_t record_count = 0, dups = 0;
- uint64_t key_bytes = 0, data_bytes = 0;
-
- if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) {
- print(" ! abort processing '%s' due to a previous error\n",
- dbi_name ? dbi_name : "@MAIN");
- return MDBX_BAD_TXN;
- }
-
- if (dbi_handle == ~0u) {
- rc = mdbx_dbi_open_ex(
- txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle,
- (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr,
- (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr);
- if (rc) {
- if (!dbi_name ||
- rc !=
- MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ {
- error("mdbx_dbi_open('%s') failed, error %d %s\n",
- dbi_name ? dbi_name : "main", rc, mdbx_strerror(rc));
- }
- return rc;
- }
- }
-
- if (dbi_handle >= CORE_DBS && dbi_name && only_subdb &&
- strcmp(only_subdb, dbi_name) != 0) {
- if (verbose) {
- print("Skip processing '%s'...\n", dbi_name);
- fflush(nullptr);
- }
- skipped_subdb++;
- return MDBX_SUCCESS;
- }
-
- if (!silent && verbose) {
- print("Processing '%s'...\n", dbi_name ? dbi_name : "@MAIN");
- fflush(nullptr);
- }
-
- rc = mdbx_dbi_flags(txn, dbi_handle, &flags);
- if (rc) {
- error("mdbx_dbi_flags() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return rc;
- }
-
- rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms));
- if (rc) {
- error("mdbx_dbi_stat() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return rc;
- }
-
- if (!silent && verbose) {
- print(" - key-value kind: %s-key => %s-value", db_flags2keymode(flags),
- db_flags2valuemode(flags));
- if (verbose > 1) {
- print(", flags:");
- if (!flags)
- print(" none");
- else {
- for (i = 0; dbflags[i].bit; i++)
- if (flags & dbflags[i].bit)
- print(" %s", dbflags[i].name);
- }
- if (verbose > 2)
- print(" (0x%02X), dbi-id %d", flags, dbi_handle);
- }
- print("\n");
- if (ms.ms_mod_txnid)
- print(" - last modification txn#%" PRIu64 "\n", ms.ms_mod_txnid);
- if (verbose > 1) {
- print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize,
- ms.ms_entries);
- print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64
- ", overflow %" PRIu64 "\n",
- ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages,
- ms.ms_overflow_pages);
- }
- }
-
- walk_dbi_t *dbi = (dbi_handle < CORE_DBS)
- ? &walk.dbi[dbi_handle]
- : pagemap_lookup_dbi(dbi_name, true);
- if (!dbi) {
- error("too many DBIs or out of memory\n");
- return MDBX_ENOMEM;
- }
- if (!dont_traversal) {
- const uint64_t subtotal_pages =
- ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages;
- if (subtotal_pages != dbi->pages.total)
- error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
- "subtotal", subtotal_pages, dbi->pages.total);
- if (ms.ms_branch_pages != dbi->pages.branch)
- error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch",
- ms.ms_branch_pages, dbi->pages.branch);
- const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed;
- if (ms.ms_leaf_pages != allleaf_pages)
- error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
- "all-leaf", ms.ms_leaf_pages, allleaf_pages);
- if (ms.ms_overflow_pages != dbi->pages.large_volume)
- error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n",
- "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume);
- }
- rc = mdbx_cursor_open(txn, dbi_handle, &mc);
- if (rc) {
- error("mdbx_cursor_open() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return rc;
- }
-
- if (ignore_wrong_order) { /* for debugging with enabled assertions */
- mc->mc_flags |= C_SKIPORD;
- if (mc->mc_xcursor)
- mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
- }
-
- const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, flags);
- saved_list = problems_push();
- prev_key.iov_base = nullptr;
- prev_key.iov_len = 0;
- prev_data.iov_base = nullptr;
- prev_data.iov_len = 0;
- rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST);
- while (rc == MDBX_SUCCESS) {
- rc = check_user_break();
- if (rc)
- goto bailout;
-
- bool bad_key = false;
- if (key.iov_len > maxkeysize) {
- problem_add("entry", record_count, "key length exceeds max-key-size",
- "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize);
- bad_key = true;
- } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) &&
- key.iov_len != sizeof(uint32_t)) {
- problem_add("entry", record_count, "wrong key length",
- "%" PRIuPTR " != 4or8", key.iov_len);
- bad_key = true;
- }
-
- bool bad_data = false;
- if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) &&
- data.iov_len != sizeof(uint32_t)) {
- problem_add("entry", record_count, "wrong data length",
- "%" PRIuPTR " != 4or8", data.iov_len);
- bad_data = true;
- }
-
- if (prev_key.iov_base) {
- if (prev_data.iov_base && !bad_data && (flags & MDBX_DUPFIXED) &&
- prev_data.iov_len != data.iov_len) {
- problem_add("entry", record_count, "different data length",
- "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len,
- data.iov_len);
- bad_data = true;
- }
-
- if (!bad_key) {
- int cmp = mdbx_cmp(txn, dbi_handle, &key, &prev_key);
- if (cmp == 0) {
- ++dups;
- if ((flags & MDBX_DUPSORT) == 0) {
- problem_add("entry", record_count, "duplicated entries", nullptr);
- if (prev_data.iov_base && data.iov_len == prev_data.iov_len &&
- memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) {
- problem_add("entry", record_count, "complete duplicate", nullptr);
- }
- } else if (!bad_data && prev_data.iov_base) {
- cmp = mdbx_dcmp(txn, dbi_handle, &data, &prev_data);
- if (cmp == 0) {
- problem_add("entry", record_count, "complete duplicate", nullptr);
- } else if (cmp < 0 && !ignore_wrong_order) {
- problem_add("entry", record_count, "wrong order of multi-values",
- nullptr);
- }
- }
- } else if (cmp < 0 && !ignore_wrong_order) {
- problem_add("entry", record_count, "wrong order of entries", nullptr);
- }
- }
- }
-
- if (handler) {
- rc = handler(record_count, &key, &data);
- if (MDBX_IS_ERROR(rc))
- goto bailout;
- }
-
- record_count++;
- key_bytes += key.iov_len;
- data_bytes += data.iov_len;
-
- if (!bad_key) {
- if (verbose && (flags & MDBX_INTEGERKEY) && !prev_key.iov_base)
- print(" - fixed key-size %" PRIuPTR "\n", key.iov_len);
- prev_key = key;
- }
- if (!bad_data) {
- if (verbose && (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) &&
- !prev_data.iov_base)
- print(" - fixed data-size %" PRIuPTR "\n", data.iov_len);
- prev_data = data;
- }
- rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT);
- }
- if (rc != MDBX_NOTFOUND)
- error("mdbx_cursor_get() failed, error %d %s\n", rc, mdbx_strerror(rc));
- else
- rc = 0;
-
- if (record_count != ms.ms_entries)
- problem_add("entry", record_count, "different number of entries",
- "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries);
-bailout:
- problems_count = problems_pop(saved_list);
- if (!silent && verbose) {
- print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64
- " key's bytes, %" PRIu64 " data's "
- "bytes, %" PRIu64 " problems\n",
- record_count, dups, key_bytes, data_bytes, problems_count);
- fflush(nullptr);
- }
-
- mdbx_cursor_close(mc);
- return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS;
-}
-
-static void usage(char *prog) {
- fprintf(stderr,
- "usage: %s [-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s subdb] "
- "dbpath\n"
- " -V\t\tprint version and exit\n"
- " -v\t\tmore verbose, could be used multiple times\n"
- " -q\t\tbe quiet\n"
- " -c\t\tforce cooperative mode (don't try exclusive)\n"
- " -w\t\twrite-mode checking\n"
- " -d\t\tdisable page-by-page traversal of B-tree\n"
- " -i\t\tignore wrong order errors (for custom comparators case)\n"
- " -s subdb\tprocess a specific subdatabase only\n"
- " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n"
- " -t\t\tturn to a specified meta-page on successful check\n"
- " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n",
- prog);
- exit(EXIT_INTERRUPTED);
-}
-
-static bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b,
- uint64_t sign_b, const bool wanna_steady) {
- if (txn_a == txn_b)
- return SIGN_IS_STEADY(sign_b);
-
- if (wanna_steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b))
- return SIGN_IS_STEADY(sign_b);
-
- return txn_a < txn_b;
-}
-
-static bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b,
- uint64_t sign_b) {
- if (!txn_a || txn_a != txn_b)
- return false;
-
- if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b))
- return false;
-
- return true;
-}
-
-static int meta_recent(const bool wanna_steady) {
- if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady))
- return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
- envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady)
- ? 1
- : 2;
- else
- return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, wanna_steady)
- ? 2
- : 0;
-}
-
-static int meta_tail(int head) {
- switch (head) {
- case 0:
- return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
- envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true)
- ? 1
- : 2;
- case 1:
- return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true)
- ? 0
- : 2;
- case 2:
- return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true)
- ? 0
- : 1;
- default:
- assert(false);
- return -1;
- }
-}
-
-static int meta_head(void) { return meta_recent(false); }
-
-void verbose_meta(int num, txnid_t txnid, uint64_t sign, uint64_t bootid_x,
- uint64_t bootid_y) {
- const bool have_bootid = (bootid_x | bootid_y) != 0;
- const bool bootid_match = bootid_x == envinfo.mi_bootid.current.x &&
- bootid_y == envinfo.mi_bootid.current.y;
-
- print(" - meta-%d: ", num);
- switch (sign) {
- case MDBX_DATASIGN_NONE:
- print("no-sync/legacy");
- break;
- case MDBX_DATASIGN_WEAK:
- print("weak-%s", bootid_match ? (have_bootid ? "intact (same boot-id)"
- : "unknown (no boot-id")
- : "dead");
- break;
- default:
- print("steady");
- break;
- }
- print(" txn#%" PRIu64, txnid);
-
- const int head = meta_head();
- if (num == head)
- print(", head");
- else if (num == meta_tail(head))
- print(", tail");
- else
- print(", stay");
-
- if (stuck_meta >= 0) {
- if (num == stuck_meta)
- print(", forced for checking");
- } else if (txnid > envinfo.mi_recent_txnid &&
- (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE)
- print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")",
- txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid);
- print("\n");
-}
-
-static uint64_t get_meta_txnid(const unsigned meta_id) {
- switch (meta_id) {
- default:
- assert(false);
- error("unexpected meta_id %u\n", meta_id);
- return 0;
- case 0:
- return envinfo.mi_meta0_txnid;
- case 1:
- return envinfo.mi_meta1_txnid;
- case 2:
- return envinfo.mi_meta2_txnid;
- }
-}
-
-static void print_size(const char *prefix, const uint64_t value,
- const char *suffix) {
- const char sf[] =
- "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */
- double k = 1024.0;
- size_t i;
- for (i = 0; sf[i + 1] && value / k > 1000.0; ++i)
- k *= 1024;
- print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix);
-}
-
-int main(int argc, char *argv[]) {
- int rc;
- char *prog = argv[0];
- char *envname;
- unsigned problems_maindb = 0, problems_freedb = 0, problems_meta = 0;
- bool write_locked = false;
- bool turn_meta = false;
- bool force_turn_meta = false;
-
- double elapsed;
-#if defined(_WIN32) || defined(_WIN64)
- uint64_t timestamp_start, timestamp_finish;
- timestamp_start = GetMilliseconds();
-#else
- struct timespec timestamp_start, timestamp_finish;
- if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) {
- rc = errno;
- error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return EXIT_FAILURE_SYS;
- }
-#endif
-
- dbi_meta.name = "@META";
- dbi_free.name = "@GC";
- dbi_main.name = "@MAIN";
- atexit(pagemap_cleanup);
-
- if (argc < 2)
- usage(prog);
-
- for (int i; (i = getopt(argc, argv,
- "0"
- "1"
- "2"
- "T"
- "V"
- "v"
- "q"
- "n"
- "w"
- "c"
- "t"
- "d"
- "i"
- "s:")) != EOF;) {
- switch (i) {
- case 'V':
- printf("mdbx_chk version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- case 'v':
- verbose++;
- break;
- case '0':
- stuck_meta = 0;
- break;
- case '1':
- stuck_meta = 1;
- break;
- case '2':
- stuck_meta = 2;
- break;
- case 't':
- turn_meta = true;
- break;
- case 'T':
- turn_meta = force_turn_meta = true;
- quiet = false;
- if (verbose < 2)
- verbose = 2;
- break;
- case 'q':
- quiet = true;
- break;
- case 'n':
- break;
- case 'w':
- envflags &= ~MDBX_RDONLY;
-#if MDBX_MMAP_INCOHERENT_FILE_WRITE
- /* Temporary `workaround` for OpenBSD kernel's flaw.
- * See todo4recovery://erased_by_github/libmdbx/issues/67 */
- envflags |= MDBX_WRITEMAP;
-#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
- break;
- case 'c':
- envflags = (envflags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE;
- break;
- case 'd':
- dont_traversal = true;
- break;
- case 's':
- if (only_subdb && strcmp(only_subdb, optarg))
- usage(prog);
- only_subdb = optarg;
- break;
- case 'i':
- ignore_wrong_order = true;
- break;
- default:
- usage(prog);
- }
- }
-
- if (optind != argc - 1)
- usage(prog);
-
- rc = MDBX_SUCCESS;
- if (stuck_meta >= 0 && (envflags & MDBX_EXCLUSIVE) == 0) {
- error("exclusive mode is required to using specific meta-page(%d) for "
- "checking.\n",
- stuck_meta);
- rc = EXIT_INTERRUPTED;
- }
- if (turn_meta) {
- if (stuck_meta < 0) {
- error("meta-page must be specified (by -0, -1 or -2 options) to turn to "
- "it.\n");
- rc = EXIT_INTERRUPTED;
- }
- if (envflags & MDBX_RDONLY) {
- error("write-mode must be enabled to turn to the specified meta-page.\n");
- rc = EXIT_INTERRUPTED;
- }
- if (only_subdb || dont_traversal) {
- error("whole database checking with tree-traversal are required to turn "
- "to the specified meta-page.\n");
- rc = EXIT_INTERRUPTED;
- }
- }
- if (rc)
- exit(rc);
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- envname = argv[optind];
- print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, envname,
- (envflags & MDBX_RDONLY) ? "only" : "write");
- fflush(nullptr);
- mdbx_setup_debug((verbose < MDBX_LOG_TRACE - 1)
- ? (MDBX_log_level_t)(verbose + 1)
- : MDBX_LOG_TRACE,
- MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, logger);
-
- rc = mdbx_env_create(&env);
- if (rc) {
- error("mdbx_env_create() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return rc < 0 ? EXIT_FAILURE_MDBX : EXIT_FAILURE_SYS;
- }
-
- rc = mdbx_env_set_maxdbs(env, MDBX_MAX_DBI);
- if (rc) {
- error("mdbx_env_set_maxdbs() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- if (stuck_meta >= 0) {
- rc = mdbx_env_open_for_recovery(env, envname, stuck_meta,
- (envflags & MDBX_RDONLY) ? false : true);
- } else {
- rc = mdbx_env_open(env, envname, envflags, 0);
- if ((envflags & MDBX_EXCLUSIVE) &&
- (rc == MDBX_BUSY ||
-#if defined(_WIN32) || defined(_WIN64)
- rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION
-#else
- rc == EBUSY || rc == EAGAIN
-#endif
- )) {
- envflags &= ~MDBX_EXCLUSIVE;
- rc = mdbx_env_open(env, envname, envflags | MDBX_ACCEDE, 0);
- }
- }
-
- if (rc) {
- error("mdbx_env_open() failed, error %d %s\n", rc, mdbx_strerror(rc));
- if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY))
- print("Please run %s in the read-write mode (with '-w' option).\n", prog);
- goto bailout;
- }
- if (verbose)
- print(" - %s mode\n",
- (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative");
-
- if ((envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0) {
- rc = mdbx_txn_lock(env, false);
- if (rc != MDBX_SUCCESS) {
- error("mdbx_txn_lock() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- write_locked = true;
- }
-
- rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
- if (rc) {
- error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo));
- if (rc) {
- error("mdbx_env_info_ex() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
- if (verbose) {
- print(" - current boot-id ");
- if (envinfo.mi_bootid.current.x | envinfo.mi_bootid.current.y)
- print("%016" PRIx64 "-%016" PRIx64 "\n", envinfo.mi_bootid.current.x,
- envinfo.mi_bootid.current.y);
- else
- print("unavailable\n");
- }
-
- mdbx_filehandle_t dxb_fd;
- rc = mdbx_env_get_fd(env, &dxb_fd);
- if (rc) {
- error("mdbx_env_get_fd() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- uint64_t dxb_filesize = 0;
-#if defined(_WIN32) || defined(_WIN64)
- {
- BY_HANDLE_FILE_INFORMATION info;
- if (!GetFileInformationByHandle(dxb_fd, &info))
- rc = GetLastError();
- else
- dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
- }
-#else
- {
- struct stat st;
- STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- if (fstat(dxb_fd, &st))
- rc = errno;
- else
- dxb_filesize = st.st_size;
- }
-#endif
- if (rc) {
- error("mdbx_filesize() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- errno = 0;
- const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize;
- alloc_pages = txn->mt_next_pgno;
- backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize;
- if (backed_pages > dxbfile_pages) {
- print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
- backed_pages, dxbfile_pages);
- ++problems_meta;
- }
- if (dxbfile_pages < NUM_METAS)
- print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS);
- if (backed_pages < NUM_METAS)
- print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS);
- if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS)
- goto bailout;
- if (backed_pages > MAX_PAGENO + 1) {
- print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n",
- backed_pages, MAX_PAGENO + 1);
- ++problems_meta;
- backed_pages = MAX_PAGENO + 1;
- }
-
- if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) {
- if (backed_pages > dxbfile_pages) {
- print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
- backed_pages, dxbfile_pages);
- ++problems_meta;
- backed_pages = dxbfile_pages;
- }
- if (alloc_pages > backed_pages) {
- print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n",
- alloc_pages, backed_pages);
- ++problems_meta;
- alloc_pages = backed_pages;
- }
- } else {
- /* LY: DB may be shrinked by writer down to the allocated pages. */
- if (alloc_pages > backed_pages) {
- print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n",
- alloc_pages, backed_pages);
- ++problems_meta;
- alloc_pages = backed_pages;
- }
- if (alloc_pages > dxbfile_pages) {
- print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n",
- alloc_pages, dxbfile_pages);
- ++problems_meta;
- alloc_pages = dxbfile_pages;
- }
- if (backed_pages > dxbfile_pages)
- backed_pages = dxbfile_pages;
- }
-
- if (verbose) {
- print(" - pagesize %u (%u system), max keysize %d..%d"
- ", max readers %u\n",
- envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize,
- mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT),
- mdbx_env_get_maxkeysize_ex(env, 0), envinfo.mi_maxreaders);
- print_size(" - mapsize ", envinfo.mi_mapsize, "\n");
- if (envinfo.mi_geo.lower == envinfo.mi_geo.upper)
- print_size(" - fixed datafile: ", envinfo.mi_geo.current, "");
- else {
- print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, "");
- print_size(" .. ", envinfo.mi_geo.upper, ", ");
- print_size("+", envinfo.mi_geo.grow, ", ");
- print_size("-", envinfo.mi_geo.shrink, "\n");
- print_size(" - current datafile: ", envinfo.mi_geo.current, "");
- }
- printf(", %" PRIu64 " pages\n",
- envinfo.mi_geo.current / envinfo.mi_dxb_pagesize);
-#if defined(_WIN32) || defined(_WIN64)
- if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper)
- print(
- " WARNING: Due Windows system limitations a "
- "file couldn't\n be truncated while the database "
- "is opened. So, the size\n database file "
- "of may by large than the database itself,\n "
- "until it will be closed or reopened in read-write mode.\n");
-#endif
- verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_bootid.meta0.x, envinfo.mi_bootid.meta0.y);
- verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
- envinfo.mi_bootid.meta1.x, envinfo.mi_bootid.meta1.y);
- verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
- envinfo.mi_bootid.meta2.x, envinfo.mi_bootid.meta2.y);
- }
-
- if (stuck_meta >= 0) {
- if (verbose) {
- print(" - skip checking meta-pages since the %u"
- " is selected for verification\n",
- stuck_meta);
- print(" - transactions: recent %" PRIu64
- ", selected for verification %" PRIu64 ", lag %" PRIi64 "\n",
- envinfo.mi_recent_txnid, get_meta_txnid(stuck_meta),
- envinfo.mi_recent_txnid - get_meta_txnid(stuck_meta));
- }
- } else {
- if (verbose > 1)
- print(" - performs check for meta-pages clashes\n");
- if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign,
- envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) {
- print(" ! meta-%d and meta-%d are clashed\n", 0, 1);
- ++problems_meta;
- }
- if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign,
- envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) {
- print(" ! meta-%d and meta-%d are clashed\n", 1, 2);
- ++problems_meta;
- }
- if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign,
- envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) {
- print(" ! meta-%d and meta-%d are clashed\n", 2, 0);
- ++problems_meta;
- }
-
- const unsigned steady_meta_id = meta_recent(true);
- const uint64_t steady_meta_txnid = get_meta_txnid(steady_meta_id);
- const unsigned weak_meta_id = meta_recent(false);
- const uint64_t weak_meta_txnid = get_meta_txnid(weak_meta_id);
- if (envflags & MDBX_EXCLUSIVE) {
- if (verbose > 1)
- print(" - performs full check recent-txn-id with meta-pages\n");
- if (steady_meta_txnid != envinfo.mi_recent_txnid) {
- print(" ! steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64
- " != %" PRIi64 ")\n",
- steady_meta_id, steady_meta_txnid, envinfo.mi_recent_txnid);
- ++problems_meta;
- }
- } else if (write_locked) {
- if (verbose > 1)
- print(" - performs lite check recent-txn-id with meta-pages (not a "
- "monopolistic mode)\n");
- if (weak_meta_txnid != envinfo.mi_recent_txnid) {
- print(" ! weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64
- " != %" PRIi64 ")\n",
- weak_meta_id, weak_meta_txnid, envinfo.mi_recent_txnid);
- ++problems_meta;
- }
- } else if (verbose) {
- print(" - skip check recent-txn-id with meta-pages (monopolistic or "
- "read-write mode only)\n");
- }
- total_problems += problems_meta;
-
- if (verbose)
- print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64
- ", lag %" PRIi64 "\n",
- envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid,
- envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid);
- }
-
- if (!dont_traversal) {
- struct problem *saved_list;
- size_t traversal_problems;
- uint64_t empty_pages, lost_bytes;
-
- print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid);
- fflush(nullptr);
- walk.pagemap = mdbx_calloc((size_t)backed_pages, sizeof(*walk.pagemap));
- if (!walk.pagemap) {
- rc = errno ? errno : MDBX_ENOMEM;
- error("calloc() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- saved_list = problems_push();
- rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr,
- true /* always skip key ordering checking to avoid
- MDBX_CORRUPTED when using custom comparators */);
- traversal_problems = problems_pop(saved_list);
-
- if (rc) {
- if (rc != MDBX_EINTR || !check_user_break())
- error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc));
- goto bailout;
- }
-
- for (uint64_t n = 0; n < alloc_pages; ++n)
- if (!walk.pagemap[n])
- unused_pages += 1;
-
- empty_pages = lost_bytes = 0;
- for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name;
- ++dbi) {
- empty_pages += dbi->pages.empty;
- lost_bytes += dbi->lost_bytes;
- }
-
- if (verbose) {
- uint64_t total_page_bytes = walk.pgcount * envinfo.mi_dxb_pagesize;
- print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n",
- walk.pgcount, unused_pages);
- if (verbose > 1) {
- for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
- ++dbi) {
- print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total);
- if (dbi->pages.other && dbi->pages.other != dbi->pages.total)
- print(", other %" PRIu64, dbi->pages.other);
- if (dbi->pages.branch)
- print(", branch %" PRIu64, dbi->pages.branch);
- if (dbi->pages.large_count)
- print(", large %" PRIu64, dbi->pages.large_count);
- uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed;
- if (all_leaf) {
- print(", leaf %" PRIu64, all_leaf);
- if (verbose > 2 &&
- (dbi->pages.subleaf_dupsort | dbi->pages.leaf_dupfixed |
- dbi->pages.subleaf_dupfixed))
- print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64
- ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")",
- dbi->pages.leaf, dbi->pages.subleaf_dupsort,
- dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed);
- }
- print("\n");
- }
- }
-
- if (verbose > 1)
- print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64
- " (%.1f%%), unused "
- "%" PRIu64 " (%.1f%%)\n",
- total_page_bytes, walk.total_payload_bytes,
- walk.total_payload_bytes * 100.0 / total_page_bytes,
- total_page_bytes - walk.total_payload_bytes,
- (total_page_bytes - walk.total_payload_bytes) * 100.0 /
- total_page_bytes);
- if (verbose > 2) {
- for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
- ++dbi)
- if (dbi->pages.total) {
- uint64_t dbi_bytes = dbi->pages.total * envinfo.mi_dxb_pagesize;
- print(" %s: subtotal %" PRIu64 " bytes (%.1f%%),"
- " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)",
- dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes,
- dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes,
- dbi_bytes - dbi->payload_bytes,
- (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes);
- if (dbi->pages.empty)
- print(", %" PRIu64 " empty pages", dbi->pages.empty);
- if (dbi->lost_bytes)
- print(", %" PRIu64 " bytes lost", dbi->lost_bytes);
- print("\n");
- } else
- print(" %s: empty\n", dbi->name);
- }
- print(" - summary: average fill %.1f%%",
- walk.total_payload_bytes * 100.0 / total_page_bytes);
- if (empty_pages)
- print(", %" PRIu64 " empty pages", empty_pages);
- if (lost_bytes)
- print(", %" PRIu64 " bytes lost", lost_bytes);
- print(", %" PRIuPTR " problems\n", traversal_problems);
- }
- } else if (verbose) {
- print("Skipping b-tree walk...\n");
- fflush(nullptr);
- }
-
- if (!verbose)
- print("Iterating DBIs...\n");
- if (data_tree_problems) {
- print("Skip processing %s since tree is corrupted (%u problems)\n", "@MAIN",
- data_tree_problems);
- problems_maindb = data_tree_problems;
- } else
- problems_maindb = process_db(~0u, /* MAIN_DBI */ nullptr, nullptr, false);
-
- if (gc_tree_problems) {
- print("Skip processing %s since tree is corrupted (%u problems)\n", "@GC",
- gc_tree_problems);
- problems_freedb = gc_tree_problems;
- } else
- problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb, false);
-
- if (verbose) {
- uint64_t value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize;
- double percent = value / 100.0;
- print(" - space: %" PRIu64 " total pages", value);
- print(", backed %" PRIu64 " (%.1f%%)", backed_pages,
- backed_pages / percent);
- print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages,
- alloc_pages / percent);
-
- if (verbose > 1) {
- value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages;
- print(", remained %" PRIu64 " (%.1f%%)", value, value / percent);
-
- value = dont_traversal ? alloc_pages - gc_pages : walk.pgcount;
- print(", used %" PRIu64 " (%.1f%%)", value, value / percent);
-
- print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent);
-
- value = gc_pages - reclaimable_pages;
- print(", detained %" PRIu64 " (%.1f%%)", value, value / percent);
-
- print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages,
- reclaimable_pages / percent);
- }
-
- value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages +
- reclaimable_pages;
- print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent);
- }
-
- if (problems_maindb == 0 && problems_freedb == 0) {
- if (!dont_traversal &&
- (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) {
- if (walk.pgcount != alloc_pages - gc_pages) {
- error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64
- "(allocated - GC))\n",
- walk.pgcount, alloc_pages - gc_pages);
- }
- if (unused_pages != gc_pages) {
- error("gc pages mismatch (%" PRIu64 "(expected) != %" PRIu64 "(GC))\n",
- unused_pages, gc_pages);
- }
- } else if (verbose) {
- print(" - skip check used and gc pages (btree-traversal with "
- "monopolistic or read-write mode only)\n");
- }
-
- if (!process_db(MAIN_DBI, nullptr, handle_maindb, true)) {
- if (!userdb_count && verbose)
- print(" - does not contain multiple databases\n");
- }
- }
-
- if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal &&
- (envflags & MDBX_RDONLY) == 0 && !only_subdb && stuck_meta < 0 &&
- get_meta_txnid(meta_recent(true)) < envinfo.mi_recent_txnid) {
- print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64
- "\n",
- envinfo.mi_recent_txnid);
- fflush(nullptr);
- if (write_locked) {
- mdbx_txn_unlock(env);
- write_locked = false;
- }
- rc = mdbx_env_sync_ex(env, true, false);
- if (rc != MDBX_SUCCESS)
- error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc));
- else {
- total_problems -= 1;
- problems_meta -= 1;
- }
- }
-
- if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb &&
- (envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) {
- const bool successful_check = (rc | total_problems | problems_meta) == 0;
- if (successful_check || force_turn_meta) {
- fflush(nullptr);
- print(" = Performing turn to the specified meta-page (%d) due to %s!\n",
- stuck_meta,
- successful_check ? "successful check" : "the -T option was given");
- fflush(nullptr);
- rc = mdbx_env_turn_for_recovery(env, stuck_meta);
- if (rc != MDBX_SUCCESS)
- error("mdbx_env_turn_for_recovery() failed, error %d %s\n", rc,
- mdbx_strerror(rc));
- } else {
- print(" = Skipping turn to the specified meta-page (%d) due to "
- "unsuccessful check!\n",
- stuck_meta);
- }
- }
-
-bailout:
- if (txn)
- mdbx_txn_abort(txn);
- if (write_locked) {
- mdbx_txn_unlock(env);
- write_locked = false;
- }
- if (env) {
- const bool dont_sync = rc != 0 || total_problems;
- mdbx_env_close_ex(env, dont_sync);
- }
- fflush(nullptr);
- if (rc) {
- if (rc < 0)
- return user_break ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS;
- return EXIT_FAILURE_MDBX;
- }
-
-#if defined(_WIN32) || defined(_WIN64)
- timestamp_finish = GetMilliseconds();
- elapsed = (timestamp_finish - timestamp_start) * 1e-3;
-#else
- if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) {
- rc = errno;
- error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc));
- return EXIT_FAILURE_SYS;
- }
- elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec +
- (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9;
-#endif /* !WINDOWS */
-
- if (total_problems) {
- print("Total %u error%s detected, elapsed %.3f seconds.\n", total_problems,
- (total_problems > 1) ? "s are" : " is", elapsed);
- if (problems_meta || problems_maindb || problems_freedb)
- return EXIT_FAILURE_CHECK_MAJOR;
- return EXIT_FAILURE_CHECK_MINOR;
- }
- print("No error is detected, elapsed %.3f seconds\n", elapsed);
- return EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/mdbx_copy.c b/3rdparty/libmdbx/src/mdbx_copy.c
deleted file mode 100644
index 4b40b558ba305c275a4f563a1ca8a490e56b2640..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_copy.c
+++ /dev/null
@@ -1,142 +0,0 @@
-/* mdbx_copy.c - memory-mapped database backup tool */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = true;
- return true;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-static void usage(const char *prog) {
- fprintf(stderr,
- "usage: %s [-V] [-q] [-c] src_path [dest_path]\n"
- " -V\t\tprint version and exit\n"
- " -q\t\tbe quiet\n"
- " -c\t\tenable compactification (skip unused pages)\n"
- " src_path\tsource database\n"
- " dest_path\tdestination (stdout if not specified)\n",
- prog);
- exit(EXIT_FAILURE);
-}
-
-int main(int argc, char *argv[]) {
- int rc;
- MDBX_env *env = NULL;
- const char *progname = argv[0], *act;
- unsigned flags = MDBX_RDONLY;
- unsigned cpflags = 0;
- bool quiet = false;
-
- for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
- if (argv[1][1] == 'n' && argv[1][2] == '\0')
- flags |= MDBX_NOSUBDIR;
- else if (argv[1][1] == 'c' && argv[1][2] == '\0')
- cpflags |= MDBX_CP_COMPACT;
- else if (argv[1][1] == 'q' && argv[1][2] == '\0')
- quiet = true;
- else if ((argv[1][1] == 'h' && argv[1][2] == '\0') ||
- strcmp(argv[1], "--help") == 0)
- usage(progname);
- else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
- printf("mdbx_copy version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- } else
- argc = 0;
- }
-
- if (argc < 2 || argc > 3)
- usage(progname);
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- if (!quiet) {
- fprintf((argc == 2) ? stderr : stdout,
- "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]);
- fflush(NULL);
- }
-
- act = "opening environment";
- rc = mdbx_env_create(&env);
- if (rc == MDBX_SUCCESS)
- rc = mdbx_env_open(env, argv[1], flags, 0);
-
- if (rc == MDBX_SUCCESS) {
- act = "copying";
- if (argc == 2) {
- mdbx_filehandle_t fd;
-#if defined(_WIN32) || defined(_WIN64)
- fd = GetStdHandle(STD_OUTPUT_HANDLE);
-#else
- fd = fileno(stdout);
-#endif
- rc = mdbx_env_copy2fd(env, fd, cpflags);
- } else
- rc = mdbx_env_copy(env, argv[2], cpflags);
- }
- if (rc)
- fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
- mdbx_strerror(rc));
- mdbx_env_close(env);
-
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/mdbx_drop.c b/3rdparty/libmdbx/src/mdbx_drop.c
deleted file mode 100644
index 9b0a18b539cab56a9cd5b98f3b21375a73cf2ff4..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_drop.c
+++ /dev/null
@@ -1,197 +0,0 @@
-/* mdbx_drop.c - memory-mapped database delete tool */
-
-/*
- * Copyright 2021 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- *
- * Copyright 2016-2022 Howard Chu, Symas Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-#include <ctype.h>
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = true;
- return true;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-static char *prog;
-bool quiet = false;
-static void usage(void) {
- fprintf(stderr,
- "usage: %s [-V] [-q] [-d] [-s name] dbpath\n"
- " -V\t\tprint version and exit\n"
- " -q\t\tbe quiet\n"
- " -d\t\tdelete the specified database, don't just empty it\n"
- " -s name\tdrop the specified named subDB\n"
- " \t\tby default empty the main DB\n",
- prog);
- exit(EXIT_FAILURE);
-}
-
-static void error(const char *func, int rc) {
- if (!quiet)
- fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
- mdbx_strerror(rc));
-}
-
-int main(int argc, char *argv[]) {
- int i, rc;
- MDBX_env *env;
- MDBX_txn *txn;
- MDBX_dbi dbi;
- char *envname = nullptr;
- char *subname = nullptr;
- int envflags = MDBX_ACCEDE;
- bool delete = false;
-
- prog = argv[0];
- if (argc < 2)
- usage();
-
- while ((i = getopt(argc, argv,
- "d"
- "s:"
- "n"
- "q"
- "V")) != EOF) {
- switch (i) {
- case 'V':
- printf("mdbx_drop version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- case 'q':
- quiet = true;
- break;
- case 'd':
- delete = true;
- break;
- case 'n':
- break;
- case 's':
- subname = optarg;
- break;
- default:
- usage();
- }
- }
-
- if (optind != argc - 1)
- usage();
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- envname = argv[optind];
- if (!quiet) {
- printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, envname, subname ? subname : "@MAIN");
- fflush(nullptr);
- }
-
- rc = mdbx_env_create(&env);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_create", rc);
- return EXIT_FAILURE;
- }
-
- if (subname) {
- rc = mdbx_env_set_maxdbs(env, 2);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_set_maxdbs", rc);
- goto env_close;
- }
- }
-
- rc = mdbx_env_open(env, envname, envflags, 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_open", rc);
- goto env_close;
- }
-
- rc = mdbx_txn_begin(env, NULL, 0, &txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_begin", rc);
- goto env_close;
- }
-
- rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_open", rc);
- goto txn_abort;
- }
-
- rc = mdbx_drop(txn, dbi, delete);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_drop", rc);
- goto txn_abort;
- }
-
- rc = mdbx_txn_commit(txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_commit", rc);
- goto txn_abort;
- }
- txn = nullptr;
-
-txn_abort:
- if (txn)
- mdbx_txn_abort(txn);
-env_close:
- mdbx_env_close(env);
-
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/mdbx_dump.c b/3rdparty/libmdbx/src/mdbx_dump.c
deleted file mode 100644
index 6eec6fc361111f414e65149640cc5a1eb8636990..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_dump.c
+++ /dev/null
@@ -1,502 +0,0 @@
-/* mdbx_dump.c - memory-mapped database dump tool */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-#include <ctype.h>
-
-#define PRINT 1
-#define GLOBAL 2
-static int mode = GLOBAL;
-
-typedef struct flagbit {
- int bit;
- char *name;
-} flagbit;
-
-flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
- {MDBX_DUPSORT, "dupsort"},
- {MDBX_INTEGERKEY, "integerkey"},
- {MDBX_DUPFIXED, "dupfixed"},
- {MDBX_INTEGERDUP, "integerdup"},
- {MDBX_REVERSEDUP, "reversedup"},
- {0, nullptr}};
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = true;
- return true;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-static const char hexc[] = "0123456789abcdef";
-
-static void dumpbyte(unsigned char c) {
- putchar(hexc[c >> 4]);
- putchar(hexc[c & 0xf]);
-}
-
-static void text(MDBX_val *v) {
- unsigned char *c, *end;
-
- putchar(' ');
- c = v->iov_base;
- end = c + v->iov_len;
- while (c < end) {
- if (isprint(*c) && *c != '\\') {
- putchar(*c);
- } else {
- putchar('\\');
- dumpbyte(*c);
- }
- c++;
- }
- putchar('\n');
-}
-
-static void dumpval(MDBX_val *v) {
- unsigned char *c, *end;
-
- putchar(' ');
- c = v->iov_base;
- end = c + v->iov_len;
- while (c < end)
- dumpbyte(*c++);
- putchar('\n');
-}
-
-bool quiet = false, rescue = false;
-const char *prog;
-static void error(const char *func, int rc) {
- if (!quiet)
- fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
- mdbx_strerror(rc));
-}
-
-/* Dump in BDB-compatible format */
-static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
- unsigned int flags;
- int rc = mdbx_dbi_flags(txn, dbi, &flags);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_flags", rc);
- return rc;
- }
-
- MDBX_stat ms;
- rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_stat", rc);
- return rc;
- }
-
- MDBX_envinfo info;
- rc = mdbx_env_info_ex(mdbx_txn_env(txn), txn, &info, sizeof(info));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_info_ex", rc);
- return rc;
- }
-
- printf("VERSION=3\n");
- if (mode & GLOBAL) {
- mode -= GLOBAL;
- if (info.mi_geo.upper != info.mi_geo.lower)
- printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
- ",g%" PRIu64 "\n",
- info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper,
- info.mi_geo.shrink, info.mi_geo.grow);
- printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper);
- printf("maxreaders=%u\n", info.mi_maxreaders);
-
- MDBX_canary canary;
- rc = mdbx_canary_get(txn, &canary);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_canary_get", rc);
- return rc;
- }
- if (canary.v)
- printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n",
- canary.v, canary.x, canary.y, canary.z);
- }
- printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
- if (name)
- printf("database=%s\n", name);
- printf("type=btree\n");
- printf("db_pagesize=%u\n", ms.ms_psize);
- /* if (ms.ms_mod_txnid)
- printf("txnid=%" PRIaTXN "\n", ms.ms_mod_txnid);
- else if (!name)
- printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */
-
- printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED |
- MDBX_INTEGERDUP | MDBX_REVERSEDUP))
- ? 1
- : 0);
- for (int i = 0; dbflags[i].bit; i++)
- if (flags & dbflags[i].bit)
- printf("%s=1\n", dbflags[i].name);
-
- uint64_t sequence;
- rc = mdbx_dbi_sequence(txn, dbi, &sequence, 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_sequence", rc);
- return rc;
- }
- if (sequence)
- printf("sequence=%" PRIu64 "\n", sequence);
-
- printf("HEADER=END\n"); /*-------------------------------------------------*/
-
- MDBX_cursor *cursor;
- MDBX_val key, data;
- rc = mdbx_cursor_open(txn, dbi, &cursor);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_open", rc);
- return rc;
- }
- if (MDBX_DEBUG > 0 && rescue) {
- cursor->mc_flags |= C_SKIPORD;
- if (cursor->mc_xcursor)
- cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
- }
-
- while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
- MDBX_SUCCESS) {
- if (user_break) {
- rc = MDBX_EINTR;
- break;
- }
- if (mode & PRINT) {
- text(&key);
- text(&data);
- } else {
- dumpval(&key);
- dumpval(&data);
- }
- }
- printf("DATA=END\n");
- if (rc == MDBX_NOTFOUND)
- rc = MDBX_SUCCESS;
- if (unlikely(rc != MDBX_SUCCESS))
- error("mdbx_cursor_get", rc);
-
- mdbx_cursor_close(cursor);
- return rc;
-}
-
-static void usage(void) {
- fprintf(stderr,
- "usage: %s [-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s subdb] "
- "dbpath\n"
- " -V\t\tprint version and exit\n"
- " -q\t\tbe quiet\n"
- " -f\t\twrite to file instead of stdout\n"
- " -l\t\tlist subDBs and exit\n"
- " -p\t\tuse printable characters\n"
- " -r\t\trescue mode (ignore errors to dump corrupted DB)\n"
- " -a\t\tdump main DB and all subDBs\n"
- " -s name\tdump only the specified named subDB\n"
- " \t\tby default dump only the main DB\n",
- prog);
- exit(EXIT_FAILURE);
-}
-
-static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
- return (a->iov_len == b->iov_len &&
- memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
- ? 0
- : 1;
-}
-
-int main(int argc, char *argv[]) {
- int i, rc;
- MDBX_env *env;
- MDBX_txn *txn;
- MDBX_dbi dbi;
- prog = argv[0];
- char *envname;
- char *subname = nullptr, *buf4free = nullptr;
- unsigned envflags = 0;
- bool alldbs = false, list = false;
-
- if (argc < 2)
- usage();
-
- while ((i = getopt(argc, argv,
- "a"
- "f:"
- "l"
- "n"
- "p"
- "s:"
- "V"
- "r"
- "q")) != EOF) {
- switch (i) {
- case 'V':
- printf("mdbx_dump version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- case 'l':
- list = true;
- /*FALLTHROUGH*/;
- __fallthrough;
- case 'a':
- if (subname)
- usage();
- alldbs = true;
- break;
- case 'f':
- if (freopen(optarg, "w", stdout) == nullptr) {
- fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
- mdbx_strerror(errno));
- exit(EXIT_FAILURE);
- }
- break;
- case 'n':
- break;
- case 'p':
- mode |= PRINT;
- break;
- case 's':
- if (alldbs)
- usage();
- subname = optarg;
- break;
- case 'q':
- quiet = true;
- break;
- case 'r':
- rescue = true;
- break;
- default:
- usage();
- }
- }
-
- if (optind != argc - 1)
- usage();
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- envname = argv[optind];
- if (!quiet) {
- fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, envname);
- fflush(nullptr);
- }
-
- rc = mdbx_env_create(&env);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_create", rc);
- return EXIT_FAILURE;
- }
-
- if (alldbs || subname) {
- rc = mdbx_env_set_maxdbs(env, 2);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_set_maxdbs", rc);
- goto env_close;
- }
- }
-
- rc = mdbx_env_open(
- env, envname,
- envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE : MDBX_RDONLY), 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_open", rc);
- goto env_close;
- }
-
- rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_begin", rc);
- goto env_close;
- }
-
- rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_open", rc);
- goto txn_abort;
- }
-
- if (alldbs) {
- assert(dbi == MAIN_DBI);
-
- MDBX_cursor *cursor;
- rc = mdbx_cursor_open(txn, MAIN_DBI, &cursor);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_open", rc);
- goto txn_abort;
- }
- if (MDBX_DEBUG > 0 && rescue) {
- cursor->mc_flags |= C_SKIPORD;
- if (cursor->mc_xcursor)
- cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD;
- }
-
- bool have_raw = false;
- int count = 0;
- MDBX_val key;
- while (MDBX_SUCCESS ==
- (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
- if (user_break) {
- rc = MDBX_EINTR;
- break;
- }
-
- if (memchr(key.iov_base, '\0', key.iov_len))
- continue;
- subname = mdbx_realloc(buf4free, key.iov_len + 1);
- if (!subname) {
- rc = MDBX_ENOMEM;
- break;
- }
-
- buf4free = subname;
- memcpy(subname, key.iov_base, key.iov_len);
- subname[key.iov_len] = '\0';
-
- MDBX_dbi sub_dbi;
- rc = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi,
- rescue ? equal_or_greater : nullptr,
- rescue ? equal_or_greater : nullptr);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_INCOMPATIBLE) {
- have_raw = true;
- continue;
- }
- error("mdbx_dbi_open", rc);
- if (!rescue)
- break;
- } else {
- count++;
- if (list) {
- printf("%s\n", subname);
- } else {
- rc = dump_sdb(txn, sub_dbi, subname);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (!rescue)
- break;
- if (!quiet)
- fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog,
- envname, mdbx_strerror(rc), subname);
- /* Here is a hack for rescue mode, don't do that:
- * - we should restart transaction in case error due
- * database corruption;
- * - but we won't close cursor, reopen and re-positioning it
- * for new a transaction;
- * - this is possible since DB is opened in read-only exclusive
- * mode and transaction is the same, i.e. has the same address
- * and so on. */
- rc = mdbx_txn_reset(txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_reset", rc);
- goto env_close;
- }
- rc = mdbx_txn_renew(txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_renew", rc);
- goto env_close;
- }
- }
- }
- rc = mdbx_dbi_close(env, sub_dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_close", rc);
- break;
- }
- }
- }
- mdbx_cursor_close(cursor);
- cursor = nullptr;
-
- if (have_raw && (!count /* || rescue */))
- rc = dump_sdb(txn, MAIN_DBI, nullptr);
- else if (!count) {
- if (!quiet)
- fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
- envname);
- rc = MDBX_NOTFOUND;
- }
- } else {
- rc = dump_sdb(txn, dbi, subname);
- }
-
- switch (rc) {
- case MDBX_NOTFOUND:
- rc = MDBX_SUCCESS;
- case MDBX_SUCCESS:
- break;
- case MDBX_EINTR:
- if (!quiet)
- fprintf(stderr, "Interrupted by signal/user\n");
- break;
- default:
- if (unlikely(rc != MDBX_SUCCESS))
- error("mdbx_cursor_get", rc);
- }
-
- mdbx_dbi_close(env, dbi);
-txn_abort:
- mdbx_txn_abort(txn);
-env_close:
- mdbx_env_close(env);
- free(buf4free);
-
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/mdbx_load.c b/3rdparty/libmdbx/src/mdbx_load.c
deleted file mode 100644
index 10e54750f9c37aabf47e662f41f0d08c8c0dd412..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_load.c
+++ /dev/null
@@ -1,873 +0,0 @@
-/* mdbx_load.c - memory-mapped database load tool */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-#include <ctype.h>
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = true;
- return true;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-static char *prog;
-static bool quiet = false;
-static size_t lineno;
-static void error(const char *func, int rc) {
- if (!quiet) {
- if (lineno)
- fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n",
- prog, lineno, func, rc, mdbx_strerror(rc));
- else
- fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
- mdbx_strerror(rc));
- }
-}
-
-static char *valstr(char *line, const char *item) {
- const size_t len = strlen(item);
- if (strncmp(line, item, len) != 0)
- return nullptr;
- if (line[len] != '=') {
- if (line[len] > ' ')
- return nullptr;
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog,
- lineno, item);
- exit(EXIT_FAILURE);
- }
- char *ptr = strchr(line, '\n');
- if (ptr)
- *ptr = '\0';
- return line + len + 1;
-}
-
-static bool valnum(char *line, const char *item, uint64_t *value) {
- char *str = valstr(line, item);
- if (!str)
- return false;
-
- char *end = nullptr;
- *value = strtoull(str, &end, 0);
- if (end && *end) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n",
- prog, lineno, item);
- exit(EXIT_FAILURE);
- }
- return true;
-}
-
-static bool valbool(char *line, const char *item, bool *value) {
- uint64_t u64;
- if (!valnum(line, item, &u64))
- return false;
-
- if (u64 > 1) {
- if (!quiet)
- fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n",
- prog, lineno, item);
- exit(EXIT_FAILURE);
- }
- *value = u64 != 0;
- return true;
-}
-
-/*----------------------------------------------------------------------------*/
-
-static char *subname = nullptr;
-static int dbi_flags;
-static txnid_t txnid;
-static uint64_t sequence;
-static MDBX_canary canary;
-static MDBX_envinfo envinfo;
-
-#define PRINT 1
-#define NOHDR 2
-#define GLOBAL 4
-static int mode = GLOBAL;
-
-static MDBX_val kbuf, dbuf;
-
-#define STRLENOF(s) (sizeof(s) - 1)
-
-typedef struct flagbit {
- unsigned bit;
- unsigned len;
- char *name;
-} flagbit;
-
-#define S(s) STRLENOF(s), s
-
-flagbit dbflags[] = {
- {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")},
- {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")},
- {MDBX_DUPFIXED, S("dupfixed")}, {MDBX_INTEGERDUP, S("integerdup")},
- {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}};
-
-static int readhdr(void) {
- /* reset parameters */
- if (subname) {
- free(subname);
- subname = nullptr;
- }
- dbi_flags = 0;
- txnid = 0;
- sequence = 0;
-
- while (true) {
- errno = 0;
- if (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) == nullptr)
- return errno ? errno : EOF;
- if (user_break)
- return MDBX_EINTR;
-
- lineno++;
- uint64_t u64;
-
- if (valnum(dbuf.iov_base, "VERSION", &u64)) {
- if (u64 != 3) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unsupported value %" PRIu64
- " for %s\n",
- prog, lineno, u64, "VERSION");
- exit(EXIT_FAILURE);
- }
- continue;
- }
-
- if (valnum(dbuf.iov_base, "db_pagesize", &u64)) {
- if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore value %" PRIu64
- " for '%s' in non-global context\n",
- prog, lineno, u64, "db_pagesize");
- } else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64
- " for %s\n",
- prog, lineno, u64, "db_pagesize");
- } else
- envinfo.mi_dxb_pagesize = (uint32_t)u64;
- continue;
- }
-
- char *str = valstr(dbuf.iov_base, "format");
- if (str) {
- if (strcmp(str, "print") == 0) {
- mode |= PRINT;
- continue;
- }
- if (strcmp(str, "bytevalue") == 0) {
- mode &= ~PRINT;
- continue;
- }
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog,
- lineno, str, "format");
- exit(EXIT_FAILURE);
- }
-
- str = valstr(dbuf.iov_base, "database");
- if (str) {
- if (*str) {
- free(subname);
- subname = mdbx_strdup(str);
- if (!subname) {
- if (!quiet)
- perror("strdup()");
- exit(EXIT_FAILURE);
- }
- }
- continue;
- }
-
- str = valstr(dbuf.iov_base, "type");
- if (str) {
- if (strcmp(str, "btree") != 0) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n",
- prog, lineno, str, "type");
- free(subname);
- exit(EXIT_FAILURE);
- }
- continue;
- }
-
- if (valnum(dbuf.iov_base, "mapaddr", &u64)) {
- if (u64) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
- " for %s\n",
- prog, lineno, u64, "mapaddr");
- }
- continue;
- }
-
- if (valnum(dbuf.iov_base, "mapsize", &u64)) {
- if (!(mode & GLOBAL)) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore value %" PRIu64
- " for '%s' in non-global context\n",
- prog, lineno, u64, "mapsize");
- } else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
- " for %s\n",
- prog, lineno, u64, "mapsize");
- } else
- envinfo.mi_mapsize = (size_t)u64;
- continue;
- }
-
- if (valnum(dbuf.iov_base, "maxreaders", &u64)) {
- if (!(mode & GLOBAL)) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore value %" PRIu64
- " for '%s' in non-global context\n",
- prog, lineno, u64, "maxreaders");
- } else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
- " for %s\n",
- prog, lineno, u64, "maxreaders");
- } else
- envinfo.mi_maxreaders = (int)u64;
- continue;
- }
-
- if (valnum(dbuf.iov_base, "txnid", &u64)) {
- if (u64 < MIN_TXNID || u64 > MAX_TXNID) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
- " for %s\n",
- prog, lineno, u64, "txnid");
- } else
- txnid = u64;
- continue;
- }
-
- if (valnum(dbuf.iov_base, "sequence", &u64)) {
- sequence = u64;
- continue;
- }
-
- str = valstr(dbuf.iov_base, "geometry");
- if (str) {
- if (!(mode & GLOBAL)) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore values %s"
- " for '%s' in non-global context\n",
- prog, lineno, str, "geometry");
- } else if (sscanf(str,
- "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
- ",g%" PRIu64,
- &envinfo.mi_geo.lower, &envinfo.mi_geo.current,
- &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink,
- &envinfo.mi_geo.grow) != 5) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
- prog, lineno, "geometry");
- exit(EXIT_FAILURE);
- }
- continue;
- }
-
- str = valstr(dbuf.iov_base, "canary");
- if (str) {
- if (!(mode & GLOBAL)) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": ignore values %s"
- " for '%s' in non-global context\n",
- prog, lineno, str, "canary");
- } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64,
- &canary.v, &canary.x, &canary.y, &canary.z) != 4) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
- prog, lineno, "canary");
- exit(EXIT_FAILURE);
- }
- continue;
- }
-
- for (int i = 0; dbflags[i].bit; i++) {
- bool value = false;
- if (valbool(dbuf.iov_base, dbflags[i].name, &value)) {
- if (value)
- dbi_flags |= dbflags[i].bit;
- else
- dbi_flags &= ~dbflags[i].bit;
- goto next;
- }
- }
-
- str = valstr(dbuf.iov_base, "HEADER");
- if (str) {
- if (strcmp(str, "END") == 0)
- return MDBX_SUCCESS;
- }
-
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
- prog, lineno, (char *)dbuf.iov_base);
- next:;
- }
- return EOF;
-}
-
-static int badend(void) {
- if (!quiet)
- fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
- lineno);
- return errno ? errno : MDBX_ENODATA;
-}
-
-static __inline int unhex(unsigned char *c2) {
- int x, c;
- x = *c2++ & 0x4f;
- if (x & 0x40)
- x -= 55;
- c = x << 4;
- x = *c2 & 0x4f;
- if (x & 0x40)
- x -= 55;
- c |= x;
- return c;
-}
-
-__hot static int readline(MDBX_val *out, MDBX_val *buf) {
- unsigned char *c1, *c2, *end;
- size_t len, l2;
- int c;
-
- if (user_break)
- return MDBX_EINTR;
-
- errno = 0;
- if (!(mode & NOHDR)) {
- c = fgetc(stdin);
- if (c == EOF)
- return errno ? errno : EOF;
- if (c != ' ') {
- lineno++;
- errno = 0;
- if (fgets(buf->iov_base, (int)buf->iov_len, stdin)) {
- if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
- return EOF;
- }
- return badend();
- }
- }
- if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == nullptr)
- return errno ? errno : EOF;
- lineno++;
-
- c1 = buf->iov_base;
- len = strlen((char *)c1);
- l2 = len;
-
- /* Is buffer too short? */
- while (c1[len - 1] != '\n') {
- buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
- if (!buf->iov_base) {
- if (!quiet)
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog,
- lineno);
- return MDBX_ENOMEM;
- }
- c1 = buf->iov_base;
- c1 += l2;
- errno = 0;
- if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == nullptr)
- return errno ? errno : EOF;
- buf->iov_len *= 2;
- len = strlen((char *)c1);
- l2 += len;
- }
- c1 = c2 = buf->iov_base;
- len = l2;
- c1[--len] = '\0';
- end = c1 + len;
-
- if (mode & PRINT) {
- while (c2 < end) {
- if (unlikely(*c2 == '\\')) {
- if (c2[1] == '\\') {
- *c1++ = '\\';
- } else {
- if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2]))
- return badend();
- *c1++ = (char)unhex(++c2);
- }
- c2 += 2;
- } else {
- /* copies are redundant when no escapes were used */
- *c1++ = *c2++;
- }
- }
- } else {
- /* odd length not allowed */
- if (len & 1)
- return badend();
- while (c2 < end) {
- if (!isxdigit(*c2) || !isxdigit(c2[1]))
- return badend();
- *c1++ = (char)unhex(c2);
- c2 += 2;
- }
- }
- c2 = out->iov_base = buf->iov_base;
- out->iov_len = c1 - c2;
-
- return MDBX_SUCCESS;
-}
-
-static void usage(void) {
- fprintf(stderr,
- "usage: %s "
- "[-V] [-q] [-a] [-f file] [-s name] [-N] [-p] [-T] [-r] [-n] dbpath\n"
- " -V\t\tprint version and exit\n"
- " -q\t\tbe quiet\n"
- " -a\t\tappend records in input order (required for custom "
- "comparators)\n"
- " -f file\tread from file instead of stdin\n"
- " -s name\tload into specified named subDB\n"
- " -N\t\tdon't overwrite existing records when loading, just skip "
- "ones\n"
- " -p\t\tpurge subDB before loading\n"
- " -T\t\tread plaintext\n"
- " -r\t\trescue mode (ignore errors to load corrupted DB dump)\n"
- " -n\t\tdon't use subdirectory for newly created database "
- "(MDBX_NOSUBDIR)\n",
- prog);
- exit(EXIT_FAILURE);
-}
-
-static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
- return (a->iov_len == b->iov_len &&
- memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
- ? 0
- : 1;
-}
-
-int main(int argc, char *argv[]) {
- int i, rc;
- MDBX_env *env = nullptr;
- MDBX_txn *txn = nullptr;
- MDBX_cursor *mc = nullptr;
- MDBX_dbi dbi;
- char *envname = nullptr;
- int envflags = MDBX_SAFE_NOSYNC | MDBX_ACCEDE, putflags = MDBX_UPSERT;
- bool rescue = false;
- bool purge = false;
-
- prog = argv[0];
- if (argc < 2)
- usage();
-
- while ((i = getopt(argc, argv,
- "a"
- "f:"
- "n"
- "s:"
- "N"
- "p"
- "T"
- "V"
- "r"
- "q")) != EOF) {
- switch (i) {
- case 'V':
- printf("mdbx_load version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- case 'a':
- putflags |= MDBX_APPEND;
- break;
- case 'f':
- if (freopen(optarg, "r", stdin) == nullptr) {
- if (!quiet)
- fprintf(stderr, "%s: %s: open: %s\n", prog, optarg,
- mdbx_strerror(errno));
- exit(EXIT_FAILURE);
- }
- break;
- case 'n':
- envflags |= MDBX_NOSUBDIR;
- break;
- case 's':
- subname = mdbx_strdup(optarg);
- break;
- case 'N':
- putflags |= MDBX_NOOVERWRITE | MDBX_NODUPDATA;
- break;
- case 'p':
- purge = true;
- break;
- case 'T':
- mode |= NOHDR | PRINT;
- break;
- case 'q':
- quiet = true;
- break;
- case 'r':
- rescue = true;
- break;
- default:
- usage();
- }
- }
-
- if (optind != argc - 1)
- usage();
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- envname = argv[optind];
- if (!quiet)
- printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, envname);
- fflush(nullptr);
-
- dbuf.iov_len = 4096;
- dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
- if (!dbuf.iov_base) {
- rc = MDBX_ENOMEM;
- error("value-buffer", rc);
- goto env_close;
- }
-
- /* read first header for mapsize= */
- if (!(mode & NOHDR)) {
- rc = readhdr();
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == EOF)
- rc = MDBX_ENODATA;
- error("readheader", rc);
- goto env_close;
- }
- }
-
- rc = mdbx_env_create(&env);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_create", rc);
- return EXIT_FAILURE;
- }
-
- mdbx_env_set_maxdbs(env, 2);
- if (envinfo.mi_maxreaders) {
- rc = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_set_maxreaders", rc);
- goto env_close;
- }
- }
-
- if (envinfo.mi_geo.current | envinfo.mi_mapsize) {
- if (envinfo.mi_geo.current) {
- rc = mdbx_env_set_geometry(
- env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current,
- (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink,
- (intptr_t)envinfo.mi_geo.grow,
- envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
- } else {
- if (envinfo.mi_mapsize > MAX_MAPSIZE) {
- if (!quiet)
- fprintf(
- stderr,
- "Database size is too large for current system (mapsize=%" PRIu64
- " is great than system-limit %zu)\n",
- envinfo.mi_mapsize, (size_t)MAX_MAPSIZE);
- goto env_close;
- }
- rc = mdbx_env_set_geometry(
- env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize,
- (intptr_t)envinfo.mi_mapsize, 0, 0,
- envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
- }
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_set_geometry", rc);
- goto env_close;
- }
- }
-
- rc = mdbx_env_open(env, envname, envflags, 0664);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_open", rc);
- goto env_close;
- }
-
- kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, 0) + 1;
- if (kbuf.iov_len >= INTPTR_MAX / 2) {
- if (!quiet)
- fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n",
- kbuf.iov_len);
- goto env_close;
- }
-
- kbuf.iov_base = malloc(kbuf.iov_len);
- if (!kbuf.iov_base) {
- rc = MDBX_ENOMEM;
- error("key-buffer", rc);
- goto env_close;
- }
-
- while (rc == MDBX_SUCCESS) {
- if (user_break) {
- rc = MDBX_EINTR;
- break;
- }
-
- rc = mdbx_txn_begin(env, nullptr, 0, &txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_begin", rc);
- goto env_close;
- }
-
- if (mode & GLOBAL) {
- mode -= GLOBAL;
- if (canary.v | canary.x | canary.y | canary.z) {
- rc = mdbx_canary_put(txn, &canary);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_canary_put", rc);
- goto txn_abort;
- }
- }
- }
-
- const char *const dbi_name = subname ? subname : "@MAIN";
- rc =
- mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
- (putflags & MDBX_APPEND) ? equal_or_greater : nullptr,
- (putflags & MDBX_APPEND) ? equal_or_greater : nullptr);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_open_ex", rc);
- goto txn_abort;
- }
-
- uint64_t present_sequence;
- rc = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_sequence", rc);
- goto txn_abort;
- }
- if (present_sequence > sequence) {
- if (!quiet)
- fprintf(stderr,
- "present sequence for '%s' value (%" PRIu64
- ") is greater than loaded (%" PRIu64 ")\n",
- dbi_name, present_sequence, sequence);
- rc = MDBX_RESULT_TRUE;
- goto txn_abort;
- }
- if (present_sequence < sequence) {
- rc = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_sequence", rc);
- goto txn_abort;
- }
- }
-
- if (purge) {
- rc = mdbx_drop(txn, dbi, false);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_drop", rc);
- goto txn_abort;
- }
- }
-
- if (putflags & MDBX_APPEND)
- putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP
- : putflags & ~MDBX_APPENDDUP;
-
- rc = mdbx_cursor_open(txn, dbi, &mc);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_open", rc);
- goto txn_abort;
- }
-
- int batch = 0;
- while (rc == MDBX_SUCCESS) {
- MDBX_val key, data;
- rc = readline(&key, &kbuf);
- if (rc == EOF)
- break;
-
- if (rc == MDBX_SUCCESS)
- rc = readline(&data, &dbuf);
- if (rc) {
- if (!quiet)
- fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
- prog, lineno);
- goto txn_abort;
- }
-
- rc = mdbx_cursor_put(mc, &key, &data, putflags);
- if (rc == MDBX_KEYEXIST && putflags)
- continue;
- if (rc == MDBX_BAD_VALSIZE && rescue) {
- if (!quiet)
- fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno,
- mdbx_strerror(rc));
- continue;
- }
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_put", rc);
- goto txn_abort;
- }
- batch++;
-
- MDBX_txn_info txn_info;
- rc = mdbx_txn_info(txn, &txn_info, false);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_info", rc);
- goto txn_abort;
- }
-
- if (batch == 10000 || txn_info.txn_space_dirty > MEGABYTE * 256) {
- rc = mdbx_txn_commit(txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_commit", rc);
- goto env_close;
- }
- batch = 0;
-
- rc = mdbx_txn_begin(env, nullptr, 0, &txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_begin", rc);
- goto env_close;
- }
- rc = mdbx_cursor_bind(txn, mc, dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_bind", rc);
- goto txn_abort;
- }
- }
- }
-
- mdbx_cursor_close(mc);
- mc = nullptr;
- rc = mdbx_txn_commit(txn);
- txn = nullptr;
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_commit", rc);
- goto env_close;
- }
- if (subname) {
- assert(dbi != MAIN_DBI);
- rc = mdbx_dbi_close(env, dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_close", rc);
- goto env_close;
- }
- } else {
- assert(dbi == MAIN_DBI);
- }
-
- /* try read next header */
- if (!(mode & NOHDR))
- rc = readhdr();
- else if (ferror(stdin) || feof(stdin))
- break;
- }
-
- switch (rc) {
- case EOF:
- rc = MDBX_SUCCESS;
- case MDBX_SUCCESS:
- break;
- case MDBX_EINTR:
- if (!quiet)
- fprintf(stderr, "Interrupted by signal/user\n");
- break;
- default:
- if (unlikely(rc != MDBX_SUCCESS))
- error("readline", rc);
- }
-
-txn_abort:
- mdbx_cursor_close(mc);
- mdbx_txn_abort(txn);
-env_close:
- mdbx_env_close(env);
- free(kbuf.iov_base);
- free(dbuf.iov_base);
-
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/mdbx_stat.c b/3rdparty/libmdbx/src/mdbx_stat.c
deleted file mode 100644
index fa229435389a596b26b94aef7ee521a1b41696e1..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/mdbx_stat.c
+++ /dev/null
@@ -1,523 +0,0 @@
-/* mdbx_stat.c - memory-mapped database status tool */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>. */
-
-#ifdef _MSC_VER
-#if _MSC_VER > 1800
-#pragma warning(disable : 4464) /* relative include path contains '..' */
-#endif
-#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
-#endif /* _MSC_VER (warnings) */
-
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "internals.h"
-
-#if defined(_WIN32) || defined(_WIN64)
-#include "wingetopt.h"
-
-static volatile BOOL user_break;
-static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
- (void)dwCtrlType;
- user_break = true;
- return true;
-}
-
-#else /* WINDOWS */
-
-static volatile sig_atomic_t user_break;
-static void signal_handler(int sig) {
- (void)sig;
- user_break = 1;
-}
-
-#endif /* !WINDOWS */
-
-static void print_stat(MDBX_stat *ms) {
- printf(" Pagesize: %u\n", ms->ms_psize);
- printf(" Tree depth: %u\n", ms->ms_depth);
- printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
- printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
- printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
- printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
-}
-
-static void usage(const char *prog) {
- fprintf(stderr,
- "usage: %s [-V] [-q] [-e] [-f[f[f]]] [-r[r]] [-a|-s name] dbpath\n"
- " -V\t\tprint version and exit\n"
- " -q\t\tbe quiet\n"
- " -p\t\tshow statistics of page operations for current session\n"
- " -e\t\tshow whole DB info\n"
- " -f\t\tshow GC info\n"
- " -r\t\tshow readers\n"
- " -a\t\tprint stat of main DB and all subDBs\n"
- " -s name\tprint stat of only the specified named subDB\n"
- " \t\tby default print stat of only the main DB\n",
- prog);
- exit(EXIT_FAILURE);
-}
-
-static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid,
- mdbx_tid_t thread, uint64_t txnid, uint64_t lag,
- size_t bytes_used, size_t bytes_retained) {
- (void)ctx;
- if (num == 1)
- printf("Reader Table\n"
- " #\tslot\t%6s %*s %20s %10s %13s %13s\n",
- "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used",
- "retained");
-
- printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid,
- (int)sizeof(size_t) * 2, (uintptr_t)thread);
- if (txnid)
- printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag,
- bytes_used / 1048576.0, bytes_retained / 1048576.0);
- else
- printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0");
-
- return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
-}
-
-const char *prog;
-bool quiet = false;
-static void error(const char *func, int rc) {
- if (!quiet)
- fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
- mdbx_strerror(rc));
-}
-
-int main(int argc, char *argv[]) {
- int opt, rc;
- MDBX_env *env;
- MDBX_txn *txn;
- MDBX_dbi dbi;
- MDBX_envinfo mei;
- prog = argv[0];
- char *envname;
- char *subname = nullptr;
- bool alldbs = false, envinfo = false, pgop = false;
- int freinfo = 0, rdrinfo = 0;
-
- if (argc < 2)
- usage(prog);
-
- while ((opt = getopt(argc, argv,
- "V"
- "q"
- "p"
- "a"
- "e"
- "f"
- "n"
- "r"
- "s:")) != EOF) {
- switch (opt) {
- case 'V':
- printf("mdbx_stat version %d.%d.%d.%d\n"
- " - source: %s %s, commit %s, tree %s\n"
- " - anchor: %s\n"
- " - build: %s for %s by %s\n"
- " - flags: %s\n"
- " - options: %s\n",
- mdbx_version.major, mdbx_version.minor, mdbx_version.release,
- mdbx_version.revision, mdbx_version.git.describe,
- mdbx_version.git.datetime, mdbx_version.git.commit,
- mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
- mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
- mdbx_build.options);
- return EXIT_SUCCESS;
- case 'q':
- quiet = true;
- break;
- case 'p':
- pgop = true;
- break;
- case 'a':
- if (subname)
- usage(prog);
- alldbs = true;
- break;
- case 'e':
- envinfo = true;
- break;
- case 'f':
- freinfo += 1;
- break;
- case 'n':
- break;
- case 'r':
- rdrinfo += 1;
- break;
- case 's':
- if (alldbs)
- usage(prog);
- subname = optarg;
- break;
- default:
- usage(prog);
- }
- }
-
- if (optind != argc - 1)
- usage(prog);
-
-#if defined(_WIN32) || defined(_WIN64)
- SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
-#else
-#ifdef SIGPIPE
- signal(SIGPIPE, signal_handler);
-#endif
-#ifdef SIGHUP
- signal(SIGHUP, signal_handler);
-#endif
- signal(SIGINT, signal_handler);
- signal(SIGTERM, signal_handler);
-#endif /* !WINDOWS */
-
- envname = argv[optind];
- envname = argv[optind];
- if (!quiet) {
- printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n",
- mdbx_version.git.describe, mdbx_version.git.datetime,
- mdbx_version.git.tree, envname);
- fflush(nullptr);
- }
-
- rc = mdbx_env_create(&env);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_create", rc);
- return EXIT_FAILURE;
- }
-
- if (alldbs || subname) {
- rc = mdbx_env_set_maxdbs(env, 2);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_set_maxdbs", rc);
- goto env_close;
- }
- }
-
- rc = mdbx_env_open(env, envname, MDBX_RDONLY, 0);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_open", rc);
- goto env_close;
- }
-
- rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_txn_begin", rc);
- goto txn_abort;
- }
-
- if (envinfo || freinfo || pgop) {
- rc = mdbx_env_info_ex(env, txn, &mei, sizeof(mei));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_env_info_ex", rc);
- goto txn_abort;
- }
- } else {
- /* LY: zap warnings from gcc */
- memset(&mei, 0, sizeof(mei));
- }
-
- if (pgop) {
- printf("Page Operations (for current session):\n");
- printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n",
- mei.mi_pgop_stat.newly);
- printf(" CoW: %8" PRIu64
- "\t// quantity of pages copied for altering\n",
- mei.mi_pgop_stat.cow);
- printf(" Clone: %8" PRIu64 "\t// quantity of parent's dirty pages "
- "clones for nested transactions\n",
- mei.mi_pgop_stat.clone);
- printf(" Split: %8" PRIu64
- "\t// page splits during insertions or updates\n",
- mei.mi_pgop_stat.split);
- printf(" Merge: %8" PRIu64
- "\t// page merges during deletions or updates\n",
- mei.mi_pgop_stat.merge);
- printf(" Spill: %8" PRIu64 "\t// quantity of spilled/ousted `dirty` "
- "pages during large transactions\n",
- mei.mi_pgop_stat.spill);
- printf(" Unspill: %8" PRIu64 "\t// quantity of unspilled/redone `dirty` "
- "pages during large transactions\n",
- mei.mi_pgop_stat.unspill);
- printf(" WOP: %8" PRIu64
- "\t// number of explicit write operations (not a pages) to a disk\n",
- mei.mi_pgop_stat.wops);
- }
-
- if (envinfo) {
- printf("Environment Info\n");
- printf(" Pagesize: %u\n", mei.mi_dxb_pagesize);
- if (mei.mi_geo.lower != mei.mi_geo.upper) {
- printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
- "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
- "/-%" PRIu64 ")\n",
- mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
- mei.mi_geo.shrink, mei.mi_geo.lower / mei.mi_dxb_pagesize,
- mei.mi_geo.upper / mei.mi_dxb_pagesize,
- mei.mi_geo.grow / mei.mi_dxb_pagesize,
- mei.mi_geo.shrink / mei.mi_dxb_pagesize);
- printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
- mei.mi_mapsize, mei.mi_mapsize / mei.mi_dxb_pagesize);
- printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
- mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
-#if defined(_WIN32) || defined(_WIN64)
- if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper)
- printf(" WARNING: Due Windows system limitations a "
- "file couldn't\n be truncated while database "
- "is opened. So, the size of\n database file "
- "may by large than the database itself,\n "
- "until it will be closed or reopened in read-write mode.\n");
-#endif
- } else {
- printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
- mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
- }
- printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
- printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
- mei.mi_latter_reader_txnid,
- mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
- printf(" Max readers: %u\n", mei.mi_maxreaders);
- printf(" Number of reader slots uses: %u\n", mei.mi_numreaders);
- }
-
- if (rdrinfo) {
- rc = mdbx_reader_list(env, reader_list_func, nullptr);
- if (MDBX_IS_ERROR(rc)) {
- error("mdbx_reader_list", rc);
- goto txn_abort;
- }
- if (rc == MDBX_RESULT_TRUE)
- printf("Reader Table is empty\n");
- else if (rc == MDBX_SUCCESS && rdrinfo > 1) {
- int dead;
- rc = mdbx_reader_check(env, &dead);
- if (MDBX_IS_ERROR(rc)) {
- error("mdbx_reader_check", rc);
- goto txn_abort;
- }
- if (rc == MDBX_RESULT_TRUE) {
- printf(" %d stale readers cleared.\n", dead);
- rc = mdbx_reader_list(env, reader_list_func, nullptr);
- if (rc == MDBX_RESULT_TRUE)
- printf(" Now Reader Table is empty\n");
- } else
- printf(" No stale readers.\n");
- }
- if (!(subname || alldbs || freinfo))
- goto txn_abort;
- }
-
- if (freinfo) {
- printf("Garbage Collection\n");
- dbi = 0;
- MDBX_cursor *cursor;
- rc = mdbx_cursor_open(txn, dbi, &cursor);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_open", rc);
- goto txn_abort;
- }
-
- MDBX_stat mst;
- rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_stat", rc);
- goto txn_abort;
- }
- print_stat(&mst);
-
- pgno_t pages = 0, *iptr;
- pgno_t reclaimable = 0;
- MDBX_val key, data;
- while (MDBX_SUCCESS ==
- (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) {
- if (user_break) {
- rc = MDBX_EINTR;
- break;
- }
- iptr = data.iov_base;
- const pgno_t number = *iptr++;
-
- pages += number;
- if (envinfo && mei.mi_latter_reader_txnid > *(txnid_t *)key.iov_base)
- reclaimable += number;
-
- if (freinfo > 1) {
- char *bad = "";
- pgno_t prev =
- MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
- pgno_t span = 1;
- for (unsigned i = 0; i < number; ++i) {
- pgno_t pg = iptr[i];
- if (MDBX_PNL_DISORDERED(prev, pg))
- bad = " [bad sequence]";
- prev = pg;
- while (i + span < number &&
- iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
- : pgno_sub(pg, span)))
- ++span;
- }
- printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
- " pages, maxspan %" PRIaPGNO "%s\n",
- *(txnid_t *)key.iov_base, number, span, bad);
- if (freinfo > 2) {
- for (unsigned i = 0; i < number; i += span) {
- const pgno_t pg = iptr[i];
- for (span = 1;
- i + span < number &&
- iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
- : pgno_sub(pg, span));
- ++span)
- ;
- if (span > 1)
- printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span);
- else
- printf(" %9" PRIaPGNO "\n", pg);
- }
- }
- }
- }
- mdbx_cursor_close(cursor);
- cursor = nullptr;
-
- switch (rc) {
- case MDBX_SUCCESS:
- case MDBX_NOTFOUND:
- break;
- case MDBX_EINTR:
- if (!quiet)
- fprintf(stderr, "Interrupted by signal/user\n");
- goto txn_abort;
- default:
- error("mdbx_cursor_get", rc);
- goto txn_abort;
- }
-
- if (envinfo) {
- uint64_t value = mei.mi_mapsize / mei.mi_dxb_pagesize;
- double percent = value / 100.0;
- printf("Page Usage\n");
- printf(" Total: %" PRIu64 " 100%%\n", value);
-
- value = mei.mi_geo.current / mei.mi_dxb_pagesize;
- printf(" Backed: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = mei.mi_last_pgno + 1;
- printf(" Allocated: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1);
- printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = mei.mi_last_pgno + 1 - pages;
- printf(" Used: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = pages;
- printf(" GC: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = pages - reclaimable;
- printf(" Retained: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = reclaimable;
- printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
-
- value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) +
- reclaimable;
- printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
- } else
- printf(" GC: %" PRIaPGNO " pages\n", pages);
- }
-
- rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_open", rc);
- goto txn_abort;
- }
-
- MDBX_stat mst;
- rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_stat", rc);
- goto txn_abort;
- }
- printf("Status of %s\n", subname ? subname : "Main DB");
- print_stat(&mst);
-
- if (alldbs) {
- MDBX_cursor *cursor;
- rc = mdbx_cursor_open(txn, dbi, &cursor);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_cursor_open", rc);
- goto txn_abort;
- }
-
- MDBX_val key;
- while (MDBX_SUCCESS ==
- (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
- MDBX_dbi subdbi;
- if (memchr(key.iov_base, '\0', key.iov_len))
- continue;
- subname = mdbx_malloc(key.iov_len + 1);
- memcpy(subname, key.iov_base, key.iov_len);
- subname[key.iov_len] = '\0';
- rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &subdbi);
- if (rc == MDBX_SUCCESS)
- printf("Status of %s\n", subname);
- mdbx_free(subname);
- if (unlikely(rc != MDBX_SUCCESS)) {
- if (rc == MDBX_INCOMPATIBLE)
- continue;
- error("mdbx_dbi_open", rc);
- goto txn_abort;
- }
-
- rc = mdbx_dbi_stat(txn, subdbi, &mst, sizeof(mst));
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_stat", rc);
- goto txn_abort;
- }
- print_stat(&mst);
-
- rc = mdbx_dbi_close(env, subdbi);
- if (unlikely(rc != MDBX_SUCCESS)) {
- error("mdbx_dbi_close", rc);
- goto txn_abort;
- }
- }
- mdbx_cursor_close(cursor);
- cursor = nullptr;
- }
-
- switch (rc) {
- case MDBX_SUCCESS:
- case MDBX_NOTFOUND:
- break;
- case MDBX_EINTR:
- if (!quiet)
- fprintf(stderr, "Interrupted by signal/user\n");
- break;
- default:
- if (unlikely(rc != MDBX_SUCCESS))
- error("mdbx_cursor_get", rc);
- }
-
- mdbx_dbi_close(env, dbi);
-txn_abort:
- mdbx_txn_abort(txn);
-env_close:
- mdbx_env_close(env);
-
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/options.h b/3rdparty/libmdbx/src/options.h
deleted file mode 100644
index 30ffdf1be7686eaf92ccd7cd07eb975b11966a2a..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/options.h
+++ /dev/null
@@ -1,427 +0,0 @@
-/*******************************************************************************
- *******************************************************************************
- *******************************************************************************
- *
- *
- * #### ##### ##### # #### # # ####
- * # # # # # # # # ## # #
- * # # # # # # # # # # # ####
- * # # ##### # # # # # # # #
- * # # # # # # # # ## # #
- * #### # # # #### # # ####
- *
- *
- */
-
-/** \defgroup build_option Build options
- * The libmdbx build options.
- @{ */
-
-/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
-#define MDBX_OSX_WANNA_DURABILITY 0
-/** Using fsync() with chance of data lost on power failure */
-#define MDBX_OSX_WANNA_SPEED 1
-
-#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
-/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
- * for OSX & iOS */
-#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
-#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
-
-/** Controls checking PID against reuse DB environment after the fork() */
-#ifndef MDBX_ENV_CHECKPID
-#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
-/* PID check could be omitted:
- * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork()
- * mapped pages will not be available for child process.
- * - in Windows where fork() not available. */
-#define MDBX_ENV_CHECKPID 0
-#else
-#define MDBX_ENV_CHECKPID 1
-#endif
-#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
-#else
-#define MDBX_ENV_CHECKPID_CONFIG MDBX_STRINGIFY(MDBX_ENV_CHECKPID)
-#endif /* MDBX_ENV_CHECKPID */
-
-/** Controls checking transaction owner thread against misuse transactions from
- * other threads. */
-#ifndef MDBX_TXN_CHECKOWNER
-#define MDBX_TXN_CHECKOWNER 1
-#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
-#else
-#define MDBX_TXN_CHECKOWNER_CONFIG MDBX_STRINGIFY(MDBX_TXN_CHECKOWNER)
-#endif /* MDBX_TXN_CHECKOWNER */
-
-/** Does a system have battery-backed Real-Time Clock or just a fake. */
-#ifndef MDBX_TRUST_RTC
-#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
- defined(__OpenBSD__)
-#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
-#else
-#define MDBX_TRUST_RTC 1
-#endif
-#define MDBX_TRUST_RTC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_TRUST_RTC)
-#else
-#define MDBX_TRUST_RTC_CONFIG MDBX_STRINGIFY(MDBX_TRUST_RTC)
-#endif /* MDBX_TRUST_RTC */
-
-/** Controls online database auto-compactification during write-transactions. */
-#ifndef MDBX_ENABLE_REFUND
-#define MDBX_ENABLE_REFUND 1
-#elif !(MDBX_ENABLE_REFUND == 0 || MDBX_ENABLE_REFUND == 1)
-#error MDBX_ENABLE_REFUND must be defined as 0 or 1
-#endif /* MDBX_ENABLE_REFUND */
-
-/** Controls gathering statistics for page operations. */
-#ifndef MDBX_ENABLE_PGOP_STAT
-#define MDBX_ENABLE_PGOP_STAT 1
-#elif !(MDBX_ENABLE_PGOP_STAT == 0 || MDBX_ENABLE_PGOP_STAT == 1)
-#error MDBX_ENABLE_PGOP_STAT must be defined as 0 or 1
-#endif /* MDBX_ENABLE_PGOP_STAT */
-
-/** Controls use of POSIX madvise() hints and friends. */
-#ifndef MDBX_ENABLE_MADVISE
-#define MDBX_ENABLE_MADVISE 1
-#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1)
-#error MDBX_ENABLE_MADVISE must be defined as 0 or 1
-#endif /* MDBX_ENABLE_MADVISE */
-
-/** Disable some checks to reduce an overhead and detection probability of
- * database corruption to a values closer to the LMDB. */
-#ifndef MDBX_DISABLE_PAGECHECKS
-#define MDBX_DISABLE_PAGECHECKS 0
-#elif !(MDBX_DISABLE_PAGECHECKS == 0 || MDBX_DISABLE_PAGECHECKS == 1)
-#error MDBX_DISABLE_PAGECHECKS must be defined as 0 or 1
-#endif /* MDBX_DISABLE_PAGECHECKS */
-
-#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
-#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
-#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
- MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
-#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
-#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
-
-#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
-#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
-#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
- MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
-#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
-#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
-
-/** Basically, this build-option is for TODO. Guess it should be replaced
- * with MDBX_ENABLE_WRITEMAP_SPILLING with the three variants:
- * 0/OFF = Don't track dirty pages at all and don't spilling ones.
- * This should be by-default on Linux and may-be other systems
- * (not sure: Darwin/OSX, FreeBSD, Windows 10) where kernel provides
- * properly LRU tracking and async writing on-demand.
- * 1/ON = Lite tracking of dirty pages but with LRU labels and explicit
- * spilling with msync(MS_ASYNC). */
-#ifndef MDBX_FAKE_SPILL_WRITEMAP
-#if defined(__linux__) || defined(__gnu_linux__)
-#define MDBX_FAKE_SPILL_WRITEMAP 1 /* msync(MS_ASYNC) is no-op on Linux */
-#else
-#define MDBX_FAKE_SPILL_WRITEMAP 0
-#endif
-#elif !(MDBX_FAKE_SPILL_WRITEMAP == 0 || MDBX_FAKE_SPILL_WRITEMAP == 1)
-#error MDBX_FAKE_SPILL_WRITEMAP must be defined as 0 or 1
-#endif /* MDBX_FAKE_SPILL_WRITEMAP */
-
-/** Controls sort order of internal page number lists.
- * This mostly experimental/advanced option with not for regular MDBX users.
- * \warning The database format depend on this option and libmdbx builded with
- * different option value are incompatible. */
-#ifndef MDBX_PNL_ASCENDING
-#define MDBX_PNL_ASCENDING 0
-#elif !(MDBX_PNL_ASCENDING == 0 || MDBX_PNL_ASCENDING == 1)
-#error MDBX_PNL_ASCENDING must be defined as 0 or 1
-#endif /* MDBX_PNL_ASCENDING */
-
-/** Avoid dependence from MSVC CRT and use ntdll.dll instead. */
-#ifndef MDBX_WITHOUT_MSVC_CRT
-#define MDBX_WITHOUT_MSVC_CRT 1
-#elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1)
-#error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1
-#endif /* MDBX_WITHOUT_MSVC_CRT */
-
-/** Size of buffer used during copying a environment/database file. */
-#ifndef MDBX_ENVCOPY_WRITEBUF
-#define MDBX_ENVCOPY_WRITEBUF 1048576u
-#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
- MDBX_ENVCOPY_WRITEBUF % 65536u
-#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
-#endif /* MDBX_ENVCOPY_WRITEBUF */
-
-/** Forces assertion checking */
-#ifndef MDBX_FORCE_ASSERTIONS
-#define MDBX_FORCE_ASSERTIONS 0
-#elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1)
-#error MDBX_FORCE_ASSERTIONS must be defined as 0 or 1
-#endif /* MDBX_FORCE_ASSERTIONS */
-
-/** Presumed malloc size overhead for each allocation
- * to adjust allocations to be more aligned. */
-#ifndef MDBX_ASSUME_MALLOC_OVERHEAD
-#ifdef __SIZEOF_POINTER__
-#define MDBX_ASSUME_MALLOC_OVERHEAD (__SIZEOF_POINTER__ * 2u)
-#else
-#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
-#endif
-#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
- MDBX_ASSUME_MALLOC_OVERHEAD % 4
-#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
-#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
-
-/** If defined then enables integration with Valgrind,
- * a memory analyzing tool. */
-#ifndef MDBX_USE_VALGRIND
-#endif /* MDBX_USE_VALGRIND */
-
-/** If defined then enables use C11 atomics,
- * otherwise detects ones availability automatically. */
-#ifndef MDBX_HAVE_C11ATOMICS
-#endif /* MDBX_HAVE_C11ATOMICS */
-
-//------------------------------------------------------------------------------
-
-/** Win32 File Locking API for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_WIN32FILES -1
-
-/** SystemV IPC semaphores for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_SYSV 5
-
-/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_POSIX1988 1988
-
-/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_POSIX2001 2001
-
-/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_POSIX2008 2008
-
-/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
-#define MDBX_LOCKING_BENAPHORE 1995
-
-/** Advanced: Choices the locking implementation (autodetection by default). */
-#if defined(_WIN32) || defined(_WIN64)
-#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
-#else
-#ifndef MDBX_LOCKING
-#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
- _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
-
-/* Some platforms define the EOWNERDEAD error code even though they
- * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
-#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
- ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
- _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
- (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
- _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
- defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
- (!defined(__GLIBC__) || \
- __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
-#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
-#else
-#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
-#endif
-#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
-#define MDBX_LOCKING MDBX_LOCKING_POSIX1988
-#else
-#define MDBX_LOCKING MDBX_LOCKING_SYSV
-#endif
-#define MDBX_LOCKING_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_LOCKING)
-#else
-#define MDBX_LOCKING_CONFIG MDBX_STRINGIFY(MDBX_LOCKING)
-#endif /* MDBX_LOCKING */
-#endif /* !Windows */
-
-/** Advanced: Using POSIX OFD-locks (autodetection by default). */
-#ifndef MDBX_USE_OFDLOCKS
-#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
- !defined(MDBX_SAFE4QEMU) && \
- !defined(__sun) /* OFD-lock are broken on Solaris */
-#define MDBX_USE_OFDLOCKS 1
-#else
-#define MDBX_USE_OFDLOCKS 0
-#endif
-#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
-#else
-#define MDBX_USE_OFDLOCKS_CONFIG MDBX_STRINGIFY(MDBX_USE_OFDLOCKS)
-#endif /* MDBX_USE_OFDLOCKS */
-
-/** Advanced: Using sendfile() syscall (autodetection by default). */
-#ifndef MDBX_USE_SENDFILE
-#if ((defined(__linux__) || defined(__gnu_linux__)) && \
- !defined(__ANDROID_API__)) || \
- (defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
-#define MDBX_USE_SENDFILE 1
-#else
-#define MDBX_USE_SENDFILE 0
-#endif
-#endif /* MDBX_USE_SENDFILE */
-
-/** Advanced: Using copy_file_range() syscall (autodetection by default). */
-#ifndef MDBX_USE_COPYFILERANGE
-#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE)
-#define MDBX_USE_COPYFILERANGE 1
-#else
-#define MDBX_USE_COPYFILERANGE 0
-#endif
-#endif /* MDBX_USE_COPYFILERANGE */
-
-/** Advanced: Using sync_file_range() syscall (autodetection by default). */
-#ifndef MDBX_USE_SYNCFILERANGE
-#if ((defined(__linux__) || defined(__gnu_linux__)) && \
- defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \
- (defined(__ANDROID_API__) && __ANDROID_API__ >= 26)
-#define MDBX_USE_SYNCFILERANGE 1
-#else
-#define MDBX_USE_SYNCFILERANGE 0
-#endif
-#endif /* MDBX_USE_SYNCFILERANGE */
-
-//------------------------------------------------------------------------------
-
-#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
-#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
- defined(__hppa__) || defined(DOXYGEN)
-#define MDBX_CPU_WRITEBACK_INCOHERENT 0
-#else
-#define MDBX_CPU_WRITEBACK_INCOHERENT 1
-#endif
-#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
-
-#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
-#ifdef __OpenBSD__
-#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1
-#else
-#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
-#endif
-#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
-
-#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
-#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
- defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
- defined(__MWERKS__) || defined(__sgi)
-/* MIPS has cache coherency issues. */
-#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
-#else
-/* LY: assume no relevant mmap/dcache issues. */
-#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
-#endif
-#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
-
-#ifndef MDBX_64BIT_ATOMIC
-#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
-#define MDBX_64BIT_ATOMIC 1
-#else
-#define MDBX_64BIT_ATOMIC 0
-#endif
-#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
-#else
-#define MDBX_64BIT_ATOMIC_CONFIG MDBX_STRINGIFY(MDBX_64BIT_ATOMIC)
-#endif /* MDBX_64BIT_ATOMIC */
-
-#ifndef MDBX_64BIT_CAS
-#if defined(ATOMIC_LLONG_LOCK_FREE)
-#if ATOMIC_LLONG_LOCK_FREE > 1
-#define MDBX_64BIT_CAS 1
-#else
-#define MDBX_64BIT_CAS 0
-#endif
-#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE)
-#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1
-#define MDBX_64BIT_CAS 1
-#else
-#define MDBX_64BIT_CAS 0
-#endif
-#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE)
-#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1
-#define MDBX_64BIT_CAS 1
-#else
-#define MDBX_64BIT_CAS 0
-#endif
-#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
-#define MDBX_64BIT_CAS 1
-#else
-#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
-#endif
-#define MDBX_64BIT_CAS_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_64BIT_CAS)
-#else
-#define MDBX_64BIT_CAS_CONFIG MDBX_STRINGIFY(MDBX_64BIT_CAS)
-#endif /* MDBX_64BIT_CAS */
-
-#ifndef MDBX_UNALIGNED_OK
-#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__)
-#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
-#elif defined(__ARM_FEATURE_UNALIGNED)
-#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
-#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
-/* expecting an optimization will well done, also this
- * hushes false-positives from UBSAN (undefined behaviour sanitizer) */
-#define MDBX_UNALIGNED_OK 0
-#elif defined(__e2k__) || defined(__elbrus__)
-#if __iset__ > 4
-#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
-#else
-#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */
-#endif
-#elif defined(__ia32__)
-#define MDBX_UNALIGNED_OK 8 /* ok unaligned for 64-bit words */
-#else
-#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
-#endif
-#elif MDBX_UNALIGNED_OK == 1
-#undef MDBX_UNALIGNED_OK
-#define MDBX_UNALIGNED_OK 32 /* any unaligned access allowed */
-#endif /* MDBX_UNALIGNED_OK */
-
-#ifndef MDBX_CACHELINE_SIZE
-#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
-#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
-#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
-#define MDBX_CACHELINE_SIZE 128
-#else
-#define MDBX_CACHELINE_SIZE 64
-#endif
-#endif /* MDBX_CACHELINE_SIZE */
-
-/** @} end of build options */
-/*******************************************************************************
- *******************************************************************************
- ******************************************************************************/
-
-#ifndef DOXYGEN
-
-/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */
-#ifndef MDBX_DEBUG
-#ifdef NDEBUG
-#define MDBX_DEBUG 0
-#else
-#define MDBX_DEBUG 1
-#endif
-#endif /* MDBX_DEBUG */
-
-#else
-
-/* !!! Actually this is a fake definitions for Doxygen !!! */
-
-/** Controls enabling of debugging features.
- *
- * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
- * including logging and assertion controls.
- * Logging level and corresponding debug flags changing
- * by \ref mdbx_setup_debug() will not have effect.
- * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
- * assertions checking and internal audit).
- * Simultaneously sets the default logging level
- * to the `MDBX_DEBUG` value.
- * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
- *
- * \ingroup build_option */
-#define MDBX_DEBUG 0...7
-
-/** Disables using of GNU libc extensions. */
-#define MDBX_DISABLE_GNU_SOURCE 0 or 1
-
-#endif /* DOXYGEN */
diff --git a/3rdparty/libmdbx/src/osal.c b/3rdparty/libmdbx/src/osal.c
deleted file mode 100644
index d19c3d362c23f3fadc469b4ade98c506966bab32..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/osal.c
+++ /dev/null
@@ -1,2630 +0,0 @@
-/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "internals.h"
-
-#if defined(_WIN32) || defined(_WIN64)
-
-#include <winioctl.h>
-
-static int waitstatus2errcode(DWORD result) {
- switch (result) {
- case WAIT_OBJECT_0:
- return MDBX_SUCCESS;
- case WAIT_FAILED:
- return (int)GetLastError();
- case WAIT_ABANDONED:
- return ERROR_ABANDONED_WAIT_0;
- case WAIT_IO_COMPLETION:
- return ERROR_USER_APC;
- case WAIT_TIMEOUT:
- return ERROR_TIMEOUT;
- default:
- return ERROR_UNHANDLED_ERROR;
- }
-}
-
-/* Map a result from an NTAPI call to WIN32 error code. */
-static int ntstatus2errcode(NTSTATUS status) {
- DWORD dummy;
- OVERLAPPED ov;
- memset(&ov, 0, sizeof(ov));
- ov.Internal = status;
- return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS
- : (int)GetLastError();
-}
-
-/* We use native NT APIs to setup the memory map, so that we can
- * let the DB file grow incrementally instead of always preallocating
- * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
- * but those headers are meant for driver-level development and
- * conflict with the regular user-level headers, so we explicitly
- * declare them here. Using these APIs also means we must link to
- * ntdll.dll, which is not linked by default in user code. */
-
-extern NTSTATUS NTAPI NtCreateSection(
- OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess,
- IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes,
- IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection,
- IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle);
-
-typedef struct _SECTION_BASIC_INFORMATION {
- ULONG Unknown;
- ULONG SectionAttributes;
- LARGE_INTEGER SectionSize;
-} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
-
-extern NTSTATUS NTAPI NtMapViewOfSection(
- IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress,
- IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize,
- IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize,
- IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType,
- IN ULONG Win32Protect);
-
-extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle,
- IN OPTIONAL PVOID BaseAddress);
-
-extern NTSTATUS NTAPI NtClose(HANDLE Handle);
-
-extern NTSTATUS NTAPI NtAllocateVirtualMemory(
- IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits,
- IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect);
-
-extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle,
- IN PVOID *BaseAddress,
- IN OUT PSIZE_T RegionSize,
- IN ULONG FreeType);
-
-#ifndef WOF_CURRENT_VERSION
-typedef struct _WOF_EXTERNAL_INFO {
- DWORD Version;
- DWORD Provider;
-} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO;
-#endif /* WOF_CURRENT_VERSION */
-
-#ifndef WIM_PROVIDER_CURRENT_VERSION
-#define WIM_PROVIDER_HASH_SIZE 20
-
-typedef struct _WIM_PROVIDER_EXTERNAL_INFO {
- DWORD Version;
- DWORD Flags;
- LARGE_INTEGER DataSourceId;
- BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE];
-} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO;
-#endif /* WIM_PROVIDER_CURRENT_VERSION */
-
-#ifndef FILE_PROVIDER_CURRENT_VERSION
-typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 {
- ULONG Version;
- ULONG Algorithm;
- ULONG Flags;
-} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1;
-#endif /* FILE_PROVIDER_CURRENT_VERSION */
-
-#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED
-#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL)
-#endif
-#ifndef STATUS_INVALID_DEVICE_REQUEST
-#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L)
-#endif
-#ifndef STATUS_NOT_SUPPORTED
-#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL)
-#endif
-
-#ifndef FILE_DEVICE_FILE_SYSTEM
-#define FILE_DEVICE_FILE_SYSTEM 0x00000009
-#endif
-
-#ifndef FSCTL_GET_EXTERNAL_BACKING
-#define FSCTL_GET_EXTERNAL_BACKING \
- CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS)
-#endif
-
-#ifndef ERROR_NOT_CAPABLE
-#define ERROR_NOT_CAPABLE 775L
-#endif
-
-#endif /* _WIN32 || _WIN64 */
-
-/*----------------------------------------------------------------------------*/
-
-#if defined(__UCLIBC__)
-__extern_C void __assert(const char *, const char *, unsigned int, const char *)
-#ifdef __THROW
- __THROW
-#else
- __nothrow
-#endif /* __THROW */
- MDBX_NORETURN;
-#define __assert_fail(assertion, file, line, function) \
- __assert(assertion, file, line, function)
-
-#elif _POSIX_C_SOURCE > 200212 && \
- /* workaround for avoid musl libc wrong prototype */ ( \
- defined(__GLIBC__) || defined(__GNU_LIBRARY__))
-/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */
-__extern_C void __assert_fail(const char *assertion, const char *file,
- unsigned line, const char *function)
-#ifdef __THROW
- __THROW
-#else
- __nothrow
-#endif /* __THROW */
- MDBX_NORETURN;
-
-#elif defined(__APPLE__) || defined(__MACH__)
-__extern_C void __assert_rtn(const char *function, const char *file, int line,
- const char *assertion) /* __nothrow */
-#ifdef __dead2
- __dead2
-#else
- MDBX_NORETURN
-#endif /* __dead2 */
-#ifdef __disable_tail_calls
- __disable_tail_calls
-#endif /* __disable_tail_calls */
- ;
-
-#define __assert_fail(assertion, file, line, function) \
- __assert_rtn(function, file, line, assertion)
-#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
-__extern_C void __assert_c99(const char *assection, const char *file, int line,
- const char *function) MDBX_NORETURN;
-#define __assert_fail(assertion, file, line, function) \
- __assert_c99(assertion, file, line, function)
-#elif defined(__OpenBSD__)
-__extern_C __dead void __assert2(const char *file, int line,
- const char *function,
- const char *assertion) /* __nothrow */;
-#define __assert_fail(assertion, file, line, function) \
- __assert2(file, line, function, assertion)
-#elif defined(__NetBSD__)
-__extern_C __dead void __assert13(const char *file, int line,
- const char *function,
- const char *assertion) /* __nothrow */;
-#define __assert_fail(assertion, file, line, function) \
- __assert13(file, line, function, assertion)
-#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \
- defined(__DragonFly__)
-__extern_C void __assert(const char *function, const char *file, int line,
- const char *assertion) /* __nothrow */
-#ifdef __dead2
- __dead2
-#else
- MDBX_NORETURN
-#endif /* __dead2 */
-#ifdef __disable_tail_calls
- __disable_tail_calls
-#endif /* __disable_tail_calls */
- ;
-#define __assert_fail(assertion, file, line, function) \
- __assert(function, file, line, assertion)
-
-#endif /* __assert_fail */
-
-#if !defined(__ANDROID_API__) || MDBX_DEBUG
-
-__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg,
- const char *func, int line) {
-#if MDBX_DEBUG
- if (env && env->me_assert_func) {
- env->me_assert_func(env, msg, func, line);
- return;
- }
-#else
- (void)env;
-#endif /* MDBX_DEBUG */
-
- if (mdbx_debug_logger)
- mdbx_debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg);
- else {
-#if defined(_WIN32) || defined(_WIN64)
- char *message = nullptr;
- const int num = mdbx_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u",
- msg, func ? func : "unknown", line);
- if (num < 1 || !message)
- message = "<troubles with assertion-message preparation>";
- OutputDebugStringA(message);
- if (IsDebuggerPresent())
- DebugBreak();
-#elif defined(__ANDROID_API__)
- __android_log_assert(msg, "mdbx", "%s:%u", func, line);
-#else
- __assert_fail(msg, "mdbx", line, func);
-#endif
- }
-
-#if defined(_WIN32) || defined(_WIN64)
- FatalExit(ERROR_UNHANDLED_ERROR);
-#else
- abort();
-#endif
-}
-
-#endif /* __ANDROID_API__ || MDBX_DEBUG */
-
-__cold void mdbx_panic(const char *fmt, ...) {
- va_list ap;
- va_start(ap, fmt);
-
- char *message = nullptr;
- const int num = mdbx_vasprintf(&message, fmt, ap);
- va_end(ap);
- const char *const const_message =
- (num < 1 || !message) ? "<troubles with panic-message preparation>"
- : message;
-
-#if defined(_WIN32) || defined(_WIN64)
- OutputDebugStringA("\r\nMDBX-PANIC: ");
- OutputDebugStringA(const_message);
- if (IsDebuggerPresent())
- DebugBreak();
- FatalExit(ERROR_UNHANDLED_ERROR);
-#else
-#if defined(__ANDROID_API__)
- __android_log_assert("panic", "mdbx", "%s", const_message);
-#else
- __assert_fail(const_message, "mdbx", 0, "panic");
-#endif /* __ANDROID_API__ */
- abort();
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-#ifndef mdbx_vasprintf
-MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt,
- va_list ap) {
- va_list ones;
- va_copy(ones, ap);
- int needed = vsnprintf(nullptr, 0, fmt, ap);
-
- if (unlikely(needed < 0 || needed >= INT_MAX)) {
- *strp = nullptr;
- va_end(ones);
- return needed;
- }
-
- *strp = mdbx_malloc(needed + 1);
- if (unlikely(*strp == nullptr)) {
- va_end(ones);
-#if defined(_WIN32) || defined(_WIN64)
- SetLastError(MDBX_ENOMEM);
-#else
- errno = MDBX_ENOMEM;
-#endif
- return -1;
- }
-
- int actual = vsnprintf(*strp, needed + 1, fmt, ones);
- va_end(ones);
-
- assert(actual == needed);
- if (unlikely(actual < 0)) {
- mdbx_free(*strp);
- *strp = nullptr;
- }
- return actual;
-}
-#endif /* mdbx_vasprintf */
-
-#ifndef mdbx_asprintf
-MDBX_INTERNAL_FUNC int mdbx_asprintf(char **strp, const char *fmt, ...) {
- va_list ap;
- va_start(ap, fmt);
- int rc = mdbx_vasprintf(strp, fmt, ap);
- va_end(ap);
- return rc;
-}
-#endif /* mdbx_asprintf */
-
-#ifndef mdbx_memalign_alloc
-MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
- void **result) {
- assert(is_powerof2(alignment) && alignment >= sizeof(void *));
-#if defined(_WIN32) || defined(_WIN64)
- (void)alignment;
- *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
- return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */;
-#elif defined(_ISOC11_SOURCE)
- *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment));
- return *result ? MDBX_SUCCESS : errno;
-#elif _POSIX_VERSION >= 200112L && \
- (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17)
- *result = nullptr;
- return posix_memalign(result, alignment, bytes);
-#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L
- *result = memalign(alignment, bytes);
- return *result ? MDBX_SUCCESS : errno;
-#else
-#error FIXME
-#endif
-}
-#endif /* mdbx_memalign_alloc */
-
-#ifndef mdbx_memalign_free
-MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr) {
-#if defined(_WIN32) || defined(_WIN64)
- VirtualFree(ptr, 0, MEM_RELEASE);
-#else
- mdbx_free(ptr);
-#endif
-}
-#endif /* mdbx_memalign_free */
-
-#ifndef mdbx_strdup
-char *mdbx_strdup(const char *str) {
- if (!str)
- return NULL;
- size_t bytes = strlen(str) + 1;
- char *dup = mdbx_malloc(bytes);
- if (dup)
- memcpy(dup, str, bytes);
- return dup;
-}
-#endif /* mdbx_strdup */
-
-/*----------------------------------------------------------------------------*/
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair) {
- int rc;
- memset(condpair, 0, sizeof(mdbx_condpair_t));
-#if defined(_WIN32) || defined(_WIN64)
- if ((condpair->mutex = CreateMutexW(NULL, FALSE, NULL)) == NULL) {
- rc = (int)GetLastError();
- goto bailout_mutex;
- }
- if ((condpair->event[0] = CreateEventW(NULL, FALSE, FALSE, NULL)) == NULL) {
- rc = (int)GetLastError();
- goto bailout_event;
- }
- if ((condpair->event[1] = CreateEventW(NULL, FALSE, FALSE, NULL)) != NULL)
- return MDBX_SUCCESS;
-
- rc = (int)GetLastError();
- (void)CloseHandle(condpair->event[0]);
-bailout_event:
- (void)CloseHandle(condpair->mutex);
-#else
- rc = pthread_mutex_init(&condpair->mutex, NULL);
- if (unlikely(rc != 0))
- goto bailout_mutex;
- rc = pthread_cond_init(&condpair->cond[0], NULL);
- if (unlikely(rc != 0))
- goto bailout_cond;
- rc = pthread_cond_init(&condpair->cond[1], NULL);
- if (likely(rc == 0))
- return MDBX_SUCCESS;
-
- (void)pthread_cond_destroy(&condpair->cond[0]);
-bailout_cond:
- (void)pthread_mutex_destroy(&condpair->mutex);
-#endif
-bailout_mutex:
- memset(condpair, 0, sizeof(mdbx_condpair_t));
- return rc;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair) {
-#if defined(_WIN32) || defined(_WIN64)
- int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
- rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError();
- rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError();
-#else
- int err, rc = pthread_mutex_destroy(&condpair->mutex);
- rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc;
- rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc;
-#endif
- memset(condpair, 0, sizeof(mdbx_condpair_t));
- return rc;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD code = WaitForSingleObject(condpair->mutex, INFINITE);
- return waitstatus2errcode(code);
-#else
- return mdbx_pthread_mutex_lock(&condpair->mutex);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair) {
-#if defined(_WIN32) || defined(_WIN64)
- return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
-#else
- return pthread_mutex_unlock(&condpair->mutex);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
- bool part) {
-#if defined(_WIN32) || defined(_WIN64)
- return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError();
-#else
- return pthread_cond_signal(&condpair->cond[part]);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair,
- bool part) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part],
- INFINITE, FALSE);
- if (code == WAIT_OBJECT_0) {
- code = WaitForSingleObject(condpair->mutex, INFINITE);
- if (code == WAIT_OBJECT_0)
- return MDBX_SUCCESS;
- }
- return waitstatus2errcode(code);
-#else
- return pthread_cond_wait(&condpair->cond[part], &condpair->mutex);
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex) {
-#if defined(_WIN32) || defined(_WIN64)
- InitializeCriticalSection(fastmutex);
- return MDBX_SUCCESS;
-#else
- return pthread_mutex_init(fastmutex, NULL);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex) {
-#if defined(_WIN32) || defined(_WIN64)
- DeleteCriticalSection(fastmutex);
- return MDBX_SUCCESS;
-#else
- return pthread_mutex_destroy(fastmutex);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex) {
-#if defined(_WIN32) || defined(_WIN64)
- __try {
- EnterCriticalSection(fastmutex);
- } __except (
- (GetExceptionCode() ==
- 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
- ? EXCEPTION_EXECUTE_HANDLER
- : EXCEPTION_CONTINUE_SEARCH) {
- return ERROR_POSSIBLE_DEADLOCK;
- }
- return MDBX_SUCCESS;
-#else
- return mdbx_pthread_mutex_lock(fastmutex);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex) {
-#if defined(_WIN32) || defined(_WIN64)
- LeaveCriticalSection(fastmutex);
- return MDBX_SUCCESS;
-#else
- return pthread_mutex_unlock(fastmutex);
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname) {
-#if defined(_WIN32) || defined(_WIN64)
- const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
- if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
- return ERROR_INVALID_NAME;
- wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
- if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
- return ERROR_INVALID_NAME;
- return DeleteFileW(pathnameW) ? MDBX_SUCCESS : (int)GetLastError();
-#else
- return unlink(pathname) ? errno : MDBX_SUCCESS;
-#endif
-}
-
-#if !(defined(_WIN32) || defined(_WIN64))
-static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); }
-#endif /*! Windows */
-
-MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname) {
-#if defined(_WIN32) || defined(_WIN64)
- const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
- if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
- return ERROR_INVALID_NAME;
- wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
- if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
- return ERROR_INVALID_NAME;
- return RemoveDirectoryW(pathnameW) ? MDBX_SUCCESS : (int)GetLastError();
-#else
- return rmdir(pathname) ? errno : MDBX_SUCCESS;
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
- const MDBX_env *env, const char *pathname,
- mdbx_filehandle_t *fd,
- mdbx_mode_t unix_mode_bits) {
- *fd = INVALID_HANDLE_VALUE;
-
-#if defined(_WIN32) || defined(_WIN64)
- const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX);
- if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX)
- return ERROR_INVALID_NAME;
- wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t));
- if (wlen != mbstowcs(pathnameW, pathname, wlen + 1))
- return ERROR_INVALID_NAME;
-
- DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING;
- DWORD FlagsAndAttributes =
- FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED;
- DWORD DesiredAccess = FILE_READ_ATTRIBUTES;
- DWORD ShareMode = (env->me_flags & MDBX_EXCLUSIVE)
- ? 0
- : (FILE_SHARE_READ | FILE_SHARE_WRITE);
-
- switch (purpose) {
- default:
- return ERROR_INVALID_PARAMETER;
- case MDBX_OPEN_LCK:
- CreationDisposition = OPEN_ALWAYS;
- DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
- FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY;
- break;
- case MDBX_OPEN_DXB_READ:
- CreationDisposition = OPEN_EXISTING;
- DesiredAccess |= GENERIC_READ;
- ShareMode |= FILE_SHARE_READ;
- break;
- case MDBX_OPEN_DXB_LAZY:
- DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
- break;
- case MDBX_OPEN_DXB_DSYNC:
- CreationDisposition = OPEN_EXISTING;
- DesiredAccess |= GENERIC_WRITE;
- FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH;
- break;
- case MDBX_OPEN_COPY:
- CreationDisposition = CREATE_NEW;
- ShareMode = 0;
- DesiredAccess |= GENERIC_WRITE;
- FlagsAndAttributes |=
- (env->me_psize < env->me_os_psize) ? 0 : FILE_FLAG_NO_BUFFERING;
- break;
- case MDBX_OPEN_DELETE:
- CreationDisposition = OPEN_EXISTING;
- ShareMode |= FILE_SHARE_DELETE;
- DesiredAccess =
- FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE;
- break;
- }
-
- *fd = CreateFileW(pathnameW, DesiredAccess, ShareMode, NULL,
- CreationDisposition, FlagsAndAttributes, NULL);
- if (*fd == INVALID_HANDLE_VALUE)
- return (int)GetLastError();
-
- BY_HANDLE_FILE_INFORMATION info;
- if (!GetFileInformationByHandle(*fd, &info)) {
- int err = (int)GetLastError();
- CloseHandle(*fd);
- *fd = INVALID_HANDLE_VALUE;
- return err;
- }
- const DWORD AttributesDiff =
- (info.dwFileAttributes ^ FlagsAndAttributes) &
- (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
- FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED);
- if (AttributesDiff)
- (void)SetFileAttributesW(pathnameW, info.dwFileAttributes ^ AttributesDiff);
-
-#else
- int flags = unix_mode_bits ? O_CREAT : 0;
- switch (purpose) {
- default:
- return EINVAL;
- case MDBX_OPEN_LCK:
- flags |= O_RDWR;
- break;
- case MDBX_OPEN_DXB_READ:
- flags = O_RDONLY;
- break;
- case MDBX_OPEN_DXB_LAZY:
- flags |= O_RDWR;
- break;
- case MDBX_OPEN_COPY:
- flags = O_CREAT | O_WRONLY | O_EXCL;
- break;
- case MDBX_OPEN_DXB_DSYNC:
- flags |= O_WRONLY;
-#if defined(O_DSYNC)
- flags |= O_DSYNC;
-#elif defined(O_SYNC)
- flags |= O_SYNC;
-#elif defined(O_FSYNC)
- flags |= O_FSYNC;
-#endif
- break;
- case MDBX_OPEN_DELETE:
- flags = O_RDWR;
- break;
- }
-
- const bool direct_nocache_for_copy =
- env->me_psize >= env->me_os_psize && purpose == MDBX_OPEN_COPY;
- if (direct_nocache_for_copy) {
-#if defined(O_DIRECT)
- flags |= O_DIRECT;
-#endif /* O_DIRECT */
-#if defined(O_NOCACHE)
- flags |= O_NOCACHE;
-#endif /* O_NOCACHE */
- }
-
-#ifdef O_CLOEXEC
- flags |= O_CLOEXEC;
-#endif /* O_CLOEXEC */
-
- /* Safeguard for todo4recovery://erased_by_github/libmdbx/issues/144 */
-#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
- int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1;
- static const char dev_null[] = "/dev/null";
- if (!is_valid_fd(STDIN_FILENO)) {
- mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN",
- STDIN_FILENO, dev_null);
- stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY);
- }
- if (!is_valid_fd(STDOUT_FILENO)) {
- mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub",
- "OUT", STDOUT_FILENO, dev_null);
- stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY);
- }
- if (!is_valid_fd(STDERR_FILENO)) {
- mdbx_warning("STD%s_FILENO/%d is invalid, open %s for temporary stub",
- "ERR", STDERR_FILENO, dev_null);
- stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY);
- }
-#else
-#error "Unexpected or unsupported UNIX or POSIX system"
-#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
-
- *fd = open(pathname, flags, unix_mode_bits);
-#if defined(O_DIRECT)
- if (*fd < 0 && (flags & O_DIRECT) &&
- (errno == EINVAL || errno == EAFNOSUPPORT)) {
- flags &= ~(O_DIRECT | O_EXCL);
- *fd = open(pathname, flags, unix_mode_bits);
- }
-#endif /* O_DIRECT */
-
- /* Safeguard for todo4recovery://erased_by_github/libmdbx/issues/144 */
-#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
- if (*fd == STDIN_FILENO) {
- mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN",
- STDIN_FILENO);
- assert(stub_fd0 == -1);
- *fd = dup(stub_fd0 = *fd);
- }
- if (*fd == STDOUT_FILENO) {
- mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT",
- STDOUT_FILENO);
- assert(stub_fd1 == -1);
- *fd = dup(stub_fd1 = *fd);
- }
- if (*fd == STDERR_FILENO) {
- mdbx_warning("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR",
- STDERR_FILENO);
- assert(stub_fd2 == -1);
- *fd = dup(stub_fd2 = *fd);
- }
- if (stub_fd0 != -1)
- close(stub_fd0);
- if (stub_fd1 != -1)
- close(stub_fd1);
- if (stub_fd2 != -1)
- close(stub_fd2);
- if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) {
- mdbx_error(
- "Rejecting the use of a FD in the range "
- "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption",
- STDIN_FILENO, STDERR_FILENO);
- close(*fd);
- return EBADF;
- }
-#else
-#error "Unexpected or unsupported UNIX or POSIX system"
-#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
-
- if (*fd < 0)
- return errno;
-
-#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC)
- const int fd_flags = fcntl(*fd, F_GETFD);
- if (fd_flags != -1)
- (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC);
-#endif /* FD_CLOEXEC && !O_CLOEXEC */
-
- if (direct_nocache_for_copy) {
-#if defined(F_NOCACHE) && !defined(O_NOCACHE)
- (void)fcntl(*fd, F_NOCACHE, 1);
-#endif /* F_NOCACHE */
- }
-
-#endif
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd) {
-#if defined(_WIN32) || defined(_WIN64)
- return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError();
-#else
- assert(fd > STDERR_FILENO);
- return (close(fd) == 0) ? MDBX_SUCCESS : errno;
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t bytes,
- uint64_t offset) {
- if (bytes > MAX_WRITE)
- return MDBX_EINVAL;
-#if defined(_WIN32) || defined(_WIN64)
- OVERLAPPED ov;
- ov.hEvent = 0;
- ov.Offset = (DWORD)offset;
- ov.OffsetHigh = HIGH_DWORD(offset);
-
- DWORD read = 0;
- if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) {
- int rc = (int)GetLastError();
- return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc;
- }
-#else
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- intptr_t read = pread(fd, buf, bytes, offset);
- if (read < 0) {
- int rc = errno;
- return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc;
- }
-#endif
- return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
- size_t bytes, uint64_t offset) {
- while (true) {
-#if defined(_WIN32) || defined(_WIN64)
- OVERLAPPED ov;
- ov.hEvent = 0;
- ov.Offset = (DWORD)offset;
- ov.OffsetHigh = HIGH_DWORD(offset);
-
- DWORD written;
- if (unlikely(!WriteFile(
- fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE,
- &written, &ov)))
- return (int)GetLastError();
- if (likely(bytes == written))
- return MDBX_SUCCESS;
-#else
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- const intptr_t written =
- pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset);
- if (likely(bytes == (size_t)written))
- return MDBX_SUCCESS;
- if (written < 0) {
- const int rc = errno;
- if (rc != EINTR)
- return rc;
- continue;
- }
-#endif
- bytes -= written;
- offset += written;
- buf = (char *)buf + written;
- }
-}
-
-MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
- size_t bytes) {
- while (true) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD written;
- if (unlikely(!WriteFile(
- fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE,
- &written, nullptr)))
- return (int)GetLastError();
- if (likely(bytes == written))
- return MDBX_SUCCESS;
-#else
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- const intptr_t written =
- write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE);
- if (likely(bytes == (size_t)written))
- return MDBX_SUCCESS;
- if (written < 0) {
- const int rc = errno;
- if (rc != EINTR)
- return rc;
- continue;
- }
-#endif
- bytes -= written;
- buf = (char *)buf + written;
- }
-}
-
-int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt,
- uint64_t offset, size_t expected_written) {
-#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || \
- (defined(__ANDROID_API__) && __ANDROID_API__ < 24)
- size_t written = 0;
- for (int i = 0; i < iovcnt; ++i) {
- int rc = mdbx_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);
- if (unlikely(rc != MDBX_SUCCESS))
- return rc;
- written += iov[i].iov_len;
- offset += iov[i].iov_len;
- }
- return (expected_written == written) ? MDBX_SUCCESS
- : MDBX_EIO /* ERROR_WRITE_FAULT */;
-#else
- int rc;
- intptr_t written;
- do {
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- written = pwritev(fd, iov, iovcnt, offset);
- if (likely(expected_written == (size_t)written))
- return MDBX_SUCCESS;
- rc = errno;
- } while (rc == EINTR);
- return (written < 0) ? rc : MDBX_EIO /* Use which error code? */;
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
- enum mdbx_syncmode_bits mode_bits) {
-#if defined(_WIN32) || defined(_WIN64)
- if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd))
- return (int)GetLastError();
- return MDBX_SUCCESS;
-#else
-
-#if defined(__APPLE__) && \
- MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY
- if (mode_bits & MDBX_SYNC_IODQ)
- return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno;
-#endif /* MacOS */
-
- /* LY: This approach is always safe and without appreciable performance
- * degradation, even on a kernel with fdatasync's bug.
- *
- * For more info about of a corresponding fdatasync() bug
- * see http://www.spinics.net/lists/linux-ext4/msg33714.html */
- while (1) {
- switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) {
- case MDBX_SYNC_NONE:
- return MDBX_SUCCESS /* nothing to do */;
-#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
- case MDBX_SYNC_DATA:
- if (fdatasync(fd) == 0)
- return MDBX_SUCCESS;
- break /* error */;
-#if defined(__linux__) || defined(__gnu_linux__)
- case MDBX_SYNC_SIZE:
- if (mdbx_linux_kernel_version >= 0x03060000)
- return MDBX_SUCCESS;
- __fallthrough /* fall through */;
-#endif /* Linux */
-#endif /* _POSIX_SYNCHRONIZED_IO > 0 */
- default:
- if (fsync(fd) == 0)
- return MDBX_SUCCESS;
- }
-
- int rc = errno;
- if (rc != EINTR)
- return rc;
- }
-#endif
-}
-
-int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length) {
-#if defined(_WIN32) || defined(_WIN64)
- BY_HANDLE_FILE_INFORMATION info;
- if (!GetFileInformationByHandle(fd, &info))
- return (int)GetLastError();
- *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
-#else
- struct stat st;
-
- STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- if (fstat(fd, &st))
- return errno;
-
- *length = st.st_size;
-#endif
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd) {
-#if defined(_WIN32) || defined(_WIN64)
- switch (GetFileType(fd)) {
- case FILE_TYPE_DISK:
- return MDBX_RESULT_FALSE;
- case FILE_TYPE_CHAR:
- case FILE_TYPE_PIPE:
- return MDBX_RESULT_TRUE;
- default:
- return (int)GetLastError();
- }
-#else
- struct stat info;
- if (fstat(fd, &info))
- return errno;
- switch (info.st_mode & S_IFMT) {
- case S_IFBLK:
- case S_IFREG:
- return MDBX_RESULT_FALSE;
- case S_IFCHR:
- case S_IFIFO:
- case S_IFSOCK:
- return MDBX_RESULT_TRUE;
- case S_IFDIR:
- case S_IFLNK:
- default:
- return MDBX_INCOMPATIBLE;
- }
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) {
-#if defined(_WIN32) || defined(_WIN64)
- if (mdbx_SetFileInformationByHandle) {
- FILE_END_OF_FILE_INFO EndOfFileInfo;
- EndOfFileInfo.EndOfFile.QuadPart = length;
- return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo,
- &EndOfFileInfo,
- sizeof(FILE_END_OF_FILE_INFO))
- ? MDBX_SUCCESS
- : (int)GetLastError();
- } else {
- LARGE_INTEGER li;
- li.QuadPart = length;
- return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd))
- ? MDBX_SUCCESS
- : (int)GetLastError();
- }
-#else
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno;
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos) {
-#if defined(_WIN32) || defined(_WIN64)
- LARGE_INTEGER li;
- li.QuadPart = pos;
- return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS
- : (int)GetLastError();
-#else
- STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t),
- "libmdbx requires 64-bit file I/O on 64-bit systems");
- return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS;
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-MDBX_INTERNAL_FUNC int
-mdbx_thread_create(mdbx_thread_t *thread,
- THREAD_RESULT(THREAD_CALL *start_routine)(void *),
- void *arg) {
-#if defined(_WIN32) || defined(_WIN64)
- *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL);
- return *thread ? MDBX_SUCCESS : (int)GetLastError();
-#else
- return pthread_create(thread, NULL, start_routine, arg);
-#endif
-}
-
-MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD code = WaitForSingleObject(thread, INFINITE);
- return waitstatus2errcode(code);
-#else
- void *unused_retval = &unused_retval;
- return pthread_join(thread, &unused_retval);
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
- size_t length,
- enum mdbx_syncmode_bits mode_bits) {
- uint8_t *ptr = (uint8_t *)map->address + offset;
-#if defined(_WIN32) || defined(_WIN64)
- if (!FlushViewOfFile(ptr, length))
- return (int)GetLastError();
-#else
-#if defined(__linux__) || defined(__gnu_linux__)
- if (mode_bits == MDBX_SYNC_NONE && mdbx_linux_kernel_version > 0x02061300)
- /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly
- * tracks dirty pages and flushes them to storage as necessary. */
- return MDBX_SUCCESS;
-#endif /* Linux */
- if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC))
- return errno;
- mode_bits &= ~MDBX_SYNC_DATA;
-#endif
- return mdbx_fsync(map->fd, mode_bits);
-}
-
-MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
- const char *pathname, int err) {
-#if defined(_WIN32) || defined(_WIN64)
- (void)pathname;
- (void)err;
- if (!mdbx_GetVolumeInformationByHandleW)
- return MDBX_ENOSYS;
- DWORD unused, flags;
- if (!mdbx_GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused,
- &flags, nullptr, 0))
- return (int)GetLastError();
- if ((flags & FILE_READ_ONLY_VOLUME) == 0)
- return MDBX_EACCESS;
-#else
- struct statvfs info;
- if (err != MDBX_ENOFILE) {
- if (statvfs(pathname, &info))
- return errno;
- if ((info.f_flag & ST_RDONLY) == 0)
- return err;
- }
- if (fstatvfs(handle, &info))
- return errno;
- if ((info.f_flag & ST_RDONLY) == 0)
- return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err;
-#endif /* !Windows */
- return MDBX_SUCCESS;
-}
-
-static int mdbx_check_fs_local(mdbx_filehandle_t handle, int flags) {
-#if defined(_WIN32) || defined(_WIN64)
- if (mdbx_RunningUnderWine() && !(flags & MDBX_EXCLUSIVE))
- return ERROR_NOT_CAPABLE /* workaround for Wine */;
-
- if (GetFileType(handle) != FILE_TYPE_DISK)
- return ERROR_FILE_OFFLINE;
-
- if (mdbx_GetFileInformationByHandleEx) {
- FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo;
- if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo,
- &RemoteProtocolInfo,
- sizeof(RemoteProtocolInfo))) {
- if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) &&
- !(flags & MDBX_RDONLY))
- return ERROR_FILE_OFFLINE;
- if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) &&
- !(flags & MDBX_EXCLUSIVE))
- return ERROR_REMOTE_STORAGE_MEDIA_ERROR;
- }
- }
-
- if (mdbx_NtFsControlFile) {
- NTSTATUS rc;
- struct {
- WOF_EXTERNAL_INFO wof_info;
- union {
- WIM_PROVIDER_EXTERNAL_INFO wim_info;
- FILE_PROVIDER_EXTERNAL_INFO_V1 file_info;
- };
- size_t reserved_for_microsoft_madness[42];
- } GetExternalBacking_OutputBuffer;
- IO_STATUS_BLOCK StatusBlock;
- rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock,
- FSCTL_GET_EXTERNAL_BACKING, NULL, 0,
- &GetExternalBacking_OutputBuffer,
- sizeof(GetExternalBacking_OutputBuffer));
- if (NT_SUCCESS(rc)) {
- if (!(flags & MDBX_EXCLUSIVE))
- return ERROR_REMOTE_STORAGE_MEDIA_ERROR;
- } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED &&
- rc != STATUS_INVALID_DEVICE_REQUEST &&
- rc != STATUS_NOT_SUPPORTED)
- return ntstatus2errcode(rc);
- }
-
- if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) {
- WCHAR *PathBuffer = mdbx_malloc(sizeof(WCHAR) * INT16_MAX);
- if (!PathBuffer)
- return MDBX_ENOMEM;
-
- int rc = MDBX_SUCCESS;
- DWORD VolumeSerialNumber, FileSystemFlags;
- if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX,
- &VolumeSerialNumber, NULL,
- &FileSystemFlags, NULL, 0)) {
- rc = (int)GetLastError();
- goto bailout;
- }
-
- if ((flags & MDBX_RDONLY) == 0) {
- if (FileSystemFlags &
- (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME |
- FILE_VOLUME_IS_COMPRESSED)) {
- rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
- goto bailout;
- }
- }
-
- if (!mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX,
- FILE_NAME_NORMALIZED |
- VOLUME_NAME_NT)) {
- rc = (int)GetLastError();
- goto bailout;
- }
-
- if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) {
- if (!(flags & MDBX_EXCLUSIVE)) {
- rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
- goto bailout;
- }
- } else if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX,
- FILE_NAME_NORMALIZED |
- VOLUME_NAME_DOS)) {
- UINT DriveType = GetDriveTypeW(PathBuffer);
- if (DriveType == DRIVE_NO_ROOT_DIR &&
- _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 &&
- _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) {
- PathBuffer[7] = 0;
- DriveType = GetDriveTypeW(PathBuffer + 4);
- }
- switch (DriveType) {
- case DRIVE_CDROM:
- if (flags & MDBX_RDONLY)
- break;
- // fall through
- case DRIVE_UNKNOWN:
- case DRIVE_NO_ROOT_DIR:
- case DRIVE_REMOTE:
- default:
- if (!(flags & MDBX_EXCLUSIVE))
- rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR;
- // fall through
- case DRIVE_REMOVABLE:
- case DRIVE_FIXED:
- case DRIVE_RAMDISK:
- break;
- }
- }
- bailout:
- mdbx_free(PathBuffer);
- return rc;
- }
-
-#else
-
- struct statvfs statvfs_info;
- if (fstatvfs(handle, &statvfs_info))
- return errno;
-#if defined(ST_LOCAL) || defined(ST_EXPORTED)
- const unsigned long st_flags = statvfs_info.f_flag;
-#endif /* ST_LOCAL || ST_EXPORTED */
-
-#if defined(__NetBSD__)
- const unsigned type = 0;
- const char *const name = statvfs_info.f_fstypename;
- const size_t name_len = VFS_NAMELEN;
-#elif defined(_AIX) || defined(__OS400__)
- const char *const name = statvfs_info.f_basetype;
- const size_t name_len = sizeof(statvfs_info.f_basetype);
- struct stat st;
- if (fstat(handle, &st))
- return errno;
- const unsigned type = st.st_vfstype;
- if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE))
- return MDBX_EREMOTE;
-#elif defined(FSTYPSZ) || defined(_FSTYPSZ)
- const unsigned type = 0;
- const char *const name = statvfs_info.f_basetype;
- const size_t name_len = sizeof(statvfs_info.f_basetype);
-#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
- defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ)
- const unsigned type = 0;
- struct stat st;
- if (fstat(handle, &st))
- return errno;
- const char *const name = st.st_fstype;
- const size_t name_len = strlen(name);
-#else
- struct statfs statfs_info;
- if (fstatfs(handle, &statfs_info))
- return errno;
-#if defined(__OpenBSD__)
- const unsigned type = 0;
-#else
- const unsigned type = statfs_info.f_type;
-#endif
-#if defined(MNT_LOCAL) || defined(MNT_EXPORTED)
- const unsigned long mnt_flags = statfs_info.f_flags;
-#endif /* MNT_LOCAL || MNT_EXPORTED */
-#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
- defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
- defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
- defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
- const char *const name = statfs_info.f_fstypename;
- const size_t name_len = sizeof(statfs_info.f_fstypename);
-#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21
- const char *const name = "";
- const unsigned name_len = 0;
-#else
-
- const char *name = "";
- unsigned name_len = 0;
-
- struct stat st;
- if (fstat(handle, &st))
- return errno;
-
- char pathbuf[PATH_MAX];
- FILE *mounted = nullptr;
-#if defined(__linux__) || defined(__gnu_linux__)
- mounted = setmntent("/proc/mounts", "r");
-#endif /* Linux */
- if (!mounted)
- mounted = setmntent("/etc/mtab", "r");
- if (mounted) {
- const struct mntent *ent;
-#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \
- (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19))
- struct mntent entbuf;
- const bool should_copy = false;
- while (nullptr !=
- (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf))))
-#else
- const bool should_copy = true;
- while (nullptr != (ent = getmntent(mounted)))
-#endif
- {
- struct stat mnt;
- if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) {
- if (should_copy) {
- name =
- strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1);
- pathbuf[name_len] = 0;
- } else {
- name = ent->mnt_fsname;
- name_len = strlen(name);
- }
- break;
- }
- }
- endmntent(mounted);
- }
-#endif /* !xBSD && !Android/Bionic */
-#endif
-
- if (name_len) {
- if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) ||
- strncasecmp("cifs", name, name_len) == 0 ||
- strncasecmp("ncpfs", name, name_len) == 0 ||
- strncasecmp("smbfs", name, name_len) == 0 ||
- strcasecmp("9P" /* WSL2 */, name) == 0 ||
- ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) &&
- strncasecmp("fuseblk", name, name_len) != 0)) &&
- !(flags & MDBX_EXCLUSIVE))
- return MDBX_EREMOTE;
- if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 ||
- strcasecmp("sshfs", name) == 0)
- return MDBX_EREMOTE;
- }
-
-#ifdef ST_LOCAL
- if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
- return MDBX_EREMOTE;
-#elif defined(MNT_LOCAL)
- if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
- return MDBX_EREMOTE;
-#endif /* ST/MNT_LOCAL */
-
-#ifdef ST_EXPORTED
- if ((st_flags & ST_EXPORTED) != 0 && !(flags & MDBX_RDONLY))
- return MDBX_EREMOTE;
-#elif defined(MNT_EXPORTED)
- if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & MDBX_RDONLY))
- return MDBX_EREMOTE;
-#endif /* ST/MNT_EXPORTED */
-
- switch (type) {
- case 0xFF534D42 /* CIFS_MAGIC_NUMBER */:
- case 0x6969 /* NFS_SUPER_MAGIC */:
- case 0x564c /* NCP_SUPER_MAGIC */:
- case 0x517B /* SMB_SUPER_MAGIC */:
-#if defined(__digital__) || defined(__osf__) || defined(__osf)
- case 0x0E /* Tru64 NFS */:
-#endif
-#ifdef ST_FST_NFS
- case ST_FST_NFS:
-#endif
- if ((flags & MDBX_EXCLUSIVE) == 0)
- return MDBX_EREMOTE;
- case 0:
- default:
- break;
- }
-#endif /* Unix */
-
- return MDBX_SUCCESS;
-}
-
-static int check_mmap_limit(const size_t limit) {
- const bool should_check =
-#if defined(__SANITIZE_ADDRESS__)
- true;
-#else
- RUNNING_ON_VALGRIND;
-#endif /* __SANITIZE_ADDRESS__ */
-
- if (should_check) {
- intptr_t pagesize, total_ram_pages, avail_ram_pages;
- int err =
- mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- const int log2page = log2n_powerof2(pagesize);
- if ((limit >> (log2page + 7)) > (size_t)total_ram_pages ||
- (limit >> (log2page + 6)) > (size_t)avail_ram_pages) {
- mdbx_error(
- "%s (%zu pages) is too large for available (%zu pages) or total "
- "(%zu pages) system RAM",
- "database upper size limit", limit >> log2page, avail_ram_pages,
- total_ram_pages);
- return MDBX_TOO_LARGE;
- }
- }
-
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
- const size_t size, const size_t limit,
- const unsigned options) {
- assert(size <= limit);
- map->limit = 0;
- map->current = 0;
- map->address = nullptr;
- map->filesize = 0;
-#if defined(_WIN32) || defined(_WIN64)
- map->section = NULL;
-#endif /* Windows */
-
- int err = mdbx_check_fs_local(map->fd, flags);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- err = check_mmap_limit(limit);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) {
- err = mdbx_ftruncate(map->fd, size);
- if (err != MDBX_SUCCESS)
- return err;
- map->filesize = size;
-#if !(defined(_WIN32) || defined(_WIN64))
- map->current = size;
-#endif /* !Windows */
- } else {
- err = mdbx_filesize(map->fd, &map->filesize);
- if (err != MDBX_SUCCESS)
- return err;
-#if !(defined(_WIN32) || defined(_WIN64))
- map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
-#endif /* !Windows */
- }
-
-#if defined(_WIN32) || defined(_WIN64)
- LARGE_INTEGER SectionSize;
- SectionSize.QuadPart = size;
- err = NtCreateSection(
- &map->section,
- /* DesiredAccess */
- (flags & MDBX_WRITEMAP)
- ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
- SECTION_MAP_WRITE
- : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
- /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize,
- /* SectionPageProtection */
- (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
- /* AllocationAttributes */ SEC_RESERVE, map->fd);
- if (!NT_SUCCESS(err))
- return ntstatus2errcode(err);
-
- SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0
- : mdbx_RunningUnderWine() ? size
- : limit;
- err = NtMapViewOfSection(
- map->section, GetCurrentProcess(), &map->address,
- /* ZeroBits */ 0,
- /* CommitSize */ 0,
- /* SectionOffset */ NULL, &ViewSize,
- /* InheritDisposition */ ViewUnmap,
- /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
- /* Win32Protect */
- (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
- if (!NT_SUCCESS(err)) {
- NtClose(map->section);
- map->section = 0;
- map->address = nullptr;
- return ntstatus2errcode(err);
- }
- assert(map->address != MAP_FAILED);
-
- map->current = (size_t)SectionSize.QuadPart;
- map->limit = ViewSize;
-
-#else /* Windows */
-
-#ifndef MAP_TRYFIXED
-#define MAP_TRYFIXED 0
-#endif
-
-#ifndef MAP_HASSEMAPHORE
-#define MAP_HASSEMAPHORE 0
-#endif
-
-#ifndef MAP_CONCEAL
-#define MAP_CONCEAL 0
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0
-#endif
-
-#ifndef MAP_FIXED_NOREPLACE
-#define MAP_FIXED_NOREPLACE 0
-#endif
-
-#ifndef MAP_NORESERVE
-#define MAP_NORESERVE 0
-#endif
-
- map->address = mmap(
- NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ,
- MAP_SHARED | MAP_FILE | MAP_NORESERVE |
- (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) |
- ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC
- : MAP_CONCEAL),
- map->fd, 0);
-
- if (unlikely(map->address == MAP_FAILED)) {
- map->limit = 0;
- map->current = 0;
- map->address = nullptr;
- return errno;
- }
- map->limit = limit;
-
-#if MDBX_ENABLE_MADVISE
-#ifdef MADV_DONTFORK
- if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0))
- return errno;
-#endif /* MADV_DONTFORK */
-#ifdef MADV_NOHUGEPAGE
- (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE);
-#endif /* MADV_NOHUGEPAGE */
-#endif /* MDBX_ENABLE_MADVISE */
-
-#endif /* ! Windows */
-
- VALGRIND_MAKE_MEM_DEFINED(map->address, map->current);
- MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address, map->current);
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map) {
- VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
- /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
- * when this memory will re-used by malloc or another mmapping.
- * See todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
- */
- MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address,
- (map->filesize && map->filesize < map->limit)
- ? map->filesize
- : map->limit);
-#if defined(_WIN32) || defined(_WIN64)
- if (map->section)
- NtClose(map->section);
- NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->address);
- if (!NT_SUCCESS(rc))
- ntstatus2errcode(rc);
-#else
- if (unlikely(munmap(map->address, map->limit)))
- return errno;
-#endif /* ! Windows */
-
- map->limit = 0;
- map->current = 0;
- map->address = nullptr;
- return MDBX_SUCCESS;
-}
-
-MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
- size_t size, size_t limit) {
- assert(size <= limit);
-#if defined(_WIN32) || defined(_WIN64)
- assert(size != map->current || limit != map->limit || size < map->filesize);
-
- NTSTATUS status;
- LARGE_INTEGER SectionSize;
- int err, rc = MDBX_SUCCESS;
-
- if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current &&
- /* workaround for Wine */ mdbx_NtExtendSection) {
- /* growth rw-section */
- SectionSize.QuadPart = size;
- status = mdbx_NtExtendSection(map->section, &SectionSize);
- if (!NT_SUCCESS(status))
- return ntstatus2errcode(status);
- map->current = size;
- if (map->filesize < size)
- map->filesize = size;
- return MDBX_SUCCESS;
- }
-
- if (limit > map->limit) {
- err = check_mmap_limit(limit);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- /* check ability of address space for growth before unmap */
- PVOID BaseAddress = (PBYTE)map->address + map->limit;
- SIZE_T RegionSize = limit - map->limit;
- status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0,
- &RegionSize, MEM_RESERVE, PAGE_NOACCESS);
- if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
- return MDBX_UNABLE_EXTEND_MAPSIZE;
- if (!NT_SUCCESS(status))
- return ntstatus2errcode(status);
-
- status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize,
- MEM_RELEASE);
- if (!NT_SUCCESS(status))
- return ntstatus2errcode(status);
- }
-
- /* Windows unable:
- * - shrink a mapped file;
- * - change size of mapped view;
- * - extend read-only mapping;
- * Therefore we should unmap/map entire section. */
- if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0)
- return MDBX_EPERM;
-
- /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
- * when this memory will re-used by malloc or another mmapping.
- * See todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
- */
- MDBX_ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit);
- status = NtUnmapViewOfSection(GetCurrentProcess(), map->address);
- if (!NT_SUCCESS(status))
- return ntstatus2errcode(status);
- status = NtClose(map->section);
- map->section = NULL;
- PVOID ReservedAddress = NULL;
- SIZE_T ReservedSize = limit;
-
- if (!NT_SUCCESS(status)) {
- bailout_ntstatus:
- err = ntstatus2errcode(status);
- bailout:
- map->address = NULL;
- map->current = map->limit = 0;
- if (ReservedAddress) {
- ReservedSize = 0;
- status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress,
- &ReservedSize, MEM_RELEASE);
- assert(NT_SUCCESS(status));
- (void)status;
- }
- return err;
- }
-
-retry_file_and_section:
- /* resizing of the file may take a while,
- * therefore we reserve address space to avoid occupy it by other threads */
- ReservedAddress = map->address;
- status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0,
- &ReservedSize, MEM_RESERVE, PAGE_NOACCESS);
- if (!NT_SUCCESS(status)) {
- ReservedAddress = NULL;
- if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
- goto bailout_ntstatus /* no way to recovery */;
-
- if (flags & MDBX_MRESIZE_MAY_MOVE)
- /* the base address could be changed */
- map->address = NULL;
- }
-
- err = mdbx_filesize(map->fd, &map->filesize);
- if (err != MDBX_SUCCESS)
- goto bailout;
-
- if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) {
- err = mdbx_ftruncate(map->fd, size);
- if (err == MDBX_SUCCESS)
- map->filesize = size;
- /* ignore error, because Windows unable shrink file
- * that already mapped (by another process) */
- }
-
- SectionSize.QuadPart = size;
- status = NtCreateSection(
- &map->section,
- /* DesiredAccess */
- (flags & MDBX_WRITEMAP)
- ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
- SECTION_MAP_WRITE
- : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
- /* ObjectAttributes */ NULL,
- /* MaximumSize (InitialSize) */ &SectionSize,
- /* SectionPageProtection */
- (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
- /* AllocationAttributes */ SEC_RESERVE, map->fd);
-
- if (!NT_SUCCESS(status))
- goto bailout_ntstatus;
-
- if (ReservedAddress) {
- /* release reserved address space */
- ReservedSize = 0;
- status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress,
- &ReservedSize, MEM_RELEASE);
- ReservedAddress = NULL;
- if (!NT_SUCCESS(status))
- goto bailout_ntstatus;
- }
-
-retry_mapview:;
- SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit;
- status = NtMapViewOfSection(
- map->section, GetCurrentProcess(), &map->address,
- /* ZeroBits */ 0,
- /* CommitSize */ 0,
- /* SectionOffset */ NULL, &ViewSize,
- /* InheritDisposition */ ViewUnmap,
- /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
- /* Win32Protect */
- (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
-
- if (!NT_SUCCESS(status)) {
- if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 &&
- map->address && (flags & MDBX_MRESIZE_MAY_MOVE) != 0) {
- /* try remap at another base address */
- map->address = NULL;
- goto retry_mapview;
- }
- NtClose(map->section);
- map->section = NULL;
-
- if (map->address && (size != map->current || limit != map->limit)) {
- /* try remap with previously size and limit,
- * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
- rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
- size = map->current;
- ReservedSize = limit = map->limit;
- goto retry_file_and_section;
- }
-
- /* no way to recovery */
- goto bailout_ntstatus;
- }
- assert(map->address != MAP_FAILED);
-
- map->current = (size_t)SectionSize.QuadPart;
- map->limit = ViewSize;
-
-#else /* Windows */
-
- map->filesize = 0;
- int rc = mdbx_filesize(map->fd, &map->filesize);
- if (rc != MDBX_SUCCESS)
- return rc;
-
- if (flags & MDBX_RDONLY) {
- map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
- if (map->current != size)
- rc = (size > map->current) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
- } else {
- if (map->filesize != size) {
- rc = mdbx_ftruncate(map->fd, size);
- if (rc != MDBX_SUCCESS)
- return rc;
- map->filesize = size;
- }
-
- if (map->current > size) {
- /* Clearing asan's bitmask for the region which released in shrinking,
- * since:
- * - after the shrinking we will get an exception when accessing
- * this region and (therefore) do not need the help of ASAN.
- * - this allows us to clear the mask only within the file size
- * when closing the mapping. */
- MDBX_ASAN_UNPOISON_MEMORY_REGION(
- (char *)map->address + size,
- ((map->current < map->limit) ? map->current : map->limit) - size);
- }
- map->current = size;
- }
-
- if (limit == map->limit)
- return rc;
-
- if (limit < map->limit) {
- /* unmap an excess at end of mapping. */
- // coverity[offset_free : FALSE]
- if (unlikely(munmap(map->dxb + limit, map->limit - limit)))
- return errno;
- map->limit = limit;
- return rc;
- }
-
- int err = check_mmap_limit(limit);
- if (unlikely(err != MDBX_SUCCESS))
- return err;
-
- assert(limit > map->limit);
- uint8_t *ptr = MAP_FAILED;
-
-#if defined(MREMAP_MAYMOVE)
- ptr = mremap(map->address, map->limit, limit,
- (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE : 0);
- if (ptr == MAP_FAILED) {
- err = errno;
- switch (err) {
- default:
- return err;
- case EAGAIN:
- case ENOMEM:
- return MDBX_UNABLE_EXTEND_MAPSIZE;
- case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
- break;
- }
- }
-#endif /* MREMAP_MAYMOVE */
-
- const unsigned mmap_flags =
- MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE |
- (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0);
- const unsigned mmap_prot =
- (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ;
-
- if (ptr == MAP_FAILED) {
- /* Try to mmap additional space beyond the end of mapping. */
- ptr = mmap(map->dxb + map->limit, limit - map->limit, mmap_prot,
- mmap_flags | MAP_FIXED_NOREPLACE, map->fd, map->limit);
- if (ptr == map->dxb + map->limit)
- ptr = map->dxb;
- else if (ptr != MAP_FAILED) {
- /* the desired address is busy, unmap unsuitable one */
- if (unlikely(munmap(ptr, limit - map->limit)))
- return errno;
- ptr = MAP_FAILED;
- } else {
- err = errno;
- switch (err) {
- default:
- return err;
- case EAGAIN:
- case ENOMEM:
- return MDBX_UNABLE_EXTEND_MAPSIZE;
- case EEXIST: /* address busy */
- case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */
- break;
- }
- }
- }
-
- if (ptr == MAP_FAILED) {
- /* unmap and map again whole region */
- if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
- /* TODO: Perhaps here it is worth to implement suspend/resume threads
- * and perform unmap/map as like for Windows. */
- return MDBX_UNABLE_EXTEND_MAPSIZE;
- }
-
- if (unlikely(munmap(map->address, map->limit)))
- return errno;
-
- // coverity[pass_freed_arg : FALSE]
- ptr = mmap(map->address, limit, mmap_prot,
- (flags & MDBX_MRESIZE_MAY_MOVE)
- ? mmap_flags
- : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE
- : MAP_FIXED),
- map->fd, 0);
- if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED &&
- unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) &&
- errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
- // coverity[pass_freed_arg : FALSE]
- ptr = mmap(map->address, limit, mmap_prot, mmap_flags | MAP_FIXED,
- map->fd, 0);
-
- if (unlikely(ptr == MAP_FAILED)) {
- /* try to restore prev mapping */
- // coverity[pass_freed_arg : FALSE]
- ptr = mmap(map->address, map->limit, mmap_prot,
- (flags & MDBX_MRESIZE_MAY_MOVE)
- ? mmap_flags
- : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE
- : MAP_FIXED),
- map->fd, 0);
- if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED &&
- unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) &&
- errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
- // coverity[pass_freed_arg : FALSE]
- ptr = mmap(map->address, map->limit, mmap_prot, mmap_flags | MAP_FIXED,
- map->fd, 0);
- if (unlikely(ptr == MAP_FAILED)) {
- VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
- /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
- * when this memory will re-used by malloc or another mmapping.
- * See
- * todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
- */
- MDBX_ASAN_UNPOISON_MEMORY_REGION(
- map->address,
- (map->current < map->limit) ? map->current : map->limit);
- map->limit = 0;
- map->current = 0;
- map->address = nullptr;
- return errno;
- }
- rc = MDBX_UNABLE_EXTEND_MAPSIZE;
- limit = map->limit;
- }
- }
-
- assert(ptr && ptr != MAP_FAILED);
- if (map->address != ptr) {
- VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current);
- /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
- * when this memory will re-used by malloc or another mmapping.
- * See
- * todo4recovery://erased_by_github/libmdbx/pull/93#issuecomment-613687203
- */
- MDBX_ASAN_UNPOISON_MEMORY_REGION(
- map->address, (map->current < map->limit) ? map->current : map->limit);
-
- VALGRIND_MAKE_MEM_DEFINED(ptr, map->current);
- MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current);
- map->address = ptr;
- }
- map->limit = limit;
-
-#if MDBX_ENABLE_MADVISE
-#ifdef MADV_DONTFORK
- if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0))
- return errno;
-#endif /* MADV_DONTFORK */
-#ifdef MADV_NOHUGEPAGE
- (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE);
-#endif /* MADV_NOHUGEPAGE */
-#endif /* MDBX_ENABLE_MADVISE */
-
-#endif /* POSIX / Windows */
-
- return rc;
-}
-
-/*----------------------------------------------------------------------------*/
-
-__cold MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny) {
- for (;;) {
-#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
- defined(__x86_64__)
- const unsigned salt = 277u * (unsigned)__rdtsc();
-#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT
- static ULONG state;
- const unsigned salt = (unsigned)RtlRandomEx(&state);
-#else
- const unsigned salt = rand();
-#endif
-
- const unsigned coin = salt % (tiny ? 29u : 43u);
- if (coin < 43 / 3)
- break;
-#if defined(_WIN32) || defined(_WIN64)
- SwitchToThread();
- if (coin > 43 * 2 / 3)
- Sleep(1);
-#else
- sched_yield();
- if (coin > 43 * 2 / 3)
- usleep(coin);
-#endif
- }
-}
-
-#if defined(_WIN32) || defined(_WIN64)
-#elif defined(__APPLE__) || defined(__MACH__)
-#include <mach/mach_time.h>
-#elif defined(__linux__) || defined(__gnu_linux__)
-__cold static clockid_t choice_monoclock(void) {
- struct timespec probe;
-#if defined(CLOCK_BOOTTIME)
- if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0)
- return CLOCK_BOOTTIME;
-#elif defined(CLOCK_MONOTONIC_RAW)
- if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0)
- return CLOCK_MONOTONIC_RAW;
-#elif defined(CLOCK_MONOTONIC_COARSE)
- if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0)
- return CLOCK_MONOTONIC_COARSE;
-#endif
- return CLOCK_MONOTONIC;
-}
-#endif
-
-/*----------------------------------------------------------------------------*/
-
-#if defined(_WIN32) || defined(_WIN64)
-static LARGE_INTEGER performance_frequency;
-#elif defined(__APPLE__) || defined(__MACH__)
-static uint64_t ratio_16dot16_to_monotine;
-#endif
-
-MDBX_INTERNAL_FUNC uint64_t
-mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16) {
-#if defined(_WIN32) || defined(_WIN64)
- if (unlikely(performance_frequency.QuadPart == 0))
- QueryPerformanceFrequency(&performance_frequency);
- const uint64_t ratio = performance_frequency.QuadPart;
-#elif defined(__APPLE__) || defined(__MACH__)
- if (unlikely(ratio_16dot16_to_monotine == 0)) {
- mach_timebase_info_data_t ti;
- mach_timebase_info(&ti);
- ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer;
- }
- const uint64_t ratio = ratio_16dot16_to_monotine;
-#else
- const uint64_t ratio = UINT64_C(1000000000);
-#endif
- const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16;
- return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1;
-}
-
-MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime) {
- static uint64_t limit;
- if (unlikely(monotime > limit)) {
- if (limit != 0)
- return UINT32_MAX;
- limit = mdbx_osal_16dot16_to_monotime(UINT32_MAX - 1);
- if (monotime > limit)
- return UINT32_MAX;
- }
- const uint32_t ret =
-#if defined(_WIN32) || defined(_WIN64)
- (uint32_t)((monotime << 16) / performance_frequency.QuadPart);
-#elif defined(__APPLE__) || defined(__MACH__)
- (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine);
-#else
- (uint32_t)(monotime * 128 / 1953125);
-#endif
- return likely(ret || monotime == 0) ? ret : /* fix underflow */ 1;
-}
-
-MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void) {
-#if defined(_WIN32) || defined(_WIN64)
- LARGE_INTEGER counter;
- counter.QuadPart = 0;
- QueryPerformanceCounter(&counter);
- return counter.QuadPart;
-#elif defined(__APPLE__) || defined(__MACH__)
- return mach_absolute_time();
-#else
-
-#if defined(__linux__) || defined(__gnu_linux__)
- static clockid_t posix_clockid = -1;
- if (unlikely(posix_clockid < 0))
- posix_clockid = choice_monoclock();
-#elif defined(CLOCK_MONOTONIC)
-#define posix_clockid CLOCK_MONOTONIC
-#else
-#define posix_clockid CLOCK_REALTIME
-#endif
-
- struct timespec ts;
- if (unlikely(clock_gettime(posix_clockid, &ts) != 0)) {
- ts.tv_nsec = 0;
- ts.tv_sec = 0;
- }
- return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-
-static void bootid_shake(bin128_t *p) {
- /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */
- const uint32_t e = p->a - (p->b << 23 | p->b >> 9);
- p->a = p->b ^ (p->c << 16 | p->c >> 16);
- p->b = p->c + (p->d << 11 | p->d >> 21);
- p->c = p->d + e;
- p->d = e + p->a;
-}
-
-static void bootid_collect(bin128_t *p, const void *s, size_t n) {
- p->y += UINT64_C(64526882297375213);
- bootid_shake(p);
- for (size_t i = 0; i < n; ++i) {
- bootid_shake(p);
- p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i];
- bootid_shake(p);
- p->y += 14621231;
- }
- bootid_shake(p);
-
- /* minor non-linear tomfoolery */
- const unsigned z = p->x % 61;
- p->y = p->y << z | p->y >> (64 - z);
- bootid_shake(p);
- bootid_shake(p);
- const unsigned q = p->x % 59;
- p->y = p->y << q | p->y >> (64 - q);
- bootid_shake(p);
- bootid_shake(p);
- bootid_shake(p);
-}
-
-#if defined(_WIN32) || defined(_WIN64)
-
-static uint64_t windows_systemtime_ms() {
- FILETIME ft;
- GetSystemTimeAsFileTime(&ft);
- return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul;
-}
-
-static uint64_t windows_bootime(void) {
- unsigned confirmed = 0;
- uint64_t boottime = 0;
- uint64_t up0 = mdbx_GetTickCount64();
- uint64_t st0 = windows_systemtime_ms();
- for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) {
- YieldProcessor();
- const uint64_t up1 = mdbx_GetTickCount64();
- const uint64_t st1 = windows_systemtime_ms();
- if (st1 > fuse && st1 == st0 && up1 == up0) {
- uint64_t diff = st1 - up1;
- if (boottime == diff) {
- if (++confirmed > 4)
- return boottime;
- } else {
- confirmed = 0;
- boottime = diff;
- }
- fuse = st1;
- Sleep(1);
- }
- st0 = st1;
- up0 = up1;
- }
- return 0;
-}
-
-static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue,
- PVOID pvData, LPDWORD pcbData) {
- LSTATUS rc;
- if (!mdbx_RegGetValueA) {
- /* an old Windows 2000/XP */
- HKEY hSubKey;
- rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey);
- if (rc == ERROR_SUCCESS) {
- rc = RegQueryValueExA(hSubKey, lpValue, NULL, NULL, pvData, pcbData);
- RegCloseKey(hSubKey);
- }
- return rc;
- }
-
- rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, NULL, pvData,
- pcbData);
- if (rc != ERROR_FILE_NOT_FOUND)
- return rc;
-
- rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue,
- RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */,
- NULL, pvData, pcbData);
- if (rc != ERROR_FILE_NOT_FOUND)
- return rc;
- return mdbx_RegGetValueA(hKey, lpSubKey, lpValue,
- RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */,
- NULL, pvData, pcbData);
-}
-#endif
-
-__cold MDBX_MAYBE_UNUSED static bool
-bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) {
- if (n > 31) {
- unsigned bits = 0;
- for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ {
- uint8_t c = ((const uint8_t *)p)[i];
- if (c >= '0' && c <= '9')
- c -= '0';
- else if (c >= 'a' && c <= 'f')
- c -= 'a' - 10;
- else if (c >= 'A' && c <= 'F')
- c -= 'A' - 10;
- else
- continue;
- assert(c <= 15);
- c ^= s->y >> 60;
- s->y = s->y << 4 | s->x >> 60;
- s->x = s->x << 4 | c;
- bits += 4;
- }
- if (bits > 42 * 3)
- /* UUID parsed successfully */
- return true;
- }
-
- if (n > 15) /* is enough handle it as a binary? */ {
- if (n == sizeof(bin128_t)) {
- bin128_t aligned;
- memcpy(&aligned, p, sizeof(bin128_t));
- s->x += aligned.x;
- s->y += aligned.y;
- } else
- bootid_collect(s, p, n);
- return true;
- }
-
- if (n)
- bootid_collect(s, p, n);
- return false;
-}
-
-__cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) {
- bin128_t bin = {{0, 0}};
- bool got_machineid = false, got_boottime = false, got_bootseq = false;
-
-#if defined(__linux__) || defined(__gnu_linux__)
- {
- const int fd =
- open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_NOFOLLOW);
- if (fd != -1) {
- struct statfs fs;
- char buf[42];
- const ssize_t len =
- (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0)
- ? read(fd, buf, sizeof(buf))
- : -1;
- const int err = close(fd);
- assert(err == 0);
- (void)err;
- if (len > 0 && bootid_parse_uuid(&bin, buf, len))
- return bin;
- }
- }
-#endif /* Linux */
-
-#if defined(__APPLE__) || defined(__MACH__)
- {
- char buf[42];
- size_t len = sizeof(buf);
- if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) &&
- bootid_parse_uuid(&bin, buf, len))
- return bin;
-
-#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
- __MAC_OS_X_VERSION_MIN_REQUIRED > 1050
- uuid_t uuid;
- struct timespec wait = {0, 1000000000u / 42};
- if (!gethostuuid(uuid, &wait) &&
- bootid_parse_uuid(&bin, uuid, sizeof(uuid)))
- got_machineid = true;
-#endif /* > 10.5 */
-
- struct timeval boottime;
- len = sizeof(boottime);
- if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) &&
- len == sizeof(boottime) && boottime.tv_sec)
- got_boottime = true;
- }
-#endif /* Apple/Darwin */
-
-#if defined(_WIN32) || defined(_WIN64)
- {
- union buf {
- DWORD BootId;
- DWORD BaseTime;
- SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo;
- struct {
- LARGE_INTEGER BootTime;
- LARGE_INTEGER CurrentTime;
- LARGE_INTEGER TimeZoneBias;
- ULONG TimeZoneId;
- ULONG Reserved;
- ULONGLONG BootTimeBias;
- ULONGLONG SleepTimeBias;
- } SysTimeOfDayInfoHacked;
- wchar_t MachineGuid[42];
- char DigitalProductId[248];
- } buf;
-
- static const char HKLM_MicrosoftCryptography[] =
- "SOFTWARE\\Microsoft\\Cryptography";
- DWORD len = sizeof(buf);
- /* Windows is madness and must die */
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography,
- "MachineGuid", &buf.MachineGuid,
- &len) == ERROR_SUCCESS &&
- len < sizeof(buf))
- got_machineid = bootid_parse_uuid(&bin, &buf.MachineGuid, len);
-
- if (!got_machineid) {
- /* again, Windows is madness */
- static const char HKLM_WindowsNT[] =
- "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
- static const char HKLM_WindowsNT_DPK[] =
- "SOFTWARE\\Microsoft\\Windows "
- "NT\\CurrentVersion\\DefaultProductKey";
- static const char HKLM_WindowsNT_DPK2[] =
- "SOFTWARE\\Microsoft\\Windows "
- "NT\\CurrentVersion\\DefaultProductKey2";
-
- len = sizeof(buf);
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT,
- "DigitalProductId", &buf.DigitalProductId,
- &len) == ERROR_SUCCESS &&
- len > 42 && len < sizeof(buf)) {
- bootid_collect(&bin, &buf.DigitalProductId, len);
- got_machineid = true;
- }
- len = sizeof(buf);
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK,
- "DigitalProductId", &buf.DigitalProductId,
- &len) == ERROR_SUCCESS &&
- len > 42 && len < sizeof(buf)) {
- bootid_collect(&bin, &buf.DigitalProductId, len);
- got_machineid = true;
- }
- len = sizeof(buf);
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2,
- "DigitalProductId", &buf.DigitalProductId,
- &len) == ERROR_SUCCESS &&
- len > 42 && len < sizeof(buf)) {
- bootid_collect(&bin, &buf.DigitalProductId, len);
- got_machineid = true;
- }
- }
-
- static const char HKLM_PrefetcherParams[] =
- "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory "
- "Management\\PrefetchParameters";
- len = sizeof(buf);
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId",
- &buf.BootId, &len) == ERROR_SUCCESS &&
- len > 1 && len < sizeof(buf)) {
- bootid_collect(&bin, &buf.BootId, len);
- got_bootseq = true;
- }
-
- len = sizeof(buf);
- if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime",
- &buf.BaseTime, &len) == ERROR_SUCCESS &&
- len >= sizeof(buf.BaseTime) && buf.BaseTime) {
- bootid_collect(&bin, &buf.BaseTime, len);
- got_boottime = true;
- }
-
- /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */
- NTSTATUS status = NtQuerySystemInformation(
- 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo,
- sizeof(buf.SysTimeOfDayInfo), &len);
- if (NT_SUCCESS(status) &&
- len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) +
- sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) &&
- buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) {
- const uint64_t UnbiasedBootTime =
- buf.SysTimeOfDayInfoHacked.BootTime.QuadPart -
- buf.SysTimeOfDayInfoHacked.BootTimeBias;
- if (UnbiasedBootTime) {
- bootid_collect(&bin, &UnbiasedBootTime, sizeof(UnbiasedBootTime));
- got_boottime = true;
- }
- }
-
- if (!got_boottime) {
- uint64_t boottime = windows_bootime();
- if (boottime) {
- bootid_collect(&bin, &boottime, sizeof(boottime));
- got_boottime = true;
- }
- }
- }
-#endif /* Windows */
-
-#if defined(CTL_HW) && defined(HW_UUID)
- if (!got_machineid) {
- static const int mib[] = {CTL_HW, HW_UUID};
- char buf[42];
- size_t len = sizeof(buf);
- if (sysctl(
-#ifdef SYSCTL_LEGACY_NONCONST_MIB
- (int *)
-#endif
- mib,
- ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0)
- got_machineid = bootid_parse_uuid(&bin, buf, len);
- }
-#endif /* CTL_HW && HW_UUID */
-
-#if defined(CTL_KERN) && defined(KERN_HOSTUUID)
- if (!got_machineid) {
- static const int mib[] = {CTL_KERN, KERN_HOSTUUID};
- char buf[42];
- size_t len = sizeof(buf);
- if (sysctl(
-#ifdef SYSCTL_LEGACY_NONCONST_MIB
- (int *)
-#endif
- mib,
- ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0)
- got_machineid = bootid_parse_uuid(&bin, buf, len);
- }
-#endif /* CTL_KERN && KERN_HOSTUUID */
-
-#if defined(__NetBSD__)
- if (!got_machineid) {
- char buf[42];
- size_t len = sizeof(buf);
- if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, NULL, 0) == 0)
- got_machineid = bootid_parse_uuid(&bin, buf, len);
- }
-#endif /* __NetBSD__ */
-
-#if _XOPEN_SOURCE_EXTENDED
- if (!got_machineid) {
- const int hostid = gethostid();
- if (hostid > 0) {
- bootid_collect(&bin, &hostid, sizeof(hostid));
- got_machineid = true;
- }
- }
-#endif /* _XOPEN_SOURCE_EXTENDED */
-
- if (!got_machineid) {
- lack:
- bin.x = bin.y = 0;
- return bin;
- }
-
- /*--------------------------------------------------------------------------*/
-
-#if defined(CTL_KERN) && defined(KERN_BOOTTIME)
- if (!got_boottime) {
- static const int mib[] = {CTL_KERN, KERN_BOOTTIME};
- struct timeval boottime;
- size_t len = sizeof(boottime);
- if (sysctl(
-#ifdef SYSCTL_LEGACY_NONCONST_MIB
- (int *)
-#endif
- mib,
- ARRAY_LENGTH(mib), &boottime, &len, NULL, 0) == 0 &&
- len == sizeof(boottime) && boottime.tv_sec) {
- bootid_collect(&bin, &boottime, len);
- got_boottime = true;
- }
- }
-#endif /* CTL_KERN && KERN_BOOTTIME */
-
-#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
- if (!got_boottime) {
- kstat_ctl_t *kc = kstat_open();
- if (kc) {
- kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc");
- if (kp && kstat_read(kc, kp, 0) != -1) {
- kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time");
- if (kn) {
- switch (kn->data_type) {
- case KSTAT_DATA_INT32:
- case KSTAT_DATA_UINT32:
- bootid_collect(&bin, &kn->value, sizeof(int32_t));
- got_boottime = true;
- case KSTAT_DATA_INT64:
- case KSTAT_DATA_UINT64:
- bootid_collect(&bin, &kn->value, sizeof(int64_t));
- got_boottime = true;
- }
- }
- }
- kstat_close(kc);
- }
- }
-#endif /* SunOS / Solaris */
-
-#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME)
- if (!got_boottime) {
- setutxent();
- const struct utmpx id = {.ut_type = BOOT_TIME};
- const struct utmpx *entry = getutxid(&id);
- if (entry) {
- bootid_collect(&bin, entry, sizeof(*entry));
- got_boottime = true;
- while (unlikely((entry = getutxid(&id)) != nullptr)) {
- /* have multiple reboot records, assuming we can distinguish next
- * bootsession even if RTC is wrong or absent */
- bootid_collect(&bin, entry, sizeof(*entry));
- got_bootseq = true;
- }
- }
- endutxent();
- }
-#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */
-
- if (!got_bootseq) {
- if (!got_boottime || !MDBX_TRUST_RTC)
- goto lack;
-
-#if defined(_WIN32) || defined(_WIN64)
- FILETIME now;
- GetSystemTimeAsFileTime(&now);
- if (0x1CCCCCC > now.dwHighDateTime)
-#else
- struct timespec mono, real;
- if (clock_gettime(CLOCK_MONOTONIC, &mono) ||
- clock_gettime(CLOCK_REALTIME, &real) ||
- /* wrong time, RTC is mad or absent */
- 1555555555l > real.tv_sec ||
- /* seems no adjustment by RTC/NTP, i.e. a fake time */
- real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec ||
- (real.tv_sec - mono.tv_sec) % 900u == 0)
-#endif
- goto lack;
- }
-
- return bin;
-}
-
-__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages,
- intptr_t *avail_pages) {
- if (!page_size && !total_pages && !avail_pages)
- return MDBX_EINVAL;
- if (total_pages)
- *total_pages = -1;
- if (avail_pages)
- *avail_pages = -1;
-
- const intptr_t pagesize = mdbx_syspagesize();
- if (page_size)
- *page_size = pagesize;
- if (unlikely(pagesize < MIN_PAGESIZE || !is_powerof2(pagesize)))
- return MDBX_INCOMPATIBLE;
-
- MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize);
- assert(pagesize == (INT64_C(1) << log2page));
- (void)log2page;
-
-#if defined(_WIN32) || defined(_WIN64)
- MEMORYSTATUSEX info;
- memset(&info, 0, sizeof(info));
- info.dwLength = sizeof(info);
- if (!GlobalMemoryStatusEx(&info))
- return (int)GetLastError();
-#endif
-
- if (total_pages) {
-#if defined(_WIN32) || defined(_WIN64)
- const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page);
-#elif defined(_SC_PHYS_PAGES)
- const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES);
- if (total_ram_pages == -1)
- return errno;
-#elif defined(_SC_AIX_REALMEM)
- const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM);
- if (total_ram_Kb == -1)
- return errno;
- const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page;
-#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || \
- defined(HW_PHYSMEM)
- size_t ram, len = sizeof(ram);
- static const int mib[] = {
- CTL_HW,
-#if defined(HW_USERMEM)
- HW_USERMEM
-#elif defined(HW_PHYSMEM64)
- HW_PHYSMEM64
-#elif defined(HW_MEMSIZE)
- HW_MEMSIZE
-#else
- HW_PHYSMEM
-#endif
- };
- if (sysctl(
-#ifdef SYSCTL_LEGACY_NONCONST_MIB
- (int *)
-#endif
- mib,
- ARRAY_LENGTH(mib), &ram, &len, NULL, 0) != 0)
- return errno;
- if (len != sizeof(ram))
- return MDBX_ENOSYS;
- const intptr_t total_ram_pages = (intptr_t)(ram >> log2page);
-#else
-#error "FIXME: Get User-accessible or physical RAM"
-#endif
- *total_pages = total_ram_pages;
- if (total_ram_pages < 1)
- return MDBX_ENOSYS;
- }
-
- if (avail_pages) {
-#if defined(_WIN32) || defined(_WIN64)
- const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page);
-#elif defined(_SC_AVPHYS_PAGES)
- const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES);
- if (avail_ram_pages == -1)
- return errno;
-#elif defined(__MACH__)
- mach_msg_type_number_t count = HOST_VM_INFO_COUNT;
- vm_statistics_data_t vmstat;
- mach_port_t mport = mach_host_self();
- kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO,
- (host_info_t)&vmstat, &count);
- mach_port_deallocate(mach_task_self(), mport);
- if (unlikely(kerr != KERN_SUCCESS))
- return MDBX_ENOSYS;
- const intptr_t avail_ram_pages = vmstat.free_count;
-#elif defined(VM_TOTAL) || defined(VM_METER)
- struct vmtotal info;
- size_t len = sizeof(info);
- static const int mib[] = {
- CTL_VM,
-#if defined(VM_TOTAL)
- VM_TOTAL
-#elif defined(VM_METER)
- VM_METER
-#endif
- };
- if (sysctl(
-#ifdef SYSCTL_LEGACY_NONCONST_MIB
- (int *)
-#endif
- mib,
- ARRAY_LENGTH(mib), &info, &len, NULL, 0) != 0)
- return errno;
- if (len != sizeof(info))
- return MDBX_ENOSYS;
- const intptr_t avail_ram_pages = info.t_free;
-#else
-#error "FIXME: Get Available RAM"
-#endif
- *avail_pages = avail_ram_pages;
- if (avail_ram_pages < 1)
- return MDBX_ENOSYS;
- }
-
- return MDBX_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/src/osal.h b/3rdparty/libmdbx/src/osal.h
deleted file mode 100644
index 6a7992bb6b7bd36e436c16efe002311194b796c1..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/osal.h
+++ /dev/null
@@ -1,1028 +0,0 @@
-/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
-
-/*
- * Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-/*----------------------------------------------------------------------------*/
-/* Microsoft compiler generates a lot of warning for self includes... */
-
-#ifdef _MSC_VER
-#pragma warning(push, 1)
-#pragma warning(disable : 4548) /* expression before comma has no effect; \
- expected expression with side - effect */
-#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
- * semantics are not enabled. Specify /EHsc */
-#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
- * mode specified; termination on exception is \
- * not guaranteed. Specify /EHsc */
-#endif /* _MSC_VER (warnings) */
-
-#if defined(_WIN32) || defined(_WIN64)
-#if !defined(_CRT_SECURE_NO_WARNINGS)
-#define _CRT_SECURE_NO_WARNINGS
-#endif /* _CRT_SECURE_NO_WARNINGS */
-#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
- !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
-#define _NO_CRT_STDIO_INLINE
-#endif
-#elif !defined(_POSIX_C_SOURCE)
-#define _POSIX_C_SOURCE 200809L
-#endif /* Windows */
-
-/*----------------------------------------------------------------------------*/
-/* C99 includes */
-#include <inttypes.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdlib.h>
-
-#include <assert.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
-
-/* C11' alignas() */
-#if __has_include(<stdalign.h>)
-#include <stdalign.h>
-#endif
-#if defined(alignas) || defined(__cplusplus)
-#define MDBX_ALIGNAS(N) alignas(N)
-#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
-#define MDBX_ALIGNAS(N) _Alignas(N)
-#elif defined(_MSC_VER)
-#define MDBX_ALIGNAS(N) __declspec(align(N))
-#elif __has_attribute(__aligned__) || defined(__GNUC__)
-#define MDBX_ALIGNAS(N) __attribute__((__aligned__(N)))
-#else
-#error "FIXME: Required alignas() or equivalent."
-#endif /* MDBX_ALIGNAS */
-
-/*----------------------------------------------------------------------------*/
-/* Systems includes */
-
-#ifdef __APPLE__
-#include <TargetConditionals.h>
-#endif /* Apple OSX & iOS */
-
-#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
- defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
- defined(__APPLE__) || defined(__MACH__)
-#include <sys/cdefs.h>
-#include <sys/mount.h>
-#include <sys/sysctl.h>
-#include <sys/types.h>
-#if defined(__FreeBSD__) || defined(__DragonFly__)
-#include <vm/vm_param.h>
-#elif defined(__OpenBSD__) || defined(__NetBSD__)
-#include <uvm/uvm_param.h>
-#else
-#define SYSCTL_LEGACY_NONCONST_MIB
-#endif
-#ifndef __MACH__
-#include <sys/vmmeter.h>
-#endif
-#else
-#include <malloc.h>
-#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
- defined(_WIN32) || defined(_WIN64))
-#include <mntent.h>
-#endif /* !Solaris */
-#endif /* !xBSD */
-
-#if defined(__FreeBSD__) || __has_include(<malloc_np.h>)
-#include <malloc_np.h>
-#endif
-
-#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
-#include <malloc/malloc.h>
-#endif /* MacOS */
-
-#if defined(__MACH__)
-#include <mach/host_info.h>
-#include <mach/mach_host.h>
-#include <mach/mach_port.h>
-#include <uuid/uuid.h>
-#endif
-
-#if defined(__linux__) || defined(__gnu_linux__)
-#include <sched.h>
-#include <sys/sendfile.h>
-#include <sys/statfs.h>
-#endif /* Linux */
-
-#ifndef _XOPEN_SOURCE
-#define _XOPEN_SOURCE 0
-#endif
-
-#ifndef _XOPEN_SOURCE_EXTENDED
-#define _XOPEN_SOURCE_EXTENDED 0
-#else
-#include <utmpx.h>
-#endif /* _XOPEN_SOURCE_EXTENDED */
-
-#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
-#include <kstat.h>
-#include <sys/mnttab.h>
-/* On Solaris, it's easier to add a missing prototype rather than find a
- * combination of #defines that break nothing. */
-__extern_C key_t ftok(const char *, int);
-#endif /* SunOS/Solaris */
-
-#if defined(_WIN32) || defined(_WIN64)
-#ifndef _WIN32_WINNT
-#define _WIN32_WINNT 0x0601 /* Windows 7 */
-#elif _WIN32_WINNT < 0x0500
-#error At least 'Windows 2000' API is required for libmdbx.
-#endif /* _WIN32_WINNT */
-#if (defined(__MINGW32__) || defined(__MINGW64__)) && \
- !defined(__USE_MINGW_ANSI_STDIO)
-#define __USE_MINGW_ANSI_STDIO 1
-#endif /* MinGW */
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif /* WIN32_LEAN_AND_MEAN */
-#include <excpt.h>
-#include <tlhelp32.h>
-#include <windows.h>
-#include <winnt.h>
-#include <winternl.h>
-#define HAVE_SYS_STAT_H
-#define HAVE_SYS_TYPES_H
-typedef HANDLE mdbx_thread_t;
-typedef unsigned mdbx_thread_key_t;
-#define MAP_FAILED NULL
-#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
-#define THREAD_CALL WINAPI
-#define THREAD_RESULT DWORD
-typedef struct {
- HANDLE mutex;
- HANDLE event[2];
-} mdbx_condpair_t;
-typedef CRITICAL_SECTION mdbx_fastmutex_t;
-
-#if !defined(_MSC_VER) && !defined(__try)
-/* *INDENT-OFF* */
-/* clang-format off */
-#define __try
-#define __except(COND) if(false)
-/* *INDENT-ON* */
-/* clang-format on */
-#endif /* stub for MSVC's __try/__except */
-
-#if MDBX_WITHOUT_MSVC_CRT
-
-#ifndef mdbx_malloc
-static inline void *mdbx_malloc(size_t bytes) {
- return HeapAlloc(GetProcessHeap(), 0, bytes);
-}
-#endif /* mdbx_malloc */
-
-#ifndef mdbx_calloc
-static inline void *mdbx_calloc(size_t nelem, size_t size) {
- return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, nelem * size);
-}
-#endif /* mdbx_calloc */
-
-#ifndef mdbx_realloc
-static inline void *mdbx_realloc(void *ptr, size_t bytes) {
- return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
- : HeapAlloc(GetProcessHeap(), 0, bytes);
-}
-#endif /* mdbx_realloc */
-
-#ifndef mdbx_free
-static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
-#endif /* mdbx_free */
-
-#else /* MDBX_WITHOUT_MSVC_CRT */
-
-#define mdbx_malloc malloc
-#define mdbx_calloc calloc
-#define mdbx_realloc realloc
-#define mdbx_free free
-#define mdbx_strdup _strdup
-
-#endif /* MDBX_WITHOUT_MSVC_CRT */
-
-#ifndef snprintf
-#define snprintf _snprintf /* ntdll */
-#endif
-
-#ifndef vsnprintf
-#define vsnprintf _vsnprintf /* ntdll */
-#endif
-
-#else /*----------------------------------------------------------------------*/
-
-#include <unistd.h>
-#if !defined(_POSIX_MAPPED_FILES) || _POSIX_MAPPED_FILES < 1
-#error "libmdbx requires the _POSIX_MAPPED_FILES feature"
-#endif /* _POSIX_MAPPED_FILES */
-
-#include <pthread.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <sys/file.h>
-#include <sys/ipc.h>
-#include <sys/mman.h>
-#include <sys/param.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <sys/uio.h>
-typedef pthread_t mdbx_thread_t;
-typedef pthread_key_t mdbx_thread_key_t;
-#define INVALID_HANDLE_VALUE (-1)
-#define THREAD_CALL
-#define THREAD_RESULT void *
-typedef struct {
- pthread_mutex_t mutex;
- pthread_cond_t cond[2];
-} mdbx_condpair_t;
-typedef pthread_mutex_t mdbx_fastmutex_t;
-#define mdbx_malloc malloc
-#define mdbx_calloc calloc
-#define mdbx_realloc realloc
-#define mdbx_free free
-#define mdbx_strdup strdup
-#endif /* Platform */
-
-#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
-/* malloc_usable_size() already provided */
-#elif defined(__APPLE__)
-#define malloc_usable_size(ptr) malloc_size(ptr)
-#elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT
-#define malloc_usable_size(ptr) _msize(ptr)
-#endif /* malloc_usable_size */
-
-#ifdef __ANDROID_API__
-#include <android/log.h>
-#if __ANDROID_API__ >= 21
-#include <sys/sendfile.h>
-#endif
-#endif /* Android */
-
-/* *INDENT-OFF* */
-/* clang-format off */
-#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
-#include <sys/stat.h>
-#endif
-#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
-#include <sys/types.h>
-#endif
-#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
-#include <sys/file.h>
-#endif
-/* *INDENT-ON* */
-/* clang-format on */
-
-#ifndef SSIZE_MAX
-#define SSIZE_MAX INTPTR_MAX
-#endif
-
-#if !defined(MADV_DODUMP) && defined(MADV_CORE)
-#define MADV_DODUMP MADV_CORE
-#endif /* MADV_CORE -> MADV_DODUMP */
-
-#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
-#define MADV_DONTDUMP MADV_NOCORE
-#endif /* MADV_NOCORE -> MADV_DONTDUMP */
-
-#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
- defined(i486) || defined(__i486) || defined(__i486__) || \
- defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
- defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
- defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
- defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
- defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
- defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
-#ifndef __ia32__
-/* LY: define neutral __ia32__ for x86 and x86-64 */
-#define __ia32__ 1
-#endif /* __ia32__ */
-#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
- defined(__amd64) || defined(_M_X64))
-/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
-#define __amd64__ 1
-#endif /* __amd64__ */
-#endif /* all x86 */
-
-#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
-#error \
- "Sanity checking failed: Two's complement, reasonably sized integer types"
-#endif
-
-#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
-#define MDBX_WORDBITS 64
-#else
-#define MDBX_WORDBITS 32
-#endif /* MDBX_WORDBITS */
-
-#if defined(__ANDROID_API__) || defined(ANDROID)
-#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS
-#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS)
-#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS
-#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS)
-#endif
-#endif /* Android */
-
-/*----------------------------------------------------------------------------*/
-/* Compiler's includes for builtins/intrinsics */
-
-#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
-#include <intrin.h>
-#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
-#if defined(__ia32__) || defined(__e2k__)
-#include <x86intrin.h>
-#endif /* __ia32__ */
-#if defined(__ia32__)
-#include <cpuid.h>
-#endif /* __ia32__ */
-#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
-#include <mbarrier.h>
-#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
- (defined(HP_IA64) || defined(__ia64))
-#include <machine/sys/inline.h>
-#elif defined(__IBMC__) && defined(__powerpc)
-#include <atomic.h>
-#elif defined(_AIX)
-#include <builtins.h>
-#include <sys/atomic_op.h>
-#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
-#include <c_asm.h>
-#include <machine/builtins.h>
-#elif defined(__MWERKS__)
-/* CodeWarrior - troubles ? */
-#pragma gcc_extensions
-#elif defined(__SNC__)
-/* Sony PS3 - troubles ? */
-#elif defined(__hppa__) || defined(__hppa)
-#include <machine/inline.h>
-#else
-#error Unsupported C compiler, please use GNU C 4.4 or newer
-#endif /* Compiler */
-
-/*----------------------------------------------------------------------------*/
-/* Byteorder */
-
-#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
- !defined(__ORDER_BIG_ENDIAN__)
-
-/* *INDENT-OFF* */
-/* clang-format off */
-#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \
- defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
-#include <endian.h>
-#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
- defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
-#include <machine/endian.h>
-#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
-#include <sys/isa_defs.h>
-#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
- (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
-#include <sys/endian.h>
-#include <sys/types.h>
-#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
- defined(__NetBSD__) || \
- defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
-#include <sys/param.h>
-#endif /* OS */
-/* *INDENT-ON* */
-/* clang-format on */
-
-#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
-#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
-#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
-#define __BYTE_ORDER__ __BYTE_ORDER
-#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
-#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
-#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
-#define __BYTE_ORDER__ _BYTE_ORDER
-#else
-#define __ORDER_LITTLE_ENDIAN__ 1234
-#define __ORDER_BIG_ENDIAN__ 4321
-
-#if defined(__LITTLE_ENDIAN__) || \
- (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
- defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
- defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
- defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
- defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
- defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
- defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
- defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
- defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
- defined(__WINDOWS__)
-#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
-
-#elif defined(__BIG_ENDIAN__) || \
- (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
- defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
- defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
- defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
- defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
- defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
- defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
-#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
-
-#else
-#error __BYTE_ORDER__ should be defined.
-#endif /* Arch */
-
-#endif
-#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
-
-/* Get the size of a memory page for the system.
- * This is the basic size that the platform's memory manager uses, and is
- * fundamental to the use of memory-mapped files. */
-MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t
-mdbx_syspagesize(void) {
-#if defined(_WIN32) || defined(_WIN64)
- SYSTEM_INFO si;
- GetSystemInfo(&si);
- return si.dwPageSize;
-#else
- return sysconf(_SC_PAGE_SIZE);
-#endif
-}
-
-typedef struct mdbx_mmap_param {
- union {
- void *address;
- uint8_t *dxb;
- struct MDBX_lockinfo *lck;
- };
- mdbx_filehandle_t fd;
- size_t limit; /* mapping length, but NOT a size of file nor DB */
- size_t current; /* mapped region size, i.e. the size of file and DB */
- uint64_t filesize /* in-process cache of a file size */;
-#if defined(_WIN32) || defined(_WIN64)
- HANDLE section; /* memory-mapped section handle */
-#endif
-} mdbx_mmap_t;
-
-typedef union bin128 {
- __anonymous_struct_extension__ struct { uint64_t x, y; };
- __anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
-} bin128_t;
-
-#if defined(_WIN32) || defined(_WIN64)
-typedef union MDBX_srwlock {
- __anonymous_struct_extension__ struct {
- long volatile readerCount;
- long volatile writerCount;
- };
- RTL_SRWLOCK native;
-} MDBX_srwlock;
-#endif /* Windows */
-
-#ifndef __cplusplus
-
-MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
-MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
-
-/*----------------------------------------------------------------------------*/
-/* Atomics */
-
-#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && (__has_include(<cstdatomic>) || __has_extension(cxx_atomic))
-#include <cstdatomic>
-#define MDBX_HAVE_C11ATOMICS
-#elif !defined(__cplusplus) && \
- (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
- !defined(__STDC_NO_ATOMICS__) && \
- (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
- !(defined(__GNUC__) || defined(__clang__)))
-#include <stdatomic.h>
-#define MDBX_HAVE_C11ATOMICS
-#elif defined(__GNUC__) || defined(__clang__)
-#elif defined(_MSC_VER)
-#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
-#pragma warning(disable : 4133) /* 'function': incompatible types - from \
- 'size_t' to 'LONGLONG' */
-#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
- 'std::size_t', possible loss of data */
-#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
- 'long', possible loss of data */
-#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
-#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
-#elif defined(__APPLE__)
-#include <libkern/OSAtomic.h>
-#else
-#error FIXME atomic-ops
-#endif
-
-/*----------------------------------------------------------------------------*/
-/* Memory/Compiler barriers, cache coherence */
-
-#if __has_include(<sys/cachectl.h>)
-#include <sys/cachectl.h>
-#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
- defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
- defined(__MWERKS__) || defined(__sgi)
-/* MIPS should have explicit cache control */
-#include <sys/cachectl.h>
-#endif
-
-MDBX_MAYBE_UNUSED static __inline void mdbx_compiler_barrier(void) {
-#if defined(__clang__) || defined(__GNUC__)
- __asm__ __volatile__("" ::: "memory");
-#elif defined(_MSC_VER)
- _ReadWriteBarrier();
-#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
- __memory_barrier();
-#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
- __compiler_barrier();
-#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
- (defined(HP_IA64) || defined(__ia64))
- _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
-#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
- defined(__ppc64__) || defined(__powerpc64__)
- __fence();
-#else
-#error "Could not guess the kind of compiler, please report to us."
-#endif
-}
-
-MDBX_MAYBE_UNUSED static __inline void mdbx_memory_barrier(void) {
-#ifdef MDBX_HAVE_C11ATOMICS
- atomic_thread_fence(memory_order_seq_cst);
-#elif defined(__ATOMIC_SEQ_CST)
-#ifdef __clang__
- __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
-#else
- __atomic_thread_fence(__ATOMIC_SEQ_CST);
-#endif
-#elif defined(__clang__) || defined(__GNUC__)
- __sync_synchronize();
-#elif defined(_WIN32) || defined(_WIN64)
- MemoryBarrier();
-#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
-#if defined(__ia32__)
- _mm_mfence();
-#else
- __mf();
-#endif
-#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
- __machine_rw_barrier();
-#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
- (defined(HP_IA64) || defined(__ia64))
- _Asm_mf();
-#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
- defined(__ppc64__) || defined(__powerpc64__)
- __lwsync();
-#else
-#error "Could not guess the kind of compiler, please report to us."
-#endif
-}
-
-/*----------------------------------------------------------------------------*/
-/* libc compatibility stuff */
-
-#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
- (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
-#define mdbx_asprintf asprintf
-#define mdbx_vasprintf vasprintf
-#else
-MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC
- MDBX_PRINTF_ARGS(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
-MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
-#endif
-
-/*----------------------------------------------------------------------------*/
-/* OS abstraction layer stuff */
-
-/* max bytes to write in one call */
-#if defined(_WIN32) || defined(_WIN64)
-#define MAX_WRITE UINT32_C(0x01000000)
-#else
-#define MAX_WRITE UINT32_C(0x3fff0000)
-#endif
-
-#if defined(__linux__) || defined(__gnu_linux__)
-MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
-MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */;
-#endif /* Linux */
-
-#ifndef mdbx_strdup
-LIBMDBX_API char *mdbx_strdup(const char *str);
-#endif
-
-MDBX_MAYBE_UNUSED static __inline int mdbx_get_errno(void) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD rc = GetLastError();
-#else
- int rc = errno;
-#endif
- return rc;
-}
-
-#ifndef mdbx_memalign_alloc
-MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
- void **result);
-#endif
-#ifndef mdbx_memalign_free
-MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
-#endif
-
-MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair);
-MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair);
-MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair);
-MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair,
- bool part);
-MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part);
-MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair);
-
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
-MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
-
-MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
- int iovcnt, uint64_t offset,
- size_t expected_written);
-MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
- uint64_t offset);
-MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
- size_t count, uint64_t offset);
-MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
- size_t count);
-
-MDBX_INTERNAL_FUNC int
-mdbx_thread_create(mdbx_thread_t *thread,
- THREAD_RESULT(THREAD_CALL *start_routine)(void *),
- void *arg);
-MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
-
-enum mdbx_syncmode_bits {
- MDBX_SYNC_NONE = 0,
- MDBX_SYNC_DATA = 1,
- MDBX_SYNC_SIZE = 2,
- MDBX_SYNC_IODQ = 4
-};
-
-MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd,
- const enum mdbx_syncmode_bits mode_bits);
-MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
-MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
-MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
-
-enum mdbx_openfile_purpose {
- MDBX_OPEN_DXB_READ = 0,
- MDBX_OPEN_DXB_LAZY = 1,
- MDBX_OPEN_DXB_DSYNC = 2,
- MDBX_OPEN_LCK = 3,
- MDBX_OPEN_COPY = 4,
- MDBX_OPEN_DELETE = 5
-};
-
-MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose,
- const MDBX_env *env, const char *pathname,
- mdbx_filehandle_t *fd,
- mdbx_mode_t unix_mode_bits);
-MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
-MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
-MDBX_INTERNAL_FUNC int mdbx_removedirectory(const char *pathname);
-MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
-MDBX_INTERNAL_FUNC int mdbx_lockfile(mdbx_filehandle_t fd, bool wait);
-
-#define MMAP_OPTION_TRUNCATE 1
-#define MMAP_OPTION_SEMAPHORE 2
-MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
- const size_t must, const size_t limit,
- const unsigned options);
-MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
-#define MDBX_MRESIZE_MAY_MOVE 0x00000100
-#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
-MDBX_INTERNAL_FUNC int mdbx_mresize(const int flags, mdbx_mmap_t *map,
- size_t size, size_t limit);
-#if defined(_WIN32) || defined(_WIN64)
-typedef struct {
- unsigned limit, count;
- HANDLE handles[31];
-} mdbx_handle_array_t;
-MDBX_INTERNAL_FUNC int
-mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
-MDBX_INTERNAL_FUNC int
-mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
-#endif /* Windows */
-MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
- size_t length,
- enum mdbx_syncmode_bits mode_bits);
-MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle,
- const char *pathname, int err);
-
-MDBX_MAYBE_UNUSED static __inline uint32_t mdbx_getpid(void) {
- STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
-#if defined(_WIN32) || defined(_WIN64)
- return GetCurrentProcessId();
-#else
- STATIC_ASSERT(sizeof(pid_t) <= sizeof(uint32_t));
- return getpid();
-#endif
-}
-
-MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
- mdbx_tid_t thunk;
- STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk));
-#if defined(_WIN32) || defined(_WIN64)
- thunk = GetCurrentThreadId();
-#else
- thunk = pthread_self();
-#endif
- return (uintptr_t)thunk;
-}
-
-#if !defined(_WIN32) && !defined(_WIN64)
-#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
-MDBX_INTERNAL_FUNC int mdbx_check_tid4bionic(void);
-#else
-static __inline int mdbx_check_tid4bionic(void) { return 0; }
-#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
-
-MDBX_MAYBE_UNUSED static __inline int
-mdbx_pthread_mutex_lock(pthread_mutex_t *mutex) {
- int err = mdbx_check_tid4bionic();
- return unlikely(err) ? err : pthread_mutex_lock(mutex);
-}
-#endif /* !Windows */
-
-MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
-MDBX_INTERNAL_FUNC uint64_t
-mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
-MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
-
-MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
-/*----------------------------------------------------------------------------*/
-/* lck stuff */
-
-/// \brief Initialization of synchronization primitives linked with MDBX_env
-/// instance both in LCK-file and within the current process.
-/// \param
-/// global_uniqueness_flag = true - denotes that there are no other processes
-/// working with DB and LCK-file. Thus the function MUST initialize
-/// shared synchronization objects in memory-mapped LCK-file.
-/// global_uniqueness_flag = false - denotes that at least one process is
-/// already working with DB and LCK-file, including the case when DB
-/// has already been opened in the current process. Thus the function
-/// MUST NOT initialize shared synchronization objects in memory-mapped
-/// LCK-file that are already in use.
-/// \return Error code or zero on success.
-MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
- MDBX_env *inprocess_neighbor,
- int global_uniqueness_flag);
-
-/// \brief Disconnects from shared interprocess objects and destructs
-/// synchronization objects linked with MDBX_env instance
-/// within the current process.
-/// \param
-/// inprocess_neighbor = NULL - if the current process does not have other
-/// instances of MDBX_env linked with the DB being closed.
-/// Thus the function MUST check for other processes working with DB or
-/// LCK-file, and keep or destroy shared synchronization objects in
-/// memory-mapped LCK-file depending on the result.
-/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
-/// (anyone of there is several) working with DB or LCK-file within the
-/// current process. Thus the function MUST NOT try to acquire exclusive
-/// lock and/or try to destruct shared synchronization objects linked with
-/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
-/// of other instances of MDBX_env within the current process, e.g.
-/// restore POSIX-fcntl locks after the closing of file descriptors.
-/// \return Error code (MDBX_PANIC) or zero on success.
-MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
- MDBX_env *inprocess_neighbor);
-
-/// \brief Connects to shared interprocess locking objects and tries to acquire
-/// the maximum lock level (shared if exclusive is not available)
-/// Depending on implementation or/and platform (Windows) this function may
-/// acquire the non-OS super-level lock (e.g. for shared synchronization
-/// objects initialization), which will be downgraded to OS-exclusive or
-/// shared via explicit calling of mdbx_lck_downgrade().
-/// \return
-/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
-/// the current process is the first and only after the last use of DB.
-/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
-/// DB has already been opened and now is used by other processes.
-/// Otherwise (not 0 and not -1) - error code.
-MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
-
-/// \brief Downgrades the level of initially acquired lock to
-/// operational level specified by argument. The reson for such downgrade:
-/// - unblocking of other processes that are waiting for access, i.e.
-/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
-/// should be made aware that access is unavailable rather than
-/// wait for it.
-/// - freeing locks that interfere file operation (especially for Windows)
-/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
-/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
-/// operational lock.
-/// \return Error code or zero on success
-MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
-
-/// \brief Locks LCK-file or/and table of readers for (de)registering.
-/// \return Error code or zero on success
-MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
-
-/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
-MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
-
-/// \brief Acquires lock for DB change (on writing transaction start)
-/// Reading transactions will not be blocked.
-/// Declared as LIBMDBX_API because it is used in mdbx_chk.
-/// \return Error code or zero on success
-LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait);
-
-/// \brief Releases lock once DB changes is made (after writing transaction
-/// has finished).
-/// Declared as LIBMDBX_API because it is used in mdbx_chk.
-LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
-
-/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
-/// the current process. The function does no more than needed for
-/// the correct working of mdbx_rpid_check() in other processes.
-/// \return Error code or zero on success
-MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
-
-/// \brief Resets alive-flag of reader presence (indicative lock)
-/// for PID of the current process. The function does no more than needed
-/// for the correct working of mdbx_rpid_check() in other processes.
-/// \return Error code or zero on success
-MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
-
-/// \brief Checks for reading process status with the given pid with help of
-/// alive-flag of presence (indicative lock) or using another way.
-/// \return
-/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
-/// and working with DB (indicative lock is present).
-/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
-/// or not working with DB (indicative lock is not present).
-/// Otherwise (not 0 and not -1) - error code.
-MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
-
-#if defined(_WIN32) || defined(_WIN64)
-
-typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
-MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
- mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
- mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
-
-#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
-typedef enum _FILE_INFO_BY_HANDLE_CLASS {
- FileBasicInfo,
- FileStandardInfo,
- FileNameInfo,
- FileRenameInfo,
- FileDispositionInfo,
- FileAllocationInfo,
- FileEndOfFileInfo,
- FileStreamInfo,
- FileCompressionInfo,
- FileAttributeTagInfo,
- FileIdBothDirectoryInfo,
- FileIdBothDirectoryRestartInfo,
- FileIoPriorityHintInfo,
- FileRemoteProtocolInfo,
- MaximumFileInfoByHandleClass
-} FILE_INFO_BY_HANDLE_CLASS,
- *PFILE_INFO_BY_HANDLE_CLASS;
-
-typedef struct _FILE_END_OF_FILE_INFO {
- LARGE_INTEGER EndOfFile;
-} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;
-
-#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
-#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002
-
-typedef struct _FILE_REMOTE_PROTOCOL_INFO {
- USHORT StructureVersion;
- USHORT StructureSize;
- DWORD Protocol;
- USHORT ProtocolMajorVersion;
- USHORT ProtocolMinorVersion;
- USHORT ProtocolRevision;
- USHORT Reserved;
- DWORD Flags;
- struct {
- DWORD Reserved[8];
- } GenericReserved;
- struct {
- DWORD Reserved[16];
- } ProtocolSpecificReserved;
-} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;
-
-#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
-
-typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
- _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
- _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
-MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
- mdbx_GetFileInformationByHandleEx;
-
-typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
- _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
- _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
- _Out_opt_ LPDWORD lpMaximumComponentLength,
- _Out_opt_ LPDWORD lpFileSystemFlags,
- _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
-MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
- mdbx_GetVolumeInformationByHandleW;
-
-typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
- _Out_ LPWSTR lpszFilePath,
- _In_ DWORD cchFilePath,
- _In_ DWORD dwFlags);
-MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
-
-typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
- _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
- _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
-MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
- mdbx_SetFileInformationByHandle;
-
-typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
- IN HANDLE FileHandle, IN OUT HANDLE Event,
- IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
- OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
- IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
- OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
-MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
-
-typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
-MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
-
-#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
-typedef struct _WIN32_MEMORY_RANGE_ENTRY {
- PVOID VirtualAddress;
- SIZE_T NumberOfBytes;
-} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
-#endif /* Windows 8.x */
-
-typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
- HANDLE hProcess, ULONG_PTR NumberOfEntries,
- PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
-MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
-
-typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
-
-typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
- IN PLARGE_INTEGER NewSectionSize);
-MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection;
-
-static __inline bool mdbx_RunningUnderWine(void) {
- return !mdbx_NtExtendSection;
-}
-
-typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
- LPCSTR lpValue, DWORD dwFlags,
- LPDWORD pdwType, PVOID pvData,
- LPDWORD pcbData);
-MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA;
-
-NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
-
-#endif /* Windows */
-
-#endif /* !__cplusplus */
-
-/*----------------------------------------------------------------------------*/
-
-#if defined(_MSC_VER) && _MSC_VER >= 1900
-/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
- * for internal format-args checker. */
-#undef PRIuPTR
-#undef PRIiPTR
-#undef PRIdPTR
-#undef PRIxPTR
-#define PRIuPTR "Iu"
-#define PRIiPTR "Ii"
-#define PRIdPTR "Id"
-#define PRIxPTR "Ix"
-#define PRIuSIZE "zu"
-#define PRIiSIZE "zi"
-#define PRIdSIZE "zd"
-#define PRIxSIZE "zx"
-#endif /* fix PRI*PTR for _MSC_VER */
-
-#ifndef PRIuSIZE
-#define PRIuSIZE PRIuPTR
-#define PRIiSIZE PRIiPTR
-#define PRIdSIZE PRIdPTR
-#define PRIxSIZE PRIxPTR
-#endif /* PRI*SIZE macros for MSVC */
-
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
diff --git a/3rdparty/libmdbx/src/version.c.in b/3rdparty/libmdbx/src/version.c.in
deleted file mode 100644
index 9686a9b7bcd514f621d32b16cbaf7c3a244d15a0..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/version.c.in
+++ /dev/null
@@ -1,46 +0,0 @@
-/* This is CMake-template for libmdbx's version.c
- ******************************************************************************/
-
-#include "internals.h"
-
-#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \
- MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR}
-#error "API version mismatch! Had `git fetch --tags` done?"
-#endif
-
-static const char sourcery[] = MDBX_STRINGIFY(MDBX_BUILD_SOURCERY);
-
-__dll_export
-#ifdef __attribute_used__
- __attribute_used__
-#elif defined(__GNUC__) || __has_attribute(__used__)
- __attribute__((__used__))
-#endif
-#ifdef __attribute_externally_visible__
- __attribute_externally_visible__
-#elif (defined(__GNUC__) && !defined(__clang__)) || \
- __has_attribute(__externally_visible__)
- __attribute__((__externally_visible__))
-#endif
- const struct MDBX_version_info mdbx_version = {
- ${MDBX_VERSION_MAJOR},
- ${MDBX_VERSION_MINOR},
- ${MDBX_VERSION_RELEASE},
- ${MDBX_VERSION_REVISION},
- {"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
- "@MDBX_GIT_DESCRIBE@"},
- sourcery};
-
-__dll_export
-#ifdef __attribute_used__
- __attribute_used__
-#elif defined(__GNUC__) || __has_attribute(__used__)
- __attribute__((__used__))
-#endif
-#ifdef __attribute_externally_visible__
- __attribute_externally_visible__
-#elif (defined(__GNUC__) && !defined(__clang__)) || \
- __has_attribute(__externally_visible__)
- __attribute__((__externally_visible__))
-#endif
- const char *const mdbx_sourcery_anchor = sourcery;
diff --git a/3rdparty/libmdbx/src/wingetopt.c b/3rdparty/libmdbx/src/wingetopt.c
deleted file mode 100644
index 4f27d648714807152a65dc97987f15aa22d04f1b..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/wingetopt.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * POSIX getopt for Windows
- *
- * AT&T Public License
- *
- * Code given out at the 1985 UNIFORUM conference in Dallas.
- */
-
-/*----------------------------------------------------------------------------*/
-/* Microsoft compiler generates a lot of warning for self includes... */
-
-#ifdef _MSC_VER
-#pragma warning(push, 1)
-#pragma warning(disable : 4548) /* expression before comma has no effect; \
- expected expression with side - effect */
-#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
- * semantics are not enabled. Specify /EHsc */
-#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
- * mode specified; termination on exception is \
- * not guaranteed. Specify /EHsc */
-#if !defined(_CRT_SECURE_NO_WARNINGS)
-#define _CRT_SECURE_NO_WARNINGS
-#endif
-#endif /* _MSC_VER (warnings) */
-
-#include "wingetopt.h"
-#include <stdio.h>
-#include <string.h>
-
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
-/*----------------------------------------------------------------------------*/
-
-#ifndef NULL
-#define NULL 0
-#endif
-
-#ifndef EOF
-#define EOF (-1)
-#endif
-
-int optind = 1;
-int optopt;
-char *optarg;
-
-int getopt(int argc, char *const argv[], const char *opts) {
- static int sp = 1;
- int c;
- const char *cp;
-
- if (sp == 1) {
- if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
- return EOF;
- else if (strcmp(argv[optind], "--") == 0) {
- optind++;
- return EOF;
- }
- }
- optopt = c = argv[optind][sp];
- if (c == ':' || (cp = strchr(opts, c)) == NULL) {
- fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c);
- if (argv[optind][++sp] == '\0') {
- optind++;
- sp = 1;
- }
- return '?';
- }
- if (*++cp == ':') {
- if (argv[optind][sp + 1] != '\0')
- optarg = &argv[optind++][sp + 1];
- else if (++optind >= argc) {
- fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
- c);
- sp = 1;
- return '?';
- } else
- optarg = argv[optind++];
- sp = 1;
- } else {
- if (argv[optind][++sp] == '\0') {
- sp = 1;
- optind++;
- }
- optarg = NULL;
- }
- return c;
-}
diff --git a/3rdparty/libmdbx/src/wingetopt.h b/3rdparty/libmdbx/src/wingetopt.h
deleted file mode 100644
index d328e38ccc2252d5d226454787d8deaae139d05a..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/src/wingetopt.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * POSIX getopt for Windows
- *
- * AT&T Public License
- *
- * Code given out at the 1985 UNIFORUM conference in Dallas.
- */
-
-#ifndef _WINGETOPT_H_
-#define _WINGETOPT_H_
-
-/* Bit of madness for Windows console */
-#define mdbx_strerror mdbx_strerror_ANSI2OEM
-#define mdbx_strerror_r mdbx_strerror_r_ANSI2OEM
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int opterr;
-extern int optind;
-extern int optopt;
-extern char *optarg;
-int getopt(int argc, char *const argv[], const char *optstring);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _GETOPT_H_ */
diff --git a/3rdparty/libmdbx/test/CMakeLists.txt b/3rdparty/libmdbx/test/CMakeLists.txt
deleted file mode 100644
index 5e7066e164f81dbc350d7302510999c0ce15e13f..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/CMakeLists.txt
+++ /dev/null
@@ -1,139 +0,0 @@
-enable_language(CXX)
-include(../cmake/compiler.cmake)
-
-set(LIBMDBX_TEST_SOURCES
- base.h
- cases.cc
- chrono.cc
- chrono.h
- config.cc
- config.h
- copy.cc
- dead.cc
- hill.cc
- jitter.cc
- keygen.cc
- keygen.h
- log.cc
- log.h
- main.cc
- osal.h
- osal-unix.cc
- osal-windows.cc
- test.cc
- test.h
- try.cc
- utils.cc
- utils.h
- append.cc
- ttl.cc
- nested.cc
- )
-
-if(NOT MDBX_BUILD_CXX)
- probe_libcxx_filesystem()
- list(APPEND LIBMDBX_TEST_SOURCES "${MDBX_SOURCE_DIR}/mdbx.c++" ../mdbx.h++)
-endif()
-
-add_executable(mdbx_test ${LIBMDBX_TEST_SOURCES})
-
-if(MDBX_CXX_STANDARD)
- set_target_properties(mdbx_test PROPERTIES
- CXX_STANDARD ${MDBX_CXX_STANDARD} CXX_STANDARD_REQUIRED ON)
-endif()
-
-set_target_properties(mdbx_test PROPERTIES
- INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
-target_setup_options(mdbx_test)
-
-if(NOT MDBX_BUILD_CXX AND LIBCXX_FILESYSTEM)
- if(CMAKE_COMPILER_IS_ELBRUSCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 1.25.23
- AND NOT CMAKE_VERSION VERSION_LESS 3.13)
- target_link_options(mdbx_test PRIVATE "-Wl,--allow-multiple-definition")
- endif()
- target_link_libraries(mdbx_test ${LIBCXX_FILESYSTEM})
-endif()
-
-target_link_libraries(mdbx_test ${TOOL_MDBX_LIB} ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT})
-if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
- target_link_libraries(mdbx_test winmm.lib)
-endif()
-
-if(UNIX AND NOT SUBPROJECT)
- add_executable(pcrf_test pcrf/pcrf_test.c)
- target_include_directories(pcrf_test PRIVATE "${PROJECT_SOURCE_DIR}")
- target_link_libraries(pcrf_test ${TOOL_MDBX_LIB})
-endif()
-
-################################################################################
-
-if (CMAKE_CROSSCOMPILING AND NOT CMAKE_CROSSCOMPILING_EMULATOR)
- message(WARNING "No emulator to run cross-compiled tests")
- add_test(NAME fake_since_no_crosscompiling_emulator COMMAND ${CMAKE_COMMAND} -E
- echo "No emulator to run cross-compiled tests")
-else()
-
- string(RANDOM LENGTH 9 ALPHABET "1234567890" test_seed)
- message(STATUS "The ${test_seed} will be used for seeding tests. Re-run cmake to re-seed it.")
-
- add_test(NAME smoke COMMAND ${MDBX_OUTPUT_DIR}/mdbx_test
- --loglevel=verbose
- --keygen.seed=${test_seed}
- --progress --console=no --pathname=smoke.db --dont-cleanup-after basic)
- set_tests_properties(smoke PROPERTIES
- TIMEOUT 600
- RUN_SERIAL OFF)
- if(MDBX_BUILD_TOOLS)
- add_test(NAME smoke_chk COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv smoke.db)
- set_tests_properties(smoke_chk PROPERTIES
- DEPENDS smoke
- TIMEOUT 60
- REQUIRED_FILES smoke.db)
- add_test(NAME smoke_chk_copy COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv smoke.db-copy)
- set_tests_properties(smoke_chk_copy PROPERTIES
- DEPENDS smoke
- TIMEOUT 60
- REQUIRED_FILES smoke.db-copy)
- endif()
-
- add_test(NAME dupsort_writemap COMMAND ${MDBX_OUTPUT_DIR}/mdbx_test
- --loglevel=notice
- --keygen.seed=${test_seed}
- --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no
- --repeat=2 --pathname=dupsort_writemap.db --dont-cleanup-after basic)
- set_tests_properties(dupsort_writemap PROPERTIES
- TIMEOUT 600
- RUN_SERIAL OFF)
- if(MDBX_BUILD_TOOLS)
- add_test(NAME dupsort_writemap_chk COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv dupsort_writemap.db)
- set_tests_properties(dupsort_writemap_chk PROPERTIES
- DEPENDS dupsort_writemap
- TIMEOUT 60
- REQUIRED_FILES dupsort_writemap.db)
- add_test(NAME dupsort_writemap_chk_copy COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv dupsort_writemap.db-copy)
- set_tests_properties(dupsort_writemap_chk_copy PROPERTIES
- DEPENDS dupsort_writemap
- TIMEOUT 60
- REQUIRED_FILES dupsort_writemap.db-copy)
- endif()
-
- add_test(NAME uniq_nested COMMAND ${MDBX_OUTPUT_DIR}/mdbx_test
- --loglevel=notice
- --mode=-writemap,-nosync-safe,-lifo --progress --console=no --repeat=2 --pathname=uniq_nested.db --dont-cleanup-after basic)
- set_tests_properties(uniq_nested PROPERTIES
- TIMEOUT 1800
- RUN_SERIAL OFF)
- if(MDBX_BUILD_TOOLS)
- add_test(NAME uniq_nested_chk COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv uniq_nested.db)
- set_tests_properties(uniq_nested_chk PROPERTIES
- DEPENDS uniq_nested
- TIMEOUT 60
- REQUIRED_FILES uniq_nested.db)
- add_test(NAME uniq_nested_chk_copy COMMAND ${MDBX_OUTPUT_DIR}/mdbx_chk -nvv uniq_nested.db-copy)
- set_tests_properties(uniq_nested_chk_copy PROPERTIES
- DEPENDS uniq_nested
- TIMEOUT 60
- REQUIRED_FILES uniq_nested.db-copy)
- endif()
-
-endif()
diff --git a/3rdparty/libmdbx/test/append.cc b/3rdparty/libmdbx/test/append.cc
deleted file mode 100644
index d75e0231ca10184a59680c02aa6a71b4e5bf5ab0..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/append.cc
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-class testcase_append : public testcase {
-public:
- testcase_append(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-
- static bool review_params(actor_params ¶ms) {
- return testcase::review_params(params) && params.make_keygen_linear();
- }
-};
-REGISTER_TESTCASE(append);
-
-bool testcase_append::run() {
- const bool reverse = flipcoin();
- const char *const caption = reverse ? "ahead" : "append";
- log_notice("the '%s' scenario is selected", caption);
-
- int err = db_open__begin__table_create_open_clean(dbi);
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("%s: bailout-prepare due '%s'", caption, mdbx_strerror(err));
- return true;
- }
-
- cursor_open(dbi);
- keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
- /* LY: теÑÑ‚ Ð½Ð°Ð¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† в append-режиме,
- * при котором запиÑи добавлÑÑŽÑ‚ÑÑ Ñтрого в конец (в порÑдке Ñортировки) */
- const MDBX_put_flags_t flags =
- reverse
- ? ((config.params.table_flags & MDBX_DUPSORT) ? MDBX_UPSERT
- : MDBX_NOOVERWRITE)
- : ((config.params.table_flags & MDBX_DUPSORT)
- ? (flipcoin() ? MDBX_APPEND | MDBX_APPENDDUP : MDBX_APPENDDUP)
- : MDBX_APPEND);
-
- key = keygen::alloc(config.params.keylen_max);
- data = keygen::alloc(config.params.datalen_max);
-
- simple_checksum inserted_checksum;
- uint64_t inserted_number = 0;
- uint64_t serial_count = 0;
- if (reverse)
- keyvalue_maker.seek2end(serial_count);
-
- unsigned txn_nops = 0;
- uint64_t committed_inserted_number = inserted_number;
- simple_checksum committed_inserted_checksum = inserted_checksum;
- while (should_continue()) {
- const keygen::serial_t serial = serial_count;
- const bool turn_key = (config.params.table_flags & MDBX_DUPSORT) == 0 ||
- flipcoin_n(config.params.keygen.split);
- if (turn_key
- ? !keyvalue_maker.increment_key_part(serial_count, reverse ? -1 : 1)
- : !keyvalue_maker.increment(serial_count, reverse ? -1 : 1)) {
- // дошли до границы проÑтранÑтва ключей
- break;
- }
-
- log_trace("%s: insert-a %" PRIu64, caption, serial);
- generate_pair(serial);
- // keygen::log_pair(logging::verbose, "append.", key, data);
-
- bool expect_key_mismatch = false;
- if (flags & (MDBX_APPEND | MDBX_APPENDDUP)) {
- MDBX_val ge_key = key->value;
- MDBX_val ge_data = data->value;
- err = mdbx_get_equal_or_great(txn_guard.get(), dbi, &ge_key, &ge_data);
-
- if (err == MDBX_SUCCESS /* exact match */) {
- expect_key_mismatch = true;
- assert(inserted_number > 0);
- assert(mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0);
- assert((config.params.table_flags & MDBX_DUPSORT) == 0 ||
- mdbx_dcmp(txn_guard.get(), dbi, &data->value, &ge_data) == 0);
- assert(inserted_number > 0);
- } else if (err == MDBX_RESULT_TRUE /* have key-value pair great than */) {
- assert(mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) < 0 ||
- ((config.params.table_flags & MDBX_DUPSORT) &&
- mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0 &&
- mdbx_dcmp(txn_guard.get(), dbi, &data->value, &ge_data) < 0));
- switch (int(flags)) {
- default:
- abort();
-#if CONSTEXPR_ENUM_FLAGS_OPERATIONS
- case MDBX_APPEND | MDBX_APPENDDUP:
-#else
- case int(MDBX_APPEND) | int(MDBX_APPENDDUP):
-#endif
- assert((config.params.table_flags & MDBX_DUPSORT) != 0);
- __fallthrough;
- // fall through
- case MDBX_APPEND:
- expect_key_mismatch = true;
- break;
- case MDBX_APPENDDUP:
- assert((config.params.table_flags & MDBX_DUPSORT) != 0);
- expect_key_mismatch =
- mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0;
- break;
- }
- } else if (err == MDBX_NOTFOUND /* all pair are less than */) {
- switch (int(flags)) {
- default:
- abort();
- case MDBX_APPENDDUP:
-#if CONSTEXPR_ENUM_FLAGS_OPERATIONS
- case MDBX_APPEND | MDBX_APPENDDUP:
-#else
- case int(MDBX_APPEND) | int(MDBX_APPENDDUP):
-#endif
- assert((config.params.table_flags & MDBX_DUPSORT) != 0);
- __fallthrough;
- // fall through
- case MDBX_APPEND:
- expect_key_mismatch = false;
- break;
- }
- } else
- failure_perror("mdbx_get_equal_or_great()", err);
-
- assert(!expect_key_mismatch);
- }
-
- err = mdbx_cursor_put(cursor_guard.get(), &key->value, &data->value, flags);
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("%s: bailout-insert due '%s'", caption, mdbx_strerror(err));
- txn_end(true);
- inserted_number = committed_inserted_number;
- inserted_checksum = committed_inserted_checksum;
- break;
- }
-
- if (!expect_key_mismatch) {
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_put(insert-a)", err);
- ++inserted_number;
- inserted_checksum.push((uint32_t)inserted_number, key->value);
- inserted_checksum.push(10639, data->value);
- } else if (unlikely(err != MDBX_EKEYMISMATCH))
- failure_perror("mdbx_cursor_put(insert-a) != MDBX_EKEYMISMATCH", err);
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("%s: bailout-commit due '%s'", caption, mdbx_strerror(err));
- inserted_number = committed_inserted_number;
- inserted_checksum = committed_inserted_checksum;
- break;
- }
- committed_inserted_number = inserted_number;
- committed_inserted_checksum = inserted_checksum;
- txn_nops = 0;
- }
-
- report(1);
- }
-
- if (txn_guard) {
- err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("%s: bailout-commit due '%s'", caption, mdbx_strerror(err));
- inserted_number = committed_inserted_number;
- inserted_checksum = committed_inserted_checksum;
- }
- }
- //----------------------------------------------------------------------------
- txn_begin(true);
- cursor_renew();
-
- MDBX_val check_key, check_data;
- err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data,
- reverse ? MDBX_LAST : MDBX_FIRST);
- if (likely(inserted_number)) {
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
- }
-
- simple_checksum read_checksum;
- uint64_t read_count = 0;
- while (err == MDBX_SUCCESS) {
- ++read_count;
- read_checksum.push((uint32_t)read_count, check_key);
- read_checksum.push(10639, check_data);
-
- err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data,
- reverse ? MDBX_PREV : MDBX_NEXT);
- }
-
- if (unlikely(err != MDBX_NOTFOUND))
- failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err);
-
- if (unlikely(read_count != inserted_number))
- failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
- read_count, inserted_number);
-
- if (unlikely(read_checksum.value != inserted_checksum.value))
- failure("read_checksum(0x%016" PRIu64 ") "
- "!= inserted_checksum(0x%016" PRIu64 ")",
- read_checksum.value, inserted_checksum.value);
-
- cursor_close();
- txn_end(true);
- //----------------------------------------------------------------------------
-
- if (dbi) {
- if (config.params.drop_table && !mode_readonly()) {
- txn_begin(false);
- db_table_drop(dbi);
- err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("%s: bailout-clean due '%s'", caption, mdbx_strerror(err));
- return true;
- }
- } else
- db_table_close(dbi);
- }
- return true;
-}
diff --git a/3rdparty/libmdbx/test/base.h b/3rdparty/libmdbx/test/base.h
deleted file mode 100644
index 8d378c015cbc154f7c531b842114b5c0d82898a9..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/base.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
-#ifndef _WIN32_WINNT
-#define _WIN32_WINNT 0x0601 /* Windows 7 */
-#endif
-#ifdef _MSC_VER
-#ifndef _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_WARNINGS
-#endif /* _CRT_SECURE_NO_WARNINGS */
-#pragma warning(push, 1)
-#pragma warning(disable : 4548) /* expression before comma has no effect; \
- expected expression with side - effect */
-#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
- semantics are not enabled. Specify /EHsc */
-#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
- mode specified; termination on exception \
- is not guaranteed. Specify /EHsc */
-#endif /* _MSC_VER (warnings) */
-
-/* If you wish to build your application for a previous Windows platform,
- * include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
- * wish to support before including SDKDDKVer.h.
- *
- * TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
-#include <SDKDDKVer.h>
-#endif /* WINDOWS */
-
-#ifdef __APPLE__
-#define _DARWIN_C_SOURCE
-#endif
-
-#include <errno.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
-#include <io.h>
-#else
-#include <fcntl.h>
-#include <sys/param.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-
-#ifdef _BSD_SOURCE
-#include <endian.h>
-#endif
-
-#include <algorithm>
-#include <cassert>
-#include <cinttypes> // for PRId64, PRIu64
-#include <cstdarg>
-#include <cstddef>
-#include <cstdint>
-#include <map>
-#include <memory>
-#include <set>
-#include <string>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#define MDBX_INTERNAL_FUNC
-#define MDBX_INTERNAL_VAR extern
-#define xMDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "../mdbx.h++"
-#include "../src/defs.h"
-#include "../src/osal.h"
-
-#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
-#define __thread __declspec(thread)
-#endif /* __thread */
-
-#include "../src/options.h"
-
-#ifdef _MSC_VER
-#pragma warning(pop)
-#pragma warning(disable : 4201) /* nonstandard extension used : \
- nameless struct / union */
-#pragma warning(disable : 4127) /* conditional expression is constant */
-#if _MSC_VER < 1900
-#pragma warning(disable : 4510) /* default constructor could \
- not be generated */
-#pragma warning(disable : 4512) /* assignment operator could \
- not be generated */
-#pragma warning(disable : 4610) /* user-defined constructor required */
-#ifndef snprintf
-#define snprintf(buffer, buffer_size, format, ...) \
- _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
-#endif
-#ifndef vsnprintf
-#define vsnprintf(buffer, buffer_size, format, args) \
- _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
-#endif
-#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
- may be unsafe */
-#endif
-#endif /* _MSC_VER */
diff --git a/3rdparty/libmdbx/test/cases.cc b/3rdparty/libmdbx/test/cases.cc
deleted file mode 100644
index cf31344b45f4aceeddf09bf5650d6548ff42528b..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/cases.cc
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-registry *registry::instance() {
- static registry *singleton;
- if (!singleton)
- singleton = new registry();
- return singleton;
-}
-
-bool registry::add(const record *item) {
- auto const singleton = instance();
- assert(singleton->name2id.count(std::string(item->name)) == 0);
- assert(singleton->id2record.count(item->id) == 0);
- if (singleton->name2id.count(std::string(item->name)) +
- singleton->id2record.count(item->id) ==
- 0) {
- singleton->name2id[std::string(item->name)] = item;
- singleton->id2record[item->id] = item;
- return true;
- }
- return false;
-}
-
-testcase *registry::create_actor(const actor_config &config,
- const mdbx_pid_t pid) {
- return instance()->id2record.at(config.testcase)->constructor(config, pid);
-}
-
-bool registry::review_actor_params(const actor_testcase id,
- actor_params ¶ms) {
- return instance()->id2record.at(id)->review_params(params);
-}
-
-//-----------------------------------------------------------------------------
-
-void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
- const char *space_id_cstr, actor_params params) {
- unsigned wait4id = 0;
- if (params.waitfor_nops) {
- for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
- if (i->is_waitable(params.waitfor_nops)) {
- if (i->signal_nops && i->signal_nops != params.waitfor_nops)
- failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
- i->actor_id, i->signal_nops);
- wait4id = i->actor_id;
- i->signal_nops = params.waitfor_nops;
- break;
- }
- }
- if (!wait4id)
- failure("No previous waitable actor for %u-ops\n", params.waitfor_nops);
- }
-
- unsigned space_id = 0;
- if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0)
- space_id = last_space_id + 1;
- else {
- char *end = nullptr;
- errno = 0;
- space_id = strtoul(space_id_cstr, &end, 0);
- if (errno)
- failure_perror("Expects an integer value for space-id\n", errno);
- if (end && *end)
- failure("The '%s' is unexpected for space-id\n", end);
- }
-
- if (!registry::review_actor_params(testcase, params))
- failure("Actor config-review failed for space-id %u\n", space_id);
-
- if (space_id > ACTOR_ID_MAX)
- failure("Invalid space-id %u\n", space_id);
- last_space_id = space_id;
-
- log_trace("configure_actor: space %u for %s", space_id,
- testcase2str(testcase));
- global::actors.emplace_back(
- actor_config(testcase, params, space_id, wait4id));
- global::databases.insert(params.pathname_db);
-}
-
-void testcase_setup(const char *casename, const actor_params ¶ms,
- unsigned &last_space_id) {
- if (strcmp(casename, "basic") == 0) {
- log_notice(">>> testcase_setup(%s)", casename);
- configure_actor(last_space_id, ac_nested, nullptr, params);
- configure_actor(last_space_id, ac_hill, nullptr, params);
- configure_actor(last_space_id, ac_ttl, nullptr, params);
- configure_actor(last_space_id, ac_copy, nullptr, params);
- configure_actor(last_space_id, ac_append, nullptr, params);
- configure_actor(last_space_id, ac_jitter, nullptr, params);
- configure_actor(last_space_id, ac_try, nullptr, params);
- configure_actor(last_space_id, ac_jitter, nullptr, params);
- configure_actor(last_space_id, ac_try, nullptr, params);
- log_notice("<<< testcase_setup(%s): done", casename);
- } else {
- failure("unknown testcase `%s`", casename);
- }
-}
-
-void keycase_setup(const char *casename, actor_params ¶ms) {
- if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) {
- log_notice(">>> keycase_setup(%s)", casename);
- params.keygen.keycase = kc_random;
- // TODO
- log_notice("<<< keycase_setup(%s): done", casename);
- } else if (strcmp(casename, "dashes") == 0 ||
- strcmp(casename, "aside") == 0) {
- log_notice(">>> keycase_setup(%s)", casename);
- params.keygen.keycase = kc_dashes;
- // TODO
- log_notice("<<< keycase_setup(%s): done", casename);
- } else if (strcmp(casename, "custom") == 0) {
- log_notice("=== keycase_setup(%s): skip", casename);
- params.keygen.keycase = kc_custom;
- } else {
- failure("unknown keycase `%s`", casename);
- }
-}
-
-/* TODO */
diff --git a/3rdparty/libmdbx/test/chrono.cc b/3rdparty/libmdbx/test/chrono.cc
deleted file mode 100644
index c84d603cceef6573c9c80c398054597c5a959e65..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/chrono.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-namespace chrono {
-
-#ifndef NSEC_PER_SEC
-#define NSEC_PER_SEC 1000000000u
-#endif /* NSEC_PER_SEC */
-
-uint32_t ns2fractional(uint32_t ns) {
- assert(ns < NSEC_PER_SEC);
- /* LY: здеÑÑŒ и далее иÑпользуетÑÑ "длинное деление", которое
- * Ð´Ð»Ñ ÑÑноÑти кода оÑтавлено как еÑÑ‚ÑŒ (без ручной оптимизации). Так как
- * GCC, Clang и даже MSVC Ñами давно умеют конвертировать деление на
- * конÑтанту в быÑтрую reciprocal-форму. */
- return ((uint64_t)ns << 32) / NSEC_PER_SEC;
-}
-
-uint32_t fractional2ns(uint32_t fractional) {
- return (fractional * (uint64_t)NSEC_PER_SEC) >> 32;
-}
-
-#ifndef USEC_PER_SEC
-#define USEC_PER_SEC 1000000u
-#endif /* USEC_PER_SEC */
-uint32_t us2fractional(uint32_t us) {
- assert(us < USEC_PER_SEC);
- return ((uint64_t)us << 32) / USEC_PER_SEC;
-}
-
-uint32_t fractional2us(uint32_t fractional) {
- return (fractional * (uint64_t)USEC_PER_SEC) >> 32;
-}
-
-#ifndef MSEC_PER_SEC
-#define MSEC_PER_SEC 1000u
-#endif /* MSEC_PER_SEC */
-uint32_t ms2fractional(uint32_t ms) {
- assert(ms < MSEC_PER_SEC);
- return ((uint64_t)ms << 32) / MSEC_PER_SEC;
-}
-
-uint32_t fractional2ms(uint32_t fractional) {
- return (fractional * (uint64_t)MSEC_PER_SEC) >> 32;
-}
-
-time from_ns(uint64_t ns) {
- time result;
- result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) |
- ns2fractional((uint32_t)(ns % NSEC_PER_SEC));
- return result;
-}
-
-time from_us(uint64_t us) {
- time result;
- result.fixedpoint = ((us / USEC_PER_SEC) << 32) |
- us2fractional((uint32_t)(us % USEC_PER_SEC));
- return result;
-}
-
-time from_ms(uint64_t ms) {
- time result;
- result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) |
- ms2fractional((uint32_t)(ms % MSEC_PER_SEC));
- return result;
-}
-
-#if __GNUC_PREREQ(8, 0) && \
- (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wcast-function-type"
-#endif /* GCC/MINGW */
-
-time now_realtime() {
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
- static void(WINAPI * query_time)(LPFILETIME);
- if (!query_time) {
- query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
- GetModuleHandle(TEXT("kernel32.dll")),
- "GetSystemTimePreciseAsFileTime");
- if (!query_time)
- query_time = GetSystemTimeAsFileTime;
- }
-
- FILETIME filetime;
- query_time(&filetime);
- uint64_t ns100 =
- (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
- return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
-#else
- struct timespec ts;
- if (unlikely(clock_gettime(CLOCK_REALTIME, &ts)))
- failure_perror("clock_gettime(CLOCK_REALTIME", errno);
-
- return from_timespec(ts);
-#endif
-}
-
-time now_monotonic() {
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
- static uint64_t reciprocal;
- static LARGE_INTEGER Frequency;
- if (reciprocal == 0) {
- if (!QueryPerformanceFrequency(&Frequency))
- failure_perror("QueryPerformanceFrequency()", GetLastError());
- reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
- Frequency.QuadPart);
- assert(reciprocal);
- }
-
- LARGE_INTEGER Counter;
- if (!QueryPerformanceCounter(&Counter))
- failure_perror("QueryPerformanceCounter()", GetLastError());
-
- time result;
- result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32;
- uint64_t mod = Counter.QuadPart % Frequency.QuadPart;
- result.fixedpoint += (mod * reciprocal) >> 16;
- return result;
-#else
- struct timespec ts;
- if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
- failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno);
-
- return from_timespec(ts);
-#endif
-}
-
-#if __GNUC_PREREQ(8, 0) && \
- (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
-#pragma GCC diagnostic pop
-#endif /* GCC/MINGW */
-
-} /* namespace chrono */
diff --git a/3rdparty/libmdbx/test/chrono.h b/3rdparty/libmdbx/test/chrono.h
deleted file mode 100644
index 5ee08856b6538643373eb7c4574395ccdf6a7828..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/chrono.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-#include "utils.h"
-
-namespace chrono {
-
-#pragma pack(push, 4)
-
-typedef union time {
- uint64_t fixedpoint;
- __anonymous_struct_extension__ struct {
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- uint32_t fractional;
- union {
- uint32_t utc;
- uint32_t integer;
- };
-#else
- union {
- uint32_t utc;
- uint32_t integer;
- };
- uint32_t fractional;
-#endif
- };
-
- void reset() { fixedpoint = 0; }
- uint32_t seconds() const { return utc; }
-} time;
-
-#pragma pack(pop)
-
-uint32_t ns2fractional(uint32_t);
-uint32_t fractional2ns(uint32_t);
-uint32_t us2fractional(uint32_t);
-uint32_t fractional2us(uint32_t);
-uint32_t ms2fractional(uint32_t);
-uint32_t fractional2ms(uint32_t);
-
-time from_ns(uint64_t us);
-time from_us(uint64_t ns);
-time from_ms(uint64_t ms);
-
-inline time from_seconds(uint64_t seconds) {
- assert(seconds < UINT32_MAX);
- time result;
- result.fixedpoint = seconds << 32;
- return result;
-}
-
-inline time from_utc(time_t utc) {
- assert(utc >= 0);
- return from_seconds((uint64_t)utc);
-}
-
-inline time infinite() {
- time result;
- result.fixedpoint = UINT64_MAX;
- return result;
-}
-
-#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
- defined(CLOCK_REALTIME)
-inline time from_timespec(const struct timespec &ts) {
- time result;
- result.fixedpoint =
- ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
- return result;
-}
-#endif /* HAVE_TIMESPEC_TV_NSEC */
-
-#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
-inline time from_timeval(const struct timeval &tv) {
- time result;
- result.fixedpoint =
- ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
- return result;
-}
-#endif /* HAVE_TIMEVAL_TV_USEC */
-
-time now_realtime();
-time now_monotonic();
-
-} /* namespace chrono */
diff --git a/3rdparty/libmdbx/test/config.cc b/3rdparty/libmdbx/test/config.cc
deleted file mode 100644
index 61b299b4d683398518f5cea04597068d19c4a6ea..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/config.cc
+++ /dev/null
@@ -1,729 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-#if defined(_MSC_VER) && !defined(strcasecmp)
-#define strcasecmp(str, len) _stricmp(str, len)
-#endif /* _MSC_VER && strcasecmp() */
-
-namespace config {
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- const char **value, const char *default_value) {
- assert(narg < argc);
- const char *current = argv[narg];
- const size_t optlen = strlen(option);
-
- if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen))
- return false;
-
- if (!value) {
- if (current[optlen + 2] == '=')
- failure("Option '--%s' doesn't accept any value\n", option);
- return true;
- }
-
- *value = nullptr;
- if (current[optlen + 2] == '=') {
- *value = ¤t[optlen + 3];
- return true;
- }
-
- if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) {
- *value = argv[narg + 1];
- if (strcmp(*value, "default") == 0) {
- if (!default_value)
- failure("Option '--%s' doesn't accept default value\n", option);
- *value = default_value;
- }
- ++narg;
- return true;
- }
-
- if (default_value) {
- *value = default_value;
- return true;
- }
-
- failure("No value given for '--%s' option\n", option);
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- std::string &value, bool allow_empty) {
- return parse_option(argc, argv, narg, option, value, allow_empty,
- allow_empty ? "" : nullptr);
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- std::string &value, bool allow_empty,
- const char *default_value) {
- const char *value_cstr;
- if (!parse_option(argc, argv, narg, option, &value_cstr, default_value))
- return false;
-
- if (!allow_empty && strlen(value_cstr) == 0)
- failure("Value for option '--%s' couldn't be empty\n", option);
-
- value = value_cstr;
- return true;
-}
-
-template <>
-bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
- const char *option, unsigned &mask,
- const option_verb *verbs) {
- const char *list;
- if (!parse_option(argc, argv, narg, option, &list))
- return false;
-
- unsigned clear = 0;
- while (*list) {
- if (*list == ',' || *list == ' ' || *list == '\t') {
- ++list;
- continue;
- }
-
- const char *const comma = strchr(list, ',');
- const bool strikethrough = *list == '-' || *list == '~';
- if (strikethrough || *list == '+')
- ++list;
- else
- mask = clear;
- const size_t len = (comma) ? comma - list : strlen(list);
- const option_verb *scan = verbs;
-
- while (true) {
- if (!scan->verb)
- failure("Unknown verb '%.*s', for option '--%s'\n", (int)len, list,
- option);
- if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
- mask = strikethrough ? mask & ~scan->mask : mask | scan->mask;
- clear = strikethrough ? clear & ~scan->mask : clear | scan->mask;
- list += len;
- break;
- }
- ++scan;
- }
- }
-
- return true;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- uint64_t &value, const scale_mode scale,
- const uint64_t minval, const uint64_t maxval,
- const uint64_t default_value) {
-
- const char *value_cstr;
- if (!parse_option(argc, argv, narg, option, &value_cstr))
- return false;
-
- if (default_value && strcmp(value_cstr, "default") == 0) {
- value = default_value;
- return true;
- }
-
- if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) {
- value = minval;
- return true;
- }
-
- if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) {
- value = maxval;
- return true;
- }
-
- char *suffix = nullptr;
- errno = 0;
- unsigned long long raw = strtoull(value_cstr, &suffix, 0);
- if ((suffix && *suffix) || errno) {
- suffix = nullptr;
- errno = 0;
- raw = strtoull(value_cstr, &suffix, 10);
- }
- if (errno)
- failure("Option '--%s' expects a numeric value (%s)\n", option,
- test_strerror(errno));
-
- uint64_t multiplier = 1;
- if (suffix && *suffix) {
- if (scale == no_scale)
- failure("Option '--%s' doesn't accepts suffixes, so '%s' is unexpected\n",
- option, suffix);
- if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
- multiplier = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
- else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
- multiplier =
- (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
- else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
- multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
- : UINT64_C(1024) * 1024 * 1024;
- else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
- multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
- : UINT64_C(1024) * 1024 * 1024 * 1024;
- else if (scale == duration &&
- (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
- multiplier = 1;
- else if (scale == duration &&
- (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
- multiplier = 60;
- else if (scale == duration &&
- (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
- multiplier = 3600;
- else if (scale == duration &&
- (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
- multiplier = 3600 * 24;
- else
- failure(
- "Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
- "suffixes, but '%s' is unexpected\n",
- option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
- suffix);
- }
-
- if (raw >= UINT64_MAX / multiplier)
- failure("The value for option '--%s' is too huge\n", option);
-
- value = raw * multiplier;
- if (maxval && value > maxval)
- failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
- maxval);
- if (value < minval)
- failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
- minval);
- return true;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- unsigned &value, const scale_mode scale,
- const unsigned minval, const unsigned maxval,
- const unsigned default_value) {
-
- uint64_t huge;
- if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
- default_value))
- return false;
- value = unsigned(huge);
- return true;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- uint8_t &value, const uint8_t minval, const uint8_t maxval,
- const uint8_t default_value) {
-
- uint64_t huge;
- if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
- default_value))
- return false;
- value = uint8_t(huge);
- return true;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- int64_t &value, const int64_t minval, const int64_t maxval,
- const int64_t default_value) {
- uint64_t proxy = uint64_t(value);
- if (parse_option(argc, argv, narg, option, proxy, config::binary,
- uint64_t(minval), uint64_t(maxval),
- uint64_t(default_value))) {
- value = int64_t(proxy);
- return true;
- }
- return false;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- int32_t &value, const int32_t minval, const int32_t maxval,
- const int32_t default_value) {
- uint64_t proxy = uint64_t(value);
- if (parse_option(argc, argv, narg, option, proxy, config::binary,
- uint64_t(minval), uint64_t(maxval),
- uint64_t(default_value))) {
- value = int32_t(proxy);
- return true;
- }
- return false;
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- logging::loglevel &loglevel) {
- const char *value_cstr;
- if (!parse_option(argc, argv, narg, option, &value_cstr))
- return false;
-
- if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0 ||
- strcmp(value_cstr, "fatal") == 0) {
- loglevel = logging::failure;
- return true;
- }
-
- if (strcmp(value_cstr, "error") == 0 || strcmp(value_cstr, "err") == 0) {
- loglevel = logging::error;
- return true;
- }
-
- if (strcmp(value_cstr, "warning") == 0 || strcmp(value_cstr, "warn") == 0) {
- loglevel = logging::error;
- return true;
- }
-
- if (strcmp(value_cstr, "default") == 0 || strcmp(value_cstr, "notice") == 0) {
- loglevel = logging::notice;
- return true;
- }
-
- if (strcmp(value_cstr, "verbose") == 0) {
- loglevel = logging::verbose;
- return true;
- }
-
- if (strcmp(value_cstr, "debug") == 0) {
- loglevel = logging::debug;
- return true;
- }
-
- if (strcmp(value_cstr, "trace") == 0) {
- loglevel = logging::trace;
- return true;
- }
-
- if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0 ||
- strcmp(value_cstr, "extra") == 0) {
- loglevel = logging::extra;
- return true;
- }
-
- char *suffix = nullptr;
- unsigned long long raw = strtoull(value_cstr, &suffix, 0);
- if ((suffix && *suffix) || errno) {
- suffix = nullptr;
- errno = 0;
- raw = strtoull(value_cstr, &suffix, 10);
- }
- if ((!suffix || !*suffix) && !errno && raw < 8) {
- loglevel = static_cast<logging::loglevel>(raw);
- return true;
- }
-
- failure("Unknown log-level '%s', for option '--%s'\n", value_cstr, option);
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- bool &value) {
- const char *value_cstr = nullptr;
- if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
- const char *current = argv[narg];
- if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) {
- value = false;
- return true;
- }
- if (strncmp(current, "--dont-", 7) == 0 &&
- strcmp(current + 7, option) == 0) {
- value = false;
- return true;
- }
- return false;
- }
-
- if (!value_cstr) {
- value = true;
- return true;
- }
-
- if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) {
- value = true;
- return true;
- }
-
- if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) {
- value = false;
- return true;
- }
-
- failure(
- "Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
- option, value_cstr);
-}
-
-//-----------------------------------------------------------------------------
-
-const struct option_verb mode_bits[] = {
- {"rdonly", unsigned(MDBX_RDONLY)},
- {"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)},
- {"nosubdir", unsigned(MDBX_NOSUBDIR)},
- {"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)},
- {"nometasync", unsigned(MDBX_NOMETASYNC)},
- {"writemap", unsigned(MDBX_WRITEMAP)},
- {"notls", unsigned(MDBX_NOTLS)},
- {"nordahead", unsigned(MDBX_NORDAHEAD)},
- {"nomeminit", unsigned(MDBX_NOMEMINIT)},
- {"coalesce", unsigned(MDBX_COALESCE)},
- {"lifo", unsigned(MDBX_LIFORECLAIM)},
- {"perturb", unsigned(MDBX_PAGEPERTURB)},
- {"accede", unsigned(MDBX_ACCEDE)},
- {"exclusive", unsigned(MDBX_EXCLUSIVE)},
- {nullptr, 0}};
-
-const struct option_verb table_bits[] = {
- {"key.reverse", unsigned(MDBX_REVERSEKEY)},
- {"key.integer", unsigned(MDBX_INTEGERKEY)},
- {"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)},
- {"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)},
- {"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)},
- {"data.dups", unsigned(MDBX_DUPSORT)},
- {nullptr, 0}};
-
-static void dump_verbs(const char *caption, size_t bits,
- const struct option_verb *verbs) {
- log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
-
- const char *comma = "";
- while (verbs->mask && bits) {
- if ((bits & verbs->mask) == verbs->mask) {
- logging::feed("%s%s", comma, verbs->verb);
- bits -= verbs->mask;
- comma = ", ";
- }
- ++verbs;
- }
-
- logging::feed("%s\n", (*comma == '\0') ? "none" : "");
-}
-
-static void dump_duration(const char *caption, unsigned duration) {
- log_verbose("%s: ", caption);
- if (duration) {
- if (duration > 24 * 3600)
- logging::feed("%u_", duration / (24 * 3600));
- if (duration > 3600)
- logging::feed("%02u:", (duration % (24 * 3600)) / 3600);
- logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60);
- } else {
- logging::feed("INFINITE");
- }
- logging::feed("\n");
-}
-
-void dump(const char *title) {
- logging::local_suffix indent(title);
-
- for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
- log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id,
- testcase2str(i->testcase), i->space_id);
- indent.push();
-
- if (i->params.loglevel) {
- log_verbose("log: level %u, %s\n", i->params.loglevel,
- i->params.pathname_log.empty()
- ? "console"
- : i->params.pathname_log.c_str());
- }
-
- log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR
- ", %i %i, %i]\n",
- i->params.pathname_db.c_str(), i->params.size_now,
- i->params.size_lower, i->params.size_upper,
- i->params.shrink_threshold, i->params.growth_step,
- i->params.pagesize);
-
- dump_verbs("mode", i->params.mode_flags, mode_bits);
- log_verbose("random-writemap: %s\n",
- i->params.random_writemap ? "Yes" : "No");
- dump_verbs("table", i->params.table_flags, table_bits);
-
- if (i->params.test_nops)
- log_verbose("iterations/records %u\n", i->params.test_nops);
- else
- dump_duration("duration", i->params.test_duration);
-
- if (i->params.nrepeat)
- log_verbose("repeat %u\n", i->params.nrepeat);
- else
- log_verbose("repeat ETERNALLY\n");
-
- log_verbose("threads %u\n", i->params.nthreads);
-
- log_verbose(
- "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64
- ", split %u/%u\n",
- keygencase2str(i->params.keygen.keycase), i->params.keygen.width,
- i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset,
- i->params.keygen.split,
- i->params.keygen.width - i->params.keygen.split);
- log_verbose("keygen.seed: %u\n", i->params.keygen.seed);
- log_verbose("keygen.zerofill: %s\n",
- i->params.keygen.zero_fill ? "Yes" : "No");
- log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min,
- i->params.keylen_max);
- log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min,
- i->params.datalen_max);
-
- log_verbose("batch: read %u, write %u\n", i->params.batch_read,
- i->params.batch_write);
-
- if (i->params.waitfor_nops)
- log_verbose("wait: actor %u for %u ops\n", i->wait4id,
- i->params.waitfor_nops);
- else if (i->params.delaystart)
- dump_duration("delay", i->params.delaystart);
- else
- log_verbose("no-delay\n");
-
- if (i->params.inject_writefaultn)
- log_verbose("inject-writefault on %u ops\n",
- i->params.inject_writefaultn);
- else
- log_verbose("no-inject-writefault\n");
-
- log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n",
- i->params.max_readers, i->params.max_tables,
- mdbx_limits_txnsize_max(i->params.pagesize));
-
- log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
- log_verbose("ignore MDBX_MAP_FULL error: %s\n",
- i->params.ignore_dbfull ? "Yes" : "No");
- log_verbose("verifying by speculum: %s\n",
- i->params.speculum ? "Yes" : "No");
-
- indent.pop();
- }
-
- dump_duration("timeout", global::config::timeout_duration_seconds);
- log_verbose("cleanup: before %s, after %s\n",
- global::config::cleanup_before ? "Yes" : "No",
- global::config::cleanup_after ? "Yes" : "No");
-
- log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No");
- log_verbose("progress indicator: %s\n",
- global::config::progress_indicator ? "Yes" : "No");
- log_verbose("console mode: %s\n",
- global::config::console_mode ? "Yes" : "No");
- log_verbose("geometry jitter: %s\n",
- global::config::geometry_jitter ? "Yes" : "No");
-}
-
-} /* namespace config */
-
-//-----------------------------------------------------------------------------
-
-using namespace config;
-
-actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms,
- unsigned space_id, unsigned wait4id)
- : actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id,
- wait4id),
- params(params) {}
-
-const std::string actor_config::serialize(const char *prefix) const {
- simple_checksum checksum;
- std::string result;
-
- if (prefix)
- result.append(prefix);
-
- checksum.push(params.pathname_db);
- result.append(params.pathname_db);
- result.push_back('|');
-
- checksum.push(params.pathname_log);
- result.append(params.pathname_log);
- result.push_back('|');
-
-#if __cplusplus > 201400
- static_assert(std::is_trivially_copyable<actor_params_pod>::value,
- "actor_params_pod should by POD");
-#else
- static_assert(std::is_standard_layout<actor_params_pod>::value,
- "actor_params_pod should by POD");
-#endif
- result.append(data2hex(static_cast<const actor_params_pod *>(¶ms),
- sizeof(actor_params_pod), checksum));
- result.push_back('|');
-
-#if __cplusplus > 201400
- static_assert(std::is_trivially_copyable<actor_config_pod>::value,
- "actor_config_pod should by POD");
-#else
- static_assert(std::is_standard_layout<actor_config_pod>::value,
- "actor_config_pod should by POD");
-#endif
- result.append(data2hex(static_cast<const actor_config_pod *>(this),
- sizeof(actor_config_pod), checksum));
- result.push_back('|');
- result.push_back(global::config::progress_indicator ? 'Y' : 'N');
- checksum.push(global::config::progress_indicator);
- result.push_back(global::config::console_mode ? 'Y' : 'N');
- checksum.push(global::config::console_mode);
- result.push_back(global::config::geometry_jitter ? 'Y' : 'N');
- checksum.push(global::config::geometry_jitter);
- result.push_back('|');
-
- result.append(osal_serialize(checksum));
- result.push_back('|');
-
- result.append(std::to_string(checksum.value));
- return result;
-}
-
-bool actor_config::deserialize(const char *str, actor_config &config) {
- simple_checksum checksum;
-
- TRACE(">> actor_config::deserialize: %s\n", str);
-
- const char *slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-1\n");
- return false;
- }
- config.params.pathname_db.assign(str, slash - str);
- checksum.push(config.params.pathname_db);
- str = slash + 1;
-
- slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-2\n");
- return false;
- }
- config.params.pathname_log.assign(str, slash - str);
- checksum.push(config.params.pathname_log);
- str = slash + 1;
-
- slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-3\n");
- return false;
- }
-#if __cplusplus > 201400
- static_assert(std::is_trivially_copyable<actor_params_pod>::value,
- "actor_params_pod should by POD");
-#else
- static_assert(std::is_standard_layout<actor_params_pod>::value,
- "actor_params_pod should by POD");
-#endif
- if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
- sizeof(actor_params_pod), checksum)) {
- TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
- (int)(slash - str), str);
- return false;
- }
- str = slash + 1;
-
- slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-4\n");
- return false;
- }
-#if __cplusplus > 201400
- static_assert(std::is_trivially_copyable<actor_config_pod>::value,
- "actor_config_pod should by POD");
-#else
- static_assert(std::is_standard_layout<actor_config_pod>::value,
- "actor_config_pod should by POD");
-#endif
- if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
- sizeof(actor_config_pod), checksum)) {
- TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
- (int)(slash - str), str);
- return false;
- }
- str = slash + 1;
-
- slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-5\n");
- return false;
- }
- if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N') &&
- (str[2] == 'Y' || str[2] == 'N')) {
- global::config::progress_indicator = str[0] == 'Y';
- checksum.push(global::config::progress_indicator);
- global::config::console_mode = str[1] == 'Y';
- checksum.push(global::config::console_mode);
- global::config::geometry_jitter = str[2] != 'N';
- checksum.push(global::config::geometry_jitter);
- str = slash + 1;
-
- slash = strchr(str, '|');
- if (!slash) {
- TRACE("<< actor_config::deserialize: slash-6\n");
- return false;
- }
- }
-
- if (!config.osal_deserialize(str, slash, checksum)) {
- TRACE("<< actor_config::deserialize: osal\n");
- return false;
- }
- str = slash + 1;
-
- uint64_t verify = std::stoull(std::string(str));
- if (checksum.value != verify) {
- TRACE("<< actor_config::deserialize: checksum mismatch\n");
- return false;
- }
-
- TRACE("<< actor_config::deserialize: OK\n");
- return true;
-}
-
-unsigned actor_params::mdbx_keylen_min() const {
- return (table_flags & MDBX_INTEGERKEY) ? 4 : 0;
-}
-
-unsigned actor_params::mdbx_keylen_max() const {
- return unsigned(mdbx_limits_keysize_max(pagesize, table_flags));
-}
-
-unsigned actor_params::mdbx_datalen_min() const {
- return (table_flags & MDBX_INTEGERDUP) ? 4 : 0;
-}
-
-unsigned actor_params::mdbx_datalen_max() const {
- return std::min(unsigned(UINT16_MAX),
- unsigned(mdbx_limits_valsize_max(pagesize, table_flags)));
-}
-
-bool actor_params::make_keygen_linear() {
- const auto base = serial_base();
- keygen.mesh = (table_flags & MDBX_DUPSORT) ? 0 : keygen.split;
- keygen.rotate = 0;
- keygen.offset = 0;
- const auto max_serial = serial_mask(keygen.width) + base;
- const auto max_key_serial = (keygen.split && (table_flags & MDBX_DUPSORT))
- ? max_serial >> keygen.split
- : max_serial;
- const auto max_value_serial = (keygen.split && (table_flags & MDBX_DUPSORT))
- ? serial_mask(keygen.split)
- : 0;
-
- while (keylen_min < 8 &&
- (keylen_min == 0 || serial_mask(keylen_min * 8) < max_key_serial)) {
- keylen_min += (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1;
- if (keylen_max < keylen_min)
- keylen_max = keylen_min;
- }
-
- if (table_flags & MDBX_DUPSORT)
- while (
- datalen_min < 8 &&
- (datalen_min == 0 || serial_mask(datalen_min * 8) < max_value_serial)) {
- datalen_min +=
- (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1;
- if (datalen_max < datalen_min)
- datalen_max = datalen_min;
- }
-
- return true;
-}
diff --git a/3rdparty/libmdbx/test/config.h b/3rdparty/libmdbx/test/config.h
deleted file mode 100644
index 8c93981e9255855b8b4d90b38c0a4433e299cb9d..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/config.h
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-#include "log.h"
-#include "utils.h"
-
-#define ACTOR_ID_MAX INT16_MAX
-
-enum actor_testcase {
- ac_none,
- ac_hill,
- ac_deadread,
- ac_deadwrite,
- ac_jitter,
- ac_try,
- ac_copy,
- ac_append,
- ac_ttl,
- ac_nested
-};
-
-enum actor_status {
- as_unknown,
- as_debugging,
- as_running,
- as_successful,
- as_killed,
- as_failed,
- as_coredump,
-};
-
-const char *testcase2str(const actor_testcase);
-const char *status2str(actor_status status);
-
-enum keygen_case {
- kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */
- kc_dashes, /* [ 0123.. 4567.. ] */
- kc_custom,
- /* TODO: more cases */
-};
-
-const char *keygencase2str(const keygen_case);
-
-//-----------------------------------------------------------------------------
-
-namespace config {
-
-enum scale_mode { no_scale, decimal, binary, duration };
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- const char **value, const char *default_value = nullptr);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- std::string &value, bool allow_empty = false);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- std::string &value, bool allow_empty,
- const char *default_value);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- bool &value);
-
-struct option_verb {
- const char *const verb;
- unsigned mask;
-};
-
-template <typename MASK>
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- MASK &mask, const option_verb *verbs) {
- static_assert(sizeof(MASK) <= sizeof(unsigned), "WTF?");
- unsigned u = unsigned(mask);
- if (parse_option<unsigned>(argc, argv, narg, option, u, verbs)) {
- mask = MASK(u);
- return true;
- }
- return false;
-}
-
-template <>
-bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
- const char *option, unsigned &mask,
- const option_verb *verbs);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- uint64_t &value, const scale_mode scale,
- const uint64_t minval = 0, const uint64_t maxval = INT64_MAX,
- const uint64_t default_value = 0);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- unsigned &value, const scale_mode scale,
- const unsigned minval = 0, const unsigned maxval = INT32_MAX,
- const unsigned default_value = 0);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- uint8_t &value, const uint8_t minval = 0,
- const uint8_t maxval = 255, const uint8_t default_value = 0);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- int64_t &value, const int64_t minval, const int64_t maxval,
- const int64_t default_value = -1);
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- int32_t &value, const int32_t minval, const int32_t maxval,
- const int32_t default_value = -1);
-
-inline bool parse_option_intptr(int argc, char *const argv[], int &narg,
- const char *option, intptr_t &value,
- const intptr_t minval, const intptr_t maxval,
- const intptr_t default_value = -1) {
- static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?");
- if (sizeof(intptr_t) == 8)
- return parse_option(argc, argv, narg, option,
- *reinterpret_cast<int64_t *>(&value), int64_t(minval),
- int64_t(maxval), int64_t(default_value));
- else
- return parse_option(argc, argv, narg, option,
- *reinterpret_cast<int32_t *>(&value), int32_t(minval),
- int32_t(maxval), int32_t(default_value));
-}
-
-bool parse_option(int argc, char *const argv[], int &narg, const char *option,
- logging::loglevel &);
-//-----------------------------------------------------------------------------
-
-struct keygen_params_pod {
- /* Параметры генератора пар key-value. Также может быть полезным опиÑание
- * алгоритма генерации в keygen.h
- *
- * Ключи и Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð³ÐµÐ½ÐµÑ€Ð¸Ñ€ÑƒÑŽÑ‚ÑÑ Ð¿Ð¾ задаваемым параметрам на оÑнове "плоÑкой"
- * иÑходной координаты. При Ñтом, в общем Ñлучае, в процеÑÑе теÑтов иÑходнаÑ
- * координата поÑледовательно итерируетÑÑ Ð² заданном диапазоне, а необходимые
- * паттерны/поÑледовательноÑти/узоры получаютÑÑ Ð·Ð° Ñчет преобразованиÑ
- * иÑходной координаты, ÑоглаÑно опиÑанным ниже параметрам.
- *
- * Стоит отметить, что порÑдок опиÑÐ°Ð½Ð¸Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð¾Ð² Ð´Ð»Ñ ÑƒÐ´Ð¾Ð±Ñтва Ñовпадает Ñ
- * порÑдком их иÑпользованиÑ, Ñ‚.е. Ñ Ð¿Ð¾Ñ€Ñдком ÑоответÑтвующих преобразований.
- *
- * Второе важное замечание каÑаетÑÑ Ð¾Ð³Ñ€Ð°Ð½Ð¸Ñ‡ÐµÐ½Ð¸Ð¹ одновременной координированной
- * генерации паттеров как Ð´Ð»Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹, так и Ð´Ð»Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹. Суть в том, что
- * Ñ‚Ð°ÐºÐ°Ñ Ð²Ð¾Ð·Ð¼Ð¾Ð¶Ð½Ð¾ÑÑ‚ÑŒ не нужна по Ñледующим причинам:
- * - libmdbx поддерживает два ÑущеÑтвенно различающихÑÑ Ð²Ð¸Ð´Ð° таблиц,
- * "уникальные" (без дубликатов и без multi-value), и так называемые
- * "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸" (c multi-value).
- * - Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† "без дубликатов" только размер ÑвÑзанных Ñ ÐºÐ»ÑŽÑ‡Ð°Ð¼Ð¸ значений
- * (данных) оказывает влиÑÐ½Ð¸Ñ Ð½Ð° работу движка, непоÑредÑтвенно Ñодержимое
- * данных не анализируетÑÑ Ð´Ð²Ð¸Ð¶ÐºÐ¾Ð¼ и не оказывает влиÑÐ½Ð¸Ñ Ð½Ð° его работу.
- * - Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸", при наличии более одного Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð´Ð»Ñ
- * некоторого ключа, формируетÑÑ Ð´Ð¾Ñ‡ÐµÑ€Ð½ÐµÐµ btree-поддерево. Ðто дерево
- * формируетÑÑ Ð²Ð¾ вложенной Ñтранице или отдельном "куÑте" Ñтраниц,
- * и обÑлуживаетÑÑ Ð½ÐµÐ·Ð°Ð²Ð¸Ñимо от Ð¾ÐºÑ€ÑƒÐ¶ÐµÐ½Ð¸Ñ Ñ€Ð¾Ð´Ð¸Ñ‚ÐµÐ»ÑŒÑкого ключа.
- * - Таким образом, паттерн генерации значений имеет ÑмыÑл только длÑ
- * таблиц "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸" и только в контекÑте одного Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡Ð°.
- * Иначе говорÑ, не имеет ÑмыÑла Ð²Ð·Ð°Ð¸Ð¼Ð½Ð°Ñ ÐºÐ¾Ð¾Ñ€Ð´Ð¸Ð½Ð°Ñ†Ð¸Ñ Ð¿Ñ€Ð¸ генерации
- * значений Ð´Ð»Ñ Ñ€Ð°Ð·Ð½Ñ‹Ñ… ключей. ПоÑтому генерацию значений Ñледует
- * раÑÑматривать только в контекÑте ÑвÑзки Ñ Ð¾Ð´Ð½Ð¸Ð¼ значением ключа.
- * - Тем не менее, во вÑех ÑлучаÑÑ… доÑтаточно важным ÑвлÑетÑÑ Ñ€Ð°Ð²Ð½Ð¾Ð²ÐµÑ€Ð¾Ñтное
- * раÑпределение вÑех возможных Ñочетаний длин ключей и данных.
- *
- * width:
- * БольшинÑтво теÑтов предполагают Ñоздание или итерирование некоторого
- * количеÑтва запиÑей. При Ñтом требуетÑÑ Ð¸Ñ‚ÐµÑ€Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ðµ или генерациÑ
- * значений и ключей из некоторого ограниченного проÑтранÑтва вариантов.
- *
- * Параметр width задает такую ширину проÑтранÑтва вариантов в битах.
- * Таким образом мощноÑÑ‚ÑŒ проÑтранÑтва вариантов (пока) вÑегда равна
- * Ñтепени двойки. Ðто ограничение можно ÑнÑÑ‚ÑŒ, но ценой увеличениÑ
- * вычиÑлительной ÑложноÑти, Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ Ð¿Ð¾Ñ‚ÐµÑ€ÑŽ проÑтоты и прозрачноÑти.
- *
- * С другой Ñтороны, не-n-битовый width может быть полезен:
- * - Позволит генерировать ключи/Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð² точно задаваемом диапазоне.
- * Ðапример, перебрать в пÑевдо-Ñлучайном порÑдке 10001 значение.
- * - Позволит поровну разделÑÑ‚ÑŒ заданное проÑтранÑтво (диапазон)
- * ключей/значений между количеÑтвом потоков некратным Ñтепени двойки.
- *
- * mesh и seed:
- * ПозволÑÑŽÑ‚ получить пÑевдо-Ñлучайные поÑледовательноÑти ключей/значений.
- * Параметр mesh задает Ñколько младших бит иÑходной плоÑкой координаты
- * будет "перемешано" (инъективно отображено), а параметр seed позволÑет
- * выбрать конкретный вариант "перемешиваниÑ".
- *
- * Перемешивание выполнÑетÑÑ Ð¿Ñ€Ð¸ ненулевом значении mesh. Перемешивание
- * реализуетÑÑ Ð¿Ð¾ÑредÑтвом Ð¿Ñ€Ð¸Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð´Ð²ÑƒÑ… инъективных функций длÑ
- * заданного количеÑтва бит:
- * - применÑетÑÑ Ð¿ÐµÑ€Ð²Ð°Ñ Ð¸Ð½ÑŠÐµÐºÑ‚Ð¸Ð²Ð½Ð°Ñ Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ;
- * - к результату добавлÑетÑÑ salt полученный из seed;
- * - применÑетÑÑ Ð²Ñ‚Ð¾Ñ€Ð°Ñ Ð¸Ð½ÑŠÐµÐºÑ‚Ð¸Ð²Ð½Ð°Ñ Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ;
- *
- * Следует отметить, что mesh умышленно позволÑет перемешать только младшую
- * чаÑÑ‚ÑŒ, что при ненулевом значении split (Ñм далее) не позволÑет получать
- * пÑевдо-Ñлучайные значений ключей без пÑевдо-ÑлучайноÑти в значениÑÑ….
- *
- * Такое ограничение ÑоответÑтвуют внутренней алгоритмике libmdbx. Проще
- * говорÑ, мы можем проверить движок пÑевдо-Ñлучайной поÑледовательноÑтью
- * ключей на таблицах без дубликатов (без multi-value), а затем проверить
- * корректноÑÑ‚ÑŒ работу пÑевдо-Ñлучайной поÑледовательноÑтью значений на
- * таблицах Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸ (Ñ multi-value), опционально добавлÑÑ
- * пÑевдо-ÑлучайноÑти к поÑледовательноÑти ключей. Однако, нет ÑмыÑла
- * генерировать пÑевдо-Ñлучайные ключи, одновременно Ñ Ñ„Ð¾Ñ€Ð¼Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸ÐµÐ¼
- * какого-либо паттерна в значениÑÑ…, так как Ñодержимое в данных либо
- * не будет иметь Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ (Ð´Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† без дубликатов), либо будет
- * обрабатыватьÑÑ Ð² отдельных btree-поддеревьÑÑ….
- *
- * rotate и offset:
- * Ð”Ð»Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¸ ÑлиÑÐ½Ð¸Ñ Ð¸ Ñ€Ð°Ð·Ð´ÐµÐ»ÐµÐ½Ð¸Ñ Ñтраниц внутри движка требуютÑÑ
- * Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹/значений в виде не-Ñмежных поÑледовательноÑтей, как-бы
- * в виде "пунктира", который поÑтепенно заполнÑет веÑÑŒ заданный диапазон.
- *
- * Параметры позволÑÑŽÑ‚ генерировать такой "пунктир". СоответÑтвенно rotate
- * задает цикличеÑкий Ñдвиг вправо, а offset задает Ñмещение, точнее говорÑ
- * Ñложение по модулю внутри диапазона заданного поÑредÑтвом width.
- *
- * Ðапример, при rotate равном 1 (цикличеÑкий Ñдвиг вправо на 1 бит),
- * четные и нечетные иÑходные Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ ÑложатÑÑ Ð² две линейные
- * поÑледовательноÑти, которые поÑтепенно закроют Ñтаршую и младшую
- * половины диапазона.
- *
- * split:
- * Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† без дубликатов (без multi-value ключей) фактичеÑки требуетÑÑ
- * Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñ‚Ð¾Ð»ÑŒÐºÐ¾ ключей, а данные могут быть поÑтоÑнным. Ðо Ð´Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† Ñ
- * дубликатами (Ñ multi-value ключами) также требуетÑÑ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹.
- *
- * Ðенулевое значение параметра split фактичеÑки включает генерацию значений,
- * при Ñтом значение split определÑет Ñколько бит иÑходного абÑтрактного
- * номера будет отрезано Ð´Ð»Ñ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ð¸ значениÑ.
- */
-
- uint8_t width{0};
- uint8_t mesh{0};
- uint8_t rotate{0};
- uint8_t split{0};
- uint32_t seed{0};
- uint64_t offset{0};
- keygen_case keycase{kc_random};
- bool zero_fill{false};
-};
-
-struct actor_params_pod {
- MDBX_env_flags_t mode_flags{MDBX_ENV_DEFAULTS};
- MDBX_db_flags_t table_flags{MDBX_DB_DEFAULTS};
- intptr_t size_lower{0};
- intptr_t size_now{0};
- intptr_t size_upper{0};
- int shrink_threshold{0};
- int growth_step{0};
- int pagesize{0};
-
- unsigned test_duration{0};
- unsigned test_nops{0};
- unsigned nrepeat{0};
- unsigned nthreads{0};
-
- unsigned keylen_min{0}, keylen_max{0};
- unsigned datalen_min{0}, datalen_max{0};
-
- unsigned batch_read{0};
- unsigned batch_write{0};
-
- unsigned delaystart{0};
- unsigned waitfor_nops{0};
- unsigned inject_writefaultn{0};
-
- unsigned max_readers{0};
- unsigned max_tables{0};
- keygen_params_pod keygen;
-
- uint8_t loglevel{0};
- bool drop_table{false};
- bool ignore_dbfull{false};
- bool speculum{false};
- bool random_writemap{true};
-
- uint64_t serial_base() const {
- // FIXME: TODO
- return 0;
- }
- static MDBX_PURE_FUNCTION uint64_t serial_mask(unsigned bits) {
- assert(bits > 0 && bits <= 64);
- return (~(uint64_t)0u) >> (64 - bits);
- }
-};
-
-struct actor_config_pod {
- unsigned actor_id{0}, space_id{0};
- actor_testcase testcase{ac_none};
- unsigned wait4id{0};
- unsigned signal_nops{0};
-
- actor_config_pod() = default;
- actor_config_pod(unsigned actor_id, actor_testcase testcase,
- unsigned space_id, unsigned wait4id)
- : actor_id(actor_id), space_id(space_id), testcase(testcase),
- wait4id(wait4id) {}
-};
-
-extern const struct option_verb mode_bits[];
-extern const struct option_verb table_bits[];
-void dump(const char *title = "config-dump: ");
-
-} /* namespace config */
-
-struct actor_params : public config::actor_params_pod {
- std::string pathname_log;
- std::string pathname_db;
- actor_params() = default;
-
- void set_defaults(const std::string &tmpdir);
- bool make_keygen_linear();
- unsigned mdbx_keylen_min() const;
- unsigned mdbx_keylen_max() const;
- unsigned mdbx_datalen_min() const;
- unsigned mdbx_datalen_max() const;
-};
-
-struct actor_config : public config::actor_config_pod {
- actor_params params;
-
- bool wanna_event4signalling() const { return true /* TODO ? */; }
-
- actor_config() = default;
- actor_config(actor_testcase testcase, const actor_params ¶ms,
- unsigned space_id, unsigned wait4id);
-
- actor_config(const char *str) : actor_config() {
- if (!deserialize(str, *this))
- failure("Invalid internal parameter '%s'\n", str);
- }
-
- const std::string osal_serialize(simple_checksum &) const;
- bool osal_deserialize(const char *str, const char *end, simple_checksum &);
-
- const std::string serialize(const char *prefix) const;
- static bool deserialize(const char *str, actor_config &config);
-
- bool is_waitable(size_t nops) const {
- switch (testcase) {
- case ac_hill:
- if (!params.test_nops || params.test_nops >= nops)
- return true;
- __fallthrough;
- default:
- return false;
- }
- }
-};
diff --git a/3rdparty/libmdbx/test/copy.cc b/3rdparty/libmdbx/test/copy.cc
deleted file mode 100644
index 37c58a248668f0f3e195cd544247783901f6203e..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/copy.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-#include "test.h"
-
-class testcase_copy : public testcase {
- const std::string copy_pathname;
- void copy_db(const bool with_compaction);
-
-public:
- testcase_copy(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid),
- copy_pathname(config.params.pathname_db + "-copy") {}
- bool run() override;
-};
-REGISTER_TESTCASE(copy);
-
-void testcase_copy::copy_db(const bool with_compaction) {
- int err = mdbx_env_delete(copy_pathname.c_str(), MDBX_ENV_JUST_DELETE);
- if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE)
- failure_perror("mdbx_removefile()", err);
-
- err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(),
- with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)"
- : "mdbx_env_copy(MDBX_CP_ASIS)",
- err);
-}
-
-bool testcase_copy::run() {
- jitter_delay();
- db_open();
- assert(!txn_guard);
- const bool order = flipcoin();
- jitter_delay();
- copy_db(order);
- jitter_delay();
- copy_db(!order);
- return true;
-}
diff --git a/3rdparty/libmdbx/test/dead.cc b/3rdparty/libmdbx/test/dead.cc
deleted file mode 100644
index 05304bc22dece3143b74637f9a8f5d019121ef73..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/dead.cc
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-class testcase_deadread : public testcase {
-public:
- testcase_deadread(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-};
-REGISTER_TESTCASE(deadread);
-
-bool testcase_deadread::run() {
- db_open();
- txn_begin(true);
- cursor_guard.reset();
- txn_guard.reset();
- db_guard.reset();
- return true;
-}
-
-//-----------------------------------------------------------------------------
-
-class testcase_deadwrite : public testcase {
-public:
- testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-};
-
-REGISTER_TESTCASE(deadwrite);
-
-bool testcase_deadwrite::run() {
- db_open();
- txn_begin(false);
- cursor_guard.reset();
- txn_guard.reset();
- db_guard.reset();
- return true;
-}
diff --git a/3rdparty/libmdbx/test/dump-load.sh b/3rdparty/libmdbx/test/dump-load.sh
deleted file mode 100755
index 55fa5c7f3389abc826f3ab1949d0946764d1347e..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/dump-load.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-echo "------------------------------------------------------------------------------"
-
-if [ -z "$1" ]; then
- echo "No mdbx-db pathname given";
- exit 2
-elif [ ! -e "$1" ]; then
- echo "The mdbx-db '$1' don't exists";
- exit 2
-else
- echo ">>>>>>>>>> $1"
- RECO="$1.recovered"
- rm -f dump1.txt dump2.txt "$RECO"
- if ./mdbx_chk "$1"; then
- echo ">>>>>>>>>> SOURCE VALID"
- (./mdbx_dump -a "$1" > dump1.txt && \
- ./mdbx_load -nf dump1.txt "$RECO" && \
- ./mdbx_chk "$RECO" && \
- echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1)
- REMOVE_RECO=1
- elif ./mdbx_chk -i "$1"; then
- echo ">>>>>>>>>> SOURCE HAS WRONG-ORDER, TRY RECOVERY"
- (./mdbx_dump -a "$1" > dump1.txt && \
- ./mdbx_load -anf dump1.txt "$RECO" && \
- ./mdbx_chk -i "$RECO" && \
- echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1)
- REMOVE_RECO=0
- else
- echo ">>>>>>>>>> SOURCE CORRUPTED, TRY RECOVERY"
- (./mdbx_dump -ar "$1" > dump1.txt && \
- ./mdbx_load -ranf dump1.txt "$RECO" && \
- ./mdbx_chk -i "$RECO" && \
- echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1)
- REMOVE_RECO=0
- fi
- ./mdbx_dump -a "$RECO" > dump2.txt && diff -u dump1.txt dump2.txt && \
- rm -f dump1.txt dump2.txt && [ $REMOVE_RECO -ne 0 ] && rm -f "$RECO"
- exit 0
-fi
diff --git a/3rdparty/libmdbx/test/hill.cc b/3rdparty/libmdbx/test/hill.cc
deleted file mode 100644
index 5aea4d71850e9c2a699fd53bdfe932229550a56d..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/hill.cc
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-/* LY: теÑÑ‚ "холмиком":
- * - Ñначала наполнÑем таблицу цикличеÑкими CRUD-манипулÑциÑми,
- * которые в каждом цикле делают неÑколько операций, Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ,
- * но в результате добавлÑÑŽÑ‚ запиÑи.
- * - затем очищаем таблицу также CRUD-манипулÑциÑми, но уже Ñ Ð´Ñ€ÑƒÐ³Ð¾Ð¹
- * пропорцией удалений.
- *
- * При Ñтом очень многое завиÑит от порÑдка перебора ключей:
- * - (пÑевдо)Ñлучайное раÑпределение требуетÑÑ Ð»Ð¸ÑˆÑŒ Ð´Ð»Ñ Ð¿Ð¾Ð»Ð½Ð¾Ñ‚Ñ‹ картины,
- * но в целом не покрывает важных кейÑов.
- * - кроме (пÑевдо)Ñлучайного перебора требуетÑÑ Ð¿Ð¾Ñледовательное
- * итерирование ключей интервалами различной ширины, Ñ Ñ‚ÐµÐ¼ чтобы
- * проверить различные варианты как разделениÑ, так и ÑлиÑÐ½Ð¸Ñ Ñтраниц
- * внутри движка.
- * - при не-уникальных ключах (MDBX_DUPSORT Ñ Ð¿Ð¾Ð´Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ð°Ð¼Ð¸), Ð´Ð»Ñ ÐºÐ°Ð¶Ð´Ð¾Ð³Ð¾
- * повтора внутри движка формируетÑÑ Ð²Ð»Ð¾Ð¶ÐµÐ½Ð½Ð¾Ðµ btree-дерево,
- * ÑоответÑтвенно требуетÑÑ Ñоблюдение аналогичных принципов
- * Ð¸Ñ‚ÐµÑ€Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð´Ð»Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹.
- */
-
-class testcase_hill : public testcase {
-public:
- testcase_hill(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-};
-REGISTER_TESTCASE(hill);
-
-bool testcase_hill::run() {
- int err = db_open__begin__table_create_open_clean(dbi);
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
- return false;
- }
- speculum.clear();
- speculum_committed.clear();
-
- /* TODO: работа в неÑколько потоков */
- keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
-
- keygen::buffer a_key = keygen::alloc(config.params.keylen_max);
- keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max);
- keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max);
- keygen::buffer b_key = keygen::alloc(config.params.keylen_max);
- keygen::buffer b_data = keygen::alloc(config.params.datalen_max);
-
- const MDBX_put_flags_t insert_flags =
- (config.params.table_flags & MDBX_DUPSORT)
- ? MDBX_NODUPDATA
- : MDBX_NODUPDATA | MDBX_NOOVERWRITE;
- const MDBX_put_flags_t update_flags =
- (config.params.table_flags & MDBX_DUPSORT)
- ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE
- : MDBX_NODUPDATA;
-
- uint64_t serial_count = 0;
- uint64_t committed_serial = serial_count;
- unsigned txn_nops = 0;
-
- bool rc = speculum_verify();
- if (!rc) {
- log_notice("uphill: bailout before main loop");
- goto bailout;
- }
-
- while (should_continue()) {
- const keygen::serial_t a_serial = serial_count;
- if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
- log_notice("uphill: unexpected key-space overflow");
- break;
- }
-
- const keygen::serial_t b_serial = serial_count;
- assert(b_serial > a_serial);
-
- // Ñоздаем первую запиÑÑŒ из пары
- const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
- log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
- a_serial);
- generate_pair(a_serial, a_key, a_data_1, age_shift);
-
- err = insert(a_key, a_data_1, insert_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
- txn_restart(true, false);
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_put(insert-a.1)", err);
- }
- if (!speculum_verify()) {
- log_notice("uphill: bailout after insert-a, before commit");
- goto bailout;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- committed_serial = a_serial;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("uphill: bailout after insert-a, after commit");
- goto bailout;
- }
- }
-
- // Ñоздаем вторую запиÑÑŒ из пары
- log_trace("uphill: insert-b %" PRIu64, b_serial);
- generate_pair(b_serial, b_key, b_data, 0);
- err = insert(b_key, b_data, insert_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
- txn_restart(true, false);
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_put(insert-b)", err);
- }
- if (!speculum_verify()) {
- log_notice("uphill: bailout after insert-b, before commit");
- goto bailout;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- committed_serial = a_serial;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("uphill: bailout after insert-b, after commit");
- goto bailout;
- }
- }
-
- // обновлÑем данные в первой запиÑи
- log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift,
- a_serial);
- generate_pair(a_serial, a_key, a_data_0, 0);
- checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
- err = replace(a_key, a_data_0, a_data_1, update_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
- txn_restart(true, false);
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_replace(update-a: 1->0)", err);
- }
- if (!speculum_verify()) {
- log_notice("uphill: bailout after update-a, before commit");
- goto bailout;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- committed_serial = a_serial;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("uphill: bailout after update-a, after commit");
- goto bailout;
- }
- }
-
- // удалÑем вторую запиÑÑŒ
- log_trace("uphill: delete-b %" PRIu64, b_serial);
- checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
- err = remove(b_key, b_data);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
- txn_restart(true, false);
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_del(b)", err);
- }
- if (!speculum_verify()) {
- log_notice("uphill: bailout after delete-b, before commit");
- goto bailout;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
- serial_count = committed_serial;
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- committed_serial = a_serial;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("uphill: bailout after delete-b, after commit");
- goto bailout;
- }
- }
-
- report(1);
- if (!keyvalue_maker.increment(serial_count, 1)) {
- // дошли до границы проÑтранÑтва ключей
- serial_count = a_serial;
- goto overflow;
- }
- }
-
- if (txn_guard) {
- MDBX_stat stat;
- err = mdbx_dbi_stat(txn_guard.get(), dbi, &stat, sizeof(stat));
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_dbi_stat()", err);
-
- uint32_t nested_deepmask;
- err = mdbx_dbi_dupsort_depthmask(txn_guard.get(), dbi, &nested_deepmask);
- if (unlikely(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE))
- failure_perror("mdbx_dbi_stat_nested_deepmask()", err);
-
- if (err != MDBX_SUCCESS) {
- log_notice("hill: reached %d tree depth", stat.ms_depth);
- } else {
- std::string str;
- int prev = -2, i = 0;
- do {
- while (!(nested_deepmask & 1))
- ++i, nested_deepmask >>= 1;
- if (prev + 1 == i) {
- if (str.back() != '-')
- str.push_back('-');
- prev = i;
- continue;
- }
- if (!str.empty()) {
- if (str.back() == '-')
- str.append(std::to_string(prev));
- str.push_back(',');
- }
- str.append(std::to_string(i));
- prev = i;
- } while (++i, nested_deepmask >>= 1);
- if (str.back() == '-')
- str.append(std::to_string(prev));
-
- log_notice("hill: reached %d tree depth & %s sub-tree depth(s)",
- stat.ms_depth, str.c_str());
- }
-
- if ((config.params.table_flags & MDBX_DUPSORT) == 0) {
- if (!check_batch_get())
- failure("batch-get verification failed");
- }
- }
-
- while (serial_count > 1) {
- if (unlikely(!keyvalue_maker.increment(serial_count, -2)))
- failure("downhill: unexpected key-space underflow");
-
- overflow:
- const keygen::serial_t a_serial = serial_count;
- const keygen::serial_t b_serial = a_serial + 1;
- assert(b_serial > a_serial);
-
- // обновлÑем первую запиÑÑŒ из пары
- const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
- log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
- a_serial);
- generate_pair(a_serial, a_key, a_data_0, 0);
- generate_pair(a_serial, a_key, a_data_1, age_shift);
- checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
- err = replace(a_key, a_data_1, a_data_0, update_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("downhill: bailout at update-a due '%s'",
- mdbx_strerror(err));
- txn_end(true);
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_put(update-a: 0->1)", err);
- }
- if (!speculum_verify()) {
- log_notice("downhill: bailout after update-a, before commit");
- break;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("downhill: bailout after update-a, after commit");
- break;
- }
- }
-
- // Ñоздаем вторую запиÑÑŒ из пары
- log_trace("downhill: insert-b %" PRIu64, b_serial);
- generate_pair(b_serial, b_key, b_data, 0);
- err = insert(b_key, b_data, insert_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("downhill: bailout at insert-a due '%s'",
- mdbx_strerror(err));
- txn_end(true);
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_put(insert-b)", err);
- }
- if (!speculum_verify()) {
- log_notice("downhill: bailout after insert-b, before commit");
- break;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("downhill: bailout after insert-b, after commit");
- break;
- }
- }
-
- // удалÑем первую запиÑÑŒ
- log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
- a_serial);
- checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
- err = remove(a_key, a_data_1);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("downhill: bailout at delete-a due '%s'",
- mdbx_strerror(err));
- txn_end(true);
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_del(a)", err);
- }
- if (!speculum_verify()) {
- log_notice("downhill: bailout after delete-a, before commit");
- break;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("downhill: bailout after delete-a, after commit");
- break;
- }
- }
-
- // удалÑем вторую запиÑÑŒ
- log_trace("downhill: delete-b %" PRIu64, b_serial);
- checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
- err = remove(b_key, b_data);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("downhill: bailout at delete-b due '%s'",
- mdbx_strerror(err));
- txn_end(true);
- speculum = speculum_committed;
- break;
- }
- failure_perror("mdbx_del(b)", err);
- }
- if (!speculum_verify()) {
- log_notice("downhill: bailout after delete-b, before commit");
- break;
- }
-
- if (++txn_nops >= config.params.batch_write) {
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
- speculum = speculum_committed;
- break;
- }
- speculum_committed = speculum;
- txn_nops = 0;
- if (!speculum_verify()) {
- log_notice("downhill: bailout after delete-b, after commit");
- goto bailout;
- }
- }
-
- report(1);
- }
-
- rc = speculum_verify();
-bailout:
- if (txn_guard) {
- err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS))
- log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
- }
-
- if (!rc && err == MDBX_MAP_FULL && config.params.ignore_dbfull)
- rc = true;
- if (dbi) {
- if (config.params.drop_table && !mode_readonly()) {
- txn_begin(false);
- db_table_drop(dbi);
- err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
- if (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)
- rc = false;
- }
- } else
- db_table_close(dbi);
- }
- return rc;
-}
diff --git a/3rdparty/libmdbx/test/jitter.cc b/3rdparty/libmdbx/test/jitter.cc
deleted file mode 100644
index 2c781d7fc847bb6574acbe690ae042bc4f284ea0..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/jitter.cc
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-class testcase_jitter : public testcase {
-protected:
- void check_dbi_error(int expect, const char *stage);
-
-public:
- testcase_jitter(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-};
-REGISTER_TESTCASE(jitter);
-
-void testcase_jitter::check_dbi_error(int expect, const char *stage) {
- MDBX_stat stat;
- int err = mdbx_dbi_stat(txn_guard.get(), dbi, &stat, sizeof(stat));
- if (err != expect)
- failure("unexpected result for %s dbi-handle: expect %d, got %d", stage,
- expect, err);
-}
-
-bool testcase_jitter::run() {
- int err;
- size_t upper_limit = config.params.size_upper;
- if (upper_limit < 1)
- upper_limit = config.params.size_now * 2;
-
- while (should_continue()) {
- jitter_delay();
- db_open();
-
- if (!dbi && !mode_readonly()) {
- // create table
- txn_begin(false);
- dbi = db_table_open(true);
- check_dbi_error(MDBX_SUCCESS, "created-uncommitted");
- // note: here and below the 4-byte length keys and value are used
- // to be compatible with any Db-flags given from command line.
- MDBX_val k = {(void *)"k000", 4}, v = {(void *)"v001", 4};
- err = mdbx_put(txn_guard.get(), dbi, &k, &v, MDBX_UPSERT);
- if (err != MDBX_SUCCESS)
- failure_perror("jitter.put-1", err);
- txn_end(false);
-
- // drop & re-create table, but abort txn
- txn_begin(false);
- check_dbi_error(MDBX_SUCCESS, "created-committed");
- err = mdbx_drop(txn_guard.get(), dbi, true);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_drop(delete=true)", err);
- check_dbi_error(MDBX_BAD_DBI, "dropped-uncommitted");
- dbi = db_table_open(true);
- check_dbi_error(MDBX_SUCCESS, "recreated-uncommitted");
- txn_end(true);
-
- // check after aborted txn
- txn_begin(false);
- v = {(void *)"v002", 4};
- err = mdbx_put(txn_guard.get(), dbi, &k, &v, MDBX_UPSERT);
- if (err != MDBX_BAD_DBI)
- failure_perror("jitter.put-2", err);
- check_dbi_error(MDBX_BAD_DBI, "dropped-recreated-aborted");
- // restore DBI
- dbi = db_table_open(false);
- check_dbi_error(MDBX_SUCCESS, "dropped-recreated-aborted+reopened");
- v = {(void *)"v003", 4};
- err = mdbx_put(txn_guard.get(), dbi, &k, &v, MDBX_UPSERT);
- if (err != MDBX_SUCCESS)
- failure_perror("jitter.put-3", err);
- txn_end(false);
- }
-
- if (upper_limit < 1) {
- MDBX_envinfo info;
- err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &info,
- sizeof(info));
- if (err)
- failure_perror("mdbx_env_info_ex()", err);
- upper_limit = (info.mi_geo.upper < INTPTR_MAX)
- ? (intptr_t)info.mi_geo.upper
- : INTPTR_MAX;
- }
-
- if (flipcoin()) {
- jitter_delay();
- txn_begin(true);
- fetch_canary();
- jitter_delay();
- txn_end(flipcoin());
- }
-
- const bool coin4size = flipcoin();
- jitter_delay();
- txn_begin(mode_readonly());
- jitter_delay();
- if (!mode_readonly()) {
- fetch_canary();
- update_canary(1);
- if (global::config::geometry_jitter) {
- err = mdbx_env_set_geometry(
- db_guard.get(), -1, -1,
- coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1);
- if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
- err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM)
- failure_perror("mdbx_env_set_geometry-1", err);
- }
- }
- txn_end(flipcoin());
-
- if (global::config::geometry_jitter) {
- err = mdbx_env_set_geometry(
- db_guard.get(), -1, -1,
- !coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1);
- if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
- err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM)
- failure_perror("mdbx_env_set_geometry-2", err);
- }
-
- if (flipcoin()) {
- jitter_delay();
- txn_begin(true);
- jitter_delay();
- txn_end(flipcoin());
- }
-
- if (global::config::geometry_jitter) {
- jitter_delay();
- err = mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1,
- -1);
- if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
- err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM)
- failure_perror("mdbx_env_set_geometry-3", err);
- }
-
- db_close();
-
- /* just 'align' nops with other tests with batching */
- const auto batching =
- std::max(config.params.batch_read, config.params.batch_write);
- report(std::max(1u, batching / 2));
- }
- return true;
-}
diff --git a/3rdparty/libmdbx/test/keygen.cc b/3rdparty/libmdbx/test/keygen.cc
deleted file mode 100644
index 807954f3fd95b19bae65458fa21861bb9cb5394f..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/keygen.cc
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-namespace keygen {
-
-/* LY: https://en.wikipedia.org/wiki/Injective_function */
-serial_t injective(const serial_t serial,
- const unsigned bits /* at least serial_minwith (8) */,
- const serial_t salt) {
- assert(bits >= serial_minwith && bits <= serial_maxwith);
-
- /* LY: All these "magic" prime numbers were found
- * and verified with a bit of brute force. */
-
- static const uint64_t m[64 - serial_minwith + 1] = {
- /* 8 - 24 */
- 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937,
- 309277, 834497, 1499933, 4373441, 10184137,
- /* 25 - 64 */
- 10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573,
- 6924051577, 19204053433, 45840188887, 53625693977, 73447827913,
- 141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197,
- 5871903036137, 11238507001417, 45264020802263, 105008404482889,
- 81921776907059, 199987980256399, 307207457507641, 946769023178273,
- 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153,
- 23171543400565993, 53353226456762893, 155627817337932409,
- 227827205384840249, 816509268558278821, 576933057762605689,
- 2623957345935638441, 5048241705479929949, 4634245581946485653,
- 4613509448041658233, 4952535426879925961};
- static const uint8_t s[64 - serial_minwith + 1] = {
- /* 8 - 24 */
- 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11,
- /* 25 - 64 */
- 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20,
- 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
- 19};
-
- const auto mult = m[bits - 8];
- const auto shift = s[bits - 8];
- serial_t result = serial * mult;
- if (salt) {
- const unsigned left = bits / 2;
- const unsigned right = bits - left;
- result = (result << left) |
- ((result & actor_params::serial_mask(bits)) >> right);
- result = (result ^ salt) * mult;
- }
-
- result ^= result << shift;
- result &= actor_params::serial_mask(bits);
- log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
- " => %" PRIu64 "/%u",
- serial, bits, mult, shift, salt, result, bits);
- return result;
-}
-
-void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
- serial_t value_age, const bool keylen_changeable) {
- assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith);
- assert(mapping.split <= mapping.width);
- assert(mapping.mesh <= mapping.width);
- assert(mapping.rotate <= mapping.width);
- assert(mapping.offset <= actor_params::serial_mask(mapping.width));
- assert(!(key_essentials.flags &
- ~(essentials::prng_fill_flag |
- unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT))));
- assert(!(value_essentials.flags &
- ~(essentials::prng_fill_flag |
- unsigned(MDBX_INTEGERDUP | MDBX_REVERSEDUP))));
-
- log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
- value_age);
-
- if (mapping.mesh >= serial_minwith) {
- serial = (serial & ~actor_params::serial_mask(mapping.mesh)) |
- injective(serial, mapping.mesh, salt);
- log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial);
- }
-
- if (mapping.rotate) {
- const unsigned right = mapping.rotate;
- const unsigned left = mapping.width - right;
- serial = (serial << left) |
- ((serial & actor_params::serial_mask(mapping.width)) >> right);
- log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64,
- mapping.rotate, serial, serial);
- }
-
- if (mapping.offset) {
- serial =
- (serial + mapping.offset) & actor_params::serial_mask(mapping.width);
- log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset,
- serial);
- }
- if (base) {
- serial += base;
- log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial);
- }
-
- serial_t key_serial = serial;
- serial_t value_serial = value_age << mapping.split;
- if (mapping.split) {
- if (MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) {
- key_serial >>= mapping.split;
- value_serial += serial & actor_params::serial_mask(mapping.split);
- } else {
- /* Без MDBX_DUPSORT требуетÑÑ ÑƒÐ½Ð¸ÐºÐ°Ð»ÑŒÐ½Ð¾ÑÑ‚ÑŒ ключей, а Ð´Ð»Ñ Ñтого нельзÑ
- * отбраÑывать какие-либо биты serial поÑле инъективного преобразованиÑ.
- * ПоÑтому key_serial не трогаем, а в value_serial нелинейно вмешиваем
- * запрошенное количеÑтво бит из serial */
- value_serial +=
- (serial ^ (serial >> mapping.split) * UINT64_C(57035339200100753)) &
- actor_params::serial_mask(mapping.split);
- }
-
- value_serial |= value_age << mapping.split;
- log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split,
- key_serial, value_serial);
- }
-
- log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial,
- value_serial);
- mk_begin(key_serial, key_essentials, *key);
- mk_begin(value_serial, value_essentials, *value);
-
-#if 0 /* unused for now */
- if (key->value.iov_len + value->value.iov_len > pair_maxlen) {
- unsigned extra = key->value.iov_len + value->value.iov_len - pair_maxlen;
- if (keylen_changeable &&
- key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) {
-#if defined(__GNUC__) || defined(__clang__)
- const bool coin = __builtin_parityll(serial) != 0;
-#else
- const bool coin = INT64_C(0xF2CEECA9989BD96A) * int64_t(serial) < 0;
-#endif
- if (coin) {
- const unsigned gap =
- key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen);
- const unsigned chop = std::min(gap, extra);
- log_trace("keygen-pair: chop %u key-len %u -> %u", chop,
- (unsigned)key->value.iov_len,
- (unsigned)key->value.iov_len - chop);
- key->value.iov_len -= chop;
- extra -= chop;
- }
- }
- if (extra && value->value.iov_len >
- std::max(8u, (unsigned)value_essentials.minlen)) {
- const unsigned gap = value->value.iov_len -
- std::max(8u, (unsigned)value_essentials.minlen);
- const unsigned chop = std::min(gap, extra);
- log_trace("keygen-pair: chop %u value-len %u -> %u", chop,
- (unsigned)value->value.iov_len,
- (unsigned)value->value.iov_len - chop);
- value->value.iov_len -= chop;
- extra -= chop;
- }
- if (keylen_changeable && extra &&
- key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) {
- const unsigned gap =
- key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen);
- const unsigned chop = std::min(gap, extra);
- log_trace("keygen-pair: chop %u key-len %u -> %u", chop,
- (unsigned)key->value.iov_len,
- (unsigned)key->value.iov_len - chop);
- key->value.iov_len -= chop;
- extra -= chop;
- }
- }
-#else
- (void)keylen_changeable;
-#endif /* unused for now */
-
- mk_continue(key_serial, key_essentials, *key);
- mk_continue(value_serial, value_essentials, *value);
- log_pair(logging::trace, "kv", key, value);
-}
-
-void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
- unsigned thread_number) {
-#if CONSTEXPR_ENUM_FLAGS_OPERATIONS
- static_assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT |
- MDBX_INTEGERDUP | MDBX_REVERSEDUP) < UINT16_MAX,
- "WTF?");
-#else
- assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT |
- MDBX_INTEGERDUP | MDBX_REVERSEDUP) < UINT16_MAX);
-#endif
- key_essentials.flags = uint16_t(
- actor.table_flags &
- MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT));
- assert(actor.keylen_min <= UINT16_MAX);
- key_essentials.minlen = uint16_t(actor.keylen_min);
- assert(actor.keylen_max <= UINT32_MAX);
- key_essentials.maxlen =
- std::min(uint32_t(actor.keylen_max),
- uint32_t(mdbx_limits_keysize_max(
- actor.pagesize, MDBX_db_flags_t(key_essentials.flags))));
-
- value_essentials.flags = uint16_t(
- actor.table_flags & MDBX_db_flags_t(MDBX_INTEGERDUP | MDBX_REVERSEDUP));
- assert(actor.datalen_min <= UINT16_MAX);
- value_essentials.minlen = uint16_t(actor.datalen_min);
- assert(actor.datalen_max <= UINT32_MAX);
- value_essentials.maxlen =
- std::min(uint32_t(actor.datalen_max),
- uint32_t(mdbx_limits_valsize_max(
- actor.pagesize, MDBX_db_flags_t(key_essentials.flags))));
-
- if (!actor.keygen.zero_fill) {
- key_essentials.flags |= essentials::prng_fill_flag;
- value_essentials.flags |= essentials::prng_fill_flag;
- }
-
- (void)thread_number;
- mapping = actor.keygen;
- salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569);
-
- base = actor.serial_base();
-}
-
-bool maker::is_unordered() const {
- return mapping.rotate ||
- mapping.mesh > ((MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT)
- ? 0
- : mapping.split);
-}
-
-void maker::seek2end(serial_t &serial) const {
- serial = actor_params::serial_mask(mapping.width) - 1;
-}
-
-bool maker::increment(serial_t &serial, int64_t delta) const {
- if (serial > actor_params::serial_mask(mapping.width)) {
- log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
- actor_params::serial_mask(mapping.width));
- return false;
- }
-
- serial_t target = serial + delta;
- if (target > actor_params::serial_mask(mapping.width) ||
- ((delta > 0) ? target < serial : target > serial)) {
- log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64
- ", overflow",
- serial, delta, target);
- return false;
- }
-
- log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64
- ", continue",
- serial, delta, target);
- serial = target;
- return true;
-}
-
-//-----------------------------------------------------------------------------
-
-MDBX_NOTHROW_PURE_FUNCTION static inline unsigned length(serial_t serial) {
-#if defined(__clang__) && __clang__ > 8
- unsigned n = 0;
- if (serial > UINT32_MAX) {
- n = 4;
- serial >>= 32;
- }
- if (serial > UINT16_MAX) {
- n += 2;
- serial >>= 16;
- }
- if (serial > UINT8_MAX) {
- n += 1;
- serial >>= 8;
- }
-#else
- unsigned n = (serial > UINT32_MAX) ? 4 : 0;
- serial = (serial > UINT32_MAX) ? serial >> 32 : serial;
-
- n += (serial > UINT16_MAX) ? 2 : 0;
- serial = (serial > UINT16_MAX) ? serial >> 16 : serial;
-
- n += (serial > UINT8_MAX);
- serial = (serial > UINT8_MAX) ? serial >> 8 : serial;
-#endif
- return n + (serial > 0);
-}
-
-buffer alloc(size_t limit) {
- result *ptr = (result *)malloc(sizeof(result) + limit + 8);
- if (unlikely(ptr == nullptr))
- failure_perror("malloc(keyvalue_buffer)", errno);
- ptr->value.iov_base = ptr->bytes;
- ptr->value.iov_len = 0;
- ptr->limit = limit + 8;
- return buffer(ptr);
-}
-
-void __hot maker::mk_begin(const serial_t serial, const essentials ¶ms,
- result &out) {
- assert(out.limit >= params.maxlen);
- assert(params.maxlen >= params.minlen);
- assert(params.maxlen >= length(serial));
-
- out.value.iov_len = std::max(unsigned(params.minlen), length(serial));
- const auto variation = params.maxlen - params.minlen;
- if (variation) {
- if (serial % (variation + 1)) {
- auto refix = serial * UINT64_C(48835288005252737);
- refix ^= refix >> 32;
- out.value.iov_len = std::max(
- out.value.iov_len, params.minlen + 1 + size_t(refix) % variation);
- }
- }
-
- assert(length(serial) <= out.value.iov_len);
- assert(out.value.iov_len >= params.minlen);
- assert(out.value.iov_len <= params.maxlen);
-}
-
-void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms,
- result &out) {
-#if CONSTEXPR_ENUM_FLAGS_OPERATIONS
- static_assert(
- (essentials::prng_fill_flag &
- unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY |
- MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0,
- "WTF?");
-#else
- assert((essentials::prng_fill_flag &
- unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY |
- MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0);
-#endif
- assert(length(serial) <= out.value.iov_len);
- out.value.iov_base = out.bytes;
- if (MDBX_db_flags_t(params.flags) & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
- assert(params.maxlen == params.minlen);
- if (MDBX_db_flags_t(params.flags) & (MDBX_INTEGERKEY | MDBX_INTEGERDUP))
- assert(params.minlen == 4 || params.minlen == 8);
- out.u64 = serial;
- if (!is_byteorder_le() && out.value.iov_len != 8)
- out.u32 = uint32_t(serial);
- } else {
- const auto prefix =
- std::max(std::min(unsigned(params.minlen), 8u), length(serial));
- out.u64 = htobe64(serial);
- out.value.iov_base = out.bytes + 8 - prefix;
- if (out.value.iov_len > prefix) {
- if (params.flags & essentials::prng_fill_flag) {
- uint64_t state = serial ^ UINT64_C(0x923ab47b7ee6f6e4);
- prng_fill(state, out.bytes + 8, out.value.iov_len - prefix);
- } else
- memset(out.bytes + 8, '\0', out.value.iov_len - prefix);
- }
- if (unlikely(MDBX_db_flags_t(params.flags) &
- (MDBX_REVERSEKEY | MDBX_REVERSEDUP)))
- std::reverse((char *)out.value.iov_base,
- (char *)out.value.iov_base + out.value.iov_len);
- }
-
- assert(out.value.iov_len >= params.minlen);
- assert(out.value.iov_len <= params.maxlen);
- assert(out.value.iov_len >= length(serial));
- assert(out.value.iov_base >= out.bytes);
- assert((char *)out.value.iov_base + out.value.iov_len <=
- (char *)&out.bytes + out.limit);
-}
-
-void log_pair(logging::loglevel level, const char *prefix, const buffer &key,
- buffer &value) {
- if (log_enabled(level)) {
- char dump_key[4096], dump_value[4096];
- logging::output(
- level, "%s-pair: key %s, value %s", prefix,
- mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&value->value, dump_value, sizeof(dump_value)));
- }
-}
-
-} /* namespace keygen */
diff --git a/3rdparty/libmdbx/test/keygen.h b/3rdparty/libmdbx/test/keygen.h
deleted file mode 100644
index 53219f5d718e269621147233fd580f07d15a1793..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/keygen.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-#include "config.h"
-#include "log.h"
-#include "utils.h"
-
-namespace keygen {
-
-/* Под "генерацией ключей" здеÑÑŒ понимаетÑÑ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ð¾Ð±Ð¾Ð¸Ñ… значений длÑ
- * пар key-value, Ñ‚.е. не только ключей, но и аÑÑоциированных Ñ Ð½Ð¸Ð¼Ð¸ данных.
- *
- * Генерацию ключей Ð½ÐµÐ»ÑŒÐ·Ñ Ð¾Ñ‚Ð½ÐµÑти к проÑтым задачам, так как требованиÑ
- * примерно Ñледующие:
- * - Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñ€Ð°Ð·Ð½Ð¾Ð³Ð¾ количеÑтва уникальных ключей различной длины
- * в задаваемом диапазоне;
- * - возможноÑÑ‚ÑŒ выбора как пÑевдо-Ñлучайного порÑдка ключей,
- * так и по некоторым ÑпецифичеÑким законам (ограниченными упорÑдоченными
- * поÑледовательноÑÑ‚Ñми, в шахматном порÑдке по граница диапазона и Ñ‚.д.);
- * - возможноÑÑ‚ÑŒ генерации дубликатов Ñ Ð·Ð°Ð´Ð°Ð²Ð°ÐµÐ¼Ñ‹Ð¼ законом раÑпределениÑ;
- * - возможноÑÑ‚ÑŒ генерации непереÑекающимиÑÑ ÐºÐ»Ð°Ñтерами Ð´Ð»Ñ Ð¿Ð°Ñ€Ð°Ð»Ð»ÐµÐ»ÑŒÐ½Ð¾Ð³Ð¾
- * иÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ð½Ð¸Ñ Ð² неÑкольких потоках;
- * - иÑпользовать минимум реÑурÑов, как CPU, так и RAM, в том чиÑле
- * Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ cache pollution и ram bandwidth.
- *
- * При Ñтом заведомо извеÑтно, что Ð´Ð»Ñ MDBX не имеет значениÑ:
- * - иÑпользуемый алфавит (Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð±Ð°Ð¹Ñ‚Ð¾Ð²);
- * - чаÑтотное раÑпределение по алфавиту;
- * - абÑолютное значение ключей или разноÑÑ‚ÑŒ между отдельными значениÑми;
- *
- * СоответÑтвенно, в общих чертах, Ñхема генерации ÑледующаÑ:
- * - вводитÑÑ Ð¿Ð»Ð¾ÑÐºÐ°Ñ Ð¾Ð´Ð½Ð¾Ð¼ÐµÑ€Ð½Ð°Ñ "координата" serial (uint64_t);
- * - Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ ÑпецифичеÑких паттернов (поÑледовательноÑтей)
- * реализуетÑÑ Ð¿Ð¾ÑредÑтвом ÑоответÑтвующих преобразований "координат", при
- * Ñтом вÑе подобные Ð¿Ñ€ÐµÐ¾Ð±Ñ€Ð°Ð·Ð¾Ð²Ð°Ð½Ð¸Ñ Ð²Ñ‹Ð¿Ð¾Ð»Ð½ÑÑŽÑ‚ÑÑ Ñ‚Ð¾Ð»ÑŒÐºÐ¾ над "координатой";
- * - Ð¸Ñ‚Ð¾Ð³Ð¾Ð²Ð°Ñ "координата" преобразуетÑÑ Ð² 8-байтное Ñуррогатное значение
- * ключа;
- * - Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹ длиной МЕÐЕЕ 8 байт Ñуррогат может уÑекатьÑÑ
- * до ненулевых байт, в том чиÑле до нулевой длины;
- * - Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹ длиной БОЛЕЕ 8 байт Ñуррогат дополнÑетÑÑ
- * нулÑми или пÑевдоÑлучайной поÑледовательноÑтью;
- *
- * Механизм генерации паттернов:
- * - реализованный механизм ÑвлÑетÑÑ ÐºÐ¾Ð¼Ð¿Ñ€Ð¾Ð¼Ð¸ÑÑом между ÑкороÑтью/проÑтотой
- * и гибкоÑтью, необходимой Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ Ð¿Ð¾ÑледовательноÑтей, которых
- * будет доÑтаточно Ð´Ð»Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¸ Ñценариев Ñ€Ð°Ð·Ð´ÐµÐ»ÐµÐ½Ð¸Ñ Ð¸ ÑлиÑÐ½Ð¸Ñ Ñтраниц
- * Ñ Ð´Ð°Ð½Ð½Ñ‹Ð¼Ð¸ внутри mdbx;
- * - пÑевдо-Ñлучайные паттерны реализуютÑÑ Ð¿Ð¾ÑредÑтвом набора инъективных
- * отображающих функций;
- * - не-пÑевдо-Ñлучайные паттерны реализуютÑÑ Ð¿Ð¾ÑредÑтвом параметризируемого
- * трех-Ñтапного преобразованиÑ:
- * 1) Ñмещение (Ñложение) по модулю;
- * 2) цикличеÑкий Ñдвиг;
- * 3) добавление абÑолютного ÑÐ¼ÐµÑ‰ÐµÐ½Ð¸Ñ (базы);
- *
- * Также Ñм. опиÑание параметров генератора ключей и значений в config.h */
-
-typedef uint64_t serial_t;
-
-enum : serial_t {
- serial_minwith = 8,
- serial_maxwith = sizeof(serial_t) * 8,
- serial_allones = ~(serial_t)0u
-};
-
-struct result {
- MDBX_val value;
- size_t limit;
- union {
- uint8_t bytes[sizeof(uint64_t)];
- uint32_t u32;
- uint64_t u64;
- };
-
- std::string as_string() const {
- return std::string((const char *)value.iov_base, value.iov_len);
- }
-};
-
-//-----------------------------------------------------------------------------
-
-struct buffer_deleter /* : public std::unary_function<void, result *> */ {
- void operator()(result *buffer) const { free(buffer); }
-};
-
-typedef std::unique_ptr<result, buffer_deleter> buffer;
-
-buffer alloc(size_t limit);
-
-class maker {
- config::keygen_params_pod mapping;
- serial_t base{0};
- serial_t salt{0};
-
- struct essentials {
- uint16_t minlen{0};
- enum { prng_fill_flag = 1 };
- uint16_t flags{0};
- uint32_t maxlen{0};
- } key_essentials, value_essentials;
-
- static void mk_begin(const serial_t serial, const essentials ¶ms,
- result &out);
- static void mk_continue(const serial_t serial, const essentials ¶ms,
- result &out);
- static void mk(const serial_t serial, const essentials ¶ms, result &out) {
- mk_begin(serial, params, out);
- mk_continue(serial, params, out);
- }
-
-public:
- void pair(serial_t serial, const buffer &key, buffer &value,
- serial_t value_age, const bool keylen_changeable);
- void setup(const config::actor_params_pod &actor, unsigned actor_id,
- unsigned thread_number);
- bool is_unordered() const;
- void seek2end(serial_t &serial) const;
-
- bool increment(serial_t &serial, int64_t delta) const;
- bool increment_key_part(serial_t &serial, int64_t delta,
- bool reset_value_part = true) const {
- if (reset_value_part) {
- serial_t value_part_bits = ((serial_t(1) << mapping.split) - 1);
- serial |= value_part_bits;
- if (delta >= 0)
- serial &= ~value_part_bits;
- }
- return increment(serial, int64_t(uint64_t(delta) << mapping.split));
- }
-};
-
-void log_pair(logging::loglevel level, const char *prefix, const buffer &key,
- buffer &value);
-
-} /* namespace keygen */
diff --git a/3rdparty/libmdbx/test/log.cc b/3rdparty/libmdbx/test/log.cc
deleted file mode 100644
index beb962031947082b64f0e6835c7a36eed8d17d74..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/log.cc
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-static void fflushall() { fflush(nullptr); }
-
-void failure(const char *fmt, ...) {
- va_list ap;
- va_start(ap, fmt);
- fflushall();
- logging::output_nocheckloglevel_ap(logging::failure, fmt, ap);
- va_end(ap);
- fflushall();
- exit(EXIT_FAILURE);
-}
-
-const char *test_strerror(int errnum) {
- static __thread char buf[1024];
- return mdbx_strerror_r(errnum, buf, sizeof(buf));
-}
-
-MDBX_NORETURN void failure_perror(const char *what, int errnum) {
- failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum);
-}
-
-//-----------------------------------------------------------------------------
-
-static void mdbx_logger(MDBX_log_level_t priority, const char *function,
- int line, const char *fmt,
- va_list args) MDBX_CXX17_NOEXCEPT {
- if (function) {
- if (priority == MDBX_LOG_FATAL)
- log_error("mdbx: fatal failure: %s, %d", function, line);
- logging::output_nocheckloglevel(
- logging::loglevel(priority),
- strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function);
- logging::feed_ap(fmt, args);
- } else
- logging::feed_ap(fmt, args);
-}
-
-namespace logging {
-
-static std::string prefix;
-static std::string suffix;
-static loglevel level;
-static FILE *last;
-
-void setlevel(loglevel priority) {
- level = priority;
- int rc = mdbx_setup_debug(MDBX_log_level_t(priority),
- MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER,
- mdbx_logger);
- log_trace("set mdbx debug-opts: 0x%02x", rc);
-}
-
-void setup(loglevel priority, const std::string &_prefix) {
- setlevel(priority);
- prefix = _prefix;
-}
-
-void setup(const std::string &_prefix) { prefix = _prefix; }
-
-const char *level2str(const loglevel alevel) {
- switch (alevel) {
- default:
- return "invalid/unknown";
- case extra:
- return "extra";
- case trace:
- return "trace";
- case debug:
- return "debug";
- case verbose:
- return "verbose";
- case notice:
- return "notice";
- case warning:
- return "warning";
- case error:
- return "error";
- case failure:
- return "failure";
- }
-}
-
-bool output(const loglevel priority, const char *format, ...) {
- if (lower(priority, level))
- return false;
-
- va_list ap;
- va_start(ap, format);
- output_nocheckloglevel_ap(priority, format, ap);
- va_end(ap);
- return true;
-}
-
-void output_nocheckloglevel_ap(const logging::loglevel priority,
- const char *format, va_list ap) {
- if (last) {
- putc('\n', last);
- fflush(last);
- if (last == stderr) {
- putc('\n', stdout);
- fflush(stdout);
- }
- last = nullptr;
- }
-
- chrono::time now = chrono::now_realtime();
- struct tm tm;
-#ifdef _MSC_VER
- int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc);
-#elif defined(_WIN32) || defined(_WIN64)
- const time_t time_proxy = now.utc;
- int rc = localtime_s(&tm, &time_proxy);
-#else
- time_t time = now.utc;
- int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno;
-#endif
- if (rc != MDBX_SUCCESS)
- failure_perror("localtime_r()", rc);
-
- last = stdout;
- fprintf(last,
- "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05lu %-10s %.4s ] %s" /* TODO */,
- tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
- tm.tm_sec, chrono::fractional2us(now.fractional), (long)osal_getpid(),
- prefix.c_str(), level2str(priority), suffix.c_str());
-
- va_list ones;
- memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */;
- if (same_or_higher(priority, error))
- va_copy(ones, ap);
- vfprintf(last, format, ap);
-
- size_t len = strlen(format);
- char end = len ? format[len - 1] : '\0';
-
- switch (end) {
- default:
- putc('\n', last);
- // fall through
- case '\n':
- fflush(last);
- last = nullptr;
- // fall through
- case ' ':
- case '_':
- case ':':
- case '|':
- case ',':
- case '\t':
- case '\b':
- case '\r':
- case '\0':
- break;
- }
-
- if (same_or_higher(priority, error)) {
- if (last != stderr) {
- fprintf(stderr, "[ %05lu %-10s %.4s ] %s", (long)osal_getpid(),
- prefix.c_str(), level2str(priority), suffix.c_str());
- vfprintf(stderr, format, ones);
- if (end == '\n')
- fflush(stderr);
- else
- last = stderr;
- }
- va_end(ones);
- }
-}
-
-bool feed_ap(const char *format, va_list ap) {
- if (!last)
- return false;
-
- if (last == stderr) {
- va_list ones;
- va_copy(ones, ap);
- vfprintf(stdout, format, ones);
- va_end(ones);
- }
- vfprintf(last, format, ap);
- size_t len = strlen(format);
- if (len && format[len - 1] == '\n') {
- fflush(last);
- if (last == stderr)
- fflush(stdout);
- last = nullptr;
- }
- return true;
-}
-
-bool feed(const char *format, ...) {
- if (!last)
- return false;
-
- va_list ap;
- va_start(ap, format);
- feed_ap(format, ap);
- va_end(ap);
- return true;
-}
-
-local_suffix::local_suffix(const char *c_str)
- : trim_pos(suffix.size()), indent(0) {
- suffix.append(c_str);
-}
-
-local_suffix::local_suffix(const std::string &str)
- : trim_pos(suffix.size()), indent(0) {
- suffix.append(str);
-}
-
-void local_suffix::push() {
- indent += 1;
- suffix.push_back('\t');
-}
-
-void local_suffix::pop() {
- assert(indent > 0);
- if (indent > 0) {
- indent -= 1;
- suffix.pop_back();
- }
-}
-
-local_suffix::~local_suffix() { suffix.erase(trim_pos); }
-
-void progress_canary(bool active) {
- static chrono::time progress_timestamp;
- chrono::time now = chrono::now_monotonic();
-
- if (now.fixedpoint - progress_timestamp.fixedpoint <
- chrono::from_ms(42).fixedpoint)
- return;
-
- if (osal_progress_push(active)) {
- progress_timestamp = now;
- return;
- }
-
- if (progress_timestamp.fixedpoint == 0) {
- putc('>', stderr);
- progress_timestamp = now;
- } else if (global::config::console_mode) {
- if (active) {
- static int last_point = -1;
- int point = (now.fixedpoint >> 29) & 3;
- if (point != last_point) {
- progress_timestamp = now;
- fprintf(stderr, "%c\b", "-\\|/"[last_point = point]);
- }
- } else if (now.fixedpoint - progress_timestamp.fixedpoint >
- chrono::from_seconds(2).fixedpoint) {
- progress_timestamp = now;
- fprintf(stderr, "%c\b", "@*"[now.utc & 1]);
- }
- } else {
- static int count;
- if (active && now.fixedpoint - progress_timestamp.fixedpoint >
- chrono::from_seconds(1).fixedpoint) {
- putc('.', stderr);
- progress_timestamp = now;
- ++count;
- } else if (now.fixedpoint - progress_timestamp.fixedpoint >
- chrono::from_seconds(5).fixedpoint) {
- putc("@*"[now.utc & 1], stderr);
- progress_timestamp = now;
- ++count;
- }
- if (count == 60) {
- count = 0;
- putc('\n', stderr);
- }
- }
- fflush(stderr);
-}
-
-} // namespace logging
-
-void log_extra(const char *msg, ...) {
- if (logging::same_or_higher(logging::extra, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::extra, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_trace(const char *msg, ...) {
- if (logging::same_or_higher(logging::trace, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::trace, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_debug(const char *msg, ...) {
- if (logging::same_or_higher(logging::debug, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::debug, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_verbose(const char *msg, ...) {
- if (logging::same_or_higher(logging::verbose, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::verbose, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_notice(const char *msg, ...) {
- if (logging::same_or_higher(logging::notice, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::notice, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_warning(const char *msg, ...) {
- if (logging::same_or_higher(logging::warning, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::warning, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_error(const char *msg, ...) {
- if (logging::same_or_higher(logging::error, logging::level)) {
- va_list ap;
- va_start(ap, msg);
- logging::output_nocheckloglevel_ap(logging::error, msg, ap);
- va_end(ap);
- } else
- logging::last = nullptr;
-}
-
-void log_trouble(const char *where, const char *what, int errnum) {
- log_error("%s: %s %s", where, what, test_strerror(errnum));
-}
-
-bool log_enabled(const logging::loglevel priority) {
- return logging::same_or_higher(priority, logging::level);
-}
-
-void log_flush(void) { fflushall(); }
diff --git a/3rdparty/libmdbx/test/log.h b/3rdparty/libmdbx/test/log.h
deleted file mode 100644
index b7dcce8eedc1dbfb486ae5baf35c276d57e679b3..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/log.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-#include "chrono.h"
-
-MDBX_NORETURN void usage(void);
-MDBX_NORETURN void MDBX_PRINTF_ARGS(1, 2) failure(const char *fmt, ...);
-MDBX_NORETURN void failure_perror(const char *what, int errnum);
-const char *test_strerror(int errnum);
-
-namespace logging {
-
-enum loglevel {
- extra = MDBX_LOG_EXTRA,
- trace = MDBX_LOG_TRACE,
- debug = MDBX_LOG_DEBUG,
- verbose = MDBX_LOG_VERBOSE,
- notice = MDBX_LOG_NOTICE,
- warning = MDBX_LOG_WARN,
- error = MDBX_LOG_ERROR,
- failure = MDBX_LOG_FATAL
-};
-
-inline bool lower(loglevel left, loglevel right) {
- static_assert(MDBX_LOG_EXTRA > MDBX_LOG_FATAL, "WTF?");
- return left > right;
-}
-
-inline bool same_or_higher(loglevel left, loglevel right) {
- return left <= right;
-}
-
-const char *level2str(const loglevel level);
-void setup(loglevel priority, const std::string &prefix);
-void setup(const std::string &prefix);
-void setlevel(loglevel priority);
-
-void output_nocheckloglevel_ap(const loglevel priority, const char *format,
- va_list ap);
-bool MDBX_PRINTF_ARGS(2, 3)
- output(const loglevel priority, const char *format, ...);
-bool feed_ap(const char *format, va_list ap);
-bool MDBX_PRINTF_ARGS(1, 2) feed(const char *format, ...);
-
-void inline MDBX_PRINTF_ARGS(2, 3)
- output_nocheckloglevel(const loglevel priority, const char *format, ...) {
- va_list ap;
- va_start(ap, format);
- output_nocheckloglevel_ap(priority, format, ap);
- va_end(ap);
-}
-
-void progress_canary(bool active);
-
-class local_suffix {
-protected:
- size_t trim_pos;
- int indent;
-
-public:
- local_suffix(const local_suffix &) = delete;
- local_suffix(const local_suffix &&) = delete;
- const local_suffix &operator=(const local_suffix &) = delete;
-
- local_suffix(const char *c_str);
- local_suffix(const std::string &str);
- void push();
- void pop();
- ~local_suffix();
-};
-
-} // namespace logging
-
-void MDBX_PRINTF_ARGS(1, 2) log_extra(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_trace(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_debug(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_verbose(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_notice(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_warning(const char *msg, ...);
-void MDBX_PRINTF_ARGS(1, 2) log_error(const char *msg, ...);
-
-void log_trouble(const char *where, const char *what, int errnum);
-void log_flush(void);
-bool log_enabled(const logging::loglevel priority);
-
-#ifdef _DEBUG
-#define TRACE(...) log_trace(__VA_ARGS__)
-#else
-#define TRACE(...) __noop(__VA_ARGS__)
-#endif
diff --git a/3rdparty/libmdbx/test/long_stochastic.sh b/3rdparty/libmdbx/test/long_stochastic.sh
deleted file mode 100755
index 04b9976e2447ee90e6e5e20767bf9ba086ec2291..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/long_stochastic.sh
+++ /dev/null
@@ -1,419 +0,0 @@
-#!/usr/bin/env bash
-if ! which make cc c++ tee >/dev/null; then
- echo "Please install the following prerequisites: make cc c++ tee banner" >&2
- exit 1
-fi
-
-LIST=basic
-FROM=1
-UPTO=9999999
-MONITOR=
-LOOPS=
-SKIP_MAKE=no
-GEOMETRY_JITTER=yes
-BANNER="$(which banner 2>/dev/null | echo echo)"
-UNAME="$(uname -s 2>/dev/null || echo Unknown)"
-DB_UPTO_MB=17408
-
-while [ -n "$1" ]
-do
- case "$1" in
- --help)
- echo "--multi Engage multi-process test scenario (default)"
- echo "--single Execute series of single-process tests (for QEMU, etc)"
- echo "--nested Execute only 'nested' testcase"
- echo "--hill Execute only 'hill' testcase"
- echo "--append Execute only 'append' testcase"
- echo "--ttl Execute only 'ttl' testcase"
- echo "--with-valgrind Run tests under Valgrind's memcheck tool"
- echo "--skip-make Don't (re)build libmdbx and test's executable"
- echo "--from NN Start iterating from the NN ops per test case"
- echo "--upto NN Don't run tests with more than NN ops per test case"
- echo "--loops NN Stop after the NN loops"
- echo "--dir PATH Specifies directory for test DB and other files (it will be cleared)"
- echo "--db-upto-mb NN Limits upper size of test DB to the NN megabytes"
- echo "--no-geometry-jitter Disable jitter for geometry upper-size"
- echo "--help Print this usage help and exit"
- exit -2
- ;;
- --multi)
- LIST=basic
- ;;
- --single)
- LIST="--nested --hill --append --ttl --copy"
- ;;
- --nested)
- LIST="--nested"
- ;;
- --hill)
- LIST="--hill"
- ;;
- --append)
- LIST="--append"
- ;;
- --ttl)
- LIST="--ttl"
- ;;
- --with-valgrind)
- echo " NOTE: Valgrind could produce some false-positive warnings"
- echo " in multi-process environment with shared memory."
- echo " For instance, when the process 'A' explicitly marks a memory"
- echo " region as 'undefined', the process 'B' fill it,"
- echo " and after this process 'A' read such region, etc."
- MONITOR="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt"
- rm -f valgrind-*.log
- ;;
- --skip-make)
- SKIP_MAKE=yes
- ;;
- --from)
- FROM=$(($2))
- if [ -z "$FROM" -o "$FROM" -lt 1 ]; then
- echo "Invalid value '$FROM' for --from option"
- exit -2
- fi
- shift
- ;;
- --upto)
- UPTO=$(($2))
- if [ -z "$UPTO" -o "$UPTO" -lt 1 ]; then
- echo "Invalid value '$UPTO' for --upto option"
- exit -2
- fi
- shift
- ;;
- --loops)
- LOOPS=$(($2))
- if [ -z "$LOOPS" -o "$LOOPS" -lt 1 -o "$LOOPS" -gt 99 ]; then
- echo "Invalid value '$LOOPS' for --loops option"
- exit -2
- fi
- shift
- ;;
- --dir)
- TESTDB_DIR="$2"
- if [ -z "$TESTDB_DIR" ]; then
- echo "Invalid value '$TESTDB_DIR' for --dir option"
- exit -2
- fi
- shift
- ;;
- --db-upto-mb)
- DB_UPTO_MB=$(($2))
- if [ -z "$DB_UPTO_MB" -o "$DB_UPTO_MB" -lt 1 -o "$DB_UPTO_MB" -gt 4194304 ]; then
- echo "Invalid value '$DB_UPTO_MB' for --db-upto-mb option"
- exit -2
- fi
- shift
- ;;
- --no-geometry-jitter)
- GEOMETRY_JITTER=no
- ;;
- *)
- echo "Unknown option '$1'"
- exit -2
- ;;
- esac
- shift
-done
-
-set -euo pipefail
-if [ -z "$MONITOR" ]; then
- if which time >/dev/null 2>/dev/null; then
- MONITOR=$(which time)
- if $MONITOR -o /dev/stdout true >/dev/null 2>/dev/null; then
- MONITOR="$MONITOR -o /dev/stdout"
- fi
- fi
- export MALLOC_CHECK_=7 MALLOC_PERTURB_=42
-fi
-
-###############################################################################
-# 1. clean data from prev runs and examine available RAM
-
-WANNA_MOUNT=0
-case ${UNAME} in
- Linux)
- MAKE=make
- if [ -z "${TESTDB_DIR:-}" ]; then
- for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]* 2>/dev/null); do
- rm -rf $old_test_dir
- done
- TESTDB_DIR="/dev/shm/mdbx-test.$$"
- fi
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
-
- if LC_ALL=C free | grep -q -i available; then
- ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s '[:blank:]' ' ' | cut -d ' ' -f 7) / 1024))
- else
- ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s '[:blank:]' ' ' | cut -d ' ' -f 4) / 1024))
- fi
- ;;
-
- FreeBSD)
- MAKE=gmake
- if [ -z "${TESTDB_DIR:-}" ]; then
- for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]* 2>/dev/null); do
- umount $old_test_dir && rm -r $old_test_dir
- done
- TESTDB_DIR="/tmp/mdbx-test.$$"
- rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR
- WANNA_MOUNT=1
- else
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
- fi
- ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024))
- ;;
-
- Darwin)
- MAKE=make
- if [ -z "${TESTDB_DIR:-}" ]; then
- for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst 2>/dev/null); do
- disk=$(mount | grep $vol | cut -d ' ' -f 1)
- echo "umount: volume $vol disk $disk"
- hdiutil unmount $vol -force
- hdiutil detach $disk
- done
- TESTDB_DIR="/Volumes/mdx$$tst"
- WANNA_MOUNT=1
- else
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
- fi
- pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024))
- freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1)
- ram_avail_mb=$((pagesize * freepages / 1024))
- echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}"
- ;;
-
- *)
- echo "FIXME: ${UNAME} not supported by this script"
- exit 2
- ;;
-esac
-
-rm -f ${TESTDB_DIR}/*
-
-###############################################################################
-# 2. estimate reasonable RAM space for test-db
-
-echo "=== ${ram_avail_mb}M RAM available"
-ram_reserve4logs_mb=1234
-if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then
- echo "=== At least ${ram_reserve4logs_mb}Mb RAM required"
- exit 3
-fi
-
-#
-# Ð’ режимах отличных от MDBX_WRITEMAP Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð´Ð¾ запиÑи в файл
-# будут накапливатьÑÑ Ð² памÑти, что может потребовать Ñвободной
-# памÑти размером Ñ Ð‘Ð”. Кроме Ñтого, в теÑÑ‚ входит Ñценарий
-# ÑÐ¾Ð·Ð´Ð°Ð½Ð¸Ñ ÐºÐ¾Ð¿Ð¸Ñ Ð‘Ð” на ходу. ПоÑтому БД не может быть больше 1/3
-# от доÑтупной памÑти. Однако, Ñледует учеÑÑ‚ÑŒ что malloc() будет
-# не Ñразу возвращать выделенную памÑÑ‚ÑŒ ÑиÑтеме, а также
-# предуÑмотреть меÑта Ð´Ð»Ñ Ð»Ð¾Ð³Ð¾Ð².
-#
-# In non-MDBX_WRITEMAP modes, updates (dirty pages) will
-# accumulate in memory before writing to the disk, which may
-# require a free memory up to the size of a whole database. In
-# addition, the test includes a script create a copy of the
-# database on the go. Therefore, the database cannot be more 1/3
-# of available memory. Moreover, should be taken into account
-# that malloc() will not return the allocated memory to the
-# system immediately, as well some space is required for logs.
-#
-db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4))
-if [ $db_size_mb -gt $DB_UPTO_MB ]; then
- db_size_mb=$DB_UPTO_MB
-fi
-echo "=== use ${db_size_mb}M for DB"
-
-###############################################################################
-# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required
-case ${UNAME} in
- Linux)
- ulimit -c unlimited
- if [ "$(cat /proc/sys/kernel/core_pattern)" != "core.%p" ]; then
- echo "core.%p > /proc/sys/kernel/core_pattern" >&2
- echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern || true
- fi
- ;;
-
- FreeBSD)
- if [[ WANNA_MOUNT ]]; then
- mount -t tmpfs tmpfs $TESTDB_DIR
- fi
- ;;
-
- Darwin)
- if [[ WANNA_MOUNT ]]; then
- ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb))
- number_of_sectors=$((ramdisk_size_mb * 2048))
- ramdev=$(hdiutil attach -nomount ram://${number_of_sectors})
- diskutil erasevolume ExFAT "mdx$$tst" ${ramdev}
- fi
- ;;
-
- *)
- echo "FIXME: ${UNAME} not supported by this script"
- exit 2
- ;;
-esac
-
-###############################################################################
-# 4. build the test executables
-
-if [ "$SKIP_MAKE" != "yes" ]; then
- ${MAKE} -j$(which nproc >/dev/null 2>/dev/null && nproc || echo 2) build-test
-fi
-
-###############################################################################
-# 5. run stochastic iterations
-
-if which lz4 >/dev/null; then
- function logger {
- lz4 > ${TESTDB_DIR}/long.log.lz4
- }
-elif which gzip >/dev/null; then
- function logger {
- gzip > ${TESTDB_DIR}/long.log.gz
- }
-else
- function logger {
- cat > ${TESTDB_DIR}/long.log
- }
-fi
-
-syncmodes=("" ,+nosync-safe ,+nosync-utterly)
-options=(writemap coalesce lifo notls perturb)
-
-function join { local IFS="$1"; shift; echo "$*"; }
-
-function bits2options {
- local bits=$1
- local i
- local list=()
- for ((i = 0; i < ${#options[@]}; ++i)); do
- list[$i]=$( (( (bits & (1 << i)) != 0 )) && echo -n '+' || echo -n '-'; echo ${options[$i]})
- done
- join , ${list[@]}
-}
-
-function failed {
- echo "FAILED" >&2
- exit 1
-}
-
-function check_deep {
- if [ "$case" = "basic" -o "$case" = "--hill" ]; then
- tee >(logger) | grep -e reach -e achieve
- else
- logger
- fi
-}
-
-function probe {
- echo "----------------------------------------------- $(date)"
- echo "${caption}"
- rm -f ${TESTDB_DIR}/* || failed
- for case in $LIST
- do
- echo "Run ./mdbx_test ${speculum} --random-writemap=no --ignore-dbfull --repeat=11 --pathname=${TESTDB_DIR}/long.db --cleanup-after=no --geometry-jitter=${GEOMETRY_JITTER} $@ $case"
- ${MONITOR} ./mdbx_test ${speculum} --random-writemap=no --ignore-dbfull --repeat=11 --pathname=${TESTDB_DIR}/long.db --cleanup-after=no --geometry-jitter=${GEOMETRY_JITTER} "$@" $case | check_deep \
- && ${MONITOR} ./mdbx_chk ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \
- && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${MONITOR} ./mdbx_chk ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \
- || failed
- done
-}
-
-#------------------------------------------------------------------------------
-
-count=0
-loop=0
-cases='?'
-for nops in 10 33 100 333 1000 3333 10000 33333 100000 333333 1000000 3333333 10000000 33333333 100000000 333333333 1000000000; do
- if [ $nops -lt $FROM ]; then continue; fi
- if [ $nops -gt $UPTO ]; then echo "The '--upto $UPTO' limit reached"; break; fi
- if [ -n "$LOOPS" ] && [ $loop -ge "$LOOPS" ]; then echo "The '--loops $LOOPS' limit reached"; break; fi
- echo "======================================================================="
- wbatch=$((nops / 7 + 1))
- speculum=$([ $nops -le 1000 ] && echo '--speculum' || true)
- while true; do
- echo "======================================================================="
- ${BANNER} "$nops / $wbatch"
- subcase=0
- for ((bits=2**${#options[@]}; --bits >= 0; )); do
- seed=$(($(date +%s) + RANDOM))
-
- split=30
- caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=24
- caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=16
- caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=4
- caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=min --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- done # options
- loop=$((loop + 1))
- if [ -n "$LOOPS" ] && [ $loop -ge "$LOOPS" ]; then break; fi
- cases="${subcase}"
- wbatch=$(((wbatch > 7) ? wbatch / 7 : 1))
- if [ $wbatch -eq 1 -o $((nops / wbatch)) -gt 1000 ]; then break; fi
- done # batch (write-ops per txn)
-done # n-ops
-
-echo "=== ALL DONE ====================== $(date)"
diff --git a/3rdparty/libmdbx/test/main.cc b/3rdparty/libmdbx/test/main.cc
deleted file mode 100644
index b4b8022b0041bdfaea924bef1c46827ce01e5562..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/main.cc
+++ /dev/null
@@ -1,742 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-#if !(defined(_WIN32) || defined(_WIN64))
-#include <sys/resource.h>
-#include <sys/time.h>
-#endif /* !Windows */
-
-MDBX_NORETURN void usage(void) {
- puts(
- "usage:\n"
- " --help or -h Show this text\n"
- "Common parameters:\n"
- " --loglevel=[0-7]|[extra..fatal]"
- " --pathname=... Path and/or name of database files\n"
- " --repeat=N Set repeat counter\n"
- " --threads=N Number of thread (unsupported for now)\n"
- " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n"
- " --failfast[=YES/no] Lill all actors on first failure/error\n"
- " --max-readers=N See mdbx_env_set_maxreaders() description\n"
- " --max-tables=N Se mdbx_env_set_maxdbs() description\n"
- " --dump-config[=YES/no] Dump entire test config before run\n"
- " --progress[=YES/no] Enable/disable progress `canary`\n"
- " --console[=yes/no] Enable/disable console-like output\n"
- " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n"
- " --cleanup-after[=YES/no] Cleanup/remove database after completion\n"
- "Database size control:\n"
- " --pagesize=... Database page size: min, max, 256..65536\n"
- " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n"
- " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n"
- " --size Initial size in Kb/Mb/Gb/Tb\n"
- " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n"
- " --growth-step Grow step in Kb/Mb/Gb/Tb\n"
- "Predefined complex scenarios/cases:\n"
- " --case=... Only `basic` scenario implemented for now\n"
- " basic == Simultaneous multi-process execution\n"
- " of test-actors: nested,hill,ttl,copy,append,jitter,try\n"
- "Test actors:\n"
- " --hill Fill-up and empty-down\n"
- " by CRUD-operation quads\n"
- " --ttl Stochastic time-to-live simulation\n"
- " --nested Nested transactionы\n"
- " with stochastic-size bellows\n"
- " --jitter Jitter/delays simulation\n"
- " --try Try write-transaction, no more\n"
- " --copy Online copy/backup\n"
- " --append Append-mode insertions\n"
- " --dead.reader Dead-reader simulator\n"
- " --dead.writer Dead-writer simulator\n"
- "Actor options:\n"
- " --batch.read=N Read-operations batch size\n"
- " --batch.write=N Write-operations batch size\n"
- " --delay=N | --no-delay (no)Delay test-actor before start\n"
- " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n"
- " completes # ops before start\n"
- " --duration=N[s|m|h|d] Define running duration\n"
- " --nops=N[K|M|G|T] Define number of operations/steps\n"
- " --inject-writefault[=yes|NO] TBD (see the source code)\n"
- " --drop[=yes|NO] Drop key-value space/table on "
- "completion\n"
- " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n"
- " --speculum[=yes|NO] Use internal `speculum` to check "
- "dataset\n"
- " --geometry-jitter[=YES|no] Use jitter for geometry upper-limit\n"
- "Keys and Value:\n"
- " --keylen.min=N Minimal keys length\n"
- " --keylen.max=N Miximal keys length\n"
- " --keylen=N Set both min/max for keys length\n"
- " --datalen.min=N Minimal data length\n"
- " --datalen.max=N Miximal data length\n"
- " --datalen=N Set both min/max for data length\n"
- " --keygen.width=N TBD (see the source code)\n"
- " --keygen.mesh=N TBD (see the source code)\n"
- " --keygen.seed=N TBD (see the source code)\n"
- " --keygen.zerofill=yes|NO TBD (see the source code)\n"
- " --keygen.split=N TBD (see the source code)\n"
- " --keygen.rotate=N TBD (see the source code)\n"
- " --keygen.offset=N TBD (see the source code)\n"
- " --keygen.case=random Generator case (only `random` for now)\n"
- "Database operation mode:\n"
- " --mode={[+-]FLAG}[,[+-]FLAG]...\n"
- " nosubdir == MDBX_NOSUBDIR\n"
- " rdonly == MDBX_RDONLY\n"
- " exclusive == MDBX_EXCLUSIVE\n"
- " accede == MDBX_ACCEDE\n"
- " nometasync == MDBX_NOMETASYNC\n"
- " lifo == MDBX_LIFORECLAIM\n"
- " coalesce == MDBX_COALESCE\n"
- " nosync-safe == MDBX_SAFE_NOSYNC\n"
- " writemap == MDBX_WRITEMAP\n"
- " nosync-utterly == MDBX_UTTERLY_NOSYNC\n"
- " perturb == MDBX_PAGEPERTURB\n"
- " notls == MDBX_NOTLS\n"
- " nordahead == MDBX_NORDAHEAD\n"
- " nomeminit == MDBX_NOMEMINIT\n"
- " --random-writemap[=YES|no] Toggle MDBX_WRITEMAP randomly\n"
- "Key-value space/table options:\n"
- " --table={[+-]FLAG}[,[+-]FLAG]...\n"
- " key.reverse == MDBX_REVERSEKEY\n"
- " key.integer == MDBX_INTEGERKEY\n"
- " data.dups == MDBX_DUPSORT\n"
- " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n"
- " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n"
- " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n");
- exit(EXIT_FAILURE);
-}
-
-//-----------------------------------------------------------------------------
-
-void actor_params::set_defaults(const std::string &tmpdir) {
- pathname_log = "";
- loglevel =
-#if defined(NDEBUG) || defined(_WIN32) || defined(_WIN64)
- logging::verbose;
-#else
- logging::trace;
-#endif
-
- pathname_db = tmpdir + "mdbx-test.db";
- mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_SAFE_NOSYNC |
- MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM | MDBX_ACCEDE;
- table_flags = MDBX_DUPSORT;
-
- size_lower = -1;
- size_now = intptr_t((table_flags & MDBX_DUPSORT) ? 256 : 1024) << 20;
- size_now = std::min(size_now, mdbx_limits_dbsize_max(-1));
- size_upper = -1;
- shrink_threshold = -1;
- growth_step = -1;
- pagesize = -1;
-
- keygen.seed = 1;
- keygen.zero_fill = false;
- keygen.keycase = kc_random;
- keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64;
- keygen.mesh = keygen.width;
- keygen.split = keygen.width / 2;
- keygen.rotate = 3;
- keygen.offset = 41;
-
- test_duration = 0;
- test_nops = 1000;
- nrepeat = 1;
- nthreads = 1;
-
- keylen_min = mdbx_keylen_min();
- keylen_max = mdbx_keylen_max();
- datalen_min = mdbx_datalen_min();
- datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42);
-
- batch_read = 42;
- batch_write = 42;
-
- delaystart = 0;
- waitfor_nops = 0;
- inject_writefaultn = 0;
-
- drop_table = false;
- ignore_dbfull = false;
- speculum = false;
- random_writemap = true;
-
- max_readers = 42;
- max_tables = 42;
-
- global::config::timeout_duration_seconds = 0 /* infinite */;
- global::config::dump_config = true;
- global::config::cleanup_before = true;
- global::config::cleanup_after = true;
- global::config::failfast = true;
- global::config::progress_indicator = true;
- global::config::console_mode = osal_istty(STDERR_FILENO);
- global::config::geometry_jitter = true;
-}
-
-namespace global {
-
-std::vector<actor_config> actors;
-std::unordered_map<unsigned, actor_config *> events;
-std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
-std::set<std::string> databases;
-unsigned nactors;
-chrono::time start_monotonic;
-chrono::time deadline_monotonic;
-bool singlemode;
-
-namespace config {
-unsigned timeout_duration_seconds;
-bool dump_config;
-bool cleanup_before;
-bool cleanup_after;
-bool failfast;
-bool progress_indicator;
-bool console_mode;
-bool geometry_jitter;
-} /* namespace config */
-
-} /* namespace global */
-
-//-----------------------------------------------------------------------------
-
-const char global::thunk_param_prefix[] = "--execute=";
-
-std::string thunk_param(const actor_config &config) {
- return config.serialize(global::thunk_param_prefix);
-}
-
-void cleanup() {
- log_trace(">> cleanup");
- for (const auto &db_path : global::databases) {
- int err = mdbx_env_delete(db_path.c_str(), MDBX_ENV_JUST_DELETE);
- if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE)
- failure_perror(db_path.c_str(), err);
- }
- log_trace("<< cleanup");
-}
-
-static void fixup4qemu(actor_params ¶ms) {
-#ifdef MDBX_SAFE4QEMU
-#if MDBX_WORDBITS == 32
- intptr_t safe4qemu_limit = size_t(512) << 20 /* 512 megabytes */;
-#if defined(__SANITIZE_ADDRESS__)
- safe4qemu_limit >>= 1;
-#else
- if (RUNNING_ON_VALGRIND)
- safe4qemu_limit >>= 1;
-#endif /* __SANITIZE_ADDRESS__ */
-
- if (params.size_lower > safe4qemu_limit ||
- params.size_now > safe4qemu_limit ||
- params.size_upper > safe4qemu_limit) {
- params.size_upper = std::min(params.size_upper, safe4qemu_limit);
- params.size_now = std::min(params.size_now, params.size_upper);
- params.size_lower = std::min(params.size_lower, params.size_now);
- log_notice("workaround: for conformance 32-bit build with "
- "QEMU/ASAN/Valgrind database size reduced to %zu megabytes",
- safe4qemu_limit >> 20);
- }
-#endif /* MDBX_WORDBITS == 32 */
-
-#if defined(__alpha__) || defined(__alpha) || defined(__sparc__) || \
- defined(__sparc) || defined(__sparc64__) || defined(__sparc64)
- if (params.size_lower != params.size_upper) {
- log_notice(
- "workaround: for conformance Alpha/Sparc build with QEMU/ASAN/Valgrind "
- "enforce fixed database size %zu megabytes",
- params.size_upper >> 20);
- params.size_lower = params.size_now = params.size_upper;
- }
-#endif /* Alpha || Sparc */
-#endif /* MDBX_SAFE4QEMU */
- (void)params;
-}
-
-int main(int argc, char *const argv[]) {
-
-#ifdef _DEBUG
- log_trace("#argc = %d", argc);
- for (int i = 0; i < argc; ++i)
- log_trace("#argv[%d] = %s", i, argv[i]);
-#endif /* _DEBUG */
-
- if (argc < 2)
- failure("No parameters given. Try --help\n");
-
- if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix,
- strlen(global::thunk_param_prefix)) == 0)
- return test_execute(
- actor_config(argv[1] + strlen(global::thunk_param_prefix)))
- ? EXIT_SUCCESS
- : EXIT_FAILURE;
-
- if (argc == 2 &&
- (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0))
- usage();
-
- actor_params params;
- params.set_defaults(osal_tempdir());
- global::config::dump_config = true;
- logging::setup((logging::loglevel)params.loglevel, "main");
- unsigned last_space_id = 0;
-
- for (int narg = 1; narg < argc; ++narg) {
- if (config::parse_option(argc, argv, narg, "dump-config",
- global::config::dump_config))
- continue;
- if (config::parse_option(argc, argv, narg, "cleanup-before",
- global::config::cleanup_before))
- continue;
- if (config::parse_option(argc, argv, narg, "cleanup-after",
- global::config::cleanup_after))
- continue;
- if (config::parse_option(argc, argv, narg, "failfast",
- global::config::failfast))
- continue;
- if (config::parse_option(argc, argv, narg, "progress",
- global::config::progress_indicator))
- continue;
- if (config::parse_option(argc, argv, narg, "console",
- global::config::console_mode))
- continue;
- if (config::parse_option(argc, argv, narg, "geometry-jitter",
- global::config::geometry_jitter))
- continue;
- if (config::parse_option(argc, argv, narg, "timeout",
- global::config::timeout_duration_seconds,
- config::duration, 1))
- continue;
-
- logging::loglevel loglevel;
- if (config::parse_option(argc, argv, narg, "loglevel", loglevel)) {
- logging::setup(loglevel, "main");
- params.loglevel = static_cast<uint8_t>(loglevel);
- continue;
- }
-
- const char *value = nullptr;
- if (config::parse_option(argc, argv, narg, "case", &value)) {
- fixup4qemu(params);
- testcase_setup(value, params, last_space_id);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db))
- continue;
- if (config::parse_option(argc, argv, narg, "mode", params.mode_flags,
- config::mode_bits))
- continue;
- if (config::parse_option(argc, argv, narg, "random-writemap",
- params.random_writemap))
- continue;
- if (config::parse_option(argc, argv, narg, "table", params.table_flags,
- config::table_bits)) {
- if ((params.table_flags & MDBX_DUPFIXED) == 0)
- params.table_flags &= ~MDBX_INTEGERDUP;
- if ((params.table_flags & MDBX_DUPSORT) == 0)
- params.table_flags &=
- ~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP);
- const unsigned keylen_max = params.mdbx_keylen_max();
- if (params.keylen_min > keylen_max)
- params.keylen_min = keylen_max;
- if (params.keylen_max > keylen_max)
- params.keylen_max = keylen_max;
- const unsigned keylen_min = params.mdbx_keylen_min();
- if (params.keylen_min < keylen_min)
- params.keylen_min = keylen_min;
- if (params.keylen_max < keylen_min)
- params.keylen_max = keylen_min;
-
- const unsigned datalen_max = params.mdbx_datalen_max();
- if (params.datalen_min > datalen_max)
- params.datalen_min = datalen_max;
- if (params.datalen_max > datalen_max)
- params.datalen_max = datalen_max;
- const unsigned datalen_min = params.mdbx_datalen_min();
- if (params.datalen_min < datalen_min)
- params.datalen_min = datalen_min;
- if (params.datalen_max < datalen_min)
- params.datalen_max = datalen_min;
- continue;
- }
-
- if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize,
- int(mdbx_limits_pgsize_min()),
- int(mdbx_limits_pgsize_max()))) {
- const unsigned keylen_max = params.mdbx_keylen_max();
- if (params.keylen_min > keylen_max)
- params.keylen_min = keylen_max;
- if (params.keylen_max > keylen_max)
- params.keylen_max = keylen_max;
- const unsigned datalen_max = params.mdbx_datalen_max();
- if (params.datalen_min > datalen_max)
- params.datalen_min = datalen_max;
- if (params.datalen_max > datalen_max)
- params.datalen_max = datalen_max;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat,
- config::no_scale))
- continue;
- if (config::parse_option(argc, argv, narg, "threads", params.nthreads,
- config::no_scale, 1, 64))
- continue;
-
- if (config::parse_option_intptr(argc, argv, narg, "size-lower",
- params.size_lower,
- mdbx_limits_dbsize_min(params.pagesize),
- mdbx_limits_dbsize_max(params.pagesize)))
- continue;
- int64_t i64 = params.size_upper;
- if (config::parse_option(argc, argv, narg, "size-upper-upto", i64,
- int64_t(mdbx_limits_dbsize_min(params.pagesize)),
- INT64_MAX, -1)) {
- if (i64 > mdbx_limits_dbsize_max(params.pagesize))
- i64 = mdbx_limits_dbsize_max(params.pagesize);
- params.size_upper = intptr_t(i64);
- continue;
- }
- if (config::parse_option_intptr(argc, argv, narg, "size-upper",
- params.size_upper,
- mdbx_limits_dbsize_min(params.pagesize),
- mdbx_limits_dbsize_max(params.pagesize)))
- continue;
- if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now,
- mdbx_limits_dbsize_min(params.pagesize),
- mdbx_limits_dbsize_max(params.pagesize)))
- continue;
- if (config::parse_option(
- argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0,
- (int)std::min((intptr_t)INT_MAX,
- mdbx_limits_dbsize_max(params.pagesize) -
- mdbx_limits_dbsize_min(params.pagesize))))
- continue;
- if (config::parse_option(
- argc, argv, narg, "growth-step", params.growth_step, 0,
- (int)std::min((intptr_t)INT_MAX,
- mdbx_limits_dbsize_max(params.pagesize) -
- mdbx_limits_dbsize_min(params.pagesize))))
- continue;
-
- if (config::parse_option(argc, argv, narg, "keygen.width",
- params.keygen.width, 8, 64))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.mesh",
- params.keygen.mesh, 0, 64))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.seed",
- params.keygen.seed, config::no_scale))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.zerofill",
- params.keygen.zero_fill))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.split",
- params.keygen.split, 0, 63))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.rotate",
- params.keygen.rotate, 0, 63))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.offset",
- params.keygen.offset, config::binary))
- continue;
- if (config::parse_option(argc, argv, narg, "keygen.case", &value)) {
- keycase_setup(value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min,
- config::no_scale, params.mdbx_keylen_min(),
- params.mdbx_keylen_max())) {
- if ((params.table_flags & MDBX_INTEGERKEY) ||
- params.keylen_max < params.keylen_min)
- params.keylen_max = params.keylen_min;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max,
- config::no_scale, params.mdbx_keylen_min(),
- params.mdbx_keylen_max())) {
- if ((params.table_flags & MDBX_INTEGERKEY) ||
- params.keylen_min > params.keylen_max)
- params.keylen_min = params.keylen_max;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "keylen", params.keylen_min,
- config::no_scale, params.mdbx_keylen_min(),
- params.mdbx_keylen_max())) {
- params.keylen_max = params.keylen_min;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "datalen.min",
- params.datalen_min, config::no_scale,
- params.mdbx_datalen_min(),
- params.mdbx_datalen_max())) {
- if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) ||
- params.datalen_max < params.datalen_min)
- params.datalen_max = params.datalen_min;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "datalen.max",
- params.datalen_max, config::no_scale,
- params.mdbx_datalen_min(),
- params.mdbx_datalen_max())) {
- if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) ||
- params.datalen_min > params.datalen_max)
- params.datalen_min = params.datalen_max;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "datalen", params.datalen_min,
- config::no_scale, params.mdbx_datalen_min(),
- params.mdbx_datalen_max())) {
- params.datalen_max = params.datalen_min;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read,
- config::no_scale, 1))
- continue;
- if (config::parse_option(argc, argv, narg, "batch.write",
- params.batch_write, config::no_scale, 1))
- continue;
- if (config::parse_option(argc, argv, narg, "delay", params.delaystart,
- config::duration))
- continue;
- if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops,
- config::decimal))
- continue;
- if (config::parse_option(argc, argv, narg, "inject-writefault",
- params.inject_writefaultn, config::decimal))
- continue;
- if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
- continue;
- if (config::parse_option(argc, argv, narg, "ignore-dbfull",
- params.ignore_dbfull))
- continue;
- if (config::parse_option(argc, argv, narg, "speculum", params.speculum))
- continue;
- if (config::parse_option(argc, argv, narg, "max-readers",
- params.max_readers, config::no_scale, 1, 255))
- continue;
- if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables,
- config::no_scale, 1, INT16_MAX))
- continue;
-
- if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) {
- params.delaystart = 0;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "no-wait4ops", nullptr)) {
- params.waitfor_nops = 0;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "duration", params.test_duration,
- config::duration, 1)) {
- params.test_nops = 0;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "nops", params.test_nops,
- config::decimal, 1)) {
- params.test_duration = 0;
- continue;
- }
- if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_hill, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "jitter", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_jitter, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_deadread, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_deadwrite, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "try", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_try, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "copy", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_copy, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "append", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_append, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "ttl", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_ttl, value, params);
- continue;
- }
- if (config::parse_option(argc, argv, narg, "nested", nullptr)) {
- fixup4qemu(params);
- configure_actor(last_space_id, ac_nested, value, params);
- continue;
- }
-
- if (*argv[narg] != '-') {
- fixup4qemu(params);
- testcase_setup(argv[narg], params, last_space_id);
- } else
- failure("Unknown option '%s'. Try --help\n", argv[narg]);
- }
-
- if (global::config::dump_config)
- config::dump();
-
- //--------------------------------------------------------------------------
-
- if (global::actors.empty()) {
- log_notice("no testcase(s) configured, exiting");
- return EXIT_SUCCESS;
- }
-
- bool failed = false;
- global::start_monotonic = chrono::now_monotonic();
- global::deadline_monotonic.fixedpoint =
- (global::config::timeout_duration_seconds == 0)
- ? chrono::infinite().fixedpoint
- : global::start_monotonic.fixedpoint +
- chrono::from_seconds(global::config::timeout_duration_seconds)
- .fixedpoint;
-
- if (global::config::cleanup_before)
- cleanup();
-
- if (global::actors.size() == 1) {
- logging::setup("main");
- global::singlemode = true;
- if (!test_execute(global::actors.front()))
- failed = true;
- } else {
- logging::setup("overlord");
-
- log_trace("=== preparing...");
- log_trace(">> osal_setup");
- osal_setup(global::actors);
- log_trace("<< osal_setup");
-
- for (auto &a : global::actors) {
- mdbx_pid_t pid;
- log_trace(">> actor_start");
- int rc = osal_actor_start(a, pid);
- log_trace("<< actor_start");
- if (rc) {
- log_trace(">> killall_actors: (%s)", "start failed");
- osal_killall_actors();
- log_trace("<< killall_actors");
- failure("Failed to start actor #%u (%s)\n", a.actor_id,
- test_strerror(rc));
- }
- global::pid2actor[pid] = &a;
- }
-
- log_trace("=== ready to start...");
- atexit(osal_killall_actors);
- log_trace(">> wait4barrier");
- osal_wait4barrier();
- log_trace("<< wait4barrier");
-
- size_t left = global::actors.size();
- log_trace("=== polling...");
- while (left > 0) {
- unsigned timeout_seconds_left = INT_MAX;
- chrono::time now_monotonic = chrono::now_monotonic();
- if (now_monotonic.fixedpoint >= global::deadline_monotonic.fixedpoint)
- timeout_seconds_left = 0;
- else {
- chrono::time left_monotonic;
- left_monotonic.fixedpoint =
- global::deadline_monotonic.fixedpoint - now_monotonic.fixedpoint;
- timeout_seconds_left = left_monotonic.seconds();
- }
-
- mdbx_pid_t pid;
- int rc = osal_actor_poll(pid, timeout_seconds_left);
- if (rc)
- failure("Poll error: %s (%d)\n", test_strerror(rc), rc);
-
- if (pid) {
- actor_status status = osal_actor_info(pid);
- actor_config *actor = global::pid2actor.at(pid);
- if (!actor)
- continue;
-
- log_verbose("actor #%u, id %d, pid %ld: %s\n", actor->actor_id,
- actor->space_id, (long)pid, status2str(status));
- if (status > as_running) {
- left -= 1;
- if (status != as_successful) {
- if (global::config::failfast && !failed) {
- log_trace(">> killall_actors: (%s)", "failfast");
- osal_killall_actors();
- log_trace("<< killall_actors");
- }
- failed = true;
- }
- }
- } else {
- if (timeout_seconds_left == 0)
- failure("Timeout\n");
- }
- }
- log_trace("=== done...");
- }
-
- log_notice("RESULT: %s\n", failed ? "Failed" : "Successful");
- if (global::config::cleanup_after) {
- if (failed)
- log_verbose("skip cleanup");
- else
- cleanup();
- }
-
-#if !(defined(_WIN32) || defined(_WIN64))
- struct rusage spent;
- if (!getrusage(global::singlemode ? RUSAGE_SELF : RUSAGE_CHILDREN, &spent)) {
- log_notice("%6s: user %f, system %f", "CPU",
- spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6,
- spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6);
-#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \
- defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \
- defined(__bsdi__) || defined(__DragonFly__) || defined(__APPLE__) || \
- defined(__MACH__) || defined(__sun)
- log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock,
- spent.ru_oublock);
- if (spent.ru_maxrss > 0)
- log_notice("%6s: %ld Kb", "RAM",
- spent.ru_maxrss
-#if defined(__sun)
- * getpagesize() / 1024u
-#elif defined(__APPLE__)
- / 1024u
-#endif
- );
- log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging",
- spent.ru_minflt, spent.ru_majflt, spent.ru_nswap);
-#endif /* Linux */
- }
-#endif /* !Windows */
-
- return failed ? EXIT_FAILURE : EXIT_SUCCESS;
-}
diff --git a/3rdparty/libmdbx/test/nested.cc b/3rdparty/libmdbx/test/nested.cc
deleted file mode 100644
index 098eada09d0dbde64901de119f9260ac6695ca50..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/nested.cc
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-#include <cmath>
-
-/* LY: теÑÑ‚ "ÑмулÑцией time-to-live" Ñ Ð²Ð»Ð¾Ð¶ÐµÐ½Ð½Ñ‹Ð¼Ð¸ транзакциÑми:
- * - организуетÑÑ "ÑкользÑщее окно", которое каждую транзакцию ÑдвигаетÑÑ
- * вперед вдоль чиÑловой оÑи.
- * - по переднему краю "ÑкользÑщего окна" запиÑи добавлÑÑŽÑ‚ÑÑ Ð² таблицу,
- * а по заднему удалÑÑŽÑ‚ÑÑ.
- * - количеÑтво добавлÑемых/удалÑемых запиÑей пÑевдоÑлучайно завиÑит
- * от номера транзакции, но Ñ ÑкÑпоненциальным раÑпределением.
- * - размер "ÑкользÑщего окна" также пÑевдоÑлучайно завиÑит от номера
- * транзакции Ñ "отрицательным" ÑкÑпоненциальным раÑпределением
- * MAX_WIDTH - exp(rnd(N)), при уменьшении окна ÑдвигаетÑÑ Ð·Ð°Ð´Ð½Ð¸Ð¹
- * край и удалÑÑŽÑ‚ÑÑ Ð·Ð°Ð¿Ð¸Ñи позади него.
- * - групповое добавление данных в начало окна и групповое удаление в конце,
- * преимущеÑтвенно выполнÑÑŽÑ‚ÑÑ Ð²Ð¾ вложенных транзакциÑÑ….
- * - Ð¼ÐµÐ½ÑŒÑˆÐ°Ñ Ñ‡Ð°ÑÑ‚ÑŒ запуÑкаемых вложенных транзакций отменÑетÑÑ, Ñ Ð¿Ð¾Ñледующим
- * продолжением итераций Ñ ÑоÑтоÑÐ½Ð¸Ñ Ð¿Ñ€ÐµÐ´Ñ‹Ð´ÑƒÑ‰Ð¸ÐµÐ³Ð¾ коммита.
- *
- * Таким образом имитируетÑÑ Ð¿Ð¾Ð²ÐµÐ´ÐµÐ½Ð¸Ðµ таблицы Ñ TTL: запиÑи ÑтохаÑтичеÑки
- * добавлÑÑŽÑ‚ÑÑ Ð¸ удалÑÑŽÑ‚ÑÑ, и изредка проиÑходÑÑ‚ маÑÑивные удалениÑ. */
-
-class testcase_nested : public testcase_ttl {
- using inherited = testcase_ttl;
- using FIFO = std::deque<std::pair<uint64_t, unsigned>>;
-
- uint64_t serial{0};
- unsigned clear_wholetable_passed{0};
- unsigned clear_stepbystep_passed{0};
- unsigned dbfull_passed{0};
- bool keyspace_overflow{false};
- FIFO fifo;
- std::stack<std::tuple<scoped_txn_guard, uint64_t, FIFO, SET>> stack;
-
- bool trim_tail(unsigned window_width);
- bool grow_head(unsigned head_count);
- bool pop_txn(bool abort);
- bool pop_txn() {
- return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3()
- : flipcoin_x2());
- }
- void push_txn();
- bool stochastic_breakable_restart_with_nested(bool force_restart = false);
-
-public:
- testcase_nested(const actor_config &config, const mdbx_pid_t pid)
- : inherited(config, pid) {}
- bool setup() override;
- bool run() override;
- bool teardown() override;
-};
-REGISTER_TESTCASE(nested);
-
-bool testcase_nested::setup() {
- if (!inherited::setup())
- return false;
- int err = db_open__begin__table_create_open_clean(dbi);
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("nested: bailout-prepare due '%s'", mdbx_strerror(err));
- return false;
- }
-
- keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
- key = keygen::alloc(config.params.keylen_max);
- data = keygen::alloc(config.params.datalen_max);
- serial = 0;
- fifo.clear();
- speculum.clear();
- assert(stack.empty());
- stack.emplace(nullptr, serial, fifo, speculum);
- return true;
-}
-
-bool testcase_nested::teardown() {
- while (!stack.empty())
- pop_txn(true);
-
- bool ok = true;
- if (dbi) {
- if (config.params.drop_table && !mode_readonly()) {
- txn_begin(false);
- db_table_drop(dbi);
- int err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS) &&
- (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)) {
- log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err));
- ok = false;
- }
- } else {
- if (txn_guard)
- txn_end(false);
- db_table_close(dbi);
- }
- dbi = 0;
- }
- return inherited::teardown() && ok;
-}
-
-void testcase_nested::push_txn() {
- MDBX_txn *nested_txn;
- MDBX_txn_flags_t flags = MDBX_txn_flags_t(
- prng32() & uint32_t(MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC));
- int err = mdbx_txn_begin(db_guard.get(), txn_guard.get(), flags, &nested_txn);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_txn_begin(nested)", err);
- /* CLANG/LLVM C++ library could stupidly copy std::set<> item-by-item,
- * i.e. with insertion(s) & comparison(s), which will cause null dereference
- * during call mdbx_cmp() with zero txn. So it is the workaround for this:
- * - explicitly set txn_guard with the new nested txn;
- * - explicitly copy the `speculum` (an instance of std::set<>). */
- scoped_txn_guard nested_txn_guard(nested_txn);
- txn_guard.swap(nested_txn_guard);
- SET speculum_snapshot(speculum);
- stack.emplace(std::move(nested_txn_guard), serial, fifo,
- std::move(speculum_snapshot));
- log_verbose("begin level#%zu txn #%" PRIu64 ", flags 0x%x, serial %" PRIu64,
- stack.size(), mdbx_txn_id(nested_txn), flags, serial);
- if (!dbi && stack.size() == 1)
- dbi = db_table_open(true);
-}
-
-bool testcase_nested::pop_txn(bool abort) {
- assert(txn_guard && !stack.empty());
- bool should_continue = true;
- MDBX_txn *txn = txn_guard.release();
- bool committed = false;
- if (abort) {
- log_verbose(
- "abort level#%zu txn #%" PRIu64 ", undo serial %" PRIu64 " <- %" PRIu64,
- stack.size(), mdbx_txn_id(txn), serial, std::get<1>(stack.top()));
- if (dbi > 0 && stack.size() == 1 &&
- is_handle_created_in_current_txn(dbi, txn))
- dbi = 0;
- int err = mdbx_txn_abort(txn);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_txn_abort()", err);
- } else {
- log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64,
- stack.size(), serial, std::get<1>(stack.top()));
- int err = mdbx_txn_commit(txn);
- if (likely(err == MDBX_SUCCESS))
- committed = true;
- else {
- should_continue = false;
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- err = mdbx_txn_abort(txn);
- if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH &&
- err != MDBX_BAD_TXN))
- failure_perror("mdbx_txn_abort()", err);
- } else
- failure_perror("mdbx_txn_commit()", err);
- }
- }
-
- std::swap(txn_guard, std::get<0>(stack.top()));
- if (!committed) {
- serial = std::get<1>(stack.top());
- std::swap(fifo, std::get<2>(stack.top()));
- std::swap(speculum, std::get<3>(stack.top()));
- }
- stack.pop();
- return should_continue;
-}
-
-bool testcase_nested::stochastic_breakable_restart_with_nested(
- bool force_restart) {
- log_trace(">> stochastic_breakable_restart_with_nested%s",
- force_restart ? ": force_restart" : "");
-
- if (force_restart)
- while (txn_guard)
- pop_txn(true);
-
- bool should_continue = true;
- while (!stack.empty() &&
- (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42))
- should_continue &= pop_txn();
-
- if (flipcoin_x3()) {
- unsigned period;
- int err = mdbx_env_get_syncperiod(db_guard.get(), &period);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_env_get_syncperiod()", err);
-
- size_t bytes;
- err = mdbx_env_get_syncbytes(db_guard.get(), &bytes);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_env_get_syncbytes()", err);
-
- err = mdbx_env_set_syncperiod(db_guard.get(), period ^ 42);
- if (unlikely(err != MDBX_SUCCESS) && err != MDBX_BUSY)
- failure_perror("mdbx_env_set_syncperiod()", err);
-
- err = mdbx_env_set_syncbytes(db_guard.get(), bytes ^ 42000);
- if (unlikely(err != MDBX_SUCCESS) && err != MDBX_BUSY)
- failure_perror("mdbx_env_set_syncbytes()", err);
- }
-
- if (should_continue)
- while (stack.empty() ||
- (is_nested_txn_available() && flipcoin() && stack.size() < 5))
- push_txn();
-
- log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s",
- should_continue ? "yes" : "no");
- return should_continue;
-}
-
-bool testcase_nested::trim_tail(unsigned window_width) {
- if (window_width || flipcoin()) {
- clear_stepbystep_passed += window_width == 0;
- while (fifo.size() > window_width) {
- uint64_t tail_serial = fifo.back().first;
- const unsigned tail_count = fifo.back().second;
- log_verbose("nested: trim-tail (serial %" PRIu64 ", count %u)",
- tail_serial, tail_count);
- fifo.pop_back();
- for (unsigned n = 0; n < tail_count; ++n) {
- log_trace("nested: remove-tail %" PRIu64, tail_serial);
- generate_pair(tail_serial);
- int err = remove(key, data);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("nested: tail-bailout due '%s'", mdbx_strerror(err));
- return false;
- }
- failure_perror("mdbx_del(tail)", err);
- }
- if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
- failure("nested: unexpected key-space overflow on the tail");
- }
- report(tail_count);
- }
- } else if (!fifo.empty()) {
- log_verbose("nested: purge state %" PRIu64 " - %" PRIu64 ", fifo-items %zu",
- fifo.front().first, fifo.back().first + fifo.back().second,
- fifo.size());
- db_table_clear(dbi, txn_guard.get());
- fifo.clear();
- clear_wholetable_passed += 1;
- report(1);
- }
- return true;
-}
-
-bool testcase_nested::grow_head(unsigned head_count) {
- const MDBX_put_flags_t insert_flags =
- (config.params.table_flags & MDBX_DUPSORT)
- ? MDBX_NODUPDATA
- : MDBX_NODUPDATA | MDBX_NOOVERWRITE;
-retry:
- fifo.push_front(std::make_pair(serial, head_count));
- for (unsigned n = 0; n < head_count; ++n) {
- log_trace("nested: insert-head %" PRIu64, serial);
- generate_pair(serial);
- int err = insert(key, data, insert_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err));
- head_count = n;
- stochastic_breakable_restart_with_nested(true);
- dbfull_passed += 1;
- goto retry;
- }
- failure_perror("mdbx_put(head)", err);
- }
-
- if (unlikely(!keyvalue_maker.increment(serial, 1))) {
- log_notice("nested: unexpected key-space overflow");
- keyspace_overflow = true;
- head_count = n;
- stochastic_breakable_restart_with_nested(true);
- goto retry;
- }
- }
-
- return true;
-}
-
-bool testcase_nested::run() {
- uint64_t seed =
- prng64_map2_white(config.params.keygen.seed) + config.actor_id;
-
- clear_wholetable_passed = 0;
- clear_stepbystep_passed = 0;
- dbfull_passed = 0;
- unsigned loops = 0;
- while (true) {
- const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
- const unsigned window_width =
- (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt);
- const unsigned head_count = edge2count(salt);
- log_debug("nested: step #%" PRIu64 " (serial %" PRIu64
- ", window %u, count %u) salt %" PRIu64,
- nops_completed, serial, window_width, head_count, salt);
-
- if (!trim_tail(window_width))
- return false;
- if (!stochastic_breakable_restart_with_nested()) {
- log_notice("nested: bailout at commit/restart after tail-trim");
- return false;
- }
- if (!speculum_verify()) {
- log_notice("nested: bailout after tail-trim");
- return false;
- }
-
- if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed ||
- !clear_stepbystep_passed)) {
- unsigned underutilization_x256 =
- txn_underutilization_x256(txn_guard.get());
- if (dbfull_passed > underutilization_x256) {
- log_notice("nested: skip head-grow to avoid one more dbfull (was %u, "
- "underutilization %.2f%%)",
- dbfull_passed, underutilization_x256 / 2.560);
- continue;
- }
- if (!grow_head(head_count))
- return false;
- if (!stochastic_breakable_restart_with_nested())
- log_notice("nested: skip commit/restart after head-grow");
- if (!speculum_verify()) {
- log_notice("nested: bailout after head-grow");
- return false;
- }
- loops += 1;
- } else if (fifo.empty()) {
- log_notice("nested: done %u whole loops, %" PRIu64 " ops, %" PRIu64
- " items",
- loops, nops_completed, serial);
- break;
- } else {
- log_notice("nested: done, wait for empty, skip head-grow");
- }
- }
-
- while (!stack.empty())
- pop_txn(false);
-
- return speculum_verify();
-}
diff --git a/3rdparty/libmdbx/test/osal-unix.cc b/3rdparty/libmdbx/test/osal-unix.cc
deleted file mode 100644
index fd20bb0b7a17cf0ce3a48e1172d0e5d9bf1a7e54..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/osal-unix.cc
+++ /dev/null
@@ -1,661 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-#if !(defined(_WIN32) || defined(_WIN64))
-
-#include <atomic>
-#include <pthread.h>
-#include <signal.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-#include <atomic>
-
-#ifndef MDBX_LOCKING
-#error "Oops, MDBX_LOCKING is undefined!"
-#endif
-
-#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008)
-#include "stub/pthread_barrier.c"
-#endif /* __APPLE__ && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */
-
-#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 && \
- (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008)
-#include "stub/pthread_barrier.c"
-#endif /* __ANDROID_API__ < 24 && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */
-
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
-#include <sys/ipc.h>
-#include <sys/sem.h>
-#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
-
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
-#include <semaphore.h>
-
-#if __cplusplus >= 201103L
-#include <atomic>
-MDBX_MAYBE_UNUSED static __inline int atomic_decrement(std::atomic_int *p) {
- return std::atomic_fetch_sub(p, 1) - 1;
-}
-#else
-MDBX_MAYBE_UNUSED static __inline int atomic_decrement(volatile int *p) {
-#if defined(__GNUC__) || defined(__clang__)
- return __sync_sub_and_fetch(p, 1);
-#elif defined(_MSC_VER)
- STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile int));
- return _InterlockedDecrement((volatile long *)p);
-#elif defined(__APPLE__)
- return OSAtomicDecrement32Barrier((volatile int *)p);
-#else
-#error FIXME: Unsupported compiler
-#endif
-}
-#endif /* C++11 */
-#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX1988 */
-
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
-static int ipc;
-static pid_t ipc_overlord_pid;
-static void ipc_remove(void) {
- if (ipc_overlord_pid == getpid())
- semctl(ipc, 0, IPC_RMID, nullptr);
-}
-
-#else
-struct shared_t {
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- pthread_barrier_t barrier;
- pthread_mutex_t mutex;
- size_t count;
- pthread_cond_t events[1];
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- struct {
-#if __cplusplus >= 201103L
- std::atomic_int countdown;
-#else
- volatile int countdown;
-#endif /* C++11 */
- sem_t sema;
- } barrier;
- size_t count;
- sem_t events[1];
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-};
-static shared_t *shared;
-#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */
-
-void osal_wait4barrier(void) {
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- struct sembuf op;
- op.sem_num = 0;
- op.sem_op = -1;
- op.sem_flg = IPC_NOWAIT;
- if (semop(ipc, &op, 1))
- failure_perror("semop(dec)", errno);
- op.sem_op = 0;
- op.sem_flg = 0;
- if (semop(ipc, &op, 1))
- failure_perror("semop(wait)", errno);
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- assert(shared != nullptr && shared != MAP_FAILED);
- int err = pthread_barrier_wait(&shared->barrier);
- if (err != 0 && err != PTHREAD_BARRIER_SERIAL_THREAD)
- failure_perror("pthread_barrier_wait(shared)", err);
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- assert(shared != nullptr && shared != MAP_FAILED);
- int err = (atomic_decrement(&shared->barrier.countdown) > 0 &&
- sem_wait(&shared->barrier.sema))
- ? errno
- : 0;
- if (err != 0)
- failure_perror("sem_wait(shared)", err);
- if (sem_post(&shared->barrier.sema))
- failure_perror("sem_post(shared)", errno);
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-}
-
-void osal_setup(const std::vector<actor_config> &actors) {
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- if (ipc_overlord_pid)
- failure("ipc already created by %ld pid", (long)ipc_overlord_pid);
- ipc_overlord_pid = getpid();
-#ifndef SEM_A
-#define SEM_A S_IRUSR
-#endif
-#ifndef SEM_R
-#define SEM_R S_IWUSR
-#endif
- ipc = semget(IPC_PRIVATE, actors.size() + 2, IPC_CREAT | SEM_A | SEM_R);
- if (ipc < 0)
- failure_perror("semget(IPC_PRIVATE, shared_sems)", errno);
- if (atexit(ipc_remove))
- failure_perror("atexit(ipc_remove)", errno);
- if (semctl(ipc, 0, SETVAL, (int)(actors.size() + 1)))
- failure_perror("semctl(SETVAL.0, shared_sems)", errno);
- for (size_t i = 1; i < actors.size() + 2; ++i)
- if (semctl(ipc, i, SETVAL, 1))
- failure_perror("semctl(SETVAL.N, shared_sems)", errno);
-#else
- assert(shared == nullptr);
- shared = (shared_t *)mmap(
- nullptr, sizeof(shared_t) + actors.size() * sizeof(shared->events[0]),
- PROT_READ | PROT_WRITE,
- MAP_SHARED | MAP_ANONYMOUS
-#ifdef MAP_HASSEMAPHORE
- | MAP_HASSEMAPHORE
-#endif
- ,
- -1, 0);
- if (MAP_FAILED == (void *)shared)
- failure_perror("mmap(shared)", errno);
-
- shared->count = actors.size() + 1;
-
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- pthread_barrierattr_t barrierattr;
- int err = pthread_barrierattr_init(&barrierattr);
- if (err)
- failure_perror("pthread_barrierattr_init()", err);
- err = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED);
- if (err)
- failure_perror("pthread_barrierattr_setpshared()", err);
-
- err = pthread_barrier_init(&shared->barrier, &barrierattr, shared->count);
- if (err)
- failure_perror("pthread_barrier_init(shared)", err);
- pthread_barrierattr_destroy(&barrierattr);
-
- pthread_mutexattr_t mutexattr;
- err = pthread_mutexattr_init(&mutexattr);
- if (err)
- failure_perror("pthread_mutexattr_init()", err);
- err = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
- if (err)
- failure_perror("pthread_mutexattr_setpshared()", err);
-
- pthread_condattr_t condattr;
- err = pthread_condattr_init(&condattr);
- if (err)
- failure_perror("pthread_condattr_init()", err);
- err = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED);
- if (err)
- failure_perror("pthread_condattr_setpshared()", err);
-
- err = pthread_mutex_init(&shared->mutex, &mutexattr);
- if (err)
- failure_perror("pthread_mutex_init(shared)", err);
-
- for (size_t i = 0; i < shared->count; ++i) {
- pthread_cond_t *event = &shared->events[i];
- err = pthread_cond_init(event, &condattr);
- if (err)
- failure_perror("pthread_cond_init(shared)", err);
- log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i,
- __Wpedantic_format_voidptr(event));
- }
- pthread_condattr_destroy(&condattr);
- pthread_mutexattr_destroy(&mutexattr);
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- shared->barrier.countdown = shared->count;
- if (sem_init(&shared->barrier.sema, true, 1))
- failure_perror("sem_init(shared.barrier)", errno);
- for (size_t i = 0; i < shared->count; ++i) {
- sem_t *event = &shared->events[i];
- if (sem_init(event, true, 0))
- failure_perror("sem_init(shared.event)", errno);
- log_trace("osal_setup: event(shared sem_init) %" PRIuPTR " -> %p", i,
- __Wpedantic_format_voidptr(event));
- }
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */
-}
-
-void osal_broadcast(unsigned id) {
- log_trace("osal_broadcast: event %u", id);
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- if (semctl(ipc, id + 1, SETVAL, 0))
- failure_perror("semctl(SETVAL)", errno);
-#else
- assert(shared != nullptr && shared != MAP_FAILED);
- if (id >= shared->count)
- failure("osal_broadcast: id > limit");
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- int err = pthread_cond_broadcast(shared->events + id);
- if (err)
- failure_perror("pthread_cond_broadcast(shared)", err);
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- if (sem_post(shared->events + id))
- failure_perror("sem_post(shared)", errno);
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */
-}
-
-int osal_waitfor(unsigned id) {
- log_trace("osal_waitfor: event %u", id);
-#if MDBX_LOCKING == MDBX_LOCKING_SYSV
- struct sembuf op;
- memset(&op, 0, sizeof(op));
- op.sem_num = (short)(id + 1);
- int rc = semop(ipc, &op, 1) ? errno : MDBX_SUCCESS;
-#else
- assert(shared != nullptr && shared != MAP_FAILED);
- if (id >= shared->count)
- failure("osal_waitfor: id > limit");
-
-#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
- MDBX_LOCKING == MDBX_LOCKING_POSIX2008
- int rc = pthread_mutex_lock(&shared->mutex);
- if (rc != 0)
- failure_perror("pthread_mutex_lock(shared)", rc);
-
- rc = pthread_cond_wait(shared->events + id, &shared->mutex);
- if (rc && rc != EINTR)
- failure_perror("pthread_cond_wait(shared)", rc);
-
- rc = pthread_mutex_unlock(&shared->mutex);
- if (rc != 0)
- failure_perror("pthread_mutex_unlock(shared)", rc);
-#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
- int rc = sem_wait(shared->events + id) ? errno : 0;
- if (rc == 0 && sem_post(shared->events + id))
- failure_perror("sem_post(shared)", errno);
-#else
-#error "FIXME"
-#endif /* MDBX_LOCKING */
-#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */
-
- return (rc == 0) ? true : false;
-}
-
-//-----------------------------------------------------------------------------
-
-const std::string
-actor_config::osal_serialize(simple_checksum &checksum) const {
- (void)checksum;
- /* not used in workload, but just for testing */
- return "unix.fork";
-}
-
-bool actor_config::osal_deserialize(const char *str, const char *end,
- simple_checksum &checksum) {
- (void)checksum;
- /* not used in workload, but just for testing */
- return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end;
-}
-
-//-----------------------------------------------------------------------------
-
-static pid_t overlord_pid;
-
-static std::atomic_int sigusr1_head, sigusr2_head;
-static void handler_SIGUSR(int signum) {
- switch (signum) {
- case SIGUSR1:
- ++sigusr1_head;
- return;
- case SIGUSR2:
- ++sigusr2_head;
- return;
- default:
- abort();
- }
-}
-
-bool osal_progress_push(bool active) {
- if (overlord_pid) {
- if (kill(overlord_pid, active ? SIGUSR1 : SIGUSR2))
- failure_perror("osal_progress_push: kill(overload)", errno);
- return true;
- }
-
- return false;
-}
-
-//-----------------------------------------------------------------------------
-
-static std::unordered_map<pid_t, actor_status> children;
-
-static std::atomic_int sigalarm_head;
-static void handler_SIGCHLD(int signum) {
- if (signum == SIGALRM)
- ++sigalarm_head;
-}
-
-mdbx_pid_t osal_getpid(void) { return getpid(); }
-
-int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; }
-
-int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
- if (children.empty()) {
- struct sigaction act;
- memset(&act, 0, sizeof(act));
- act.sa_handler = handler_SIGCHLD;
- sigaction(SIGCHLD, &act, nullptr);
- sigaction(SIGALRM, &act, nullptr);
- act.sa_handler = handler_SIGUSR;
- sigaction(SIGUSR1, &act, nullptr);
- sigaction(SIGUSR2, &act, nullptr);
-
- sigset_t mask;
- sigemptyset(&mask);
- sigaddset(&mask, SIGCHLD);
- sigaddset(&mask, SIGUSR1);
- sigaddset(&mask, SIGUSR2);
- sigprocmask(SIG_UNBLOCK, &mask, nullptr);
- }
-
- pid = fork();
-
- if (pid == 0) {
- overlord_pid = getppid();
- const bool result = test_execute(config);
- exit(result ? EXIT_SUCCESS : EXIT_FAILURE);
- }
-
- if (pid < 0)
- return errno;
-
- log_trace("osal_actor_start: fork pid %ld for %u", (long)pid,
- config.actor_id);
- children[pid] = as_running;
- return 0;
-}
-
-actor_status osal_actor_info(const mdbx_pid_t pid) { return children.at(pid); }
-
-void osal_killall_actors(void) {
- for (auto &pair : children) {
- kill(pair.first, SIGKILL);
- pair.second = as_killed;
- }
-}
-
-static const char *signal_name(const int sig) {
- if (sig == SIGHUP)
- return "HUP";
- if (sig == SIGINT)
- return "INT";
- if (sig == SIGQUIT)
- return "QUIT";
- if (sig == SIGILL)
- return "ILL";
- if (sig == SIGTRAP)
- return "TRAP";
- if (sig == SIGABRT)
- return "ABRT";
- if (sig == SIGBUS)
- return "BUS";
- if (sig == SIGKILL)
- return "KILL";
- if (sig == SIGUSR1)
- return "USR1";
- if (sig == SIGSEGV)
- return "SEGV";
- if (sig == SIGUSR2)
- return "USR2";
- if (sig == SIGPIPE)
- return "PIPE";
- if (sig == SIGALRM)
- return "ALRM";
- if (sig == SIGTERM)
- return "TERM";
- if (sig == SIGCHLD)
- return "CHLD";
- if (sig == SIGCONT)
- return "CONT";
- if (sig == SIGSTOP)
- return "STOP";
-#ifdef SIGPOLL
- if (sig == SIGPOLL)
- return "POLL";
-#endif
-#ifdef SIGFPE
- if (sig == SIGFPE)
- return "FPE";
-#endif
-#ifdef SIGEMT
- if (sig == SIGEMT)
- return "EMT";
-#endif
-#ifdef SIGSTKFLT
- if (sig == SIGSTKFLT)
- return "STKFLT";
-#endif
-#ifdef SIGTSTP
- if (sig == SIGTSTP)
- return "TSTP";
-#endif
-#ifdef SIGTTIN
- if (sig == SIGTTIN)
- return "TTIN";
-#endif
-#ifdef SIGTTOU
- if (sig == SIGTTOU)
- return "TTOU";
-#endif
-#ifdef SIGURG
- if (sig == SIGURG)
- return "URG";
-#endif
-#ifdef SIGXCPU
- if (sig == SIGXCPU)
- return "XCPU";
-#endif
-#ifdef SIGXFSZ
- if (sig == SIGXFSZ)
- return "XFSZ";
-#endif
-#ifdef SIGVTALRM
- if (sig == SIGVTALRM)
- return "VTALRM";
-#endif
-#ifdef SIGPROF
- if (sig == SIGPROF)
- return "PROF";
-#endif
-#ifdef SIGWINCH
- if (sig == SIGWINCH)
- return "WINCH";
-#endif
-#ifdef SIGIO
- if (sig == SIGIO)
- return "IO";
-#endif
-#ifdef SIGPWR
- if (sig == SIGPWR)
- return "PWR";
-#endif
-#ifdef SIGSYS
- if (sig == SIGSYS)
- return "SYS";
-#endif
- static char buf[32];
- snprintf(buf, sizeof(buf), "%u", sig);
- return buf;
-}
-
-int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
- static sig_atomic_t sigalarm_tail;
- alarm(0) /* cancel prev timeout */;
- sigalarm_tail = sigalarm_head /* reset timeout flag */;
-
- int options = WNOHANG;
- if (timeout) {
- alarm((timeout > INT_MAX) ? INT_MAX : timeout);
- options = 0;
- }
-
-#ifdef WUNTRACED
- options |= WUNTRACED;
-#endif
-#ifdef WCONTINUED
- options |= WCONTINUED;
-#endif
-
- while (sigalarm_tail == sigalarm_head) {
- int status;
- pid = waitpid(0, &status, options);
- const int err = errno;
-
- if (pid > 0) {
- if (WIFEXITED(status))
- children[pid] =
- (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed;
- else if (WIFSIGNALED(status)) {
-#ifdef WCOREDUMP
- if (WCOREDUMP(status))
- children[pid] = as_coredump;
- else
-#endif /* WCOREDUMP */
- switch (WTERMSIG(status)) {
- case SIGABRT:
- case SIGBUS:
- case SIGFPE:
- case SIGILL:
- case SIGSEGV:
- log_notice("child pid %lu terminated by SIG%s", (long)pid,
- signal_name(WTERMSIG(status)));
- children[pid] = as_coredump;
- break;
- default:
- log_notice("child pid %lu killed by SIG%s", (long)pid,
- signal_name(WTERMSIG(status)));
- children[pid] = as_killed;
- }
- } else if (WIFSTOPPED(status))
- children[pid] = as_debugging;
- else if (WIFCONTINUED(status))
- children[pid] = as_running;
- else {
- assert(false);
- }
- return 0;
- }
-
- static sig_atomic_t sigusr1_tail, sigusr2_tail;
- if (sigusr1_tail != sigusr1_head) {
- sigusr1_tail = sigusr1_head;
- logging::progress_canary(true);
- if (pid < 0 && err == EINTR)
- continue;
- }
- if (sigusr2_tail != sigusr2_head) {
- sigusr2_tail = sigusr2_head;
- logging::progress_canary(false);
- if (pid < 0 && err == EINTR)
- continue;
- }
-
- if (pid == 0)
- break;
-
- if (err != EINTR)
- return err;
- }
- return 0 /* timeout */;
-}
-
-void osal_yield(void) {
- if (sched_yield())
- failure_perror("sched_yield()", errno);
-}
-
-void osal_udelay(unsigned us) {
- chrono::time until, now = chrono::now_monotonic();
- until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
- struct timespec ts;
-
- static unsigned threshold_us;
- if (threshold_us == 0) {
-#if defined(_POSIX_CPUTIME) && _POSIX_CPUTIME > -1 && \
- defined(CLOCK_PROCESS_CPUTIME_ID)
- if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) {
- int rc = errno;
- log_warning("clock_getres(CLOCK_PROCESS_CPUTIME_ID), failed errno %d",
- rc);
- }
-#endif /* CLOCK_PROCESS_CPUTIME_ID */
- if (threshold_us == 0 && clock_getres(CLOCK_MONOTONIC, &ts)) {
- int rc = errno;
- failure_perror("clock_getres(CLOCK_MONOTONIC)", rc);
- }
- chrono::time threshold = chrono::from_timespec(ts);
- assert(threshold.seconds() == 0);
-
- threshold_us = chrono::fractional2us(threshold.fractional);
- if (threshold_us < 1000)
- threshold_us = 1000;
- }
-
- ts.tv_sec = ts.tv_nsec = 0;
- if (us > threshold_us) {
- ts.tv_sec = us / 1000000u;
- ts.tv_nsec = (us % 1000000u) * 1000u;
- }
-
- do {
- if (us > threshold_us) {
- if (nanosleep(&ts, &ts)) {
- int rc = errno;
- /* if (rc == EINTR) { ... } ? */
- failure_perror("usleep()", rc);
- }
- us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u;
- }
- cpu_relax();
-
- now = chrono::now_monotonic();
- } while (until.fixedpoint > now.fixedpoint);
-}
-
-bool osal_istty(int fd) { return isatty(fd) == 1; }
-
-std::string osal_tempdir(void) {
- const char *tempdir = getenv("TMPDIR");
- if (!tempdir)
- tempdir = getenv("TMP");
- if (!tempdir)
- tempdir = getenv("TEMPDIR");
- if (!tempdir)
- tempdir = getenv("TEMP");
- if (tempdir && *tempdir) {
- std::string dir(tempdir);
- if (dir.back() != '/')
- dir.append("/");
- return dir;
- }
- if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0)
- return "/dev/shm/";
- return "";
-}
-
-#endif /* !Windows */
diff --git a/3rdparty/libmdbx/test/osal-windows.cc b/3rdparty/libmdbx/test/osal-windows.cc
deleted file mode 100644
index db42ba2b5baee0690f5a96ee8d4c74adbe388954..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/osal-windows.cc
+++ /dev/null
@@ -1,455 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-#if defined(_WIN32) || defined(_WIN64)
-
-static std::unordered_map<unsigned, HANDLE> events;
-static HANDLE hBarrierSemaphore, hBarrierEvent;
-static HANDLE hProgressActiveEvent, hProgressPassiveEvent;
-
-static int waitstatus2errcode(DWORD result) {
- switch (result) {
- case WAIT_OBJECT_0:
- return MDBX_SUCCESS;
- case WAIT_FAILED:
- return GetLastError();
- case WAIT_ABANDONED:
- return ERROR_ABANDONED_WAIT_0;
- case WAIT_IO_COMPLETION:
- return ERROR_USER_APC;
- case WAIT_TIMEOUT:
- return ERROR_TIMEOUT;
- default:
- return ERROR_UNHANDLED_ERROR;
- }
-}
-
-void osal_wait4barrier(void) {
- DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0);
- switch (rc) {
- default:
- failure_perror("WaitForSingleObject(BarrierSemaphore)",
- waitstatus2errcode(rc));
- case WAIT_OBJECT_0:
- rc = WaitForSingleObject(hBarrierEvent, INFINITE);
- if (rc != WAIT_OBJECT_0)
- failure_perror("WaitForSingleObject(BarrierEvent)",
- waitstatus2errcode(rc));
- break;
- case WAIT_TIMEOUT:
- if (!SetEvent(hBarrierEvent))
- failure_perror("SetEvent(BarrierEvent)", GetLastError());
- break;
- }
-}
-
-static HANDLE make_inheritable(HANDLE hHandle) {
- assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE);
- if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(),
- &hHandle, 0, TRUE,
- DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS))
- failure_perror("DuplicateHandle()", GetLastError());
- return hHandle;
-}
-
-void osal_setup(const std::vector<actor_config> &actors) {
- assert(events.empty());
- const size_t n = actors.size() + 1;
- events.reserve(n);
-
- for (unsigned i = 0; i < n; ++i) {
- HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
- if (!hEvent)
- failure_perror("CreateEvent()", GetLastError());
- hEvent = make_inheritable(hEvent);
- log_trace("osal_setup: event %u -> %p", i, hEvent);
- events[i] = hEvent;
- }
-
- hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL);
- if (!hBarrierSemaphore)
- failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError());
- hBarrierSemaphore = make_inheritable(hBarrierSemaphore);
-
- hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
- if (!hBarrierEvent)
- failure_perror("CreateEvent(BarrierEvent)", GetLastError());
- hBarrierEvent = make_inheritable(hBarrierEvent);
-
- hProgressActiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
- if (!hProgressActiveEvent)
- failure_perror("CreateEvent(ProgressActiveEvent)", GetLastError());
- hProgressActiveEvent = make_inheritable(hProgressActiveEvent);
-
- hProgressPassiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
- if (!hProgressPassiveEvent)
- failure_perror("CreateEvent(ProgressPassiveEvent)", GetLastError());
- hProgressPassiveEvent = make_inheritable(hProgressPassiveEvent);
-}
-
-void osal_broadcast(unsigned id) {
- log_trace("osal_broadcast: event %u", id);
- if (!SetEvent(events.at(id)))
- failure_perror("SetEvent()", GetLastError());
-}
-
-int osal_waitfor(unsigned id) {
- log_trace("osal_waitfor: event %u", id);
- DWORD rc = WaitForSingleObject(events.at(id), INFINITE);
- return waitstatus2errcode(rc);
-}
-
-mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); }
-
-int osal_delay(unsigned seconds) {
- Sleep(seconds * 1000u);
- return 0;
-}
-
-//-----------------------------------------------------------------------------
-
-const std::string
-actor_config::osal_serialize(simple_checksum &checksum) const {
- checksum.push(hBarrierSemaphore);
- checksum.push(hBarrierEvent);
- checksum.push(hProgressActiveEvent);
- checksum.push(hProgressPassiveEvent);
-
- HANDLE hWait = INVALID_HANDLE_VALUE;
- if (wait4id) {
- hWait = events.at(wait4id);
- checksum.push(hWait);
- }
-
- HANDLE hSignal = INVALID_HANDLE_VALUE;
- if (wanna_event4signalling()) {
- hSignal = events.at(actor_id);
- checksum.push(hSignal);
- }
-
- return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait,
- hSignal, hProgressActiveEvent, hProgressPassiveEvent);
-}
-
-bool actor_config::osal_deserialize(const char *str, const char *end,
- simple_checksum &checksum) {
-
- std::string copy(str, end - str);
- TRACE(">> osal_deserialize(%s)\n", copy.c_str());
-
- assert(hBarrierSemaphore == 0);
- assert(hBarrierEvent == 0);
- assert(hProgressActiveEvent == 0);
- assert(hProgressPassiveEvent == 0);
- assert(events.empty());
-
- HANDLE hWait, hSignal;
- if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore,
- &hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent,
- &hProgressPassiveEvent) != 6) {
- TRACE("<< osal_deserialize: failed\n");
- return false;
- }
-
- checksum.push(hBarrierSemaphore);
- checksum.push(hBarrierEvent);
- checksum.push(hProgressActiveEvent);
- checksum.push(hProgressPassiveEvent);
-
- if (wait4id) {
- checksum.push(hWait);
- events[wait4id] = hWait;
- }
-
- if (wanna_event4signalling()) {
- checksum.push(hSignal);
- events[actor_id] = hSignal;
- }
-
- TRACE("<< osal_deserialize: OK\n");
- return true;
-}
-
-//-----------------------------------------------------------------------------
-
-typedef std::pair<HANDLE, actor_status> child;
-static std::unordered_map<mdbx_pid_t, child> children;
-
-bool osal_progress_push(bool active) {
- if (!children.empty()) {
- if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent))
- failure_perror("osal_progress_push: SetEvent(overlord.progress)",
- GetLastError());
- return true;
- }
-
- return false;
-}
-
-static void ArgvQuote(std::string &CommandLine, const std::string &Argument,
- bool Force = false)
-
-/*++
-
-https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
-
-Routine Description:
-
- This routine appends the given argument to a command line such
- that CommandLineToArgvW will return the argument string unchanged.
- Arguments in a command line should be separated by spaces; this
- function does not add these spaces.
-
-Arguments:
-
- Argument - Supplies the argument to encode.
-
- CommandLine - Supplies the command line to which we append the encoded
-argument string.
-
- Force - Supplies an indication of whether we should quote
- the argument even if it does not contain any characters that would
- ordinarily require quoting.
-
-Return Value:
-
- None.
-
-Environment:
-
- Arbitrary.
-
---*/
-
-{
- //
- // Unless we're told otherwise, don't quote unless we actually
- // need to do so --- hopefully avoid problems if programs won't
- // parse quotes properly
- //
-
- if (Force == false && Argument.empty() == false &&
- Argument.find_first_of(" \t\n\v\"") == Argument.npos) {
- CommandLine.append(Argument);
- } else {
- CommandLine.push_back('"');
-
- for (auto It = Argument.begin();; ++It) {
- unsigned NumberBackslashes = 0;
-
- while (It != Argument.end() && *It == '\\') {
- ++It;
- ++NumberBackslashes;
- }
-
- if (It == Argument.end()) {
- //
- // Escape all backslashes, but let the terminating
- // double quotation mark we add below be interpreted
- // as a metacharacter.
- //
- CommandLine.append(NumberBackslashes * 2, '\\');
- break;
- } else if (*It == L'"') {
- //
- // Escape all backslashes and the following
- // double quotation mark.
- //
- CommandLine.append(NumberBackslashes * 2 + 1, '\\');
- CommandLine.push_back(*It);
- } else {
- //
- // Backslashes aren't special here.
- //
- CommandLine.append(NumberBackslashes, '\\');
- CommandLine.push_back(*It);
- }
- }
-
- CommandLine.push_back('"');
- }
-}
-
-int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
- if (children.size() == MAXIMUM_WAIT_OBJECTS)
- failure("Couldn't manage more that %u actors on Windows\n",
- MAXIMUM_WAIT_OBJECTS);
-
- _flushall();
-
- STARTUPINFOA StartupInfo;
- GetStartupInfoA(&StartupInfo);
-
- char exename[_MAX_PATH + 1];
- DWORD exename_size = sizeof(exename);
- if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename,
- &exename_size))
- failure_perror("QueryFullProcessImageName()", GetLastError());
-
- if (exename[1] != ':') {
- exename_size = GetModuleFileName(NULL, exename, sizeof(exename));
- if (exename_size >= sizeof(exename))
- return ERROR_BAD_LENGTH;
- }
-
- std::string cmdline = "$ ";
- ArgvQuote(cmdline, thunk_param(config));
-
- if (cmdline.size() >= 32767)
- return ERROR_BAD_LENGTH;
-
- PROCESS_INFORMATION ProcessInformation;
- if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()),
- NULL, // Retuned process handle is not inheritable.
- NULL, // Retuned thread handle is not inheritable.
- TRUE, // Child inherits all inheritable handles.
- NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY,
- NULL, // Inherit the parent's environment.
- NULL, // Inherit the parent's current directory.
- &StartupInfo, &ProcessInformation))
- failure_perror(exename, GetLastError());
-
- CloseHandle(ProcessInformation.hThread);
- pid = ProcessInformation.dwProcessId;
- children[pid] = std::make_pair(ProcessInformation.hProcess, as_running);
- return 0;
-}
-
-actor_status osal_actor_info(const mdbx_pid_t pid) {
- actor_status status = children.at(pid).second;
- if (status > as_running)
- return status;
-
- DWORD ExitCode;
- if (!GetExitCodeProcess(children.at(pid).first, &ExitCode))
- failure_perror("GetExitCodeProcess()", GetLastError());
-
- switch (ExitCode) {
- case STILL_ACTIVE:
- return as_running;
- case EXIT_SUCCESS:
- status = as_successful;
- break;
- case EXCEPTION_BREAKPOINT:
- case EXCEPTION_SINGLE_STEP:
- status = as_debugging;
- break;
- case STATUS_CONTROL_C_EXIT:
- status = as_killed;
- break;
- case EXCEPTION_ACCESS_VIOLATION:
- case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
- case EXCEPTION_DATATYPE_MISALIGNMENT:
- case EXCEPTION_STACK_OVERFLOW:
- case EXCEPTION_INVALID_DISPOSITION:
- case EXCEPTION_ILLEGAL_INSTRUCTION:
- case EXCEPTION_NONCONTINUABLE_EXCEPTION:
- status = as_coredump;
- break;
- default:
- status = as_failed;
- break;
- }
-
- children.at(pid).second = status;
- return status;
-}
-
-void osal_killall_actors(void) {
- for (auto &pair : children)
- TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT);
-}
-
-int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
- std::vector<HANDLE> handles;
- handles.reserve(children.size() + 2);
- handles.push_back(hProgressActiveEvent);
- handles.push_back(hProgressPassiveEvent);
- for (const auto &pair : children)
- if (pair.second.second <= as_running)
- handles.push_back(pair.second.first);
-
- while (true) {
- DWORD rc =
- MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0],
- (timeout > 60) ? 60 * 1000 : timeout * 1000,
- QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0);
-
- if (rc == WAIT_OBJECT_0) {
- logging::progress_canary(true);
- continue;
- }
- if (rc == WAIT_OBJECT_0 + 1) {
- logging::progress_canary(false);
- continue;
- }
-
- if (rc >= WAIT_OBJECT_0 + 2 && rc < WAIT_OBJECT_0 + handles.size()) {
- pid = 0;
- for (const auto &pair : children)
- if (pair.second.first == handles[rc - WAIT_OBJECT_0]) {
- pid = pair.first;
- break;
- }
- return 0;
- }
-
- if (rc == WAIT_TIMEOUT) {
- pid = 0;
- return 0;
- }
-
- return waitstatus2errcode(rc);
- }
-}
-
-void osal_yield(void) { SwitchToThread(); }
-
-void osal_udelay(unsigned us) {
- chrono::time until, now = chrono::now_monotonic();
- until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
-
- static unsigned threshold_us;
- if (threshold_us == 0) {
- unsigned timeslice_ms = 1;
- while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO)
- ++timeslice_ms;
- threshold_us = timeslice_ms * 1500u;
- assert(threshold_us > 0);
- }
-
- do {
- if (us > threshold_us && us > 1000) {
- DWORD rc = SleepEx(us / 1000, TRUE);
- if (rc)
- failure_perror("SleepEx()", waitstatus2errcode(rc));
- us = 0;
- }
-
- YieldProcessor();
- now = chrono::now_monotonic();
- } while (now.fixedpoint < until.fixedpoint);
-}
-
-bool osal_istty(int fd) { return _isatty(fd) != 0; }
-
-std::string osal_tempdir(void) {
- char buf[MAX_PATH + 1];
- DWORD len = GetTempPathA(sizeof(buf), buf);
- return std::string(buf, len);
-}
-
-#endif /* Windows */
diff --git a/3rdparty/libmdbx/test/osal.h b/3rdparty/libmdbx/test/osal.h
deleted file mode 100644
index b35c9d4cfd689419c0a8f56909d3aea1479f3c68..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/osal.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-
-void osal_setup(const std::vector<actor_config> &actors);
-void osal_broadcast(unsigned id);
-int osal_waitfor(unsigned id);
-
-int osal_actor_start(const actor_config &config, mdbx_pid_t &pid);
-actor_status osal_actor_info(const mdbx_pid_t pid);
-void osal_killall_actors(void);
-int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout);
-void osal_wait4barrier(void);
-
-bool osal_progress_push(bool active);
-
-mdbx_pid_t osal_getpid(void);
-int osal_delay(unsigned seconds);
-void osal_udelay(unsigned us);
-void osal_yield(void);
-bool osal_istty(int fd);
-std::string osal_tempdir(void);
-
-#ifdef _MSC_VER
-#ifndef STDIN_FILENO
-#define STDIN_FILENO _fileno(stdin)
-#endif
-#ifndef STDOUT_FILENO
-#define STDOUT_FILENO _fileno(stdout)
-#endif
-#ifndef STDERR_FILENO
-#define STDERR_FILENO _fileno(stderr)
-#endif
-#endif /* _MSC_VER */
diff --git a/3rdparty/libmdbx/test/pcrf/README.md b/3rdparty/libmdbx/test/pcrf/README.md
deleted file mode 100644
index b2c9b5ce950f3645f498cac92fabbd9efea167e7..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/pcrf/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-PCRF Session DB emulation test
-
diff --git a/3rdparty/libmdbx/test/pcrf/pcrf_test.c b/3rdparty/libmdbx/test/pcrf/pcrf_test.c
deleted file mode 100644
index 96bb631dd4b900eea341b3152ffcbbdb3eb6d842..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/pcrf/pcrf_test.c
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Copyright 2016-2022 Leonid Yuriev <leo@yuriev.ru>.
- * Copyright 2015 Vladimir Romanov
- * <https://www.linkedin.com/in/vladimirromanov>, Yota Lab.
- *
- * This file is part of libmdbx.
- *
- * ReOpenMDBX is free software; you can redistribute it and/or modify it under
- * the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation; either version 3 of the License, or
- * (at your option) any later version.
- *
- * ReOpenMDBX is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <sys/stat.h>
-#include <sys/time.h>
-
-#include "mdbx.h"
-#include <assert.h>
-#include <inttypes.h>
-#include <limits.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#define IP_PRINTF_ARG_HOST(addr) \
- (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \
- (int)((addr)&0xff)
-
-char opt_db_path[PATH_MAX] = "./mdbx_bench2";
-static MDBX_env *env;
-#define REC_COUNT 10240000
-int64_t ids[REC_COUNT * 10];
-int32_t ids_count = 0;
-
-int64_t mdbx_add_count = 0;
-int64_t mdbx_del_count = 0;
-uint64_t mdbx_add_time = 0;
-uint64_t mdbx_del_time = 0;
-int64_t obj_id = 0;
-int64_t mdbx_data_size = 0;
-int64_t mdbx_key_size = 0;
-
-typedef struct {
- char session_id1[100];
- char session_id2[100];
- char ip[20];
- uint8_t fill[100];
-} session_data_t;
-
-typedef struct {
- int64_t obj_id;
- int8_t event_type;
-} __attribute__((__packed__)) event_data_t;
-
-static void add_id_to_pool(int64_t id) {
- ids[ids_count] = id;
- ids_count++;
-}
-
-static inline int64_t getClockUs(void) {
- struct timespec val;
-#ifdef CYGWIN
- clock_gettime(CLOCK_REALTIME, &val);
-#else
- clock_gettime(CLOCK_MONOTONIC, &val);
-#endif
- return val.tv_sec * ((int64_t)1000000) + val.tv_nsec / 1000;
-}
-
-static int64_t get_id_from_pool() {
- if (ids_count == 0) {
- return -1;
- }
- int32_t index = rand() % ids_count;
- int64_t id = ids[index];
- ids[index] = ids[ids_count - 1];
- ids_count--;
- return id;
-}
-
-#define MDBX_CHECK(x) \
- do { \
- const int rc = (x); \
- if (rc != MDBX_SUCCESS) { \
- printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \
- __FILE__, __LINE__); \
- exit(EXIT_FAILURE); \
- } \
- } while (0)
-
-static void db_connect() {
- MDBX_dbi dbi_session;
- MDBX_dbi dbi_session_id;
- MDBX_dbi dbi_event;
- MDBX_dbi dbi_ip;
-
- MDBX_CHECK(mdbx_env_create(&env));
- MDBX_CHECK(mdbx_env_set_geometry(
- env, 0, 0, REC_COUNT * sizeof(session_data_t) * 10, -1, -1, -1));
- MDBX_CHECK(mdbx_env_set_maxdbs(env, 30));
- MDBX_CHECK(mdbx_env_open(env, opt_db_path,
- MDBX_CREATE | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC |
- MDBX_LIFORECLAIM,
- 0664));
- MDBX_txn *txn;
-
- // transaction init
- MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
- // open database in read-write mode
- MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
- MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
- MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
- MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
- // transaction commit
- MDBX_CHECK(mdbx_txn_commit(txn));
- printf("Connection open\n");
-}
-
-static void create_record(uint64_t record_id) {
- MDBX_dbi dbi_session;
- MDBX_dbi dbi_session_id;
- MDBX_dbi dbi_event;
- MDBX_dbi dbi_ip;
- event_data_t event;
- MDBX_txn *txn;
- session_data_t data;
- // transaction init
- snprintf(data.session_id1, sizeof(data.session_id1),
- "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64,
- (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1,
- record_id);
- snprintf(data.session_id2, sizeof(data.session_id2),
- "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id,
- (record_id + UINT64_C(1442695040888963407)) %
- UINT64_C(6364136223846793005));
- snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d",
- IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
- event.obj_id = record_id;
- event.event_type = 1;
-
- MDBX_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
- MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
- MDBX_val _ip_rec = {data.ip, strlen(data.ip)};
- MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
- MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) +
- (rand() % sizeof(data.fill))};
- MDBX_val _event_rec = {&event, sizeof(event)};
-
- uint64_t start = getClockUs();
- MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
- MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
- MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
- MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
- MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
- MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec,
- MDBX_NOOVERWRITE | MDBX_NODUPDATA));
- MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec,
- MDBX_NOOVERWRITE | MDBX_NODUPDATA));
- MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec,
- MDBX_NOOVERWRITE | MDBX_NODUPDATA));
- MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
- MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
- MDBX_CHECK(mdbx_txn_commit(txn));
-
- mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
- mdbx_key_size +=
- (_obj_id_rec.iov_len + _session_id1_rec.iov_len +
- _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
-
- // transaction commit
- mdbx_add_count++;
- mdbx_add_time += (getClockUs() - start);
-}
-
-static void delete_record(int64_t record_id) {
- MDBX_dbi dbi_session;
- MDBX_dbi dbi_session_id;
- MDBX_dbi dbi_event;
- MDBX_dbi dbi_ip;
- event_data_t event;
- MDBX_txn *txn;
-
- // transaction init
- uint64_t start = getClockUs();
- MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
- // open database in read-write mode
- MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
- MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
- MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
- MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
- // put data
- MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
- MDBX_val _data_rec;
- // get data
- MDBX_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec));
- session_data_t *data = (session_data_t *)_data_rec.iov_base;
-
- MDBX_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
- MDBX_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
- MDBX_val _ip_rec = {data->ip, strlen(data->ip)};
- MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL));
- MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL));
- MDBX_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL));
- event.obj_id = record_id;
- event.event_type = 1;
- MDBX_val _event_rec = {&event, sizeof(event)};
- MDBX_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL));
- MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL));
-
- mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
- mdbx_key_size -=
- (_obj_id_rec.iov_len + _session_id1_rec.iov_len +
- _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
-
- // transaction commit
- MDBX_CHECK(mdbx_txn_commit(txn));
- mdbx_del_count++;
- mdbx_del_time += (getClockUs() - start);
-}
-
-static void db_disconnect() {
- mdbx_env_close(env);
- printf("Connection closed\n");
-}
-
-static void get_db_stat(const char *db, int64_t *ms_branch_pages,
- int64_t *ms_leaf_pages) {
- MDBX_txn *txn;
- MDBX_stat stat;
- MDBX_dbi dbi;
-
- MDBX_CHECK(mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &txn));
- MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi));
- MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat)));
- mdbx_txn_abort(txn);
- printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64
- " | %11" PRIu64 " |\n",
- db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries,
- stat.ms_leaf_pages, stat.ms_overflow_pages);
- (*ms_branch_pages) += stat.ms_branch_pages;
- (*ms_leaf_pages) += stat.ms_leaf_pages;
-}
-
-static void periodic_stat(void) {
- int64_t ms_branch_pages = 0;
- int64_t ms_leaf_pages = 0;
- MDBX_stat mst;
- MDBX_envinfo mei;
- MDBX_CHECK(mdbx_env_stat_ex(env, NULL, &mst, sizeof(mst)));
- MDBX_CHECK(mdbx_env_info_ex(env, NULL, &mei, sizeof(mei)));
- printf("Environment Info\n");
- printf(" Pagesize: %u\n", mst.ms_psize);
- if (mei.mi_geo.lower != mei.mi_geo.upper) {
- printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
- "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
- "/-%" PRIu64 ")\n",
- mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
- mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
- mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
- mei.mi_geo.shrink / mst.ms_psize);
- printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
- mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
- } else {
- printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
- mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
- }
- printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
- mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
- printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
- printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
- printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
- mei.mi_latter_reader_txnid,
- mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
- printf(" Max readers: %u\n", mei.mi_maxreaders);
- printf(" Number of readers used: %u\n", mei.mi_numreaders);
-
- printf(" Name | ms_branch_pages | depth | entries | leaf_pages "
- "| overf_pages |\n");
- get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
- get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
- get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
- get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
- printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "",
- ms_branch_pages, "", "", ms_leaf_pages, "");
-
- static int64_t prev_add_count;
- static int64_t prev_del_count;
- static uint64_t prev_add_time;
- static uint64_t prev_del_time;
- static int64_t t = -1;
- if (t > 0) {
- int64_t delta = (getClockUs() - t);
- printf("CPS: add %" PRIu64 ", delete %" PRIu64
- ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64
- "K\n",
- (mdbx_add_count - prev_add_count) * 1000000 / delta,
- (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024,
- mdbx_data_size / 1024, mdbx_key_size / 1024);
- printf("usage data=%" PRIu64 "%%",
- ((mdbx_data_size + mdbx_key_size) * 100) /
- ((ms_leaf_pages + ms_branch_pages) * 4096));
- if (prev_add_time != mdbx_add_time) {
- printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) *
- 1000000 /
- (mdbx_add_time - prev_add_time));
- }
- if (prev_del_time != mdbx_del_time) {
- printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) *
- 1000000 /
- (mdbx_del_time - prev_del_time));
- }
- if (mdbx_add_time) {
- printf(" tAdd : %" PRIu64 " c/s",
- mdbx_add_count * 1000000 / mdbx_add_time);
- }
- if (mdbx_del_time) {
- printf(" tDel : %" PRIu64 " c/s",
- mdbx_del_count * 1000000 / mdbx_del_time);
- }
- puts("");
- }
- t = getClockUs();
- prev_add_count = mdbx_add_count;
- prev_del_count = mdbx_del_count;
- prev_add_time = mdbx_add_time;
- prev_del_time = mdbx_del_time;
-}
-
-#if 0 /* unused */
-static void periodic_add_rec() {
- for (int i = 0; i < 10240; i++) {
- if (ids_count <= REC_COUNT) {
- int64_t id = obj_id++;
- create_record(id);
- add_id_to_pool(id);
- }
- if (ids_count > REC_COUNT) {
- int64_t id = get_id_from_pool();
- delete_record(id);
- }
- }
- periodic_stat();
-}
-#endif
-
-int main(int argc, char **argv) {
- (void)argc;
- (void)argv;
-
- char filename[PATH_MAX];
- int i;
-
- strcpy(filename, opt_db_path);
- strcat(filename, MDBX_DATANAME);
- remove(filename);
-
- strcpy(filename, opt_db_path);
- strcat(filename, MDBX_LOCKNAME);
- remove(filename);
-
- puts("Open DB...");
- db_connect();
- puts("Create data...");
- int64_t t = getClockUs();
- for (i = 0; i < REC_COUNT; i++) {
- int64_t id = obj_id++;
- create_record(id);
- add_id_to_pool(id);
- if (i % 1000 == 0) {
- int64_t now = getClockUs();
- if ((now - t) > 1000000L) {
- periodic_stat();
- t = now;
- }
- }
- }
- periodic_stat();
- while (1) {
- int i;
- for (i = 0; i < 1000; i++) {
- int64_t id = obj_id++;
- create_record(id);
- add_id_to_pool(id);
- id = get_id_from_pool();
- delete_record(id);
- }
- for (i = 0; i < 50; i++) {
- int64_t id = obj_id++;
- create_record(id);
- add_id_to_pool(id);
- }
- int64_t id = obj_id++;
- create_record(id);
- add_id_to_pool(id);
- int64_t now = getClockUs();
- if ((now - t) > 10000000L) {
- periodic_stat();
- t = now;
- }
- }
- db_disconnect();
- return 0;
-}
diff --git a/3rdparty/libmdbx/test/stochastic_small.sh b/3rdparty/libmdbx/test/stochastic_small.sh
deleted file mode 100755
index 8c9bba5ff1eca29ace9cd55842840c8d46ca9a1e..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/stochastic_small.sh
+++ /dev/null
@@ -1,370 +0,0 @@
-#!/usr/bin/env bash
-if ! which make cc c++ tee >/dev/null; then
- echo "Please install the following prerequisites: make cc c++ tee banner" >&2
- exit 1
-fi
-
-LIST=--hill
-FROM=1
-UPTO=9999
-MONITOR=
-LOOPS=
-SKIP_MAKE=no
-BANNER="$(which banner 2>/dev/null | echo echo)"
-UNAME="$(uname -s 2>/dev/null || echo Unknown)"
-DB_UPTO_MB=17408
-
-while [ -n "$1" ]
-do
- case "$1" in
- --help)
- echo "--multi Engage multi-process test scenario (default)"
- echo "--single Execute series of single-process tests (for QEMU, etc)"
- echo "--with-valgrind Run tests under Valgrind's memcheck tool"
- echo "--skip-make Don't (re)build libmdbx and test's executable"
- echo "--from NN Start iterating from the NN ops per test case"
- echo "--upto NN Don't run tests with more than NN ops per test case"
- echo "--loops NN Stop after the NN loops"
- echo "--dir PATH Specifies directory for test DB and other files (it will be cleared)"
- echo "--db-upto-mb NN Limits upper size of test DB to the NN megabytes"
- echo "--help Print this usage help and exit"
- exit -2
- ;;
- --multi)
- LIST=basic
- ;;
- --single)
- LIST="--nested --hill --append --ttl --copy"
- ;;
- --with-valgrind)
- echo " NOTE: Valgrind could produce some false-positive warnings"
- echo " in multi-process environment with shared memory."
- echo " For instance, when the process 'A' explicitly marks a memory"
- echo " region as 'undefined', the process 'B' fill it,"
- echo " and after this process 'A' read such region, etc."
- MONITOR="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt"
- rm -f valgrind-*.log
- ;;
- --skip-make)
- SKIP_MAKE=yes
- ;;
- --from)
- FROM=$(($2))
- if [ -z "$FROM" -o "$FROM" -lt 1 ]; then
- echo "Invalid value '$FROM' for --from option"
- exit -2
- fi
- shift
- ;;
- --upto)
- UPTO=$(($2))
- if [ -z "$UPTO" -o "$UPTO" -lt 1 ]; then
- echo "Invalid value '$UPTO' for --upto option"
- exit -2
- fi
- shift
- ;;
- --loops)
- LOOPS=$(($2))
- if [ -z "$LOOPS" -o "$LOOPS" -lt 1 -o "$LOOPS" -gt 99 ]; then
- echo "Invalid value '$LOOPS' for --loops option"
- exit -2
- fi
- shift
- ;;
- --dir)
- TESTDB_DIR="$2"
- if [ -z "$TESTDB_DIR" ]; then
- echo "Invalid value '$TESTDB_DIR' for --dir option"
- exit -2
- fi
- shift
- ;;
- --db-upto-mb)
- DB_UPTO_MB=$(($2))
- if [ -z "$DB_UPTO_MB" -o "$DB_UPTO_MB" -lt 1 -o "$DB_UPTO_MB" -gt 4194304 ]; then
- echo "Invalid value '$DB_UPTO_MB' for --db-upto-mb option"
- exit -2
- fi
- shift
- ;;
- *)
- echo "Unknown option '$1'"
- exit -2
- ;;
- esac
- shift
-done
-
-set -euo pipefail
-if [ -z "$MONITOR" ]; then
- if which time >/dev/null 2>/dev/null; then
- MONITOR=$(which time)
- if $MONITOR -o /dev/stdout true >/dev/null 2>/dev/null; then
- MONITOR="$MONITOR -o /dev/stdout"
- fi
- fi
- export MALLOC_CHECK_=7 MALLOC_PERTURB_=42
-fi
-
-###############################################################################
-# 1. clean data from prev runs and examine available RAM
-
-WANNA_MOUNT=0
-case ${UNAME} in
- Linux)
- MAKE=make
- if [ -z "${TESTDB_DIR:-}" ]; then
- for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]* 2>/dev/null); do
- rm -rf $old_test_dir
- done
- TESTDB_DIR="/dev/shm/mdbx-test.$$"
- fi
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
-
- if LC_ALL=C free | grep -q -i available; then
- ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024))
- else
- ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024))
- fi
- ;;
-
- FreeBSD)
- MAKE=gmake
- if [ -z "${TESTDB_DIR:-}" ]; then
- for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]* 2>/dev/null); do
- umount $old_test_dir && rm -r $old_test_dir
- done
- TESTDB_DIR="/tmp/mdbx-test.$$"
- rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR
- WANNA_MOUNT=1
- else
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
- fi
- ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024))
- ;;
-
- Darwin)
- MAKE=make
- if [ -z "${TESTDB_DIR:-}" ]; then
- for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst 2>/dev/null); do
- disk=$(mount | grep $vol | cut -d ' ' -f 1)
- echo "umount: volume $vol disk $disk"
- hdiutil unmount $vol -force
- hdiutil detach $disk
- done
- TESTDB_DIR="/Volumes/mdx$$tst"
- WANNA_MOUNT=1
- else
- mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
- fi
- pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024))
- freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1)
- ram_avail_mb=$((pagesize * freepages / 1024))
- echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}"
- ;;
-
- *)
- echo "FIXME: ${UNAME} not supported by this script"
- exit 2
- ;;
-esac
-
-rm -f ${TESTDB_DIR}/*
-
-###############################################################################
-# 2. estimate reasonable RAM space for test-db
-
-echo "=== ${ram_avail_mb}M RAM available"
-ram_reserve4logs_mb=1234
-if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then
- echo "=== At least ${ram_reserve4logs_mb}Mb RAM required"
- exit 3
-fi
-
-#
-# Ð’ режимах отличных от MDBX_WRITEMAP Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð´Ð¾ запиÑи в файл
-# будут накапливатьÑÑ Ð² памÑти, что может потребовать Ñвободной
-# памÑти размером Ñ Ð‘Ð”. Кроме Ñтого, в теÑÑ‚ входит Ñценарий
-# ÑÐ¾Ð·Ð´Ð°Ð½Ð¸Ñ ÐºÐ¾Ð¿Ð¸Ñ Ð‘Ð” на ходу. ПоÑтому БД не может быть больше 1/3
-# от доÑтупной памÑти. Однако, Ñледует учеÑÑ‚ÑŒ что malloc() будет
-# не Ñразу возвращать выделенную памÑÑ‚ÑŒ ÑиÑтеме, а также
-# предуÑмотреть меÑта Ð´Ð»Ñ Ð»Ð¾Ð³Ð¾Ð².
-#
-# In non-MDBX_WRITEMAP modes, updates (dirty pages) will
-# accumulate in memory before writing to the disk, which may
-# require a free memory up to the size of a whole database. In
-# addition, the test includes a script create a copy of the
-# database on the go. Therefore, the database cannot be more 1/3
-# of available memory. Moreover, should be taken into account
-# that malloc() will not return the allocated memory to the
-# system immediately, as well some space is required for logs.
-#
-db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4))
-if [ $db_size_mb -gt $DB_UPTO_MB ]; then
- db_size_mb=$DB_UPTO_MB
-fi
-echo "=== use ${db_size_mb}M for DB"
-
-###############################################################################
-# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required
-case ${UNAME} in
- Linux)
- ulimit -c unlimited
- if [ "$(cat /proc/sys/kernel/core_pattern)" != "core.%p" ]; then
- echo "core.%p > /proc/sys/kernel/core_pattern" >&2
- echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern || true
- fi
- ;;
-
- FreeBSD)
- if [[ WANNA_MOUNT ]]; then
- mount -t tmpfs tmpfs $TESTDB_DIR
- fi
- ;;
-
- Darwin)
- if [[ WANNA_MOUNT ]]; then
- ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb))
- number_of_sectors=$((ramdisk_size_mb * 2048))
- ramdev=$(hdiutil attach -nomount ram://${number_of_sectors})
- diskutil erasevolume ExFAT "mdx$$tst" ${ramdev}
- fi
- ;;
-
- *)
- echo "FIXME: ${UNAME} not supported by this script"
- exit 2
- ;;
-esac
-
-###############################################################################
-# 4. build the test executables
-
-if [ "$SKIP_MAKE" != "yes" ]; then
- ${MAKE} -j$(which nproc >/dev/null 2>/dev/null && nproc || echo 2) build-test
-fi
-
-###############################################################################
-# 5. run stochastic iterations
-
-if which lz4 >/dev/null; then
- function logger {
- lz4 > ${TESTDB_DIR}/long.log.lz4
- }
-elif which gzip >/dev/null; then
- function logger {
- gzip > ${TESTDB_DIR}/long.log.gz
- }
-else
- function logger {
- cat > ${TESTDB_DIR}/long.log
- }
-fi
-
-syncmodes=("" ,+nosync-safe ,+nosync-utterly)
-options=(writemap coalesce lifo notls perturb)
-
-function join { local IFS="$1"; shift; echo "$*"; }
-
-function bits2options {
- local bits=$1
- local i
- local list=()
- for ((i = 0; i < ${#options[@]}; ++i)); do
- list[$i]=$( (( (bits & (1 << i)) != 0 )) && echo -n '+' || echo -n '-'; echo ${options[$i]})
- done
- join , ${list[@]}
-}
-
-function failed {
- echo "FAILED" >&2
- exit 1
-}
-
-function check_deep {
- if [ "$case" = "basic" -o "$case" = "--hill" ]; then
- tee >(logger) | grep -e reach -e achieve
- else
- logger
- fi
-}
-
-function probe {
- echo "----------------------------------------------- $(date)"
- echo "${caption}"
- rm -f ${TESTDB_DIR}/* || failed
- for case in $LIST
- do
- echo "Run ./mdbx_test ${speculum} --random-writemap=no --ignore-dbfull --repeat=1 --pathname=${TESTDB_DIR}/long.db --cleanup-after=no $@ $case"
- ${MONITOR} ./mdbx_test ${speculum} --random-writemap=no --ignore-dbfull --repeat=1 --pathname=${TESTDB_DIR}/long.db --cleanup-before=yes --cleanup-after=no "$@" $case | check_deep \
- && ${MONITOR} ./mdbx_chk ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \
- && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${MONITOR} ./mdbx_chk ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \
- || failed
- done
-}
-
-#------------------------------------------------------------------------------
-
-count=0
-loop=0
-cases='?'
-for ((wbatch=FROM; wbatch<=UPTO; ++wbatch)); do
- if [ -n "$LOOPS" ] && [ $loop -ge "$LOOPS" ]; then echo "The '--loops $LOOPS' limit reached"; break; fi
- echo "======================================================================="
- speculum=$([ $wbatch -le 1000 ] && echo '--speculum' || true)
- nops=$((wbatch/7 + 1))
- for ((rep=1; rep < 11; ++rep)); do
- echo "======================================================================="
- ${BANNER} "$nops / $wbatch, repeat $rep"
- subcase=0
- for ((bits=2**${#options[@]}; --bits >= 0; )); do
- seed=$(($(date +%s) + RANDOM))
-
- split=30
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=24
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=16
- caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
-
- split=4
- caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \
- --pagesize=4K --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
- --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \
- --keygen.seed=${seed}
- done # options
- cases="${subcase}"
- done # repeats
- loop=$((loop + 1))
- if [ -n "$LOOPS" ] && [ $loop -ge "$LOOPS" ]; then break; fi
-done # wbatch
-
-echo "=== ALL DONE ====================== $(date)"
diff --git a/3rdparty/libmdbx/test/stub/LICENSE b/3rdparty/libmdbx/test/stub/LICENSE
deleted file mode 100644
index 6a0dd3066ba6908d2ace7213a175026aaf4d476f..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/stub/LICENSE
+++ /dev/null
@@ -1,24 +0,0 @@
-Copyright (c) 2015, Aleksey Demakov
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
diff --git a/3rdparty/libmdbx/test/stub/README.md b/3rdparty/libmdbx/test/stub/README.md
deleted file mode 100644
index a6a8fd1a915d8c7ec93a68931deac30b50a16e7f..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/stub/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# DarwinPthreadBarrier
-
-A pthread_barrier_t implementation for Mac OS/X
-
-There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes
-this omission by providing a simple-minded barrier implementation based
-on a pair of pthread_mutex_t and pthread_cond_t.
-
diff --git a/3rdparty/libmdbx/test/stub/pthread_barrier.c b/3rdparty/libmdbx/test/stub/pthread_barrier.c
deleted file mode 100644
index 2e37900a6484fc4306f87af2f6476f7b6ee523b3..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/stub/pthread_barrier.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright (c) 2015, Aleksey Demakov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "pthread_barrier.h"
-
-#include <errno.h>
-
-int pthread_barrierattr_init(pthread_barrierattr_t *attr) {
- memset(attr, 0, sizeof(pthread_barrierattr_t));
- int m = pthread_mutexattr_init(&attr->mattr);
- int c = pthread_condattr_init(&attr->cattr);
- return m ? m : c;
-}
-
-int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) {
- int c = pthread_condattr_destroy(&attr->cattr);
- int m = pthread_mutexattr_destroy(&attr->mattr);
- return m ? m : c;
-}
-
-int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
- int *__restrict pshared) {
- return pthread_condattr_getpshared(&attr->cattr, pshared);
-}
-
-int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) {
- int m = pthread_mutexattr_setpshared(&attr->mattr, pshared);
- int c = pthread_condattr_setpshared(&attr->cattr, pshared);
- return m ? m : c;
-}
-
-int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
- const pthread_barrierattr_t *__restrict attr,
- unsigned count) {
- if (count == 0)
- return errno = EINVAL;
-
- int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0);
- if (rc)
- return rc;
-
- rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0);
- if (rc) {
- int errno_save = errno;
- pthread_mutex_destroy(&barrier->mutex);
- errno = errno_save;
- return rc;
- }
-
- barrier->limit = count;
- barrier->count = 0;
- barrier->phase = 0;
- return 0;
-}
-
-int pthread_barrier_destroy(pthread_barrier_t *barrier) {
- pthread_mutex_destroy(&barrier->mutex);
- pthread_cond_destroy(&barrier->cond);
- return 0;
-}
-
-int pthread_barrier_wait(pthread_barrier_t *barrier) {
- int rc = pthread_mutex_lock(&barrier->mutex);
- if (rc)
- return rc;
-
- barrier->count++;
- if (barrier->count >= barrier->limit) {
- barrier->phase++;
- barrier->count = 0;
- pthread_cond_broadcast(&barrier->cond);
- pthread_mutex_unlock(&barrier->mutex);
- return PTHREAD_BARRIER_SERIAL_THREAD;
- } else {
- unsigned phase = barrier->phase;
- do
- pthread_cond_wait(&barrier->cond, &barrier->mutex);
- while (phase == barrier->phase);
- pthread_mutex_unlock(&barrier->mutex);
- return 0;
- }
-}
diff --git a/3rdparty/libmdbx/test/stub/pthread_barrier.h b/3rdparty/libmdbx/test/stub/pthread_barrier.h
deleted file mode 100644
index b9e0dd7c338904805a2b85feeb72febc6d022d5b..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/stub/pthread_barrier.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2015, Aleksey Demakov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef PTHREAD_BARRIER_H
-#define PTHREAD_BARRIER_H
-
-#include <pthread.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if !defined(PTHREAD_BARRIER_SERIAL_THREAD)
-#define PTHREAD_BARRIER_SERIAL_THREAD (1)
-#endif
-
-#if !defined(PTHREAD_PROCESS_PRIVATE)
-#define PTHREAD_PROCESS_PRIVATE (42)
-#endif
-#if !defined(PTHREAD_PROCESS_SHARED)
-#define PTHREAD_PROCESS_SHARED (43)
-#endif
-
-typedef struct {
- pthread_mutexattr_t mattr;
- pthread_condattr_t cattr;
-} pthread_barrierattr_t;
-
-typedef struct {
- pthread_mutex_t mutex;
- pthread_cond_t cond;
- unsigned int limit;
- unsigned int count;
- unsigned int phase;
-} pthread_barrier_t;
-
-int pthread_barrierattr_init(pthread_barrierattr_t *attr);
-int pthread_barrierattr_destroy(pthread_barrierattr_t *attr);
-
-int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
- int *__restrict pshared);
-int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared);
-
-int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
- const pthread_barrierattr_t *__restrict attr,
- unsigned int count);
-int pthread_barrier_destroy(pthread_barrier_t *barrier);
-
-int pthread_barrier_wait(pthread_barrier_t *barrier);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* PTHREAD_BARRIER_H */
diff --git a/3rdparty/libmdbx/test/test.cc b/3rdparty/libmdbx/test/test.cc
deleted file mode 100644
index fd8f987d5a03c841749788d8216288c2b21428eb..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/test.cc
+++ /dev/null
@@ -1,1252 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-
-const char *testcase2str(const actor_testcase testcase) {
- switch (testcase) {
- default:
- assert(false);
- return "?!";
- case ac_none:
- return "none";
- case ac_hill:
- return "hill";
- case ac_deadread:
- return "deadread";
- case ac_deadwrite:
- return "deadwrite";
- case ac_jitter:
- return "jitter";
- case ac_try:
- return "try";
- case ac_copy:
- return "copy";
- case ac_append:
- return "append";
- case ac_ttl:
- return "ttl";
- case ac_nested:
- return "nested";
- }
-}
-
-const char *status2str(actor_status status) {
- switch (status) {
- default:
- assert(false);
- return "?!";
- case as_debugging:
- return "debugging";
- case as_running:
- return "running";
- case as_successful:
- return "successful";
- case as_killed:
- return "killed";
- case as_failed:
- return "failed";
- case as_coredump:
- return "coredump";
- }
-}
-
-const char *keygencase2str(const keygen_case keycase) {
- switch (keycase) {
- default:
- assert(false);
- return "?!";
- case kc_random:
- return "random";
- case kc_dashes:
- return "dashes";
- case kc_custom:
- return "custom";
- }
-}
-
-//-----------------------------------------------------------------------------
-
-int testcase::hsr_callback(const MDBX_env *env, const MDBX_txn *txn,
- mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard,
- unsigned gap, size_t space,
- int retry) MDBX_CXX17_NOEXCEPT {
- (void)txn;
- testcase *self = (testcase *)mdbx_env_get_userctx(env);
-
- if (retry == 0)
- log_notice("hsr_callback: waitfor pid %lu, thread %" PRIuPTR
- ", txn #%" PRIu64 ", gap %d, space %zu",
- (long)pid, (size_t)tid, laggard, gap, space);
-
- MDBX_envinfo info;
- int rc = mdbx_env_info_ex(env, txn, &info, sizeof(info));
- if (rc != MDBX_SUCCESS)
- return rc;
-
- if (self->should_continue(true) &&
- (space > size_t(info.mi_geo.grow) * 2 ||
- info.mi_geo.current >= info.mi_geo.upper)) {
- osal_yield();
- if (retry > 0)
- osal_udelay(retry * 100);
- return MDBX_RESULT_FALSE /* retry / wait until reader done */;
- }
-
- /* allow growth or MDBX_MAP_FULL */
- return MDBX_RESULT_TRUE;
-}
-
-void testcase::db_prepare() {
- log_trace(">> db_prepare");
- assert(!db_guard);
-
- MDBX_env *env = nullptr;
- int rc = mdbx_env_create(&env);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_create()", rc);
-
- assert(env != nullptr);
- db_guard.reset(env);
-
- rc = mdbx_env_set_userctx(env, this);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_set_userctx()", rc);
-
- rc = mdbx_env_set_maxreaders(env, config.params.max_readers);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_set_maxreaders()", rc);
-
- rc = mdbx_env_set_maxdbs(env, config.params.max_tables);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_set_maxdbs()", rc);
-
- rc = mdbx_env_set_hsr(env, testcase::hsr_callback);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_set_hsr()", rc);
-
- rc = mdbx_env_set_geometry(
- env, config.params.size_lower, config.params.size_now,
- config.params.size_upper, config.params.growth_step,
- config.params.shrink_threshold, config.params.pagesize);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_set_mapsize()", rc);
-
- log_trace("<< db_prepare");
-}
-
-void testcase::db_open() {
- log_trace(">> db_open");
-
- if (!db_guard)
- db_prepare();
-
- jitter_delay(true);
-
- MDBX_env_flags_t mode = config.params.mode_flags;
- if (config.params.random_writemap && flipcoin())
- mode ^= MDBX_WRITEMAP;
-
- actual_env_mode = mode;
- int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(),
- mode, 0640);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_env_open()", rc);
-
- rc = mdbx_env_set_syncperiod(db_guard.get(), unsigned(0.042 * 65536));
- if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_BUSY)
- failure_perror("mdbx_env_set_syncperiod()", rc);
-
- rc = mdbx_env_set_syncbytes(db_guard.get(), INT_MAX / 421);
- if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_BUSY)
- failure_perror("mdbx_env_set_syncbytes()", rc);
-
- log_trace("<< db_open");
-}
-
-void testcase::db_close() {
- log_trace(">> db_close");
- cursor_guard.reset();
- txn_guard.reset();
- db_guard.reset();
- log_trace("<< db_close");
-}
-
-void testcase::txn_begin(bool readonly, MDBX_txn_flags_t flags) {
- assert((flags & MDBX_TXN_RDONLY) == 0);
- log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
- flags);
- assert(!txn_guard);
-
- MDBX_txn *txn = nullptr;
- int rc = mdbx_txn_begin(db_guard.get(), nullptr,
- readonly ? flags | MDBX_TXN_RDONLY : flags, &txn);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_txn_begin()", rc);
- txn_guard.reset(txn);
- need_speculum_assign = config.params.speculum && !readonly;
-
- log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
- flags);
-}
-
-int testcase::breakable_commit() {
- log_trace(">> txn_commit");
- assert(txn_guard);
-
- /* CLANG/LLVM C++ library could stupidly copy std::set<> item-by-item,
- * i.e. with insertion(s) & comparison(s), which will cause null dereference
- * during call mdbx_cmp() with zero txn. So it is the workaround for this:
- * - explicitly make copies of the `speculums`;
- * - explicitly move relevant copy after transaction commit. */
- SET speculum_committed_copy(ItemCompare(this)),
- speculum_copy(ItemCompare(this));
- if (need_speculum_assign) {
- speculum_committed_copy = speculum_committed;
- speculum_copy = speculum;
- }
-
- MDBX_txn *txn = txn_guard.release();
- txn_inject_writefault(txn);
- int rc = mdbx_txn_commit(txn);
- if (unlikely(rc != MDBX_SUCCESS) &&
- (rc != MDBX_MAP_FULL || !config.params.ignore_dbfull))
- failure_perror("mdbx_txn_commit()", rc);
-
- if (need_speculum_assign) {
- need_speculum_assign = false;
- if (unlikely(rc != MDBX_SUCCESS))
- speculum = std::move(speculum_committed_copy);
- else
- speculum_committed = std::move(speculum_copy);
- }
-
- log_trace("<< txn_commit: %s", rc ? "failed" : "Ok");
- return rc;
-}
-
-unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const {
- if (txn) {
- MDBX_txn_info info;
- int err = mdbx_txn_info(txn, &info, false);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_txn_info()", err);
- const size_t left = size_t(info.txn_space_leftover);
- const size_t total =
- size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty);
- return (unsigned)(left / (total >> 8));
- }
- return 0;
-}
-
-void testcase::txn_end(bool abort) {
- log_trace(">> txn_end(%s)", abort ? "abort" : "commit");
- assert(txn_guard);
-
- MDBX_txn *txn = txn_guard.release();
- if (abort) {
- int err = mdbx_txn_abort(txn);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_txn_abort()", err);
- if (need_speculum_assign)
- speculum = speculum_committed;
- } else {
- txn_inject_writefault(txn);
- int err = mdbx_txn_commit(txn);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_txn_commit()", err);
- if (need_speculum_assign)
- speculum_committed = speculum;
- }
-
- log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
-}
-
-void testcase::cursor_open(MDBX_dbi handle) {
- log_trace(">> cursor_open(%u)", handle);
- assert(!cursor_guard);
- assert(txn_guard);
-
- MDBX_cursor *cursor = nullptr;
- int rc = mdbx_cursor_open(txn_guard.get(), handle, &cursor);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_open()", rc);
- cursor_guard.reset(cursor);
-
- log_trace("<< cursor_open(%u)", handle);
-}
-
-void testcase::cursor_close() {
- log_trace(">> cursor_close()");
- assert(cursor_guard);
- MDBX_cursor *cursor = cursor_guard.release();
- mdbx_cursor_close(cursor);
- log_trace("<< cursor_close()");
-}
-
-void testcase::cursor_renew() {
- log_trace(">> cursor_renew()");
- assert(cursor_guard);
- int err = mdbx_cursor_renew(txn_guard.get(), cursor_guard.get());
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_renew()", err);
- log_trace("<< cursor_renew()");
-}
-
-int testcase::breakable_restart() {
- int rc = MDBX_SUCCESS;
- if (txn_guard)
- rc = breakable_commit();
- txn_begin(false, MDBX_TXN_READWRITE);
- if (cursor_guard)
- cursor_renew();
- return rc;
-}
-
-void testcase::txn_restart(bool abort, bool readonly, MDBX_txn_flags_t flags) {
- if (txn_guard)
- txn_end(abort);
- txn_begin(readonly, flags);
- if (cursor_guard)
- cursor_renew();
-}
-
-void testcase::txn_inject_writefault(void) {
- if (txn_guard)
- txn_inject_writefault(txn_guard.get());
-}
-
-void testcase::txn_inject_writefault(MDBX_txn *txn) {
- if (config.params.inject_writefaultn && txn) {
- if (config.params.inject_writefaultn <= nops_completed &&
- (MDBX_txn_flags_t(mdbx_txn_flags(txn)) & MDBX_TXN_RDONLY) == 0) {
- log_verbose(
- "== txn_inject_writefault(): got %u nops or more, inject FAULT",
- config.params.inject_writefaultn);
- log_flush();
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
- TerminateProcess(GetCurrentProcess(), 42);
-#else
- raise(SIGKILL);
-#endif
- }
- }
-}
-
-bool testcase::wait4start() {
- if (config.wait4id) {
- log_trace(">> wait4start(%u)", config.wait4id);
- assert(!global::singlemode);
- int rc = osal_waitfor(config.wait4id);
- if (rc) {
- log_trace("<< wait4start(%u), failed %s", config.wait4id,
- test_strerror(rc));
- return false;
- }
- } else {
- log_trace("== skip wait4start: not needed");
- }
-
- if (config.params.delaystart) {
- int rc = osal_delay(config.params.delaystart);
- if (rc) {
- log_trace("<< delay(%u), failed %s", config.params.delaystart,
- test_strerror(rc));
- return false;
- }
- } else {
- log_trace("== skip delay: not needed");
- }
-
- return true;
-}
-
-void testcase::kick_progress(bool active) const {
- if (!global::config::progress_indicator)
- return;
- logging::progress_canary(active);
-}
-
-void testcase::report(size_t nops_done) {
- assert(nops_done > 0);
- if (!nops_done)
- return;
-
- nops_completed += nops_done;
- log_debug("== complete +%" PRIuPTR " iteration, total %" PRIu64 " done",
- nops_done, nops_completed);
-
- kick_progress(true);
-
- if (config.signal_nops && !signalled &&
- config.signal_nops <= nops_completed) {
- log_trace(">> signal(n-ops %" PRIu64 ")", nops_completed);
- if (!global::singlemode)
- osal_broadcast(config.actor_id);
- signalled = true;
- log_trace("<< signal(n-ops %" PRIu64 ")", nops_completed);
- }
-}
-
-void testcase::signal() {
- if (!signalled) {
- log_trace(">> signal(forced)");
- if (!global::singlemode)
- osal_broadcast(config.actor_id);
- signalled = true;
- log_trace("<< signal(forced)");
- }
-}
-
-bool testcase::setup() {
- db_prepare();
- if (!wait4start())
- return false;
-
- start_timestamp = chrono::now_monotonic();
- nops_completed = 0;
- return true;
-}
-
-bool testcase::teardown() {
- log_trace(">> testcase::teardown");
- signal();
- db_close();
- log_trace("<< testcase::teardown");
- return true;
-}
-
-bool testcase::should_continue(bool check_timeout_only) const {
- bool result = true;
-
- if (config.params.test_duration) {
- chrono::time since;
- since.fixedpoint =
- chrono::now_monotonic().fixedpoint - start_timestamp.fixedpoint;
- if (since.seconds() >= config.params.test_duration)
- result = false;
- }
-
- if (!check_timeout_only && config.params.test_nops &&
- nops_completed >= config.params.test_nops)
- result = false;
-
- if (result)
- kick_progress(false);
-
- return result;
-}
-
-void testcase::fetch_canary() {
- MDBX_canary canary_now;
- log_trace(">> fetch_canary");
-
- int rc = mdbx_canary_get(txn_guard.get(), &canary_now);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_canary_get()", rc);
-
- if (canary_now.v < last.canary.v)
- failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64
- "(canary-last.v)",
- canary_now.v, last.canary.v);
- if (canary_now.y < last.canary.y)
- failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64
- "(canary-last.y)",
- canary_now.y, last.canary.y);
-
- last.canary = canary_now;
- log_trace("<< fetch_canary: db-sequence %" PRIu64
- ", db-sequence.txnid %" PRIu64,
- last.canary.y, last.canary.v);
-}
-
-void testcase::update_canary(uint64_t increment) {
- MDBX_canary canary_now = last.canary;
-
- log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y,
- increment);
- canary_now.y += increment;
-
- int rc = mdbx_canary_put(txn_guard.get(), &canary_now);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_canary_put()", rc);
-
- log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y);
-}
-
-bool testcase::is_handle_created_in_current_txn(const MDBX_dbi handle,
- MDBX_txn *txn) {
- unsigned flags, state;
- int err = mdbx_dbi_flags_ex(txn, handle, &flags, &state);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_dbi_flags_ex()", err);
- return (state & MDBX_DBI_CREAT) != 0;
-}
-
-int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) {
- db_open();
-
- int err, retry_left = 42;
- for (;;) {
- txn_begin(false);
- handle = db_table_open(true);
-
- if (is_handle_created_in_current_txn(handle, txn_guard.get()))
- return MDBX_SUCCESS;
- db_table_clear(handle);
- err = breakable_commit();
- if (likely(err == MDBX_SUCCESS)) {
- txn_begin(false);
- return MDBX_SUCCESS;
- }
- if (--retry_left == 0)
- break;
- jitter_delay(true);
- }
- log_notice("db_begin_table_create_open_clean: bailout due '%s'",
- mdbx_strerror(err));
- return err;
-}
-
-MDBX_dbi testcase::db_table_open(bool create) {
- log_trace(">> testcase::db_table_create");
-
- char tablename_buf[16];
- const char *tablename = nullptr;
- if (config.space_id) {
- int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u",
- config.space_id);
- if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1)
- failure("snprintf(tablename): %d", rc);
- tablename = tablename_buf;
- }
- log_debug("use %s table", tablename ? tablename : "MAINDB");
-
- MDBX_dbi handle = 0;
- int rc = mdbx_dbi_open(txn_guard.get(), tablename,
- (create ? MDBX_CREATE : MDBX_DB_DEFAULTS) |
- config.params.table_flags,
- &handle);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_dbi_open()", rc);
-
- log_trace("<< testcase::db_table_create, handle %u", handle);
- return handle;
-}
-
-void testcase::db_table_drop(MDBX_dbi handle) {
- log_trace(">> testcase::db_table_drop, handle %u", handle);
-
- if (config.params.drop_table) {
- int rc = mdbx_drop(txn_guard.get(), handle, true);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_drop(delete=true)", rc);
- speculum.clear();
- log_trace("<< testcase::db_table_drop");
- } else {
- log_trace("<< testcase::db_table_drop: not needed");
- }
-}
-
-void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) {
- log_trace(">> testcase::db_table_clear, handle %u", handle);
- int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_drop(delete=false)", rc);
- speculum.clear();
- log_trace("<< testcase::db_table_clear");
-}
-
-void testcase::db_table_close(MDBX_dbi handle) {
- log_trace(">> testcase::db_table_close, handle %u", handle);
- assert(!txn_guard);
- int rc = mdbx_dbi_close(db_guard.get(), handle);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_dbi_close()", rc);
- log_trace("<< testcase::db_table_close");
-}
-
-void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
- MDBX_val expected_valued) {
- MDBX_val actual_value = expected_valued;
- int rc = mdbx_get_equal_or_great(txn_guard.get(), handle, &key2check,
- &actual_value);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror(step, rc);
- if (!is_samedata(&actual_value, &expected_valued))
- failure("%s data mismatch", step);
-}
-
-//-----------------------------------------------------------------------------
-
-bool test_execute(const actor_config &config_const) {
- const mdbx_pid_t pid = osal_getpid();
- actor_config config = config_const;
-
- if (global::singlemode) {
- logging::setup(format("single_%s", testcase2str(config.testcase)));
- } else {
- logging::setup((logging::loglevel)config.params.loglevel,
- format("child_%u.%u", config.actor_id, config.space_id));
- log_trace(">> wait4barrier");
- osal_wait4barrier();
- log_trace("<< wait4barrier");
- }
-
- try {
- std::unique_ptr<testcase> test(registry::create_actor(config, pid));
- size_t iter = 0;
- do {
- iter++;
- if (!test->setup()) {
- log_notice("test setup failed");
- return false;
- }
- if (!test->run()) {
- log_notice("test failed");
- return false;
- }
- if (!test->teardown()) {
- log_notice("test teardown failed");
- return false;
- }
-
- if (config.params.nrepeat == 1)
- log_verbose("test successfully");
- else {
- if (config.params.nrepeat)
- log_verbose("test successfully (iteration %zi of %zi)", iter,
- size_t(config.params.nrepeat));
- else
- log_verbose("test successfully (iteration %zi)", iter);
- config.params.keygen.seed += INT32_C(0xA4F4D37B);
- log_verbose("turn keygen to %u", config.params.keygen.seed);
- }
-
- } while (config.params.nrepeat == 0 || iter < config.params.nrepeat);
- return true;
- } catch (const std::exception &pipets) {
- failure("***** Exception: %s *****", pipets.what());
- return false;
- }
-}
-
-//-----------------------------------------------------------------------------
-
-enum speculum_cursors : int {
- lowerbound = 0,
- prev = 1,
- prev_prev = 2,
- next = 3,
- next_next = 4,
- seek_check = 5
-};
-
-bool testcase::is_same(const Item &a, const Item &b) const {
- if (!is_samedata(dataview2iov(a.first), dataview2iov(b.first)))
- return false;
- if ((config.params.table_flags & MDBX_DUPSORT) &&
- !is_samedata(dataview2iov(a.second), dataview2iov(b.second)))
- return false;
- return true;
-}
-
-bool testcase::is_same(const testcase::SET::const_iterator &it,
- const MDBX_val &k, const MDBX_val &v) const {
-
- return is_samedata(dataview2iov(it->first), k) &&
- is_samedata(dataview2iov(it->second), v);
-}
-
-void testcase::verbose(const char *where, const char *stage,
- const testcase::SET::const_iterator &it) const {
- if (it == speculum.end())
- log_verbose("speculum-%s: %s expect END", where, stage);
- else {
- char dump_key[32], dump_value[32];
- MDBX_val it_key = dataview2iov(it->first);
- MDBX_val it_data = dataview2iov(it->second);
- log_verbose("speculum-%s: %s expect {%s, %s}", where, stage,
- mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
- }
-}
-
-void testcase::verbose(const char *where, const char *stage, const MDBX_val &k,
- const MDBX_val &v, int err) const {
- char dump_key[32], dump_value[32];
- if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE)
- log_verbose("speculum-%s: %s cursor {%d, %s}", where, stage, err,
- mdbx_strerror(err));
- else
- log_verbose("speculum-%s: %s cursor {%s, %s}", where, stage,
- mdbx_dump_val(&k, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&v, dump_value, sizeof(dump_value)));
-}
-
-void testcase::speculum_check_iterator(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- const MDBX_val &k,
- const MDBX_val &v) const {
- char dump_key[32], dump_value[32];
- MDBX_val it_key = dataview2iov(it->first);
- MDBX_val it_data = dataview2iov(it->second);
- // log_verbose("speculum-%s: %s expect {%s, %s}", where, stage,
- // mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
- // mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
- if (!is_samedata(it_key, k))
- failure("speculum-%s: %s key mismatch %s (must) != %s", where, stage,
- mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&k, dump_value, sizeof(dump_value)));
- if (!is_samedata(it_data, v))
- failure("speculum-%s: %s data mismatch %s (must) != %s", where, stage,
- mdbx_dump_val(&it_data, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&v, dump_value, sizeof(dump_value)));
-}
-
-#if SPECULUM_CURSORS
-void testcase::speculum_check_cursor(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- int cursor_err, const MDBX_val &cursor_key,
- const MDBX_val &cursor_data) const {
- // verbose(where, stage, cursor_key, cursor_data, cursor_err);
- // verbose(where, stage, it);
- if (cursor_err != MDBX_SUCCESS && cursor_err != MDBX_NOTFOUND &&
- cursor_err != MDBX_RESULT_TRUE && cursor_err != MDBX_ENODATA)
- failure("speculum-%s: %s %s %d %s", where, stage, "cursor-get", cursor_err,
- mdbx_strerror(cursor_err));
-
- char dump_key[32], dump_value[32];
- if (it == speculum.end() && cursor_err != MDBX_NOTFOUND)
- failure("speculum-%s: %s extra pair {%s, %s}", where, stage,
- mdbx_dump_val(&cursor_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&cursor_data, dump_value, sizeof(dump_value)));
- else if (it != speculum.end() && cursor_err == MDBX_NOTFOUND) {
- MDBX_val it_key = dataview2iov(it->first);
- MDBX_val it_data = dataview2iov(it->second);
- failure("speculum-%s: %s lack pair {%s, %s}", where, stage,
- mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
- } else if (cursor_err == MDBX_SUCCESS || cursor_err == MDBX_RESULT_TRUE)
- speculum_check_iterator(where, stage, it, cursor_key, cursor_data);
-}
-
-void testcase::speculum_check_cursor(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- MDBX_cursor *cursor,
- const MDBX_cursor_op op) const {
- MDBX_val cursor_key = {0, 0};
- MDBX_val cursor_data = {0, 0};
- int err;
- if (it != speculum.end() && std::next(it) == speculum.end() &&
- op == MDBX_PREV && (config.params.table_flags & MDBX_DUPSORT)) {
- /* Workaround for MDBX/LMDB flaw */
- err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, MDBX_LAST);
- if (err == MDBX_SUCCESS)
- err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, MDBX_LAST_DUP);
- } else
- err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, op);
- return speculum_check_cursor(where, stage, it, err, cursor_key, cursor_data);
-}
-
-void testcase::speculum_prepare_cursors(const Item &item) {
- int err;
- assert(config.params.speculum);
- if (speculum_cursors[lowerbound])
- for (auto &guard : speculum_cursors) {
- if (txn_guard.get() != mdbx_cursor_txn(guard.get()) ||
- dbi != mdbx_cursor_dbi(guard.get())) {
- err = mdbx_cursor_bind(txn_guard.get(), guard.get(), dbi);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_bind()", err);
- }
- }
- else
- for (auto &guard : speculum_cursors) {
- MDBX_cursor *cursor = nullptr;
- err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
- if (unlikely(err != MDBX_SUCCESS))
- failure_perror("mdbx_cursor_open()", err);
- guard.reset(cursor);
- }
-
- const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
- const MDBX_val item_key = dataview2iov(item.first),
- item_data = dataview2iov(item.second);
- MDBX_val lowerbound_key = item_key;
- MDBX_val lowerbound_data = item_data;
- // verbose("prepare-cursors", "item", item_key, item_data);
- err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key, &lowerbound_data,
- MDBX_SET_LOWERBOUND);
- // verbose("prepare-cursors", "lowerbound", lowerbound_key, lowerbound_data,
- // err);
- if (unlikely(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE &&
- err != MDBX_NOTFOUND))
- failure("speculum-%s: %s %s %d %s", "prepare-cursors", "lowerbound",
- "cursor-get", err, mdbx_strerror(err));
-
- auto it_lowerbound = speculum.lower_bound(item);
- // verbose("prepare-cursors", "lowerbound", it_lowerbound);
- speculum_check_cursor("prepare-cursors", "lowerbound", it_lowerbound, err,
- lowerbound_key, lowerbound_data);
-
- const auto cursor_prev = speculum_cursors[prev].get();
- err = mdbx_cursor_copy(cursor_lowerbound, cursor_prev);
- if (unlikely(err != MDBX_SUCCESS))
- failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev",
- "cursor-copy", err, mdbx_strerror(err));
- auto it_prev = it_lowerbound;
- if (it_prev != speculum.begin()) {
- speculum_check_cursor("prepare-cursors", "prev", --it_prev, cursor_prev,
- MDBX_PREV);
- } else if ((err = mdbx_cursor_on_first(cursor_prev)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev", err,
- mdbx_strerror(err));
-
- const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
- err = mdbx_cursor_copy(cursor_prev, cursor_prev_prev);
- if (unlikely(err != MDBX_SUCCESS))
- failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev-prev",
- "cursor-copy", err, mdbx_strerror(err));
- auto it_prev_prev = it_prev;
- if (it_prev_prev != speculum.begin()) {
- speculum_check_cursor("prepare-cursors", "prev-prev", --it_prev_prev,
- cursor_prev_prev, MDBX_PREV);
- } else if ((err = mdbx_cursor_on_first(cursor_prev_prev)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev-prev",
- err, mdbx_strerror(err));
-
- const auto cursor_next = speculum_cursors[next].get();
- err = mdbx_cursor_copy(cursor_lowerbound, cursor_next);
- if (unlikely(err != MDBX_SUCCESS))
- failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next",
- "cursor-copy", err, mdbx_strerror(err));
- auto it_next = it_lowerbound;
- if (it_next != speculum.end()) {
- speculum_check_cursor("prepare-cursors", "next", ++it_next, cursor_next,
- MDBX_NEXT);
- } else if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next", err,
- mdbx_strerror(err));
-
- const auto cursor_next_next = speculum_cursors[next_next].get();
- err = mdbx_cursor_copy(cursor_next, cursor_next_next);
- if (unlikely(err != MDBX_SUCCESS))
- failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next-next",
- "cursor-copy", err, mdbx_strerror(err));
- auto it_next_next = it_next;
- if (it_next_next != speculum.end()) {
- speculum_check_cursor("prepare-cursors", "next-next", ++it_next_next,
- cursor_next_next, MDBX_NEXT);
- } else if ((err = mdbx_cursor_on_last(cursor_next_next)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next-next",
- err, mdbx_strerror(err));
-}
-#endif /* SPECULUM_CURSORS */
-
-int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata,
- MDBX_put_flags_t flags) {
- int err;
- bool rc = true;
- Item item;
-#if SPECULUM_CURSORS
- MDBX_cursor *check_seek_cursor = nullptr;
- MDBX_val seek_check_key, seek_check_data;
- int seek_check_err = 42;
-#endif /* SPECULUM_CURSORS */
- if (config.params.speculum) {
- item.first = iov2dataview(akey);
- item.second = iov2dataview(adata);
-#if SPECULUM_CURSORS
- speculum_prepare_cursors(item);
- check_seek_cursor = speculum_cursors[seek_check].get();
- seek_check_key = akey->value;
- seek_check_data = adata->value;
- seek_check_err = mdbx_cursor_get(
- check_seek_cursor, &seek_check_key, &seek_check_data,
- (config.params.table_flags & MDBX_DUPSORT) ? MDBX_GET_BOTH
- : MDBX_SET_KEY);
- if (seek_check_err != MDBX_SUCCESS && seek_check_err != MDBX_NOTFOUND)
- failure("speculum-%s: %s pre-insert %d %s", "insert", "seek",
- seek_check_err, mdbx_strerror(seek_check_err));
-#endif /* SPECULUM_CURSORS */
- }
-
- err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
- if (err != MDBX_SUCCESS && err != MDBX_KEYEXIST)
- return err;
-
- if (config.params.speculum) {
- char dump_key[32], dump_value[32];
- const auto insertion_result = speculum.insert(item);
- if (err == MDBX_KEYEXIST && insertion_result.second) {
- log_error("speculum.insert: unexpected %s {%s, %s}", "MDBX_KEYEXIST",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
- rc = false;
- }
- if (err == MDBX_SUCCESS && !insertion_result.second) {
- log_error("speculum.insert: unexpected %s {%s, %s}", "MDBX_SUCCESS",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
- rc = false;
- }
-
-#if SPECULUM_CURSORS
- if (insertion_result.second) {
- if (seek_check_err != MDBX_NOTFOUND) {
- log_error(
- "speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err,
- mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value)));
- rc = false;
- }
- } else {
- if (seek_check_err != MDBX_SUCCESS) {
- log_error(
- "speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err,
- mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value)));
- speculum_check_iterator("insert", "pre-seek", insertion_result.first,
- seek_check_key, seek_check_data);
- rc = false;
- }
- }
-
- if (insertion_result.first != speculum.begin()) {
- const auto cursor_prev = speculum_cursors[prev].get();
- auto it_prev = insertion_result.first;
- speculum_check_cursor("after-insert", "prev", --it_prev, cursor_prev,
- MDBX_GET_CURRENT);
- if (it_prev != speculum.begin()) {
- const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
- auto it_prev_prev = it_prev;
- speculum_check_cursor("after-insert", "prev-prev", --it_prev_prev,
- cursor_prev_prev, MDBX_GET_CURRENT);
- }
- }
-
- auto it_lowerbound = insertion_result.first;
- if (++it_lowerbound != speculum.end()) {
- const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
- speculum_check_cursor("after-insert", "lowerbound", it_lowerbound,
- cursor_lowerbound, MDBX_GET_CURRENT);
-
- auto it_next = it_lowerbound;
- if (++it_next != speculum.end()) {
- const auto cursor_next = speculum_cursors[next].get();
- speculum_check_cursor("after-insert", "next", it_next, cursor_next,
- MDBX_GET_CURRENT);
-
- auto it_next_next = it_next;
- if (++it_next_next != speculum.end()) {
- const auto cursor_next_next = speculum_cursors[next_next].get();
- speculum_check_cursor("after-insert", "next-next", it_next_next,
- cursor_next_next, MDBX_GET_CURRENT);
- }
- }
- }
-#endif /* SPECULUM_CURSORS */
- }
-
- return rc ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
-}
-
-int testcase::replace(const keygen::buffer &akey,
- const keygen::buffer &new_data,
- const keygen::buffer &old_data, MDBX_put_flags_t flags) {
- if (config.params.speculum) {
- const auto S_key = iov2dataview(akey);
- const auto S_old = iov2dataview(old_data);
- const auto S_new = iov2dataview(new_data);
- const auto removed = speculum.erase(SET::key_type(S_key, S_old));
- if (unlikely(removed != 1)) {
- char dump_key[128], dump_value[128];
- log_error(
- "speculum-%s: %s old value {%s, %s}", "replace",
- (removed > 1) ? "multi" : "no",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&old_data->value, dump_value, sizeof(dump_value)));
- }
- if (unlikely(!speculum.emplace(S_key, S_new).second)) {
- char dump_key[128], dump_value[128];
- log_error(
- "speculum-replace: new pair not inserted {%s, %s}",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&new_data->value, dump_value, sizeof(dump_value)));
- }
- }
- return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
- &old_data->value, flags);
-}
-
-int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) {
- int err;
- bool rc = true;
- Item item;
- if (config.params.speculum) {
- item.first = iov2dataview(akey);
- item.second = iov2dataview(adata);
-#if SPECULUM_CURSORS
- speculum_prepare_cursors(item);
-#endif /* SPECULUM_CURSORS */
- }
-
- err = mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
- if (err != MDBX_NOTFOUND && err != MDBX_SUCCESS)
- return err;
-
- if (config.params.speculum) {
- char dump_key[32], dump_value[32];
- const auto it_found = speculum.find(item);
- if (it_found == speculum.end()) {
- if (err != MDBX_NOTFOUND) {
- log_error("speculum.remove: unexpected %s {%s, %s}", "MDBX_SUCCESS",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
- rc = false;
- }
- } else {
- if (err != MDBX_SUCCESS) {
- log_error("speculum.remove: unexpected %s {%s, %s}", "MDBX_NOTFOUND",
- mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
- rc = false;
- }
-
-#if SPECULUM_CURSORS
- if (it_found != speculum.begin()) {
- const auto cursor_prev = speculum_cursors[prev].get();
- auto it_prev = it_found;
- speculum_check_cursor("after-remove", "prev", --it_prev, cursor_prev,
- MDBX_GET_CURRENT);
- if (it_prev != speculum.begin()) {
- const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
- auto it_prev_prev = it_prev;
- speculum_check_cursor("after-remove", "prev-prev", --it_prev_prev,
- cursor_prev_prev, MDBX_GET_CURRENT);
- }
- }
-
- auto it_next = it_found;
- const auto cursor_next = speculum_cursors[next].get();
- const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
- if (++it_next != speculum.end()) {
- speculum_check_cursor("after-remove", "next", it_next, cursor_next,
- MDBX_GET_CURRENT);
- speculum_check_cursor("after-remove", "lowerbound", it_next,
- cursor_lowerbound, MDBX_NEXT);
-
- auto it_next_next = it_next;
- const auto cursor_next_next = speculum_cursors[next_next].get();
- if (++it_next_next != speculum.end()) {
- speculum_check_cursor("after-remove", "next-next", it_next_next,
- cursor_next_next, MDBX_GET_CURRENT);
- } else if ((err = mdbx_cursor_on_last(cursor_next_next)) !=
- MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-last %d %s", "after-remove", "next-next",
- err, mdbx_strerror(err));
- } else {
- if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-last %d %s", "after-remove", "next", err,
- mdbx_strerror(err));
- if ((err = mdbx_cursor_on_last(cursor_lowerbound)) != MDBX_RESULT_TRUE)
- failure("speculum-%s: %s on-last %d %s", "after-remove", "lowerbound",
- err, mdbx_strerror(err));
- }
-#endif /* SPECULUM_CURSORS */
-
- speculum.erase(it_found);
- }
- }
-
- return rc ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
-}
-
-bool testcase::speculum_verify() {
- if (!config.params.speculum)
- return true;
-
- if (!txn_guard)
- txn_begin(true);
-
- char dump_key[128], dump_value[128];
- char dump_mkey[128], dump_mvalue[128];
-
- MDBX_cursor *cursor;
- int eof, err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
- if (err != MDBX_SUCCESS)
- failure_perror("mdbx_cursor_open()", err);
-
- bool rc = true;
- MDBX_val akey, avalue;
- MDBX_val mkey, mvalue;
- err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
- if (err == MDBX_NOTFOUND) {
- err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_GET_CURRENT);
- if (err == MDBX_ENODATA)
- err = MDBX_NOTFOUND;
- else {
- log_error("unexpected %d for MDBX_GET_CURRENT on empty DB", err);
- rc = false;
- }
- }
-
- unsigned extra = 0, lost = 0, n = 0;
- assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this)));
- auto it = speculum.cbegin();
- while (true) {
- if (err != MDBX_SUCCESS) {
- akey.iov_len = avalue.iov_len = 0;
- akey.iov_base = avalue.iov_base = nullptr;
- } else {
- eof = mdbx_cursor_eof(cursor);
- if (eof != MDBX_RESULT_FALSE) {
- log_error("false-positive cursor-eof %u/%u: db{%s, %s}, rc %i", n,
- extra, mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), eof);
- rc = false;
- }
- }
- const auto S_key = iov2dataview(akey);
- const auto S_data = iov2dataview(avalue);
- if (it != speculum.cend()) {
- mkey = it->first;
- mvalue = it->second;
- }
- if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first &&
- S_data == it->second) {
- ++it;
- err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
- } else if (err == MDBX_SUCCESS &&
- (it == speculum.cend() || S_key < it->first ||
- (S_key == it->first && S_data < it->second))) {
- extra += 1;
- if (it != speculum.cend()) {
- log_error("extra pair %u/%u: db{%s, %s} < mi{%s, %s}", n, extra,
- mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
- mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
- mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
- } else {
- log_error("extra pair %u/%u: db{%s, %s} < mi.END", n, extra,
- mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
- }
- err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
- rc = false;
- } else if (it != speculum.cend() &&
- (err == MDBX_NOTFOUND || S_key > it->first ||
- (S_key == it->first && S_data > it->second))) {
- lost += 1;
- if (err == MDBX_NOTFOUND) {
- log_error("lost pair %u/%u: db.END > mi{%s, %s}", n, lost,
- mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
- mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
- } else {
- log_error("lost pair %u/%u: db{%s, %s} > mi{%s, %s}", n, lost,
- mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
- mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
- mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
- }
- ++it;
- rc = false;
- } else if (err == MDBX_NOTFOUND && it == speculum.cend()) {
- break;
- } else if (err != MDBX_SUCCESS) {
- failure_perror("mdbx_cursor_get()", err);
- } else {
- assert(!"WTF?");
- }
- n += 1;
- }
-
- if (err == MDBX_NOTFOUND) {
- eof = mdbx_cursor_eof(cursor);
- if (eof != MDBX_RESULT_TRUE) {
- eof = mdbx_cursor_eof(cursor);
- log_error("false-negative cursor-eof: %u, rc %i", n, eof);
- rc = false;
- }
- }
- mdbx_cursor_close(cursor);
- return rc;
-}
-
-bool testcase::check_batch_get() {
- char dump_key[128], dump_value[128];
- char dump_key_batch[128], dump_value_batch[128];
-
- MDBX_cursor *cursor;
- int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
- if (err != MDBX_SUCCESS)
- failure_perror("mdbx_cursor_open()", err);
-
- MDBX_cursor *batch_cursor;
- err = mdbx_cursor_open(txn_guard.get(), dbi, &batch_cursor);
- if (err != MDBX_SUCCESS)
- failure_perror("mdbx_cursor_open()", err);
-
- MDBX_val pairs[42];
- size_t count = 0xDeadBeef;
- err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, ARRAY_LENGTH(pairs),
- MDBX_FIRST);
- bool rc = true;
- size_t i, n = 0;
- while (err == MDBX_SUCCESS) {
- for (i = 0; i < count; i += 2) {
- mdbx::slice k, v;
- int err2 = mdbx_cursor_get(cursor, &k, &v, MDBX_NEXT);
- if (err2 != MDBX_SUCCESS)
- failure_perror("mdbx_cursor_open()", err2);
- if (k != pairs[i] || v != pairs[i + 1]) {
- log_error(
- "batch-get pair mismatch %zu/%zu: sequential{%s, %s} != "
- "batch{%s, %s}",
- n + i / 2, i, mdbx_dump_val(&k, dump_key, sizeof(dump_key)),
- mdbx_dump_val(&v, dump_value, sizeof(dump_value)),
- mdbx_dump_val(&pairs[i], dump_key_batch, sizeof(dump_key_batch)),
- mdbx_dump_val(&pairs[i + 1], dump_value_batch,
- sizeof(dump_value_batch)));
- rc = false;
- }
- }
- n += i / 2;
- err = mdbx_cursor_get_batch(batch_cursor, &count, pairs,
- ARRAY_LENGTH(pairs), MDBX_NEXT);
- }
- if (err != MDBX_NOTFOUND)
- failure_perror("mdbx_cursor_get_batch()", err);
-
- err = mdbx_cursor_eof(batch_cursor);
- if (err != MDBX_RESULT_TRUE) {
- log_error("batch-get %s cursor not-eof %d", "batch", err);
- rc = false;
- }
- err = mdbx_cursor_on_last(batch_cursor);
- if (err != MDBX_RESULT_TRUE) {
- log_error("batch-get %s cursor not-on-last %d", "batch", err);
- rc = false;
- }
-
- err = mdbx_cursor_on_last(cursor);
- if (err != MDBX_RESULT_TRUE) {
- log_error("batch-get %s cursor not-on-last %d", "checked", err);
- rc = false;
- }
- mdbx_cursor_close(cursor);
- mdbx_cursor_close(batch_cursor);
- return rc;
-}
diff --git a/3rdparty/libmdbx/test/test.h b/3rdparty/libmdbx/test/test.h
deleted file mode 100644
index 40bb01ac2c82b0b4467de87a19dc2e4875b8e1fc..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/test.h
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-
-#include "base.h"
-#include "chrono.h"
-#include "config.h"
-#include "keygen.h"
-#include "log.h"
-#include "osal.h"
-#include "utils.h"
-
-#include <deque>
-#include <set>
-#include <stack>
-#include <tuple>
-
-#ifndef HAVE_cxx17_std_string_view
-#if __cplusplus >= 201703L && __has_include(<string_view>)
-#include <string_view>
-#define HAVE_cxx17_std_string_view 1
-#else
-#define HAVE_cxx17_std_string_view 0
-#endif
-#endif /* HAVE_cxx17_std_string_view */
-
-#if HAVE_cxx17_std_string_view
-#include <string_view>
-#endif
-
-bool test_execute(const actor_config &config);
-std::string thunk_param(const actor_config &config);
-void testcase_setup(const char *casename, const actor_params ¶ms,
- unsigned &last_space_id);
-void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
- const char *space_id_cstr, actor_params params);
-void keycase_setup(const char *casename, actor_params ¶ms);
-
-namespace global {
-
-extern const char thunk_param_prefix[];
-extern std::vector<actor_config> actors;
-extern std::unordered_map<unsigned, actor_config *> events;
-extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
-extern std::set<std::string> databases;
-extern unsigned nactors;
-extern chrono::time start_monotonic;
-extern chrono::time deadline_monotonic;
-extern bool singlemode;
-
-namespace config {
-extern unsigned timeout_duration_seconds;
-extern bool dump_config;
-extern bool cleanup_before;
-extern bool cleanup_after;
-extern bool failfast;
-extern bool progress_indicator;
-extern bool console_mode;
-extern bool geometry_jitter;
-} /* namespace config */
-
-} /* namespace global */
-
-//-----------------------------------------------------------------------------
-
-struct db_deleter /* : public std::unary_function<void, MDBX_env *> */ {
- void operator()(MDBX_env *env) const { mdbx_env_close(env); }
-};
-
-struct txn_deleter /* : public std::unary_function<void, MDBX_txn *> */ {
- void operator()(MDBX_txn *txn) const {
- int rc = mdbx_txn_abort(txn);
- if (rc)
- log_trouble(__func__, "mdbx_txn_abort()", rc);
- }
-};
-
-struct cursor_deleter /* : public std::unary_function<void, MDBX_cursor *> */ {
- void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); }
-};
-
-using scoped_db_guard = std::unique_ptr<MDBX_env, db_deleter>;
-using scoped_txn_guard = std::unique_ptr<MDBX_txn, txn_deleter>;
-using scoped_cursor_guard = std::unique_ptr<MDBX_cursor, cursor_deleter>;
-
-//-----------------------------------------------------------------------------
-
-class testcase;
-
-class registry {
- struct record {
- actor_testcase id;
- std::string name;
- bool (*review_params)(actor_params &);
- testcase *(*constructor)(const actor_config &, const mdbx_pid_t);
- };
- std::unordered_map<std::string, const record *> name2id;
- std::unordered_map<int, const record *> id2record;
- static bool add(const record *item);
- static registry *instance();
-
-public:
- template <class TESTCASE> struct factory : public record {
- factory(const actor_testcase id, const char *name) {
- this->id = id;
- this->name = name;
- review_params = TESTCASE::review_params;
- constructor = [](const actor_config &config,
- const mdbx_pid_t pid) -> testcase * {
- return new TESTCASE(config, pid);
- };
- add(this);
- }
- };
- static bool review_actor_params(const actor_testcase id,
- actor_params ¶ms);
- static testcase *create_actor(const actor_config &config,
- const mdbx_pid_t pid);
-};
-
-#define REGISTER_TESTCASE(NAME) \
- static registry::factory<testcase_##NAME> gRegister_##NAME( \
- ac_##NAME, MDBX_STRINGIFY(NAME))
-
-class testcase {
-protected:
- using data_view = mdbx::slice;
- static inline data_view iov2dataview(const MDBX_val &v) {
- return (v.iov_base && v.iov_len)
- ? data_view(static_cast<const char *>(v.iov_base), v.iov_len)
- : data_view();
- }
- static inline data_view iov2dataview(const keygen::buffer &b) {
- return iov2dataview(b->value);
- }
-
- using Item = std::pair<::mdbx::buffer<>, ::mdbx::buffer<>>;
-
- static MDBX_val dataview2iov(const data_view &v) {
- MDBX_val r;
- r.iov_base = (void *)v.data();
- r.iov_len = v.size();
- return r;
- }
- struct ItemCompare {
- const testcase *context;
- ItemCompare(const testcase *owner) : context(owner) {
- /* The context->txn_guard may be empty/null here */
- }
-
- bool operator()(const Item &a, const Item &b) const {
- MDBX_val va = dataview2iov(a.first), vb = dataview2iov(b.first);
- assert(context->txn_guard.get() != nullptr);
- int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
- if (cmp == 0 &&
- (context->config.params.table_flags & MDBX_DUPSORT) != 0) {
- va = dataview2iov(a.second);
- vb = dataview2iov(b.second);
- cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
- }
- return cmp < 0;
- }
- };
-
- // for simplify the set<pair<key,value>>
- // is used instead of multimap<key,value>
- using SET = std::set<Item, ItemCompare>;
-
- const actor_config &config;
- const mdbx_pid_t pid;
-
- MDBX_dbi dbi{0};
- scoped_db_guard db_guard;
- scoped_txn_guard txn_guard;
- scoped_cursor_guard cursor_guard;
- bool signalled{false};
- bool need_speculum_assign{false};
-
- uint64_t nops_completed{0};
- chrono::time start_timestamp;
- keygen::buffer key;
- keygen::buffer data;
- keygen::maker keyvalue_maker;
-
- struct {
- MDBX_canary canary;
- } last;
-
- SET speculum{ItemCompare(this)}, speculum_committed{ItemCompare(this)};
-#ifndef SPECULUM_CURSORS
-#define SPECULUM_CURSORS 1
-#endif /* SPECULUM_CURSORS */
-#if SPECULUM_CURSORS
- scoped_cursor_guard speculum_cursors[5 + 1];
- void speculum_prepare_cursors(const Item &item);
- void speculum_check_cursor(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- int cursor_err, const MDBX_val &cursor_key,
- const MDBX_val &cursor_data) const;
- void speculum_check_cursor(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- MDBX_cursor *cursor,
- const MDBX_cursor_op op) const;
-#endif /* SPECULUM_CURSORS */
- void speculum_check_iterator(const char *where, const char *stage,
- const testcase::SET::const_iterator &it,
- const MDBX_val &k, const MDBX_val &v) const;
-
- void verbose(const char *where, const char *stage,
- const testcase::SET::const_iterator &it) const;
- void verbose(const char *where, const char *stage, const MDBX_val &k,
- const MDBX_val &v, int err = MDBX_SUCCESS) const;
-
- bool is_same(const Item &a, const Item &b) const;
- bool is_same(const SET::const_iterator &it, const MDBX_val &k,
- const MDBX_val &v) const;
-
- bool speculum_verify();
- bool check_batch_get();
- int insert(const keygen::buffer &akey, const keygen::buffer &adata,
- MDBX_put_flags_t flags);
- int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
- const keygen::buffer &old_value, MDBX_put_flags_t flags);
- int remove(const keygen::buffer &akey, const keygen::buffer &adata);
-
- static int hsr_callback(const MDBX_env *env, const MDBX_txn *txn,
- mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard,
- unsigned gap, size_t space,
- int retry) MDBX_CXX17_NOEXCEPT;
-
- MDBX_env_flags_t actual_env_mode{MDBX_ENV_DEFAULTS};
- bool is_nested_txn_available() const {
- return (actual_env_mode & MDBX_WRITEMAP) == 0;
- }
- void kick_progress(bool active) const;
- void db_prepare();
- void db_open();
- void db_close();
- void txn_begin(bool readonly, MDBX_txn_flags_t flags = MDBX_TXN_READWRITE);
- int breakable_commit();
- void txn_end(bool abort);
- int breakable_restart();
- void txn_restart(bool abort, bool readonly,
- MDBX_txn_flags_t flags = MDBX_TXN_READWRITE);
- void cursor_open(MDBX_dbi handle);
- void cursor_close();
- void cursor_renew();
- void txn_inject_writefault(void);
- void txn_inject_writefault(MDBX_txn *txn);
- void fetch_canary();
- void update_canary(uint64_t increment);
- void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
- MDBX_val expected_valued);
- unsigned txn_underutilization_x256(MDBX_txn *txn) const;
-
- MDBX_dbi db_table_open(bool create);
- void db_table_drop(MDBX_dbi handle);
- void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr);
- void db_table_close(MDBX_dbi handle);
- int db_open__begin__table_create_open_clean(MDBX_dbi &handle);
- bool is_handle_created_in_current_txn(const MDBX_dbi handle, MDBX_txn *txn);
-
- bool wait4start();
- void report(size_t nops_done);
- void signal();
- bool should_continue(bool check_timeout_only = false) const;
-
- void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key,
- keygen::buffer &out_value, keygen::serial_t data_age) {
- keyvalue_maker.pair(serial, out_key, out_value, data_age, false);
- }
-
- void generate_pair(const keygen::serial_t serial) {
- keyvalue_maker.pair(serial, key, data, 0, true);
- }
-
- bool mode_readonly() const {
- return (config.params.mode_flags & MDBX_RDONLY) ? true : false;
- }
-
-public:
- testcase(const actor_config &config, const mdbx_pid_t pid)
- : config(config), pid(pid) {
- start_timestamp.reset();
- memset(&last, 0, sizeof(last));
- }
-
- static bool review_params(actor_params ¶ms) {
- // silently fix key/data length for fixed-length modes
- if ((params.table_flags & MDBX_INTEGERKEY) &&
- params.keylen_min != params.keylen_max)
- params.keylen_min = params.keylen_max;
- if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) &&
- params.datalen_min != params.datalen_max)
- params.datalen_min = params.datalen_max;
- return true;
- }
-
- virtual bool setup();
- virtual bool run() { return true; }
- virtual bool teardown();
- virtual ~testcase() {}
-};
-
-//-----------------------------------------------------------------------------
-
-class testcase_ttl : public testcase {
- using inherited = testcase;
-
-protected:
- struct {
- unsigned max_window_size{0};
- unsigned max_step_size{0};
- } sliding;
- unsigned edge2window(uint64_t edge);
- unsigned edge2count(uint64_t edge);
-
-public:
- testcase_ttl(const actor_config &config, const mdbx_pid_t pid)
- : inherited(config, pid) {}
- bool setup() override;
- bool run() override;
-};
diff --git a/3rdparty/libmdbx/test/try.cc b/3rdparty/libmdbx/test/try.cc
deleted file mode 100644
index da81e6313a07614615c5892949da5ba6456b0b69..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/try.cc
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "test.h"
-
-class testcase_try : public testcase {
-public:
- testcase_try(const actor_config &config, const mdbx_pid_t pid)
- : testcase(config, pid) {}
- bool run() override;
-};
-REGISTER_TESTCASE(try);
-
-bool testcase_try::run() {
- db_open();
- assert(!txn_guard);
-
- MDBX_txn *txn = nullptr;
- MDBX_txn *txn2 = nullptr;
- int rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TXN_READWRITE, &txn);
- if (unlikely(rc != MDBX_SUCCESS))
- failure_perror("mdbx_txn_begin(MDBX_TXN_TRY)", rc);
- else {
- rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TXN_TRY, &txn2);
- if (unlikely(rc != MDBX_BUSY))
- failure_perror("mdbx_txn_begin(MDBX_TXN_TRY)", rc);
- }
-
- txn_guard.reset(txn);
- return true;
-}
diff --git a/3rdparty/libmdbx/test/ttl.cc b/3rdparty/libmdbx/test/ttl.cc
deleted file mode 100644
index 06498f5ee5b30c5680bbdef3756abb5fd24734aa..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/ttl.cc
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-#include <cmath>
-#include <deque>
-
-/* LY: теÑÑ‚ "ÑмулÑцией time-to-live":
- * - организуетÑÑ "ÑкользÑщее окно", которое двигаетÑÑ Ð²Ð¿ÐµÑ€ÐµÐ´ вдоль
- * чиÑловой оÑи каждую транзакцию.
- * - по переднему краю "ÑкользÑщего окна" запиÑи добавлÑÑŽÑ‚ÑÑ Ð² таблицу,
- * а по заднему удалÑÑŽÑ‚ÑÑ.
- * - количеÑтво добавлÑемых/удалÑемых запиÑей пÑевдоÑлучайно завиÑит
- * от номера транзакции, но Ñ ÑкÑпоненциальным раÑпределением.
- * - размер "ÑкользÑщего окна" также пÑевдоÑлучайно завиÑит от номера
- * транзакции Ñ "отрицательным" ÑкÑпоненциальным раÑпределением
- * MAX_WIDTH - exp(rnd(N)), при уменьшении окна ÑдвигаетÑÑ Ð·Ð°Ð´Ð½Ð¸Ð¹
- * край и удалÑÑŽÑ‚ÑÑ Ð·Ð°Ð¿Ð¸Ñи позади него.
- *
- * Таким образом имитируетÑÑ Ð¿Ð¾Ð²ÐµÐ´ÐµÐ½Ð¸Ðµ таблицы Ñ TTL: запиÑи ÑтохаÑтичеÑки
- * добавлÑÑŽÑ‚ÑÑ Ð¸ удалÑÑŽÑ‚ÑÑ, но изредка проиÑходит маÑÑивное удаление.
- */
-REGISTER_TESTCASE(ttl);
-
-unsigned testcase_ttl::edge2count(uint64_t edge) {
- const double rnd = u64_to_double1(prng64_map1_white(edge));
- const unsigned count = std::lrint(std::pow(sliding.max_step_size, rnd));
- // average value: (X - 1) / ln(X), where X = sliding.max_step_size
- return count;
-}
-
-unsigned testcase_ttl::edge2window(uint64_t edge) {
- const double rnd = u64_to_double1(bleach64(edge));
- const unsigned size = sliding.max_window_size -
- std::lrint(std::pow(sliding.max_window_size, rnd));
- // average value: Y - (Y - 1) / ln(Y), where Y = sliding.max_window_size
- return size;
-}
-
-static inline double estimate(const double x, const double y) {
- /* Ñреднее кол-во операций N = X' * Y', где X' и Y' Ñредние значениÑ
- * размера окна и кол-ва добавлÑемых за один шаг запиÑей:
- * X' = (X - 1) / ln(X), где X = sliding.max_step_size
- * Y' = Y - (Y - 1) / ln(Y), где Y = sliding.max_window_size */
- return (x - 1) / std::log(x) * (y - (y - 1) / std::log(y));
-}
-
-bool testcase_ttl::setup() {
- const unsigned window_top_lower =
- 7 /* нижний предел Ð´Ð»Ñ Ð²ÐµÑ€Ñ…Ð½ÐµÐ¹ границы диапазона, в котором будет
- ÑтохаÑтичеÑки колебатьÑÑ Ñ€Ð°Ð·Ð¼ÐµÑ€ окна */
- ;
- const unsigned count_top_lower =
- 7 /* нижний предел Ð´Ð»Ñ Ð²ÐµÑ€Ñ…Ð½ÐµÐ¹ границы диапазона, в котором будет
- ÑтохаÑтичеÑки колебатьÑÑ ÐºÐ¾Ð»-во запиÑей добавлÑемых на одном шаге */
- ;
-
- /* Ð´Ð»Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð¸Ð·Ð°Ñ†Ð¸Ð¸ иÑпользуем подходÑщие параметры,
- * которые не имеют здеÑÑŒ ÑмыÑла в первоначальном значении. */
- const double ratio =
- double(config.params.batch_read ? config.params.batch_read : 1) /
- double(config.params.batch_write ? config.params.batch_write : 1);
-
- /* проще найти двоичным поиÑком (Ð²Ð°Ñ€Ð¸Ð°Ñ†Ð¸Ñ Ð¼ÐµÑ‚Ð¾Ð´Ð° Ðьютона) */
- double hi = config.params.test_nops, lo = 1;
- double x = std::sqrt(hi + lo) / ratio;
- while (hi > lo) {
- const double n = estimate(x, x * ratio);
- if (n > config.params.test_nops)
- hi = x - 1;
- else
- lo = x + 1;
- x = (hi + lo) / 2;
- }
-
- sliding.max_step_size = std::lrint(x);
- if (sliding.max_step_size < count_top_lower)
- sliding.max_step_size = count_top_lower;
- sliding.max_window_size = std::lrint(x * ratio);
- if (sliding.max_window_size < window_top_lower)
- sliding.max_window_size = window_top_lower;
-
- while (estimate(sliding.max_step_size, sliding.max_window_size) >
- config.params.test_nops * 2.0) {
- if (ratio * sliding.max_step_size > sliding.max_window_size) {
- if (sliding.max_step_size < count_top_lower)
- break;
- sliding.max_step_size = sliding.max_step_size * 7 / 8;
- } else {
- if (sliding.max_window_size < window_top_lower)
- break;
- sliding.max_window_size = sliding.max_window_size * 7 / 8;
- }
- }
-
- log_verbose("come up window_max %u from `batch_read`",
- sliding.max_window_size);
- log_verbose("come up step_max %u from `batch_write`", sliding.max_step_size);
- return inherited::setup();
-}
-
-bool testcase_ttl::run() {
- int err = db_open__begin__table_create_open_clean(dbi);
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));
- return false;
- }
-
- uint64_t seed =
- prng64_map2_white(config.params.keygen.seed) + config.actor_id;
- keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
- key = keygen::alloc(config.params.keylen_max);
- data = keygen::alloc(config.params.datalen_max);
- const MDBX_put_flags_t insert_flags =
- (config.params.table_flags & MDBX_DUPSORT)
- ? MDBX_NODUPDATA
- : MDBX_NODUPDATA | MDBX_NOOVERWRITE;
-
- std::deque<std::pair<uint64_t, unsigned>> fifo;
- uint64_t serial = 0;
- bool rc = false;
- unsigned clear_wholetable_passed = 0;
- unsigned clear_stepbystep_passed = 0;
- unsigned dbfull_passed = 0;
- unsigned loops = 0;
- bool keyspace_overflow = false;
- while (true) {
- const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
-
- const unsigned window_width =
- (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt);
- unsigned head_count = edge2count(salt);
- log_debug("ttl: step #%" PRIu64 " (serial %" PRIu64
- ", window %u, count %u) salt %" PRIu64,
- nops_completed, serial, window_width, head_count, salt);
-
- if (window_width || flipcoin()) {
- clear_stepbystep_passed += window_width == 0;
- while (fifo.size() > window_width) {
- uint64_t tail_serial = fifo.back().first;
- const unsigned tail_count = fifo.back().second;
- log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial,
- tail_count);
- fifo.pop_back();
- for (unsigned n = 0; n < tail_count; ++n) {
- log_trace("ttl: remove-tail %" PRIu64, tail_serial);
- generate_pair(tail_serial);
- err = remove(key, data);
- if (unlikely(err != MDBX_SUCCESS)) {
- if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
- log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err));
- goto bailout;
- }
- failure_perror("mdbx_del(tail)", err);
- }
- if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
- failure("ttl: unexpected key-space overflow on the tail");
- }
- report(tail_count);
- }
- } else {
- log_trace("ttl: purge state");
- db_table_clear(dbi);
- fifo.clear();
- clear_wholetable_passed += 1;
- report(1);
- }
-
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err));
- break;
- }
- if (!speculum_verify()) {
- log_notice("ttl: bailout after tail-trim");
- return false;
- }
-
- if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed ||
- !clear_stepbystep_passed)) {
- unsigned underutilization_x256 =
- txn_underutilization_x256(txn_guard.get());
- if (dbfull_passed > underutilization_x256) {
- log_notice("ttl: skip head-grow to avoid one more dbfull (was %u, "
- "underutilization %.2f%%)",
- dbfull_passed, underutilization_x256 / 2.560);
- continue;
- }
- fifo.push_front(std::make_pair(serial, head_count));
- retry:
- for (unsigned n = 0; n < head_count; ++n) {
- log_trace("ttl: insert-head %" PRIu64, serial);
- generate_pair(serial);
- err = insert(key, data, insert_flags);
- if (unlikely(err != MDBX_SUCCESS)) {
- if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) &&
- config.params.ignore_dbfull) {
- log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
- txn_restart(true, false);
- serial = fifo.front().first;
- fifo.front().second = head_count = n;
- dbfull_passed += 1;
- goto retry;
- }
- failure_perror("mdbx_put(head)", err);
- }
-
- if (unlikely(!keyvalue_maker.increment(serial, 1))) {
- log_notice("ttl: unexpected key-space overflow");
- keyspace_overflow = true;
- txn_restart(true, false);
- serial = fifo.front().first;
- fifo.front().second = head_count = n;
- goto retry;
- }
- }
- err = breakable_restart();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
- serial = fifo.front().first;
- fifo.pop_front();
- }
- if (!speculum_verify()) {
- log_notice("ttl: bailout after head-grow");
- return false;
- }
- loops += 1;
- } else if (fifo.empty()) {
- log_notice("ttl: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items",
- loops, nops_completed, serial);
- rc = true;
- break;
- } else {
- log_notice("ttl: done, wait for empty, skip head-grow");
- }
- }
-
-bailout:
- if (!rc && err == MDBX_MAP_FULL && config.params.ignore_dbfull)
- rc = true;
- txn_end(true);
- if (dbi) {
- if (config.params.drop_table && !mode_readonly()) {
- txn_begin(false);
- db_table_drop(dbi);
- err = breakable_commit();
- if (unlikely(err != MDBX_SUCCESS)) {
- log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err));
- if (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)
- rc = false;
- }
- } else
- db_table_close(dbi);
- }
- return rc;
-}
diff --git a/3rdparty/libmdbx/test/utils.cc b/3rdparty/libmdbx/test/utils.cc
deleted file mode 100644
index 920d2ef6d9f867a33db513efd84d1e23b25c7dcf..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/utils.cc
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#include "test.h"
-#include <float.h>
-#if defined(HAVE_IEEE754_H) || __has_include(<ieee754.h>)
-#include <ieee754.h>
-#endif
-#if defined(__APPLE__) || defined(__MACH__)
-#include <mach/mach_time.h>
-#endif /* defined(__APPLE__) || defined(__MACH__) */
-
-std::string format(const char *fmt, ...) {
- va_list ap, ones;
- va_start(ap, fmt);
- va_copy(ones, ap);
-#ifdef _MSC_VER
- int needed = _vscprintf(fmt, ap);
-#else
- int needed = vsnprintf(nullptr, 0, fmt, ap);
-#endif
- assert(needed >= 0);
- va_end(ap);
- std::string result;
- result.reserve((size_t)needed + 1);
- result.resize((size_t)needed, '\0');
- int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones);
- assert(actual == needed);
- (void)actual;
- va_end(ones);
- return result;
-}
-
-std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) {
- std::string result;
- if (bytes > 0) {
- const uint8_t *data = (const uint8_t *)ptr;
- checksum.push(data, bytes);
- result.reserve(bytes * 2);
- const uint8_t *const end = data + bytes;
- do {
- char h = *data >> 4;
- char l = *data & 15;
- result.push_back((l < 10) ? l + '0' : l - 10 + 'a');
- result.push_back((h < 10) ? h + '0' : h - 10 + 'a');
- } while (++data < end);
- }
- assert(result.size() == bytes * 2);
- return result;
-}
-
-bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
- size_t bytes, simple_checksum &checksum) {
- if (bytes * 2 != (size_t)(hex_end - hex_begin))
- return false;
-
- uint8_t *data = (uint8_t *)ptr;
- for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) {
- unsigned l = hex[0], h = hex[1];
-
- if (l >= '0' && l <= '9')
- l = l - '0';
- else if (l >= 'A' && l <= 'F')
- l = l - 'A' + 10;
- else if (l >= 'a' && l <= 'f')
- l = l - 'a' + 10;
- else
- return false;
-
- if (h >= '0' && h <= '9')
- h = h - '0';
- else if (h >= 'A' && h <= 'F')
- h = h - 'A' + 10;
- else if (h >= 'a' && h <= 'f')
- h = h - 'a' + 10;
- else
- return false;
-
- uint32_t c = l + (h << 4);
- checksum.push(c);
- *data = (uint8_t)c;
- }
- return true;
-}
-
-bool is_samedata(const MDBX_val *a, const MDBX_val *b) {
- return a->iov_len == b->iov_len &&
- memcmp(a->iov_base, b->iov_base, a->iov_len) == 0;
-}
-
-//-----------------------------------------------------------------------------
-
-uint64_t prng64_white(uint64_t &state) {
- state = prng64_map2_careless(state);
- return bleach64(state);
-}
-
-uint32_t prng32(uint64_t &state) {
- return (uint32_t)(prng64_careless(state) >> 32);
-}
-
-void prng_fill(uint64_t &state, void *ptr, size_t bytes) {
- uint32_t u32 = prng32(state);
-
- while (bytes >= 4) {
- memcpy(ptr, &u32, 4);
- ptr = (uint32_t *)ptr + 1;
- bytes -= 4;
- u32 = prng32(state);
- }
-
- switch (bytes & 3) {
- case 3:
- memcpy(ptr, &u32, 3);
- break;
- case 2:
- memcpy(ptr, &u32, 2);
- break;
- case 1:
- memcpy(ptr, &u32, 1);
- break;
- case 0:
- break;
- }
-}
-
-static __thread uint64_t prng_state;
-
-void prng_seed(uint64_t seed) { prng_state = bleach64(seed); }
-
-uint32_t prng32(void) { return prng32(prng_state); }
-
-uint64_t prng64(void) { return prng64_white(prng_state); }
-
-void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); }
-
-double double_from_lower(uint64_t salt) {
-#ifdef IEEE754_DOUBLE_BIAS
- ieee754_double r;
- r.ieee.negative = 0;
- r.ieee.exponent = IEEE754_DOUBLE_BIAS;
- r.ieee.mantissa0 = (unsigned)(salt >> 32);
- r.ieee.mantissa1 = (unsigned)salt;
- return r.d;
-#else
- const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
- const double scale = 1.0 / (double)top;
- return (salt & top) * scale;
-#endif
-}
-
-double double_from_upper(uint64_t salt) {
-#ifdef IEEE754_DOUBLE_BIAS
- ieee754_double r;
- r.ieee.negative = 0;
- r.ieee.exponent = IEEE754_DOUBLE_BIAS;
- salt >>= 64 - DBL_MANT_DIG;
- r.ieee.mantissa0 = unsigned(salt >> 32);
- r.ieee.mantissa1 = unsigned(salt);
- return r.d;
-#else
- const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
- const double scale = 1.0 / (double)top;
- return (salt >> (64 - DBL_MANT_DIG)) * scale;
-#endif
-}
-
-bool flipcoin() { return prng32() & 1; }
-bool flipcoin_x2() { return (prng32() & 3) == 0; }
-bool flipcoin_x3() { return (prng32() & 7) == 0; }
-bool flipcoin_x4() { return (prng32() & 15) == 0; }
-bool flipcoin_n(unsigned n) {
- return (prng64() & ((UINT64_C(1) << n) - 1)) == 0;
-}
-
-bool jitter(unsigned probability_percent) {
- const uint32_t top = UINT32_MAX - UINT32_MAX % 100;
- uint32_t dice, edge = (top) / 100 * probability_percent;
- do
- dice = prng32();
- while (dice >= top);
- return dice < edge;
-}
-
-void jitter_delay(bool extra) {
- unsigned dice = prng32() & 3;
- if (dice == 0) {
- log_trace("== jitter.no-delay");
- } else {
- log_trace(">> jitter.delay: dice %u", dice);
- do {
- cpu_relax();
- memory_barrier();
- cpu_relax();
- if (dice > 1) {
- osal_yield();
- cpu_relax();
- if (dice > 2) {
- unsigned us =
- prng32() & (extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */);
- log_trace("== jitter.delay: %0.6f", us / 1000000.0);
- osal_udelay(us);
- }
- }
- } while (flipcoin());
- log_trace("<< jitter.delay: dice %u", dice);
- }
-}
diff --git a/3rdparty/libmdbx/test/utils.h b/3rdparty/libmdbx/test/utils.h
deleted file mode 100644
index dd52dc003c962d30c4ed4b966b5b3470ad4f0c69..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/utils.h
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Copyright 2017-2022 Leonid Yuriev <leo@yuriev.ru>
- * and other libmdbx authors: please see AUTHORS file.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-
-#pragma once
-#include "base.h"
-
-#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
- !defined(__ORDER_BIG_ENDIAN__)
-#error __BYTE_ORDER__ should be defined.
-#endif
-
-#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
- __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
-#error Unsupported byte order.
-#endif
-
-#if __GNUC_PREREQ(4, 4) || defined(__clang__)
-#ifndef bswap64
-#define bswap64(v) __builtin_bswap64(v)
-#endif
-#ifndef bswap32
-#define bswap32(v) __builtin_bswap32(v)
-#endif
-#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \
- !defined(bswap16)
-#define bswap16(v) __builtin_bswap16(v)
-#endif
-
-#elif defined(_MSC_VER)
-
-#if _MSC_FULL_VER < 190024215
-#pragma message( \
- "It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
-#endif
-
-#define bswap64(v) _byteswap_uint64(v)
-#define bswap32(v) _byteswap_ulong(v)
-#define bswap16(v) _byteswap_ushort(v)
-#define rot64(v, s) _rotr64(v, s)
-#define rot32(v, s) _rotr(v, s)
-
-#if defined(_M_X64) || defined(_M_IA64)
-#pragma intrinsic(_umul128)
-#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
-#pragma intrinsic(__umulh)
-#define mul_64x64_high(a, b) __umulh(a, b)
-#endif
-
-#if defined(_M_IX86)
-#pragma intrinsic(__emulu)
-#define mul_32x32_64(a, b) __emulu(a, b)
-#elif defined(_M_ARM)
-#define mul_32x32_64(a, b) _arm_umull(a, b)
-#endif
-
-#endif /* compiler */
-
-#ifndef bswap64
-#ifdef __bswap_64
-#define bswap64(v) __bswap_64(v)
-#else
-static __inline uint64_t bswap64(uint64_t v) {
- return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
- ((v << 24) & UINT64_C(0x0000ff0000000000)) |
- ((v << 8) & UINT64_C(0x000000ff00000000)) |
- ((v >> 8) & UINT64_C(0x00000000ff0000000)) |
- ((v >> 24) & UINT64_C(0x0000000000ff0000)) |
- ((v >> 40) & UINT64_C(0x000000000000ff00));
-}
-#endif
-#endif /* bswap64 */
-
-#ifndef bswap32
-#ifdef __bswap_32
-#define bswap32(v) __bswap_32(v)
-#else
-static __inline uint32_t bswap32(uint32_t v) {
- return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
- ((v >> 8) & UINT32_C(0x0000ff00));
-}
-#endif
-#endif /* bswap32 */
-
-#ifndef bswap16
-#ifdef __bswap_16
-#define bswap16(v) __bswap_16(v)
-#else
-static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
-#endif
-#endif /* bswap16 */
-
-#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-
-#ifndef htole16
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define htobe16(v) bswap16(v)
-#define htole16(v) (v)
-#define be16toh(v) bswap16(v)
-#define le16toh(v) (v)
-#else
-#define htobe16(v) (v)
-#define htole16(v) bswap16(v)
-#define be16toh(v) (v)
-#define le16toh(v) bswap16(v)
-#endif
-#endif /* htole16 */
-
-#ifndef htole32
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define htobe32(v) bswap32(v)
-#define htole32(v) (v)
-#define be32toh(v) bswap32(v)
-#define le32toh(v) (v)
-#else
-#define htobe32(v) (v)
-#define htole32(v) bswap32(v)
-#define be32toh(v) (v)
-#define le32toh(v) bswap32(v)
-#endif
-#endif /* htole32 */
-
-#ifndef htole64
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define htobe64(v) bswap64(v)
-#define htole64(v) (v)
-#define be64toh(v) bswap64(v)
-#define le64toh(v) (v)
-#else
-#define htobe64(v) (v)
-#define htole64(v) bswap_64(v)
-#define be64toh(v) (v)
-#define le64toh(v) bswap_64(v)
-#endif
-#endif /* htole64 */
-
-namespace unaligned {
-
-template <typename T> static __inline T load(const void *ptr) {
- if (MDBX_UNALIGNED_OK >= sizeof(T))
- return *(const T *)ptr;
- else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- return *(const T __unaligned *)ptr;
-#else
- T local;
- memcpy(&local, (const T *)ptr, sizeof(T));
- return local;
-#endif /* _MSC_VER || __unaligned */
- }
-}
-
-template <typename T> static __inline void store(void *ptr, const T &value) {
- if (MDBX_UNALIGNED_OK >= sizeof(T))
- *(T *)ptr = value;
- else {
-#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
- defined(_M_X64) || defined(_M_IA64)
- *((T __unaligned *)ptr) = value;
-#else
- memcpy(ptr, &value, sizeof(T));
-#endif /* _MSC_VER || __unaligned */
- }
-}
-
-} /* namespace unaligned */
-
-//-----------------------------------------------------------------------------
-
-#ifndef rot64
-static __inline uint64_t rot64(uint64_t v, unsigned s) {
- return (v >> s) | (v << (64 - s));
-}
-#endif /* rot64 */
-
-static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; }
-
-#undef roundup2
-static __inline size_t roundup2(size_t value, size_t granularity) {
- assert(is_power2(granularity));
- return (value + granularity - 1) & ~(granularity - 1);
-}
-
-//-----------------------------------------------------------------------------
-
-static __inline void memory_barrier(void) {
-#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
- __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
-#elif defined(__ATOMIC_SEQ_CST)
- __atomic_thread_fence(__ATOMIC_SEQ_CST);
-#elif defined(__clang__) || defined(__GNUC__)
- __sync_synchronize();
-#elif defined(_MSC_VER)
- MemoryBarrier();
-#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
-#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
- __mf();
-#elif defined(__ia32__)
- _mm_mfence();
-#else
-#error "Unknown target for Intel Compiler, please report to us."
-#endif
-#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
- __machine_rw_barrier();
-#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
- (defined(HP_IA64) || defined(__ia64))
- _Asm_mf();
-#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
- defined(__ppc64__) || defined(__powerpc64__)
- __lwsync();
-#else
-#error "Could not guess the kind of compiler, please report to us."
-#endif
-}
-
-static __inline void cpu_relax() {
-#if defined(__ia32__)
- _mm_pause();
-#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \
- defined(YieldProcessor)
- YieldProcessor();
-#else
-/* nope */
-#endif
-}
-
-//-----------------------------------------------------------------------------
-
-struct simple_checksum {
- uint64_t value{0};
-
- simple_checksum() = default;
-
- void push(const uint32_t &data) {
- value += data * UINT64_C(9386433910765580089) + 1;
- value ^= value >> 41;
- value *= UINT64_C(0xBD9CACC22C6E9571);
- }
-
- void push(const uint64_t &data) {
- push((uint32_t)data);
- push((uint32_t)(data >> 32));
- }
-
- void push(const bool data) {
- push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E));
- }
-
- void push(const void *ptr, size_t bytes) {
- const uint8_t *data = (const uint8_t *)ptr;
- for (size_t i = 0; i < bytes; ++i)
- push((uint32_t)data[i]);
- }
-
- void push(const double &data) { push(&data, sizeof(double)); }
- void push(const char *cstr) { push(cstr, strlen(cstr)); }
- void push(const std::string &str) { push(str.data(), str.size()); }
-
- void push(unsigned salt, const MDBX_val &val) {
- push(unsigned(val.iov_len));
- push(salt);
- push(val.iov_base, val.iov_len);
- }
-
-#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
- void push(const HANDLE &handle) { push(&handle, sizeof(handle)); }
-#endif /* _WINDOWS */
-};
-
-std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum);
-bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
- size_t bytes, simple_checksum &checksum);
-bool is_samedata(const MDBX_val *a, const MDBX_val *b);
-inline bool is_samedata(const MDBX_val &a, const MDBX_val &b) {
- return is_samedata(&a, &b);
-}
-std::string format(const char *fmt, ...);
-
-static inline uint64_t bleach64(uint64_t v) {
- // Tommy Ettinger, https://www.blogger.com/profile/04953541827437796598
- // http://mostlymangling.blogspot.com/2019/01/better-stronger-mixer-and-test-procedure.html
- v ^= rot64(v, 25) ^ rot64(v, 50);
- v *= UINT64_C(0xA24BAED4963EE407);
- v ^= rot64(v, 24) ^ rot64(v, 49);
- v *= UINT64_C(0x9FB21C651E98DF25);
- return v ^ v >> 28;
-}
-
-static inline uint32_t bleach32(uint32_t x) {
- // https://github.com/skeeto/hash-prospector
- // exact bias: 0.17353355999581582
- x ^= x >> 16;
- x *= UINT32_C(0x7feb352d);
- x ^= 0x3027C563 ^ (x >> 15);
- x *= UINT32_C(0x846ca68b);
- x ^= x >> 16;
- return x;
-}
-
-static inline uint64_t prng64_map1_careless(uint64_t state) {
- return state * UINT64_C(6364136223846793005) + 1;
-}
-
-static inline uint64_t prng64_map2_careless(uint64_t state) {
- return (state + UINT64_C(1442695040888963407)) *
- UINT64_C(6364136223846793005);
-}
-
-static inline uint64_t prng64_map1_white(uint64_t state) {
- return bleach64(prng64_map1_careless(state));
-}
-
-static inline uint64_t prng64_map2_white(uint64_t state) {
- return bleach64(prng64_map2_careless(state));
-}
-
-static inline uint64_t prng64_careless(uint64_t &state) {
- state = prng64_map1_careless(state);
- return state;
-}
-
-static inline double u64_to_double1(uint64_t v) {
- union {
- uint64_t u64;
- double d;
- } casting;
-
- casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12);
- assert(casting.d >= 1.0 && casting.d < 2.0);
- return casting.d - 1.0;
-}
-
-uint64_t prng64_white(uint64_t &state);
-uint32_t prng32(uint64_t &state);
-void prng_fill(uint64_t &state, void *ptr, size_t bytes);
-
-void prng_seed(uint64_t seed);
-uint32_t prng32(void);
-uint64_t prng64(void);
-void prng_fill(void *ptr, size_t bytes);
-
-bool flipcoin();
-bool flipcoin_x2();
-bool flipcoin_x3();
-bool flipcoin_x4();
-bool flipcoin_n(unsigned n);
-bool jitter(unsigned probability_percent);
-void jitter_delay(bool extra = false);
diff --git a/3rdparty/libmdbx/test/valgrind_suppress.txt b/3rdparty/libmdbx/test/valgrind_suppress.txt
deleted file mode 100644
index fbaf2bc5e0ba8b3ad685382768593e1aa919d3ce..0000000000000000000000000000000000000000
--- a/3rdparty/libmdbx/test/valgrind_suppress.txt
+++ /dev/null
@@ -1,587 +0,0 @@
-{
- msync-whole-mmap-1
- Memcheck:Param
- msync(start)
- fun:msync
- ...
- fun:mdbx_sync_locked
-}
-{
- msync-whole-mmap-2
- Memcheck:Param
- msync(start)
- fun:msync
- ...
- fun:mdbx_env_sync_internal
-}
-{
- msync-whole-mmap-3
- Memcheck:Param
- msync(start)
- fun:msync
- ...
- fun:mdbx_mapresize
-}
-{
- msync-wipe-steady
- Memcheck:Param
- msync(start)
- fun:msync
- ...
- fun:mdbx_wipe_steady
-}
-
-# memcmp() inside mdbx_iov_write() as workaround for todo4recovery://erased_by_github/libmdbx/issues/269
-{
- write-page-check-bcmp
- Memcheck:Cond
- fun:bcmp
- fun:mdbx_iov_write*
-}
-{
- write-page-check-memcmp
- Memcheck:Cond
- fun:memcmp*
- fun:mdbx_iov_write*
-}
-
-# single-page flush by pwrite()
-{
- pwrite-page-flush
- Memcheck:Param
- pwrite(buf)
- fun:pwrite
- ...
- fun:mdbx_iov_write*
-}
-{
- pwrite64-page-flush
- Memcheck:Param
- pwrite64(buf)
- fun:pwrite
- ...
- fun:mdbx_iov_write*
-}
-
-# modern Valgrind don't support the `vector[...]` pattern
-#{
-# pwritev-page-flush
-# Memcheck:Param
-# pwritev(vector[...])
-# fun:pwritev
-# ...
-# fun:mdbx_iov_write*
-#}
-# for((i=0;i<64;++i)); do echo -e "{\n pwritev-page-flush-$i\n Memcheck:Param\n pwritev(vector[$i])\n fun:pwritev\n ...\n fun:mdbx_iov_write*\n}"; done >> valgrind_suppress.txt
-{
- pwritev-page-flush-0
- Memcheck:Param
- pwritev(vector[0])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-1
- Memcheck:Param
- pwritev(vector[1])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-2
- Memcheck:Param
- pwritev(vector[2])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-3
- Memcheck:Param
- pwritev(vector[3])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-4
- Memcheck:Param
- pwritev(vector[4])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-5
- Memcheck:Param
- pwritev(vector[5])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-6
- Memcheck:Param
- pwritev(vector[6])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-7
- Memcheck:Param
- pwritev(vector[7])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-8
- Memcheck:Param
- pwritev(vector[8])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-9
- Memcheck:Param
- pwritev(vector[9])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-10
- Memcheck:Param
- pwritev(vector[10])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-11
- Memcheck:Param
- pwritev(vector[11])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-12
- Memcheck:Param
- pwritev(vector[12])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-13
- Memcheck:Param
- pwritev(vector[13])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-14
- Memcheck:Param
- pwritev(vector[14])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-15
- Memcheck:Param
- pwritev(vector[15])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-16
- Memcheck:Param
- pwritev(vector[16])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-17
- Memcheck:Param
- pwritev(vector[17])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-18
- Memcheck:Param
- pwritev(vector[18])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-19
- Memcheck:Param
- pwritev(vector[19])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-20
- Memcheck:Param
- pwritev(vector[20])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-21
- Memcheck:Param
- pwritev(vector[21])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-22
- Memcheck:Param
- pwritev(vector[22])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-23
- Memcheck:Param
- pwritev(vector[23])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-24
- Memcheck:Param
- pwritev(vector[24])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-25
- Memcheck:Param
- pwritev(vector[25])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-26
- Memcheck:Param
- pwritev(vector[26])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-27
- Memcheck:Param
- pwritev(vector[27])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-28
- Memcheck:Param
- pwritev(vector[28])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-29
- Memcheck:Param
- pwritev(vector[29])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-30
- Memcheck:Param
- pwritev(vector[30])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-31
- Memcheck:Param
- pwritev(vector[31])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-32
- Memcheck:Param
- pwritev(vector[32])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-33
- Memcheck:Param
- pwritev(vector[33])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-34
- Memcheck:Param
- pwritev(vector[34])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-35
- Memcheck:Param
- pwritev(vector[35])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-36
- Memcheck:Param
- pwritev(vector[36])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-37
- Memcheck:Param
- pwritev(vector[37])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-38
- Memcheck:Param
- pwritev(vector[38])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-39
- Memcheck:Param
- pwritev(vector[39])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-40
- Memcheck:Param
- pwritev(vector[40])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-41
- Memcheck:Param
- pwritev(vector[41])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-42
- Memcheck:Param
- pwritev(vector[42])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-43
- Memcheck:Param
- pwritev(vector[43])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-44
- Memcheck:Param
- pwritev(vector[44])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-45
- Memcheck:Param
- pwritev(vector[45])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-46
- Memcheck:Param
- pwritev(vector[46])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-47
- Memcheck:Param
- pwritev(vector[47])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-48
- Memcheck:Param
- pwritev(vector[48])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-49
- Memcheck:Param
- pwritev(vector[49])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-50
- Memcheck:Param
- pwritev(vector[50])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-51
- Memcheck:Param
- pwritev(vector[51])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-52
- Memcheck:Param
- pwritev(vector[52])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-53
- Memcheck:Param
- pwritev(vector[53])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-54
- Memcheck:Param
- pwritev(vector[54])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-55
- Memcheck:Param
- pwritev(vector[55])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-56
- Memcheck:Param
- pwritev(vector[56])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-57
- Memcheck:Param
- pwritev(vector[57])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-58
- Memcheck:Param
- pwritev(vector[58])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-59
- Memcheck:Param
- pwritev(vector[59])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-60
- Memcheck:Param
- pwritev(vector[60])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-61
- Memcheck:Param
- pwritev(vector[61])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-62
- Memcheck:Param
- pwritev(vector[62])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}
-{
- pwritev-page-flush-63
- Memcheck:Param
- pwritev(vector[63])
- fun:pwritev
- ...
- fun:mdbx_iov_write*
-}