.gitignore

 # Prerequisites
 build/*
+cmake-build-*/
 build_stretch/*
 test/build
 *.txt.user
@@ -67,4 +68,9 @@ Makefile
 cmake_install.cmake
 
 # Editor's temp files
-*~
\ No newline at end of file
+*~
+.idea/
+
+# OS files
+.DS_Store
+

.gitlab-ci.yml

+variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+
 stages:
-    - prepare
-    - analyze
     - build
-    - test
+    - deploy
 
-variables:
-    GIT_SUBMODULE_STRATEGY: normal
-
-cellframe-sdk-analyze:
-    stage: analyze
-    tags:
-    - cellframe-sdk
-    script: ~/production/integration/PVS-studio/analyze.sh 
-    artifacts:
-      paths:
-      - report/issues.txt
-      when: always
+.ci-polygon:
+  tags:
+     - ci-polygon
+  
+.tests:  
+  extends: .ci-polygon
+  stage: build
+  timeout: 3 hours 30 minutes
+  dependencies: []
+  
+  
+tests:amd64.gcc:
+    extends: .tests
+    image: demlabs/debian/amd64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh amd64-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+
+tests:amd64.clang:
+    extends: .tests
+    image: demlabs/debian/amd64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh amd64-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_C_COMPILER=clang -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+
+tests:arm64.gcc:
+    extends: .tests
+    image: demlabs/debian/arm64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh arm64-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+tests:arm64.clang:
+    extends: .tests
+    image: demlabs/debian/arm64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh arm64-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_C_COMPILER=clang -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+tests:arm32.gcc:
+    extends: .tests
+    image: demlabs/debian/arm32:qt5
+    before_script: /opt/buildtools/prepare_environment.sh arm32v7-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+tests:arm32.clang:
+    extends: .tests
+    image: demlabs/debian/arm32:qt5
+    before_script: /opt/buildtools/prepare_environment.sh arm32v7-linux
+    script:
+      - mkdir build
+      - cd build && cmake .. -DCMAKE_C_COMPILER=clang -DCMAKE_BUILD_TYPE=Release -DBUILD_CELLFRAME_SDK_TESTS=ON -DOFF_CORE_CELLFRAME_SDK_TESTS_OFF=network-monitor -DOFF_CRYPTO_CELLFRAME_SDK_TESTS_OFF="multisign" && make -j$(nproc) && ctest --verbose
+
+build:windows:
+    extends: .tests
+    image: demlabs/windows/amd64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh amd64-windows
+    script:
+      - mkdir build
+      - cd build && export PATH=${MXE_ROOT}/usr/bin:$PATH && x86_64-w64-mingw32.static-cmake .. -DCMAKE_BUILD_TYPE=Release && make -j$(nproc)
+    
+
+build:macos:
+    extends: .tests
+    image: demlabs/macos/amd64:qt5
+    before_script: /opt/buildtools/prepare_environment.sh amd64-windows
+    script:
+      - mkdir build
+      - cd build && $(${OSXCROSS_ROOT}/bin/osxcross-conf) && export OSXCROSS_NO_INCLUDE_PATH_WARNINGS=1 && export OSXCROSS_HOST=x86_64-apple-darwin20.4 && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=${OSXCROSS_ROOT}/toolchain.cmake -DBoost_INCLUDE_DIR=$BOOST_INCLUDE_DIR .. && make -j$(nproc)
+
+
+build:android:
+    extends: .tests
+    image: demlabs/android/any:qt5
+    before_script: /opt/buildtools/prepare_environment.sh amd64-windows
+    script:
+      - ./prod_build/build.sh --target android

.gitmodules

+[submodule "dap-sdk"]
+	path = dap-sdk
+	url = ../../dap/dap-sdk.git
+	branch = develop

3rdparty/crc32c_adler/README

+
+Extended version of CRC32C implementation by Mark Adler, providing both hardware (SSE 4.2) and software algorithms with auto-detection.
+
+Source:
+https://stackoverflow.com/a/17646775
+
+According to Ferry Toth, Adler's implementation is 'highly optimized':
+https://stackoverflow.com/a/39862799
+https://github.com/htot/crc32c
+
+Revised software algorithm by Robert Važan:
+https://stackoverflow.com/a/21915223
+https://crc32c.machinezoo.com/
+https://bitbucket.org/robertvazan/crc32c-hw/src

3rdparty/crc32c_adler/crc32c_adler.c

+/*
+  CRC32C_ADLER -- Computes CRC32C Checksums
+  Version 1.2, Date 05/21/18
+  Copyright (C) 2013 Mark Adler <madler@alumni.caltech.edu>
+  Copyright (C) 2018 Fonic <https://github.com/fonic>
+
+  Provides both a hardware-accelerated algorithm (*) and a software algorithm.
+  Note that this computes CRC32C checksums, not CRC32 (without 'C') checksums
+  used by Ethernet, gzip, etc.
+
+  (*) CRC instruction on Intel SSE 4.2 processors.  SSE 4.2 was first supported
+      by Nehalem processors introduced in November, 2008.
+
+  Version history:
+  1.0  10 Feb 2013  First version
+  1.1   1 Aug 2013  Correct comments on why three crc instructions in parallel
+  1.2  21 May 2018  Add header file, revise hardware support check, eliminate
+                    pthreads, restructure code, revise comments and description
+
+  Version 1.1 by Mark Adler was originally published here:
+  https://stackoverflow.com/a/17646775
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software.  If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "crc32c_adler.h"
+
+/* CRC32C (iSCSI) polynomial in reversed bit order. */
+#define POLY 0x82f63b78
+
+
+/******************************************************************************
+ *                                                                            *
+ *  Software Algorithm (1) - Table-driven, 8 Bytes / Iteration                *
+ *                                                                            *
+ ******************************************************************************/
+
+/* Table for software algorithm. */
+static uint32_t crc32c_table[8][256];
+
+/* Flag to indicate if crc32c_init_sw() has been called. */
+static int crc32c_sw_initialized = 0;
+
+/* Initialize table for software algorithm. */
+static void crc32c_init_sw(void)
+{
+    uint32_t n, crc, k;
+
+    for (n = 0; n < 256; n++) {
+        crc = n;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
+        crc32c_table[0][n] = crc;
+    }
+    for (n = 0; n < 256; n++) {
+        crc = crc32c_table[0][n];
+        for (k = 1; k < 8; k++) {
+            crc = crc32c_table[0][crc & 0xff] ^ (crc >> 8);
+            crc32c_table[k][n] = crc;
+        }
+    }
+    crc32c_sw_initialized = 1;
+}
+
+/* Compute CRC32C checksum. */
+uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len)
+{
+    const unsigned char *next = buf;
+    uint64_t crc;
+
+    if (!crc32c_sw_initialized)
+        crc32c_init_sw();
+
+    crc = crci ^ 0xffffffff;
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = crc32c_table[7][crc & 0xff] ^
+              crc32c_table[6][(crc >> 8) & 0xff] ^
+              crc32c_table[5][(crc >> 16) & 0xff] ^
+              crc32c_table[4][(crc >> 24) & 0xff] ^
+              crc32c_table[3][(crc >> 32) & 0xff] ^
+              crc32c_table[2][(crc >> 40) & 0xff] ^
+              crc32c_table[1][(crc >> 48) & 0xff] ^
+              crc32c_table[0][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+    while (len) {
+        crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    return (uint32_t)crc ^ 0xffffffff;
+}
+
+
+/******************************************************************************
+ *                                                                            *
+ *  Software Algorithm (2) - Table-driven, 16 Bytes / Iteration               *
+ *                                                                            *
+ ******************************************************************************/
+
+/* Table for software algorithm. */
+static uint32_t crc32c_table2[16][256];
+
+/* Flag to indicate if crc32c_init_sw2() has been called. */
+static int crc32c_table2_initialized = 0;
+
+/* Initialize table for software algorithm. */
+static void crc32c_init_sw2(void)
+{
+    for(int i = 0; i < 256; i++)
+    {
+        uint32_t res = (uint32_t)i;
+        for(int t = 0; t < 16; t++) {
+            for (int k = 0; k < 8; k++) res = (res & 1) == 1 ? POLY ^ (res >> 1) : (res >> 1);
+            crc32c_table2[t][i] = res;
+        }
+    }
+    crc32c_table2_initialized = 1;
+}
+
+/* Compute CRC32C checksum. */
+uint32_t crc32c_sw2(uint32_t crci, const void *buf, size_t len)
+{
+    const unsigned char *next = buf;
+#ifdef __x86_64__
+    uint64_t crc;
+#else
+    uint32_t crc;
+#endif
+
+    if(!crc32c_table2_initialized)
+        crc32c_init_sw2();
+
+    crc = crci ^ 0xffffffff;
+#ifdef __x86_64__
+    while (len && ((uintptr_t)next & 7) != 0)
+    {
+        crc = crc32c_table2[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        --len;
+    }
+    while (len >= 16)
+    {
+        crc ^= *(uint64_t *)next;
+        uint64_t high = *(uint64_t *)(next + 8);
+        crc = crc32c_table2[15][crc & 0xff]
+            ^ crc32c_table2[14][(crc >> 8) & 0xff]
+            ^ crc32c_table2[13][(crc >> 16) & 0xff]
+            ^ crc32c_table2[12][(crc >> 24) & 0xff]
+            ^ crc32c_table2[11][(crc >> 32) & 0xff]
+            ^ crc32c_table2[10][(crc >> 40) & 0xff]
+            ^ crc32c_table2[9][(crc >> 48) & 0xff]
+            ^ crc32c_table2[8][crc >> 56]
+            ^ crc32c_table2[7][high & 0xff]
+            ^ crc32c_table2[6][(high >> 8) & 0xff]
+            ^ crc32c_table2[5][(high >> 16) & 0xff]
+            ^ crc32c_table2[4][(high >> 24) & 0xff]
+            ^ crc32c_table2[3][(high >> 32) & 0xff]
+            ^ crc32c_table2[2][(high >> 40) & 0xff]
+            ^ crc32c_table2[1][(high >> 48) & 0xff]
+            ^ crc32c_table2[0][high >> 56];
+        next += 16;
+        len -= 16;
+    }
+#else
+    while (len && ((uintptr_t)next & 3) != 0)
+    {
+        crc = crc32c_table2[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        --len;
+    }
+    while (len >= 12)
+    {
+        crc ^= *(uint32_t *)next;
+        uint32_t high = *(uint32_t *)(next + 4);
+        uint32_t high2 = *(uint32_t *)(next + 8);
+        crc = crc32c_table2[11][crc & 0xff]
+            ^ crc32c_table2[10][(crc >> 8) & 0xff]
+            ^ crc32c_table2[9][(crc >> 16) & 0xff]
+            ^ crc32c_table2[8][crc >> 24]
+            ^ crc32c_table2[7][high & 0xff]
+            ^ crc32c_table2[6][(high >> 8) & 0xff]
+            ^ crc32c_table2[5][(high >> 16) & 0xff]
+            ^ crc32c_table2[4][high >> 24]
+            ^ crc32c_table2[3][high2 & 0xff]
+            ^ crc32c_table2[2][(high2 >> 8) & 0xff]
+            ^ crc32c_table2[1][(high2 >> 16) & 0xff]
+            ^ crc32c_table2[0][high2 >> 24];
+        next += 12;
+        len -= 12;
+    }
+#endif
+    while (len)
+    {
+        crc = crc32c_table2[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        --len;
+    }
+    return (uint32_t)crc ^ 0xffffffff;
+}
+
+
+/******************************************************************************
+ *                                                                            *
+ *  Hardware Algorithm - SSE 4.2                                              *
+ *                                                                            *
+ ******************************************************************************/
+
+/* Multiply a matrix times a vector over the Galois field of two elements,
+   GF(2).  Each element is a bit in an unsigned integer.  mat must have at
+   least as many entries as the power of two for most significant one bit in
+   vec. */
+static inline uint32_t gf2_matrix_times(uint32_t *mat, uint32_t vec)
+{
+    uint32_t sum;
+
+    sum = 0;
+    while (vec) {
+        if (vec & 1)
+            sum ^= *mat;
+        vec >>= 1;
+        mat++;
+    }
+    return sum;
+}
+
+/* Multiply a matrix by itself over GF(2).  Both mat and square must have 32
+   rows. */
+static inline void gf2_matrix_square(uint32_t *square, uint32_t *mat)
+{
+    int n;
+
+    for (n = 0; n < 32; n++)
+        square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+/* Construct an operator to apply len zeros to a crc.  len must be a power of
+   two.  If len is not a power of two, then the result is the same as for the
+   largest power of two less than len.  The result for len == 0 is the same as
+   for len == 1.  A variant of this routine could be easily written for any
+   len, but that is not needed for this application. */
+static void crc32c_zeros_op(uint32_t *even, size_t len)
+{
+    int n;
+    uint32_t row;
+    uint32_t odd[32];           /* odd-power-of-two zeros operator */
+
+    /* Put operator for one zero bit in odd. */
+    odd[0] = POLY;              /* CRC32C polynomial */
+    row = 1;
+    for (n = 1; n < 32; n++) {
+        odd[n] = row;
+        row <<= 1;
+    }
+
+    /* Put operator for two zero bits in even. */
+    gf2_matrix_square(even, odd);
+
+    /* Put operator for four zero bits in odd. */
+    gf2_matrix_square(odd, even);
+
+    /* First square will put the operator for one zero byte (eight zero bits),
+       in even -- next square puts operator for two zero bytes in odd, and so
+       on, until len has been rotated down to zero. */
+    do {
+        gf2_matrix_square(even, odd);
+        len >>= 1;
+        if (len == 0)
+            return;
+        gf2_matrix_square(odd, even);
+        len >>= 1;
+    } while (len);
+
+    /* Answer ended up in odd -- copy to even. */
+    for (n = 0; n < 32; n++)
+        even[n] = odd[n];
+}
+
+/* Take a length and build four lookup tables for applying the zeros operator
+   for that length, byte-by-byte on the operand. */
+static void crc32c_zeros(uint32_t zeros[][256], size_t len)
+{
+    uint32_t n;
+    uint32_t op[32];
+
+    crc32c_zeros_op(op, len);
+    for (n = 0; n < 256; n++) {
+        zeros[0][n] = gf2_matrix_times(op, n);
+        zeros[1][n] = gf2_matrix_times(op, n << 8);
+        zeros[2][n] = gf2_matrix_times(op, n << 16);
+        zeros[3][n] = gf2_matrix_times(op, n << 24);
+    }
+}
+
+/* Apply the zeros operator table to crc. */
+static inline uint32_t crc32c_shift(uint32_t zeros[][256], uint32_t crc)
+{
+    return zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^
+           zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24];
+}
+
+/* Block sizes for three-way parallel crc computation.  LONG and SHORT must
+   both be powers of two.  The associated string constants must be set
+   accordingly, for use in constructing the assembler instructions. */
+#define LONG 8192
+#define LONGx1 "8192"
+#define LONGx2 "16384"
+#define SHORT 256
+#define SHORTx1 "256"
+#define SHORTx2 "512"
+
+/* Tables for hardware algorithm that shift a crc by LONG and SHORT zeros. */
+static uint32_t crc32c_long[4][256];
+static uint32_t crc32c_short[4][256];
+
+/* Flag to indicate if crc32c_init_hw() has been called. */
+static int crc32c_hw_initialized = 0;
+
+
+
+
+#if defined(__x86_64__)                 /* @RRL: to compile for ARM */
+
+
+
+static void crc32c_init_hw(void)
+{
+    crc32c_zeros(crc32c_long, LONG);
+    crc32c_zeros(crc32c_short, SHORT);
+    crc32c_hw_initialized = 1;
+}
+
+/* Compute CRC32C checksum. */
+uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len)
+{
+    const unsigned char *next = buf;
+    const unsigned char *end;
+    uint64_t crc0, crc1, crc2;      /* need to be 64 bits for crc32q */
+
+    /* Populate shift tables the first time through. */
+    if (!crc32c_hw_initialized)
+        crc32c_init_hw();
+
+    /* Pre-process the crc. */
+    crc0 = crc ^ 0xffffffff;
+
+    /* Compute the crc for up to seven leading bytes to bring the data pointer
+       to an eight-byte boundary. */
+    while (len && ((uintptr_t)next & 7) != 0) {
+        __asm__("crc32b\t" "(%1), %0"
+                : "=r"(crc0)
+                : "r"(next), "0"(crc0));
+        next++;
+        len--;
+    }
+
+    /* Compute the crc on sets of LONG*3 bytes, executing three independent crc
+       instructions, each on LONG bytes -- this is optimized for the Nehalem,
+       Westmere, Sandy Bridge, and Ivy Bridge architectures, which have a
+       throughput of one crc per cycle, but a latency of three cycles. */
+    while (len >= LONG*3) {
+        crc1 = 0;
+        crc2 = 0;
+        end = next + LONG;
+        do {
+            __asm__("crc32q\t" "(%3), %0\n\t"
+                    "crc32q\t" LONGx1 "(%3), %1\n\t"
+                    "crc32q\t" LONGx2 "(%3), %2"
+                    : "=r"(crc0), "=r"(crc1), "=r"(crc2)
+                    : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2));
+            next += 8;
+        } while (next < end);
+        crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1;
+        crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2;
+        next += LONG*2;
+        len -= LONG*3;
+    }
+
+    /* Do the same thing, but now on SHORT*3 blocks for the remaining data less
+       than a LONG*3 block. */
+    while (len >= SHORT*3) {
+        crc1 = 0;
+        crc2 = 0;
+        end = next + SHORT;
+        do {
+            __asm__("crc32q\t" "(%3), %0\n\t"
+                    "crc32q\t" SHORTx1 "(%3), %1\n\t"
+                    "crc32q\t" SHORTx2 "(%3), %2"
+                    : "=r"(crc0), "=r"(crc1), "=r"(crc2)
+                    : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2));
+            next += 8;
+        } while (next < end);
+        crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1;
+        crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2;
+        next += SHORT*2;
+        len -= SHORT*3;
+    }
+
+    /* Compute the crc on the remaining eight-byte units less than a SHORT*3
+       block. */
+    end = next + (len - (len & 7));
+    while (next < end) {
+        __asm__("crc32q\t" "(%1), %0"
+                : "=r"(crc0)
+                : "r"(next), "0"(crc0));
+        next += 8;
+    }
+    len &= 7;
+
+    /* Compute the crc for up to seven trailing bytes. */
+    while (len) {
+        __asm__("crc32b\t" "(%1), %0"
+                : "=r"(crc0)
+                : "r"(next), "0"(crc0));
+        next++;
+        len--;
+    }
+
+    /* Return a post-processed crc. */
+    return (uint32_t)crc0 ^ 0xffffffff;
+}
+
+
+
+#endif /* @RRL: to compile for ARM */
+
+/******************************************************************************
+ *                                                                            *
+ *  Other Functions                                                           *
+ *                                                                            *
+ ******************************************************************************/
+
+/* Variables to store information on hardware support. */
+static int crc32c_hardware_support = 0;
+static int crc32c_hardware_checked = 0;
+
+/* Check for hardware support (SSE 4.2).  Note that this does not check for
+   the existence of the cpuid instruction itself, which was introduced on the
+   486SL in 1992, so this will fail on earlier x86 processors.  cpuid works
+   on all Pentium and later processors. */
+int crc32c_hw_support()
+{
+#if defined(__x86_64__)                 /* @RRL: to compile for ARM */
+    if (!crc32c_hardware_checked) {
+        do {
+            uint32_t eax, ecx;
+            eax = 1;
+            __asm__("cpuid"
+                    : "=c"(ecx)
+                    : "a"(eax)
+                    : "%ebx", "%edx");
+            (crc32c_hardware_support) = (ecx >> 20) & 1;
+        } while (0);
+        crc32c_hardware_checked = 1;
+    }
+    return crc32c_hardware_support;
+#else
+    return  0;
+#endif
+
+}
+
+/* Disable hardware algorithm even if supported by hardware. */
+void crc32c_hw_disable()
+{
+    crc32c_hardware_support = 0;
+    crc32c_hardware_checked = 1;
+}
+
+/* Compute CRC32C checksum. Use hardware algorithm if supported,
+   fall back on software algorithm otherwise. */
+uint32_t crc32c(uint32_t crc, const void *buf, size_t len)
+{
+#if defined(__x86_64__)                 /* @RRL: to compile for ARM */
+    return crc32c_hw_support() ? crc32c_hw(crc, buf, len) : crc32c_sw(crc, buf, len);
+#else
+    return crc32c_sw(crc, buf, len);
+#endif
+}

3rdparty/crc32c_adler/crc32c_adler.h

+/*
+  CRC32C_ADLER -- Computes CRC32C Checksums
+  Version 1.2, Date 05/21/18
+  Copyright (C) 2013 Mark Adler <madler@alumni.caltech.edu>
+  Copyright (C) 2018 Fonic <https://github.com/fonic>
+
+  Provides both a hardware-accelerated algorithm (*) and a software algorithm.
+  Note that this computes CRC32C checksums, not CRC32 (without 'C') checksums
+  used by Ethernet, gzip, etc.
+
+  (*) CRC instruction on Intel SSE 4.2 processors.  SSE 4.2 was first supported
+      by Nehalem processors introduced in November, 2008.
+
+  Version history:
+  1.0  10 Feb 2013  First version
+  1.1   1 Aug 2013  Correct comments on why three crc instructions in parallel
+  1.2  21 May 2018  Add header file, revise hardware support check, eliminate
+                    pthreads, restructure code, revise comments and description
+
+  Version 1.1 by Mark Adler was originally published here:
+  https://stackoverflow.com/a/17646775
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software.  If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+#ifndef CRC32C_ADLER_H
+#define CRC32C_ADLER_H
+
+#include <stdint.h>
+
+#define CRC32C_INIT 0xEDB88320
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* Compute CRC32C checksum using software algorithm (1). */
+uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len);
+
+/* Compute CRC32C checksum using software algorithm (2). */
+uint32_t crc32c_sw2(uint32_t crci, const void *buf, size_t len);
+
+/* Compute CRC32C checksum using hardware algorithm. */
+uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len);
+
+/* Check if hardware-support (i.e. SSE 4.2) is available. */
+int crc32c_hw_support();
+
+/* Disable hardware algorithm even if supported by hardware. */
+void crc32c_hw_disable();
+
+/* Compute CRC32C checksum. Use hardware algorithm if supported,
+   fall back on software algorithm otherwise. */
+uint32_t crc32c(uint32_t crc, const void *buf, size_t len);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif // CRC32C_ADLER_H

3rdparty/crc32c_adler/example-dht.c

+/*
+  TODO describe example
+*/
+#define _GNU_SOURCE     /* random, srandom*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <fcntl.h>      /* open, close etc. */
+#include <string.h>     /* memcpy */
+#include <netinet/in.h> /* AF_INET, INET_ADDRSTRLEN etc. */
+#include <arpa/inet.h>  /* inet_ntop */
+
+#include "crc32c_adler.h"
+
+/* Convert id to hex string of formate used in BEP 42. */
+char id_hex_str[20*2+2+1]; /* 20 bytes x 2 characters + 2 x ' ' + '\0' */
+const char*
+id_to_hex(const unsigned char *id)
+{
+    //const char* hex_chr = "0123456789ABCDEF";
+    const char* hex_chr = "0123456789abcdef";
+    for(int i=0,j=0; i < 20 && j < sizeof(id_hex_str)-2; i++) {
+        id_hex_str[j++] = hex_chr[ (id[i]>>4) & 0x0F ];
+        id_hex_str[j++] = hex_chr[  id[i]     & 0x0F ];
+        if (i == 2 || i == 18) {
+            id_hex_str[j++] = ' ';
+        }
+    }
+    id_hex_str[sizeof(id_hex_str)-1] = '\0';
+    return id_hex_str;
+}
+
+/* Generate node ID from IP address + predefined rand using example algorithm
+   provided in BEP 42.
+
+   Parameters:
+   ip       IPv4 or IPv6 address (network byte order)
+   iplen    number of octets to consider in ip (4 or 8)
+   id       resulting node ID
+   rand     predefined random value */
+void crc32c_id(const uint8_t* ip, int iplen, uint8_t id[20], uint32_t rand)
+{
+    uint8_t v4_mask[] = { 0x03, 0x0f, 0x3f, 0xff };
+    uint8_t v6_mask[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff };
+    uint8_t* mask = iplen == 4 ? v4_mask : v6_mask;
+
+    uint8_t ip_copy[8];
+    memcpy(ip_copy, ip, iplen);
+
+    for (int i = 0; i < iplen; ++i)
+            ip_copy[i] &= mask[i];
+
+    //uint32_t rand = random() & 0xff;
+    uint8_t r = rand & 0x7;
+    ip_copy[0] |= r << 5;
+
+    uint32_t crc = 0;
+    crc = crc32c(crc, ip_copy, iplen);
+
+    /* only take the top 21 bits from crc */
+    id[0] = (crc >> 24) & 0xff;
+    id[1] = (crc >> 16) & 0xff;
+    id[2] = ((crc >> 8) & 0xf8) | (random() & 0x7);
+    for (int i = 3; i < 19; ++i) id[i] = random();
+    id[19] = rand;
+}
+
+/* Find how many bits two ids have in common. */
+int common_bits(const unsigned char *id1, const unsigned char *id2)
+{
+    int i, j;
+    unsigned char xor;
+    for(i = 0; i < 20; i++) {
+        if(id1[i] != id2[i])
+            break;
+    }
+
+    if(i == 20)
+        return 160;
+
+    xor = id1[i] ^ id2[i];
+
+    j = 0;
+    while((xor & 0x80) == 0) {
+        xor <<= 1;
+        j++;
+    }
+
+    return 8 * i + j;
+}
+
+/* Check if a node ID is correct in the sense of BEP 42. */
+int check_id(const uint8_t id1[20], const uint8_t* ip, int iplen, uint32_t rand)
+{
+    /* Generate ID from IP + rand -> id2. */
+    uint8_t id2[20];
+    crc32c_id(ip, iplen, id2, rand);
+
+    /* Compare id1 with id2:
+       - the first 21 bits must match
+       - the last byte must match rand */
+    int cbits = common_bits(id1, id2);
+    if (cbits < 21) {
+        printf("Only the first %i bits match (expected: 21)\n", cbits);
+        return 0;
+    }
+    if (id1[19] != id2[19]) {
+        printf("Last byte does not match (expected: %u, got: %u)\n", id2[19], id1[19]);
+        return 0;
+    }
+    return 1;
+}
+
+/* Main. */
+int main(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+
+    /* Print which CRC32C algorithm is used. */
+    printf("\nUsing %s algorithm.\n\n", crc32c_hw_support() ? "hardware-accelerated (SSE 4.2)" : "software");
+
+    /* Seed random. */
+    int fd = open("/dev/urandom", O_RDONLY);
+    if(fd < 0) {
+        perror("open(random)");
+        exit(1);
+    }
+    unsigned seed;
+    read(fd, &seed, sizeof(seed));
+    srandom(seed);
+    close(fd);
+
+    /* Example IP/rand combinations as used in BEP 42. */
+    uint8_t ip[5][4] = {
+        { 124, 31, 75, 21 },
+        { 21, 75, 31, 124 },
+        { 65, 23, 51, 170 },
+        { 84, 124, 73, 14 },
+        { 43, 213, 53, 83 }
+    };
+    uint32_t rand[] = {
+        1,
+        86,
+        22,
+        65,
+        90
+    };
+    int iplen = 4;
+    uint8_t id[20];
+
+    printf("IP              rand  Node ID                                    Ok?\n");
+    printf("=============== ===== ========================================== ====\n");
+    for (int i = 0; i < 5; ++i) {
+        crc32c_id(ip[i], iplen, id, rand[i]);
+        char ipstr[INET_ADDRSTRLEN];
+        inet_ntop(AF_INET, ip[i], ipstr, sizeof(ipstr));
+        printf("%-15s  %2u   %s %s\n", ipstr, rand[i], id_to_hex(id), (check_id(id, ip[i], 4, rand[i]) ? "yes" : "no"));
+    }
+
+    return 0;
+}

3rdparty/crc32c_adler/example-stdin.c

+/*
+  TODO describe example
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+
+#include "crc32c_adler.h"
+
+#define SIZE (262144*3)
+#define CHUNK SIZE
+
+int main(int argc, char **argv)
+{
+    char *buf;
+    ssize_t got;
+    size_t off, n;
+    uint32_t crc;
+
+    (void)argv;
+    crc = 0;
+    buf = malloc(SIZE);
+    if (buf == NULL) {
+        fputs("out of memory", stderr);
+        return 1;
+    }
+    while ((got = read(0, buf, SIZE)) > 0) {
+        off = 0;
+        do {
+            n = (size_t)got - off;
+            if (n > CHUNK)
+                n = CHUNK;
+            crc = argc > 1 ? crc32c_sw(crc, buf + off, n) :
+                             crc32c(crc, buf + off, n);
+            off += n;
+        } while (off < (size_t)got);
+    }
+    free(buf);
+    if (got == -1) {
+        fputs("read error\n", stderr);
+        return 1;
+    }
+    printf("%08x\n", crc);
+    return 0;
+}

3rdparty/cuttdb/CMakeLists.txt

-cmake_minimum_required(VERSION 3.1)
-
-project(dap_cuttdb C)
-
-add_definitions ("-D_GNU_SOURCE")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
-
-file(GLOB cuttdb_src src/*.c)
-file(GLOB cuttdb_h src/*.h)
-
-# the server part ain't ported, and thus not built, so are tests.
-list(FILTER cuttdb_src EXCLUDE REGEX "ae_")
-list(FILTER cuttdb_h EXCLUDE REGEX "ae_")
-list(FILTER cuttdb_src EXCLUDE REGEX "server.")
-list(FILTER cuttdb_h EXCLUDE REGEX "server.")
-list(FILTER cuttdb_src EXCLUDE REGEX "dump.")
-list(FILTER cuttdb_h EXCLUDE REGEX "dump.")
-list(FILTER cuttdb_src EXCLUDE REGEX "builddb.")
-list(FILTER cuttdb_h EXCLUDE REGEX "builddb.")
-list(FILTER cuttdb_src EXCLUDE REGEX "test_mt.")
-list(FILTER cuttdb_h EXCLUDE REGEX "test_mt.")
-
-if(UNIX)
-   list(FILTER cuttdb_src EXCLUDE REGEX "mman.")
-   list(FILTER cuttdb_h EXCLUDE REGEX "mman.")
-endif()
-
-add_library(${PROJECT_NAME} STATIC ${cuttdb_src} ${cuttdb_h})
-set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
-set_target_properties(${PROJECT_NAME} PROPERTIES COMPILER_LANGUAGE C)
-
-set_target_properties(dap_cuttdb PROPERTIES LINKER_LANGUAGE C)
-
-target_link_libraries(${PROJECT_NAME})
-
-target_include_directories(${PROJECT_NAME} INTERFACE src)

3rdparty/cuttdb/src/ae_epoll.c

-/* Linux epoll(2) based ae.c module
- * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com
- * Released under the BSD license. See the COPYING file for more info. */
-
-#include <sys/epoll.h>
-#include <errno.h>
-
-typedef struct aeApiState {
-    int epfd;
-    struct epoll_event events[AE_SETSIZE];
-} aeApiState;
-
-static int aeApiCreate(EventLoop *eventLoop) {
-    aeApiState *state = malloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */
-    if (state->epfd == -1) return -1;
-    eventLoop->apidata = state;
-    return 0;
-}
-
-/*
-    be not referenced anywhere
-static void aeApiFree(EventLoop *eventLoop) {
-    aeApiState *state = eventLoop->apidata;
-
-    close(state->epfd);
-    free(state);
-}
-*/
-
-static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee;
-    ee.events = EPOLLONESHOT;
-    if (mask & AE_READABLE) ee.events |= EPOLLIN;
-    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.u64 = 0; /* avoid valgrind warning */
-    ee.data.fd = fd;
-    if (epoll_ctl(state->epfd, EPOLL_CTL_ADD,fd,&ee) == -1 && errno != EEXIST) {
-        fprintf(stderr, "epoll_ctl(%d,%d) failed: %d\n", EPOLL_CTL_ADD,fd,errno);
-        return -1;
-    }
-    return 0;
-}
-
-static int aeApiUpdateEvent(EventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee;
-    ee.events = EPOLLONESHOT;
-    if (mask & AE_READABLE) ee.events |= EPOLLIN;
-    if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
-    ee.data.u64 = 0; /* avoid valgrind warning */
-    ee.data.fd = fd;
-    if (epoll_ctl(state->epfd, EPOLL_CTL_MOD,fd,&ee) == -1) {
-        fprintf(stderr, "epoll_ctl(%d,%d) failed: %d\n", EPOLL_CTL_ADD,fd,errno);
-        return -1;
-    }
-    return 0;
-}
-
-static int aeApiDelEvent(EventLoop *eventLoop, int fd) {
-    aeApiState *state = eventLoop->apidata;
-    struct epoll_event ee;
-
-    ee.events = 0;
-    ee.data.u64 = 0; /* avoid valgrind warning */
-    ee.data.fd = fd;
-    /* Note, Kernel < 2.6.9 requires a non null event pointer even for
-     * EPOLL_CTL_DEL. */
-    if ( epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee) == -1 
-            && errno != ENOENT && errno != EBADF) {
-        fprintf(stderr, "epoll_ctl(%d,%d) failed: %d\n", EPOLL_CTL_DEL,fd,errno);
-        return -1;
-    }
-    return 0;
-}
-
-int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, numevents = 0;
-
-    retval = epoll_wait(state->epfd,state->events,AE_SETSIZE,
-            tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
-    if (retval > 0) {
-        int j;
-
-        numevents = retval;
-        for (j = 0; j < numevents; j++) {
-            int mask = 0;
-            struct epoll_event *e = state->events+j;
-
-            if (e->events & EPOLLIN) mask |= AE_READABLE;
-            if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
-            eventLoop->fired[j] = e->data.fd;
-        }
-    }
-    return numevents;
-}
-
-
-/*
-    be not referenced anywhere
-static char *aeApiName(void) {
-    return "epoll";
-}
-*/
-

3rdparty/cuttdb/src/ae_kqueue.c

-/* Kqueue(2)-based ae.c module
- * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
- * Released under the BSD license. See the COPYING file for more info. */
-
-#include <sys/types.h>
-#include <sys/event.h>
-#include <sys/time.h>
-
-typedef struct aeApiState {
-    int kqfd;
-    struct kevent events[AE_SETSIZE];
-} aeApiState;
-
-static int aeApiCreate(EventLoop *eventLoop) {
-    aeApiState *state = malloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    state->kqfd = kqueue();
-    if (state->kqfd == -1) return -1;
-    eventLoop->apidata = state;
-    
-    return 0;    
-}
-
-static void aeApiFree(EventLoop *eventLoop) {
-    aeApiState *state = eventLoop->apidata;
-
-    close(state->kqfd);
-    free(state);
-}
-
-static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-    struct kevent ke;
-    
-    if (mask & AE_READABLE) {
-        EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
-        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
-    }
-    if (mask & AE_WRITABLE) {
-        EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
-        if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
-    }
-    return 0;
-}
-
-static int aeApiUpdateEvent(EventLoop *eventLoop, int fd, int mask) {
-    return aeApiAddEvent(eventLoop, fd, mask);
-}
-
-static int aeApiDelEvent(EventLoop *eventLoop, int fd) {
-    aeApiState *state = eventLoop->apidata;
-    struct kevent ke;
-
-    EV_SET(&ke, fd, EVFILT_READ | EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
-    kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
-    return 0;
-}
-
-static int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, numevents = 0;
-
-    if (tvp != NULL) {
-        struct timespec timeout;
-        timeout.tv_sec = tvp->tv_sec;
-        timeout.tv_nsec = tvp->tv_usec * 1000;
-        retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, &timeout);
-    } else {
-        retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, NULL);
-    }    
-
-    if (retval > 0) {
-        int j;
-        
-        numevents = retval;
-        for(j = 0; j < numevents; j++) {
-            int mask = 0;
-            struct kevent *e = state->events+j;
-            
-            if (e->filter == EVFILT_READ) mask |= AE_READABLE;
-            if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
-            eventLoop->fired[j] = e->ident; 
-        }
-    }
-    return numevents;
-}
-
-static char *aeApiName(void) {
-    return "kqueue";
-}

3rdparty/cuttdb/src/ae_select.c

-/* Select()-based ae.c module
- * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com
- * Released under the BSD license. See the COPYING file for more info. */
-
-#include <string.h>
-
-typedef struct aeApiState {
-    fd_set rfds, wfds;
-    /* We need to have a copy of the fd sets as it's not safe to reuse
-     * FD sets after select(). */
-    fd_set _rfds, _wfds;
-} aeApiState;
-
-static int aeApiCreate(EventLoop *eventLoop) {
-    aeApiState *state = malloc(sizeof(aeApiState));
-
-    if (!state) return -1;
-    FD_ZERO(&state->rfds);
-    FD_ZERO(&state->wfds);
-    eventLoop->apidata = state;
-    return 0;
-}
-
-static void aeApiFree(EventLoop *eventLoop) {
-    free(eventLoop->apidata);
-}
-
-static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-
-    if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
-    if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
-    return 0;
-}
-
-static void aeApiDelEvent(EventLoop *eventLoop, int fd, int mask) {
-    aeApiState *state = eventLoop->apidata;
-
-    if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
-    if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
-}
-
-static int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) {
-    aeApiState *state = eventLoop->apidata;
-    int retval, j, numevents = 0;
-
-    memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
-    memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
-
-    retval = select(eventLoop->maxfd+1,
-                &state->_rfds,&state->_wfds,NULL,tvp);
-    if (retval > 0) {
-        for (j = 0; j <= eventLoop->maxfd; j++) {
-            int mask = 0;
-            aeFileEvent *fe = &eventLoop->events[j];
-
-            if (fe->mask == AE_NONE) continue;
-            if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
-                mask |= AE_READABLE;
-            if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
-                mask |= AE_WRITABLE;
-            eventLoop->fired[numevents].fd = j;
-            eventLoop->fired[numevents].mask = mask;
-            numevents++;
-        }
-    }
-    return numevents;
-}
-
-static char *aeApiName(void) {
-    return "select";
-}

3rdparty/cuttdb/src/cdb_bgtask.c

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-#include "cdb_bgtask.h"
-#include <stdlib.h>
-#include <errno.h>
-#ifndef _WIN32
-#include <sys/signal.h>
-#else
-#include <signal.h>
-#endif
-
-
-/* where thread begins */
-static void *_cdb_bgtask_func(void *arg);
-
-
-CDBBGTASK *cdb_bgtask_new()
-{
-    CDBBGTASK *bt = (CDBBGTASK *)malloc(sizeof(CDBBGTASK));
-
-    bt->tnum = 0;
-    bt->run = 0;
-    bt->tid = 0;
-    pthread_cond_init(&bt->scond, NULL);
-    pthread_mutex_init(&bt->smutex, NULL);
-    return bt;
-}
-
-
-/* add a task into task list, must called before the thread run */
-int cdb_bgtask_add(CDBBGTASK *bt, TASKFUNC func, void *arg, int intval)
-{
-    TASK *task = &bt->tasks[bt->tnum];
-
-    if (bt->tid || bt->tnum > MAXTASKNUM)
-        return -1;
-
-    task->arg = arg;
-    task->func = func;
-    task->intval = intval;
-    task->ltime = time(NULL);
-    bt->tnum++;
-    return 0;
-}
-
-
-static void *_cdb_bgtask_func(void *arg)
-{
-    CDBBGTASK *bt = (CDBBGTASK *)arg;
-#ifndef _WIN32
-    /* block all signals coming into current thread */
-    sigset_t smask;
-    sigfillset(&smask);
-    pthread_sigmask(SIG_BLOCK, &smask, NULL);
-#endif
-    /* loop */
-    while(bt->run) {
-        time_t now = time(NULL);
-        struct timespec timeout;
-        int l_cond_rc;
-
-        /* check should run some tasks every 1 second */
-        timeout.tv_sec = now + 1;
-        timeout.tv_nsec = 0;
-
-        /* iterate and run the tasks */
-        for(int i = 0; i < bt->tnum; i++) {
-            TASK *task = &bt->tasks[i];
-            if (now >= task->ltime + task->intval) {
-                task->func(task->arg);
-                task->ltime = now;
-            }
-        }
-        pthread_mutex_lock(&bt->smutex);
-        l_cond_rc = pthread_cond_timedwait(&bt->scond, &bt->smutex, &timeout);
-        pthread_mutex_unlock(&bt->smutex);
-    }
-
-    return NULL;
-}
-
-
-/* create a thread for tasks */
-void cdb_bgtask_start(CDBBGTASK *bt)
-{
-    if (bt->run)
-        return;
-
-    bt->run = 1;
-    pthread_create(&bt->tid, NULL, _cdb_bgtask_func, bt);
-    return;
-}
-
-
-/* wait for the task thread exits */
-void cdb_bgtask_stop(CDBBGTASK *bt)
-{
-    if (bt->run) {
-        void **ret = NULL;
-        bt->run = 0;
-        pthread_cond_signal(&bt->scond);
-        pthread_join(bt->tid, ret);
-    }
-
-    bt->tnum = 0;
-}
-
-
-void cdb_bgtask_destroy(CDBBGTASK *bt)
-{
-    cdb_bgtask_stop(bt);
-    pthread_cond_destroy(&bt->scond);
-    pthread_mutex_destroy(&bt->smutex);
-    free(bt);
-}
-
-
-
-

3rdparty/cuttdb/src/cdb_bgtask.h

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-#ifndef _CDB_BGTASK_H_
-#define _CDB_BGTASK_H_
-#include <time.h>
-#include <pthread.h>
-
-
-/* 16 tasks at most in a task thread */
-#define MAXTASKNUM 16
-
-typedef void (*TASKFUNC)(void *);
-
-/* struct for timer task */
-typedef struct {
-    /* task function */
-    TASKFUNC func;
-    /* task argument */
-    void *arg;
-    /* task run interval(seconds) */
-    int intval;
-    /* time of last run */
-    time_t ltime;
-} TASK;
-
-/* struct for a background task manager */
-typedef struct CDBBGTASK
-{
-    TASK tasks[MAXTASKNUM];
-    /* number of tasks */
-    int tnum;
-    /* is running? */
-    int run;
-    pthread_t tid;
-    /* for wait the thread exit */
-    pthread_mutex_t smutex;
-    pthread_cond_t scond;
-} CDBBGTASK;
-
-
-
-CDBBGTASK *cdb_bgtask_new();
-int cdb_bgtask_add(CDBBGTASK *task, TASKFUNC func, void *arg, int intval);
-void cdb_bgtask_start(CDBBGTASK *bt);
-void cdb_bgtask_stop(CDBBGTASK *task);
-void cdb_bgtask_destroy(CDBBGTASK *task);
-
-
-#endif

3rdparty/cuttdb/src/cdb_bloomfilter.c

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-#include "cdb_bloomfilter.h"
-#include <stdlib.h>
-#include <string.h>
-
-#define CDBBFHASHNUM 16
-#define CDBBFSPLITPOW 6
-
-static uint64_t BFSEEDS[CDBBFHASHNUM] = {217636919,290182597,386910137,515880193,
-                                        687840301,917120411,1222827239,1610612741,
-                                        3300450239,3300450259,3300450281,3300450289,
-                                        3221225473ul,4294967291ul,163227661,122420729,};
-
-struct CDBBLOOMFILTER
-{
-    uint8_t *bitmap[1<<CDBBFSPLITPOW];
-    uint64_t rnum;
-    uint64_t size;
-    int hnum;
-    int ratio;
-};
-
-
-CDBBLOOMFILTER *cdb_bf_new(uint64_t rnum, uint64_t size)
-{
-    CDBBLOOMFILTER *bf = (CDBBLOOMFILTER *)malloc(sizeof(CDBBLOOMFILTER));
-    bf->rnum = 0;
-    bf->size = size;
-    /* number of hash should be 0.7 * ratio */
-    bf->hnum = size * 8 * 7 / (rnum * 10);
-    /* number of hash is limit in [1, 16] */
-    if (bf->hnum > CDBBFHASHNUM)
-        bf->hnum = CDBBFHASHNUM;
-    if (bf->hnum == 0)
-        bf->hnum = 1;
-    /* avoid malloc too much memory once */
-    for(int i = 0; i < (1 << CDBBFSPLITPOW); i++) {
-        bf->bitmap[i] = (uint8_t*)malloc(size >> CDBBFSPLITPOW);
-        memset(bf->bitmap[i], 0, size >> CDBBFSPLITPOW);
-    }
-    return bf;
-}
-
-
-void cdb_bf_set(CDBBLOOMFILTER *bf, void *key, int ksize)
-{
-    uint8_t *src = (uint8_t *)key, *end = src + ksize;
-    uint64_t hval[CDBBFHASHNUM] = {0};
-
-    for(;src < end; src++) 
-        for(int i = 0; i < bf->hnum; i++) 
-            hval[i] = hval[i] * BFSEEDS[i] + *src;
-
-    for(int i = 0; i < bf->hnum; i++) {
-        uint64_t p = (hval[i] >> CDBBFSPLITPOW) % ((bf->size >> CDBBFSPLITPOW) << 3);
-        uint8_t *bitmap = bf->bitmap[hval[i] & ((1<<CDBBFSPLITPOW) - 1)];
-        bitmap[p >> 3] |= (1 << (p & 0x07));
-    }
-
-    bf->rnum++;
-}
-
-
-bool cdb_bf_exist(CDBBLOOMFILTER *bf, void *key, int ksize)
-{
-    uint8_t *src = (uint8_t *)key, *end = src + ksize;
-    uint64_t hval[CDBBFHASHNUM] = {0};
-    int exist = 0;
-
-    for(;src < end; src++) 
-        for(int i = 0; i < bf->hnum; i++) 
-            hval[i] = hval[i] * BFSEEDS[i] + *src;
-
-    for(int i = 0; i < bf->hnum; i++) {
-        uint64_t p = (hval[i] >> CDBBFSPLITPOW) % ((bf->size >> CDBBFSPLITPOW) << 3);
-        uint8_t *bitmap = bf->bitmap[hval[i] & ((1<<CDBBFSPLITPOW) - 1)];
-        if (bitmap[p >> 3] & (1 << (p & 0x07)))
-            exist++;
-        else 
-            break;
-    }
-
-    return (exist == bf->hnum);
-}
-
-void cdb_bf_clean(CDBBLOOMFILTER *bf)
-{
-    for(int i = 0; i < (1 << CDBBFSPLITPOW); i++) 
-        memset(bf->bitmap[i], 0, bf->size >> CDBBFSPLITPOW);
-
-    bf->rnum = 0;
-}
-
-
-void cdb_bf_destroy(CDBBLOOMFILTER *bf)
-{
-    for(int i = 0; i < (1 << CDBBFSPLITPOW); i++) 
-        free(bf->bitmap[i]);
-    free(bf);
-}
-
-
-#ifdef _UT_CDBBF_
-#include <stdio.h>
-#include <stdlib.h>
-#include "cdb_bloomfilter.h"
-
-int main(int argc, char *argv[])
-{
-    int size = 1048576;
-    int rnum = 1048576;
-    if (argc > 1)
-        rnum = atoi(argv[1]);
-    if (argc > 2)
-        size = atoi(argv[2]);
-
-    CDBBLOOMFILTER *bf = cdb_bf_new(rnum, size);
-    for(int i = 0; i < rnum; i++) {
-        int j = 2 * i;
-        cdb_bf_set(bf, &j, 4);
-    }
-
-    int exist = 0;
-    for(int i = 0; i < rnum; i++) {
-        int j = 2 * i;
-        if (cdb_bf_exist(bf, &j, 4))
-            exist++;
-    }
-    printf("right positive: %.2f%%%%\n", (float)exist/(float)rnum*10000);
-
-    exist = 0;
-    for(int i = 0; i < rnum * 2; i++) {
-        int j = 2 * i + 1;
-        if (cdb_bf_exist(bf, &j, 4))
-            exist++;
-    }
-
-    printf("false positive: %.2f%%%%  %d/%d\n", (float)exist/(float)rnum*5000, exist, rnum * 2);
-    printf("element num: %d\n", bf->rnum);
-    cdb_bf_destroy(bf);
-    return 0;
-}
-#endif
-

3rdparty/cuttdb/src/cdb_bloomfilter.h

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-/*
-Bloom Filter is currently not used in cuttdb
-*/
-#ifndef _CDB_BLOOMFILTER_H_
-#define _CDB_BLOOMFILTER_H_
-#include <stdbool.h>
-#include <stdint.h>
-
-typedef struct CDBBLOOMFILTER CDBBLOOMFILTER;
-
-#define CDBBFRATIO 8
-
-CDBBLOOMFILTER *cdb_bf_new(uint64_t rnum, uint64_t size);
-void cdb_bf_set(CDBBLOOMFILTER *bf, void *key, int ksize);
-bool cdb_bf_exist(CDBBLOOMFILTER *bf, void *key, int ksize);
-void cdb_bf_clean(CDBBLOOMFILTER *bf);
-void cdb_bf_destroy(CDBBLOOMFILTER *bf);
-
-#endif

3rdparty/cuttdb/src/cdb_builddb.c

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-#include "cuttdb.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <time.h>
-
-int main(int argc, char *argv[])
-{
-    CDB *db = cdb_new();
-    if (argc < 2) {
-        fprintf(stderr, "Usage: %s db_path [hsize = 2000000]\n", argv[0]);
-        return 0;
-    }
-        
-    /* 1TB memory limit(unlimited) */
-    cdb_option(db, argc >= 3? atoi(argv[2]):2000000 , 0, 1048576);
-    cdb_seterrcb(db, cdb_deferrorcb, NULL);
-    if (cdb_open(db, argv[1], CDB_CREAT | CDB_PAGEWARMUP) < 0) {
-        return -1;
-    }
-    char *buf = NULL;
-    long count = 0;
-
-    size_t size, size2;
-    while((size = getline(&buf, &size2, stdin)) != -1) {
-        /* remove the delimiter*/
-        buf[--size] = '\0';
-        int klen = -1;
-        int vlen = -1;
-        uint32_t expire = 0;
-        int parsenum = 0;
-        for(int i = 0; i < size; i++) {
-            if (buf[i] == '\t') {
-                if (klen == -1)
-                    klen = i;
-                else {
-                    vlen = i - klen - 1;
-                    parsenum = 1;
-                }
-            } else if (buf[i] >= '0' && buf[i] <= '9' && parsenum) {
-                expire = expire * 10 + buf[i] - '0';
-            }
-        }
-
-        if (klen > 0 && vlen > 0) {
-            cdb_set2(db, buf, klen, buf + klen + 1, vlen,
-                    CDB_OVERWRITE, expire > 0? expire - time(NULL): 0);
-            count++;
-        }
-        free(buf);
-        buf = NULL;
-    }
-    cdb_destroy(db);
-    fprintf(stderr, "imported %ld records\n", count);
-    return 0;
-}
-
-

3rdparty/cuttdb/src/cdb_core.c

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-#include "cuttdb.h"
-#include "cdb_crc64.h"
-#include "cdb_types.h"
-#include "cdb_hashtable.h"
-#include "cdb_bloomfilter.h"
-#include "cdb_lock.h"
-#include "cdb_bgtask.h"
-#include "cdb_errno.h"
-#include "cdb_vio.h"
-#include "cdb_core.h"
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
-#include <pthread.h>
-
-static void _cdb_pageout(CDB *db);
-static void _cdb_defparam(CDB *db);
-static void _cdb_recout(CDB *db);
-static uint32_t _pagehash(const void *key, int len);
-static void _cdb_flushdpagetask(void *arg);
-static void _cdb_timerreset(struct timespec *ts);
-static uint32_t _cdb_timermicrosec(struct timespec *ts);
-static void _cdb_pagewarmup(CDB *db, bool loadbf);
-
-
-/* it isn't necessary to rehash bid in hash table cache */
-static uint32_t _pagehash(const void *key, int len)
-{
-    (void) len;
-    return *(uint32_t*)key;
-}
-
-
-/* used to get the duration of a procedure */
-static void _cdb_timerreset(struct timespec *ts)
-{
-    clock_gettime(CLOCK_MONOTONIC, ts);
-}
-
-
-static uint32_t _cdb_timermicrosec(struct timespec *ts)
-{
-    struct timespec ts2;
-    uint32_t diff;
-    clock_gettime(CLOCK_MONOTONIC, &ts2);
-    diff = (ts2.tv_sec - ts->tv_sec) * 1000000;
-    diff += ts2.tv_nsec / 1000;
-    diff -= ts->tv_nsec / 1000;
-    return diff;
-}
-
-
-/* reset the parameters */
-static void _cdb_defparam(CDB *db)
-{
-    db->rnum = 0;
-    db->bfsize = 0;
-    db->rclimit = 128 * MB;
-    db->pclimit = 1024 * MB;
-    db->hsize = 1000000; 
-    db->rcache = db->pcache = db->dpcache = NULL;
-    db->bf = NULL;
-    db->opened = false;
-    db->vio = NULL;
-    db->mtable = NULL;
-    db->oid = 0;
-    db->roid = 0;
-    db->errcbarg = NULL;
-    db->errcb = NULL;
-    db->areadsize = 4 * KB;
-    return;
-}
-
-
-/* flush all dirty pages */
-void cdb_flushalldpage(CDB *db)
-{
-    if (db->dpcache) {
-        while (db->dpcache->num) {
-            CDBHTITEM *item = cdb_ht_poptail(db->dpcache);    
-            uint32_t bid = *(uint32_t*)cdb_ht_itemkey(db->dpcache, item);
-            FOFF off;
-            db->vio->wpage(db->vio, (CDBPAGE*)cdb_ht_itemval(db->dpcache, item), &off);
-            db->mtable[bid] = off;
-            free(item);
-        } 
-
-        db->roid = db->oid; 
-        db->vio->cleanpoint(db->vio);
-    }
-}
-
-
-/* flush oldest dirty index page to disk, it runs in another thread and triggered by timer */
-static void _cdb_flushdpagetask(void *arg)
-{
-    CDB *db = (CDB *)arg;
-    CDBHTITEM *item;
-    CDBPAGE *page;
-    time_t now = time(NULL);
-    bool cleandcache = false;
-    uint32_t bid;
-
-    if (!db->dpcache)
-        /* no dirty page cache */
-        return;
-
-    /* if there isn't too much dirty page and some time passed since last clean,
-     write out all dirty pages to make a recovery point(oid) */
-    if (db->dpcache->num < 1024 && now > db->ndpltime + 120)
-        cleandcache = true;
-        
-    while(db->dpcache->num) {
-        FOFF off;
-        cdb_lock_lock(db->dpclock);
-        item = cdb_ht_gettail(db->dpcache);
-        /* no item in dpcache after lock */
-        if (item == NULL) {
-            cdb_lock_unlock(db->dpclock);
-            return;
-        }
-        page = (CDBPAGE *)cdb_ht_itemval(db->dpcache, item);
-        /* bid = page->bid; also OK */
-        bid = *(uint32_t*)cdb_ht_itemkey(db->dpcache, item);
-        /* been dirty for too long? */
-        if (now > page->mtime + DPAGETIMEOUT || cleandcache) {
-            if (cdb_lock_trylock(db->mlock[page->bid % MLOCKNUM])) {
-                /* avoid dead lock, since dpclock is holding */
-                cdb_lock_unlock(db->dpclock);
-                return;
-            }
-            /* remove it from dpcache */
-            cdb_ht_poptail(db->dpcache);
-            cdb_lock_unlock(db->dpclock);
-
-            /* write to disk */
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            db->vio->wpage(db->vio, page, &off);
-            db->wcount++;
-            db->wtime += _cdb_timermicrosec(&ts);
-            db->mtable[bid] = off;
-
-            /* move the clean page into pcache */
-            cdb_lock_lock(db->pclock);
-            cdb_ht_insert(db->pcache, item);
-            cdb_lock_unlock(db->pclock);
-            cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-        } else {
-            /* tail in dpcache isn't expired */
-            cdb_lock_unlock(db->dpclock);
-            return;
-        }
-    }
-
-    if (db->dpcache->num == 0 && cleandcache)
-        db->ndpltime = now;
-
-    if (cleandcache) {
-        /* clean succeed if goes here, remember the recovery point */
-        /* it's not necessary to lock */
-        db->roid = db->oid; 
-        db->vio->cleanpoint(db->vio);
-    }
-}
-
-
-/* fill the index page cache, and set the bloomfilter if necessary */
-static void _cdb_pagewarmup(CDB *db, bool loadbf)
-{
-    char sbuf[SBUFSIZE];
-    void *it = db->vio->pageitfirst(db->vio, 0);
-
-    if (it == NULL)
-        return;
-
-    for(;;) {
-        CDBPAGE *page = (CDBPAGE *)sbuf;
-        if (db->vio->pageitnext(db->vio, &page, it) < 0)
-            break;
-
-        /* the page is the newest one because its offset matches the one in main table */
-        if (OFFEQ(page->ooff, db->mtable[page->bid])) {
-            if (loadbf) {
-                /* iterate key hashes in page, set to the filter */
-                cdb_lock_lock(db->bflock);
-                for(uint32_t i = 0; i < page->num; i++) {
-                    uint64_t hash = (page->bid << 24) | (page->items[i].hash.i2 << 8)
-                        | (page->items[i].hash.i1);
-                    /* bloom filter use the combined record hash as key */
-                    cdb_bf_set(db->bf, &hash, SI8);
-                }
-                cdb_lock_unlock(db->bflock);
-            }
-
-            /* set the page to pcache if it doesn't exceed the limit size */
-            if (db->pcache && db->pcache->size < db->pclimit) {
-                cdb_lock_lock(db->pclock);
-                cdb_ht_insert2(db->pcache, &page->bid, SI4, page, MPAGESIZE(page));
-                cdb_lock_unlock(db->pclock);
-            }
-        }
-        /* the page may not be still in stack */
-        if (page != (CDBPAGE *)sbuf)
-            free(page);
-
-        if (!loadbf && (db->pcache && db->pcache->size > db->pclimit))
-            break;
-    }
-
-    db->vio->pageitdestroy(db->vio, it);
-}
-
-
-/* generate an incremental global operation id */
-uint64_t cdb_genoid(CDB *db)
-{
-    uint64_t oid;
-    cdb_lock_lock(db->oidlock);
-    oid = db->oid++;
-    cdb_lock_unlock(db->oidlock);
-    return oid;
-}
-
-
-/* get a new record iterator */
-void *cdb_iterate_new(CDB *db, uint64_t oid)
-{
-    return db->vio->recitfirst(db->vio, oid);
-}
-
-
-
-/* iterate the database by callback */
-uint64_t cdb_iterate(CDB *db, CDB_ITERCALLBACK itcb, void *arg, void *iter)
-{
-    char sbuf[SBUFSIZE];
-    uint64_t cnt = 0;
-
-    if (iter == NULL)
-        return cnt;
-    for(;;) {
-        /* the rec is a copy from file, may in stack or allocated in heap */
-        CDBREC *rec = (CDBREC *)sbuf;
-        bool ret = true;
-        if (db->vio->recitnext(db->vio, &rec, iter) < 0) 
-            break;
-        
-        if (cdb_checkoff(db, CDBHASH64(rec->key, rec->ksize), rec->ooff, CDB_NOTLOCKED)) {
-            ret = itcb(arg, rec->key, rec->ksize, rec->val, rec->vsize, rec->expire, rec->oid);
-            cnt++;
-        }
-        if (rec != (CDBREC *)sbuf)
-            free(rec);
-        if (!ret) 
-            break;
-    }
-    return cnt;
-}
-
-
-
-/* destroy the iterator */
-void cdb_iterate_destroy(CDB *db, void *iter)
-{
-    db->vio->recitdestroy(db->vio, iter);
-}
-
-
-/* difficult to implement */
-/*
-static void _cdb_rcachewarmup(CDB *db)
-{
-}
-*/
-
-
-CDB *cdb_new()
-{
-    CDB *db;
-    db = (CDB *)malloc(sizeof(CDB));
-    /* I assume all operation in this layer is 'fast', so no mutex used here */
-    for(int i = 0; i < MLOCKNUM; i++) 
-        db->mlock[i] = cdb_lock_new(CDB_LOCKSPIN);
-    db->dpclock = cdb_lock_new(CDB_LOCKSPIN);
-    db->pclock = cdb_lock_new(CDB_LOCKSPIN);
-    db->rclock = cdb_lock_new(CDB_LOCKSPIN);
-    db->stlock = cdb_lock_new(CDB_LOCKSPIN);
-    db->oidlock = cdb_lock_new(CDB_LOCKSPIN);
-    db->bflock = cdb_lock_new(CDB_LOCKSPIN);
-    db->bgtask = cdb_bgtask_new();
-    /* every thread should has its own errno */
-    db->errkey = (pthread_key_t *)malloc(sizeof(pthread_key_t));
-    pthread_key_create(db->errkey, NULL);
-    /* set default parameter */
-    _cdb_defparam(db);
-    return db;
-}
-
-
-int cdb_option(CDB *db, int bnum, int rcacheMB, int pcacheMB)
-{
-    /* too small bnum is not allowed */
-    db->hsize = bnum > 4096? bnum : 4096;
-
-    if (rcacheMB >= 0)
-        db->rclimit = (uint64_t)rcacheMB * MB;
-    if (pcacheMB >= 0)
-        db->pclimit = (uint64_t)pcacheMB * MB;
-    return 0;
-}
-
-
-void cdb_option_bloomfilter(CDB *db, uint64_t size)
-{
-    db->bfsize = size;
-}
-
-void cdb_option_areadsize(CDB *db, uint32_t size)
-{
-    db->areadsize = size;
-    if (db->areadsize < 1 * KB)
-        db->areadsize = 1 * KB;
-
-    if (db->areadsize > SBUFSIZE - (sizeof(CDBREC) - RECHSIZE)) 
-        db->areadsize = SBUFSIZE - (sizeof(CDBREC) - RECHSIZE);
-}
-
-int cdb_open(CDB *db, const char *file_name, int mode)
-{
-    /* if will become into a hash table when file_name == CDB_MEMDB */
-    int memdb = (strcmp(file_name, CDB_MEMDB) == 0);
-
-    if (db->rclimit)
-        /* record cache is enabled */
-        db->rcache = cdb_ht_new(true, NULL);
-    else if (memdb) {
-        /* record cache is disabled, but in MEMDB mode */
-        cdb_seterrno(db, CDB_MEMDBNOCACHE, __FILE__, __LINE__);
-        goto ERRRET;
-    }
-
-    if (db->pclimit && !memdb) {
-        /* page cache enabled. page cache is meaningless under MEMDB  mode */
-        db->dpcache = cdb_ht_new(true, _pagehash);
-        db->pcache = cdb_ht_new(true, _pagehash);
-    }
-
-
-    if (!memdb) {
-        if (db->bfsize) {
-            /* bloom filter enabled */
-            db->bf = cdb_bf_new(db->bfsize, db->bfsize);
-        }
-        /* now only one storage format is supported */
-        db->vio = cdb_vio_new(CDBVIOAPND2);
-        db->vio->db = db;
-        if (db->vio->open(db->vio, file_name, mode) < 0)
-            goto ERRRET;
-        if (db->vio->rhead(db->vio) < 0) {
-            db->mtable = (FOFF*)malloc(sizeof(FOFF) * db->hsize);
-            memset(db->mtable, 0, sizeof(FOFF) * db->hsize);
-        }
-        /* dirty index page would be swap to disk by timer control */
-        cdb_bgtask_add(db->bgtask, _cdb_flushdpagetask, db, 1);
-        db->ndpltime = time(NULL);
-        /* start background task thread */
-        cdb_bgtask_start(db->bgtask);
-    } else {
-        /* no persistent storage under MEMDB mode */
-        db->vio = NULL;
-        db->bgtask = NULL;
-        db->mtable = NULL;
-    }
-
-    if (db->bf || ((mode & CDB_PAGEWARMUP) && db->pcache)) {
-        /* fill the bloom filter if it is enabled, and fill the page cache */
-        _cdb_pagewarmup(db, !!db->bf);
-    }
-
-    /* reset the statistic info */
-    cdb_stat(db, NULL);
-    db->opened = true;
-    return 0;
-
-ERRRET:
-    if (db->rcache)
-        cdb_ht_destroy(db->rcache);
-    if (db->pcache)
-        cdb_ht_destroy(db->pcache);
-    if (db->dpcache)
-        cdb_ht_destroy(db->dpcache);
-    if (db->bf)
-        cdb_bf_destroy(db->bf);
-    cdb_bgtask_stop(db->bgtask);
-    _cdb_defparam(db);
-    return -1;
-}
-
-
-/* check if the page cache size exceed the limit. clean oldest page if necessary */
-static void _cdb_pageout(CDB *db)
-{
-    while (PCOVERFLOW(db)) {
-        if (db->pcache->num) {
-            /* clean page cache is prior */
-            cdb_lock_lock(db->pclock);
-            cdb_ht_removetail(db->pcache);
-            cdb_lock_unlock(db->pclock);
-        } else if (db->dpcache->num) {
-            CDBHTITEM *item;
-            uint32_t bid;
-            FOFF off;
-            cdb_lock_lock(db->dpclock);
-            item = cdb_ht_gettail(db->dpcache);    
-            if (item == NULL) {
-                cdb_lock_unlock(db->dpclock);
-                break;
-            }
-
-            bid = *(uint32_t*)cdb_ht_itemkey(db->dpcache, item);
-            /* must lock the main table inside the dpclock protection */
-            if (cdb_lock_trylock(db->mlock[bid % MLOCKNUM]) < 0) {
-                /* avoid dead lock since dpclock is holding */
-                cdb_lock_unlock(db->dpclock);
-                /* do nothing this time */
-                break;
-            }
-            cdb_ht_poptail(db->dpcache);
-            cdb_lock_unlock(db->dpclock);
-
-            /* write out dirty page */
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            db->vio->wpage(db->vio, (CDBPAGE*)cdb_ht_itemval(db->dpcache, item), &off);
-            db->wcount++;
-            db->wtime += _cdb_timermicrosec(&ts);
-            db->mtable[bid] = off;
-            cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-            free(item);
-        }
-    }
-}
-
-
-/* check if the record cache size exceed the limit. clean oldest record if necessary */
-static void _cdb_recout(CDB *db)
-{
-    while (RCOVERFLOW(db)) {
-        cdb_lock_lock(db->rclock);
-        if (db->rcache->num)
-            cdb_ht_removetail(db->rcache);
-        cdb_lock_unlock(db->rclock);
-    }
-}
-
-
-/* get all offsets from index(page) by key, even if only one of them at most is valid.
- Others are due to the hash collision */
-int cdb_getoff(CDB *db, uint64_t hash, FOFF **offs, int locked) 
-{
-    char sbuf[SBUFSIZE];
-    CDBPAGE *page = NULL;
-    int rnum;
-    bool incache = true;
-    uint32_t bid = (hash >> 24) % db->hsize;
-    PHASH phash;
-
-    phash.i1 = hash & 0xff;
-    phash.i2 = (hash >> 8) & 0xffff;
-
-    if (db->bf) {
-        uint64_t bfkey = (bid << 24) | (hash & 0xffffff);
-        /* check the key-hash in bloom filter? return now if not exist */
-        cdb_lock_lock(db->bflock);
-        if (!cdb_bf_exist(db->bf, &bfkey, SI8)) {
-            cdb_lock_unlock(db->bflock);
-            return 0;
-        }
-        cdb_lock_unlock(db->bflock);
-    }
-
-    if (locked == CDB_NOTLOCKED) cdb_lock_lock(db->mlock[bid % MLOCKNUM]);
-    /* page exists in clean page cache? */
-    if (db->pcache) {
-        cdb_lock_lock(db->pclock);
-        page = cdb_ht_get2(db->pcache, &bid, SI4, true);
-        cdb_lock_unlock(db->pclock);
-    }
-
-    /* not in pcache, exists in dirty page cache? */
-    if (page == NULL && db->dpcache) {
-        cdb_lock_lock(db->dpclock);
-        page = cdb_ht_get2(db->dpcache, &bid, SI4, true);
-        cdb_lock_unlock(db->dpclock);
-    }
-
-    if (page == NULL) {
-        /* not in dpcache either, read from disk */
-        incache = false;
-        db->pcmiss++;
-        /* page stays in stack by default */
-        page = (CDBPAGE *)sbuf;
-        if (OFFNOTNULL(db->mtable[bid])) {
-            /* page offset not null in main table */
-            int ret;
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            ret = db->vio->rpage(db->vio, &page, db->mtable[bid]);
-            db->rcount++;
-            db->rtime += _cdb_timermicrosec(&ts);
-
-            /* read page error, return */
-            if (ret < 0) {
-                if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-                if (page != (CDBPAGE *)sbuf)
-                    free(page);
-                return -1;
-            }
-        } else {
-            /* no page in this bucket */
-            page->cap = page->num = 0;
-            page->osize = 0;
-            OFFZERO(page->ooff);
-        }
-    } else {
-        db->pchit++;
-    }
-
-    rnum = 0;
-    for(uint32_t i = 0; i < page->num; i++) {
-        /* compare every hash in the page */
-        if (PHASHEQ(page->items[i].hash, phash)) {
-            (*offs)[rnum] = page->items[i].off;
-            /* result offset list stays in stack by default. Allocate one in heap if 
-            it exceeds the limit */
-            if (++rnum == SFOFFNUM) {
-                /* very little possibility goes here */
-                FOFF *tmp = (FOFF*)malloc((page->num - i + SFOFFNUM + 1) * sizeof(FOFF));
-                memcpy(tmp, *offs, SFOFFNUM * sizeof(FOFF));
-                *offs = tmp;
-            } 
-        }
-    }
-
-    if (!incache) {
-        /* set into clean page cache if not exists before */
-        if (db->pcache) {
-            cdb_lock_lock(db->pclock);
-            cdb_ht_insert2(db->pcache, &bid, SI4, page, MPAGESIZE(page));
-            cdb_lock_unlock(db->pclock);
-        }
-        /* if page now points to heap memory, free it */
-        if (page != (CDBPAGE *)sbuf) {
-            free(page);
-        }
-    }
-    if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-
-    /* check page cache overflow */
-    if (PCOVERFLOW(db))
-        _cdb_pageout(db);
-
-    return rnum;
-}
-
-
-/* replace a specified record's offset, may be used at disk space recycling 
- off indicates its previous offset, noff is the new offset. return negative if not found */
-int cdb_replaceoff(CDB *db, uint64_t hash, FOFF off, FOFF noff, int locked)
-{
-    char sbuf[SBUFSIZE];
-    CDBPAGE *page = NULL;
-    CDBHTITEM *pitem = NULL;
-    bool indpcache = false;
-    uint32_t bid = (hash >> 24) % db->hsize;
-    PHASH phash;
-    bool found = false;
-
-    phash.i1 = hash & 0xff;
-    phash.i2 = (hash >> 8) & 0xffff;
-
-    if (locked == CDB_NOTLOCKED) cdb_lock_lock(db->mlock[bid % MLOCKNUM]);
-    if (db->pcache) {
-        /* in clean page cache, since it would be modified, it should be deleted from pcache */
-        cdb_lock_lock(db->pclock);
-        pitem = cdb_ht_del(db->pcache, &bid, SI4);
-        cdb_lock_unlock(db->pclock);
-        if (pitem)
-            page = (CDBPAGE *)cdb_ht_itemval(db->pcache, pitem);
-    }
-    if (page == NULL && db->dpcache) {
-        /* not in pcache, but in dirty page cache */
-        cdb_lock_lock(db->dpclock);
-        page = cdb_ht_get2(db->dpcache, &bid, SI4, true);
-        cdb_lock_unlock(db->dpclock);
-        if (page)
-            indpcache = true;
-    }
-    if (page == NULL) {
-        /* not exists either, read from disk */
-        db->pcmiss++;
-        page = (CDBPAGE *)sbuf;
-        if (OFFNOTNULL(db->mtable[bid])) {
-            int ret;
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            ret = db->vio->rpage(db->vio, &page, db->mtable[bid]);
-            db->rcount++;
-            db->rtime += _cdb_timermicrosec(&ts);
-            
-            if (ret < 0) {
-                if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-                if (page != (CDBPAGE *)sbuf)
-                    free(page);
-                return -1;
-            }
-        } else {
-            /* nullified the empty page */
-            page->cap = page->num = 0;
-            page->osize = 0;
-            OFFZERO(page->ooff);
-        }
-    } else {
-        db->pchit++;
-    }
-
-    /* check and modify */
-    for(uint32_t i = 0; i < page->num; i++) {
-        if (PHASHEQ(page->items[i].hash, phash)
-            && OFFEQ(page->items[i].off, off)) {
-                page->items[i].off = noff;
-                found = true;
-                break;
-        }
-    }
-
-    if (db->dpcache && !indpcache) {
-        /* if page already dirty in cache, need not do anything */
-        /* dirty page cache is enabled but not exists before */
-        if (pitem) {
-            /* pitem not NULL indicates it belongs to pcache */
-            if (found) {
-                /* modified page */
-                cdb_lock_lock(db->dpclock);
-                cdb_ht_insert(db->dpcache, pitem);
-                cdb_lock_unlock(db->dpclock);
-            } else {
-                /* got from pcache, but not modified */
-                cdb_lock_lock(db->pclock);
-                cdb_ht_insert(db->pcache, pitem);
-                cdb_lock_unlock(db->pclock);
-            }
-            /* page belongs to memory in 'cache', must not free */
-        } else if (page != NULL) {
-            /* page read from disk, but not in cache */
-            cdb_lock_lock(db->dpclock);
-            cdb_ht_insert2(db->dpcache, &bid, SI4, page, MPAGESIZE(page));
-            cdb_lock_unlock(db->dpclock);
-            /* the 'page' won't be use anymore */
-            if (page != (CDBPAGE *)sbuf) 
-                free(page);
-        }
-    } else if (!db->dpcache){
-        /* no page cache. Write out dirty page immediately */
-        FOFF poff;
-        struct timespec ts;
-        _cdb_timerreset(&ts);
-        db->vio->wpage(db->vio, page, &poff);
-        db->wcount++;
-        db->wtime += _cdb_timermicrosec(&ts);
-
-        db->mtable[bid] = poff;
-        if (page != (CDBPAGE *)sbuf) 
-                free(page);
-    }
-    if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-
-    /* check page cache overflow */
-    if (PCOVERFLOW(db))
-        _cdb_pageout(db);
-
-    return 0;
-}
-
-
-/* insert/delete a key-offset pair from index page */
-int cdb_updatepage(CDB *db, uint64_t hash, FOFF off, int opt, int locked)
-{
-    char sbuf[SBUFSIZE], sbuf2[SBUFSIZE];
-    CDBPAGE *page = NULL, *npage = NULL;
-    CDBHTITEM *pitem = NULL, *nitem = NULL;
-    CDBHASHTABLE *tmpcache = NULL;
-    CDBLOCK *tmpclock = NULL;
-    int npsize = 0;
-    uint32_t bid = (hash >> 24) % db->hsize;
-    PHASH phash;
-
-    phash.i1 = hash & 0xff;
-    phash.i2 = (hash >> 8) & 0xffff;
-
-    if (locked == CDB_NOTLOCKED) cdb_lock_lock(db->mlock[bid % MLOCKNUM]);
-    /* firstly, try move the page out of the cache if possible, 
-    it assumes that the page would be modified(pair exists) */
-    if (db->pcache) {
-        /* try clean page cache */
-        cdb_lock_lock(db->pclock);
-        pitem = cdb_ht_del(db->pcache, &bid, SI4);
-        cdb_lock_unlock(db->pclock);
-        if (pitem) {
-            page = (CDBPAGE *)cdb_ht_itemval(db->pcache, pitem);
-            tmpcache = db->pcache;
-            tmpclock = db->pclock;
-        }
-    }
-    if (page == NULL && db->dpcache) {
-        /* try dirty page cache */
-        cdb_lock_lock(db->dpclock);
-        pitem = cdb_ht_del(db->dpcache, &bid, SI4);
-        cdb_lock_unlock(db->dpclock);
-        if (pitem) {
-            page = (CDBPAGE *)cdb_ht_itemval(db->dpcache, pitem);
-            tmpcache = db->dpcache;
-            tmpclock = db->dpclock;
-        }
-    }
-
-    if (page == NULL) {
-        db->pcmiss++;
-        page = (CDBPAGE *)sbuf;
-        /* doesn't exist in cache, read from disk */
-        if (OFFNOTNULL(db->mtable[bid])) {
-            int ret;
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            ret = db->vio->rpage(db->vio, &page, db->mtable[bid]);
-            db->rcount++;
-            db->rtime += _cdb_timermicrosec(&ts);
-
-            if (ret < 0) {
-                if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-                if (page != (CDBPAGE *)sbuf)
-                    free(page);
-                return -1;
-            }
-        } else {
-            page->cap = 0;
-            page->num = 0;
-            page->osize = 0;
-            OFFZERO(page->ooff);
-        }
-    } else {
-        db->pchit++;
-    }
-
-    npsize = MPAGESIZE(page);
-
-    if (opt == CDB_PAGEDELETEOFF)
-    ;//    npsize = MPAGESIZE(page) - sizeof(PITEM);
-    /* do not malloc new page on deletion */
-
-    else if (opt == CDB_PAGEINSERTOFF && page->cap == page->num) {
-    /* get a new page, from dirty page cache if possible */
-        npsize = MPAGESIZE(page) + CDB_PAGEINCR * sizeof(PITEM);
-        if (db->dpcache) {
-            nitem = cdb_ht_newitem(db->dpcache, SI4, npsize);
-            *(uint32_t*)cdb_ht_itemkey(db->dpcache, nitem) = bid;
-            npage = (CDBPAGE *)cdb_ht_itemval(db->dpcache, nitem);
-        } else {
-            /* no dpcache, use stack if size fits */
-            if (npsize > SBUFSIZE) 
-                npage = (CDBPAGE *)malloc(npsize);
-            else
-                npage = (CDBPAGE *)sbuf2;
-        }
-
-        /* initialize the new page */
-    
-        npage->bid = bid;
-        npage->oid = cdb_genoid(db);
-        npage->osize = page->osize;
-        npage->ooff = page->ooff;
-        npage->mtime = time(NULL);
-        npage->cap = page->cap + CDB_PAGEINCR;
-        npage->num = page->num;
-        memcpy(npage->items, page->items, page->num * sizeof(PITEM)); 
-        /* old page got from cache */
-        if (pitem)
-            free(pitem);
-        /* old page read from disk, if in stack? */
-        else if (page != (CDBPAGE *)sbuf)
-            free(page);
-
-        page = npage;
-        pitem = nitem;
-    }
-
-    uint32_t onum = page->num;
-
-    if (opt == CDB_PAGEDELETEOFF) {
-        bool found = false;
-        for(uint32_t i = 0; i < page->num; i++) {
-            if (!found) {
-                if (PHASHEQ(page->items[i].hash, phash)
-                    && OFFEQ(page->items[i].off, off))
-                {
-                    found = true;
-                    /* records num is consistant with index */
-                    cdb_lock_lock(db->stlock);
-                    db->rnum--;
-                    cdb_lock_unlock(db->stlock);
-                }
-            }
-            if (found && i + 1 < page->num)
-                page->items[i] = page->items[i+1];
-        }
-        if (found)
-            page->num--;
-    } else if (opt == CDB_PAGEINSERTOFF) {
-        bool found = false;
-        /* check already exist? */
-        for(uint32_t i = 0; i < page->num; i++) {
-            if (PHASHEQ(page->items[i].hash, phash)
-                && OFFEQ(page->items[i].off, off)) {
-                /* avoid exceptional deduplicated item */
-                found = true;
-                break;
-            }
-        }
-
-        /* append to the tail */
-        if (!found) {
-            page->items[page->num].hash = phash;
-            page->items[page->num].off = off;
-            page->num++;
-            /* records num is consistant with index */
-            cdb_lock_lock(db->stlock);
-            db->rnum++;
-            cdb_lock_unlock(db->stlock);
-            if (db->bf) {
-                uint64_t bfkey = (((hash >> 24) % db->hsize) << 24) | (hash & 0xffffff);
-                cdb_lock_lock(db->bflock);
-                cdb_bf_set(db->bf, &bfkey, SI8);
-                cdb_lock_unlock(db->bflock);
-            }
-        }
-    }
-
-    if (page->num == onum) {
-        /* nothing done */
-        if (pitem) {
-            /* insert the item back to the cache where it belongs */
-            cdb_lock_lock(tmpclock);
-            cdb_ht_insert(tmpcache, pitem);
-            cdb_lock_unlock(tmpclock);
-        } else {
-            if (page != (CDBPAGE *)sbuf2
-                    && page != (CDBPAGE *)sbuf)
-                free(page);
-        }
-        if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-        return -1;
-    } else {
-        if (pitem) {
-            cdb_lock_lock(db->dpclock);
-            cdb_ht_insert(db->dpcache, pitem);
-            cdb_lock_unlock(db->dpclock);
-        } else {
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            db->vio->wpage(db->vio, page, &off);
-            db->wcount++;
-            db->wtime += _cdb_timermicrosec(&ts);
-
-            db->mtable[bid] = off;
-            if (page != (CDBPAGE *)sbuf2
-                    && page != (CDBPAGE *)sbuf)
-                free(page);
-        }
-    }
-
-    if (locked == CDB_NOTLOCKED) cdb_lock_unlock(db->mlock[bid % MLOCKNUM]);
-
-    /* check page cache overflow */
-    if (PCOVERFLOW(db))
-        _cdb_pageout(db);
-
-    return 0;
-}
-
-
-/* check if an record with specified key-offset exists in index */
-bool cdb_checkoff(CDB *db, uint64_t hash, FOFF off, int locked)
-{
-    FOFF soffs[SFOFFNUM];
-    FOFF *soff = (FOFF *)soffs;
-    int dupnum;
-    int ret = false;
-
-    /* get all possible offsets */
-    dupnum = cdb_getoff(db, hash, &soff, locked);
-    for(int i = 0; i < dupnum; i++) {
-        if (OFFEQ(soff[i], off)) {
-            ret = true;
-            break;
-        }
-    }
-
-    if (soff != (FOFF *)soffs) {
-        free(soff);
-    }
-
-    return ret;
-}
-
-
-/* wrapper and simplified of set operation */
-int cdb_set(CDB *db, const char *key, int ksize, const char *val, int vsize)
-{
-    return cdb_set2(db, key, ksize, val, vsize, CDB_OVERWRITE, 0);
-}
-
-
-int cdb_set2(CDB *db, const char *key, int ksize, const char *val, int vsize, int opt, int expire)
-{
-    CDBREC rec;
-    FOFF ooff, noff;
-    uint32_t now = time(NULL);
-    uint64_t hash;
-    uint32_t lockid;
-    bool expired = false;
- 
-    if (db->vio == NULL) {
-        /* if it is a memdb, just operate on the record cache and return */
-        cdb_lock_lock(db->rclock);
-        cdb_ht_insert2(db->rcache, key, ksize, val, vsize);
-        cdb_lock_unlock(db->rclock);
-        if (RCOVERFLOW(db))
-            _cdb_recout(db);
-        return 0;
-    }
-
-    hash = CDBHASH64(key, ksize);
-    lockid = (hash >> 24) % db->hsize % MLOCKNUM;
-    OFFZERO(rec.ooff);
-    OFFZERO(ooff);
-    rec.osize = 0;
-    rec.key = (char*)key;
-    rec.val = (char*)val;
-    rec.ksize = ksize;
-    rec.vsize = vsize;
-    rec.oid = cdb_genoid(db);
-    rec.expire = expire? now + expire : 0;
-        
-    cdb_lock_lock(db->mlock[lockid]);
-    if (db->rcache) {
-        /* if record already exists, get its old meta info */
-        int item_vsize;
-        char *cval;
-        uint32_t old_expire = 0;
-        cdb_lock_lock(db->rclock);
-        cval = cdb_ht_get(db->rcache, key, ksize, &item_vsize, false);
-        if (cval) {
-            /* record already exists */
-            ooff = rec.ooff = *(FOFF*)cval;
-            rec.osize = item_vsize - SFOFF - SI4;
-            old_expire = *(uint32_t*)(cval + SFOFF); 
-        }
-        cdb_lock_unlock(db->rclock);
-        if (old_expire && old_expire <= now)
-            /* once exist but expired? */
-            expired = true;
-    }
-    
-    if (OFFNULL(ooff)) {
-        FOFF soffs[SFOFFNUM];
-        FOFF *soff = soffs;
-        char sbuf[SBUFSIZE];
-        CDBREC *rrec = (CDBREC*)sbuf;
-        
-        int retnum;
-        if ((retnum = cdb_getoff(db, hash, &soff, CDB_LOCKED)) < 0) {
-            cdb_lock_unlock(db->mlock[lockid]);
-            return -1;
-        }
-            
-        for(int i = 0; i < retnum; i++) {
-            /* check for duplicate records/older version*/
-            int cret;
-            if (rrec != (CDBREC*)sbuf) {
-                free(rrec);
-                rrec = (CDBREC*)sbuf;
-            }
-            
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            cret = db->vio->rrec(db->vio, &rrec, soff[i], false);
-            db->rcount++;
-            db->rtime += _cdb_timermicrosec(&ts);
-            
-            if (cret < 0)
-                continue;
-                
-            if ((uint32_t) ksize == rrec->ksize && memcmp(rrec->key, key, ksize) == 0) {
-                /* got its old meta info */
-                rec.osize = rrec->osize;
-                rec.ooff = rrec->ooff;
-                ooff = rec.ooff;
-                if (rrec->expire <= now)
-                    expired = true;
-                break;
-            }
-        }
-        if (soff != soffs)
-            free(soff);
-        if (rrec != (CDBREC*)sbuf) 
-            free(rrec);
-    }
-    
-    if (OFFNOTNULL(ooff) && !expired) {
-        /* record already exists*/
-        if (opt & CDB_INSERTIFNOEXIST) {
-            cdb_lock_unlock(db->mlock[lockid]);
-            cdb_seterrno(db, CDB_EXIST, __FILE__, __LINE__);
-            return -2;
-        }
-    } else {
-        if (opt & CDB_INSERTIFEXIST) {
-            cdb_lock_unlock(db->mlock[lockid]);
-            cdb_seterrno(db, CDB_NOTFOUND, __FILE__, __LINE__);
-            return -3;
-        }
-    }
-    
-    struct timespec ts;
-    _cdb_timerreset(&ts);
-    if (db->vio->wrec(db->vio, &rec, &noff) < 0) {
-        cdb_lock_unlock(db->mlock[lockid]);
-        return -1;
-    }
-    db->wcount++;
-    db->wtime += _cdb_timermicrosec(&ts);
-    
-    if (OFFNOTNULL(ooff)) {
-        cdb_replaceoff(db, hash, ooff, noff, CDB_LOCKED);
-    } else {
-        cdb_updatepage(db, hash, noff, CDB_PAGEINSERTOFF, CDB_LOCKED);
-    }
-    
-    if (db->rcache) {
-        if ((opt & CDB_INSERTCACHE) == CDB_INSERTCACHE) {
-            char *cval;
-            CDBHTITEM *item = cdb_ht_newitem(db->rcache, ksize, vsize + SI4 + SFOFF);
-            memcpy(cdb_ht_itemkey(db->rcache, item), key, ksize);
-            cval = cdb_ht_itemval(db->rcache, item);
-            memcpy(cval + SI4 + SFOFF, val, vsize);
-            *(FOFF*)(cval) = rec.ooff;
-            *(uint32_t*)(cval + SFOFF) = rec.expire;
-            cdb_lock_lock(db->rclock);
-            cdb_ht_insert(db->rcache, item);
-            cdb_lock_unlock(db->rclock);
-        }
-    } 
-    cdb_lock_unlock(db->mlock[lockid]);
-    
-    if (RCOVERFLOW(db))
-        _cdb_recout(db);
-
-    cdb_seterrno(db, CDB_SUCCESS, __FILE__, __LINE__);
-    return 0;
-}
-
-
-int cdb_is(CDB *db, const char *key, int ksize)
-{
-    FOFF soffs[SFOFFNUM];
-    FOFF *offs;
-    int dupnum, ret = -3;
-    uint64_t hash;
-    //uint32_t now = time(NULL);
-    uint32_t lockid;
-
-    if (db->rcache) {
-        char *cval;
-        cdb_lock_lock(db->rclock);
-        cval = cdb_ht_get(db->rcache, key, ksize, 0, true);
-        if (cval) {
-            db->rchit++;
-            cdb_lock_unlock(db->rclock);
-            return 0;
-        } else {
-            db->rcmiss++;
-            if (db->vio == NULL) {
-                cdb_lock_unlock(db->rclock);
-                return -3;
-            }
-        }
-        cdb_lock_unlock(db->rclock);
-    }
-
-    offs = soffs;
-    hash = CDBHASH64(key, ksize);
-    lockid = (hash >> 24) % db->hsize % MLOCKNUM;
-    cdb_lock_lock(db->mlock[lockid]);
-    dupnum = cdb_getoff(db, hash, &offs, CDB_LOCKED);
-    if (dupnum <= 0) {
-        cdb_lock_unlock(db->mlock[lockid]);
-        return -1;
-    }
-    else
-        ret = 0;
-    cdb_lock_unlock(db->mlock[lockid]);
-
-    if (RCOVERFLOW(db))
-        _cdb_recout(db);
-
-    if (offs != soffs)
-        free(offs);
-
-    if (ret < 0)
-        cdb_seterrno(db, CDB_NOTFOUND, __FILE__, __LINE__);
-    else {
-        db->rcmiss++;
-        cdb_seterrno(db, CDB_SUCCESS, __FILE__, __LINE__);
-    }
-    return ret;
-}
-
-
-int cdb_get(CDB *db, const char *key, int ksize, void **val, int *vsize)
-{
-    char sbuf[SBUFSIZE];
-    CDBREC *rec = (CDBREC *)sbuf;
-    FOFF soffs[SFOFFNUM];
-    FOFF *offs;
-    int dupnum, ret = -3;
-    uint64_t hash;
-    uint32_t now = time(NULL);
-    uint32_t lockid;
-
-    *vsize = 0;
-    *val = NULL;
-    if (db->rcache) {
-        char *cval;
-        cdb_lock_lock(db->rclock);
-        cval = cdb_ht_get(db->rcache, key, ksize, vsize, true);
-        if (cval) {
-            db->rchit++;
-            if (db->vio) {
-                (*vsize) -= SI4 + SFOFF;
-                if (*(uint32_t*)(cval + SFOFF)
-                    && *(uint32_t*)(cval + SFOFF) <= now) {
-                    cdb_lock_unlock(db->rclock);
-                    /* not found no not report error now */
-                    //cdb_seterrno(db, CDB_NOTFOUND, __FILE__, __LINE__);
-                    return -3;
-                }
-                cval = (void*)(cval + SI4 + SFOFF);
-            }
-            *val = malloc(*vsize);
-            memcpy(*val, cval, *vsize);
-            cdb_lock_unlock(db->rclock);
-            return 0;
-        } else {
-            db->rcmiss++;
-            if (db->vio == NULL) {
-                cdb_lock_unlock(db->rclock);
-                return -3;
-            }
-        }
-        cdb_lock_unlock(db->rclock);
-    }
-
-    offs = soffs;
-    hash = CDBHASH64(key, ksize);
-    lockid = (hash >> 24) % db->hsize % MLOCKNUM;
-    cdb_lock_lock(db->mlock[lockid]);
-    dupnum = cdb_getoff(db, hash, &offs, CDB_LOCKED);
-    if (dupnum < 0) {
-        cdb_lock_unlock(db->mlock[lockid]);
-        return -1;
-    }
-
-    for(int i = 0; i < dupnum; i++) {
-        int cret;
-        if (rec != (CDBREC*)sbuf) {
-            free(rec);
-            rec = (CDBREC*)sbuf;
-        }
-
-        struct timespec ts;
-        _cdb_timerreset(&ts);
-        cret = db->vio->rrec(db->vio, &rec, offs[i], true);
-        db->rcount++;
-        db->rtime += _cdb_timermicrosec(&ts);
-
-        if (cret < 0)
-            continue;
-
-        if ((uint32_t) ksize == rec->ksize && memcmp(rec->key, key, ksize) == 0) {
-            if (rec->expire && rec->expire <= now) {
-                break;
-            }
-            *vsize = rec->vsize;
-            *val = malloc(*vsize);
-            memcpy(*val, rec->val, *vsize);
-            ret = 0;
-            break;
-        } 
-    }
-
-    if (ret == 0 && db->rcache) {
-        char *cval;
-        CDBHTITEM *item = cdb_ht_newitem(db->rcache, ksize, *vsize + SI4 + SFOFF);
-        memcpy(cdb_ht_itemkey(db->rcache, item), key, ksize);
-        cval = cdb_ht_itemval(db->rcache, item);
-        memcpy(cval + SI4 + SFOFF, *val, *vsize);
-        *(FOFF*)(cval) = rec->ooff;
-        *(uint32_t*)(cval + SFOFF) = rec->expire;
-        cdb_lock_lock(db->rclock);
-        cdb_ht_insert(db->rcache, item);
-        cdb_lock_unlock(db->rclock);
-    }
-    cdb_lock_unlock(db->mlock[lockid]);
-    
-    if (RCOVERFLOW(db))
-        _cdb_recout(db);
-            
-    if (offs != soffs)
-        free(offs);
-        
-    if (rec != (CDBREC*)sbuf) 
-        free(rec);
-
-    if (ret < 0)
-        cdb_seterrno(db, CDB_NOTFOUND, __FILE__, __LINE__);
-    else {
-        db->rcmiss++;
-        cdb_seterrno(db, CDB_SUCCESS, __FILE__, __LINE__);
-    }
-    return ret;
-}
-
-
-void cdb_free_val(void **val)
-{
-    if (*val) 
-        free(*val);
-    *val = NULL;
-}
-
-
-int cdb_del(CDB *db, const char *key, int ksize)
-{
-    FOFF ooff;
-    CDBREC rec;
-    uint32_t lockid;
-    uint64_t hash;
-    
-    OFFZERO(rec.ooff);
-    OFFZERO(ooff);
-    rec.osize = 0;
-    rec.key = (char*)key;
-    rec.ksize = ksize;
-    rec.val = NULL;
-    rec.vsize = 0;
-    
-    if (db->vio == NULL) {
-        /* if it is a memdb, just operate on the record cache and return */
-        cdb_lock_lock(db->rclock);
-        cdb_ht_del2(db->rcache, key, ksize);
-        cdb_lock_unlock(db->rclock);
-        if (RCOVERFLOW(db))
-            _cdb_recout(db);
-        return 0;
-    }
-    
-    hash = CDBHASH64(key, ksize);
-    lockid = (hash >> 24) % db->hsize % MLOCKNUM;
-    cdb_lock_lock(db->mlock[lockid]);
-    if (db->rcache) {
-        /* if record already exists, get its old meta info */
-        CDBHTITEM *item;
-        cdb_lock_lock(db->rclock);
-        item = cdb_ht_del(db->rcache, key, ksize);
-        cdb_lock_unlock(db->rclock);
-        if (item) {
-            char *cval = cdb_ht_itemval(db->rcache, item);
-            ooff = rec.ooff = *(FOFF*)cval;
-            rec.osize = item->vsize - SFOFF - SI4;
-            rec.expire = *(uint32_t*)(cval + SFOFF);
-            free(item);
-        }
-    }
-    
-    if (OFFNULL(ooff)) {
-        FOFF soffs[SFOFFNUM];
-        FOFF *soff = soffs;
-        char sbuf[SBUFSIZE];
-        CDBREC *rrec = (CDBREC*)sbuf;
-        
-        int retnum;
-        if ((retnum = cdb_getoff(db, hash, &soff, CDB_LOCKED)) < 0) {
-            cdb_lock_unlock(db->mlock[lockid]);
-            return -1;
-        }
-            
-        for(int i = 0; i < retnum; i++) {
-            /* check for duplicate records/older version*/
-            int cret;
-            if (rrec != (CDBREC*)sbuf) {
-                free(rrec);
-                rrec = (CDBREC*)sbuf;
-            }
-            
-            struct timespec ts;
-            _cdb_timerreset(&ts);
-            cret = db->vio->rrec(db->vio, &rrec, soff[i], false);
-            db->rcount++;
-            db->rtime += _cdb_timermicrosec(&ts);
-            
-            if (cret < 0)
-                continue;
-                
-            if ((uint32_t) ksize == rrec->ksize && memcmp(rrec->key, key, ksize) == 0) {
-                /* got its old meta info */
-                rec.osize = rrec->osize;
-                rec.ooff = rrec->ooff;
-                ooff = rec.ooff;
-                break;
-            }
-        }
-        if (soff != soffs)
-            free(soff);
-        if (rrec != (CDBREC*)sbuf) 
-            free(rrec);
-    }
-    
-    if (OFFNOTNULL(ooff)) {
-        cdb_updatepage(db, hash, ooff, CDB_PAGEDELETEOFF, CDB_LOCKED);
-        cdb_lock_unlock(db->mlock[lockid]);
-        
-        struct timespec ts;
-        _cdb_timerreset(&ts);
-        db->vio->drec(db->vio, &rec, ooff);
-        //if ( < 0)
-          // return -1;  succeed or not doesn't matter
-        db->wcount++;
-        db->wtime += _cdb_timermicrosec(&ts);
-        cdb_seterrno(db, CDB_SUCCESS, __FILE__, __LINE__);
-        return 0;
-    } else {
-        cdb_lock_unlock(db->mlock[lockid]);
-        cdb_seterrno(db, CDB_NOTFOUND, __FILE__, __LINE__);
-        return -3;
-    }
-}
-
-
-void cdb_stat(CDB *db, CDBSTAT *stat)
-{
-    if (stat == NULL) {
-        db->rchit = db->rcmiss = 0;
-        db->pchit = db->pcmiss = 0;
-        db->rcount = db->rtime = 0;
-        db->wcount = db->wtime = 0;
-    } else {
-        stat->rnum = db->rnum;
-        stat->rcnum = db->rcache? db->rcache->num : 0;
-        stat->pnum = db->hsize;
-        stat->pcnum = (db->pcache? db->pcache->num : 0) 
-            + (db->dpcache? db->dpcache->num : 0);
-        stat->rchit = db->rchit;
-        stat->rcmiss = db->rcmiss;
-        stat->pchit = db->pchit;
-        stat->pcmiss = db->pcmiss;
-        stat->rlatcy = db->rcount ? db->rtime / db->rcount : 0;
-        stat->wlatcy = db->wcount ? db->wtime / db->wcount : 0;
-    }
-}
-
-
-int cdb_close(CDB *db)
-{
-    if (!db->opened)
-        return -1;
-
-    if (db->bgtask)
-        cdb_bgtask_stop(db->bgtask);
-    if (db->rcache)
-        cdb_ht_destroy(db->rcache);
-    if (db->pcache)
-        cdb_ht_destroy(db->pcache);
-    if (db->dpcache) {
-        cdb_flushalldpage(db);
-        cdb_ht_destroy(db->dpcache);
-    }
-
-    if (db->vio) {
-        db->vio->whead(db->vio);
-        db->vio->close(db->vio);
-        cdb_vio_destroy(db->vio);
-    }
-    if (db->mtable)
-        free(db->mtable);
-    db->opened = false;
-    _cdb_defparam(db);
-    return 0;
-}
-
-
-void cdb_deferrorcb(void *arg, int errno, const char *file, int line)
-{
-    (void) arg;
-    fprintf(stderr, "DBERR: [%s:%d] %d - %s\n", file, line, errno, cdb_errmsg(errno));
-}
-
-
-int cdb_destroy(CDB *db)
-{
-    if (db->opened)
-        cdb_close(db);
-    for(int i = 0; i < MLOCKNUM; i++)
-        cdb_lock_destory(db->mlock[i]);
-    cdb_lock_destory(db->dpclock);
-    cdb_lock_destory(db->pclock);
-    cdb_lock_destory(db->rclock);
-    cdb_lock_destory(db->stlock);
-    cdb_lock_destory(db->oidlock);
-    cdb_lock_destory(db->bflock);
-    cdb_bgtask_destroy(db->bgtask);
-    pthread_key_delete(*(pthread_key_t*)db->errkey);
-    free(db->errkey);
-    free(db);
-    return 0;
-}
-
-
-

3rdparty/cuttdb/src/cdb_core.h

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-#ifndef _CDB_CORE_H_
-#define _CDB_CORE_H_
-#include "cuttdb.h"
-#include "cdb_types.h"
-#include "cdb_hashtable.h"
-#include "cdb_bloomfilter.h"
-#include "cdb_lock.h"
-#include "cdb_vio.h"
-#include "cdb_bgtask.h"
-#include <stdint.h>
-#include <stdbool.h>
-
-
-enum {
-    CDB_PAGEDELETEOFF = 0,
-    CDB_PAGEINSERTOFF = 1,
-};
-
-/* the DB object */
-struct CDB
-{
-    /* size limit for record cache */
-    uint64_t rclimit;
-    /* size limit for index page cache */
-    uint64_t pclimit;
-    /* size of bloom filter */
-    uint64_t bfsize;
-    /* record number in db */
-    uint64_t rnum;
-    /* always increment operation id */
-    uint64_t oid;
-    /* recovery point oid */
-    uint64_t roid;
-    /* hash table size */
-    uint32_t hsize;
-    /* last timestamp of no dirty page state */
-    uint32_t ndpltime;
-    /* currently the database opened or not */
-    bool opened;
-    /* the size for a disk seek&read, should not greater than SBUFSIZE */
-    uint32_t areadsize;
-
-    /* record cache */
-    CDBHASHTABLE *rcache;
-    /* (clean) index page cache */
-    CDBHASHTABLE *pcache;
-    /* dirty index page cache */
-    CDBHASHTABLE *dpcache;
-    /* Bloom Filter */
-    CDBBLOOMFILTER *bf;
-
-    /* lock for rcache */
-    CDBLOCK *rclock;
-    /* lock for pcache */
-    CDBLOCK *pclock;
-    /* lock for dpcache */
-    CDBLOCK *dpclock;
-    /* lock for hash table operation, split to MLOCKNUM groups */
-    CDBLOCK *mlock[MLOCKNUM];
-    /* lock for statistic */
-    CDBLOCK *stlock;
-    /* lock for operation id */
-    CDBLOCK *oidlock;
-    /* lock for bloom filter */
-    CDBLOCK *bflock;
-    /* background tasks in another thread */
-    CDBBGTASK *bgtask;
-
-    /* main hash table, contains 'hsize' elements */
-    FOFF *mtable;
-    /* disk i/o layer object */
-    CDBVIO *vio;
-
-    /* callback function when error occurs */
-    CDB_ERRCALLBACK errcb;
-    /* argument for callback function */
-    void *errcbarg;
-    /* key to get error code in current thread */
-    void *errkey;
-
-    /* statistics below, this fields have no lock protection */
-    /* record cache hit/miss */
-    uint64_t rchit;
-    uint64_t rcmiss;
-    /* page cache hit/miss */
-    uint64_t pchit;
-    uint64_t pcmiss;
-    /* cumulative disk read time */
-    uint64_t rtime;
-    /* number of disk read operation */
-    uint64_t rcount;
-    /* cumulative disk write time */
-    uint64_t wtime;
-    /* number of disk write operation */
-    uint64_t wcount;
-};
-
-
-bool cdb_checkoff(CDB *db, uint64_t hash, FOFF off, int locked);
-int cdb_getoff(CDB *db, uint64_t hash, FOFF **offs, int locked);
-int cdb_replaceoff(CDB *db, uint64_t hash, FOFF off, FOFF noff, int locked);
-int cdb_updatepage(CDB *db, uint64_t hash, FOFF off, int opt, int locked);
-void cdb_flushalldpage(CDB *db);
-uint64_t cdb_genoid(CDB *db);
-
-#endif
-

3rdparty/cuttdb/src/cdb_crc64.c

-/*
- *   CuttDB - a fast key-value storage engine
- *
- *
- *   http://code.google.com/p/cuttdb/
- *   
- *   Copyright (c) 2012, Siyuan Fu.  All rights reserved.
- *   Use and distribution licensed under the BSD license. 
- *   See the LICENSE file for full text
- *
- *   Author: Siyuan Fu <fusiyuan2010@gmail.com>
- *
- */
-
-
-/**************************************************************
-* *
-* Fichier : crc64.c *
-* Fonction pour calculer le CRC64 *
-* *
-**************************************************************/
-#include "cdb_crc64.h"
-
-
-#define CONST64(n) (n##ULL)
-static uint64_t CRC64_Table[256] =
-{
-    CONST64(0x0000000000000000), CONST64(0x42f0e1eba9ea3693),
-    CONST64(0x85e1c3d753d46d26), CONST64(0xc711223cfa3e5bb5),
-    CONST64(0x493366450e42ecdf), CONST64(0x0bc387aea7a8da4c),
-    CONST64(0xccd2a5925d9681f9), CONST64(0x8e224479f47cb76a),
-    CONST64(0x9266cc8a1c85d9be), CONST64(0xd0962d61b56fef2d),
-    CONST64(0x17870f5d4f51b498), CONST64(0x5577eeb6e6bb820b),
-    CONST64(0xdb55aacf12c73561), CONST64(0x99a54b24bb2d03f2),
-    CONST64(0x5eb4691841135847), CONST64(0x1c4488f3e8f96ed4),
-    CONST64(0x663d78ff90e185ef), CONST64(0x24cd9914390bb37c),
-    CONST64(0xe3dcbb28c335e8c9), CONST64(0xa12c5ac36adfde5a),
-    CONST64(0x2f0e1eba9ea36930), CONST64(0x6dfeff5137495fa3),
-    CONST64(0xaaefdd6dcd770416), CONST64(0xe81f3c86649d3285),
-    CONST64(0xf45bb4758c645c51), CONST64(0xb6ab559e258e6ac2),
-    CONST64(0x71ba77a2dfb03177), CONST64(0x334a9649765a07e4),
-    CONST64(0xbd68d2308226b08e), CONST64(0xff9833db2bcc861d),
-    CONST64(0x388911e7d1f2dda8), CONST64(0x7a79f00c7818eb3b),
-    CONST64(0xcc7af1ff21c30bde), CONST64(0x8e8a101488293d4d),
-    CONST64(0x499b3228721766f8), CONST64(0x0b6bd3c3dbfd506b),
-    CONST64(0x854997ba2f81e701), CONST64(0xc7b97651866bd192),
-    CONST64(0x00a8546d7c558a27), CONST64(0x4258b586d5bfbcb4),
-    CONST64(0x5e1c3d753d46d260), CONST64(0x1cecdc9e94ace4f3),
-    CONST64(0xdbfdfea26e92bf46), CONST64(0x990d1f49c77889d5),
-    CONST64(0x172f5b3033043ebf), CONST64(0x55dfbadb9aee082c),
-    CONST64(0x92ce98e760d05399), CONST64(0xd03e790cc93a650a),
-    CONST64(0xaa478900b1228e31), CONST64(0xe8b768eb18c8b8a2),
-    CONST64(0x2fa64ad7e2f6e317), CONST64(0x6d56ab3c4b1cd584),
-    CONST64(0xe374ef45bf6062ee), CONST64(0xa1840eae168a547d),
-    CONST64(0x66952c92ecb40fc8), CONST64(0x2465cd79455e395b),
-    CONST64(0x3821458aada7578f), CONST64(0x7ad1a461044d611c),
-    CONST64(0xbdc0865dfe733aa9), CONST64(0xff3067b657990c3a),
-    CONST64(0x711223cfa3e5bb50), CONST64(0x33e2c2240a0f8dc3),
-    CONST64(0xf4f3e018f031d676), CONST64(0xb60301f359dbe0e5),
-    CONST64(0xda050215ea6c212f), CONST64(0x98f5e3fe438617bc),
-    CONST64(0x5fe4c1c2b9b84c09), CONST64(0x1d14202910527a9a),
-    CONST64(0x93366450e42ecdf0), CONST64(0xd1c685bb4dc4fb63),
-    CONST64(0x16d7a787b7faa0d6), CONST64(0x5427466c1e109645),
-    CONST64(0x4863ce9ff6e9f891), CONST64(0x0a932f745f03ce02),
-    CONST64(0xcd820d48a53d95b7), CONST64(0x8f72eca30cd7a324),
-    CONST64(0x0150a8daf8ab144e), CONST64(0x43a04931514122dd),
-    CONST64(0x84b16b0dab7f7968), CONST64(0xc6418ae602954ffb),
-    CONST64(0xbc387aea7a8da4c0), CONST64(0xfec89b01d3679253),
-    CONST64(0x39d9b93d2959c9e6), CONST64(0x7b2958d680b3ff75),
-    CONST64(0xf50b1caf74cf481f), CONST64(0xb7fbfd44dd257e8c),
-    CONST64(0x70eadf78271b2539), CONST64(0x321a3e938ef113aa),
-    CONST64(0x2e5eb66066087d7e), CONST64(0x6cae578bcfe24bed),
-    CONST64(0xabbf75b735dc1058), CONST64(0xe94f945c9c3626cb),
-    CONST64(0x676dd025684a91a1), CONST64(0x259d31cec1a0a732),
-    CONST64(0xe28c13f23b9efc87), CONST64(0xa07cf2199274ca14),
-    CONST64(0x167ff3eacbaf2af1), CONST64(0x548f120162451c62),
-    CONST64(0x939e303d987b47d7), CONST64(0xd16ed1d631917144),
-    CONST64(0x5f4c95afc5edc62e), CONST64(0x1dbc74446c07f0bd),
-    CONST64(0xdaad56789639ab08), CONST64(0x985db7933fd39d9b),
-    CONST64(0x84193f60d72af34f), CONST64(0xc6e9de8b7ec0c5dc),
-    CONST64(0x01f8fcb784fe9e69), CONST64(0x43081d5c2d14a8fa),
-    CONST64(0xcd2a5925d9681f90), CONST64(0x8fdab8ce70822903),
-    CONST64(0x48cb9af28abc72b6), CONST64(0x0a3b7b1923564425),
-    CONST64(0x70428b155b4eaf1e), CONST64(0x32b26afef2a4998d),
-    CONST64(0xf5a348c2089ac238), CONST64(0xb753a929a170f4ab),
-    CONST64(0x3971ed50550c43c1), CONST64(0x7b810cbbfce67552),
-    CONST64(0xbc902e8706d82ee7), CONST64(0xfe60cf6caf321874),
-    CONST64(0xe224479f47cb76a0), CONST64(0xa0d4a674ee214033),
-    CONST64(0x67c58448141f1b86), CONST64(0x253565a3bdf52d15),
-    CONST64(0xab1721da49899a7f), CONST64(0xe9e7c031e063acec),
-    CONST64(0x2ef6e20d1a5df759), CONST64(0x6c0603e6b3b7c1ca),
-    CONST64(0xf6fae5c07d3274cd), CONST64(0xb40a042bd4d8425e),
-    CONST64(0x731b26172ee619eb), CONST64(0x31ebc7fc870c2f78),
-    CONST64(0xbfc9838573709812), CONST64(0xfd39626eda9aae81),
-    CONST64(0x3a28405220a4f534), CONST64(0x78d8a1b9894ec3a7),
-    CONST64(0x649c294a61b7ad73), CONST64(0x266cc8a1c85d9be0),
-    CONST64(0xe17dea9d3263c055), CONST64(0xa38d0b769b89f6c6),
-    CONST64(0x2daf4f0f6ff541ac), CONST64(0x6f5faee4c61f773f),
-    CONST64(0xa84e8cd83c212c8a), CONST64(0xeabe6d3395cb1a19),
-    CONST64(0x90c79d3fedd3f122), CONST64(0xd2377cd44439c7b1),
-    CONST64(0x15265ee8be079c04), CONST64(0x57d6bf0317edaa97),
-    CONST64(0xd9f4fb7ae3911dfd), CONST64(0x9b041a914a7b2b6e),
-    CONST64(0x5c1538adb04570db), CONST64(0x1ee5d94619af4648),
-    CONST64(0x02a151b5f156289c), CONST64(0x4051b05e58bc1e0f),
-    CONST64(0x87409262a28245ba), CONST64(0xc5b073890b687329),
-    CONST64(0x4b9237f0ff14c443), CONST64(0x0962d61b56fef2d0),
-    CONST64(0xce73f427acc0a965), CONST64(0x8c8315cc052a9ff6),
-    CONST64(0x3a80143f5cf17f13), CONST64(0x7870f5d4f51b4980),
-    CONST64(0xbf61d7e80f251235), CONST64(0xfd913603a6cf24a6),
-    CONST64(0x73b3727a52b393cc), CONST64(0x31439391fb59a55f),
-    CONST64(0xf652b1ad0167feea), CONST64(0xb4a25046a88dc879),
-    CONST64(0xa8e6d8b54074a6ad), CONST64(0xea16395ee99e903e),
-    CONST64(0x2d071b6213a0cb8b), CONST64(0x6ff7fa89ba4afd18),
-    CONST64(0xe1d5bef04e364a72), CONST64(0xa3255f1be7dc7ce1),
-    CONST64(0x64347d271de22754), CONST64(0x26c49cccb40811c7),
-    CONST64(0x5cbd6cc0cc10fafc), CONST64(0x1e4d8d2b65facc6f),
-    CONST64(0xd95caf179fc497da), CONST64(0x9bac4efc362ea149),
-    CONST64(0x158e0a85c2521623), CONST64(0x577eeb6e6bb820b0),
-    CONST64(0x906fc95291867b05), CONST64(0xd29f28b9386c4d96),
-    CONST64(0xcedba04ad0952342), CONST64(0x8c2b41a1797f15d1),
-    CONST64(0x4b3a639d83414e64), CONST64(0x09ca82762aab78f7),
-    CONST64(0x87e8c60fded7cf9d), CONST64(0xc51827e4773df90e),
-    CONST64(0x020905d88d03a2bb), CONST64(0x40f9e43324e99428),
-    CONST64(0x2cffe7d5975e55e2), CONST64(0x6e0f063e3eb46371),
-    CONST64(0xa91e2402c48a38c4), CONST64(0xebeec5e96d600e57),
-    CONST64(0x65cc8190991cb93d), CONST64(0x273c607b30f68fae),
-    CONST64(0xe02d4247cac8d41b), CONST64(0xa2dda3ac6322e288),
-    CONST64(0xbe992b5f8bdb8c5c), CONST64(0xfc69cab42231bacf),
-    CONST64(0x3b78e888d80fe17a), CONST64(0x7988096371e5d7e9),
-    CONST64(0xf7aa4d1a85996083), CONST64(0xb55aacf12c735610),
-    CONST64(0x724b8ecdd64d0da5), CONST64(0x30bb6f267fa73b36),
-    CONST64(0x4ac29f2a07bfd00d), CONST64(0x08327ec1ae55e69e),
-    CONST64(0xcf235cfd546bbd2b), CONST64(0x8dd3bd16fd818bb8),
-    CONST64(0x03f1f96f09fd3cd2), CONST64(0x41011884a0170a41),
-    CONST64(0x86103ab85a2951f4), CONST64(0xc4e0db53f3c36767),
-    CONST64(0xd8a453a01b3a09b3), CONST64(0x9a54b24bb2d03f20),
-    CONST64(0x5d45907748ee6495), CONST64(0x1fb5719ce1045206),
-    CONST64(0x919735e51578e56c), CONST64(0xd367d40ebc92d3ff),
-    CONST64(0x1476f63246ac884a), CONST64(0x568617d9ef46bed9),
-    CONST64(0xe085162ab69d5e3c), CONST64(0xa275f7c11f7768af),
-    CONST64(0x6564d5fde549331a), CONST64(0x279434164ca30589),
-    CONST64(0xa9b6706fb8dfb2e3), CONST64(0xeb46918411358470),
-    CONST64(0x2c57b3b8eb0bdfc5), CONST64(0x6ea7525342e1e956),
-    CONST64(0x72e3daa0aa188782), CONST64(0x30133b4b03f2b111),
-    CONST64(0xf7021977f9cceaa4), CONST64(0xb5f2f89c5026dc37),
-    CONST64(0x3bd0bce5a45a6b5d), CONST64(0x79205d0e0db05dce),
-    CONST64(0xbe317f32f78e067b), CONST64(0xfcc19ed95e6430e8),
-    CONST64(0x86b86ed5267cdbd3), CONST64(0xc4488f3e8f96ed40),
-    CONST64(0x0359ad0275a8b6f5), CONST64(0x41a94ce9dc428066),
-    CONST64(0xcf8b0890283e370c), CONST64(0x8d7be97b81d4019f),
-    CONST64(0x4a6acb477bea5a2a), CONST64(0x089a2aacd2006cb9),
-    CONST64(0x14dea25f3af9026d), CONST64(0x562e43b4931334fe),
-    CONST64(0x913f6188692d6f4b), CONST64(0xd3cf8063c0c759d8),
-    CONST64(0x5dedc41a34bbeeb2), CONST64(0x1f1d25f19d51d821),
-    CONST64(0xd80c07cd676f8394), CONST64(0x9afce626ce85b507)
-};
-
-
-uint64_t cdb_crc64(const void *buf, uint32_t len)
-{
-    uint32_t i;
-    uint64_t crc = 0xFFFFFFFFFFFFFFFF;
-    uint8_t *cbuf = (uint8_t *)buf;
-
-    for (i = 0; i < len; i++) {
-        crc = CRC64_Table[(uint8_t)(crc >> 56) ^ *cbuf++] ^ (crc << 8);
-    }
-    return crc;
-} 
-