From 021073907fa53eba7a68c4f6bcb7630e15729f2a Mon Sep 17 00:00:00 2001 From: dsandersoremutah Date: Fri, 26 May 2006 23:17:49 +0000 Subject: [PATCH] Added .cpp and .h files under the sql/src subdirectory git-svn-id: https://svn.code.sf.net/p/flaim/code/trunk@469 0109f412-320b-0410-ab79-c3e0c5ffbbe6 --- sql/src/btreeinfo.cpp | 727 +++ sql/src/checksum.cpp | 702 +++ sql/src/f_btpool.cpp | 106 + sql/src/f_btpool.h | 79 + sql/src/f_btree.cpp | 11672 ++++++++++++++++++++++++++++++++++++++ sql/src/f_btree.h | 1058 ++++ sql/src/f_nici.cpp | 2991 ++++++++++ sql/src/f_nici.h | 268 + sql/src/fbtrset.cpp | 583 ++ sql/src/fbtrset.h | 184 + sql/src/fcache.h | 3320 +++++++++++ sql/src/fcollate.cpp | 174 + sql/src/fcollate.h | 92 + sql/src/fdbcnfig.cpp | 1307 +++++ sql/src/fdbcopy.cpp | 884 +++ sql/src/fdbremov.cpp | 319 ++ sql/src/fdbrenam.cpp | 446 ++ sql/src/fdict.cpp | 3776 ++++++++++++ sql/src/fdict.h | 509 ++ sql/src/fdllmain.cpp | 176 + sql/src/fdynbtre.cpp | 1198 ++++ sql/src/fdynbuf.cpp | 123 + sql/src/fdynbuf.h | 85 + sql/src/fdynsset.cpp | 328 ++ sql/src/fdynsset.h | 672 +++ sql/src/ffilehdr.cpp | 454 ++ sql/src/filesys.h | 411 ++ sql/src/flaimodbc.h | 442 ++ sql/src/flaimsql.h | 1710 ++++++ sql/src/flaimsys.h | 4359 ++++++++++++++ sql/src/flbackup.cpp | 2573 +++++++++ sql/src/flclose.cpp | 224 + sql/src/flconvrt.cpp | 726 +++ sql/src/flcreate.cpp | 472 ++ sql/src/fldbglog.cpp | 325 ++ sql/src/flerror.cpp | 153 + sql/src/flgethdr.cpp | 53 + sql/src/flindex.cpp | 1120 ++++ sql/src/flkeyret.cpp | 293 + sql/src/flmstat.cpp | 1453 +++++ sql/src/flmstat.h | 135 + sql/src/flog.cpp | 112 + sql/src/flog.h | 114 + sql/src/flopen.cpp | 1993 +++++++ sql/src/flreduce.cpp | 849 +++ sql/src/fltrabrt.cpp | 449 ++ sql/src/fltrbeg.cpp | 987 ++++ sql/src/fltrcmit.cpp | 554 ++ sql/src/fnumber.cpp | 1001 ++++ sql/src/frestore.cpp | 307 + sql/src/frow.cpp | 4574 +++++++++++++++ sql/src/fsblk_u.cpp | 208 + sql/src/fscursor.cpp | 1037 ++++ sql/src/fscursor.h | 334 ++ sql/src/fsdatacu.cpp | 783 +++ sql/src/fslfile.cpp | 503 ++ sql/src/fslfileu.cpp | 1118 ++++ sql/src/fsrefupd.cpp | 100 + sql/src/fsrvlock.cpp | 1204 ++++ sql/src/fsrvlock.h | 424 ++ sql/src/fstructs.h | 1377 +++++ sql/src/fsuperfl.cpp | 1003 ++++ sql/src/fsuperfl.h | 215 + sql/src/fsysdata.cpp | 3308 +++++++++++ sql/src/funicode.cpp | 1075 ++++ sql/src/fvector.cpp | 1355 +++++ sql/src/fxml.h | 582 ++ sql/src/fxpath.h | 454 ++ sql/src/kybldkey.cpp | 2001 +++++++ sql/src/kybuild.cpp | 887 +++ sql/src/kycollat.cpp | 435 ++ sql/src/kyeword.cpp | 425 ++ sql/src/kyqsort.cpp | 1215 ++++ sql/src/kyunlock.cpp | 139 + sql/src/recover.cpp | 469 ++ sql/src/rfl.cpp | 5537 ++++++++++++++++++ sql/src/rfl.h | 664 +++ sql/src/scache.cpp | 7894 ++++++++++++++++++++++++++ sql/src/sqleval.cpp | 1486 +++++ sql/src/sqloptimize.cpp | 2762 +++++++++ sql/src/sqlparser.cpp | 577 ++ sql/src/translog.cpp | 323 ++ 82 files changed, 97516 insertions(+) create mode 100644 sql/src/btreeinfo.cpp create mode 100644 sql/src/checksum.cpp create mode 100644 sql/src/f_btpool.cpp create mode 100644 sql/src/f_btpool.h create mode 100644 sql/src/f_btree.cpp create mode 100644 sql/src/f_btree.h create mode 100644 sql/src/f_nici.cpp create mode 100644 sql/src/f_nici.h create mode 100644 sql/src/fbtrset.cpp create mode 100644 sql/src/fbtrset.h create mode 100644 sql/src/fcache.h create mode 100644 sql/src/fcollate.cpp create mode 100644 sql/src/fcollate.h create mode 100644 sql/src/fdbcnfig.cpp create mode 100644 sql/src/fdbcopy.cpp create mode 100644 sql/src/fdbremov.cpp create mode 100644 sql/src/fdbrenam.cpp create mode 100644 sql/src/fdict.cpp create mode 100644 sql/src/fdict.h create mode 100644 sql/src/fdllmain.cpp create mode 100644 sql/src/fdynbtre.cpp create mode 100644 sql/src/fdynbuf.cpp create mode 100644 sql/src/fdynbuf.h create mode 100644 sql/src/fdynsset.cpp create mode 100644 sql/src/fdynsset.h create mode 100644 sql/src/ffilehdr.cpp create mode 100644 sql/src/filesys.h create mode 100644 sql/src/flaimodbc.h create mode 100644 sql/src/flaimsql.h create mode 100644 sql/src/flaimsys.h create mode 100644 sql/src/flbackup.cpp create mode 100644 sql/src/flclose.cpp create mode 100644 sql/src/flconvrt.cpp create mode 100644 sql/src/flcreate.cpp create mode 100644 sql/src/fldbglog.cpp create mode 100644 sql/src/flerror.cpp create mode 100644 sql/src/flgethdr.cpp create mode 100644 sql/src/flindex.cpp create mode 100644 sql/src/flkeyret.cpp create mode 100644 sql/src/flmstat.cpp create mode 100644 sql/src/flmstat.h create mode 100644 sql/src/flog.cpp create mode 100644 sql/src/flog.h create mode 100644 sql/src/flopen.cpp create mode 100644 sql/src/flreduce.cpp create mode 100644 sql/src/fltrabrt.cpp create mode 100644 sql/src/fltrbeg.cpp create mode 100644 sql/src/fltrcmit.cpp create mode 100644 sql/src/fnumber.cpp create mode 100644 sql/src/frestore.cpp create mode 100644 sql/src/frow.cpp create mode 100644 sql/src/fsblk_u.cpp create mode 100644 sql/src/fscursor.cpp create mode 100644 sql/src/fscursor.h create mode 100644 sql/src/fsdatacu.cpp create mode 100644 sql/src/fslfile.cpp create mode 100644 sql/src/fslfileu.cpp create mode 100644 sql/src/fsrefupd.cpp create mode 100644 sql/src/fsrvlock.cpp create mode 100644 sql/src/fsrvlock.h create mode 100644 sql/src/fstructs.h create mode 100644 sql/src/fsuperfl.cpp create mode 100644 sql/src/fsuperfl.h create mode 100644 sql/src/fsysdata.cpp create mode 100644 sql/src/funicode.cpp create mode 100644 sql/src/fvector.cpp create mode 100644 sql/src/fxml.h create mode 100644 sql/src/fxpath.h create mode 100644 sql/src/kybldkey.cpp create mode 100644 sql/src/kybuild.cpp create mode 100644 sql/src/kycollat.cpp create mode 100644 sql/src/kyeword.cpp create mode 100644 sql/src/kyqsort.cpp create mode 100644 sql/src/kyunlock.cpp create mode 100644 sql/src/recover.cpp create mode 100644 sql/src/rfl.cpp create mode 100644 sql/src/rfl.h create mode 100644 sql/src/scache.cpp create mode 100644 sql/src/sqleval.cpp create mode 100644 sql/src/sqloptimize.cpp create mode 100644 sql/src/sqlparser.cpp create mode 100644 sql/src/translog.cpp diff --git a/sql/src/btreeinfo.cpp b/sql/src/btreeinfo.cpp new file mode 100644 index 0000000..dd06299 --- /dev/null +++ b/sql/src/btreeinfo.cpp @@ -0,0 +1,727 @@ +//------------------------------------------------------------------------------ +// Desc: Class for gathering b-tree information. +// +// Tabs: 3 +// +// Copyright (c) 2005-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: btreeinfo.cpp 3116 2006-01-19 13:31:53 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Collect information about a block. +****************************************************************************/ +RCODE F_BTreeInfo::collectBlockInfo( + F_Db * pDb, + LFILE * pLFile, + BTREE_INFO * pBTreeInfo, + F_BTREE_BLK_HDR * pBlkHdr, + F_INDEX * pIndex) +{ + RCODE rc = NE_SFLM_OK; + SFLM_BTREE_LEVEL_INFO * pLevelInfo = &pBTreeInfo->levelInfo [m_uiCurrLevel]; + FLMUINT uiLoop; + FLMBYTE * pucOffset; + FLMBYTE * pucEntry; + FLMBYTE * pucTmp; + FLMUINT uiKeyLen; + FLMUINT uiDataLen; + F_CachedBlock * pCachedBlock = NULL; + FLMUINT uiBlkAddr; + FLMUINT uiOADataLen; + FLMBYTE * pucKey; + + // Block level better be the same as our current level we are + // supposedly processing. + + flmAssert( (FLMUINT)pBlkHdr->ui8BlkLevel == m_uiCurrLevel); + + pLevelInfo->ui64BlockCount++; + pLevelInfo->ui64BlockLength += (FLMUINT64)m_uiBlockSize; + pLevelInfo->ui64ElmOffsetOverhead += ((FLMUINT64)pBlkHdr->ui16NumKeys * 2); + pLevelInfo->ui64ElmCount += (FLMUINT64)pBlkHdr->ui16NumKeys; + pLevelInfo->ui64BlockFreeSpace += pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + + // Traverse through each key. + + pucOffset = (FLMBYTE *)pBlkHdr + sizeofBTreeBlkHdr( pBlkHdr); + for (uiLoop = 0; + uiLoop < (FLMUINT)pBlkHdr->ui16NumKeys; + uiLoop++, pucOffset += 2) + { + pucEntry = ((FLMBYTE *)pBlkHdr) + FB2UW( pucOffset); + uiDataLen = 0; + uiOADataLen = 0; + uiKeyLen = 0; + switch (pBlkHdr->stdBlkHdr.ui8BlkType) + { + case BT_LEAF: + + // Elements are: + // Key Length - 2 bytes + // Key - Key Length bytes + + uiKeyLen = (FLMUINT)FB2UW( pucEntry); + pLevelInfo->ui64ElmKeyLengthOvhd += 2; + pucKey = pucEntry + 2; + break; + case BT_NON_LEAF: + + // Elements are: + // Child Blk Address - 4 bytes + // Key Length - 2 bytes + // Key - Key Length bytes + + pLevelInfo->ui64ElmChildAddrsOvhd += 4; + uiKeyLen = (FLMUINT)FB2UW( pucEntry + 4); + pLevelInfo->ui64ElmKeyLengthOvhd += 2; + pucKey = pucEntry + 6; + break; + case BT_NON_LEAF_COUNTS: + + // Elements are: + // Child Block Address - 4 bytes + // Counts - 4 bytes + // Key Length - 2 bytes + // Key - Key Length bytes + + uiKeyLen = (FLMUINT)FB2UW( pucEntry + 8); + pLevelInfo->ui64ElmKeyLengthOvhd += 2; + pLevelInfo->ui64ElmCountsOvhd += 4; + pLevelInfo->ui64ElmChildAddrsOvhd += 4; + pucKey = pucEntry + 10; + break; + case BT_LEAF_DATA: + + // Elements are: + // Flags - 1 byte + // Key Length - 1 or 2 bytes + // Data Length - 1 or 2 bytes + // Overall data Length - 0 or 4 bytes + // Key - Key Length bytes + // Data - Data Length bytes. NOTE: May be a four byte blk + // address if data is stored in data-only blocks. + + pLevelInfo->ui64ElmFlagOvhd++; + if (!(*pucEntry & BTE_FLAG_FIRST_ELEMENT)) + { + pLevelInfo->ui64ContElmCount++; + } + pucTmp = pucEntry + 1; + if (bteKeyLenFlag( pucEntry)) + { + // Two byte key length + + uiKeyLen = (FLMUINT)FB2UW( pucTmp); + pLevelInfo->ui64ElmKeyLengthOvhd += 2; + pucTmp += 2; + } + else + { + + // One byte key length + + uiKeyLen = (FLMUINT)(*pucTmp); + pLevelInfo->ui64ElmKeyLengthOvhd++; + pucTmp++; + } + + if (bteDataLenFlag( pucEntry)) + { + + // Two byte data length + + uiDataLen = (FLMUINT)FB2UW( pucTmp); + pLevelInfo->ui64ElmDataLenOvhd += 2; + pucTmp += 2; + } + else + { + + // One byte data length. + + uiDataLen = (FLMUINT)(*pucTmp); + pLevelInfo->ui64ElmDataLenOvhd++; + pucTmp++; + } + + // Check for the presence of the OverallDataLength field (4 bytes). + + if (bteOADataLenFlag( pucEntry)) + { + uiOADataLen = (FLMUINT)FB2UD( pucTmp); + pLevelInfo->ui64ElmOADataLenOvhd += 4; + pucTmp += 4; + } + pucKey = pucTmp; + + if (bteDataBlockFlag( pucEntry)) + { + flmAssert( uiDataLen == 4); + flmAssert( uiOADataLen); + + // Skip over the key to get to the data only block address. + + pucTmp += uiKeyLen; + uiBlkAddr = (FLMUINT)FB2UD( pucTmp); + while (uiBlkAddr) + { + if (RC_BAD( pDb->m_pDatabase->getBlock( pDb, pLFile, + uiBlkAddr, NULL, &pCachedBlock))) + { + goto Exit; + } + + // Block better be a data-only block. + + flmAssert( pCachedBlock->m_pBlkHdr->ui8BlkType == BT_DATA_ONLY); + pLevelInfo->ui64DataOnlyBlockCount++; + pLevelInfo->ui64DataOnlyBlockLength += (FLMUINT64)m_uiBlockSize; + pLevelInfo->ui64DataOnlyBlockFreeSpace += + (FLMUINT64)pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + + // Subtract from uiIODataLen - should go to exactly + // zero by the time we leave this loop. + + uiOADataLen -= (m_uiBlockSize - + sizeofDOBlkHdr( pCachedBlock->m_pBlkHdr) - + pCachedBlock->m_pBlkHdr->ui16BlkBytesAvail); + + uiBlkAddr = (FLMUINT)pCachedBlock->m_pBlkHdr->ui32NextBlkInChain; + + ScaReleaseCache( pCachedBlock, FALSE); + pCachedBlock = NULL; + } + + // Better have accounted for the exact amount of data that + // was given in the overall data length field. + + flmAssert( !uiOADataLen); + } + break; + + default: + pucKey = NULL; + flmAssert( 0); + break; + } + if (uiKeyLen) + { + pLevelInfo->ui64ElmKeyLength += (FLMUINT64)uiKeyLen; + } + if (uiDataLen) + { + pLevelInfo->ui64ElmDataLength += (FLMUINT64)uiDataLen; + } + + // If this is an index, parse the key. + + if (pIndex && uiKeyLen) + { + FLMUINT uiComponentLen; + FLMBYTE * pucKeyEnd = pucKey + uiKeyLen; + ICD * pIcd; + FLMUINT uiIcd; + + for (uiIcd = 0, pIcd = pIndex->pKeyIcds; + uiIcd < pIndex->uiNumKeyComponents; + uiIcd++, pIcd++) + { + flmAssert( pucKey + 2 <= pucKeyEnd); + uiComponentLen = getKeyComponentLength( pucKey); + if (uiComponentLen == KEY_HIGH_VALUE || + uiComponentLen == KEY_LOW_VALUE) + { + uiComponentLen = 0; + } + pLevelInfo->ui64KeyComponentLengthsSize += 2; + if (uiComponentLen) + { + pLevelInfo->ui64KeyDataSize += (FLMUINT64)uiComponentLen; + } + pucKey += (2 + uiComponentLen); + } + pLevelInfo->ui64KeyIdSize += (FLMUINT64)(pucKeyEnd - pucKey); + } + } + +Exit: + + if (pCachedBlock) + { + ScaReleaseCache( pCachedBlock, FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc: Collect information on a b-tree. +****************************************************************************/ +RCODE F_BTreeInfo::collectBTreeInfo( + F_Db * pDb, + LFILE * pLFile, + BTREE_INFO * pBTreeInfo, + const char * pszName, + F_INDEX * pIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNameBufSize; + FLMUINT uiLeftBlocks [MAX_LEVELS]; + F_Database * pDatabase = pDb->m_pDatabase; + F_CachedBlock * pCachedBlock = NULL; + F_BTREE_BLK_HDR * pBlkHdr; + + m_uiBlockSize = pDatabase->m_uiBlockSize; + + // Allocate a name buffer. + + uiNameBufSize = f_strlen( pszName) + 1; + if (RC_BAD( rc = m_pool.poolAlloc( uiNameBufSize, + (void **)(&pBTreeInfo->pszLfName)))) + { + goto Exit; + } + f_strcpy( pBTreeInfo->pszLfName, pszName); + m_uiCurrLfNum = pLFile->uiLfNum; + m_bIsTable = pIndex ? FALSE : TRUE; + m_pszCurrLfName = pBTreeInfo->pszLfName; + + // Reset the information for the b-tree. uiLfNum should have already + // been set by the caller. + + flmAssert( pBTreeInfo->uiLfNum == pLFile->uiLfNum); + pBTreeInfo->uiNumLevels = 0; + f_memset( &pBTreeInfo->levelInfo [0], 0, sizeof( pBTreeInfo->levelInfo)); + + // Read the root block to see how many levels are in the b-tree. + + if (RC_BAD( pDatabase->getBlock( pDb, pLFile, pLFile->uiRootBlk, + NULL, &pCachedBlock))) + { + goto Exit; + } + pBlkHdr = (F_BTREE_BLK_HDR *)pCachedBlock->m_pBlkHdr; + + pBTreeInfo->uiNumLevels = pBlkHdr->ui8BlkLevel + 1; + flmAssert( pBTreeInfo->uiNumLevels <= MAX_LEVELS); + + // Better be a root block, and better not have a prev and next + // block address. + + flmAssert( isRootBlk( pBlkHdr)); + flmAssert( pBlkHdr->stdBlkHdr.ui32BlkAddr == (FLMUINT32)pLFile->uiRootBlk); + flmAssert( !pBlkHdr->stdBlkHdr.ui32PrevBlkInChain); + flmAssert( !pBlkHdr->stdBlkHdr.ui32NextBlkInChain); + m_uiCurrLevel = pBlkHdr->ui8BlkLevel; + + // Gather information for the root block. + + if (RC_BAD( rc = collectBlockInfo( pDb, pLFile, pBTreeInfo, + pBlkHdr, pIndex))) + { + goto Exit; + } + m_ui64CurrLfBlockCount = 1; + m_ui64CurrLevelBlockCount = 1; + m_ui64TotalBlockCount = 1; + if (RC_BAD( rc = doCallback())) + { + goto Exit; + } + + // Get all of the leftmost blocks in the b-tree. + + uiLeftBlocks [pBlkHdr->ui8BlkLevel] = pLFile->uiRootBlk; + if (!pBlkHdr->ui8BlkLevel) + { + flmAssert( pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF || + pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA); + } + else + { + FLMUINT uiLevel; + FLMBYTE * pucEntry; + FLMUINT uiBlkAddr; + + // Gather up the leftmost blocks + + uiLevel = pBlkHdr->ui8BlkLevel; + while (uiLevel) + { + + uiLevel--; + + // Get the left-most down-block pointer - which is the first one + // in the array. + + pucEntry = ((FLMBYTE *)pBlkHdr) + sizeofBTreeBlkHdr( pBlkHdr); + pucEntry = ((FLMBYTE *)pBlkHdr) + FB2UW( pucEntry); + uiLeftBlocks [uiLevel] = (FLMUINT)FB2UD( pucEntry); + + // Get the leftmost block at the next level down in the b-tree + + ScaReleaseCache( pCachedBlock, FALSE); + pCachedBlock = NULL; + if (RC_BAD( pDatabase->getBlock( pDb, pLFile, + uiLeftBlocks [uiLevel], NULL, &pCachedBlock))) + { + goto Exit; + } + pBlkHdr = (F_BTREE_BLK_HDR *)pCachedBlock->m_pBlkHdr; + + // If we are at level zero, we better be on a leaf block. + + if (!uiLevel) + { + flmAssert( pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF || + pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA); + } + } + + // Now do each level of the b-tree. Have already done the root + // block, so we start one level down from it. + + m_uiCurrLevel = pBTreeInfo->uiNumLevels - 2; + for (;;) + { + uiBlkAddr = uiLeftBlocks [m_uiCurrLevel]; + + m_ui64CurrLevelBlockCount = 0; + while (uiBlkAddr) + { + if (pCachedBlock) + { + ScaReleaseCache( pCachedBlock, FALSE); + pCachedBlock = NULL; + } + if (RC_BAD( pDatabase->getBlock( pDb, pLFile, uiBlkAddr, + NULL, &pCachedBlock))) + { + goto Exit; + } + pBlkHdr = (F_BTREE_BLK_HDR *)pCachedBlock->m_pBlkHdr; + + // Gather information for the block. + + if (RC_BAD( rc = collectBlockInfo( pDb, pLFile, pBTreeInfo, + pBlkHdr, pIndex))) + { + goto Exit; + } + m_ui64CurrLfBlockCount++; + m_ui64CurrLevelBlockCount++; + m_ui64TotalBlockCount++; + if (RC_BAD( rc = doCallback())) + { + goto Exit; + } + + // Go to the next block in the chain. + + uiBlkAddr = pBlkHdr->stdBlkHdr.ui32NextBlkInChain; + } + + if (!m_uiCurrLevel) + { + break; + } + + // Go down to the next level in the b-tree. + + m_uiCurrLevel--; + } + } + +Exit: + + if (pCachedBlock) + { + ScaReleaseCache( pCachedBlock, FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc: Collect b-tree information for an index. If uiIndexNum is zero, + collect b-tree information for ALL indexes. + If we already have information on the index, we will clear the + information and get it again. +****************************************************************************/ +RCODE F_BTreeInfo::collectIndexInfo( + F_Db * pDb, + FLMUINT uiIndexNum, + IF_BTreeInfoStatus * pInfoStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + BTREE_INFO * pIndexInfo; + F_INDEX * pIndex; + FLMUINT uiLoop; + + // Start a read transaction, if no other transaction is going. + + if (pDb->getTransType() == SFLM_NO_TRANS) + { + if (RC_BAD( rc = pDb->transBegin( SFLM_READ_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + m_pInfoStatus = pInfoStatus; + m_uiCurrLfNum = 0; + m_bIsTable = FALSE; + m_pszCurrLfName = NULL; + m_uiCurrLevel = 0; + m_ui64CurrLfBlockCount = 0; + m_ui64CurrLevelBlockCount = 0; + m_ui64TotalBlockCount = 0; + + if (!uiIndexNum) + { + m_uiNumIndexes = 0; + for (uiLoop = 0, pIndex = pDb->m_pDict->m_pIndexTbl; + uiLoop < pDb->m_pDict->m_uiHighestIndexNum; + uiLoop++, pIndex++) + { + if (pIndex->uiIndexNum) + { + if (RC_BAD( rc = collectIndexInfo( pDb, pIndex->uiIndexNum, pInfoStatus))) + { + goto Exit; + } + } + } + } + else + { + // See if we can find the b-tree already in our list. + + uiLoop = 0; + pIndexInfo = m_pIndexArray; + while (uiLoop < m_uiNumIndexes && pIndexInfo->uiLfNum != uiIndexNum) + { + uiLoop++; + pIndexInfo++; + } + if (uiLoop == m_uiNumIndexes) + { + pIndexInfo = NULL; + } + + // See if the index is defined in the database + + if ((pIndex = pDb->m_pDict->getIndex( uiIndexNum)) == NULL) + { + rc = NE_SFLM_OK; + + // If we previously had the index, remove it from the array. + + if (pIndexInfo) + { + if (uiLoop < m_uiNumIndexes - 1) + { + f_memmove( pIndexInfo, &pIndexInfo[ 1], + sizeof( BTREE_INFO) * (m_uiNumIndexes - uiLoop - 1)); + } + m_uiNumIndexes--; + } + goto Exit; + } + + // If we previously had the index, reset its information. + // Otherwise, create a new index info structure and + // add it to the array. + + if (!pIndexInfo) + { + + // Allocate space for a new index info structure in the array. + + if (m_uiNumIndexes == m_uiIndexArraySize) + { + if (RC_BAD( rc = f_realloc( + sizeof( BTREE_INFO) * (m_uiIndexArraySize + 5), &m_pIndexArray))) + { + goto Exit; + } + m_uiIndexArraySize += 5; + } + pIndexInfo = &m_pIndexArray [m_uiNumIndexes]; + pIndexInfo->uiLfNum = uiIndexNum; + m_uiNumIndexes++; + } + + // Get the index information + + if (RC_BAD( rc = collectBTreeInfo( pDb, &pIndex->lfInfo, pIndexInfo, + pIndex->pszIndexName, pIndex))) + { + goto Exit; + } + } + +Exit: + + if (bStartedTrans) + { + pDb->transAbort(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Collect b-tree information for a collection. If uiTableNum is + zero, collect b-tree information for ALL collections. + + If we already have information on the collection, we will clear the + information and get it again. +****************************************************************************/ +RCODE F_BTreeInfo::collectTableInfo( + F_Db * pDb, + FLMUINT uiTableNum, + IF_BTreeInfoStatus * pInfoStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + BTREE_INFO * pTableInfo; + F_TABLE * pTable; + FLMUINT uiLoop; + + // Start a read transaction, if no other transaction is going. + + if (pDb->getTransType() == SFLM_NO_TRANS) + { + if (RC_BAD( rc = pDb->transBegin( SFLM_READ_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + m_pInfoStatus = pInfoStatus; + m_uiCurrLfNum = 0; + m_bIsTable = FALSE; + m_pszCurrLfName = NULL; + m_uiCurrLevel = 0; + m_ui64CurrLfBlockCount = 0; + m_ui64CurrLevelBlockCount = 0; + m_ui64TotalBlockCount = 0; + + if (!uiTableNum) + { + m_uiNumTables = 0; + for (uiLoop = 0, pTable = pDb->m_pDict->m_pTableTbl; + uiLoop < pDb->m_pDict->m_uiHighestTableNum; + uiLoop++, pTable++) + { + if (pTable->uiTableNum) + { + if (RC_BAD( rc = collectTableInfo( + pDb, pTable->uiTableNum, pInfoStatus))) + { + goto Exit; + } + } + } + } + else + { + // See if we can find the b-tree already in our list. + + uiLoop = 0; + pTableInfo = m_pTableArray; + while (uiLoop < m_uiNumTables && pTableInfo->uiLfNum != uiTableNum) + { + uiLoop++; + pTableInfo++; + } + if (uiLoop == m_uiNumTables) + { + pTableInfo = NULL; + } + + // See if the index is defined in the database + + if ((pTable = pDb->m_pDict->getTable( uiTableNum)) == NULL) + { + rc = NE_SFLM_OK; + + // If we previously had the table, remove it from the array. + + if (pTableInfo) + { + if (uiLoop < m_uiNumTables - 1) + { + f_memmove( pTableInfo, &pTableInfo[ 1], + sizeof( BTREE_INFO) * (m_uiNumTables - uiLoop - 1)); + } + m_uiNumTables--; + } + goto Exit; + } + + // If we previously had the table, reset its information. + // Otherwise, create a new table info structure and + // add it to the array. + + if (!pTableInfo) + { + + // Allocate space for a new index info structure in the array. + + if (m_uiNumTables == m_uiTableArraySize) + { + if (RC_BAD( rc = f_realloc( + sizeof( BTREE_INFO) * (m_uiTableArraySize + 5), + &m_pTableArray))) + { + goto Exit; + } + m_uiTableArraySize += 5; + } + pTableInfo = &m_pTableArray [m_uiNumTables]; + pTableInfo->uiLfNum = uiTableNum; + m_uiNumTables++; + } + + // Get the table information + + if (RC_BAD( rc = collectBTreeInfo( pDb, &pTable->lfInfo, + pTableInfo, pTable->pszTableName, NULL))) + { + goto Exit; + } + } + +Exit: + + if (bStartedTrans) + { + pDb->transAbort(); + } + + return( rc); +} + diff --git a/sql/src/checksum.cpp b/sql/src/checksum.cpp new file mode 100644 index 0000000..fb969fc --- /dev/null +++ b/sql/src/checksum.cpp @@ -0,0 +1,702 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the routine which calculates a block checksum. +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: checksum.cpp 3123 2006-01-24 17:19:50 -0700 (Tue, 24 Jan 2006) ahodgkinson $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#if (defined( FLM_WIN) && !defined( FLM_64BIT)) || defined( FLM_NLM) + + static unsigned long gv_mmxCheckSumFlag = 1; + + #if defined( FLM_WATCOM_NLM) + + extern void FastBlockCheckSumMMX( + void * pBlk, + unsigned long *puiChecksum, + unsigned long *puiXORdata, + unsigned long uiNumberOfBytes); + + extern void FastBlockCheckSum386( + void * pBlk, + unsigned long *puiChecksum, + unsigned long *puiXORdata, + unsigned long uiNumberOfBytes); + + extern unsigned long GetMMXSupported(void); + + #else + + static void FastBlockCheckSumMMX( + void * pBlk, + unsigned long *puiChecksum, + unsigned long *puiXORdata, + unsigned long uiNumberOfBytes); + + static void FastBlockCheckSum386( + void * pBlk, + unsigned long *puiChecksum, + unsigned long *puiXORdata, + unsigned long uiNumberOfBytes); + + static unsigned long GetMMXSupported(void); + + #endif + +#endif + +/******************************************************************** +Desc: Returns 1 if the CPU supports MMX +Ret: 0 or 1 if CPU supports MMX +*********************************************************************/ +#if defined( FLM_WATCOM_NLM) + + #pragma aux GetMMXSupported parm; + #pragma aux GetMMXSupported = \ + 0xB8 0x01 0x00 0x00 0x00 /* mov eax, 1 */\ + 0x0F 0xA2 /* CPUID */\ + 0x33 0xC0 /* xor eax, eax */\ + 0xF7 0xC2 0x00 0x00 0x80 0x00 /* test edx, (1 SHL 23) ;check for MMX Instruction support */\ + 0x0F 0x95 0xC0 /* setnz al */\ + modify exact [EAX EBX ECX EDX]; + +#elif defined( FLM_WIN) && !defined( FLM_64BIT) + + unsigned long GetMMXSupported( void) + { + unsigned long bMMXSupported; + __asm + { + mov eax, 1 + cpuid + xor eax, eax + test edx, (1 SHL 23) + setnz al + mov bMMXSupported, eax + } + return bMMXSupported; + } + +#endif + +/******************************************************************** +Desc: Performs part of the FLAIM block checksum algorithm + using MMX instructions. +*********************************************************************/ +#if defined( FLM_WATCOM_NLM) + + #pragma aux FastBlockCheckSumMMX parm [ESI] [eax] [ebx] [ecx]; + #pragma aux FastBlockCheckSumMMX = \ + 0x50 /* push eax ;save the sum pointer */\ + 0x53 /* push ebx ;save the xor pointer */\ + 0x8B 0x10 /* mov edx, [eax] ;for local add */\ + 0x81 0xE2 0xFF 0x00 0x00 0x00 /* and edx, 0ffh ;clear unneeded bits */\ + 0x8B 0x1B /* mov ebx, [ebx] ;for local xor */\ + 0x81 0xE3 0xFF 0x00 0x00 0x00 /* and ebx, 0ffh ;clear unneeded bits */\ + 0x8B 0xF9 /* mov edi, ecx ;save the amount to copy */\ + 0x83 0xF9 0x20 /* cmp ecx, 32 ;see if we have enough for the big loop */\ + 0x0F 0x82 0x63 0x00 0x00 0x00 /* jb #MediumStuff */\ + 0xC1 0xE9 0x05 /* shr ecx, 5 ;convert length to 32 byte blocks */\ + 0x83 0xE7 0x1F /* and edi, 01fh ;change saved length to remainder */\ + 0x0F 0xEF 0xED /* pxor mm5, mm5 ;wasted space to 16 byte align the loop */\ + 0x0F 0x6E 0xE2 /* movd mm4,edx */\ + 0x0F 0x6E 0xEB /* movd mm5,ebx */\ + 0x0F 0x6F 0x06 /* movq mm0, [esi] */\ + 0x0F 0x6F 0x4E 0x08 /* movq mm1, [esi + 8] */\ + 0x0F 0x6F 0x56 0x10 /* movq mm2, [esi + 16] */\ + 0x0F 0x6F 0x5E 0x18 /* movq mm3, [esi + 24] */\ + 0x83 0xC6 0x20 /* add esi, 32 ;move the data pointer ahead 32 */\ + 0x0F 0xFC 0xE0 /* paddb mm4, mm0 */\ + 0x0F 0xEF 0xE8 /* pxor mm5, mm0 */\ + 0x0F 0xFC 0xE1 /* paddb mm4, mm1 */\ + 0x0F 0xEF 0xE9 /* pxor mm5, mm1 */\ + 0x0F 0xFC 0xE2 /* paddb mm4, mm2 */\ + 0x0F 0xEF 0xEA /* pxor mm5, mm2 */\ + 0x0F 0xFC 0xE3 /* paddb mm4, mm3 */\ + 0x0F 0xEF 0xEB /* pxor mm5, mm3 */\ + 0x49 /* dec ecx ;see if there is more to do */\ + 0x75 0xD3 /* jnz #BigStuffLoop */\ + 0x0F 0x7E 0xEB /* movd ebx, mm5 */\ + 0x0F 0x73 0xD5 0x20 /* psrlq mm5, 32 */\ + 0x0F 0x7E 0xE8 /* movd eax, mm5 */\ + 0x33 0xD8 /* xor ebx, eax */\ + 0x0F 0x6F 0xC4 /* movq mm0, mm4 */\ + 0x0F 0x73 0xD0 0x20 /* psrlq mm0, 32 */\ + 0x0F 0xFC 0xE0 /* paddb mm4, mm0 */\ + 0x0F 0x6F 0xC4 /* movq mm0, mm4 */\ + 0x0F 0x73 0xD0 0x10 /* psrlq mm0, 16 */\ + 0x0F 0xFC 0xE0 /* paddb mm4, mm0 */\ + 0x0F 0x7E 0xE2 /* movd edx, mm4 */\ + 0x0F 0x77 /* emms ;end of MMX stuff */\ + 0x8B 0xCF /* mov ecx, edi ;load up the rest of the length */\ + 0x83 0xF9 0x04 /* cmp ecx, 4 */\ + 0x0F 0x82 0x1D 0x00 0x00 0x00 /* jb #SmallStuff */\ + 0xC1 0xE9 0x02 /* shr ecx, 2 */\ + 0x83 0xE7 0x03 /* and edi, 3 */\ + 0x8B 0x06 /* mov eax, [esi] */\ + 0x83 0xC6 0x04 /* add esi, 4 */\ + 0x33 0xD8 /* xor ebx, eax */\ + 0x02 0xD0 /* add dl, al */\ + 0x02 0xF4 /* add dh, ah */\ + 0xC1 0xE8 0x10 /* shr eax, 16 */\ + 0x02 0xD0 /* add dl, al */\ + 0x02 0xF4 /* add dh, ah */\ + 0x49 /* dec ecx */\ + 0x75 0xEB /* jnz #DSSumLoop */\ + 0x8B 0xCF /* mov ecx, edi ;load up the rest of the length */\ + 0x02 0xD6 /* add dl, dh ;get complete sum in dl */\ + 0x8B 0xC3 /* mov eax, ebx ;get complete xor in bl */\ + 0xC1 0xE8 0x10 /* shr eax, 16 */\ + 0x66 0x33 0xD8 /* xor bx, ax */\ + 0x32 0xDF /* xor bl, bh */\ + 0x83 0xF9 0x00 /* cmp ecx, 0 ;see if anything left to do - 3 or less bytes */\ + 0x0F 0x84 0x0A 0x00 0x00 0x00 /* jz #Done */\ + 0x8A 0x06 /* mov al, [esi] */\ + 0x46 /* inc esi */\ + 0x02 0xD0 /* add dl, al */\ + 0x32 0xD8 /* xor bl, al */\ + 0x49 /* dec ecx */\ + 0x75 0xF6 /* jnz #SmallStuffLoop */\ + 0x81 0xE2 0xFF 0x00 0x00 0x00 /* and edx, 0ffh ;clear unneeded bits */\ + 0x58 /* pop eax */\ + 0x81 0xE3 0xFF 0x00 0x00 0x00 /* and ebx, 0ffh ;clear unneeded bits */\ + 0x5F /* pop edi */\ + 0x89 0x18 /* mov [eax], ebx */\ + 0x89 0x17 /* mov [edi], edx */\ + parm [ESI] [eax] [ebx] [ecx] \ + modify exact [eax ebx ecx edx ESI EDI]; + +#elif defined( FLM_WIN) && !defined( FLM_64BIT) + + static void FastBlockCheckSumMMX( + void * pBlk, + unsigned long * puiChecksum, + unsigned long * puiXORdata, + unsigned long uiNumberOfBytes) + { + __asm + { + mov esi, pBlk + + // Load up the starting checksum values into edx (add) and ebx (XOR) + + mov eax, puiChecksum + mov edx, [eax] + and edx, 0ffh ;clear unneeded bits + mov eax, puiXORdata + mov ebx, [eax] + and ebx, 0ffh ;clear unneeded bits + mov ecx, uiNumberOfBytes + mov edi, ecx ;save the amount to copy + + cmp ecx, 32 ;see if we have enough for the big loop + jb MediumStuff + + shr ecx, 5 ;convert length to 32 byte blocks + and edi, 01fh ;change saved length to remainder + pxor mm5, mm5 ;wasted space to 16 byte align the upcoming loop - check tHIS.. + + movd mm4, edx ;set ADD + movd mm5, ebx ;set XOR + + BigStuffLoop: + ;load up mm0 - mm3 with 8 bytes each of data. + movq mm0, [esi] + movq mm1, [esi + 8] + movq mm2, [esi + 16] + movq mm3, [esi + 24] + add esi, 32 ;move the data pointer ahead 32 + ;add mm0 - mm3 to mm4 + ;xor mm0 - mm3 with mm5 + paddb mm4, mm0 + pxor mm5, mm0 + paddb mm4, mm1 + pxor mm5, mm1 + paddb mm4, mm2 + pxor mm5, mm2 + paddb mm4, mm3 + pxor mm5, mm3 + dec ecx ;see if there is more to do + jnz BigStuffLoop + ;mm4 contains the sum to this point + ;mm5 contains the xor to this point + ;edi contains the bytes left + ;esi points to data left to do + ;extract the xor value from mm5 and put it in ebx + movd ebx, mm5 + psrlq mm5, 32 + movd eax, mm5 + xor ebx, eax + ;extract the sum value from mm4 and put it in dl & dh + movq mm0, mm4 + psrlq mm0, 32 + paddb mm4, mm0 + movq mm0, mm4 + psrlq mm0, 16 + paddb mm4, mm0 + movd edx, mm4 + emms ;end of MMX stuff + + mov ecx, edi ;load up the rest of the length + ;dl contains half the sum to this point + ;dh contains half the sum to this point + ;ebx contains the xor to this point - 32 bits wide. + ;ecx contains the bytes still left to do + ;esi contains pointer to data to checksum + MediumStuff: + cmp ecx, 4 + jb SmallStuff + shr ecx, 2 + and edi, 3 + + DSSumLoop: + mov eax, [esi] + add esi, 4 + xor ebx, eax + add dl, al + add dh, ah + shr eax, 16 + add dl, al + add dh, ah + dec ecx + jnz DSSumLoop + mov ecx, edi ;load up the rest of the length + ;dl contains half the sum to this point + ;dh contains half the sum to this point + ;ebx contains the xor to this point - 32 bits wide. + ;ecx contains the bytes still left to do + ;esi contains pointer to data to checksum + SmallStuff: + add dl, dh ;get complete sum in dl + mov eax, ebx ;get complete xor in bl + shr eax, 16 + xor bx, ax + xor bl, bh + cmp ecx, 0 ;see if anything left to do - 3 or less bytes + jz Done + + SmallStuffLoop: + mov al, [esi] + inc esi + add dl, al + xor bl, al + dec ecx + jnz SmallStuffLoop + Done: + and edx, 0ffh ;clear unneeded bits + and ebx, 0ffh ;clear unneeded bits + + // Set the return values. + + mov eax, puiChecksum + mov [eax], edx + + mov eax, puiXORdata + mov [eax], ebx + } + return; + } +#endif + +/****************************************************************************** +Desc: Performs part of the FLAIM block checksum algorithm + using 386 and NOT MMX instructions. +******************************************************************************/ +#if defined( FLM_WATCOM_NLM) + + #pragma aux FastBlockCheckSum386 parm [ESI] [eax] [ebx] [ecx]; + + #pragma aux FastBlockCheckSum386 = \ + 0x50 /* push eax ;save the sum pointer */\ + 0x53 /* push ebx ;save the xor pointer */\ + 0x8B 0x10 /* mov edx, [eax] ;for local add */\ + 0x81 0xE2 0xFF 0x00 0x00 0x00 /* and edx, 0ffh ;clear unneeded bits */\ + 0x8B 0x1B /* mov ebx, [ebx] ;for local xor */\ + 0x81 0xE3 0xFF 0x00 0x00 0x00 /* and ebx, 0ffh ;clear unneeded bits */\ + /* ;dl contains the sum to this point */\ + /* ;ebx contains the xor to this point */\ + /* ;ecx contains the bytes still left to do */\ + /* ;esi contains pointer to data to checksum */\ + 0x83 0xF9 0x04 /* cmp ecx, 4 */\ + 0x0F 0x82 0x1F 0x00 0x00 0x00 /* jb #SmallStuff */\ + 0x8B 0xF9 /* mov edi, ecx */\ + 0xC1 0xE9 0x02 /* shr ecx, 2 */\ + 0x83 0xE7 0x03 /* and edi, 3 */\ + /* #DSSumLoop: */\ + 0x8B 0x06 /* mov eax, [esi] */\ + 0x83 0xC6 0x04 /* add esi, 4 */\ + 0x33 0xD8 /* xor ebx, eax */\ + 0x02 0xD0 /* add dl, al */\ + 0x02 0xF4 /* add dh, ah */\ + 0xC1 0xE8 0x10 /* shr eax, 16 */\ + 0x02 0xD0 /* add dl, al */\ + 0x02 0xF4 /* add dh, ah */\ + 0x49 /* dec ecx */\ + 0x75 0xEB /* jnz #DSSumLoop */\ + 0x8B 0xCF /* mov ecx, edi ;load up the rest of len */\ + /* ;dl contains half the sum to this point */\ + /* ;dh contains half the sum to this point */\ + /* ;ebx contains the xor to this point */\ + /* ;ecx contains the bytes still left to do */\ + /* ;esi contains pointer to data to checksum */\ + /* #SmallStuff: */\ + 0x02 0xD6 /* add dl, dh ;get complete sum in dl */\ + 0x8B 0xC3 /* mov eax, ebx ;get complete xor in bl */\ + 0xC1 0xE8 0x10 /* shr eax, 16 */\ + 0x66 0x33 0xD8 /* xor bx, ax */\ + 0x32 0xDF /* xor bl, bh */\ + 0x83 0xF9 0x00 /* cmp ecx, 0 */\ + 0x0F 0x84 0x0A 0x00 0x00 0x00 /* jz #Done */\ + /* #SmallStuffLoop: */\ + 0x8A 0x06 /* mov al, [esi] */\ + 0x46 /* inc esi */\ + 0x02 0xD0 /* add dl, al */\ + 0x32 0xD8 /* xor bl, al */\ + 0x49 /* dec ecx */\ + 0x75 0xF6 /* jnz #SmallStuffLoop */\ + /* #Done: */\ + 0x81 0xE2 0xFF 0x00 0x00 0x00 /* and edx, 0ffh ;clear unneeded bits */\ + 0x58 /* pop eax */\ + 0x81 0xE3 0xFF 0x00 0x00 0x00 /* and ebx, 0ffh ;clear unneeded bits */\ + 0x5F /* pop edi */\ + 0x89 0x18 /* mov [eax], ebx */\ + 0x89 0x17 /* mov [edi], edx */\ + parm [ESI] [eax] [ebx] [ecx] \ + modify exact [eax ebx ecx edx ESI EDI]; + +#elif defined( FLM_WIN) && !defined( FLM_64BIT) + + static void FastBlockCheckSum386( + void * pBlk, + unsigned long *puiChecksum, + unsigned long *puiXORdata, + unsigned long uiNumberOfBytes) + { + __asm + { + mov esi, pBlk + + // Load up the starting checksum values into edx (add) and ebx (XOR) + + mov eax, puiChecksum + mov edx, [eax] // Set local add + and edx, 0ffh ;clear unneeded bits + mov eax, puiXORdata + mov ebx, [eax] + and ebx, 0ffh ;clear unneeded bits + mov ecx, uiNumberOfBytes + + ;dl contains the sum to this point + ;ebx contains the xor to this point - 32 bits wide. + ;ecx contains the bytes still left to do + ;esi contains pointer to data to checksum + cmp ecx, 4 + jb SmallStuff + mov edi, ecx + shr ecx, 2 + and edi, 3 + + DSSumLoop: + mov eax, [esi] + add esi, 4 + xor ebx, eax + add dl, al + add dh, ah + shr eax, 16 + add dl, al + add dh, ah + dec ecx + jnz DSSumLoop + mov ecx, edi ;load up the rest of the length + ;dl contains half the sum to this point + ;dh contains half the sum to this point + ;ebx contains the xor to this point - 32 bits wide. + ;ecx contains the bytes still left to do + ;esi contains pointer to data to checksum + + SmallStuff: + add dl, dh ;get complete sum in dl + mov eax, ebx ;get complete xor in bl + shr eax, 16 + xor bx, ax + xor bl, bh + cmp ecx, 0 ;see if anything left to do - 3 or less bytes + jz Done + + SmallStuffLoop: + mov al, [esi] + inc esi + add dl, al + xor bl, al + dec ecx + jnz SmallStuffLoop + + Done: + and edx, 0ffh ;clear unneeded bits + and ebx, 0ffh ;clear unneeded bits + + // Set the return values. + + mov eax, puiChecksum // Address of add result/start + mov [eax], edx + + mov eax, puiXORdata // Address of xor result/start + mov [eax], ebx + } + + return; + } +#endif + +/****************************************************************************** +Desc: Performs part of the FLAIM block checksum algorithm + using MMX or 386 instructions. +Note: FastBlockCheckSum will start with the checksum and xordata you + pass in. It assumes that the data is already dword aligned. +******************************************************************************/ +#if (defined( FLM_WIN) && !defined( FLM_64BIT)) || defined( FLM_NLM) +void FastBlockCheckSum( + void * pBlk, + FLMUINT * puiChecksum, + FLMUINT * puiXORdata, + FLMUINT uiNumberOfBytes) +{ + if( gv_mmxCheckSumFlag == 1) + { + FastBlockCheckSumMMX( (void *) pBlk, (unsigned long *) puiChecksum, + (unsigned long *) puiXORdata, (unsigned long) uiNumberOfBytes); + } + else + { + FastBlockCheckSum386( (void *) pBlk, (unsigned long *) puiChecksum, + (unsigned long *) puiXORdata, (unsigned long) uiNumberOfBytes); + } +} +#endif + +/****************************************************************************** +Desc: Sets the global variable to check if MMX instructions are allowed. +******************************************************************************/ +void F_DbSystem::initFastBlockCheckSum( void) +{ +#if (defined( FLM_WIN) && !defined( FLM_64BIT)) || defined( FLM_NLM) + // NOTE that GetMMXSupported assumes that we are running on at least a + // pentium. The check to see if we are on a pentium requires that we + // modify the flags register, and we can't do that if we are running + // in ring3. Because NetWare 5 - according to our product marketing - + // requires at least a P5 90Mhz, we will be safe. When you port this + // code to NT, you may need to come up with a safe way to see if we + // can do MMX instructions - unless you can assume that even on NT you + // will be on at least a P5. + + gv_mmxCheckSumFlag = GetMMXSupported(); +#endif +} + +/******************************************************************** +Desc: Calculate the checksum for a packet. +*********************************************************************/ +FLMBYTE RflCalcChecksum( + FLMBYTE * pucPacket, + FLMUINT uiPacketBodyLen) +{ + FLMUINT uiBytesToChecksum; + FLMUINT uiChecksum; + FLMBYTE * pucStart; + + // Checksum is calculated for every byte in the packet that + // comes after the checksum byte. + + uiBytesToChecksum = (FLMUINT)(uiPacketBodyLen + + RFL_PACKET_OVERHEAD - + (RFL_PACKET_CHECKSUM_OFFSET + 1)); + + pucStart = &pucPacket [RFL_PACKET_CHECKSUM_OFFSET + 1]; + + FLMBYTE * pucEnd; + FLMBYTE * pucSectionEnd; + FLMBYTE * pucCur; + FLMBYTE ucTmp; + + uiChecksum = 0; + pucCur = pucStart; + pucEnd = pucStart + uiBytesToChecksum; + +#ifdef FLM_64BIT + pucSectionEnd = pucStart + (sizeof( FLMUINT) - ((FLMUINT)pucStart & 0x7)); +#else + pucSectionEnd = pucStart + (sizeof( FLMUINT) - ((FLMUINT)pucStart & 0x3)); +#endif + + if( pucSectionEnd > pucEnd) + { + pucSectionEnd = pucEnd; + } + + while( pucCur < pucSectionEnd) + { + uiChecksum = (uiChecksum << 8) + *pucCur++; + } + +#ifdef FLM_64BIT + pucSectionEnd = (FLMBYTE *)((FLMUINT)pucEnd & 0xFFFFFFFFFFFFFFF8); +#else + pucSectionEnd = (FLMBYTE *)((FLMUINT)pucEnd & 0xFFFFFFFC); +#endif + + while( pucCur < pucSectionEnd) + { + uiChecksum ^= *((FLMUINT *)pucCur); + pucCur += sizeof( FLMUINT); + } + + while( pucCur < pucEnd) + { + uiChecksum ^= *pucCur++; + } + + ucTmp = (FLMBYTE)uiChecksum; + + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; + + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; + +#ifdef FLM_64BIT + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; + + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; + + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; + + uiChecksum >>= 8; + ucTmp ^= (FLMBYTE)uiChecksum; +#endif + + ucTmp ^= (FLMBYTE)(uiChecksum >> 8); + + if( (uiChecksum = (FLMUINT)ucTmp) == 0) + { + uiChecksum = 1; + } + + return( (FLMBYTE)uiChecksum); +} + +/******************************************************************** +Desc: Calculate the checksum for a block. NOTE: This is ALWAYS done + on the raw image that will be written to disk. This means + that if the block needs to be converted before writing it out, + it should be done before calculating the checksum. +*********************************************************************/ +FLMUINT32 calcBlkCRC( + F_BLK_HDR * pBlkHdr, + FLMUINT uiBlkEnd) +{ + FLMUINT32 ui32SaveCRC; + FLMUINT uiAdds; + FLMUINT uiXORs; + FLMBYTE * pucBlkPtr; + + // Calculate CRC on everything except for the ui32BlkCRC value. + // To do this, we temporarily change it to zero. The saved + // value will be restored after calculating the CRC. + + ui32SaveCRC = pBlkHdr->ui32BlkCRC; + pBlkHdr->ui32BlkCRC = 0; + uiAdds = 0; + uiXORs = 0; + pucBlkPtr = (FLMBYTE *)pBlkHdr; + +#if defined( FLM_NLM) || (defined( FLM_WIN) && !defined( FLM_64BIT)) + + FastBlockCheckSum( pucBlkPtr, &uiAdds, &uiXORs, + (unsigned long)uiBlkEnd); + +#else + + FLMBYTE * pucCur = pucBlkPtr; + FLMBYTE * pucEnd = pucCur + uiBlkEnd; + + while( pucCur < pucEnd) + { + uiAdds += *pucCur; + uiXORs ^= *pucCur++; + } + + uiAdds &= 0xFF; +#endif + + // Restore the CRC that was in the block. + + pBlkHdr->ui32BlkCRC = ui32SaveCRC; + return( (FLMUINT32)((uiAdds << 16) + uiXORs)); +} + +/******************************************************************** +Desc: Calculate the CRC for the database header. +*********************************************************************/ +FLMUINT32 calcDbHdrCRC( + SFLM_DB_HDR * pDbHdr) +{ + FLMUINT32 ui32SaveCRC; + FLMUINT uiAdds; + FLMUINT uiXORs; + FLMBYTE * pucHdr; + + // Checksum everything except for the ui32HdrCRC value. + + ui32SaveCRC = pDbHdr->ui32HdrCRC; + pDbHdr->ui32HdrCRC = 0; + + uiAdds = 0; + uiXORs = 0; + pucHdr = (FLMBYTE *)pDbHdr; + +#if defined( FLM_NLM) || (defined( FLM_WIN) && !defined( FLM_64BIT)) + + FastBlockCheckSum( pucHdr, &uiAdds, &uiXORs, + (unsigned long)sizeof( SFLM_DB_HDR)); + +#else + FLMBYTE * pucCur = pucHdr; + FLMBYTE * pucEnd = pucHdr + sizeof( SFLM_DB_HDR); + + while( pucCur < pucEnd) + { + uiAdds += *pucCur; + uiXORs ^= *pucCur++; + } + + uiAdds &= 0xFF; +#endif + + // Restore the checksum that was in the header + + pDbHdr->ui32HdrCRC = ui32SaveCRC; + return( (FLMUINT32)((uiAdds << 16) + uiXORs)); +} diff --git a/sql/src/f_btpool.cpp b/sql/src/f_btpool.cpp new file mode 100644 index 0000000..a186a76 --- /dev/null +++ b/sql/src/f_btpool.cpp @@ -0,0 +1,106 @@ +//------------------------------------------------------------------------------ +// Desc: B-Tree pool class file +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_btpool.cpp 3111 2006-01-19 13:10:50 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Method for initializing the Btree Pool. +****************************************************************************/ +RCODE F_BtPool::btpInit( void) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( !m_bInitialized); + + // Create a mutex to control access to the pool. + if (RC_BAD( rc = f_mutexCreate( &m_hMutex))) + { + goto Exit; + } + + m_bInitialized = TRUE; + +Exit: + + return rc; +} + +/**************************************************************************** +Desc: Method to reserve a btree object from the pool. +****************************************************************************/ +RCODE F_BtPool::btpReserveBtree( + F_Btree ** ppBtree) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bInitialized); + + // Lock the mutex first! + f_mutexLock( m_hMutex); + + if (m_pBtreeList) + { + *ppBtree = m_pBtreeList; + m_pBtreeList = m_pBtreeList->m_pNext; + (*ppBtree)->m_pNext = NULL; + } + else + { + if (( *ppBtree = f_new F_Btree()) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + } + +Exit: + + f_mutexUnlock( m_hMutex); + + return rc; +} + +/**************************************************************************** +Desc: Method to return a Btree to the pool. +****************************************************************************/ +void F_BtPool::btpReturnBtree( + F_Btree ** ppBtree) +{ + flmAssert( m_bInitialized); + + // Close the Btree + + (*ppBtree)->btClose(); + + // Lock the mutex first! + + f_mutexLock( m_hMutex); + + (*ppBtree)->m_pNext = m_pBtreeList; + m_pBtreeList = *ppBtree; + *ppBtree = NULL; + + f_mutexUnlock( m_hMutex); +} diff --git a/sql/src/f_btpool.h b/sql/src/f_btpool.h new file mode 100644 index 0000000..063cf43 --- /dev/null +++ b/sql/src/f_btpool.h @@ -0,0 +1,79 @@ +//------------------------------------------------------------------------------ +// Desc: Header file for the B-Tree pool +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_btpool.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef F_BTPOOL_H +#define F_BTPOOL_H + +#include "f_btree.h" + +class F_BtPool : public F_Object +{ +public: + F_BtPool( void) + { + m_pBtreeList = NULL; + m_hMutex = F_MUTEX_NULL; + m_bInitialized = FALSE; + } + + ~F_BtPool( void) + { + while (m_pBtreeList) + { + F_Btree * pBtree; + + pBtree = m_pBtreeList; + m_pBtreeList = m_pBtreeList->m_pNext; + + pBtree->Release(); + } + + if (m_hMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hMutex); + } + + m_bInitialized = FALSE; + } + + RCODE btpInit( void); + + RCODE btpReserveBtree( + F_Btree ** ppBtree); + + void btpReturnBtree( + F_Btree ** ppBtree); + +private: + + F_Btree * m_pBtreeList; + F_MUTEX m_hMutex; + FLMBOOL m_bInitialized; +}; + +#endif + + + diff --git a/sql/src/f_btree.cpp b/sql/src/f_btree.cpp new file mode 100644 index 0000000..cc1d498 --- /dev/null +++ b/sql/src/f_btree.cpp @@ -0,0 +1,11672 @@ +//------------------------------------------------------------------------------ +// Desc: This class handles all of operations on a given B-Tree. +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_btree.cpp 3111 2006-01-19 13:10:50 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC FLMUINT btGetEntryDataLength( + FLMBYTE * pucEntry, + const FLMBYTE ** ppucDataRV, + FLMUINT * puiOADataLengthRV, + FLMBOOL * pbDOBlockRV); + +FSTATIC RCODE btGetEntryData( + FLMBYTE * pucEntry, + FLMBYTE * pucBufferRV, + FLMUINT uiBufferSize, + FLMUINT * puiLenDataRV); + +/*************************************************************************** +Desc: Constructor +****************************************************************************/ +F_Btree::F_Btree( void) +{ + m_bOpened = FALSE; + m_pStack = NULL; + m_uiStackLevels = 0; + m_uiRootLevel = 0; + f_memset(m_Stack, 0, sizeof(m_Stack)); + m_pLFile = NULL; + m_pDb = NULL; + m_bTempDb = FALSE; + m_pucTempBlk = NULL; + m_pucTempDefragBlk = NULL; + m_bCounts = FALSE; + m_bData = TRUE; // Default + m_bSetupForRead = FALSE; + m_bSetupForWrite = FALSE; + m_bSetupForReplace = FALSE; + m_uiBlockSize = 0; + m_uiDefragThreshold = 0; + m_uiOverflowThreshold = 0; + m_pReplaceInfo = NULL; + m_pReplaceStruct = NULL; + m_uiReplaceLevels = 0; + m_ui64CurrTransID = 0; + m_ui64LastBlkTransId = 0; + m_ui64PrimaryBlkTransId = 0; + m_uiBlkChangeCnt = 0; + m_ui64LowTransId = FLM_MAX_UINT64; + m_bMostCurrent = FALSE; + m_uiDataLength = 0; + m_uiPrimaryDataLen = 0; + m_uiOADataLength = 0; + m_uiDataRemaining = 0; + m_uiOADataRemaining = 0; + m_uiOffsetAtStart = 0; + m_bDataOnlyBlock = FALSE; + m_bOrigInDOBlocks = FALSE; + m_ui32PrimaryBlkAddr = 0; + m_uiPrimaryOffset = 0; + m_ui32DOBlkAddr = 0; + m_ui32CurBlkAddr = 0; + m_uiCurOffset = 0; + m_pucDataPtr = NULL; + m_bFirstRead = FALSE; + m_pSCache = NULL; + m_pBlkHdr = NULL; + m_uiSearchLevel = BH_MAX_LEVELS; + m_pNext = NULL; + m_pCompare = NULL; +} + +/*************************************************************************** +Desc: Destructor +****************************************************************************/ +F_Btree::~F_Btree( void) +{ + if ( m_bOpened) + { + btClose(); + } +} + +/*************************************************************************** +Desc: Function to create a new (empty) B-Tree. To do this, we create the + root block. Upon return, the Root block address and the block address + will be set in the LFile. +****************************************************************************/ +RCODE F_Btree::btCreate( + F_Db * pDb, // In + LFILE * pLFile, // In/Out + FLMBOOL bCounts, // In + FLMBOOL bData // In + ) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache = NULL; + F_BTREE_BLK_HDR * pBlkHdr = NULL; + FLMUINT16 * pui16Offset; + FLMBYTE * pucEntry; + FLMBYTE ucLEMEntry[ 3]; + FLMUINT uiFlags = 0; + FLMUINT uiLEMSize; + + // We can't create a new Btree if we have already been initialized. + + if (m_bOpened) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify that we are in an update transaction. + if (pDb->m_eTransType != SFLM_UPDATE_TRANS && !pDb->m_pDatabase->m_bTempDb) + { + rc = RC_SET_AND_ASSERT( pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Initialize the returned root block address to 0 incase of an error. + pLFile->uiRootBlk = 0; + + // Call createBlock to create a new block + if (RC_BAD( rc = pDb->m_pDatabase->createBlock( pDb, &pSCache))) + { + goto Exit; + } + + // Save the new block address as the root block address + pLFile->uiRootBlk = pSCache->m_uiBlkAddress; + + // Save the block address and identify the block as the root block. + if (RC_BAD( rc = btOpen( pDb, pLFile, bCounts, bData))) + { + goto Exit; + } + pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + + setRootBlk( pBlkHdr); + pBlkHdr->ui16LogicalFile = (FLMUINT16)pLFile->uiLfNum; + setBlkLfType( pBlkHdr, pLFile->eLfType); + pBlkHdr->ui8BlkLevel = 0; + pBlkHdr->stdBlkHdr.ui8BlkType = (bData ? BT_LEAF_DATA : BT_LEAF); + + pBlkHdr->stdBlkHdr.ui32PrevBlkInChain = 0; + pBlkHdr->stdBlkHdr.ui32NextBlkInChain = 0; + + if (pLFile->uiEncDefNum) + { + setBlockEncrypted( (F_BLK_HDR *)pBlkHdr); + } + + // Insert a LEM into the block + uiFlags = BTE_FLAG_FIRST_ELEMENT | BTE_FLAG_LAST_ELEMENT; + + if (RC_BAD( rc = buildAndStoreEntry( (bData ? BT_LEAF_DATA : BT_LEAF), + uiFlags, NULL, 0, NULL, 0, 0, 0, 0, &ucLEMEntry[0], + 3, &uiLEMSize))) + { + goto Exit; + } + + pui16Offset = BtOffsetArray((FLMBYTE *)pBlkHdr, 0); + pucEntry = (FLMBYTE *)pBlkHdr + m_uiBlockSize - uiLEMSize; + + bteSetEntryOffset( pui16Offset, 0, (FLMUINT16)(pucEntry - (FLMBYTE *)pBlkHdr)); + f_memcpy( pucEntry, ucLEMEntry, uiLEMSize); + + // Offset Entry & 2 byte LEM + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail = (FLMUINT16)(m_uiBlockSize - + sizeofBTreeBlkHdr(pBlkHdr) - + uiLEMSize - 2); + pBlkHdr->ui16HeapSize = pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + + // There is one entry now. + pBlkHdr->ui16NumKeys = 1; + +Exit: + + if (pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Btree initialization function. +****************************************************************************/ +RCODE F_Btree::btOpen( + F_Db * pDb, + LFILE * pLFile, + FLMBOOL bCounts, + FLMBOOL bData, + IF_ResultSetCompare * pCompare) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase= pDb->m_pDatabase; + + if ( m_bOpened) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + if (pDb->m_eTransType == SFLM_NO_TRANS && !pDatabase->m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + + if( !pLFile->uiRootBlk) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + m_pLFile = pLFile; + m_uiBlockSize = pDatabase->m_uiBlockSize; + m_uiDefragThreshold = m_uiBlockSize / 20; + m_uiOverflowThreshold = (m_uiBlockSize * 8) / 5; + m_bCounts = bCounts; + m_bData = bData; + m_pDb = pDb; + m_bTempDb = pDatabase->m_bTempDb; + m_pReplaceInfo = NULL; + m_uiReplaceLevels = 0; + m_ui64CurrTransID = 0; + m_ui64LastBlkTransId = 0; + m_ui64PrimaryBlkTransId = 0; + m_uiBlkChangeCnt = 0; + m_uiSearchLevel = BH_MAX_LEVELS; + + m_bSetupForRead = FALSE; + m_bSetupForWrite = FALSE; + m_bSetupForReplace = FALSE; + + // Buffer is required to hold at least the maximum number of + // offsets possible given the block size with a minimum entry size of + // 5 bytes. Each offset is 2 bytes. We only need this buffer when we + // are in an update transaction. + + if (pDb->m_eTransType == SFLM_UPDATE_TRANS || m_bTempDb) + { + m_uiBufferSize = m_uiBlockSize * 2; + } + else + { + m_uiBufferSize = 0; + } + + // If we are in an update transaction, there are certain buffers that + // are used. Make sure these are allocated. + + if( (pDb->m_eTransType == SFLM_UPDATE_TRANS || m_bTempDb) && + !pDatabase->m_pucUpdBuffer) + { + // Buffer should be at least 80% of two blocks. To make sure the other + // structures being allocated here are aligned, we allocate 2 times the + // database's block size for the update buffer. + + pDatabase->m_uiUpdBufferSize = m_uiBlockSize * 2; + flmAssert( pDatabase->m_uiUpdBufferSize >= m_uiOverflowThreshold); + + if( RC_BAD( rc = f_alloc( + pDatabase->m_uiUpdBufferSize + + (m_uiBlockSize * 2) + + m_uiBufferSize + + sizeof( BTREE_REPLACE_STRUCT) * BH_MAX_LEVELS, + &pDatabase->m_pucUpdBuffer))) + { + goto Exit; + } + + // Temporary buffers for the F_Btree class will immediately follow + // the update buffer. + + pDatabase->m_pucBTreeTmpBlk = + pDatabase->m_pucUpdBuffer + pDatabase->m_uiUpdBufferSize; + pDatabase->m_pucBTreeTmpDefragBlk = + pDatabase->m_pucBTreeTmpBlk + pDatabase->m_uiBlockSize; + pDatabase->m_pucBtreeBuffer = + pDatabase->m_pucBTreeTmpDefragBlk + pDatabase->m_uiBlockSize; + pDatabase->m_pucReplaceStruct = + pDatabase->m_pucBtreeBuffer + m_uiBufferSize; + } + + // NOTE: These temporary buffers may be NULL. They are only allocated the + // first time we detect that we are operating inside an update + // transaction - because we need to make sure that only one thread + // actually does the allocation. The assumption here is that the + // buffers are only ever used during update operations, which will + // *always* be inside update transactions. + + m_pucTempBlk = pDatabase->m_pucBTreeTmpBlk; + m_pucTempDefragBlk = pDatabase->m_pucBTreeTmpDefragBlk; + m_pucBuffer = pDatabase->m_pucBtreeBuffer; + m_pReplaceStruct = (BTREE_REPLACE_STRUCT *)pDatabase->m_pucReplaceStruct; + + flmAssert( !m_pCompare); + if ((m_pCompare = pCompare) != NULL) + { + m_pCompare->AddRef(); + } + + m_bOpened = TRUE; + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Btree close function +****************************************************************************/ +void F_Btree::btClose() +{ + FLMUINT uiLoop; + + // Ok to close multiple times. + if (!m_bOpened) + { + // Btree is not open. Just return. + return; + } + + m_pLFile = NULL; + m_pDb = NULL; + m_bTempDb = FALSE; + + for (uiLoop = 0; uiLoop < BH_MAX_LEVELS; uiLoop++) + { + m_Stack[ uiLoop].pucKeyBuf = NULL; + } + + // Release any blocks still held in the stack. + btRelease(); + + if (m_pSCache) + { + flmAssert( 0); + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + if (m_pCompare) + { + m_pCompare->Release(); + m_pCompare = NULL; + } + + m_bOpened = FALSE; +} + +/*************************************************************************** +Desc: Delete the entire tree +****************************************************************************/ +RCODE F_Btree::btDeleteTree( + IF_DeleteStatus * ifpDeleteStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumLevels; + FLMUINT puiBlkAddrs[ BH_MAX_LEVELS]; + FLMUINT uiLoop; + + flmAssert( m_bOpened); + + // Verify the transaction type + + if (m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Fill up uiBlkAddrs and calculate the number of levels. + + if( RC_BAD( rc = btGetBlockChains( puiBlkAddrs, &uiNumLevels))) + { + goto Exit; + } + + // Iterate over the list of block chains and free all of the blocks + + for( uiLoop = 0; uiLoop < uiNumLevels; uiLoop++) + { + if( RC_BAD( rc = btFreeBlockChain( + m_pDb, m_pLFile, puiBlkAddrs[ uiLoop], 0, NULL, NULL, + ifpDeleteStatus))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: +****************************************************************************/ +RCODE btFreeBlockChain( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiStartAddr, + FLMUINT uiBlocksToFree, + FLMUINT * puiBlocksFreed, + FLMUINT * puiEndAddr, + IF_DeleteStatus * ifpDeleteStatus) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = pDb->getDatabase(); + F_CachedBlock * pCurrentBlk = NULL; + F_CachedBlock * pDOSCache = NULL; + FLMBYTE * pBlk; + FLMBYTE * pucEntry; + FLMUINT uiEntryNum; + FLMUINT uiDOBlkAddr; + FLMBYTE ucDOBlkAddr[ 4]; + FLMUINT uiStatusCounter = 0; + FLMUINT uiNextBlkAddr = 0; + FLMUINT uiCurrentBlkAddr = 0; + FLMUINT uiTreeBlocksFreed = 0; + FLMUINT uiDataBlocksFreed = 0; + FLMBOOL bFreeAll = FALSE; + + if( !uiBlocksToFree) + { + bFreeAll = TRUE; + } + + // Verify the transaction type + + if( pDb->getTransType() != SFLM_UPDATE_TRANS) + { + rc = RC_SET_AND_ASSERT( pDb->getTransType() == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Now, go through the chain and delete the blocks... + + uiCurrentBlkAddr = uiStartAddr; + while( uiCurrentBlkAddr) + { + if( !bFreeAll && uiTreeBlocksFreed >= uiBlocksToFree) + { + break; + } + + if( RC_BAD( pDatabase->getBlock( pDb, pLFile, + uiCurrentBlkAddr, NULL, &pCurrentBlk))) + { + goto Exit; + } + + pBlk = (FLMBYTE *)pCurrentBlk->getBlockPtr(); + uiNextBlkAddr = ((F_BLK_HDR *)pBlk)->ui32NextBlkInChain; + + // If this is a leaf block, then there may be entries + // with data-only references that will need to be cleaned up too. + + if( getBlkType( pBlk) == BT_LEAF_DATA) + { + for( uiEntryNum = 0; + uiEntryNum < ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys; + uiEntryNum++) + { + pucEntry = BtEntry( pBlk, uiEntryNum); + + if( bteDataBlockFlag( pucEntry)) + { + // Get the data-only block address + + if( RC_BAD( rc = btGetEntryData( + pucEntry, &ucDOBlkAddr[ 0], 4, NULL))) + { + goto Exit; + } + + uiDOBlkAddr = bteGetBlkAddr( (FLMBYTE *)&ucDOBlkAddr[ 0]); + while( uiDOBlkAddr) + { + if( RC_BAD( + rc = pDatabase->getBlock( pDb, pLFile, + uiDOBlkAddr, NULL, &pDOSCache))) + { + goto Exit; + } + + uiDOBlkAddr = pDOSCache->getBlockPtr()->ui32NextBlkInChain; + rc = pDatabase->blockFree( pDb, pDOSCache); + pDOSCache = NULL; + + if (RC_BAD( rc)) + { + goto Exit; + } + + uiDataBlocksFreed++; + } + } + } + } + + rc = pDatabase->blockFree( pDb, pCurrentBlk); + pCurrentBlk = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + + if( ifpDeleteStatus && pLFile && ++uiStatusCounter >= 25) + { + uiStatusCounter = 0; + if( RC_BAD( rc = ifpDeleteStatus->reportDelete( + uiTreeBlocksFreed + uiDataBlocksFreed, + pDatabase->getBlockSize()))) + { + goto Exit; + } + } + + uiTreeBlocksFreed++; + uiCurrentBlkAddr = uiNextBlkAddr; + } + + if( puiBlocksFreed) + { + *puiBlocksFreed = uiTreeBlocksFreed; + } + + if( puiEndAddr) + { + *puiEndAddr = uiCurrentBlkAddr; + } + +Exit: + + if( pDOSCache) + { + ScaReleaseCache( pDOSCache, FALSE); + } + + if( pCurrentBlk) + { + ScaReleaseCache( pCurrentBlk, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Returns the address of the first block at each level of the tree +Note: puiBlockAddrs is assumed to point to a buffer that can store + BH_MAX_LEVELS FLMUINT values +****************************************************************************/ +RCODE F_Btree::btGetBlockChains( + FLMUINT * puiBlockAddrs, + FLMUINT * puiNumLevels) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumLevels = 0; + FLMUINT uiLFileNum; + FLMBOOL bIsIndex; + FLMUINT32 ui32NextBlkAddr; + F_CachedBlock * pCurrentBlk = NULL; + FLMBYTE * pucBlk; + FLMBYTE * pucEntry; + + flmAssert( m_bOpened); + + // Verify the transaction type + + if( m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Fill puiBlockAddrs and calculate the number of levels. + // NOTE: Normally, level 0 is the leaf level. In this function, + // puiBlockAddrs[ 0] is the ROOT and puiBlockAddrs[ uiNumLevels - 1] + // is the LEAF! + + ui32NextBlkAddr = (FLMUINT32)m_pLFile->uiRootBlk; + uiLFileNum = m_pLFile->uiLfNum; + bIsIndex = m_pLFile->eLfType == SFLM_LF_INDEX ? TRUE : FALSE; + + while( ui32NextBlkAddr) + { + puiBlockAddrs[ uiNumLevels++] = ui32NextBlkAddr; + + if( RC_BAD( m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &pCurrentBlk))) + { + goto Exit; + } + + pucBlk = (FLMBYTE *)pCurrentBlk->m_pBlkHdr; + + if( getBlkType( pucBlk) == BT_LEAF || getBlkType( pucBlk) == BT_LEAF_DATA) + { + ui32NextBlkAddr = 0; + } + else + { + // The child block address is the first part of the entry + + pucEntry = BtEntry( pucBlk, 0); + ui32NextBlkAddr = bteGetBlkAddr( pucEntry); + } + + // Release the current block + + ScaReleaseCache( pCurrentBlk, FALSE); + pCurrentBlk = NULL; + } + + *puiNumLevels = uiNumLevels; + +Exit: + + if( pCurrentBlk) + { + ScaReleaseCache( pCurrentBlk, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Function to insert an entry into the Btree. +****************************************************************************/ +RCODE F_Btree::btInsertEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bFirst, + FLMBOOL bLast, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + F_BLK_HDR * pBlkHdr; + FLMBYTE pucDOAddr[ 4]; + + if ( !m_bOpened || m_bSetupForRead || m_bSetupForReplace || + (m_bSetupForWrite && bFirst)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + flmAssert( m_uiSearchLevel >= BH_MAX_LEVELS); + flmAssert( !m_pDb->m_pDatabase->m_pRfl || + !m_pDb->m_pDatabase->m_pRfl->isLoggingEnabled()); + + if( !uiKeyLen) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Verify the transaction type + + if( m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Be sure to clear the Data Only flag. + + if( bFirst) + { + m_bDataOnlyBlock = FALSE; + } + + if( bLast) + { + + // We need to locate where we should insert the new entry. + + rc = findEntry( pucKey, uiKeyLen, FLM_EXACT); + + // Should not find this entry. If we get back anything other than + // an NE_SFLM_NOT_FOUND, then there is a problem. + + if( rc != NE_SFLM_NOT_FOUND) + { + if( RC_OK( rc)) + { + rc = RC_SET( NE_SFLM_NOT_UNIQUE); + } + goto Exit; + } + } + + if( bFirst && (!bLast || (uiKeyLen + uiDataLen > m_uiOverflowThreshold))) + { + // If bLast is not set, then we will setup to store the data in + // data only blocks. The assumption is that whenever we don't see bLast + // set when starting an insert, then the data is so large that it must + // be placed in a chain of Data Only blocks. There is no way for me to + // check the final size of the data ahead of time, so I rely on the + // calling routine to figure this out for me. + + // Get one empty block to begin with. + + flmAssert( m_pSCache == NULL); + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( m_pDb, &m_pSCache))) + { + goto Exit; + } + + // The data going in will be stored in Data-only blocks. + // Setup the block header... + + pBlkHdr = m_pSCache->m_pBlkHdr; + pBlkHdr->ui8BlkType = BT_DATA_ONLY; + pBlkHdr->ui32PrevBlkInChain = 0; + pBlkHdr->ui32NextBlkInChain = 0; + + if( m_pLFile->uiEncDefNum) + { + ((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum = (FLMUINT32)m_pLFile->uiEncDefNum; + setBlockEncrypted( pBlkHdr); + } + + pBlkHdr->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)); + + m_uiDataRemaining = m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr); + m_uiDataLength = 0; + m_uiOADataLength = 0; + m_bDataOnlyBlock = TRUE; + m_bSetupForWrite = TRUE; + m_ui32DOBlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + m_ui32CurBlkAddr = m_ui32DOBlkAddr; + } + + if( m_bDataOnlyBlock) + { + if( RC_BAD( rc = storeDataOnlyBlocks( pucKey, uiKeyLen, bFirst, + pucData, uiDataLen))) + { + goto Exit; + } + } + + if( bLast) + { + const FLMBYTE * pucLocalData; + FLMUINT uiLocalDataLen; + F_ELM_UPD_ACTION eAction; + + if( m_bDataOnlyBlock) + { + // build an entry that points to the DO block. + + UD2FBA( m_ui32DOBlkAddr, pucDOAddr); + pucLocalData = &pucDOAddr[0]; + uiLocalDataLen = m_uiOADataLength; + eAction = ELM_INSERT_DO; + + } + else + { + pucLocalData = pucData; + uiLocalDataLen = uiDataLen; + eAction = ELM_INSERT; + } + + if( RC_BAD( rc = updateEntry( pucKey, uiKeyLen, pucLocalData, + uiLocalDataLen, eAction))) + { + goto Exit; + } + + if( pui32BlkAddr) + { + *pui32BlkAddr = m_ui32PrimaryBlkAddr; + } + + if( puiOffsetIndex) + { + *puiOffsetIndex = m_uiCurOffset; + } + + m_bSetupForWrite = FALSE; + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( TRUE); + return( rc); +} + +/*************************************************************************** +Desc: Function to remove an entry into the Btree. +****************************************************************************/ +RCODE F_Btree::btRemoveEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucValue = NULL; + FLMUINT uiLen = 0; + + if ( !m_bOpened) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify the Txn type + if (m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + flmAssert( !m_pDb->m_pDatabase->m_pRfl || + !m_pDb->m_pDatabase->m_pRfl->isLoggingEnabled()); + + btResetBtree(); + + // We need to locate where we should remove the entry. + if (RC_BAD( rc = findEntry( pucKey, + uiKeyLen, + FLM_EXACT))) + { + goto Exit; + } + + if (RC_BAD( rc = updateEntry( pucKey, uiKeyLen, pucValue, uiLen, + ELM_REMOVE))) + { + goto Exit; + } + +Exit: + + releaseBlocks( TRUE); + return( rc); +} + +/*************************************************************************** +Desc: Function to provide a streaming interface for replacing large + data elements. +****************************************************************************/ +RCODE F_Btree::btReplaceEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bFirst, + FLMBOOL bLast, + FLMBOOL bTruncate, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry; + F_BLK_HDR * pBlkHdr; + const FLMBYTE * pucLocalData = NULL; + FLMUINT uiOADataLength = 0; + FLMBYTE pucDOAddr[ 4]; + + if( !m_bOpened || m_bSetupForRead || m_bSetupForWrite || + (m_bSetupForReplace && bFirst)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + flmAssert( m_uiSearchLevel >= BH_MAX_LEVELS); + flmAssert( !m_pDb->m_pDatabase->m_pRfl || + !m_pDb->m_pDatabase->m_pRfl->isLoggingEnabled()); + + if (!uiKeyLen) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Verify the Txn type + if (m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + + // Be sure to clear the Data Only flags + + if( bFirst) + { + m_bDataOnlyBlock = FALSE; + m_bOrigInDOBlocks = FALSE; + } + + if( bFirst || bLast) + { + + // We need to locate the entry we want to replace + + if( RC_BAD( rc = findEntry( pucKey, uiKeyLen, FLM_EXACT, NULL, + pui32BlkAddr, puiOffsetIndex))) + { + goto Exit; + } + + // We must first determine if the existing entry is stored + // in data only blocks. + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + btGetEntryDataLength( pucEntry, &pucLocalData, + &uiOADataLength, &m_bOrigInDOBlocks); + + } + + if( bFirst && (!bLast || (bLast && !bTruncate && m_bOrigInDOBlocks) || + (uiKeyLen + uiDataLen > m_uiOverflowThreshold))) + { + // If bLast is not set, then we will setup to store the data in + // data only blocks. + + m_bDataOnlyBlock = TRUE; + flmAssert( m_pSCache == NULL); + + if( m_bOrigInDOBlocks) + { + // Need to get the first DO block, and work from there. + + m_ui32DOBlkAddr = bteGetBlkAddr( pucLocalData); + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32DOBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + else + { + // Get one empty block to begin with + + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( m_pDb, &m_pSCache))) + { + goto Exit; + } + + // The data going in will be stored in Data-only blocks. + // Setup the block header... + + pBlkHdr = m_pSCache->m_pBlkHdr; + pBlkHdr->ui8BlkType = BT_DATA_ONLY; + pBlkHdr->ui32PrevBlkInChain = 0; + pBlkHdr->ui32NextBlkInChain = 0; + + if (m_pLFile->uiEncDefNum) + { + ((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum = (FLMUINT32)m_pLFile->uiEncDefNum; + setBlockEncrypted( pBlkHdr); + } + + pBlkHdr->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - + sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)); + } + + m_uiDataRemaining = m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr); + m_uiDataLength = 0; + m_uiOADataLength = 0; + m_bDataOnlyBlock = TRUE; + m_bSetupForReplace = TRUE; + m_ui32DOBlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + m_ui32CurBlkAddr = m_ui32DOBlkAddr; + } + + if( m_bDataOnlyBlock) + { + if( !bTruncate && !m_bOrigInDOBlocks) + { + bTruncate = TRUE; + } + + // May need to skip over the key that is stored in the first DO block. + // We only want to do this the first time in here. The test to determine + // if this is our first time in this block is to see if the m_uiDataLength + // is equal to the m_uiDataRemaining. They would only be the same on the + // first time for each DO block. + + if( m_bOrigInDOBlocks && m_pSCache && + m_pSCache->m_pBlkHdr->ui32PrevBlkInChain == 0 && !m_uiDataLength) + { + m_uiDataRemaining -= (uiKeyLen + 2); + } + + if( RC_BAD( rc = replaceDataOnlyBlocks( pucKey, uiKeyLen, + !m_bOrigInDOBlocks && bFirst, pucData, uiDataLen, bLast, + bTruncate))) + { + goto Exit; + } + } + + // If we were writing to Data Only Blocks and we are not truncating the + // data, then we are done here. + + if( m_bDataOnlyBlock && !bTruncate) + { + if( bLast && (uiOADataLength <= m_uiOADataLength)) + { + bTruncate = TRUE; + } + else + { + goto Exit; + } + } + + // Only replace the entry on the last call. + + if( bLast) + { + FLMUINT uiLocalDataLen; + F_ELM_UPD_ACTION eAction; + + if (m_bDataOnlyBlock) + { + // build an entry that points to the DO block. + + UD2FBA( m_ui32DOBlkAddr, pucDOAddr); + + pucLocalData = &pucDOAddr[0]; + uiLocalDataLen = m_uiOADataLength; + eAction = ELM_REPLACE_DO; + + } + else + { + pucLocalData = pucData; + uiLocalDataLen = uiDataLen; + eAction = ELM_REPLACE; + } + + if( RC_BAD( rc = updateEntry( pucKey, uiKeyLen, pucLocalData, + uiLocalDataLen, eAction, bTruncate))) + { + goto Exit; + } + } + +Exit: + + if (RC_OK( rc)) + { + if (pui32BlkAddr) + { + *pui32BlkAddr = m_ui32PrimaryBlkAddr; + } + + if (puiOffsetIndex) + { + *puiOffsetIndex = m_uiCurOffset; + } + } + + if( bLast) + { + m_bSetupForReplace = FALSE; + } + + if (m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( TRUE); + return( rc); +} + +/*************************************************************************** +Desc: Function to search the Btree for a specific key. +****************************************************************************/ +RCODE F_Btree::btLocateEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT uiMatch, + FLMUINT * puiPosition, // May be NULL + FLMUINT * puiDataLength, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry = NULL; + + flmAssert( pucKey && uiKeyBufSize && puiKeyLen); + + if (!m_bOpened || m_bSetupForWrite || m_bSetupForReplace) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + m_bSetupForRead = FALSE; + + // Verify the Txn type + + if (m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // Find the entry we are interested in. + if (RC_BAD(rc = findEntry( pucKey, + *puiKeyLen, + uiMatch, + puiPosition, + pui32BlkAddr, + puiOffsetIndex))) + { + goto Exit; + } + + m_ui64LowTransId = m_pStack->pBlkHdr->stdBlkHdr.ui64TransID; + m_bMostCurrent = (m_pStack->pSCache->m_ui64HighTransID == FLM_MAX_UINT64) + ? TRUE + : FALSE; + + m_ui32PrimaryBlkAddr = m_pStack->ui32BlkAddr; + m_uiPrimaryOffset = m_pStack->uiCurOffset; + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + + // Point to the entry... + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + // Return the optional data length - get the overall data length only. + if (puiDataLength && + m_pStack->pSCache->m_pBlkHdr->ui8BlkType == BT_LEAF_DATA) + { + btGetEntryDataLength( pucEntry, NULL, puiDataLength, NULL); + } + else if (puiDataLength) + { + *puiDataLength = 0; + } + + if( RC_BAD( rc = setupReadState( m_pStack->pSCache->m_pBlkHdr, pucEntry))) + { + goto Exit; + } + + // In case the returning key is not what was originally requested, such as + // in the case of FLM_FIRST, FLM_LAST, FLM_EXCL and possibly FLM_INCL, + // we will pass back the key we actually found. + + if( uiMatch != FLM_EXACT) + { + if( RC_BAD( rc = setReturnKey( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, pucKey, puiKeyLen, + uiKeyBufSize))) + { + goto Exit; + } + } + + m_bFirstRead = FALSE; + m_bSetupForRead = TRUE; + +Exit: + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Method to get the data after a call to btLocateEntry, btNextEntry, + btPrevEntry, btFirstEntry or btLastEntry. +****************************************************************************/ +RCODE F_Btree::btGetEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT uiKeyLen, + FLMBYTE * pucData, + FLMUINT uiDataBufSize, + FLMUINT * puiDataLen + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry; + + if( !m_bOpened || !m_bSetupForRead || + m_bSetupForWrite || m_bSetupForReplace) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + if( puiDataLen) + { + *puiDataLen = 0; + } + + // Is there anything there to get? + + if( m_uiOADataRemaining == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + // If the transaction Id or the Block Change Count has changed, + // we must re-sync ourselves. + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // We must call btLocateEntry so we can re-initialize the read + + if( !m_bFirstRead) + { + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, + &uiKeyLen, FLM_EXACT))) + { + goto Exit; + } + + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + else + { + rc = RC_SET(NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + } + } + + // Get the current block. It is either a DO or a Btree block. + + if( m_pSCache == NULL) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + updateTransInfo( m_pSCache->m_pBlkHdr->ui64TransID, + m_pSCache->m_ui64HighTransID); + + // Now to find where we were the last time through. + + if( !m_bDataOnlyBlock) + { + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, + m_uiCurOffset); + + btGetEntryDataLength( pucEntry, &m_pucDataPtr, NULL, NULL); + } + else + { + m_pucDataPtr = (FLMBYTE *)m_pSCache->m_pBlkHdr + + sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr); + + // May need to skip over the key that is stored in the first DO block. + // We only want to do this the first time in here. The test to determine + // if this is our first time in this block is to see if the m_uiDataLength + // is equal to the m_uiDataRemaining. They would only be the same on the + // first time for each DO block. + + if( m_pSCache && m_pSCache->m_pBlkHdr->ui32PrevBlkInChain == 0) + { + FLMUINT16 ui16KeyLen = FB2UW( m_pucDataPtr); + + // Key lengths should be the same + + flmAssert( uiKeyLen == (FLMUINT)ui16KeyLen); + + m_pucDataPtr += (ui16KeyLen + 2); + } + } + + m_pucDataPtr += (m_uiDataLength - m_uiDataRemaining); + + if( RC_BAD( rc = extractEntryData( pucKey, uiKeyLen, pucData, + uiDataBufSize, puiDataLen))) + { + goto Exit; + } + + // Mark that we have completed our first read operation. + // No more read synchronization allowed. + + m_bFirstRead = TRUE; + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Function to locate the next entry in the Btree. The key buffer and + actual size is passed in. +****************************************************************************/ +RCODE F_Btree::btNextEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry = NULL; + FLMBOOL bAdvanced = FALSE; + + if( !m_bOpened || !m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify the transaction type + + if( m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // Make sure we are looking at btree block. If the m_bDataOnlyBlock + // flag is set, then the block address in m_ui32CurBlkAddr is a + // data only block. We must reset it to the primary block address. + + if( m_bDataOnlyBlock) + { + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + } + else + { + // If the entry did not reference a DO block, then we need to + // reset the primary block and offset with where we currently + // are incase the current block is further ahead. This saves time + // so that we don't have to scan past old blocks we are not intereseted + // in. + + m_ui32PrimaryBlkAddr = m_ui32CurBlkAddr; + m_uiPrimaryOffset = m_uiCurOffset; + m_ui64PrimaryBlkTransId = m_ui64LastBlkTransId; + } + + // Do we need to resynchronize? + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + // Doing a find with FLM_EXCL will result in our being positioned at + // the next entry. + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, puiKeyLen, + FLM_EXCL, puiDataLength))) + { + goto Exit; + } + + bAdvanced = TRUE; + } + } + + // Get the current block if we need it. + + if( !m_pSCache) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + // If we have already advanced due to a resynch, then we don't need to call + // the advanceToNextElement function, however, we do need to get the + // current entry. + + if( bAdvanced) + { + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, m_uiCurOffset); + } + else + { + for (;;) + { + // Advance to the next entry in the block. We don't have a stack so + // don't advance it. + + if( RC_BAD( rc = advanceToNextElement( FALSE))) + { + goto Exit; + } + + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, m_uiCurOffset); + + if( m_bData) + { + if( bteFirstElementFlag(pucEntry)) + { + break; + } + } + else + { + break; + } + } + } + + // Return the optional data length - get the overall data length only. + + if( puiDataLength) + { + btGetEntryDataLength( pucEntry, NULL, puiDataLength, NULL); + } + + if( RC_BAD( rc = setupReadState( m_pSCache->m_pBlkHdr, pucEntry))) + { + goto Exit; + } + + // Incase the returning key is not what was originally requested, such as in + // the case of FLM_FIRST, FLM_LAST, FLM_EXCL and possibly FLM_INCL, + // we will pass back the key we actually found. + + if( RC_BAD( rc = setReturnKey( pucEntry, m_pSCache->m_pBlkHdr->ui8BlkType, + pucKey, puiKeyLen, uiKeyBufSize))) + { + goto Exit; + } + + if( pui32BlkAddr) + { + *pui32BlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + } + + if( puiOffsetIndex) + { + *puiOffsetIndex = m_uiCurOffset; + } + + m_bFirstRead = FALSE; + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Function to get the previous entry in the Btree. +****************************************************************************/ +RCODE F_Btree::btPrevEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry = NULL; + + if( !m_bOpened || !m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify the transaction type + + if( m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // Make sure we are looking at the first block of the + // current entry. Reading of the entry could have moved us + // to another block, or if it was in a DO block, we would be + // looking at the wrong block altogether. + + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + m_ui64LastBlkTransId = m_ui64PrimaryBlkTransId; + + // Do we need to resynchronize? + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + // Doing a find with FLM_INCL will allow for the possibility that + // the original entry is no longer there. We will still have + // to backup to the previous entry. + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, puiKeyLen, + FLM_INCL, puiDataLength))) + { + goto Exit; + } + } + } + + if( !m_pSCache) + { + // Fetch the current block, then backup from there. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + for (;;) + { + // Backup to the previous entry in the block. + + if( RC_BAD( rc = backupToPrevElement( FALSE))) + { + goto Exit; + } + + // Get the entry, size etc. + + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, m_uiCurOffset); + + if( m_bData) + { + if( bteFirstElementFlag( pucEntry)) + { + break; + } + } + else + { + break; + } + } + + // Return the optional data length - get the overall data length only. + + if( puiDataLength) + { + btGetEntryDataLength( pucEntry, NULL, puiDataLength, NULL); + } + + if( RC_BAD( rc = setupReadState( m_pSCache->m_pBlkHdr, pucEntry))) + { + goto Exit; + } + + // In case the returning key is not what was originally requested, such as in + // the case of FLM_FIRST, FLM_LAST, FLM_EXCL and possibly FLM_INCL, + // we will pass back the key we actually found. + + if( RC_BAD( rc = setReturnKey( pucEntry, m_pSCache->m_pBlkHdr->ui8BlkType, + pucKey, puiKeyLen, uiKeyBufSize))) + { + goto Exit; + } + + if( pui32BlkAddr) + { + *pui32BlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + } + + if( puiOffsetIndex) + { + *puiOffsetIndex = m_uiCurOffset; + } + + m_bFirstRead = FALSE; + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Locate the first entry in the Btree and return the key. +****************************************************************************/ +RCODE F_Btree::btFirstEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + + m_Stack[ 0].pucKeyBuf = pucKey; + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, puiKeyLen, + FLM_FIRST, NULL, puiDataLength, pui32BlkAddr, puiOffsetIndex))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Locate the last entry in the Btree and return the key. +****************************************************************************/ +RCODE F_Btree::btLastEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + + m_Stack[ 0].pucKeyBuf = pucKey; + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, puiKeyLen, + FLM_LAST, NULL, puiDataLength, pui32BlkAddr, puiOffsetIndex))) + { + if( rc == NE_SFLM_BOF_HIT) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + } + + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Function to search the Btree for a specific key. +****************************************************************************/ +RCODE F_Btree::btPositionTo( + FLMUINT uiPosition, + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry = NULL; + + flmAssert( pucKey && uiKeyBufSize && puiKeyLen); + + m_bSetupForRead = FALSE; + + if( !m_bOpened || !m_bCounts) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify the transaction type + + if( m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // Find the entry we are interested in. + + if( RC_BAD(rc = positionToEntry( uiPosition))) + { + goto Exit; + } + + m_ui64LowTransId = m_pStack->pBlkHdr->stdBlkHdr.ui64TransID; + m_bMostCurrent = (m_pStack->pSCache->m_ui64HighTransID == FLM_MAX_UINT64) + ? TRUE + : FALSE; + + m_ui32PrimaryBlkAddr = m_pStack->ui32BlkAddr; + m_uiPrimaryOffset = m_pStack->uiCurOffset; + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + + // Point to the entry ... + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + if( RC_BAD( rc = setupReadState( m_pStack->pSCache->m_pBlkHdr, pucEntry))) + { + goto Exit; + } + + // In case the returning key is not what was originally requested, such + // as in the case of FLM_FIRST, FLM_LAST, FLM_EXCL and + // possibly FLM_INCL, we will pass back the key we actually found. + + if( RC_BAD( rc = setReturnKey( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, pucKey, puiKeyLen, + uiKeyBufSize))) + { + goto Exit; + } + + m_bFirstRead = FALSE; + m_bSetupForRead = TRUE; + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Method to get the actual poisition of the entry. Note: Must be + maintaining counts in the Btree AND also have located to an entry + first. The key that is passed in is used only if we have to + resynchronize due to a transaction change. +****************************************************************************/ +RCODE F_Btree::btGetPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT * puiPosition) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiKeyBufSize = uiKeyLen; + FLMUINT uiLocalKeyLen = uiKeyLen; + + if( !m_bOpened || !m_bSetupForRead || !m_bCounts) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Verify the transaction type + + if( m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + *puiPosition = 0; + + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + m_ui64LastBlkTransId = m_ui64PrimaryBlkTransId; + + // Do we need to resynchronize? + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // We can get the position easily if we have to re-sync. + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, &uiLocalKeyLen, + FLM_EXACT, puiPosition))) + { + goto Exit; + } + + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + } + else + { + // To calculate the position, we will have to reconstruct the stack. + + m_pStack = &m_Stack[ m_uiStackLevels - 1]; + for (;;) + { + // Get the block at this level. + + flmAssert( m_pStack->ui32BlkAddr); + flmAssert( m_pStack->pSCache == NULL); + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_pStack->ui32BlkAddr, NULL, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + + *puiPosition += countRangeOfKeys( m_pStack, 0, m_pStack->uiCurOffset); + + if( (getBlkType( (FLMBYTE *)m_pStack->pBlkHdr) == BT_LEAF) || + (getBlkType( (FLMBYTE *)m_pStack->pBlkHdr) == BT_LEAF_DATA)) + { + break; + } + else + { + // Next level down. (stack is inverted). + + m_pStack--; + } + } + } + +Exit: + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Method to rewind back to the beginning of the current entry. +****************************************************************************/ +RCODE F_Btree::btRewind( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache = NULL; + + if( !m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + m_ui64LastBlkTransId = m_ui64PrimaryBlkTransId; + + // Do we need to resync? + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + flmAssert( m_pSCache == NULL); + + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // Won't need this block anymore. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + if( RC_BAD( rc = btLocateEntry( pucKey, uiKeyBufSize, puiKeyLen, + FLM_EXACT))) + { + goto Exit; + } + + goto Exit; + } + } + + m_uiOADataRemaining = m_uiOADataLength; // Track the overall length progress + m_uiDataLength = m_uiPrimaryDataLen; // Restore the primary block data length + m_uiDataRemaining = m_uiDataLength; // Track the local entry progress + + if( m_bDataOnlyBlock) + { + m_ui32CurBlkAddr = m_ui32DOBlkAddr; + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32DOBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + m_ui64LastBlkTransId = pSCache->m_pBlkHdr->ui64TransID; + + // Local amount of data in this block + + m_uiDataRemaining = m_uiBlockSize - + sizeofDOBlkHdr((F_BLK_HDR *)pSCache->m_pBlkHdr) - + pSCache->m_pBlkHdr->ui16BlkBytesAvail; + + // Keep the actual local data size for later. + + m_uiDataLength = m_uiDataRemaining; + + // Now release the DO Block. We will get it again when we need it. + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + + m_bFirstRead = FALSE; + m_bSetupForRead = TRUE; + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Method for computing the number of keys and blocks between two points + in the Btree. The key count is inclusive of the two end points and + the block count is exclusive of the two end points. +****************************************************************************/ +RCODE F_Btree::btComputeCounts( + F_Btree * pUntilBtree, + FLMUINT64 * pui64BlkCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness) +{ + RCODE rc = NE_SFLM_OK; + + if( !m_bSetupForRead || !pUntilBtree->m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // Ensure that both Btrees are from the same container. + + if( m_pLFile->uiRootBlk != pUntilBtree->m_pLFile->uiRootBlk) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + rc = computeCounts( m_pStack, pUntilBtree->m_pStack, pui64BlkCount, + pui64KeyCount, pbTotalsEstimated, uiAvgBlkFullness); + +Exit: + + releaseBlocks( FALSE); + pUntilBtree->releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Function to release the blocks in the stack, and optionally, reset + the stack +****************************************************************************/ +void F_Btree::releaseBlocks( + FLMBOOL bResetStack) +{ + FLMUINT uiLevel; + + // Release any blocks still held in the stack. + + for( uiLevel = 0; uiLevel <= m_uiRootLevel; uiLevel++) + { + if( m_Stack[ uiLevel].pSCache) + { + if( m_Stack[ uiLevel].pSCache->m_uiUseCount) + { + ScaReleaseCache( m_Stack[ uiLevel].pSCache, FALSE); + } + + m_Stack[ uiLevel].pSCache = NULL; + m_Stack[ uiLevel].pBlkHdr = NULL; + } + + if( bResetStack) + { + m_Stack[ uiLevel].ui32BlkAddr = 0; + m_Stack[ uiLevel].uiKeyLen = 0; + m_Stack[ uiLevel].uiCurOffset = 0; + m_Stack[ uiLevel].uiLevel = 0; + } + } + + if( bResetStack) + { + m_uiStackLevels = 0; + m_uiRootLevel = 0; + m_bStackSetup = FALSE; + m_pStack = NULL; + } +} + +/*************************************************************************** +Desc: Function to create a new block at the current level. The new block + will always be inserted previous to the current block. All entries + that sort ahead of the current insertion point will be moved into + the new block. If there is room, the new entry will be inserted + into the current block. Otherwise, if there is room, the new entry + will be inserted into the new block. If there is still not enough + room, then if possible, it try to store a partial entry in the new + block. If we still cannot store anything, we will see if we can + store a partial entry in the current block. If that does not work, + then it will set the remaining amount and return. Another block + split will be needed before we store this entry. +****************************************************************************/ +RCODE F_Btree::splitBlock( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + FLMBOOL * pbBlockSplit) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pNewSCache = NULL; + F_CachedBlock * pPrevSCache = NULL; + F_BTREE_BLK_HDR * pBlkHdr = NULL; + FLMUINT uiBlkAddr; + FLMUINT uiEntrySize; + FLMBOOL bHaveRoom; + FLMBOOL bMovedToPrev = FALSE; + FLMBOOL bLastEntry; + FLMUINT uiMinEntrySize; + FLMBOOL bDefragBlk = FALSE; + FLMBOOL bSavedReplaceInfo = FALSE; + + // If the current block is a root block, then we will have to introduce + // a new level into the B-Tree. + + if( isRootBlk( m_pStack->pBlkHdr)) + { + if( RC_BAD( rc = createNewLevel())) + { + goto Exit; + } + } + + // If the current block is empty we must insert what we can here. + // This scenario only occurs when we are engaged in a ReplaceByInsert + // operation. Normal inserts would never result in an empty block. + // Since we know we are part of a replace operation, we know that the + // parent of this block only needs the counts updated, not the key. + + if( m_pStack->uiLevel == 0 && m_pStack->pBlkHdr->ui16NumKeys == 0) + { + if( RC_BAD( rc = storePartialEntry( pucKey, uiKeyLen, pucValue, + uiLen, uiFlags, uiChildBlkAddr, uiCounts, ppucRemainingValue, + puiRemainingLen, FALSE))) + { + goto Exit; + } + + *pbBlockSplit = FALSE; + goto MoveToPrev; + } + + // Create a new block and insert it as previous to this block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( m_pDb, &pNewSCache))) + { + goto Exit; + } + + *pbBlockSplit = TRUE; + + // Setup the header ... + + pBlkHdr = (F_BTREE_BLK_HDR *)pNewSCache->m_pBlkHdr; + + unsetRootBlk( pBlkHdr); + setBlkLfType( pBlkHdr, m_pLFile->eLfType); + + pBlkHdr->ui16NumKeys = 0; + pBlkHdr->ui8BlkLevel = (FLMUINT8)m_pStack->uiLevel; + pBlkHdr->stdBlkHdr.ui8BlkType = m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType; + pBlkHdr->ui16LogicalFile = m_pStack->pBlkHdr->ui16LogicalFile; + + // Check for encrypted block. + + if( isEncryptedBlk( (F_BLK_HDR *)m_pStack->pBlkHdr)) + { + setBlockEncrypted( (F_BLK_HDR *)pBlkHdr); + } + + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - sizeofBTreeBlkHdr(pBlkHdr)); + + pBlkHdr->ui16HeapSize = + (FLMUINT16)(m_uiBlockSize - sizeofBTreeBlkHdr(pBlkHdr)); + + // We are going to make changes to the current block. The pSCache could + // have changed since making this call, so we need to update the block + // header + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = + BtOffsetArray( (FLMBYTE *)m_pStack->pBlkHdr, 0); + + // Get the current previous block if there is one. + + uiBlkAddr = m_pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain; + + if( uiBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiBlkAddr, NULL, &pPrevSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pPrevSCache))) + { + goto Exit; + } + } + + // Link the new block between the current and it's previous + + pBlkHdr->stdBlkHdr.ui32NextBlkInChain = m_pStack->ui32BlkAddr; + pBlkHdr->stdBlkHdr.ui32PrevBlkInChain = (FLMUINT32)uiBlkAddr; + + m_pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain = + pBlkHdr->stdBlkHdr.ui32BlkAddr; + + // There may not be a previous block. + + if( pPrevSCache) + { + pPrevSCache->m_pBlkHdr->ui32NextBlkInChain = + pBlkHdr->stdBlkHdr.ui32BlkAddr; + + // Release the old previous block since we no longer need it. + + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + + // We will move all entries in the current block up to but NOT including + // the entry pointed to by uiCurOffset to the new block. + + if( m_pStack->uiCurOffset > 0) + { + if( RC_BAD( rc = moveToPrev( 0, m_pStack->uiCurOffset - 1, &pNewSCache))) + { + goto Exit; + } + + // All entries prior to the old insertion point were moved. + // Therefore, the new insertion point must be at the beginning. + + m_pStack->uiCurOffset = 0; + + // If we emptied the block. This will require us to update the parent. + + if( m_pStack->pBlkHdr->ui16NumKeys == 0) + { + if (RC_BAD( rc = saveReplaceInfo( pucKey, uiKeyLen))) + { + goto Exit; + } + + bSavedReplaceInfo = TRUE; + } + } + + // If the block is now empty, we will store a partial entry in it here. + // This scenario only occurs when we are engaged in a ReplaceByInsert + // operation. Normal inserts would never result in an empty block. + // Since we know we are part of a replace operation, we know that the + // parent of this block only needs the counts updated, not the key. + + if( m_pStack->uiLevel == 0 && m_pStack->pBlkHdr->ui16NumKeys == 0) + { + if( RC_BAD( rc = storePartialEntry( pucKey, uiKeyLen, pucValue, uiLen, + uiFlags, uiChildBlkAddr, uiCounts, ppucRemainingValue, + puiRemainingLen, FALSE))) + { + goto Exit; + } + + goto MoveToPrev; + } + + // Is there room for the new entry now in the current block? + + if( RC_BAD( rc = calcNewEntrySize( uiKeyLen, uiLen, &uiEntrySize, + &bHaveRoom, &bDefragBlk))) + { + goto Exit; + } + + if( bHaveRoom) + { + if( bDefragBlk) + { + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucValue, uiLen, + uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, uiEntrySize, + &bLastEntry))) + { + goto Exit; + } + + if( bLastEntry && !bSavedReplaceInfo) + { + // Since we just added/replaced an entry to the last position of the + // current block. we will need to preserve the current stack so that + // we can finish updating the parentage later. Should only happen as + // a result of a replace operation where the new entry is larger than + // the existing one while in the upper levels. + + if( RC_BAD( rc = saveReplaceInfo( pucKey, uiKeyLen))) + { + goto Exit; + } + } + + // If we are keeping counts, we must update those too. + + if( m_bCounts && !isRootBlk( m_pStack->pBlkHdr)) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + if( m_pStack->uiLevel == 0) + { + *ppucRemainingValue = NULL; + *puiRemainingLen = 0; + } + + goto MoveToPrev; + } + + // Can we store the whole thing in the new block? + + if( uiEntrySize <= pBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + // If this block has a parent block, and the btree is maintaining counts + // we will want to update the counts on the parent block. + + if( m_bCounts && !isRootBlk( m_pStack->pBlkHdr)) + { + if (RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + // We can release the current block since it is no longer needed. + + ScaReleaseCache( m_pStack->pSCache, FALSE); + + m_pStack->pSCache = pNewSCache; + pNewSCache = NULL; + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pBlkHdr; + m_pStack->ui32BlkAddr = pBlkHdr->stdBlkHdr.ui32BlkAddr; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + // Setting the uiCurOffset to the actual number of keys will cause the + // new entry to go in as the last element. + + m_pStack->uiCurOffset = m_pStack->pBlkHdr->ui16NumKeys; + + // We don't need to check to see if we need to defragment this block + // because it is "new". Anything that just got written to it will + // be contiguous already. + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucValue, uiLen, + uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, uiEntrySize, + &bLastEntry))) + { + goto Exit; + } + + flmAssert( bLastEntry); + + if( m_pStack->uiLevel == 0) + { + *ppucRemainingValue = NULL; + *puiRemainingLen = 0; + } + + bMovedToPrev = TRUE; + goto MoveToPrev; + } + + // Can we store part of the new entry into the new block? + // Calculate the minimum entry size to store. + + if( RC_BAD( rc = calcNewEntrySize( uiKeyLen, 1, &uiMinEntrySize, + &bHaveRoom, &bDefragBlk))) + { + goto Exit; + } + + // bHaveRoom refers to the current block, and we want to put this into + // the previous block. + + if( uiMinEntrySize <= pBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + // If this block has a parent block, and the btree is maintaining counts + // we will want to update the counts on the parent block. + + if( !isRootBlk( m_pStack->pBlkHdr)) + { + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + } + + // We can release the current block since it is no longer needed. + + ScaReleaseCache( m_pStack->pSCache, FALSE); + + m_pStack->pSCache = pNewSCache; + pNewSCache = NULL; + m_pStack->pBlkHdr = pBlkHdr; + m_pStack->ui32BlkAddr = pBlkHdr->stdBlkHdr.ui32BlkAddr; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + // Setting the uiCurOffset to the actual number of keys will cause the + // new entry to go in as the last element. + + m_pStack->uiCurOffset = m_pStack->pBlkHdr->ui16NumKeys; + + if( RC_BAD( rc = storePartialEntry( pucKey, uiKeyLen, pucValue, + uiLen, uiFlags, uiChildBlkAddr, uiCounts, ppucRemainingValue, + puiRemainingLen, TRUE))) + { + goto Exit; + } + + bMovedToPrev = TRUE; + } + else if( uiMinEntrySize <= m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + // We will store part of the entry in the current block + + if( RC_BAD( rc = storePartialEntry( + pucKey, uiKeyLen, pucValue, uiLen, uiFlags, uiChildBlkAddr, uiCounts, + ppucRemainingValue, puiRemainingLen, FALSE))) + { + goto Exit; + } + } + else + { + // Couldn't store anything, so try again after updating the parents. + + *ppucRemainingValue = pucValue; + *puiRemainingLen = uiLen; + } + +MoveToPrev: + + if( *pbBlockSplit) + { + // Release the current entry if it hasn't already been released. + + if( !bMovedToPrev && RC_OK( rc)) + { + + // If this block has a parent block, and the btree is maintaining counts + // we will want to update the counts on the parent block. + + if( !isRootBlk( m_pStack->pBlkHdr) && m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + ScaReleaseCache( m_pStack->pSCache, FALSE); + + flmAssert( pNewSCache); + + m_pStack->pSCache = pNewSCache; + pNewSCache = NULL; + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pBlkHdr; + m_pStack->ui32BlkAddr = pBlkHdr->stdBlkHdr.ui32BlkAddr; + m_pStack->uiCurOffset = m_pStack->pBlkHdr->ui16NumKeys - 1; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + } + } + +Exit: + + if( *pbBlockSplit) + { + if( m_pDb->m_pDbStats) + { + SFLM_LFILE_STATS * pLFileStats; + + if( (pLFileStats = m_pDb->getLFileStatPtr( m_pLFile)) != NULL) + { + pLFileStats->bHaveStats = TRUE; + pLFileStats->ui64BlockSplits++; + } + } + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + if( pNewSCache) + { + ScaReleaseCache( pNewSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Function to create a new level in the Btree. + This function will ensure that the F_BTSK stack is consistent with + the way it was configured before the function was called. + + This function will create a new block and copy the current contents + of the root block into it. It will then insert a single entry into + the root block to point to the new child. + + Note that there is a maximum of BH_MAX_LEVELS levels to the Btree. + Any effort to exceed that level will result in an error. +****************************************************************************/ +RCODE F_Btree::createNewLevel( void) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pNewSCache = NULL; + FLMBYTE * pSrcBlk; + FLMBYTE * pDstBlk; + F_BTREE_BLK_HDR * pBlkHdr; + FLMUINT uiCounts = 0; + FLMBYTE * pucEntry; + FLMBYTE * pucNull = NULL; + FLMBYTE ucBuffer[ SFLM_MAX_KEY_SIZE + BTE_NLC_KEY_START]; + FLMUINT uiMaxNLKey = SFLM_MAX_KEY_SIZE + BTE_NLC_KEY_START; + FLMUINT uiEntrySize; + F_BTSK * pRootStack; + FLMUINT uiFlags; + + // Assert that we are looking at the root block! + + flmAssert( isRootBlk( m_pStack->pBlkHdr)); + + // Check the root level + + if( m_pStack->uiLevel >= BH_MAX_LEVELS - 1) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_FULL); + goto Exit; + } + + // Create a new block to copy the contents of the root block into + + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( m_pDb, &pNewSCache))) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + + // Log that we are about to change the root block + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + // Update the stack since the pSCache could have changed + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)m_pStack->pBlkHdr, 0); + + // Copy the data from the root block to the new block + + pSrcBlk = (FLMBYTE *)m_pStack->pui16OffsetArray; + pBlkHdr = (F_BTREE_BLK_HDR *)pNewSCache->m_pBlkHdr; + + // Check for encryption + + if( isEncryptedBlk( (F_BLK_HDR *)m_pStack->pBlkHdr)) + { + setBlockEncrypted( (F_BLK_HDR *)pBlkHdr); + } + + pDstBlk = (FLMBYTE *)BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + unsetRootBlk( pBlkHdr); + setBlkLfType( pBlkHdr, m_pLFile->eLfType); + + pBlkHdr->ui16LogicalFile = (FLMUINT16)m_pLFile->uiLfNum; + pBlkHdr->ui16NumKeys = m_pStack->pBlkHdr->ui16NumKeys; + pBlkHdr->ui8BlkLevel = m_pStack->pBlkHdr->ui8BlkLevel; + pBlkHdr->ui16HeapSize = m_pStack->pBlkHdr->ui16HeapSize; + + pBlkHdr->stdBlkHdr.ui8BlkType = + ((F_BLK_HDR *)m_pStack->pBlkHdr)->ui8BlkType; + + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail = + ((F_BLK_HDR *)m_pStack->pBlkHdr)->ui16BlkBytesAvail; + + pBlkHdr->stdBlkHdr.ui32PrevBlkInChain = 0; + pBlkHdr->stdBlkHdr.ui32NextBlkInChain = 0; + + // Copy the data from the root block to the new block. + + f_memcpy( pDstBlk, pSrcBlk, m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr)); + + // Empty out the root block data. + +#ifdef FLM_DEBUG + f_memset( BtOffsetArray( (FLMBYTE *)m_pStack->pBlkHdr, 0), + 0, m_uiBlockSize - sizeofBTreeBlkHdr( m_pStack->pBlkHdr)); +#endif + + m_pStack->pBlkHdr->ui16NumKeys = 0; + m_pStack->pBlkHdr->ui16HeapSize = + ((F_BLK_HDR *)m_pStack->pBlkHdr)->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - sizeofBTreeBlkHdr( m_pStack->pBlkHdr)); + + // Check the root block type to see if we need to change it. The root + // block may have been a leaf node. + + if( (m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF) || + (m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA)) + { + // Need to set the block type to either + // BT_NON_LEAF or BT_NON_LEAF_COUNTS + + if( m_bCounts) + { + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType = BT_NON_LEAF_COUNTS; + } + else + { + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType = BT_NON_LEAF; + } + } + + // Now add a new entry to the stack. + + pRootStack = m_pStack; + pRootStack++; + + f_memcpy( pRootStack, m_pStack, sizeof( F_BTSK)); + + // Now fix the entries in the stack. + + pRootStack->uiLevel++; + pRootStack->pBlkHdr->ui8BlkLevel++; + pRootStack->uiCurOffset = 0; // First entry + pRootStack->pui16OffsetArray = BtOffsetArray( + (FLMBYTE *)pRootStack->pBlkHdr, 0); + + m_pStack->pBlkHdr = pBlkHdr; // Point to new block + m_pStack->ui32BlkAddr = (FLMUINT32)pNewSCache->m_uiBlkAddress; + m_pStack->pSCache = pNewSCache; + pNewSCache = NULL; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + // Build a new entry for the root block that will point to the newly created + // child block. If the root block type is BT_NON_LEAF_COUNTS, then we + // need to sum the counts from the child block + + if( m_bCounts) + { + uiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + // Create and insert a LEM entry to mark the last position in the block. + + uiFlags = BTE_FLAG_LAST_ELEMENT | BTE_FLAG_FIRST_ELEMENT; + + if( RC_BAD( rc = buildAndStoreEntry( + ((F_BLK_HDR *)pRootStack->pBlkHdr)->ui8BlkType, + uiFlags, pucNull, 0, pucNull, 0, 0, m_pStack->ui32BlkAddr, + uiCounts, &ucBuffer[ 0], uiMaxNLKey, &uiEntrySize))) + { + goto Exit; + } + + // Copy the entry into the root block. + + pucEntry = (FLMBYTE *)pRootStack->pBlkHdr + m_uiBlockSize - uiEntrySize; + f_memcpy( pucEntry, &ucBuffer[ 0], uiEntrySize); + bteSetEntryOffset( pRootStack->pui16OffsetArray, 0, + (FLMUINT16)(pucEntry - (FLMBYTE *)pRootStack->pBlkHdr)); + + pRootStack->pBlkHdr->ui16NumKeys++; + + pRootStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= (FLMUINT16)uiEntrySize + 2; + + pRootStack->pBlkHdr->ui16HeapSize -= (FLMUINT16)uiEntrySize + 2; + + m_uiStackLevels++; + m_uiRootLevel++; + +Exit: + + if( pNewSCache) + { + ScaReleaseCache( pNewSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to calculate the optimal data length size to store. This + method is called when storing a partial entry, and we need to know + what the largest data size we c an store is. +****************************************************************************/ +RCODE F_Btree::calcOptimalDataLength( + FLMUINT uiKeyLen, + FLMUINT uiDataLen, + FLMUINT uiBytesAvail, + FLMUINT * puiNewDataLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFixedAmounts; + FLMUINT uiRemainder; + + switch( ((F_BLK_HDR *)m_pStack->pBlkHdr)->ui8BlkType) + { + case BT_LEAF: + case BT_NON_LEAF: + case BT_NON_LEAF_COUNTS: + { + // These blocks do not have any data. + + *puiNewDataLen = 0; + break; + } + + case BT_LEAF_DATA: + { + // These amounts don't change. Note that the overhead includes the + // Overall Data Length Field, even though it may not be there in + // the end. + + uiFixedAmounts = BTE_LEAF_DATA_OVHD + + (uiKeyLen > ONE_BYTE_SIZE ? 2 : 1) + + uiKeyLen; + + uiRemainder = uiBytesAvail - uiFixedAmounts; + + if (uiRemainder >= (ONE_BYTE_SIZE + 2)) + { + *puiNewDataLen = uiRemainder - 2; + } + else + { + *puiNewDataLen = uiRemainder - 1; + } + break; + } + + default: + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + + if( uiDataLen < *puiNewDataLen) + { + *puiNewDataLen = uiDataLen; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This function will count the total number of keys in the block. + Typically the value ui16NumKeys will yield this number, however, if + the block type is BT_NON_LEAF_COUNTS, we also want to include the + counts in each entry. +****************************************************************************/ +RCODE F_Btree::updateParentCounts( + F_CachedBlock * pChildSCache, + F_CachedBlock ** ppParentSCache, + FLMUINT uiParentElm) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCounts; + FLMBYTE * pucCounts; + F_CachedBlock * pParentSCache; + FLMBYTE * pBlk = (FLMBYTE *)pChildSCache->m_pBlkHdr; + + flmAssert( getBlkType( pBlk) == BT_NON_LEAF_COUNTS); + uiCounts = countKeys( pBlk); + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, ppParentSCache))) + { + goto Exit; + } + + pParentSCache = *ppParentSCache; + pucCounts = BtEntry( (FLMBYTE *)pParentSCache->m_pBlkHdr, uiParentElm); + pucCounts += 4; + UD2FBA( (FLMUINT32)uiCounts, pucCounts); + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This function will count the total number of keys in the block. + Typically the value ui16NumKeys will yield this number, however, if + the block type is BT_NON_LEAF_COUNTS, we also want to include the + counts in each entry. +****************************************************************************/ +FLMUINT F_Btree::countKeys( + FLMBYTE * pBlk) +{ + FLMUINT uiTotal = 0; + FLMUINT uiIndex; + FLMBYTE * pucEntry; + FLMUINT16 * puiOffsetArray; + + puiOffsetArray = BtOffsetArray( pBlk, 0); + + if( getBlkType(pBlk) != BT_NON_LEAF_COUNTS) + { + uiTotal = ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys; + } + else + { + for (uiIndex = 0; uiIndex < + ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys; uiIndex++) + { + pucEntry = BtEntry( pBlk, uiIndex); + uiTotal += FB2UD( &pucEntry[ BTE_NLC_COUNTS]); + } + } + + return( uiTotal); +} + +/*************************************************************************** +Desc: Function to store an entry in a Data-only block. +****************************************************************************/ +RCODE F_Btree::storeDataOnlyBlocks( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL bSaveKey, + const FLMBYTE * pucData, + FLMUINT uiDataLen) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pPrevSCache = NULL; + const FLMBYTE * pucLocalData = pucData; + FLMUINT uiDataToWrite = uiDataLen; + F_BLK_HDR * pBlkHdr = NULL; + FLMBYTE * pDestPtr = NULL; + FLMUINT uiAmtToCopy; + + if( bSaveKey) + { + if( !m_pSCache) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + + // Assert that the current block is empty and has no previous link. + + flmAssert( m_pSCache->m_pBlkHdr->ui16BlkBytesAvail == + m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr)); + + flmAssert( m_pSCache->m_pBlkHdr->ui32PrevBlkInChain == 0); + + pBlkHdr = m_pSCache->m_pBlkHdr; + pDestPtr = (FLMBYTE *)pBlkHdr + + sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr); + + UW2FBA( (FLMUINT16)uiKeyLen, pDestPtr); + pDestPtr += sizeof( FLMUINT16); + + f_memcpy( pDestPtr, pucKey, uiKeyLen); + pDestPtr += uiKeyLen; + + m_uiDataRemaining -= (uiKeyLen + sizeof( FLMUINT16)); + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)m_uiDataRemaining; + } + + while( uiDataToWrite > 0) + { + if( !m_pSCache) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + + if( !bSaveKey) + { + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Now copy as much of the remaining data as we can into the new block. + + pDestPtr = (FLMBYTE *)pBlkHdr + sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr ); + pDestPtr += (m_uiBlockSize - + sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr ) - + m_uiDataRemaining); + } + else + { + bSaveKey = FALSE; + } + + uiAmtToCopy = (uiDataToWrite <= m_uiDataRemaining + ? uiDataToWrite + : m_uiDataRemaining); + + f_memcpy( pDestPtr, pucLocalData, uiAmtToCopy); + + m_uiDataRemaining -= uiAmtToCopy; + m_uiOADataLength += uiAmtToCopy; + uiDataToWrite -= uiAmtToCopy; + pucLocalData += uiAmtToCopy; + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)m_uiDataRemaining; + + // Now get the next block (if needed) + + if( uiDataToWrite) + { + pPrevSCache = m_pSCache; + m_pSCache = NULL; + + // Now create a new block + + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( m_pDb, &m_pSCache))) + { + goto Exit; + } + + pBlkHdr = m_pSCache->m_pBlkHdr; + pBlkHdr->ui8BlkType = BT_DATA_ONLY; + pBlkHdr->ui32PrevBlkInChain = pPrevSCache->m_pBlkHdr->ui32BlkAddr; + pBlkHdr->ui32NextBlkInChain = 0; + + if( m_pLFile->uiEncDefNum) + { + ((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum = (FLMUINT32)m_pLFile->uiEncDefNum; + setBlockEncrypted( pBlkHdr); + } + + pBlkHdr->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - sizeofDOBlkHdr( pBlkHdr)); + + pPrevSCache->m_pBlkHdr->ui32NextBlkInChain = pBlkHdr->ui32BlkAddr; + + m_ui32CurBlkAddr = pBlkHdr->ui32BlkAddr; + m_uiDataRemaining = m_uiBlockSize - sizeofDOBlkHdr( pBlkHdr); + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + } + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Function to Replace data in data only blocks. +****************************************************************************/ +RCODE F_Btree::replaceDataOnlyBlocks( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL bSaveKey, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bLast, + FLMBOOL bTruncate) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pPrevSCache = NULL; + const FLMBYTE * pucLocalData = pucData; + FLMUINT uiDataToWrite = uiDataLen; + F_BLK_HDR * pBlkHdr = NULL; + FLMBYTE * pDestPtr = NULL; + FLMUINT uiAmtToCopy; + FLMUINT32 ui32NextBlkAddr; + + // Do we need to store the key too? + + if( bSaveKey) + { + if( !m_pSCache) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + + // Assert that the current block is empty and has no previous link. + + flmAssert( m_pSCache->m_pBlkHdr->ui16BlkBytesAvail == + m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr)); + + flmAssert( m_pSCache->m_pBlkHdr->ui32PrevBlkInChain == 0); + + pBlkHdr = m_pSCache->m_pBlkHdr; + pDestPtr = (FLMBYTE *)pBlkHdr + + sizeofDOBlkHdr( (F_BLK_HDR *)m_pSCache->m_pBlkHdr ); + + UW2FBA( (FLMUINT16)uiKeyLen, pDestPtr); + pDestPtr += sizeof( FLMUINT16); + + f_memcpy( pDestPtr, pucKey, uiKeyLen); + pDestPtr += uiKeyLen; + + m_uiDataRemaining -= (uiKeyLen + sizeof( FLMUINT16)); + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)m_uiDataRemaining; + } + + while( uiDataToWrite > 0) + { + if( !m_pSCache) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + + if( !bSaveKey) + { + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Now copy as much of the remaining data as we can into the new block. + + pDestPtr = (FLMBYTE *)pBlkHdr + sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr); + pDestPtr += (m_uiBlockSize - sizeofDOBlkHdr( + (F_BLK_HDR *)pBlkHdr ) - m_uiDataRemaining); + } + else + { + bSaveKey = FALSE; + } + + uiAmtToCopy = (uiDataToWrite <= m_uiDataRemaining + ? uiDataToWrite + : m_uiDataRemaining); + + f_memcpy( pDestPtr, pucLocalData, uiAmtToCopy); + + m_uiDataRemaining -= uiAmtToCopy; + m_uiOADataLength += uiAmtToCopy; + uiDataToWrite -= uiAmtToCopy; + pucLocalData += uiAmtToCopy; + + if( bTruncate || (m_uiDataRemaining < pBlkHdr->ui16BlkBytesAvail)) + { + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)m_uiDataRemaining; + } + + // Now get the next block (if needed) + + if( uiDataToWrite) + { + pPrevSCache = m_pSCache; + m_pSCache = NULL; + ui32NextBlkAddr = pPrevSCache->m_pBlkHdr->ui32NextBlkInChain; + + if( ui32NextBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pSCache))) + { + goto Exit; + } + pBlkHdr = m_pSCache->m_pBlkHdr; + } + else + { + // Now create a new block + + if( RC_BAD( rc = m_pDb->m_pDatabase->createBlock( + m_pDb, &m_pSCache))) + { + goto Exit; + } + + pBlkHdr = m_pSCache->m_pBlkHdr; + pBlkHdr->ui8BlkType = BT_DATA_ONLY; + pBlkHdr->ui32PrevBlkInChain = pPrevSCache->m_pBlkHdr->ui32BlkAddr; + pBlkHdr->ui32NextBlkInChain = 0; + + if( m_pLFile->uiEncDefNum) + { + setBlockEncrypted( pBlkHdr); + ((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum = (FLMUINT32)m_pLFile->uiEncDefNum; + } + + pBlkHdr->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - sizeofDOBlkHdr( pBlkHdr)); + } + + pPrevSCache->m_pBlkHdr->ui32NextBlkInChain = pBlkHdr->ui32BlkAddr; + + m_ui32CurBlkAddr = pBlkHdr->ui32BlkAddr; + m_uiDataRemaining = m_uiBlockSize - sizeofDOBlkHdr( pBlkHdr); + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + } + } + + // If this was the last pass to store the data, then see if we need to + // remove any left over blocks. We will not truncate the data if + // the bTruncate parameter is not set. + + if( bLast && bTruncate) + { + flmAssert( m_pSCache); + + ui32NextBlkAddr = m_pSCache->m_pBlkHdr->ui32NextBlkInChain; + m_pSCache->m_pBlkHdr->ui32NextBlkInChain = 0; + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + // If there are any blocks left over, they must be freed. + + while( ui32NextBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + ui32NextBlkAddr = m_pSCache->m_pBlkHdr->ui32NextBlkInChain; + + rc = m_pDb->m_pDatabase->blockFree( m_pDb, m_pSCache); + m_pSCache = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + } + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to construct a new leaf entry using the key and value + information passed in. +****************************************************************************/ +RCODE F_Btree::buildAndStoreEntry( + FLMUINT uiBlkType, + FLMUINT uiFlags, + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMUINT uiOADataLen, // If zero, it will not be used. + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + FLMBYTE * pucBuffer, + FLMUINT uiBufferSize, + FLMUINT * puiEntrySize) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucTemp = pucBuffer; + + if( puiEntrySize) + { + *puiEntrySize = calcEntrySize( uiBlkType, uiFlags, + uiKeyLen, uiDataLen, uiOADataLen); + + if( !(*puiEntrySize) || *puiEntrySize > uiBufferSize) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + } + + switch( uiBlkType) + { + case BT_LEAF: + { + // No Data in this entry, so it is easy to make. + + UW2FBA( (FLMUINT16)uiKeyLen, pucTemp); + pucTemp += 2; + + f_memcpy( pucTemp, pucKey, uiKeyLen); + break; + } + + case BT_LEAF_DATA: + { + // Make sure the correct flags are set... + + if( uiKeyLen > ONE_BYTE_SIZE) + { + uiFlags |= BTE_FLAG_KEY_LEN; + } + else + { + uiFlags &= ~BTE_FLAG_KEY_LEN; + } + + if( uiDataLen > ONE_BYTE_SIZE) + { + uiFlags |= BTE_FLAG_DATA_LEN; + } + else + { + uiFlags &= ~BTE_FLAG_DATA_LEN; + } + + // Only the first element of an entry that spans elements + // will hold an OADataLen field. + + if( uiOADataLen && (uiFlags & BTE_FLAG_FIRST_ELEMENT)) + { + uiFlags |= BTE_FLAG_OA_DATA_LEN; + } + else + { + uiFlags &= ~BTE_FLAG_OA_DATA_LEN; + } + + // Now start setting the elements of the entry. + // Flags first. + + *pucTemp = (FLMBYTE)uiFlags; + pucTemp++; + + // KeyLen + + if( uiFlags & BTE_FLAG_KEY_LEN) + { + UW2FBA( (FLMUINT16)uiKeyLen, pucTemp); + pucTemp += 2; + } + else + { + *pucTemp = (FLMBYTE)uiKeyLen; + pucTemp++; + } + + if( uiFlags & BTE_FLAG_DATA_LEN) + { + UW2FBA( (FLMUINT16)uiDataLen, pucTemp); + pucTemp += 2; + } + else + { + *pucTemp = (FLMBYTE)uiDataLen; + pucTemp++; + } + + if( uiFlags & BTE_FLAG_OA_DATA_LEN) + { + UD2FBA( (FLMUINT32)uiOADataLen, pucTemp); + pucTemp += 4; + } + + // Key + + f_memcpy( pucTemp, pucKey, uiKeyLen); + pucTemp += uiKeyLen; + + // Data + + f_memcpy( pucTemp, pucData, uiDataLen); + break; + } + + case BT_NON_LEAF: + case BT_NON_LEAF_COUNTS: + { + // Child block address - 4 bytes + + pucTemp = pucBuffer; + + flmAssert( uiChildBlkAddr); + UD2FBA( (FLMUINT32)uiChildBlkAddr, pucTemp); + pucTemp += 4; + + // Counts - 4 bytes + + if( uiBlkType == BT_NON_LEAF_COUNTS) + { + UD2FBA( (FLMUINT32)uiCounts, pucTemp); + pucTemp += 4; + } + + // KeyLen field - 2 bytes + + UW2FBA( (FLMUINT16)uiKeyLen, pucTemp); + pucTemp += 2; + + // Key - variable length (uiKeyLen) + + f_memcpy( pucTemp, pucKey, uiKeyLen); + break; + } + + default: + { + // Invalid block type + + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to remove an entry from a block. This method will delete the + entry pointed to by the current Stack. This method does NOT defragment + the block. If the entry points to any data only blocks, they will + also be removed from circulation if the parameter bDeleteDOBlocks is + set to true. Otherwise, they will not be freed. This is so we can + call this method when we are moving entries between blocks or + replacing entries etc. +****************************************************************************/ +RCODE F_Btree::remove( + FLMBOOL bDeleteDOBlocks) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT16 * pui16OffsetArray; + FLMUINT uiNumKeys; + FLMUINT uiEntrySize; + FLMUINT uiTmp; + FLMBYTE * pucEntry; + FLMBOOL bDOBlock; + F_CachedBlock * pSCache = NULL; + FLMUINT uiBlkAddr; + FLMBYTE * pucEndOfHeap; + F_BTREE_BLK_HDR * pBlkHdr; + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, + &m_pStack->pSCache))) + { + goto Exit; + } + + pBlkHdr = m_pStack->pBlkHdr = + (F_BTREE_BLK_HDR *)m_pStack->pSCache->getBlockPtr(); + + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + uiNumKeys = pBlkHdr->ui16NumKeys; + + if( !uiNumKeys) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Point to the entry... + + pucEntry = BtEntry( (FLMBYTE *)pBlkHdr, m_pStack->uiCurOffset); + uiEntrySize = getEntrySize( (FLMBYTE *)pBlkHdr, m_pStack->uiCurOffset); + + pucEndOfHeap = (FLMBYTE *)pBlkHdr + sizeofBTreeBlkHdr(pBlkHdr) + + (uiNumKeys * 2) + pBlkHdr->ui16HeapSize; + + // We are only going to have data only blocks if we are storing data + // in the btree. + + if( m_bData) + { + bDOBlock = bteDataBlockFlag( pucEntry); + + // If the data for this entry is in one or more Data Only blocks, then + // we must delete those blocks first. + + if( bDOBlock && bDeleteDOBlocks) + { + FLMBYTE ucDOBlkAddr[ 4]; + + // Get the block address of the DO Block. + + if( RC_BAD( rc = btGetEntryData( pucEntry, ucDOBlkAddr, + sizeof( FLMUINT), NULL))) + { + goto Exit; + } + + uiBlkAddr = bteGetBlkAddr( (FLMBYTE *)&ucDOBlkAddr[ 0]); + while( uiBlkAddr) + { + // We need to delete the data only blocks first. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + // Get the next block address (if any) + + uiBlkAddr = pSCache->m_pBlkHdr->ui32NextBlkInChain; + + // Now put the block into the Avail list. + + rc = m_pDb->m_pDatabase->blockFree( m_pDb, pSCache); + pSCache = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + } + } + } + + pui16OffsetArray = m_pStack->pui16OffsetArray; + + // Move the offsets around to effectively remove the entry. + + for( uiTmp = m_pStack->uiCurOffset; (uiTmp + 1) < uiNumKeys; uiTmp++) + { + bteSetEntryOffset( pui16OffsetArray, uiTmp, + bteGetEntryOffset( pui16OffsetArray, (uiTmp + 1))); + } + +#ifdef FLM_DEBUG + // Erase the last offset entry. + + bteSetEntryOffset( pui16OffsetArray, uiTmp, 0); +#endif + + pBlkHdr->ui16NumKeys--; + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail += (FLMUINT16)uiEntrySize; + pBlkHdr->ui16HeapSize += 2; // One offset was removed. + + // Was this entry we just removed adjacent to the heap space? If + // so then we can increase the heap space. + + if( pucEndOfHeap == pucEntry) + { + pBlkHdr->ui16HeapSize += (FLMUINT16)actualEntrySize(uiEntrySize); + } + +#ifdef FLM_DEBUG + // Let's erase whatever was in the entry space. + + f_memset( pucEntry, 0, actualEntrySize(uiEntrySize)); +#endif + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to remove multiple entries from a block. The entries must be + contiguous. If any entries store data in data-only blocks, they will + be freed and put into the avail list. +****************************************************************************/ +RCODE F_Btree::removeRange( + FLMUINT uiStartElm, + FLMUINT uiEndElm, + FLMBOOL bDeleteDOBlocks) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT16 * pui16OffsetArray; + FLMUINT uiNumKeys; + FLMUINT uiEntrySize; + FLMBYTE * pucEntry; + FLMBOOL bDOBlock; + F_CachedBlock * pSCache = NULL; + FLMUINT uiBlkAddr; + FLMUINT uiCurOffset; + FLMUINT uiCounter; + FLMBYTE * pucEndOfHeap; + FLMBYTE * pucStartOfHeap; + F_BTREE_BLK_HDR * pBlkHdr; + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + pBlkHdr = m_pStack->pBlkHdr = + (F_BTREE_BLK_HDR *)m_pStack->pSCache->getBlockPtr(); + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + uiNumKeys = pBlkHdr->ui16NumKeys; + + if( !uiNumKeys) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + flmAssert( uiEndElm < uiNumKeys); + + // Point to the entry ... + + for( uiCurOffset = uiStartElm; uiCurOffset <= uiEndElm; uiCurOffset++) + { + pucEntry = BtEntry( (FLMBYTE *)pBlkHdr, uiCurOffset); + uiEntrySize = getEntrySize( (FLMBYTE *)pBlkHdr, uiCurOffset); + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail += (FLMUINT16)uiEntrySize; + pBlkHdr->ui16NumKeys--; + + bDOBlock = bteDataBlockFlag(pucEntry); + + // If the data for this entry is in a Data Only block, then we must delete + // those blocks first. + + if( bDOBlock && bDeleteDOBlocks) + { + FLMBYTE ucDOBlkAddr[ 4]; + + // Get the block address of the DO Block. + + if( RC_BAD( rc = btGetEntryData( pucEntry, ucDOBlkAddr, 4, NULL))) + { + goto Exit; + } + + uiBlkAddr = bteGetBlkAddr( (FLMBYTE *)&ucDOBlkAddr[ 0]); + while( uiBlkAddr) + { + // We need to delete the data only blocks first. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + // Get the next block address (if any) + + uiBlkAddr = pSCache->m_pBlkHdr->ui32NextBlkInChain; + + // Now put the block into the Avail list. + + rc = m_pDb->m_pDatabase->blockFree( m_pDb, pSCache); + pSCache = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + } + } + + // Now erase the old entry + +#ifdef FLM_DEBUG + f_memset( pucEntry, 0, actualEntrySize(uiEntrySize)); +#endif + } + + // Move the offsets around to effectively remove the entries. + + pui16OffsetArray = m_pStack->pui16OffsetArray; + if( uiEndElm < (uiNumKeys - 1)) + { + // We will need to move the remaining offsets forward + // to delete the desired range. + + for (uiCurOffset = uiStartElm, uiCounter = 0; + uiCounter < (uiNumKeys - (uiEndElm + 1)); + uiCounter++, uiCurOffset++) + { + bteSetEntryOffset( pui16OffsetArray, uiCurOffset, + bteGetEntryOffset( pui16OffsetArray, + (uiEndElm + uiCounter + 1))); + } + } + +#ifdef FLM_DEBUG + // Erase the remaining offsets + + while (uiCurOffset < (uiNumKeys - 1)) + { + bteSetEntryOffset( pui16OffsetArray, uiCurOffset++, 0); + } + +#endif + + // We need to determine if we have gained any more heap space. We start + // by pointing to the end of the block, them moving forward until we reach + // the closest entry. + + pucEndOfHeap = (FLMBYTE *)pBlkHdr + m_uiBlockSize; + + for ( uiCurOffset = 0; uiCurOffset < pBlkHdr->ui16NumKeys; uiCurOffset++) + { + pucEntry = BtEntry( (FLMBYTE *)pBlkHdr, uiCurOffset); + + if (pucEntry < pucEndOfHeap) + { + pucEndOfHeap = pucEntry; + } + } + + // Now clean up the heap space. + + pucStartOfHeap = (FLMBYTE *)pBlkHdr + sizeofBTreeBlkHdr( pBlkHdr) + + (pBlkHdr->ui16NumKeys * 2); + + pBlkHdr->ui16HeapSize = (FLMUINT16)(pucEndOfHeap - pucStartOfHeap); + +#ifdef FLM_DEBUG + f_memset( pucStartOfHeap, 0, pBlkHdr->ui16HeapSize); +#endif + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to try to move entries (whole) from the target block to the + previous block. The entries may be moved, up to but not including + the current entry position. We do not want to change the parentage + of this block. We need to use the stack to fix up the parentage of + the previous block. Entries are moved from the lowest to highest. +****************************************************************************/ +RCODE F_Btree::moveEntriesToPrevBlk( + FLMUINT uiNewEntrySize, + F_CachedBlock ** ppPrevSCache, + FLMBOOL * pbEntriesWereMoved) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLocalAvailSpace; + FLMUINT uiAvailSpace; + FLMUINT uiHeapSize; + F_CachedBlock * pPrevSCache = NULL; + FLMUINT uiPrevBlkAddr; + FLMUINT uiOAEntrySize = 0; + FLMUINT uiStart; + FLMUINT uiFinish; + FLMUINT uiCount; + FLMUINT uiOffset; + + // Assume nothing to move. + + *pbEntriesWereMoved = FALSE; + + // If we are already at the first entry in the block, there + // is nothing that we can move since we will always insert ahead of + // the current position. + + if( !m_pStack->uiCurOffset) + { + goto Exit; + } + + // Get the previous block. + + if( (uiPrevBlkAddr = + m_pStack->pSCache->m_pBlkHdr->ui32PrevBlkInChain) == 0) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiPrevBlkAddr, NULL, &pPrevSCache))) + { + goto Exit; + } + + uiLocalAvailSpace = m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + uiAvailSpace = pPrevSCache->m_pBlkHdr->ui16BlkBytesAvail; + uiHeapSize = ((F_BTREE_BLK_HDR *)pPrevSCache->m_pBlkHdr)->ui16HeapSize; + + // If we add the available space in this block and the previous block, would + // it be enough to make room for the new entry? If so, then we will + // see if we can make that room by moving ( whole) entries. + + if( (uiAvailSpace + uiLocalAvailSpace) < uiNewEntrySize) + { + goto Exit; + } + + uiStart = 0; + uiFinish = m_pStack->uiCurOffset; + + // Get the size of each entry until we are over the available size limit + + for( uiOffset = 0, uiCount = 0 ; uiOffset < uiFinish; uiOffset++) + { + FLMUINT uiLocalEntrySize; + + uiLocalEntrySize = getEntrySize( (FLMBYTE *)m_pStack->pBlkHdr, uiOffset); + + if( (uiLocalEntrySize + uiOAEntrySize) < uiAvailSpace) + { + uiOAEntrySize += uiLocalEntrySize; + uiLocalAvailSpace += uiLocalEntrySize; + uiCount++; + } + else + { + break; + } + } + + if( !uiCount) + { + goto Exit; + } + + // It looks like we can move at least one entry. + // Will this give use enough room to store the new entry? + + if( uiLocalAvailSpace < uiNewEntrySize) + { + // Moving these entries will not benefit us, so don't bother + + goto Exit; + } + + // Do we need to defragment the block first? + + if( uiHeapSize < uiOAEntrySize) + { + flmAssert( uiHeapSize != uiAvailSpace); + if( RC_BAD( rc = defragmentBlock( &pPrevSCache))) + { + goto Exit; + } + } + + // We are going to get some benefit from moving, so let's do it... + + if (RC_BAD( rc = moveToPrev( uiStart, uiStart + uiCount - 1, &pPrevSCache))) + { + goto Exit; + } + + // We will need to return this block. + + *ppPrevSCache = pPrevSCache; + pPrevSCache = NULL; + + // Adjust the current offset in the stack so we are still pointing to the + // same entry. + + m_pStack->uiCurOffset -= uiCount; + + // If this block has a parent block, and the btree is maintaining counts + // we will want to update the counts on the parent block. + + if( !isRootBlk( m_pStack->pBlkHdr)) + { + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + } + + *pbEntriesWereMoved = TRUE; + +Exit: + + if (pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: This method will move entries beginning at uiStart, up to and + including uiFinish from the current block (m_pStack) to pPrevSCache. + As a part of this operation, both the target block and the source + block will be changed. A call to logPhysBlock will be made before + each block is changed. Never move the highest key in the block + because we don't want to have to update the parentage of the + current block... +****************************************************************************/ +RCODE F_Btree::moveToPrev( + FLMUINT uiStart, + FLMUINT uiFinish, + F_CachedBlock ** ppPrevSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT16 * pui16DstOffsetA = NULL; + F_BTREE_BLK_HDR * pSrcBlkHdr = NULL; + F_BTREE_BLK_HDR * pDstBlkHdr = NULL; + FLMBYTE * pucSrcEntry; + FLMBYTE * pucDstEntry; + FLMUINT uiEntrySize; + FLMUINT uiIndex; + F_CachedBlock * pPrevSCache; + FLMBOOL bEntriesCombined = FALSE; + + // Make sure we have logged the block we are changing. + // Note that the source block will be logged in the removeRange method. + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, ppPrevSCache))) + { + goto Exit; + } + + pPrevSCache = *ppPrevSCache; + pSrcBlkHdr = m_pStack->pBlkHdr; + pDstBlkHdr = (F_BTREE_BLK_HDR *)pPrevSCache->m_pBlkHdr; + pui16DstOffsetA = BtOffsetArray( (FLMBYTE *)pDstBlkHdr, 0); + + pucDstEntry = getBlockEnd( pDstBlkHdr); + + // Beginning at the start, copy each entry over from the source + // to the destination block. + + for( uiIndex = uiStart; uiIndex <= uiFinish; uiIndex++) + { + if( RC_BAD( rc = combineEntries( pSrcBlkHdr, uiIndex, pDstBlkHdr, + (pDstBlkHdr->ui16NumKeys + ? pDstBlkHdr->ui16NumKeys - 1 + : 0), + &bEntriesCombined, &uiEntrySize))) + { + goto Exit; + } + + if( bEntriesCombined) + { + F_BTSK tmpStack; + F_BTSK * pTmpStack; + + tmpStack.pSCache = pPrevSCache; + tmpStack.pBlkHdr = pDstBlkHdr; + tmpStack.uiCurOffset = pDstBlkHdr->ui16NumKeys - 1; // Last entry + + pTmpStack = m_pStack; + m_pStack = &tmpStack; + + rc = remove( FALSE); + m_pStack = pTmpStack; + + if( RC_BAD( rc)) + { + goto Exit; + } + + if( pDstBlkHdr->ui16HeapSize != + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + if( RC_BAD( rc = defragmentBlock( &pPrevSCache))) + { + goto Exit; + } + } + + pucDstEntry = getBlockEnd( pDstBlkHdr) - uiEntrySize; + f_memcpy( pucDstEntry, m_pucTempBlk, uiEntrySize); + + bteSetEntryOffset( pui16DstOffsetA, + pDstBlkHdr->ui16NumKeys++, + (FLMUINT16)(pucDstEntry - (FLMBYTE *)pDstBlkHdr)); + + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= + ((FLMUINT16)uiEntrySize + 2); + + pDstBlkHdr->ui16HeapSize -= ((FLMUINT16)uiEntrySize + 2); + bEntriesCombined = FALSE; + } + else + { + pucSrcEntry = BtEntry( (FLMBYTE *)pSrcBlkHdr, uiIndex); + uiEntrySize = getEntrySize( (FLMBYTE *)pSrcBlkHdr, uiIndex); + pucDstEntry -= actualEntrySize(uiEntrySize); + + f_memcpy( pucDstEntry, pucSrcEntry, actualEntrySize(uiEntrySize)); + + bteSetEntryOffset( pui16DstOffsetA, + pDstBlkHdr->ui16NumKeys++, + (FLMUINT16)(pucDstEntry - (FLMBYTE *)pDstBlkHdr)); + + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= (FLMUINT16)uiEntrySize; + pDstBlkHdr->ui16HeapSize -= (FLMUINT16)uiEntrySize; + } + } + + // Now remove the entries from the Src block. + + if( RC_BAD( rc = removeRange( uiStart, uiFinish, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to try to move entries (whole) from the target block to the + next block. The entries may be moved up to but not including + the current entry position depending on how much room is available if + any. Entries are moved from the highest to lowest. +****************************************************************************/ +RCODE F_Btree::moveEntriesToNextBlk( + FLMUINT uiNewEntrySize, + FLMBOOL * pbEntriesWereMoved) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLocalAvailSpace; + FLMUINT uiAvailSpace; + FLMUINT uiHeapSize; + F_CachedBlock * pNextSCache = NULL; + FLMUINT uiNextBlkAddr; + FLMUINT uiOAEntrySize = 0; + FLMUINT uiStart; + FLMUINT uiFinish; + FLMUINT uiCount; + FLMUINT uiOffset; + F_CachedBlock * pChildSCache = NULL; + F_CachedBlock * pParentSCache = NULL; + F_BTSK * pParentStack; + FLMUINT uiLevel; + FLMBOOL bReleaseChild = FALSE; + FLMBOOL bReleaseParent = FALSE; + FLMBOOL bCommonParent = FALSE; + + // Assume nothing to move. + + *pbEntriesWereMoved = FALSE; + + // Get the next block. + + if( (uiNextBlkAddr = m_pStack->pSCache->m_pBlkHdr->ui32NextBlkInChain) == 0) + { + goto Exit; + } + + if( (FLMUINT16)m_pStack->uiCurOffset >= (m_pStack->pBlkHdr->ui16NumKeys - 1)) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiNextBlkAddr, NULL, &pNextSCache))) + { + goto Exit; + } + + // Our first task is to determine if we can move anything at all. + // How much free space is there in the next block? + + uiLocalAvailSpace = m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + uiAvailSpace = pNextSCache->m_pBlkHdr->ui16BlkBytesAvail; + uiHeapSize = ((F_BTREE_BLK_HDR *)pNextSCache->m_pBlkHdr)->ui16HeapSize; + + // If we add the available space in this block and the next block, would + // it be enough to make room for the new entry? If so, then we will + // see if we can make that room by moving ( whole) entries. + + if( (uiAvailSpace + uiLocalAvailSpace) < uiNewEntrySize) + { + goto Exit; + } + + // Begin at the last entry and work backward. + + uiStart = m_pStack->pBlkHdr->ui16NumKeys - 1; + uiFinish = m_pStack->uiCurOffset; + + // Get the size of each entry (plus 2 for the offset entry) until we are + // over the available size limit. + + for( uiOffset = uiStart, uiCount = 0 ; uiOffset > uiFinish; uiOffset--) + { + FLMUINT uiLocalEntrySize; + + uiLocalEntrySize = getEntrySize( (FLMBYTE *)m_pStack->pBlkHdr, + uiOffset); + + if( (uiLocalEntrySize + uiOAEntrySize) < uiAvailSpace) + { + uiOAEntrySize += uiLocalEntrySize; + uiLocalAvailSpace += uiLocalEntrySize; + uiCount++; + } + else + { + break; + } + } + + if( uiCount == 0) + { + goto Exit; + } + + // It looks like we can move at least one entry. + // Will this give use enough room to store the new entry? + + if( uiLocalAvailSpace < uiNewEntrySize) + { + goto Exit; + } + + flmAssert( uiStart > uiFinish); + + // Do we need to defragment the block first before we do the move? + + if( uiHeapSize < uiOAEntrySize) + { + flmAssert( uiHeapSize != uiAvailSpace); + if( RC_BAD( rc = defragmentBlock( &pNextSCache))) + { + goto Exit; + } + } + + // We are going to get some benefit from moving, so let's do it... + + if( RC_BAD( rc = moveToNext( uiStart, uiStart - uiCount + 1, + &pNextSCache))) + { + goto Exit; + } + + // If this block has a parent block, and the btree is maintaining counts + // we will need to update the counts on the parent blocks. + + if( m_bCounts) + { + for( uiLevel = m_pStack->uiLevel; + uiLevel < m_uiStackLevels - 1; + uiLevel++) + { + pParentStack = &m_Stack[ uiLevel + 1]; + + // If we are at "current" level, then we want to use the pNextSCache + // block as the child. Otherwise, we want to use the previous parent + // block as the child. + + if( uiLevel == m_pStack->uiLevel) + { + pChildSCache = pNextSCache; + bReleaseChild = TRUE; + pNextSCache = NULL; + } + else + { + pChildSCache = pParentSCache; + bReleaseChild = bReleaseParent; + bReleaseParent = FALSE; + } + + // Check to see if the parent entry is the last entry in the + // block. If it is, then we will need to get the next block. + // If the parent block is the same for both blocks, then we + // only need to reference the next entry. We don't want to release + // the parent as it is referenced in the stack. + + if( bCommonParent || + (pParentStack->uiCurOffset < + (FLMUINT)(pParentStack->pBlkHdr->ui16NumKeys - 1))) + { + pParentSCache = pParentStack->pSCache; + bReleaseParent = FALSE; + + if (RC_BAD( rc = updateParentCounts( pChildSCache, &pParentSCache, + (bCommonParent + ? pParentStack->uiCurOffset + : pParentStack->uiCurOffset + 1)))) + { + goto Exit; + } + + // The parent has changed, so update the stack. + + pParentStack->pBlkHdr = + (F_BTREE_BLK_HDR *)pParentSCache->m_pBlkHdr; + + pParentStack->pSCache = pParentSCache; + bCommonParent = TRUE; + } + else + { + // We need to get the next block at the parent level first. We + // release the previous parent if there was one. + + uiNextBlkAddr = pParentStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain; + + flmAssert( uiNextBlkAddr); + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiNextBlkAddr, NULL, &pParentSCache))) + { + goto Exit; + } + + bReleaseParent = TRUE; + + if( RC_BAD( rc = updateParentCounts( pChildSCache, + &pParentSCache, 0))) + { + goto Exit; + } + } + + if( bReleaseChild) + { + ScaReleaseCache( pChildSCache, FALSE); + pChildSCache = NULL; + bReleaseChild = FALSE; + } + } + } + + *pbEntriesWereMoved = TRUE; + +Exit: + + if( pChildSCache && bReleaseChild) + { + ScaReleaseCache( pChildSCache, FALSE); + } + + if( pParentSCache && bReleaseParent) + { + ScaReleaseCache( pParentSCache, FALSE); + } + + if( pNextSCache) + { + ScaReleaseCache( pNextSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: This method will move entries beginning at uiStart, down to and + including uiFinish from the current block (m_pStack) to pNextSCache. + As a part of this operation, both the target block and the source + block will be changed. +****************************************************************************/ +RCODE F_Btree::moveToNext( + FLMUINT uiStart, + FLMUINT uiFinish, + F_CachedBlock ** ppNextSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT16 * pui16DstOffsetA = NULL; + F_BTREE_BLK_HDR * pSrcBlkHdr = NULL; + F_BTREE_BLK_HDR * pDstBlkHdr = NULL; + FLMBYTE * pucSrcEntry; + FLMBYTE * pucDstEntry; + FLMUINT uiEntrySize; + FLMINT iIndex; + FLMUINT uiBytesToCopy; + FLMUINT uiNumKeysToAdd; + F_CachedBlock * pNextSCache = *ppNextSCache; + FLMBOOL bEntriesCombined; + FLMBYTE * pucOffsetArray; + + // Make sure we have logged the block we are changing. + // Note that the source block will be logged in the removeRange method. + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pNextSCache))) + { + goto Exit; + } + + // SCache block may have changed. Need to pass it back. + + *ppNextSCache = pNextSCache; + + pSrcBlkHdr = m_pStack->pBlkHdr; + pDstBlkHdr = (F_BTREE_BLK_HDR *)pNextSCache->m_pBlkHdr; + + // We will need to save off the current offset array. We will do this + // by copying it into our temporary block. + + uiBytesToCopy = pDstBlkHdr->ui16NumKeys * 2; + if( uiBytesToCopy > m_uiBufferSize) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + pui16DstOffsetA = BtOffsetArray((FLMBYTE *)pDstBlkHdr, 0); + pucOffsetArray = &m_pucBuffer[ m_uiBufferSize] - uiBytesToCopy; + + f_memcpy( pucOffsetArray, (FLMBYTE *)pui16DstOffsetA, uiBytesToCopy); + + // Point to the last entry in the block. + + pucDstEntry = getBlockEnd( pDstBlkHdr); + + // Beginning at the start, copy each entry over from the Src to the Dst + // block. Note that the uiStart parameter represents a higher position + // in the block. In otherwords, we are actually copying from the end or + // highest position to a lower position in the block. Therefore we want + // to make sure the offset array is copied in the same way, otherwise it + // would reverse the order of the entries. + + uiNumKeysToAdd = uiStart - uiFinish + 1; + pui16DstOffsetA = (FLMUINT16 *)pucOffsetArray; + + for( iIndex = uiStart; iIndex >= (FLMINT)uiFinish; iIndex--) + { + if( RC_BAD( rc = combineEntries( pSrcBlkHdr, iIndex, pDstBlkHdr, + 0, &bEntriesCombined, &uiEntrySize))) + { + goto Exit; + } + + if( bEntriesCombined) + { + F_BTSK tmpStack; + F_BTSK * pTmpStack; + + tmpStack.pSCache = pNextSCache; + tmpStack.pBlkHdr = pDstBlkHdr; + tmpStack.uiCurOffset = 0; // 1st entry. + + pTmpStack = m_pStack; + m_pStack = &tmpStack; + + rc = remove( FALSE); + m_pStack = pTmpStack; + + if (RC_BAD( rc)) + { + goto Exit; + } + + if( pDstBlkHdr->ui16HeapSize != + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + if( RC_BAD( rc = defragmentBlock( &pNextSCache))) + { + goto Exit; + } + + // Refresh the saved offset array. + + uiBytesToCopy -= 2; + pucOffsetArray = &m_pucBuffer[ m_uiBufferSize] - uiBytesToCopy; + + f_memcpy( pucOffsetArray, + (FLMBYTE *)BtOffsetArray( (FLMBYTE *)pDstBlkHdr, 0), + uiBytesToCopy); + } + + pucDstEntry = getBlockEnd( pDstBlkHdr) - uiEntrySize; + f_memcpy( pucDstEntry, m_pucTempBlk, uiEntrySize); + + bteSetEntryOffset( pui16DstOffsetA, 0, + (FLMUINT16)(pucDstEntry - (FLMBYTE *)pDstBlkHdr)); + + pDstBlkHdr->ui16NumKeys++; + + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= + ((FLMUINT16)uiEntrySize + 2); + + pDstBlkHdr->ui16HeapSize -= ((FLMUINT16)uiEntrySize + 2); + + bEntriesCombined = FALSE; + } + else + { + pucSrcEntry = BtEntry( (FLMBYTE *)pSrcBlkHdr, iIndex); + uiEntrySize = getEntrySize( (FLMBYTE *)pSrcBlkHdr, iIndex); + + pucDstEntry -= actualEntrySize(uiEntrySize); + + f_memcpy( pucDstEntry, pucSrcEntry, + actualEntrySize(uiEntrySize)); + + pui16DstOffsetA--; + + bteSetEntryOffset( pui16DstOffsetA, 0, + (FLMUINT16)(pucDstEntry - (FLMBYTE *)pDstBlkHdr)); + + pDstBlkHdr->ui16NumKeys++; + pDstBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= (FLMUINT16)uiEntrySize; + pDstBlkHdr->ui16HeapSize -= (FLMUINT16)uiEntrySize; + } + } + + // Now put the new offset array into the block. + + f_memcpy( BtOffsetArray( (FLMBYTE *)pDstBlkHdr, 0), + pui16DstOffsetA, + &m_pucBuffer[ m_uiBufferSize] - (FLMBYTE *)pui16DstOffsetA); + + // Now remove the entries from the Src block. + + if( RC_BAD( rc = removeRange( uiFinish, uiStart, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to advance to the next entry. If there are no more entries + in the block, it will release the current block and get the next in + the chain. If there are no more entries, i.e. no more blocks in + the chain, NE_SFLM_EOF_HIT will be returned. +****************************************************************************/ +RCODE F_Btree::advanceToNextElement( + FLMBOOL bAdvanceStack) +{ + RCODE rc = NE_SFLM_OK; + F_BTREE_BLK_HDR * pBlkHdr; + + flmAssert( m_pSCache); + + pBlkHdr = (F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr; + + if( m_uiCurOffset + 1 >= pBlkHdr->ui16NumKeys) + { + // We are out of entries in this block, so we will release it + // and get the next block in the chain (if any). + + if( RC_BAD( rc = getNextBlock( &m_pSCache))) + { + goto Exit; + } + + m_ui32PrimaryBlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + m_uiPrimaryOffset = 0; + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = 0; + + if( bAdvanceStack) + { + if( RC_BAD( rc = moveStackToNext( m_pSCache))) + { + goto Exit; + } + + // This block now has two uses. It will be released twice. + + m_pSCache->m_uiUseCount++; + } + } + else + { + m_uiPrimaryOffset++; + m_uiCurOffset++; + m_pStack->uiCurOffset++; + } + +Exit: + + // We do not want to release the m_pSCache here. That is to be done by the + // caller. + + return( rc); +} + +/*************************************************************************** +Desc: Method to backup the stack to the previous entry. If there are no + more entries in the block, it will release the current block and get + the previous in the chain. If there are no more entries, i.e. no + more blocks in the chain, NE_SFLM_BOF_HIT will be returned. +****************************************************************************/ +RCODE F_Btree::backupToPrevElement( + FLMBOOL bBackupStack) +{ + RCODE rc = NE_SFLM_OK; + F_BTREE_BLK_HDR * pBlkHdr; + + flmAssert( m_pSCache); + + pBlkHdr = (F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr; + + if( !m_uiCurOffset) + { + // We are out of entries in this block, so we will release it + // and get the previous block in the chain (if any). + + if( RC_BAD( rc = getPrevBlock( &m_pSCache))) + { + goto Exit; + } + + m_ui32PrimaryBlkAddr = m_pSCache->m_pBlkHdr->ui32BlkAddr; + + m_uiPrimaryOffset = + ((F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr)->ui16NumKeys - 1; + + m_ui32CurBlkAddr = m_ui32PrimaryBlkAddr; + m_uiCurOffset = m_uiPrimaryOffset; + + if( bBackupStack) + { + if( RC_BAD( rc = moveStackToPrev( m_pSCache))) + { + goto Exit; + } + + // This block now has two uses. It will be released twice. + + m_pSCache->m_uiUseCount++; + } + } + else + { + m_uiPrimaryOffset--; + m_uiCurOffset--; + m_pStack->uiCurOffset--; + } + +Exit: + + // We do not want to release the m_pSCache here. That is to be done by the + // caller. + + return( rc); +} + +/*************************************************************************** +Desc: Method to extract the key length from a given entry. The optional + pucKeyRV is a buffer where we can return the address of the start of + the actual key. +****************************************************************************/ +FLMUINT F_Btree::getEntryKeyLength( + FLMBYTE * pucEntry, + FLMUINT uiBlockType, + const FLMBYTE ** ppucKeyRV) +{ + FLMUINT uiKeyLength; + FLMBYTE * pucTmp = NULL; + + // The way we get the key length depends on the type of block we have. + + switch( uiBlockType) + { + case BT_LEAF_DATA: + { + pucTmp = &pucEntry[ 1]; // skip past the flags + + if( bteKeyLenFlag( pucEntry)) + { + uiKeyLength = FB2UW( pucTmp); + pucTmp += 2; + } + else + { + uiKeyLength = *pucTmp; + pucTmp += 1; + } + + if( bteDataLenFlag(pucEntry)) + { + pucTmp += 2; + } + else + { + pucTmp += 1; + } + + // Check for the presence of the OverallDataLength field (4 bytes). + + if( bteOADataLenFlag( pucEntry)) + { + pucTmp += 4; + } + + break; + } + + case BT_LEAF: + { + uiKeyLength = FB2UW( pucEntry); + + if( ppucKeyRV) + { + pucTmp = &pucEntry[ BTE_KEY_START]; + } + + break; + } + + case BT_NON_LEAF: + { + uiKeyLength = FB2UW( &pucEntry[ BTE_NL_KEY_LEN]); + + if( ppucKeyRV) + { + pucTmp = &pucEntry[ BTE_NL_KEY_START]; + } + + break; + } + + case BT_NON_LEAF_COUNTS: + { + uiKeyLength = FB2UW( &pucEntry[ BTE_NLC_KEY_LEN]); + + if( ppucKeyRV) + { + pucTmp = &pucEntry[ BTE_NLC_KEY_START]; + } + + break; + } + + default: + { + flmAssert( 0); + uiKeyLength = 0; + pucTmp = NULL; + break; + } + } + + // Do we need to return the key pointer? + + if( ppucKeyRV) + { + *ppucKeyRV = pucTmp; + } + + return( uiKeyLength); +} + +/*************************************************************************** +Desc: Method to extract the data length from a given entry. The parameter + pucDataRV is an optional return value that will hold the address + of the beginning of the data in the entry. This method + ** assumes ** the entry is from a BT_LEAF_DATA block. No other block + type has any data. +****************************************************************************/ +FSTATIC FLMUINT btGetEntryDataLength( + FLMBYTE * pucEntry, + const FLMBYTE ** ppucDataRV, // Optional + FLMUINT * puiOADataLengthRV, // Optional + FLMBOOL * pbDOBlockRV) // Optional +{ + const FLMBYTE * pucTmp; + FLMUINT uiDataLength; + FLMUINT uiKeyLength; + + pucTmp = &pucEntry[ 1]; // skip past the flags + + if( bteKeyLenFlag( pucEntry)) + { + uiKeyLength = FB2UW( pucTmp); + pucTmp += 2; + } + else + { + uiKeyLength = *pucTmp; + pucTmp += 1; + } + + if( bteDataLenFlag(pucEntry)) + { + uiDataLength = FB2UW( pucTmp); + pucTmp += 2; + } + else + { + uiDataLength = *pucTmp; + pucTmp += 1; + } + + // Check for the presence of the OverallDataLength field (4 bytes). + + if( bteOADataLenFlag(pucEntry)) + { + if( puiOADataLengthRV) + { + *puiOADataLengthRV = FB2UD( pucTmp); + } + pucTmp += 4; + } + else if (puiOADataLengthRV) + { + *puiOADataLengthRV = uiDataLength; + } + + // Are we to return a pointer to the data? + + if( ppucDataRV) + { + // Advance to the Data since we are currently pointing to the Key. + + *ppucDataRV = (FLMBYTE *)(pucTmp + uiKeyLength); + } + + if( pbDOBlockRV) + { + *pbDOBlockRV = bteDataBlockFlag( pucEntry); + } + + return( uiDataLength); +} + +/*************************************************************************** +Desc: Method to extract the data value from a given block. This method + expects to receive a buffer to copy the data into. This method does + not read data across blocks. The puiLenDataRV is an optional + parameter that will hold the actual data size returned. +****************************************************************************/ +FSTATIC RCODE btGetEntryData( + FLMBYTE * pucEntry, // Pointer to the entry containing the data + FLMBYTE * pucBufferRV, + FLMUINT uiBufferSize, + FLMUINT * puiLenDataRV) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDataLength; + const FLMBYTE * pucData; + + // Get the data length + + uiDataLength = btGetEntryDataLength( pucEntry, &pucData, NULL, NULL); + + if( uiDataLength > uiBufferSize) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + +#ifdef FLM_DEBUG + f_memset( pucBufferRV, 0, uiBufferSize); +#endif + f_memcpy( pucBufferRV, pucData, uiDataLength); + + // Do we need to return the data length? + + if( puiLenDataRV) + { + *puiLenDataRV = uiDataLength; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This method will return the overall size of the entry at uiOffset in + pBlk. The size returned includes a two byte allowance for the offset + entry used by this entry. +****************************************************************************/ +FLMUINT F_Btree::getEntrySize( + FLMBYTE * pBlk, + FLMUINT uiOffset, + FLMBYTE ** ppucEntry) +{ + FLMBYTE * pucEntry; + FLMUINT uiEntrySize; + + // Point to the entry ... + + pucEntry = BtEntry( pBlk, uiOffset); + + if( ppucEntry) + { + *ppucEntry = pucEntry; + } + + // Different block types have different entry formats. + + switch( getBlkType( pBlk)) + { + case BT_LEAF: + { + uiEntrySize = 4 + FB2UW( pucEntry); + break; + } + case BT_LEAF_DATA: + { + FLMBYTE * pucTmp = &pucEntry[ 1]; + + // Stuff we know + + uiEntrySize = 3; + + // Get the key length + + if( *pucEntry & BTE_FLAG_KEY_LEN) + { + uiEntrySize += FB2UW( pucTmp) + 2; + pucTmp += 2; + } + else + { + uiEntrySize += (*pucTmp + 1); + pucTmp++; + } + + // Get the data length + + if( *pucEntry & BTE_FLAG_DATA_LEN) + { + // 2 byte data length field + + uiEntrySize += (FB2UW( pucTmp) + 2); + } + else + { + // 1 byte data length field + + uiEntrySize += (FLMUINT)*pucTmp + 1; + } + + // Get the Overall Data length (if present) + + if( *pucEntry & BTE_FLAG_OA_DATA_LEN) + { + uiEntrySize += 4; + } + + break; + } + + case BT_NON_LEAF: + { + uiEntrySize = 8 + FB2UW( &pucEntry[ BTE_NL_KEY_LEN]); + break; + } + + case BT_NON_LEAF_COUNTS: + { + uiEntrySize = 12 + FB2UW( &pucEntry[ BTE_NLC_KEY_LEN]); + break; + } + + default: + { + flmAssert( 0); + uiEntrySize = 0; + break; + } + } + + return( uiEntrySize); +} + +/*************************************************************************** +Desc: Method to search the BTree for a specific entry. Upon a successful + return from this method, the local stack will be setup and pointing + to either the desired entry, or if the entry does not exist, it will + be pointing to the entry that would be immediately after the desired + entry. This method therefore can be used both for reads and updates + where we want to insert a new entry into the BTree. +****************************************************************************/ +RCODE F_Btree::findEntry( + const FLMBYTE * pucKey, // In + FLMUINT uiKeyLen, // In + FLMUINT uiMatch, // In + FLMUINT * puiPosition, // Out + FLMUINT32 * pui32BlkAddr, // In/Out + FLMUINT * puiOffsetIndex) // In/Out +{ + RCODE rc = NE_SFLM_OK; + F_BTSK * pStack = NULL; + FLMUINT32 ui32BlkAddress; + F_CachedBlock * pSCache = NULL; + FLMBYTE * pucEntry; + FLMUINT uiPrevCounts = 0; + FLMUINT uiLevel; + + // Make sure the stack is clean before we start. + + btRelease(); + + // No input key is needed to get the first or last key. + + if( uiMatch == FLM_FIRST || uiMatch == FLM_LAST) + { + uiKeyLen = 0; + } + + if( uiKeyLen > SFLM_MAX_KEY_SIZE) + { + rc = RC_SET( NE_SFLM_BTREE_KEY_SIZE); + goto Exit; + } + + // Have we been passed a block address to look in? + + if( pui32BlkAddr && *pui32BlkAddr) + { + if( RC_OK( rc = findInBlock( pucKey, uiKeyLen, uiMatch, puiPosition, + pui32BlkAddr, puiOffsetIndex))) + { + goto Exit; + } + } + + // Beginning at the root node, we will scan until we find the first key + // that is greater than or equal to our target key. If we don't find any + // key that is larger than our target key, we will use the last block found. + + ui32BlkAddress = (FLMUINT32)m_pLFile->uiRootBlk; + + for( ;;) + { + // Get the block - Note that this will place a use on the block. + // It must be properly released when done. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddress, NULL, &pSCache))) + { + goto Exit; + } + + // We are building the stack inverted to make traversing it a bit easier. + + uiLevel = ((F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr)->ui8BlkLevel; + pStack = &m_Stack[ uiLevel]; + + m_uiStackLevels++; + + pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + pStack->ui32BlkAddr = ui32BlkAddress; + pStack->pSCache = pSCache; + pSCache = NULL; + pStack->uiLevel = uiLevel; + pStack->uiKeyLen = uiKeyLen; + pStack->pucKeyBuf = pucKey; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pStack->pBlkHdr, 0); + + if( isRootBlk( pStack->pBlkHdr)) + { + m_uiRootLevel = uiLevel; + } + + // Search the block for the key. When we return from this method + // the pStack will be pointing to the last entry we looked at. + + if( RC_BAD( rc = scanBlock( pStack, uiMatch))) + { + // It is okay if we couldn't find the key. Especially if + // we are still in the upper levels of the B-tree. + + if( (rc != NE_SFLM_NOT_FOUND) && (rc != NE_SFLM_EOF_HIT)) + { + goto Exit; + } + } + + // Are we at the end of our search? + + if( (pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA) || + (pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF) || + (m_uiStackLevels - 1 >= m_uiSearchLevel)) + { + if( m_bCounts && puiPosition) + { + flmAssert( m_uiSearchLevel >= BH_MAX_LEVELS); + *puiPosition = uiPrevCounts + pStack->uiCurOffset; + } + + // If this is a search for the last entry, then we should adjust the + // uiCurOffset so that it points to a valid entry. + + if( uiMatch == FLM_LAST) + { + m_pStack = pStack; + + for (;;) + { + if( RC_BAD( rc = moveStackToPrev( NULL))) + { + goto Exit; + } + + // If we are on the leaf level, we need to make sure we are + // looking at a first occurrence of an entry. + + if( pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA) + { + pucEntry = BtEntry( + (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + if( bteFirstElementFlag( pucEntry)) + { + break; + } + } + else + { + break; + } + } + } + + break; + } + else + { + if( m_bCounts && puiPosition) + { + uiPrevCounts += countRangeOfKeys( pStack, 0, pStack->uiCurOffset); + } + + // Get the Child Block Address + + pucEntry = BtEntry( + (FLMBYTE *)pStack->pSCache->m_pBlkHdr, pStack->uiCurOffset); + + ui32BlkAddress = bteGetBlkAddr( pucEntry); + } + } + + // Return the block and offset if needed. + + if( pui32BlkAddr) + { + *pui32BlkAddr = pStack->ui32BlkAddr; + } + + if( puiOffsetIndex) + { + *puiOffsetIndex = pStack->uiCurOffset; + } + + m_bStackSetup = TRUE; + +Exit: + + if( RC_OK( rc) || (rc == NE_SFLM_NOT_FOUND) || (rc == NE_SFLM_EOF_HIT)) + { + if( pStack) + { + m_pStack = pStack; + } + } + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Private method to search for a particular key in a pre-designted + block offset. If we don't find it at the given offset, we will do a + binary search for it. Note that a uiMatch of FLM_FIRST & FLM_LAST + will be ignored if we locate the entry by the puiOffsetIndex parameter. + Also, this method does not setup the full stack. Only the level where + the block address passed in resides. +****************************************************************************/ +RCODE F_Btree::findInBlock( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT uiMatch, + FLMUINT * puiPosition, + FLMUINT32 * pui32BlkAddr, + FLMUINT * puiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + F_BTSK * pStack; + F_CachedBlock * pSCache = NULL; + FLMBYTE * pucEntry; + const FLMBYTE * pucBlkKey; + FLMUINT uiBlkKeyLen; + + // Get the block - Note that this will place a use on the block. + // It must be properly released when done. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + *pui32BlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + if( !blkIsBTree( pSCache->getBlockPtr())) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + + // Verify that the block belongs to the correct collection number + + if( ((F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr)->ui16LogicalFile != + m_pLFile->uiLfNum) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + + // Verify that we are looking at the same type of block, + // i.e. collection vs index. + + if( ((F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr)->ui8BTreeFlags & BLK_IS_INDEX && + m_pLFile->eLfType != SFLM_LF_INDEX) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + + // If the block is not a leaf block, the caller will + // need to do a full search down the B-Tree + + if( ((F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr)->ui8BlkLevel != 0) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + + pStack = &m_Stack[ 0]; + m_uiStackLevels++; + + pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + pStack->ui32BlkAddr = *pui32BlkAddr; + pStack->pSCache = pSCache; + pSCache = NULL; + pStack->uiLevel = 0; + pStack->uiKeyLen = uiKeyLen; + pStack->pucKeyBuf = pucKey; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pStack->pBlkHdr, 0); + pStack->uiCurOffset = puiOffsetIndex ? *puiOffsetIndex : 0; + + if( isRootBlk( pStack->pBlkHdr)) + { + m_uiRootLevel = 0; + } + + // See if the entry we are looking for is at the passed offset + + if( puiOffsetIndex) + { + if( *puiOffsetIndex < pStack->pBlkHdr->ui16NumKeys) + { + pucEntry = BtEntry( (FLMBYTE *)pStack->pBlkHdr, *puiOffsetIndex); + + uiBlkKeyLen = getEntryKeyLength( pucEntry, + getBlkType( (FLMBYTE *)pStack->pBlkHdr), &pucBlkKey); + + if( uiKeyLen == uiBlkKeyLen) + { + if( f_memcmp( pucKey, pucBlkKey, uiKeyLen) == 0) + { + goto GotEntry; + } + } + } + } + + // Search the block for the key. When we return from this method + // the pStack will be pointing to the last entry we looked at. + + if( RC_BAD( rc = scanBlock( pStack, uiMatch))) + { + goto Exit; + } + +GotEntry: + + if( m_bCounts && puiPosition) + { + flmAssert( m_uiSearchLevel >= BH_MAX_LEVELS); + + // VISIT: These counts aren't accurate in this case. + + *puiPosition = pStack->uiCurOffset; + } + + // Verify that we are looking at an entry with the firstElement flag set. + + m_pStack = pStack; + + for (;;) + { + // If we are on the leaf level, we need to make sure we are + // looking at a first occurrence of an entry. + + if( m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA) + { + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + if( bteFirstElementFlag( pucEntry)) + { + break; + } + } + else + { + break; + } + + if( RC_BAD( rc = moveStackToPrev( NULL))) + { + goto Exit; + } + } + + *pui32BlkAddr = m_pStack->ui32BlkAddr; + + if( puiOffsetIndex) + { + *puiOffsetIndex = m_pStack->uiCurOffset; + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + if( RC_BAD( rc)) + { + btRelease(); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to search through a BTree block to find a specific key. If + that key cannot be found, then the pStack will be positioned right + after the last entry in the block. The search is a binary search that + is looking for the first key that is >= the target key. The uiMatch + parameter further qualifies the search. The FLM_FIRST & FLM_LAST + values will ignore the key altogether and just return the first or last + key respectively. The FLM_INCL value will return the key if found or the + first key following if not found. The FLM_EXACT will return an + NE_SFLM_NOT_FOUND if the key cannot be found. FLM_EXCL will return + the first key following the target key. +****************************************************************************/ +RCODE F_Btree::scanBlock( + F_BTSK * pStack, + FLMUINT uiMatch) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTop; + FLMUINT uiMid; + FLMUINT uiBottom; + FLMINT iResult; + F_CachedBlock * pSCache = NULL; + const FLMBYTE * pucBlockKey; + FLMBYTE * pucEntry; + FLMUINT uiBlockKeyLen; + + if( pStack->pBlkHdr->ui16NumKeys == 0) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + + uiTop = 0; + uiBottom = (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1); + + if( uiMatch == FLM_FIRST) + { + pStack->uiCurOffset = uiTop; + goto Exit; + } + + if( uiMatch == FLM_LAST || pStack->uiKeyLen == 0) + { + pStack->uiCurOffset = uiBottom; + goto Exit; + } + + flmAssert( uiMatch == FLM_INCL || uiMatch == FLM_EXCL || + uiMatch == FLM_EXACT); + + // Test the first entry + + pucEntry = (FLMBYTE *)pStack->pBlkHdr + + bteGetEntryOffset( pStack->pui16OffsetArray, + uiTop); + + uiBlockKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)pStack->pBlkHdr)->ui8BlkType, + &pucBlockKey); + + // Compare the entries ... + + if( !uiBlockKeyLen) + { + // The LEM entry will always sort last!! + + iResult = 1; + goto ResultGreater1; + } + else + { + if( RC_BAD( rc = compareBlkKeys( pucBlockKey, uiBlockKeyLen, + pStack->pucKeyBuf, pStack->uiKeyLen, &iResult))) + { + goto Exit; + } + } + + if( iResult >= 0) + { +ResultGreater1: + + if( iResult && uiMatch == FLM_EXACT) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + } + + uiMid = uiTop; + goto VerifyPosition; + } + + // If there is more than one entry in the block, we can skip the first + // one since we have already seen it. + + if( uiTop < uiBottom) + { + uiTop++; + } + + // Test the last + + pucEntry = (FLMBYTE *)pStack->pBlkHdr + + bteGetEntryOffset( pStack->pui16OffsetArray, + uiBottom); + + uiBlockKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)pStack->pBlkHdr)->ui8BlkType, + &pucBlockKey); + + if( !uiBlockKeyLen) + { + // The LEM entry will always sort last!! + + iResult = 1; + goto ResultGreater2; + } + else + { + if( RC_BAD( rc = compareBlkKeys( pucBlockKey, uiBlockKeyLen, + pStack->pucKeyBuf, pStack->uiKeyLen, &iResult))) + { + goto Exit; + } + } + + if( iResult <= 0) + { + if( iResult < 0 && uiMatch != FLM_INCL) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + } + + uiMid = uiBottom; + goto VerifyPosition; + } + +ResultGreater2: + + for( ;;) + { + + if( uiTop == uiBottom) + { + // We're done - didn't find it. + + if( uiMatch == FLM_EXACT) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + } + + uiMid = uiTop; + break; + } + + // Get the midpoint + + uiMid = (uiTop + uiBottom) / 2; + + pucEntry = (FLMBYTE *)pStack->pBlkHdr + + bteGetEntryOffset( pStack->pui16OffsetArray, + uiMid); + + uiBlockKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)pStack->pBlkHdr)->ui8BlkType, + &pucBlockKey); + + // Compare the entries + + if( !uiBlockKeyLen) + { + // The LEM entry will always sort last!! + + iResult = 1; + goto ResultGreater; + } + else + { + if( RC_BAD( rc = compareBlkKeys( pucBlockKey, uiBlockKeyLen, + pStack->pucKeyBuf, pStack->uiKeyLen, &iResult))) + { + goto Exit; + } + } + + if( iResult > 0) + { +ResultGreater: + + // Midpoint (block key) is > Target key + + uiBottom = uiMid; + continue; + } + + if( iResult < 0) + { + // Midpoint (block key) is < Target key + // Since we want to find the first key that is >= to the target key, + // and we have aleady visited the key at uiMid and know that it is < + // our target key, we can skip it and advance to the key that is one + // beyond it. + + flmAssert( uiMid < uiBottom); + uiTop = uiMid + 1; + continue; + } + + break; + } + +VerifyPosition: + + if( uiMatch != FLM_EXCL) + { + // Verify that we are looking at the first occurrence of this key. + + while( iResult == 0) + { + if( uiMid > 0) + { + pucEntry = (FLMBYTE *)pStack->pBlkHdr + + bteGetEntryOffset( pStack->pui16OffsetArray, + (uiMid - 1)); + + uiBlockKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)pStack->pBlkHdr)->ui8BlkType, + &pucBlockKey); + + if( !uiBlockKeyLen) + { + // The LEM entry will always sort last!! + + iResult = 1; + } + else + { + if( RC_BAD( rc = compareBlkKeys( pucBlockKey, uiBlockKeyLen, + pStack->pucKeyBuf, pStack->uiKeyLen, &iResult))) + { + goto Exit; + } + + if( iResult == 0) + { + uiMid--; + } + } + } + else + { + break; + } + } + + pStack->uiCurOffset = uiMid; + } + else if( uiMatch == FLM_EXCL) + { + // If we are at the leaf level, then we want to see if + // this is the last entry in the last block. + // If it is, then we cannot satisfy the request, otherwise + // we will position to the next key and return ok. + + if( pStack->pBlkHdr->ui8BlkLevel == 0 && + pStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == 0 && + uiMid == (FLMUINT)pStack->pBlkHdr->ui16NumKeys - 1 && + iResult == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + } + else if( pStack->pBlkHdr->ui8BlkLevel == 0) + { + // Check for the next entry at leaf level + + while( iResult == 0) + { + // Are we on the last key? + + if( uiMid == (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1)) + { + if( pStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == 0) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + } + else + { + pStack->uiCurOffset = uiMid; + m_pStack = pStack; + + if( RC_BAD( rc = moveStackToNext( NULL))) + { + goto Exit; + } + + uiMid = 0; + } + } + else + { + uiMid++; + } + + pucEntry = (FLMBYTE *)pStack->pBlkHdr + + bteGetEntryOffset( pStack->pui16OffsetArray, + uiMid); + + uiBlockKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)pStack->pBlkHdr)->ui8BlkType, + &pucBlockKey); + + if( !uiBlockKeyLen) + { + // The LEM entry will always sort last!! + + iResult = 1; + } + else + { + if( RC_BAD( rc = compareBlkKeys( pucBlockKey, uiBlockKeyLen, + pStack->pucKeyBuf, pStack->uiKeyLen, &iResult))) + { + goto Exit; + } + } + } + + pStack->uiCurOffset = uiMid; + if( uiMid == (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1) && + pStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + } + } + else + { + pStack->uiCurOffset = uiMid; + } + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: This method will compare two key fields. + Returned values: 0 - Keys are equal + 1 - Key in Block is > Target key + -1 - Key in Block is < Target key +****************************************************************************/ +RCODE F_Btree::compareKeys( + const FLMBYTE * pucKey1, + FLMUINT uiKeyLen1, + const FLMBYTE * pucKey2, + FLMUINT uiKeyLen2, + FLMINT * piCompare) +{ + RCODE rc = NE_SFLM_OK; + + if( !m_pCompare) + { + if( (*piCompare = f_memcmp( pucKey1, pucKey2, + f_min( uiKeyLen1, uiKeyLen2))) == 0) + { + *piCompare = uiKeyLen1 == uiKeyLen2 + ? 0 + : uiKeyLen1 < uiKeyLen2 + ? -1 + : 1; + } + } + else + { + if( RC_BAD( rc = m_pCompare->compare( pucKey1, uiKeyLen1, + pucKey2, uiKeyLen2, piCompare))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method for positioning to a specific entry. +****************************************************************************/ +RCODE F_Btree::positionToEntry( + FLMUINT uiPosition) +{ + RCODE rc = NE_SFLM_OK; + F_BTSK * pStack = NULL; + FLMUINT32 ui32BlkAddress; + F_CachedBlock * pSCache = NULL; + FLMUINT uiLevel; + FLMBYTE * pucEntry; + FLMUINT uiPrevCounts = 0; + + // Make sure the stack is clean before we start. + + btRelease(); + + // Beginning at the root node. + + ui32BlkAddress = (FLMUINT32)m_pLFile->uiRootBlk; + + // Get the block - Note that this will place a use on the block. + // It must be properly released when done. + + while( ui32BlkAddress) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddress, NULL, &pSCache))) + { + goto Exit; + } + + uiLevel = ((F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr)->ui8BlkLevel; + pStack = &m_Stack[ uiLevel]; + + pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + pStack->ui32BlkAddr = ui32BlkAddress; + pStack->pSCache = pSCache; + pSCache = NULL; + pStack->uiLevel = uiLevel; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pStack->pBlkHdr, 0); + + m_uiStackLevels++; + + if( RC_BAD( rc = searchBlock( pStack->pBlkHdr, &uiPrevCounts, + uiPosition, &pStack->uiCurOffset))) + { + goto Exit; + } + + if( (pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF_DATA) || + (pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_LEAF)) + { + ui32BlkAddress = 0; + } + else + { + // Get the next child block address + + pucEntry = BtEntry( (FLMBYTE *)pStack->pBlkHdr, + pStack->uiCurOffset); + + ui32BlkAddress = bteGetBlkAddr( pucEntry); + } + } + + m_uiRootLevel = m_uiStackLevels - 1; + +Exit: + + if( RC_OK( rc) || (rc == NE_SFLM_NOT_FOUND) || (rc == NE_SFLM_EOF_HIT)) + { + m_pStack = pStack; + } + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: +****************************************************************************/ +RCODE F_Btree::searchBlock( + F_BTREE_BLK_HDR * pBlkHdr, + FLMUINT * puiPrevCounts, + FLMUINT uiPosition, + FLMUINT * puiOffset) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOffset; + FLMUINT uiNumKeys; + FLMUINT uiCounts; + FLMBYTE * pucEntry; + + uiNumKeys = pBlkHdr->ui16NumKeys; + + if( getBlkType( (FLMBYTE *)pBlkHdr) != BT_NON_LEAF_COUNTS) + { + flmAssert( uiPosition >= *puiPrevCounts); + + uiOffset = uiPosition - *puiPrevCounts; + *puiPrevCounts = uiPosition; + } + else + { + for( uiOffset = 0; uiOffset < uiNumKeys; uiOffset++) + { + pucEntry = BtEntry( (FLMBYTE *)pBlkHdr, uiOffset); + pucEntry += 4; + + uiCounts = FB2UD( pucEntry); + + if( *puiPrevCounts + uiCounts >= (uiPosition + 1)) + { + break; + } + else + { + *puiPrevCounts += uiCounts; + } + } + } + + if( uiOffset >= uiNumKeys) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + } + + *puiOffset = uiOffset; + return( rc); +} + +/*************************************************************************** +Desc: Method to move all the data in the block into a contiguous space. +****************************************************************************/ +RCODE F_Btree::defragmentBlock( + F_CachedBlock ** ppSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumKeys; + FLMBOOL bSorted; + FLMBYTE * pucCurEntry; + FLMBYTE * pucPrevEntry; + FLMBYTE * pucTempEntry; + FLMUINT uiTempToMove; + FLMUINT uiIndex; + FLMUINT uiAmtToMove; + FLMUINT uiFirstHole; + FLMUINT16 ui16BlkBytesAvail; + FLMUINT16 * pui16OffsetArray; + F_CachedBlock * pSCache = *ppSCache; + F_BTREE_BLK_HDR * pBlk = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + F_BTREE_BLK_HDR * pOldBlk = NULL; + FLMBYTE * pucHeap; + FLMBYTE * pucBlkEnd; + F_CachedBlock * pOldSCache = NULL; + + flmAssert( pBlk->stdBlkHdr.ui16BlkBytesAvail != pBlk->ui16HeapSize); + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &pSCache, &pOldSCache))) + { + goto Exit; + } + + pBlk = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + *ppSCache = pSCache; + uiNumKeys = getBlkEntryCount( (FLMBYTE *)pBlk); + + // Determine if the entries are sorted + + pucPrevEntry = (FLMBYTE *)pBlk + m_uiBlockSize; + bSorted = TRUE; + uiFirstHole = 0; + pucHeap = (FLMBYTE *)pBlk + m_uiBlockSize; + + for( uiIndex = 0; uiIndex < uiNumKeys; uiIndex++) + { + pucCurEntry = BtEntry( (FLMBYTE *)pBlk, uiIndex); + + if( pucPrevEntry < pucCurEntry) + { + bSorted = FALSE; + break; + } + else + { + uiAmtToMove = actualEntrySize( + getEntrySize( (FLMBYTE *)pBlk, uiIndex)); + pucHeap -= uiAmtToMove; + + if( !uiFirstHole && pucHeap != pucCurEntry) + { + uiFirstHole = uiIndex + 1; + } + } + + pucPrevEntry = pucCurEntry; + } + + ui16BlkBytesAvail = (FLMUINT16)(m_uiBlockSize - sizeofBTreeBlkHdr( pBlk)) - + (FLMUINT16)(uiNumKeys * 2); + pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlk, 0); + pucBlkEnd = (FLMBYTE *)pBlk + m_uiBlockSize; + + if( uiFirstHole > 1) + { + uiFirstHole--; + pucHeap = BtEntry( (FLMBYTE *)pBlk, uiFirstHole - 1); + ui16BlkBytesAvail -= (FLMUINT16)(pucBlkEnd - pucHeap); + } + else + { + uiFirstHole = 0; + pucHeap = pucBlkEnd; + } + + if( !bSorted) + { + FLMUINT16 * pui16OldOffsetArray; + + // If old and new blocks are the same (because of a + // prior call to logBlock), we need to save a copy of the block + // before making changes. + + if( !pOldSCache) + { + f_memcpy( m_pucTempDefragBlk, pSCache->m_pBlkHdr, m_uiBlockSize); + pOldBlk = (F_BTREE_BLK_HDR *)m_pucTempDefragBlk; + } + else + { + pOldBlk = (F_BTREE_BLK_HDR *)pOldSCache->m_pBlkHdr; + } + + pui16OldOffsetArray = BtOffsetArray( (FLMBYTE *)pOldBlk, 0); + + // Rebuild the block so that all of the entries are in order + + for( uiIndex = uiFirstHole; uiIndex < uiNumKeys; uiIndex++) + { + pucCurEntry = BtEntry( (FLMBYTE *)pOldBlk, uiIndex); + uiAmtToMove = actualEntrySize( getEntrySize( (FLMBYTE *)pOldBlk, uiIndex)); + pucHeap -= uiAmtToMove; + bteSetEntryOffset( pui16OffsetArray, uiIndex, + (FLMUINT16)(pucHeap - (FLMBYTE *)pBlk)); + uiIndex++; + + while( uiIndex < uiNumKeys) + { + pucTempEntry = BtEntry( (FLMBYTE *)pOldBlk, uiIndex); + uiTempToMove = actualEntrySize( getEntrySize( (FLMBYTE *)pOldBlk, uiIndex)); + + if ((pucCurEntry - uiTempToMove) != pucTempEntry) + { + uiIndex--; + break; + } + else + { + pucCurEntry -= uiTempToMove; + pucHeap -= uiTempToMove; + uiAmtToMove += uiTempToMove; + bteSetEntryOffset( pui16OffsetArray, uiIndex, + (FLMUINT16)(pucHeap - (FLMBYTE *)pBlk)); + uiIndex++; + } + } + + f_memcpy( pucHeap, pucCurEntry, uiAmtToMove); + ui16BlkBytesAvail -= (FLMUINT16)uiAmtToMove; + } + } + else + { + // Work back from the first hole. Move entries to fill all of the + // holes in the block. + + for( uiIndex = uiFirstHole; uiIndex < uiNumKeys; uiIndex++) + { + pucCurEntry = BtEntry( (FLMBYTE *)pBlk, uiIndex); + uiAmtToMove = actualEntrySize( getEntrySize( (FLMBYTE *)pBlk, uiIndex)); + pucHeap -= uiAmtToMove; + + if( pucHeap != pucCurEntry) + { + // We have a hole. We don't want to move just one entry + // if we can avoid it. We would like to continue searching + // until we find either the end, or another hole. Then we + // can move a larger block of data instead of one entry. + + bteSetEntryOffset( pui16OffsetArray, uiIndex, + (FLMUINT16)(pucHeap - (FLMBYTE *)pBlk)); + uiIndex++; + + while( uiIndex < uiNumKeys) + { + pucTempEntry = BtEntry( (FLMBYTE *)pBlk, uiIndex); + uiTempToMove = actualEntrySize( + getEntrySize( (FLMBYTE *)pBlk, uiIndex)); + + if( (pucCurEntry - uiTempToMove) != pucTempEntry) + { + uiIndex--; + break; + } + else + { + pucCurEntry -= uiTempToMove; + pucHeap -= uiTempToMove; + uiAmtToMove += uiTempToMove; + bteSetEntryOffset( pui16OffsetArray, + uiIndex, (FLMUINT16)(pucHeap - (FLMBYTE *)pBlk)); + uiIndex++; + } + } + } + + // Now move the range we have determined. + + f_memmove( pucHeap, pucCurEntry, uiAmtToMove); + ui16BlkBytesAvail -= (FLMUINT16)(uiAmtToMove); + } + } + + // Set the available space. If there are no keys in this block, we should + // set the it to the calculated available space + + if( !uiNumKeys) + { + pBlk->stdBlkHdr.ui16BlkBytesAvail = ui16BlkBytesAvail; + } + + flmAssert( pBlk->stdBlkHdr.ui16BlkBytesAvail == ui16BlkBytesAvail); + pBlk->ui16HeapSize = ui16BlkBytesAvail; + + // Clean up the heap space. + +#ifdef FLM_DEBUG + f_memset( getBlockEnd( pBlk) - ui16BlkBytesAvail, 0, ui16BlkBytesAvail); +#endif + +Exit: + + if( pOldSCache) + { + ScaReleaseCache( pOldSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to handle the insertion, deletion and replacment of a single + entry in a block. + Assumption: The find method has already been called to locate the + insertion point, so the stack has already been setup. +****************************************************************************/ +RCODE F_Btree::updateEntry( + const FLMBYTE * pucKey, // In + FLMUINT uiKeyLen, // In + const FLMBYTE * pucValue, // In + FLMUINT uiLen, // In + F_ELM_UPD_ACTION eAction, + FLMBOOL bTruncate) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucRemainingValue = NULL; + FLMUINT uiRemainingLen = 0; + const FLMBYTE * pucSavKey = pucKey; + FLMUINT uiSavKeyLen = uiKeyLen; + FLMUINT uiChildBlkAddr = 0; + FLMUINT uiCounts = 0; + FLMUINT uiFlags = BTE_FLAG_FIRST_ELEMENT | BTE_FLAG_LAST_ELEMENT; + FLMBOOL bMoreToRemove = FALSE; + FLMBOOL bDone = FALSE; + FLMUINT uiOrigDataLen = uiLen; + FLMBOOL bOrigTruncate = bTruncate; + + flmAssert( m_pReplaceInfo == NULL); + + // For each level that needs modifying... + + while( !bDone) + { + + switch( eAction) + { + case ELM_INSERT_DO: + { + // In this case, the uiLen parameter represents the OADataLength. + + uiFlags = BTE_FLAG_DATA_BLOCK | + BTE_FLAG_FIRST_ELEMENT | + BTE_FLAG_LAST_ELEMENT | + BTE_FLAG_OA_DATA_LEN; + + if( RC_BAD( rc = insertEntry( &pucKey, &uiKeyLen, pucValue, + uiLen, uiFlags, &uiChildBlkAddr, &uiCounts, &pucRemainingValue, + &uiRemainingLen, &eAction))) + { + goto Exit; + } + + // Not needed for upper levels of the Btree. + + pucValue = NULL; + uiLen = 0; + break; + } + + case ELM_INSERT: + { + // This function will return all info needed to handle the next + // level up in the Btree (if anything), including setting up + // the stack. pucKey & uiKeyLen will be pointing to the key that + // the upper level needs to insert, replace or delete. + // + // It will be pointing to an entry in a lower level block, so that + // block must not be released until after we are all done. + + if( RC_BAD( rc = insertEntry( &pucKey, &uiKeyLen, pucValue, + uiLen, uiFlags, &uiChildBlkAddr, &uiCounts, &pucRemainingValue, + &uiRemainingLen, &eAction))) + { + goto Exit; + } + + // Not needed for upper levels of the Btree. + + pucValue = NULL; + uiLen = 0; + break; + } + + case ELM_REPLACE_DO: + { + // In this case, the uiLen parameter represents the OADataLength. + + uiFlags = BTE_FLAG_DATA_BLOCK | + BTE_FLAG_FIRST_ELEMENT | + BTE_FLAG_LAST_ELEMENT | + BTE_FLAG_OA_DATA_LEN; + + // Should only get here if we are able to truncate the data. + + flmAssert( bTruncate); + + if( RC_BAD( rc = replaceEntry( &pucKey, &uiKeyLen, pucValue, + uiLen, uiFlags, &uiChildBlkAddr, &uiCounts, + &pucRemainingValue, &uiRemainingLen, &eAction))) + { + goto Exit; + } + + // Not needed for upper levels of the Btree. + + pucValue = NULL; + uiLen = 0; + bTruncate = TRUE; + break; + } + + case ELM_REPLACE: + { + if( RC_BAD( rc = replaceEntry( &pucKey, &uiKeyLen, pucValue, + uiLen, uiFlags, &uiChildBlkAddr, &uiCounts, &pucRemainingValue, + &uiRemainingLen, &eAction, bTruncate))) + { + goto Exit; + } + + // Not needed for upper levels of the Btree. + + pucValue = NULL; + uiLen = 0; + bTruncate = TRUE; + break; + } + + case ELM_REMOVE: + { + if (RC_BAD( rc = removeEntry( &pucKey, &uiKeyLen, &uiChildBlkAddr, + &uiCounts, &bMoreToRemove, &eAction))) + { + goto Exit; + } + + // Not needed for upper levels of the B-Tree. + + pucValue = NULL; + uiLen = 0; + + break; + } + + case ELM_DONE: + { + if( m_pReplaceInfo) + { + // This info structure gets generated when the replaced entry in + // the upper levels is the last entry in the block and we had to + // move entries to a previous block to accommodate it. + // We will therefore need to update the parent block with this + // new information. We need to take care of this before we check + // for any additional data to store. + + if( RC_BAD( rc = restoreReplaceInfo( &pucKey, &uiKeyLen, + &uiChildBlkAddr, &uiCounts))) + { + goto Exit; + } + + bTruncate = bOrigTruncate; + eAction = ELM_REPLACE; + } + else if( bMoreToRemove) + { + eAction = ELM_REMOVE; + + // We need to locate where we should remove the entry. + + if( RC_BAD( rc = findEntry( pucSavKey, uiSavKeyLen, FLM_EXACT))) + { + goto Exit; + } + + } + else if( pucRemainingValue && uiRemainingLen) + { + eAction = ELM_INSERT; + + // We need to locate where we should insert the new entry. + + rc = findEntry( pucSavKey, uiSavKeyLen, FLM_EXCL); + + // We could find this entry. If we get back anything other than + // an NE_SFLM_EOF_HIT or NE_SFLM_OK, then there is a problem. + + if( rc != NE_SFLM_OK && rc != NE_SFLM_EOF_HIT && + rc != NE_SFLM_NOT_FOUND) + { + goto Exit; + } + + pucValue = pucRemainingValue; + uiLen = uiRemainingLen; + pucKey = pucSavKey; + uiKeyLen = uiSavKeyLen; + + // Make certain that the BTE_FIRST_ELEMENT flag is NOT set if + // the first part of the data was stored. + + if( uiOrigDataLen != uiLen) + { + uiFlags = BTE_FLAG_LAST_ELEMENT; + } + else + { + uiFlags = BTE_FLAG_FIRST_ELEMENT | BTE_FLAG_LAST_ELEMENT; + } + } + else + { + bDone = TRUE; + } + + break; + } + + // Should never get this! + + case ELM_BLK_MERGE: + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This method will coordinate inserting an entry into a block. If it + cannot fit it all in, then it may have to break the entry up so that + it spans more than one block. It will also setup for the next level + before returning. +****************************************************************************/ +RCODE F_Btree::insertEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucDataValue = pucValue; + FLMUINT uiDataLen = uiLen; + FLMUINT uiOADataLen = 0; + FLMUINT uiEntrySize = 0; + FLMBOOL bEntriesWereMoved = FALSE; + FLMBOOL bHaveRoom; + FLMBOOL bLastEntry; + const FLMBYTE * pucKey = *ppucKey; + FLMUINT uiKeyLen = *puiKeyLen; + FLMUINT uiChildBlkAddr = *puiChildBlkAddr; + FLMUINT uiCounts = *puiCounts; + F_CachedBlock * pPrevSCache = NULL; + FLMBYTE * pucEntry; + FLMBOOL bDefragBlk = FALSE; + FLMBOOL bBlockSplit; + + if( m_pStack->uiLevel == 0) + { + // We are only safe to do this when we are working on level 0 + // (leaf level) of the Btree. + + *ppucRemainingValue = NULL; + *puiRemainingLen = 0; + } + + if( *peAction == ELM_INSERT_DO) + { + // Adjust the data entry sizes as the data passed in is the + // OA Data Length. + + uiOADataLen = uiLen; + uiDataLen = 4; + } + + // Process until we are done + +StartOver: + + if( RC_BAD( rc = calcNewEntrySize( uiKeyLen, uiDataLen, &uiEntrySize, + &bHaveRoom, &bDefragBlk))) + { + goto Exit; + } + + // Does the entry fit into the block? + + if( bHaveRoom) + { + if( bDefragBlk) + { + // We will have to defragment the block before we can store the data + + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucDataValue, + uiDataLen, uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, + uiEntrySize, &bLastEntry))) + { + goto Exit; + } + + if( (bLastEntry || m_bCounts) && !isRootBlk( m_pStack->pBlkHdr)) + { + // Are we in here because of the counts only? If so, then we + // can update the counts right here, no need to continue. + + if( !bLastEntry) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + + *peAction = ELM_DONE; + } + else + { + // Ensure we are updating with the correct key. + + pucEntry = BtLastEntry( (FLMBYTE *)m_pStack->pBlkHdr); + + *puiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, ppucKey); + + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Do we need counts for the next level? + + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + m_pStack++; + *peAction = ELM_REPLACE; + } + } + else + { + *peAction = ELM_DONE; + } + + goto Exit; + } + + // Can we move entries around at all to make some room? + + if( RC_BAD( rc = moveEntriesToPrevBlk( uiEntrySize, &pPrevSCache, + &bEntriesWereMoved))) + { + goto Exit; + } + + if( bEntriesWereMoved) + { + // Only defragment the block if the heap size is not big enough. + + if( uiEntrySize > m_pStack->pBlkHdr->ui16HeapSize) + { + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + // Store the entry now because we know there is enough room + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucDataValue, + uiDataLen, uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, + uiEntrySize, &bLastEntry))) + { + goto Exit; + } + + // Ordinarily, this would NEVER be the last element in the + // block because we need to adjust the stack to take care of the + // elements we just moved! There is only one condition where we would + // insert as the last entry in the block, and that is when this + // insert is actually a part of a replace operation where the data + // is too large to fit in the block. We had to remove the entry, then + // insert the new one and we are in the upper levels of the + // btree. (i.e. not at the leaf). + // + // VISIT: Should I assert that we are not at the leaf level + // if we get in here? + + if( bLastEntry) + { + // Since we just added an entry to the last position of the + // current block. We will need to preserve the current stack so + // that we can finish updating the parentage later. Should only + // happen as a result of a replace operation where the new entry + // is larger than the existing one while in the upper levels. + + if( RC_BAD( rc = saveReplaceInfo( pucKey, uiKeyLen))) + { + goto Exit; + } + } + + // Need to update the counts of the parents if we are maintining + // counts before we abandon + + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + // This method will release any blocks no longer referenced + // in the stack. Then pull in the previous block information into + // the stack. + + if( RC_BAD( rc = moveStackToPrev( pPrevSCache))) + { + goto Exit; + } + + // If we are maintaining counts, then lets return a count of the + // current number of keys referenced below this point. + + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + flmAssert( !isRootBlk( m_pStack->pBlkHdr)); + + // Return the key to the last entry in the prevous block. + // Recall that we have changed that stack now so that it + // is referencing the changed block (pPrevSCache). + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + *puiKeyLen = getEntryKeyLength( pucEntry, + pPrevSCache->m_pBlkHdr->ui8BlkType, ppucKey); + + // Return the new child block address + + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Set up to fixup the parentage of the previous block on return... + + m_pStack++; + + // Return the new action for the parent block. + + *peAction = ELM_REPLACE; + goto Exit; + } + + // Try moving to the next block... + + if( RC_BAD( rc = moveEntriesToNextBlk( uiEntrySize, &bEntriesWereMoved))) + { + goto Exit; + } + + if( bEntriesWereMoved) + { + // Only defragment the block if the heap size is not big enough. + + if( uiEntrySize > m_pStack->pBlkHdr->ui16HeapSize) + { + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + // Store the entry now because we know there is enough room + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucDataValue, + uiDataLen, uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, + uiEntrySize, &bLastEntry))) + { + goto Exit; + } + + // Return the key to the last entry in the current block. + // Note: If bLastEntry is TRUE, we already know what the key is. + + if( !bLastEntry) + { + // Get the last key from the block. + + pucEntry = BtLastEntry( (FLMBYTE *)m_pStack->pBlkHdr); + + *puiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, ppucKey); + + } + + flmAssert( !isRootBlk( m_pStack->pBlkHdr)); + + // if we are maintaining counts, then lets return a count of the + // current number of keys referenced below this point. + + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + // Return the new child block address + + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Set up to fixup the parentage of the this block on return... + + m_pStack++; + *peAction = ELM_REPLACE; + + goto Exit; + } + + // Before we incur the expense of a block split, see if we can store this + // entry in the previous block. If we can, we will save some space. This + // will only happen if we are trying to insert at the first position in + // this block. We would only ever get into this block of code once for + // each level of the btree. + + if( m_pStack->uiCurOffset == 0 && + m_pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, m_pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain, + NULL, &pPrevSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = moveStackToPrev( pPrevSCache))) + { + goto Exit; + } + + // Increment so we point to one past the last entry. + + m_pStack->uiCurOffset++; + goto StartOver; + } + + // We will have to split the block to make room for this entry. + + if( RC_BAD( rc = splitBlock( *ppucKey, *puiKeyLen, pucDataValue, + uiDataLen, uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, + ppucRemainingValue, puiRemainingLen, &bBlockSplit))) + { + goto Exit; + } + + // Return the new key value. + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + *puiKeyLen = getEntryKeyLength( pucEntry, + ((F_BLK_HDR *)m_pStack->pBlkHdr)->ui8BlkType, ppucKey); + + // Return the child block address and the counts (if needed). + + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Return the counts if we are maintaining them + + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + // The bBlockSplit boolean will only be FALSE if we were involved in a + // ReplaceByInsert operation and the call to split resulted in an empty + // block. Thus we were able to store the new entry. In such cases, + // only the count (if any) need to be updated, not the keys. + + if( bBlockSplit) + { + *peAction = ELM_INSERT; + m_pStack++; + } + else + { + *peAction = ELM_DONE; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to handle the insertion of a single entry into a block. + Assumption: The find method has already been called to locate the + insertion point, so the stack has already been setup. +****************************************************************************/ +RCODE F_Btree::storeEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + FLMUINT uiEntrySize, + FLMBOOL * pbLastEntry) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkType = m_pStack->pSCache->m_pBlkHdr->ui8BlkType; + FLMBYTE * pucInsertAt; + FLMUINT16 * pui16OffsetArray; + FLMUINT uiNumKeys; + FLMUINT uiTmp; + F_BTREE_BLK_HDR * pBlk; + + // Assume this is not the last entry for now. + // We will change it later if needed. + + *pbLastEntry = FALSE; + + // We can go ahead and insert this entry as it is. All checking has been + // made before getting to this point. + + uiEntrySize = calcEntrySize( uiBlkType, uiFlags, + uiKeyLen, uiLen, uiOADataLen); + + // Log this block before making any changes to it. Since the + // pSCache could change, we must update the block header after the call. + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + pBlk = m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlk, 0); + uiNumKeys = getBlkEntryCount( (FLMBYTE *)pBlk); + pucInsertAt = getBlockEnd( pBlk) - uiEntrySize; + pui16OffsetArray = m_pStack->pui16OffsetArray; + + if( RC_BAD( rc = buildAndStoreEntry( uiBlkType, uiFlags, pucKey, uiKeyLen, + pucValue, uiLen, uiOADataLen, uiChildBlkAddr, uiCounts, + pucInsertAt, uiEntrySize, NULL))) + { + goto Exit; + } + + // Now to update the offset in the offset array. This will move all + // entries that sort after the new entry down by one position. + + for( uiTmp = uiNumKeys; uiTmp > m_pStack->uiCurOffset; uiTmp--) + { + bteSetEntryOffset( pui16OffsetArray, uiTmp, + bteGetEntryOffset( pui16OffsetArray, uiTmp - 1)); + } + + bteSetEntryOffset( pui16OffsetArray, m_pStack->uiCurOffset, + (FLMUINT16)(pucInsertAt - (FLMBYTE *)pBlk)); + + // Update the available space and the number of keys. + // Account for the new offset entry too. + + m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail -= + (FLMUINT16)(uiEntrySize + 2); + m_pStack->pBlkHdr->ui16HeapSize -= (FLMUINT16)(uiEntrySize + 2); + m_pStack->pBlkHdr->ui16NumKeys++; + + // Check to see if this was the last entry + + if( m_pStack->uiCurOffset == (FLMUINT)(m_pStack->pBlkHdr->ui16NumKeys - 1)) + { + *pbLastEntry = TRUE; + } + + if( !m_pStack->uiLevel && (uiFlags & BTE_FLAG_FIRST_ELEMENT)) + { + m_ui32PrimaryBlkAddr = m_pStack->ui32BlkAddr; + m_uiCurOffset = m_pStack->uiCurOffset; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This method will coordinate removing an entry from a block. If the + entry spans more than one block, it will set the flag pbMoreToRemove. + It will also setup for the next level before returning. +****************************************************************************/ +RCODE F_Btree::removeEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + FLMBOOL * pbMoreToRemove, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bLastEntry = FALSE; + FLMBYTE * pucEntry; + FLMBOOL bMergedWithPrev = FALSE; + FLMBOOL bMergedWithNext = FALSE; + + if( m_pStack->uiLevel == 0) + { + // We are only safe to do this when we are working on level 0 + // (leaf level) of the Btree. + + *pbMoreToRemove = FALSE; + } + + // Check the current entry to see if it spans more than a single block. + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + // We only need to worry about data spanning more than one block if it is + // at level zero (i.e. leaf block) and the lastElement flag is not set. + + if( (m_pStack->uiLevel == 0) && m_bData && !bteLastElementFlag( pucEntry)) + { + *pbMoreToRemove = TRUE; + } + + // Find out if we are looking at the last entry in the block. + + if( m_pStack->uiCurOffset == (FLMUINT)(m_pStack->pBlkHdr->ui16NumKeys - 1)) + { + bLastEntry = TRUE; + } + + // Now we remove the entry... Will also remove any chained Data Only blocks + + if( RC_BAD( rc = remove( TRUE))) + { + goto Exit; + } + + // If the block is now empty, we will free the block. + + if( !m_pStack->pBlkHdr->ui16NumKeys) + { + FLMBOOL bIsRoot; + + // Test for root block. + + bIsRoot = isRootBlk( m_pStack->pBlkHdr); + + if( RC_BAD( rc = deleteEmptyBlock())) + { + goto Exit; + } + + // Need to remove the parent entry referencing the deleted block. + + if( !bIsRoot) + { + *peAction = ELM_REMOVE; + m_pStack++; + } + else + { + // If we ever get here, it means we have just deleted the root block. + // I have put in the possibility, but typically, deleting the Btree + // is done by calling btDeleteTree. + + *peAction = ELM_DONE; + } + } + else + { + if( ((((F_BLK_HDR *)m_pStack->pBlkHdr)->ui16BlkBytesAvail * 100) / + m_uiBlockSize) >= BT_LOW_WATER_MARK) + { + // We will need to check to see if we can merge two blocks into one to + // conserve space. + + if( RC_BAD( rc = mergeBlocks( bLastEntry, &bMergedWithPrev, + &bMergedWithNext, peAction))) + { + goto Exit; + } + } + + // If the entry that we just removed was the last entry in the block and + // we did not merge any blocks, we will need to prep for an update to the + // parent with a new key. + + if( bLastEntry && !bMergedWithPrev && !bMergedWithNext) + { + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + // Backup to the new "last" entry (remove() does not adjust the offset + // in the stack). + + flmAssert( m_pStack->uiCurOffset > 0); + + m_pStack->uiCurOffset--; + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + *puiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, ppucKey); + + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + *peAction = ELM_REPLACE; + m_pStack++; + } + else + { + // Are we tracking counts? + + if( !bMergedWithPrev && !bMergedWithNext) + { + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + *peAction = ELM_DONE; + } + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to replace an existing entry with a new one. +****************************************************************************/ +RCODE F_Btree::replaceEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction, + FLMBOOL bTruncate) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucDataValue = pucValue; + FLMUINT uiDataLen = uiLen; + FLMUINT uiOADataLen = 0; + FLMBYTE * pucEntry = NULL; + FLMUINT32 ui32OrigDOAddr = 0; + const FLMBYTE * pucData = NULL; + + if( m_pStack->uiLevel == 0) + { + *ppucRemainingValue = NULL; + *puiRemainingLen = 0; + } + + if( *peAction == ELM_REPLACE_DO) + { + // Adjust the data entry sizes as the data passed in + // is the OA Data Length. + + uiOADataLen = uiLen; + uiDataLen = 4; + } + + if( m_pStack->uiLevel == 0 && m_bData) + { + if( m_bOrigInDOBlocks) + { + flmAssert( bTruncate); + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + btGetEntryDataLength( pucEntry, &pucData, NULL, NULL); + ui32OrigDOAddr = bteGetBlkAddr( pucData); + } + } + + // We only have to worry about updating the upper levels of the Btree + // when we are doing a replacement at a non-leaf level or we are maintaining + // counts. Replacements at the leaf level do not require a change in the + // parent block. The only exception is when the old entry spanned to + // another block, but the new one did not. This results in removing the + // excess part of the old entry unless we are not truncating the element. + // Even then, we only update the parent if the excess entry was the only key + // in the block, i.e. the block became empty as a result of the removal. + // All of this would have been handled already by the time we return from + // this call. + + // When bTruncate is FALSE we do not trim back the entry so we don't worry + // about updating the parentage. + + if( RC_BAD( rc = replaceOldEntry( ppucKey, puiKeyLen, pucDataValue, + uiDataLen, uiFlags, uiOADataLen, puiChildBlkAddr, puiCounts, + ppucRemainingValue, puiRemainingLen, peAction, bTruncate))) + { + goto Exit; + } + + // Do we need to free the original DO blocks since they are not + // used in the new entry? + + if( m_bOrigInDOBlocks && !m_bDataOnlyBlock && m_pStack->uiLevel == 0) + { + if( RC_BAD( rc = removeDOBlocks( ui32OrigDOAddr))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to handle replacing a single entry in a block. + ASSUMPTION: The find method has already been called to locate the + insertion point, so the stack has already been setup. +****************************************************************************/ +RCODE F_Btree::replaceOldEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction, + FLMBOOL bTruncate) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOldEntrySize; + FLMBYTE * pucEntry = NULL; + FLMBYTE * pucData = NULL; + FLMUINT uiEntrySize; + FLMBOOL bLastEntry = FALSE; + FLMBOOL bLastElement = TRUE; + FLMBOOL bHaveRoom; + FLMBOOL bDefragBlk; + FLMUINT uiDataLen = 0; + FLMUINT uiOldOADataLen = 0; + FLMBOOL bRemoveOADataAllowance = FALSE; + + uiOldEntrySize = actualEntrySize( + getEntrySize( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset, &pucEntry)); + + if( m_pStack->uiLevel == 0 && m_bData) + { + bLastElement = bteLastElementFlag( pucEntry); + + uiDataLen = btGetEntryDataLength( pucEntry, (const FLMBYTE **)&pucData, + &uiOldOADataLen, NULL); + + // Test to see if we need to worry about the bTruncate flag. + + if( uiDataLen == uiOldOADataLen) + { + if( uiLen > uiDataLen) + { + bTruncate = TRUE; + } + else if( uiLen <= uiDataLen && uiOADataLen == 0) + { + bRemoveOADataAllowance = TRUE; + } + } + else + { + if( uiLen > uiOldOADataLen) + { + bTruncate = TRUE; + } + } + } + + // bTruncate has no meaning if we have no data or we are not at the + // leaf level. + + if( m_pStack->uiLevel != 0 || !m_bData) + { + bTruncate = TRUE; + } + + // The calcNewEntrySize function will tack on 2 bytes for the offset. + // It also adds an extra 4 bytes for the OADataLen, even though it may + // not be needed. We will need to be aware of this here as it may affect + // our decision as to how we will replace the entry. + + if( RC_BAD( rc = calcNewEntrySize( *puiKeyLen, uiLen, &uiEntrySize, + &bHaveRoom, &bDefragBlk))) + { + goto Exit; + } + + if( bRemoveOADataAllowance) + { + uiEntrySize -= 4; + } + + // Since this is a replace operation, we don't need to know about the offset + // as that won't be a factor in what we are doing. 'actualEntrySize' will + // remove those two bytyes from the size. + + uiEntrySize = actualEntrySize( uiEntrySize); + if( uiEntrySize <= uiOldEntrySize) + { + if( !bTruncate) + { + flmAssert( uiLen <= uiDataLen); + f_memcpy( pucData, pucValue, uiLen); + + if( m_pStack->uiCurOffset == + (FLMUINT)(m_pStack->pBlkHdr->ui16NumKeys - 1)) + { + bLastEntry = TRUE; + } + } + else + { + // We can go ahead and replace this entry as it is. All checking + // has been made before getting to this point. + + if( RC_BAD( rc = buildAndStoreEntry( + m_pStack->pSCache->m_pBlkHdr->ui8BlkType, + uiFlags, *ppucKey, *puiKeyLen, pucValue, uiLen, uiOADataLen, + *puiChildBlkAddr, *puiCounts, m_pucTempBlk, m_uiBlockSize, + &uiEntrySize))) + { + goto Exit; + } + + if( RC_BAD( rc = replace( m_pucTempBlk, uiEntrySize, &bLastEntry))) + { + goto Exit; + } + } + + if( !bLastElement && bTruncate) + { + // The element that we replaced actually spans more than one entry. + // We will have to remove the remaining entries. + + if( RC_BAD( rc = removeRemainingEntries( *ppucKey, *puiKeyLen))) + { + goto Exit; + } + } + + if( (bLastEntry || m_bCounts) && !isRootBlk( m_pStack->pBlkHdr) && + (m_pStack->uiLevel != 0)) + { + // Are we in here because of the counts only? If so, then make + // sure we don't change the key in the parent. + + if( !bLastEntry) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + + *peAction = ELM_DONE; + } + else + { + // Return the key to the last entry in the block. + + pucEntry = BtLastEntry( (FLMBYTE *)m_pStack->pBlkHdr); + + *puiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, ppucKey); + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Do we need counts for the next level? + + if( m_bCounts) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + m_pStack++; + *peAction = ELM_REPLACE; + } + } + else + { + *peAction = ELM_DONE; + } + + goto Exit; + } + + // If we do not have a stack setup yet (which can happen if the replace + // is trying to shortcut to the previously known block address and offset), + // then at this point, we must build the stack, since it may be required + // to adjust the upper levels of the btree. + + if( !m_bStackSetup) + { + if( RC_BAD( rc = findEntry( *ppucKey, *puiKeyLen, FLM_EXACT))) + { + goto Exit; + } + } + + // The new entry will not fit into the original entry's space. + // If we remove the entry in the block, will there be enough room + // to put it in? + + if( bTruncate && + m_pStack->pSCache->m_pBlkHdr->ui16BlkBytesAvail + + uiOldEntrySize >= uiEntrySize) + { + // First remove the current entry. Do not delete any DO blocks chained + // to this entry. + + if( RC_BAD( rc = remove( FALSE))) + { + goto Exit; + } + + if( (m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail != + m_pStack->pBlkHdr->ui16HeapSize) && + ((uiEntrySize + 2) > m_pStack->pBlkHdr->ui16HeapSize)) + { + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + // Now insert the new entry. + + if( RC_BAD( rc = storeEntry( *ppucKey, *puiKeyLen, pucValue, uiLen, + uiFlags, uiOADataLen, *puiChildBlkAddr, *puiCounts, uiEntrySize, + &bLastEntry))) + { + goto Exit; + } + + // Check if the original element spanned more than one entry + + if( !bLastElement) + { + // The element that we replaced actually spans more than one entry. + // We will have to remove the remaining entries. + + if( RC_BAD( rc = removeRemainingEntries( *ppucKey, *puiKeyLen))) + { + goto Exit; + } + } + + if( (bLastEntry || m_bCounts) && !isRootBlk( m_pStack->pBlkHdr) && + (m_pStack->uiLevel != 0)) + { + // Are we in here because of the counts only? + + if( !bLastEntry) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + + *peAction = ELM_DONE; + } + else + { + // Set the key to the last entry in the block. + + pucEntry = BtLastEntry( (FLMBYTE *)m_pStack->pBlkHdr); + + *puiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, ppucKey); + *puiChildBlkAddr = m_pStack->ui32BlkAddr; + + // Do we need counts for the next level? + + if( m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType == BT_NON_LEAF_COUNTS) + { + *puiCounts = countKeys( (FLMBYTE *)m_pStack->pBlkHdr); + } + + m_pStack++; + *peAction = ELM_REPLACE; + } + } + else + { + *peAction = ELM_DONE; + } + + goto Exit; + } + + // If the original element does not span multiple entries and we still don't + // have room for the replacement, then we will remove this entry and insert + // the replacement. When the insert happens, it will take care of moving + // things around or splitting the block as needed to get it in. If bTruncate + // is FALSE, and the new entry is larger than the original, we can ignore it. + + if( bLastElement) + { + if( RC_BAD( rc = replaceByInsert( ppucKey, puiKeyLen, + pucValue, uiLen, uiOADataLen, uiFlags, puiChildBlkAddr, + puiCounts, ppucRemainingValue, puiRemainingLen, + peAction))) + { + goto Exit; + } + + goto Exit; + } + + if( bTruncate) + { + if( RC_BAD( rc = replaceMultiples( ppucKey, puiKeyLen, pucValue, + uiLen, uiFlags, puiChildBlkAddr, puiCounts, ppucRemainingValue, + puiRemainingLen, peAction))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = replaceMultiNoTruncate( ppucKey, puiKeyLen, + pucValue, uiLen, uiFlags, puiChildBlkAddr, puiCounts, + ppucRemainingValue, puiRemainingLen, peAction))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This method is called whenever a replacement entry will not fit in + the block, even if we remove the existing entry. It ASSUMES that the + original element does not continue to another entry, either in the + same block or in another block. +****************************************************************************/ +RCODE F_Btree::replaceByInsert( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiDataLen, + FLMUINT uiOADataLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLen = uiDataLen; + + if( *peAction == ELM_REPLACE_DO) + { + uiLen = uiOADataLen; + *peAction = ELM_INSERT_DO; + } + else + { + *peAction = ELM_INSERT; + } + + // At this point, it is clear that this new entry is larger than the + // old entry. We will remove the old entry first. Then we can treat + // this whole operation as an insert rather than as a replace. + + if( RC_BAD( rc = remove( FALSE))) + { + goto Exit; + } + + if( RC_BAD( rc = insertEntry( ppucKey, puiKeyLen, pucDataValue, uiLen, + uiFlags, puiChildBlkAddr, puiCounts, ppucRemainingValue, puiRemainingLen, + peAction))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to replace an entry in a block and update the available + space. This method expects to receive a buffer with an entry already + prepared to be written to the block. +****************************************************************************/ +RCODE F_Btree::replace( + FLMBYTE * pucEntry, + FLMUINT uiEntrySize, + FLMBOOL * pbLastEntry) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucReplaceAt; + FLMUINT uiNumKeys; + FLMBYTE * pBlk; + FLMUINT uiOldEntrySize; + + *pbLastEntry = FALSE; + + // Log this block before making any changes to it. Since the + // pSCache could change, we must update the block header after the call. + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + pBlk = (FLMBYTE *)m_pStack->pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( pBlk, 0); + + uiNumKeys = getBlkEntryCount( pBlk); + + uiOldEntrySize = actualEntrySize( getEntrySize( + pBlk, m_pStack->uiCurOffset)); + + flmAssert( uiOldEntrySize >= uiEntrySize); + + pucReplaceAt = BtEntry( pBlk, m_pStack->uiCurOffset); + + // Let's go ahead and copy the entry into the block now. + + f_memcpy( pucReplaceAt, pucEntry, uiEntrySize); + +#ifdef FLM_DEBUG + // Clean up the empty space (if any) + + if( uiOldEntrySize > uiEntrySize) + { + pucReplaceAt += uiEntrySize; + f_memset( pucReplaceAt, 0, uiOldEntrySize - uiEntrySize); + } +#endif + + // Update the available space. It may not have changed at all if the + // two entries are the same size. The Heap size will not have changed. + // This is because we write the entry into the same location as the + // original. Even though the new entry may be smaller, we start at + // the same location, possibly leaving a hole in the block. + + m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail += + (FLMUINT16)(uiOldEntrySize - uiEntrySize); + + + if( m_pStack->uiCurOffset == (FLMUINT)(m_pStack->pBlkHdr->ui16NumKeys - 1)) + { + *pbLastEntry = TRUE; + } + + // Preserve the block and offset index in case it is wanted on the way out. + + if( !m_pStack->uiLevel && bteFirstElementFlag( pucEntry)) + { + m_ui32PrimaryBlkAddr = m_pStack->ui32BlkAddr; + m_uiCurOffset = m_pStack->uiCurOffset; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to rebuild the stack so that it references the parentage of + the parameter pSCache block. The assumption is that we will begin at + whatever level m_pStack is currently sitting at. Therefore, this + method can be called for any level in the Btree. +****************************************************************************/ +RCODE F_Btree::moveStackToPrev( + F_CachedBlock * pSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkAddr; + F_BTREE_BLK_HDR * pBlkHdr; + F_BTSK * pStack = m_pStack; + F_CachedBlock * pPrevSCache = NULL; + + if( pSCache) + { + if( pStack->pSCache) + { + // Make sure the block we passed in really is the previous + // block in the chain. + + if( pSCache->m_uiBlkAddress != + pStack->pSCache->m_pBlkHdr->ui32PrevBlkInChain) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Cannot be the same block. + + if( pSCache == pStack->pSCache) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Release the current block. We don't need to fetch + // the new block because it was passed in to us. If + // we encounter this situation further up the tree, + // we will have to fetch the block as well. + + ScaReleaseCache( pStack->pSCache, FALSE); + } + + pStack->pSCache = pSCache; + pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + pStack->pBlkHdr = pBlkHdr; + pStack->ui32BlkAddr = ((F_BLK_HDR *)pBlkHdr)->ui32BlkAddr; + pStack->uiCurOffset = pBlkHdr->ui16NumKeys - 1; // Last entry + pStack->uiLevel = pBlkHdr->ui8BlkLevel; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + // Now walk up the stack until done. + + pStack++; + } + + for (;;) + { + // If we don't have this block in the stack, we must first get it. + + if( pStack->pSCache == NULL) + { + // Don't continue if we don't have this level in the stack. + + if( pStack->ui32BlkAddr == 0) + { + break; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pStack->ui32BlkAddr, NULL, &pStack->pSCache))) + { + goto Exit; + } + + pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pStack->pSCache->m_pBlkHdr; + } + + // See if we need to go to the previous block. + + if( pStack->uiCurOffset == 0) + { + // If this is the root block and we are looking at the first + // entry in the block, then we have a problem. + + if( !isRootBlk( pStack->pBlkHdr)) + { + // When the stack is pointing to the first entry, this + // means that we want the target stack to point to the previous + // block in the chain. + + uiBlkAddr = pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain; + flmAssert( uiBlkAddr); + + // Fetch the new block + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiBlkAddr, NULL, &pPrevSCache))) + { + goto Exit; + } + + // Release the old block + + ScaReleaseCache( pStack->pSCache, FALSE); + + pBlkHdr = (F_BTREE_BLK_HDR *)pPrevSCache->m_pBlkHdr; + pStack->pSCache = pPrevSCache; + pPrevSCache = NULL; + pStack->pBlkHdr = pBlkHdr; + pStack->ui32BlkAddr = ((F_BLK_HDR *)pBlkHdr)->ui32BlkAddr; + pStack->uiCurOffset = pBlkHdr->ui16NumKeys - 1; // Last Entry + pStack->uiLevel = pBlkHdr->ui8BlkLevel; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + } + else + { + // We have no previous. + + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + } + else + { + pStack->uiCurOffset--; // Previous Entry + break; + } + + pStack++; + } + +Exit: + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to rebuild the stack so that it references the parentage of + the parameter pSCache block. The assumption is that we will begin at + whatever level m_pStack is currently sitting at. Therefore, this + method can be called for any level in the Btree. +****************************************************************************/ +RCODE F_Btree::moveStackToNext( + F_CachedBlock * pSCache, + FLMBOOL bReleaseCurrent) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkAddr; + F_BTREE_BLK_HDR * pBlkHdr; + F_BTSK * pStack = m_pStack; + F_CachedBlock * pNextSCache = NULL; + + if( pSCache) + { + if( pStack->pSCache) + { + // Make sure the block we passed in really is the next in chain. + + if( pSCache->m_uiBlkAddress != + pStack->pSCache->m_pBlkHdr->ui32NextBlkInChain) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Cannot be the same block. + + if( pSCache == pStack->pSCache) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Release the current block. We don't need to fetch + // the new block because it was passed in to us. If + // we encounter this situation further up the tree, + // we will have to fetch the block as well. + + if( bReleaseCurrent) + { + ScaReleaseCache( pStack->pSCache, FALSE); + } + } + + pStack->pSCache = pSCache; + pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + pStack->pBlkHdr = pBlkHdr; + pStack->ui32BlkAddr = ((F_BLK_HDR *)pBlkHdr)->ui32BlkAddr; + pStack->uiCurOffset = 0; // First entry + pStack->uiLevel = pBlkHdr->ui8BlkLevel; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + + // Now walk up the stack until done. + + pStack++; + } + + for (;;) + { + // If we don't currently have this block, let's get it. + + if( pStack->pSCache == NULL) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pStack->ui32BlkAddr, NULL, &pStack->pSCache))) + { + goto Exit; + } + + pStack->pBlkHdr = (F_BTREE_BLK_HDR *)pStack->pSCache->m_pBlkHdr; + } + + // See if we need to go to the next block. + + if( pStack->uiCurOffset == (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1)) + { + // If this is the root block and we are looking at the last entry in the + // block, then we have a problem. + + if( !isRootBlk( pStack->pBlkHdr)) + { + // When the stack is pointing to the last entry, this + // means that we want the target stack to point the next block in + // the chain. + + uiBlkAddr = pStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain; + flmAssert( uiBlkAddr); + + // Get the next block + + if( RC_BAD( rc = getNextBlock( &pStack->pSCache))) + { + goto Exit; + } + + pBlkHdr = (F_BTREE_BLK_HDR *)pStack->pSCache->m_pBlkHdr; + pStack->pBlkHdr = pBlkHdr; + pStack->ui32BlkAddr = ((F_BLK_HDR *)pBlkHdr)->ui32BlkAddr; + pStack->uiCurOffset = 0; // First Entry + pStack->uiLevel = pBlkHdr->ui8BlkLevel; + pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pBlkHdr, 0); + } + else + { + // We should never have to attempt to get a previous block + // on the root. + + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + else + { + pStack->uiCurOffset++; // Next Entry + break; + } + + pStack++; + } + +Exit: + + if( pNextSCache) + { + ScaReleaseCache( pNextSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to calculate the actual entry size of a new entry +****************************************************************************/ +RCODE F_Btree::calcNewEntrySize( + FLMUINT uiKeyLen, + FLMUINT uiDataLen, + FLMUINT * puiEntrySize, + FLMBOOL * pbHaveRoom, + FLMBOOL * pbDefragBlk) +{ + RCODE rc = NE_SFLM_OK; + + // Calculate the entry size. + + switch( m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType) + { + case BT_LEAF: + { + // This block type is a leaf block, No Data + + *puiEntrySize = BTE_LEAF_OVHD + uiKeyLen; + break; + } + + case BT_LEAF_DATA: + { + // Leaf block with data + + *puiEntrySize = BTE_LEAF_DATA_OVHD + + (uiKeyLen > ONE_BYTE_SIZE ? 2 : 1) + + (uiDataLen > ONE_BYTE_SIZE ? 2 : 1) + + uiKeyLen + uiDataLen; + break; + } + + case BT_NON_LEAF: + { + *puiEntrySize = BTE_NON_LEAF_OVHD + uiKeyLen; + break; + } + + case BT_NON_LEAF_COUNTS: + { + *puiEntrySize = BTE_NON_LEAF_COUNTS_OVHD + uiKeyLen; + break; + } + + default: + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + *puiEntrySize = 0; + goto Exit; + } + } + + // See if we have room in the heap first. If not, maybe we can make + // room by defraging the block. + + if( *puiEntrySize <= m_pStack->pBlkHdr->ui16HeapSize) + { + *pbDefragBlk = FALSE; + *pbHaveRoom = TRUE; + } + else if( *puiEntrySize <= m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + // A defrag of the block is required to make room. We will only defrag + // if we can recover a minimum of 5% of the total block size. + + if( m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail >= m_uiDefragThreshold) + { + *pbHaveRoom = TRUE; + *pbDefragBlk = TRUE; + } + else + { + *pbHaveRoom = FALSE; + *pbDefragBlk = FALSE; + } + } + else + { + *pbHaveRoom = FALSE; + *pbDefragBlk = FALSE; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Function to save the replacement information that we could not store + on the current go round. The replace function will check for the + presence of this structure and deal with it later. +****************************************************************************/ +RCODE F_Btree::saveReplaceInfo( + const FLMBYTE * pucNewKey, + FLMUINT uiNewKeyLen) +{ + RCODE rc = NE_SFLM_OK; + BTREE_REPLACE_STRUCT * pPrev; + F_BTSK * pStack = m_pStack; + const FLMBYTE * pucParentKey; + FLMBYTE * pucEntry; + + if( m_uiReplaceLevels + 1 >= BH_MAX_LEVELS) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + pPrev = m_pReplaceInfo; + m_pReplaceInfo = &m_pReplaceStruct[ m_uiReplaceLevels++]; + m_pReplaceInfo->pPrev = (void *)pPrev; + + // We should not be at the root level already! + + flmAssert( pStack->uiLevel != m_uiStackLevels - 1); + + m_pReplaceInfo->uiParentLevel = pStack->uiLevel+1; + m_pReplaceInfo->uiNewKeyLen = uiNewKeyLen; + m_pReplaceInfo->uiChildBlkAddr = pStack->ui32BlkAddr; + + if( m_bCounts) + { + m_pReplaceInfo->uiCounts = countKeys( (FLMBYTE *)pStack->pBlkHdr); + } + else + { + m_pReplaceInfo->uiCounts = 0; + } + + f_memcpy( &m_pReplaceInfo->pucNewKey[0], pucNewKey, uiNewKeyLen); + + pStack++; + pucEntry = BtEntry( (FLMBYTE *)pStack->pBlkHdr, pStack->uiCurOffset); + + m_pReplaceInfo->uiParentKeyLen = getEntryKeyLength( pucEntry, + pStack->pBlkHdr->stdBlkHdr.ui8BlkType, &pucParentKey); + + f_memcpy( &m_pReplaceInfo->pucParentKey[0], pucParentKey, + m_pReplaceInfo->uiParentKeyLen); + + m_pReplaceInfo->uiParentChildBlkAddr = bteGetBlkAddr( pucEntry); + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to restore the stack to a state where we can finish updating + the parent with the new key information. +****************************************************************************/ +RCODE F_Btree::restoreReplaceInfo( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts) +{ + RCODE rc = NE_SFLM_OK; + RCODE rcTmp = NE_SFLM_OK; + FLMUINT uiLoop; + FLMBYTE * pucEntry; + FLMUINT uiKeyLen; + const FLMBYTE * pucKey; + FLMUINT uiSearchLevel = m_uiSearchLevel; + FLMUINT uiStackLevels = m_uiStackLevels; + + // We will need to redo our stack from the top down to + // make sure we are looking at the correct blocks. + + m_uiSearchLevel = m_uiStackLevels - m_pReplaceInfo->uiParentLevel - 1; + rcTmp = findEntry( m_pReplaceInfo->pucParentKey, + m_pReplaceInfo->uiParentKeyLen, FLM_EXACT); + + m_uiSearchLevel = uiSearchLevel; + + if ((rcTmp != NE_SFLM_OK) && + (rcTmp != NE_SFLM_NOT_FOUND) && + (rcTmp != NE_SFLM_EOF_HIT)) + { + rc = RC_SET( rcTmp); + goto Exit; + } + + // Set the stack pointer to the parent level that we want to replace. + + m_pStack = &m_Stack[ m_pReplaceInfo->uiParentLevel]; + + // There is always the possibility that the key we are searching for + // has a duplicate key ahead of it, as a result of a continuation element. + // We really must replace the entry we were looking at when the information + // was stored, therefore, we will verify that we have the right entry. + + for( ;;) + { + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + uiKeyLen = getEntryKeyLength( pucEntry, + m_pStack->pBlkHdr->stdBlkHdr.ui8BlkType, &pucKey); + + if( uiKeyLen != m_pReplaceInfo->uiParentKeyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( f_memcmp( &m_pReplaceInfo->pucParentKey[0], pucKey, uiKeyLen) == 0) + { + if( bteGetBlkAddr( pucEntry) != m_pReplaceInfo->uiParentChildBlkAddr) + { + // Try moving forward to the next entry ... + + if( RC_BAD( rc = moveStackToNext( NULL))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + else + { + break; + } + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + + // Now return the other important stuff + + *puiChildBlkAddr = m_pReplaceInfo->uiChildBlkAddr; + *puiKeyLen = m_pReplaceInfo->uiNewKeyLen; + *puiCounts = m_pReplaceInfo->uiCounts; + + for( uiLoop = 0; uiLoop < m_uiStackLevels; uiLoop++) + { + m_Stack[ uiLoop].uiKeyLen = m_pReplaceInfo->uiNewKeyLen; + } + + m_uiStackLevels = uiStackLevels; + + // Point to the key + + *ppucKey = &m_pReplaceInfo->pucNewKey[ 0]; + + // Free the current ReplaceInfo Buffer + + m_pReplaceInfo = (BTREE_REPLACE_STRUCT *)m_pReplaceInfo->pPrev; + m_uiReplaceLevels--; + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to set the key to be returned to the caller. +****************************************************************************/ +FINLINE RCODE F_Btree::setReturnKey( + FLMBYTE * pucEntry, + FLMUINT uiBlockType, + FLMBYTE * pucKey, + FLMUINT * puiKeyLen, + FLMUINT uiKeyBufSize) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiKeyLen; + const FLMBYTE * pucKeyRV; + + uiKeyLen = getEntryKeyLength( pucEntry, uiBlockType, &pucKeyRV); + + if( uiKeyLen == 0) + { + // We hit the LEM, hence the EOF error + + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + if( uiKeyLen <= uiKeyBufSize) + { + f_memcpy( pucKey, pucKeyRV, uiKeyLen); + *puiKeyLen = uiKeyLen; + } + else + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to return the data from either the BTREE block or + the DO block. It will update the tracking variables too. + This method assumes that the m_pSCache has already been setup for + the 1st go-round. +****************************************************************************/ +RCODE F_Btree::extractEntryData( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBYTE * pucBuffer, + FLMUINT uiBufSiz, + FLMUINT * puiDataLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucDestPtr = pucBuffer; + FLMUINT32 ui32BlkAddr = 0; + FLMBOOL bNewBlock; + FLMUINT uiDataLen = 0; + + flmAssert( m_pSCache); + + if( puiDataLen) + { + *puiDataLen = 0; + } + +#ifdef FLM_DEBUG + if( pucBuffer) + { + f_memset( pucBuffer, 0, uiBufSiz); + } +#endif + + // Is there anything to read? + + if( m_uiOADataRemaining == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + while( m_uiOADataRemaining && (uiDataLen < uiBufSiz)) + { + if( m_uiDataRemaining <= (uiBufSiz - uiDataLen)) + { + // Let's take what we have left in this block first. + + if( pucDestPtr) + { + f_memcpy( pucDestPtr, m_pucDataPtr, m_uiDataRemaining); + pucDestPtr += m_uiDataRemaining; + } + + uiDataLen += m_uiDataRemaining; + m_uiOADataRemaining -= m_uiDataRemaining; + m_uiDataRemaining = 0; + } + else + { + // Buffer is too small to hold everything in this block. + + if( pucDestPtr) + { + f_memcpy( pucDestPtr, m_pucDataPtr, uiBufSiz - uiDataLen); + pucDestPtr += (uiBufSiz - uiDataLen); + } + + m_pucDataPtr += (uiBufSiz - uiDataLen); + m_uiOADataRemaining -= (uiBufSiz - uiDataLen); + m_uiDataRemaining -= (uiBufSiz - uiDataLen); + uiDataLen += (uiBufSiz - uiDataLen); + } + + // If there is still more overall data remaining, we need to get the + // next DO block or standard block and setup to read it too. + // i.e. More to come, but nothing left in this block. + + if( (m_uiOADataRemaining > 0) && (m_uiDataRemaining == 0)) + { + if (!m_bDataOnlyBlock && + (m_uiCurOffset < + (FLMUINT)(((F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr)->ui16NumKeys - 1))) + { + m_uiCurOffset++; + bNewBlock = FALSE; + } + else + { + // Get the next block address + + ui32BlkAddr = m_pSCache->m_pBlkHdr->ui32NextBlkInChain; + + // Release the current block before we get the next one. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + if( ui32BlkAddr == 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + updateTransInfo( m_pSCache->m_pBlkHdr->ui64TransID, + m_pSCache->m_ui64HighTransID); + + m_ui64LastBlkTransId = m_pSCache->m_pBlkHdr->ui64TransID; + bNewBlock = TRUE; + } + + // If this is a data only block, then we can get the local data size + // from the header. + + if( m_bDataOnlyBlock) + { + flmAssert( m_pSCache->m_pBlkHdr->ui8BlkType == BT_DATA_ONLY); + + m_pucDataPtr = (FLMBYTE *)m_pSCache->m_pBlkHdr + + sizeofDOBlkHdr( m_pSCache->m_pBlkHdr); + + m_uiDataRemaining = m_uiBlockSize - + sizeofDOBlkHdr( m_pSCache->m_pBlkHdr) - + m_pSCache->m_pBlkHdr->ui16BlkBytesAvail; + + m_uiDataLength = m_uiDataRemaining; + m_ui32CurBlkAddr = ui32BlkAddr; + } + else + { + F_BTREE_BLK_HDR * pBlkHdr; + FLMBYTE * pucEntry; + + // In a BTREE block, we MUST ensure that the first entry is a + // continuation of the previous entry in the previous block. + + pBlkHdr = (F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr; + + if( pBlkHdr->ui16NumKeys == 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( bNewBlock) + { + m_uiCurOffset = 0; + } + + // Point to the first entry ... + + pucEntry = BtEntry( (FLMBYTE *)pBlkHdr, m_uiCurOffset); + + if( !checkContinuedEntry( pucKey, uiKeyLen, NULL, pucEntry, + pBlkHdr->stdBlkHdr.ui8BlkType)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + m_uiDataRemaining = btGetEntryDataLength( pucEntry, + &m_pucDataPtr, NULL, NULL); + + m_uiDataLength = m_uiDataRemaining; + + if( bNewBlock) + { + m_ui32CurBlkAddr = ui32BlkAddr; + } + } + + // Update the offset at the begining of the current entry. + + m_uiOffsetAtStart = m_uiOADataLength - m_uiOADataRemaining; + } + } + +Exit: + + if( puiDataLen) + { + *puiDataLen = uiDataLen; + } + + if( m_pSCache) + { + // We must release the SCache block + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to prepare the Btree state for reading. Since several APIs do + the same thing, this has been put into a private method. +****************************************************************************/ +RCODE F_Btree::setupReadState( + F_BLK_HDR * pBlkHdr, + FLMBYTE * pucEntry) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache = NULL; + const FLMBYTE * pucData; + + // Is there any data? Check the block type. + + if( pBlkHdr->ui8BlkType == BT_LEAF_DATA) + { + // How large is the value for this entry? + + m_uiDataLength = btGetEntryDataLength( pucEntry, &pucData, + &m_uiOADataLength, &m_bDataOnlyBlock); + + m_uiPrimaryDataLen = m_uiDataLength; + } + else + { + m_uiDataLength = 0; + m_uiOADataLength = 0; + m_bDataOnlyBlock = FALSE; + } + + // Represents the offset at the beginning entry in the first block. This + // will change as we move through the blocks. + + m_uiOffsetAtStart = 0; + + // Watch the transaction id and the transaction count during streaming + // read operations. If either changes after an initial read, then + // we abort the operation. + + m_ui64CurrTransID = m_pDb->m_ui64CurrTransID; + m_uiBlkChangeCnt = m_pDb->m_uiBlkChangeCnt; + m_ui64LastBlkTransId = pBlkHdr->ui64TransID; + m_ui64PrimaryBlkTransId = pBlkHdr->ui64TransID; + + // Track the overall length progress + + m_uiOADataRemaining = m_uiOADataLength; + + // Track the local entry progress + + m_uiDataRemaining = m_uiDataLength; + + if( m_bDataOnlyBlock) + { + m_ui32DOBlkAddr = bteGetBlkAddr( pucData); + m_ui32CurBlkAddr = m_ui32DOBlkAddr; + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32DOBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + m_ui64LastBlkTransId = pSCache->m_pBlkHdr->ui64TransID; + + // Local amount of data in this block + + m_uiDataRemaining = m_uiBlockSize - + sizeofDOBlkHdr((F_BLK_HDR *)pSCache->m_pBlkHdr) - + pSCache->m_pBlkHdr->ui16BlkBytesAvail; + + // Keep the actual local data size for later. + + m_uiDataLength = m_uiDataRemaining; + + // Adjust for the key at the beginning of the first block. + + if( pSCache->m_pBlkHdr->ui32PrevBlkInChain == 0) + { + FLMBYTE * pucPtr = (FLMBYTE *)pSCache->m_pBlkHdr + + sizeofDOBlkHdr((F_BLK_HDR *)pSCache->m_pBlkHdr); + FLMUINT16 ui16KeyLen = FB2UW( pucPtr); + + m_uiDataLength -= (ui16KeyLen + 2); + m_uiDataRemaining -= (ui16KeyLen + 2); + } + + // Now release the DO Block. We will get it again when we need it. + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to remove extra entries after a replace operation. +****************************************************************************/ +RCODE F_Btree::removeRemainingEntries( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + F_BTREE_BLK_HDR * pBlkHdr; + FLMBOOL bLastElement = FALSE; + FLMBYTE * pucEntry; + FLMBOOL bFirst = TRUE; + + // We should never get to this function when in the upper levels. + + flmAssert( m_pStack->uiLevel == 0); + + // If we do not have a stack setup yet (which can happen if the replace + // is trying to shortcut to the previously known block address and offset), + // then at this point, we must build the stack, since it may be required + // to adjust the upper levels of the btree. + + if( !m_bStackSetup) + { + if( RC_BAD( rc = findEntry( pucKey, uiKeyLen, FLM_EXACT))) + { + goto Exit; + } + } + + while( !bLastElement) + { + // Begin each iteration at the leaf level. + + m_pStack = &m_Stack[ 0]; + + // Advance the stack to the next entry. + + if (bFirst || + m_pStack->uiCurOffset >= (FLMUINT)(m_pStack->pBlkHdr->ui16NumKeys)) + { + if( RC_BAD( rc = moveStackToNext( NULL))) + { + goto Exit; + } + } + + bFirst = FALSE; + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + if( !checkContinuedEntry( pucKey, uiKeyLen, &bLastElement, + pucEntry, getBlkType( (FLMBYTE *)m_pStack->pBlkHdr))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Remove the entry from this block. + + if( RC_BAD( rc = remove( FALSE))) + { + goto Exit; + } + + pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + + // Is the block empty now? If it is, then we will want to remove this + // block and remove the entry in the parent that points to this block. + + if( pBlkHdr->ui16NumKeys == 0) + { + for (;;) + { + flmAssert( !isRootBlk( m_pStack->pBlkHdr)); + + // Remove this block, then update the parent. + + if( RC_BAD( rc = deleteEmptyBlock())) + { + goto Exit; + } + + // Now update the parent blocks + + m_pStack++; + + if( RC_BAD( rc = remove( FALSE))) + { + goto Exit; + } + + // Update the counts if keeping counts. + + if( m_bCounts && !isRootBlk(pBlkHdr)) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + if( m_pStack->pBlkHdr->ui16NumKeys > 0) + { + break; + } + } + + // Rebuild the stack to the beginning after a delete block operation. + + if( RC_BAD( findEntry( pucKey, uiKeyLen, FLM_EXACT))) + { + goto Exit; + } + + bFirst = TRUE; + } + else + { + // Update the counts if keeping counts. + + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to delete an empty block. The block that will be deleted is + the current block pointed to by m_pStack. +****************************************************************************/ +RCODE F_Btree::deleteEmptyBlock( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32PrevBlkAddr; + FLMUINT32 ui32NextBlkAddr; + F_CachedBlock * pSCache = NULL; + + // Get the previous block address so we can back everything up in the stack + + ui32PrevBlkAddr = m_pStack->pBlkHdr->stdBlkHdr.ui32PrevBlkInChain; + ui32NextBlkAddr = m_pStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain; + + // Free the block + + rc = m_pDb->m_pDatabase->blockFree(m_pDb, m_pStack->pSCache); + + m_pStack->pSCache = NULL; + m_pStack->pBlkHdr = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + + // Update the previous block. + + if( ui32PrevBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32PrevBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pSCache))) + { + goto Exit; + } + + pSCache->m_pBlkHdr->ui32NextBlkInChain = ui32NextBlkAddr; + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + + // Update the next block + + if( ui32NextBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pSCache))) + { + goto Exit; + } + + pSCache->m_pBlkHdr->ui32PrevBlkInChain = ui32PrevBlkAddr; + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to remove (free) all data only blocks that are linked to the + data only block whose address is passed in (inclusive). +****************************************************************************/ +RCODE F_Btree::removeDOBlocks( + FLMUINT32 ui32BlkAddr) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32NextBlkAddr; + F_CachedBlock * pSCache = NULL; + + ui32NextBlkAddr = ui32BlkAddr; + + while( ui32NextBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + flmAssert( getBlkType( (FLMBYTE *)pSCache->m_pBlkHdr) == BT_DATA_ONLY); + ui32NextBlkAddr = pSCache->m_pBlkHdr->ui32NextBlkInChain; + + rc = m_pDb->m_pDatabase->blockFree( m_pDb, pSCache); + pSCache = NULL; + + if( RC_BAD( rc)) + { + goto Exit; + } + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method used to replace entries where the original spans multiple + elements and we are NOT to truncate it. To do this, we will attempt + to fill each block until we have stored everything. +****************************************************************************/ +RCODE F_Btree::replaceMultiples( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiLen, + FLMUINT, //uiFlags, + FLMUINT *, //puiChildBlkAddr, + FLMUINT *, //puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bLastElement = FALSE; + FLMUINT uiRemainingData = uiLen; + const FLMBYTE * pucRemainingValue = pucDataValue; + FLMBYTE * pucEntry = NULL; + FLMBYTE * pucData; + FLMUINT uiDataLength; + FLMUINT uiOADataLength = uiLen; + FLMUINT uiOldOADataLength; + FLMUINT uiAmtCopied; + + // Must be at the leaf level! + + flmAssert( m_pStack->uiLevel == 0); + + while( uiRemainingData) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( + (FLMBYTE *)m_pStack->pBlkHdr, 0); + + // Get a pointer to the current entry + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + // Determine the data size for this entry + + uiDataLength = btGetEntryDataLength( pucEntry, (const FLMBYTE **)&pucData, + &uiOldOADataLength, NULL); + + // Now over-write as much of the data as we can + + if( uiRemainingData >= uiDataLength) + { + f_memcpy( pucData, pucRemainingValue, uiDataLength); + + uiAmtCopied = uiDataLength; + pucRemainingValue += uiDataLength; + uiRemainingData -= uiDataLength; + } + else + { + f_memcpy( pucData, pucRemainingValue, uiRemainingData); + uiAmtCopied = uiRemainingData; + pucRemainingValue += uiRemainingData; + uiRemainingData = 0; + } + + // Do we need to adjust the data length? + + if( uiDataLength > uiAmtCopied) + { + FLMBYTE * pucTmp = pucEntry; + + // Skip the flag + + pucTmp++; + + if( bteKeyLenFlag( pucEntry)) + { + pucTmp += 2; + } + else + { + pucTmp++; + } + + if( bteDataLenFlag( pucEntry)) + { + UW2FBA( (FLMUINT16)uiAmtCopied, pucTmp); + pucTmp += 2; + } + else + { + *pucTmp = (FLMBYTE)uiAmtCopied; + pucTmp++; + } + + // We need to adjust the free space in the block too. + + m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail += + (FLMUINT16)(uiDataLength - uiAmtCopied); + + +#ifdef FLM_DEBUG + // Clear the unused portion of the block now. + + pucTmp = pucData + uiAmtCopied; + f_memset( pucTmp, 0, (uiDataLength - uiAmtCopied)); +#endif + } + + // Adjust the OA Data length if needed. We only need to worry about this + // on the first element. No others have it. + + if( bteFirstElementFlag( pucEntry) && uiOADataLength != uiOldOADataLength) + { + FLMBYTE * pucTmp = pucEntry; + + flmAssert( bteOADataLenFlag( pucEntry)); + + pucTmp++; + + if( bteKeyLenFlag( pucEntry)) + { + pucTmp += 2; + } + else + { + pucTmp++; + } + + if( bteDataLenFlag( pucEntry)) + { + pucTmp += 2; + } + else + { + pucTmp++; + } + + UD2FBA( (FLMUINT32)uiOADataLength, pucTmp); + } + + // If we just updated the last member of this entry so break out. + + if( uiRemainingData == 0) + { + break; + } + + // Was this the last element for this entry? + + if( bteLastElementFlag(pucEntry)) + { + FLMBYTE * pucTmp = pucEntry; + + // Turn off the lastElement flag on this entry. + + *pucTmp &= ~BTE_FLAG_LAST_ELEMENT; + + // No more to replace, the rest is going to be new data. + + *ppucRemainingValue = pucRemainingValue; + *puiRemainingLen = uiRemainingData; + break; + } + + // Advance to the next entry, this block or the next... + // The function expects to find the block in m_pSCache, so + // let's put it there for now. + + if( RC_BAD( rc = moveStackToNext( NULL))) + { + goto Exit; + } + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + // Make sure we are still looking at the same key etc. + + if( !checkContinuedEntry( *ppucKey, *puiKeyLen, &bLastElement, + pucEntry, getBlkType( (FLMBYTE *)m_pStack->pBlkHdr))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + + // Are there any more entries to remove? + + if( !bteLastElementFlag( pucEntry) && !uiRemainingData) + { + *pucEntry |= BTE_FLAG_LAST_ELEMENT; + + if( RC_BAD( rc = removeRemainingEntries( *ppucKey, *puiKeyLen))) + { + goto Exit; + } + } + + *peAction = ELM_DONE; + +Exit: + + // Only release the m_pSCache if the use count is greater than 1. It is + // pointed to by the stack also. + + if( m_pSCache && m_pSCache->m_uiUseCount > 1) + { + ScaReleaseCache( m_pSCache, FALSE); + } + + m_pSCache = NULL; + return( rc); +} + +/*************************************************************************** +Desc: Method used to replace entries where the original spans multiple + elements and we are not to truncate it. To do this, we will attempt + to fill each block until we have stored everything. +****************************************************************************/ +RCODE F_Btree::replaceMultiNoTruncate( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiLen, + FLMUINT, //uiFlags, + FLMUINT *, //puiChildBlkAddr, + FLMUINT *, //puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bLastElement = FALSE; + FLMUINT uiRemainingData = uiLen; + const FLMBYTE * pucRemainingValue = pucDataValue; + FLMBYTE * pucEntry; + FLMBYTE * pucData; + FLMUINT uiDataLength; + + // Must be at the leaf level + + flmAssert( m_pStack->uiLevel == 0); + + while( uiRemainingData) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( + (FLMBYTE *)m_pStack->pBlkHdr, 0); + + // Get a pointer to the current entry + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + // Determine the data size for this entry + + uiDataLength = btGetEntryDataLength( pucEntry, + (const FLMBYTE **)&pucData, NULL, NULL); + + // Now over-write as much of the data as we can. + + if( uiRemainingData > uiDataLength) + { + f_memcpy( pucData, pucRemainingValue, uiDataLength); + pucRemainingValue += uiDataLength; + uiRemainingData -= uiDataLength; + } + else + { + f_memcpy( pucData, pucRemainingValue, uiRemainingData); + pucRemainingValue += uiRemainingData; + uiRemainingData = 0; + } + + // We just updated the last member of this entry so break out. + + if( uiRemainingData == 0) + { + break; + } + + // Was this the last element for this entry? + + if( bteLastElementFlag( pucEntry)) + { + // No more to replace, the rest is going to be new data. + + *ppucRemainingValue = pucRemainingValue; + *puiRemainingLen = uiRemainingData; + break; + } + + // Advance to the next entry, this block or the next... + // The function expects to find the block in m_pSCache, so + // let's put it there f or now. + + if( RC_BAD( rc = moveStackToNext( NULL))) + { + goto Exit; + } + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, + m_pStack->uiCurOffset); + + // Make sure we are still looking at the same key etc. + + if( !checkContinuedEntry( *ppucKey, *puiKeyLen, &bLastElement, + pucEntry, getBlkType( (FLMBYTE *)m_pStack->pBlkHdr))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + + *peAction = ELM_DONE; + +Exit: + + // Only release the m_pSCache if the use count is greater than 1. It is + // pointed to by the stack also. + + if( m_pSCache && m_pSCache->m_uiUseCount > 1) + { + ScaReleaseCache( m_pSCache, FALSE); + } + + m_pSCache = NULL; + return( rc); +} + +/*************************************************************************** +Desc: Private method to retrieve the next block in the chain relative to + the block that is passed in. The block that is passed in is always + released prior to getting the next block. +****************************************************************************/ +FINLINE RCODE F_Btree::getNextBlock( + F_CachedBlock ** ppSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32BlkAddr; + + ui32BlkAddr = (*ppSCache)->m_pBlkHdr->ui32NextBlkInChain; + + ScaReleaseCache( *ppSCache, FALSE); + *ppSCache = NULL; + + if( ui32BlkAddr == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddr, NULL, ppSCache))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Private method to retrieve the previous block in the chain relative to + the block that is passed in. The block that is passed in is always + released prior to getting the previous block. +****************************************************************************/ +FINLINE RCODE F_Btree::getPrevBlock( + F_CachedBlock ** ppSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32BlkAddr; + + ui32BlkAddr = (*ppSCache)->m_pBlkHdr->ui32PrevBlkInChain; + + ScaReleaseCache( *ppSCache, FALSE); + *ppSCache = NULL; + + if( ui32BlkAddr == 0) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddr, NULL, ppSCache))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Private method to verify that the entry we are looking at in the stack + is a continuation entry. The key must match the key we pass in and + the entry must be marked as a continuation, i.e. not the first + element. +****************************************************************************/ +FLMBOOL F_Btree::checkContinuedEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL * pbLastElement, + FLMBYTE * pucEntry, + FLMUINT uiBlkType) +{ + FLMBOOL bOk = TRUE; + FLMUINT uiBlkKeyLen; + const FLMBYTE * pucBlkKey; + + if( pbLastElement) + { + *pbLastElement = bteLastElementFlag( pucEntry); + } + + uiBlkKeyLen = getEntryKeyLength( pucEntry, uiBlkType, &pucBlkKey); + + // Must be the same size key! + + if( uiKeyLen != uiBlkKeyLen) + { + bOk = FALSE; + goto Exit; + } + + // Must be identical! + + if( f_memcmp( pucKey, pucBlkKey, uiKeyLen) != 0) + { + bOk = FALSE; + goto Exit; + } + + // Must not be the first element! + + if( bteFirstElementFlag( pucEntry)) + { + bOk = FALSE; + goto Exit; + } + +Exit: + + return( bOk); +} + +/*************************************************************************** +Desc: Private method to assend the tree, updating the counts for a + particular block. This method allows us to update the counts quickly + without the need to continually loop, replacing existing keys with + new counts. +****************************************************************************/ +RCODE F_Btree::updateCounts( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLevel; + + for( uiLevel = m_pStack->uiLevel; + uiLevel < m_uiStackLevels - 1; + uiLevel++) + { + if( RC_BAD( rc = updateParentCounts( m_Stack[ uiLevel].pSCache, + &m_Stack[ uiLevel + 1].pSCache, m_Stack[ uiLevel + 1].uiCurOffset))) + { + goto Exit; + } + + m_Stack[ uiLevel + 1].pBlkHdr = + (F_BTREE_BLK_HDR *)m_Stack[ uiLevel + 1].pSCache->m_pBlkHdr; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Private method to store part of an entry in a block. This method will + determine how much of the data can be stored in the block. The amount + that does not get stored will be returned in ppucRemainingValue and + puiRemainingLen. +****************************************************************************/ +RCODE F_Btree::storePartialEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + FLMBOOL bNewBlock) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNewDataLen; + FLMUINT uiOADataLen = 0; + FLMUINT uiEntrySize; + FLMBOOL bHaveRoom; + FLMBOOL bDefragBlk; + FLMBOOL bLastEntry; + + if( RC_BAD( rc = calcOptimalDataLength( uiKeyLen, + uiLen, m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail, + &uiNewDataLen))) + { + goto Exit; + } + + if( uiNewDataLen < uiLen) + { + // Turn off the last element flag. + + uiFlags &= ~BTE_FLAG_LAST_ELEMENT; + + if( uiFlags & BTE_FLAG_FIRST_ELEMENT) + { + // Store the overall data length from this point forward. + + uiOADataLen = uiLen; + } + } + + if( RC_BAD( rc = calcNewEntrySize( uiKeyLen, uiNewDataLen, &uiEntrySize, + &bHaveRoom, &bDefragBlk))) + { + goto Exit; + } + + // We will defragment the block first if the avail and heap + // are not the same size. + + if( m_pStack->pBlkHdr->ui16HeapSize != + m_pStack->pBlkHdr->stdBlkHdr.ui16BlkBytesAvail) + { + if( RC_BAD( rc = defragmentBlock( &m_pStack->pSCache))) + { + goto Exit; + } + } + + if( RC_BAD( rc = storeEntry( pucKey, uiKeyLen, pucValue, uiNewDataLen, + uiFlags, uiOADataLen, uiChildBlkAddr, uiCounts, + uiEntrySize, &bLastEntry))) + { + goto Exit; + } + + // If this block has a parent block, and the btree is maintaining counts + // we will want to update the counts on the parent block. + + if( !isRootBlk( m_pStack->pBlkHdr) && m_bCounts && !bNewBlock) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + if( uiNewDataLen < uiLen) + { + // Save the portion of the data that was not written. + // It will be written later. + + *ppucRemainingValue = pucValue + uiNewDataLen; + *puiRemainingLen = uiLen - uiNewDataLen; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Private meethod for checking the down links in the btree to make sure + they are not corrupt. +****************************************************************************/ +RCODE F_Btree::checkDownLinks( void) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pParentSCache = NULL; + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, m_pLFile->uiRootBlk, NULL, &pParentSCache))) + { + goto Exit; + } + + if( (pParentSCache->m_pBlkHdr->ui8BlkType == BT_NON_LEAF) || + (pParentSCache->m_pBlkHdr->ui8BlkType == BT_NON_LEAF_COUNTS)) + { + if( RC_BAD( rc = verifyChildLinks( pParentSCache))) + { + goto Exit; + } + } + +Exit: + + if( pParentSCache) + { + ScaReleaseCache( pParentSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Private method (recursive) that checks the child links in the given + blocks to ensure they are correct. +****************************************************************************/ +RCODE F_Btree::verifyChildLinks( + F_CachedBlock * pParentSCache) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumKeys; + F_CachedBlock * pChildSCache = NULL; + F_BTREE_BLK_HDR * pParentBlkHdr; + F_BTREE_BLK_HDR * pChildBlkHdr; + FLMUINT uiCurOffset; + FLMBYTE * pucEntry; + FLMUINT32 ui32BlkAddr; + const FLMBYTE * pucParentKey; + FLMBYTE * pucChildEntry; + const FLMBYTE * pucChildKey; + FLMUINT uiParentKeyLen; + FLMUINT uiChildKeyLen; + + pParentBlkHdr = (F_BTREE_BLK_HDR *)pParentSCache->m_pBlkHdr; + uiNumKeys = pParentBlkHdr->ui16NumKeys; + + for( uiCurOffset = 0; uiCurOffset < uiNumKeys; uiCurOffset++) + { + pucEntry = BtEntry( (FLMBYTE *)pParentBlkHdr, uiCurOffset); + + // Non-leaf nodes have children. + + ui32BlkAddr = bteGetBlkAddr( pucEntry); + flmAssert( ui32BlkAddr); + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, ui32BlkAddr, NULL, &pChildSCache))) + { + goto Exit; + } + + pChildBlkHdr = (F_BTREE_BLK_HDR *)pChildSCache->m_pBlkHdr; + + // Get key from the parent entry and compare it to the + // last key in the child block. + + uiParentKeyLen = getEntryKeyLength( + pucEntry, pParentBlkHdr->stdBlkHdr.ui8BlkType, &pucParentKey); + + // Get the last entry in the child block. + + pucChildEntry = BtLastEntry( (FLMBYTE *)pChildBlkHdr); + + uiChildKeyLen = getEntryKeyLength( + pucChildEntry, pChildBlkHdr->stdBlkHdr.ui8BlkType, &pucChildKey); + + if( uiParentKeyLen != uiChildKeyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( f_memcmp( pucParentKey, pucChildKey, uiParentKeyLen) != 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( (pChildBlkHdr->stdBlkHdr.ui8BlkType == BT_NON_LEAF) || + (pChildBlkHdr->stdBlkHdr.ui8BlkType == BT_NON_LEAF_COUNTS)) + { + if( RC_BAD( rc = verifyChildLinks( pChildSCache))) + { + goto Exit; + } + } + + ScaReleaseCache( pChildSCache, FALSE); + pChildSCache = NULL; + } + +Exit: + + if( pChildSCache) + { + ScaReleaseCache( pChildSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: This is a private method that computes the number of entries (keys) + and the number of blocks between two points in the Btree. +****************************************************************************/ +RCODE F_Btree::computeCounts( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT64 * pui64BlockCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64TotalKeys = 0; + FLMUINT64 ui64EstKeyCount = 0; + FLMUINT64 ui64TotalBlocksBetween = 0; + FLMUINT64 ui64EstBlocksBetween = 0; + FLMUINT uiBlkKeyCount; + + ui64TotalBlocksBetween = 0; + *pbTotalsEstimated = FALSE; + + // The stack that we are looking at does not hold the blocks + // we need. We first need to restore the blocks as needed. + + if( RC_BAD( rc = getCacheBlocks( pFromStack, pUntilStack))) + { + goto Exit; + } + + // Are the from and until positions in the same block? + + if( pFromStack->ui32BlkAddr == pUntilStack->ui32BlkAddr) + { + rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pUntilStack->uiCurOffset, &uiBlkKeyCount, NULL); + ui64TotalKeys = (FLMUINT64)uiBlkKeyCount; + goto Exit; + } + + // Are we maintaining counts on this Btree? If so, we can just + // use the counts we have... The blocks count may still be estimated. + + if( m_bCounts) + { + return( getStoredCounts( pFromStack, pUntilStack, pui64BlockCount, + pui64KeyCount, pbTotalsEstimated, uiAvgBlkFullness)); + } + + // Since we are not keeping counts on this Btree, we will need to + // count them and possibly estimate them. + + // Gather the counts in the from and until leaf blocks. + + if( RC_BAD( rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pFromStack->pBlkHdr->ui16NumKeys - 1, &uiBlkKeyCount, NULL))) + { + goto Exit; + } + ui64TotalKeys += (FLMUINT64)uiBlkKeyCount; + + if( RC_BAD( rc = blockCounts( pUntilStack, 0, + pUntilStack->uiCurOffset, &uiBlkKeyCount, NULL))) + { + goto Exit; + } + + ui64TotalKeys += (FLMUINT64)uiBlkKeyCount; + + // Do the obvious check to see if the blocks are neighbors. If they + // are, we are done. + + if( pFromStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == + pUntilStack->ui32BlkAddr) + { + goto Exit; + } + + // Estimate the number of elements in the parent block. + + *pbTotalsEstimated = TRUE; + + ui64EstKeyCount = (FLMUINT64)getAvgKeyCount( pFromStack, pUntilStack, uiAvgBlkFullness); + ui64EstBlocksBetween = 1; + + for (;;) + { + FLMUINT uiBlkElementCount; + FLMUINT uiTempBlkElementCount; + FLMUINT64 ui64EstElementCount; + + // Go up a b-tree level and check out how far apart the elements are. + + pFromStack++; + pUntilStack++; + + if( RC_BAD( rc = getCacheBlocks( pFromStack, pUntilStack))) + { + goto Exit; + } + + // Share the same block? + + if( pFromStack->ui32BlkAddr == pUntilStack->ui32BlkAddr) + { + if( RC_BAD( rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pUntilStack->uiCurOffset, NULL, &uiBlkElementCount))) + { + goto Exit; + } + + // Don't count the pFromStack or the pUntilStack current elements. + + uiBlkElementCount -= 2; + + ui64TotalBlocksBetween += ui64EstBlocksBetween * + (FLMUINT64)(uiBlkElementCount > 0 ? uiBlkElementCount : 1); + ui64TotalKeys += ui64EstKeyCount * + (FLMUINT64)(uiBlkElementCount > 0 ? uiBlkElementCount : 1); + goto Exit; + } + + // Gather the counts in the from and until non-leaf blocks. + + if( RC_BAD( rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pFromStack->pBlkHdr->ui16NumKeys - 1, NULL, &uiBlkElementCount))) + { + goto Exit; + } + + // Don't count the first element. + + uiBlkElementCount--; + + if( RC_BAD( rc = blockCounts( pUntilStack, 0, + pUntilStack->uiCurOffset, NULL, &uiTempBlkElementCount))) + { + goto Exit; + } + + uiBlkElementCount += (uiTempBlkElementCount - 1); + + ui64TotalBlocksBetween += ui64EstBlocksBetween * (FLMUINT64)uiBlkElementCount; + ui64TotalKeys += ui64EstKeyCount * (FLMUINT64)uiBlkElementCount; + + // Do the obvious check to see if the blocks are neighbors. + + if( (FLMUINT)pFromStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == + pUntilStack->ui32BlkAddr) + { + goto Exit; + } + + // Recompute the estimated element count on every b-tree level + // because the compression is better the lower in the b-tree we go. + + ui64EstElementCount = (FLMUINT64)getAvgKeyCount( + pFromStack, pUntilStack, uiAvgBlkFullness); + + // Adjust the estimated key/ref count to be the counts from a complete + // (not partial) block starting at this level going to the leaf. + + ui64EstKeyCount *= ui64EstElementCount; + ui64EstBlocksBetween *= ui64EstElementCount; + } + +Exit: + + if( pui64KeyCount) + { + *pui64KeyCount = ui64TotalKeys; + } + + if( pui64BlockCount) + { + *pui64BlockCount = ui64TotalBlocksBetween; + } + + return( rc); +} + +/*************************************************************************** +Desc: Private method to count the number of unique keys between two points. + The count returned is inclusive of the first and last offsets. +****************************************************************************/ +RCODE F_Btree::blockCounts( + F_BTSK * pStack, + FLMUINT uiFirstOffset, + FLMUINT uiLastOffset, + FLMUINT * puiKeyCount, + FLMUINT * puiElementCount) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiKeyCount; + FLMUINT uiElementCount; + FLMBYTE * pucBlk; + FLMBYTE * pucEntry; + + // Debug checks. + + flmAssert( uiFirstOffset <= uiLastOffset); + flmAssert( uiLastOffset <= (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1)); + + uiKeyCount = uiElementCount = 0; + pucBlk = (FLMBYTE *)pStack->pBlkHdr; + + // Loop gathering the statistics. + + while( uiFirstOffset <= uiLastOffset) + { + uiElementCount++; + + if( puiKeyCount) + { + pucEntry = BtEntry( pucBlk, uiFirstOffset); + + // We only have to worry about first key elements when we are at the + // leaf level and we are keeping data at that level. + + if( pStack->uiLevel == 0 && m_bData) + { + if( bteFirstElementFlag( pucEntry)) + { + uiKeyCount++; + } + } + else + { + uiKeyCount++; + } + } + + // Next element. + + if( uiFirstOffset == (FLMUINT)(pStack->pBlkHdr->ui16NumKeys - 1)) + { + break; + } + else + { + uiFirstOffset++; + } + } + + if( puiKeyCount) + { + *puiKeyCount = uiKeyCount; + } + + if( puiElementCount) + { + *puiElementCount = uiElementCount; + } + + return( rc); +} + +/*************************************************************************** +Desc: Similar to computeCounts, except we use the stored counts. +****************************************************************************/ +RCODE F_Btree::getStoredCounts( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT64 * pui64BlockCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64OmittedKeys; + FLMUINT64 ui64TotalKeys; + FLMUINT64 ui64EstBlocksBetween; + FLMUINT64 ui64TotalBlocksBetween; + + *pbTotalsEstimated = FALSE; + *pui64BlockCount = 0; + ui64TotalBlocksBetween = 0; + + // Are these blocks adjacent? + + if( pFromStack->pBlkHdr->stdBlkHdr.ui32NextBlkInChain == + pUntilStack->ui32BlkAddr) + { + *pui64KeyCount = (pFromStack->pBlkHdr->ui16NumKeys - + pFromStack->uiCurOffset) + pUntilStack->uiCurOffset + 1; + goto Exit; + } + + *pbTotalsEstimated = TRUE; + + // How many keys are excluded in the From and Until blocks? + + ui64OmittedKeys = (FLMUINT64)countRangeOfKeys( + pFromStack, 0, pFromStack->uiCurOffset) - 1; + + ui64OmittedKeys += (FLMUINT64)countRangeOfKeys( + pUntilStack, pUntilStack->uiCurOffset, + pUntilStack->pBlkHdr->ui16NumKeys - 1) - 1; + + ui64TotalKeys = 0; + ui64EstBlocksBetween = 1; + + for( ;;) + { + FLMUINT uiBlkElementCount; + FLMUINT uiBlkTempElementCount; + FLMUINT64 ui64EstElementCount; + + // Go up a b-tree level and check out how far apart the elements are. + + pFromStack++; + pUntilStack++; + + if( RC_BAD( rc = getCacheBlocks( pFromStack, pUntilStack))) + { + goto Exit; + } + + // Share the same block? We can get the actual key count now. + + if( pFromStack->ui32BlkAddr == pUntilStack->ui32BlkAddr) + { + + if( RC_BAD( rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pUntilStack->uiCurOffset, NULL, &uiBlkElementCount))) + { + goto Exit; + } + + // Don't count the pFromStack current element. + + uiBlkElementCount -= 2; + ui64TotalBlocksBetween += ui64EstBlocksBetween * + (FLMUINT64)(uiBlkElementCount > 0 ? uiBlkElementCount : 1); + + // Add one to the last offset to include the last entry in the count. + + ui64TotalKeys = (FLMUINT64)countRangeOfKeys( + pFromStack, pFromStack->uiCurOffset, pUntilStack->uiCurOffset); + + *pui64KeyCount = ui64TotalKeys - ui64OmittedKeys; + *pui64BlockCount = ui64TotalBlocksBetween; + goto Exit; + } + + // How many to exclude from the From & Until blocks. + + if( pFromStack->uiCurOffset) + { + ui64OmittedKeys += (FLMUINT64)countRangeOfKeys( + pFromStack, 0, pFromStack->uiCurOffset - 1); + } + + ui64OmittedKeys += (FLMUINT64)countRangeOfKeys( + pUntilStack, pUntilStack->uiCurOffset + 1, + pUntilStack->pBlkHdr->ui16NumKeys - 1); + + // Gather the counts in the from and until non-leaf blocks. + + if( RC_BAD( rc = blockCounts( pFromStack, pFromStack->uiCurOffset, + pFromStack->pBlkHdr->ui16NumKeys - 1, NULL, &uiBlkElementCount))) + { + goto Exit; + } + + // Don't count the first element. + + uiBlkElementCount--; + + if( RC_BAD( rc = blockCounts( pUntilStack, 0, + pUntilStack->uiCurOffset, NULL, &uiBlkTempElementCount))) + { + goto Exit; + } + + uiBlkElementCount += (uiBlkTempElementCount - 1); + ui64TotalBlocksBetween += ui64EstBlocksBetween * (FLMUINT64)uiBlkElementCount; + + // We are not going to check if these blocks are neighbors here because + // we want to find the common parent. That will tell us what the actual + // counts are at the leaf level. + + // Recompute the estimated element count on every b-tree level + // because the compression is better the lower in the b-tree we go. + + ui64EstElementCount = (FLMUINT64)getAvgKeyCount( + pFromStack, pUntilStack, uiAvgBlkFullness); + ui64EstBlocksBetween *= ui64EstElementCount; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Retrieve the blocks identified in the two stack entries. Used in + computing counts (btComputeCounts etc.) +****************************************************************************/ +RCODE F_Btree::getCacheBlocks( + F_BTSK * pStack1, + F_BTSK * pStack2) +{ + RCODE rc = NE_SFLM_OK; + + // If these blocks are at the root level, we must ensure that we retrieve + // the root block. The root block can potentially change address, so + // we wil reset it here to be sure. + + if( pStack1->uiLevel == m_uiRootLevel) + { + pStack1->ui32BlkAddr = (FLMUINT32)m_pLFile->uiRootBlk; + } + + if( pStack2->uiLevel == m_uiRootLevel) + { + pStack2->ui32BlkAddr = (FLMUINT32)m_pLFile->uiRootBlk; + } + + if( RC_BAD( m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, pStack1->ui32BlkAddr, NULL, &pStack1->pSCache))) + { + goto Exit; + } + + pStack1->pBlkHdr = (F_BTREE_BLK_HDR *)pStack1->pSCache->m_pBlkHdr; + + if( RC_BAD( m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, pStack2->ui32BlkAddr, NULL, &pStack2->pSCache))) + { + goto Exit; + } + + pStack2->pBlkHdr = (F_BTREE_BLK_HDR *)pStack2->pSCache->m_pBlkHdr; + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to tally the counts in a block between (inclusive) the + uiFromOffset & uiUntilOffset parameters. +****************************************************************************/ +FLMUINT F_Btree::countRangeOfKeys( + F_BTSK * pFromStack, + FLMUINT uiFromOffset, + FLMUINT uiUntilOffset) +{ + FLMUINT uiCount = 0; + FLMBYTE * pucBlk; + FLMUINT uiLoop = uiFromOffset; + FLMBYTE * pucEntry; + FLMUINT uiBlkType; + + pucBlk = (FLMBYTE *)pFromStack->pBlkHdr; + uiBlkType = getBlkType( pucBlk); + + if( uiBlkType == BT_NON_LEAF_COUNTS) + { + while( uiLoop < uiUntilOffset) + { + pucEntry = BtEntry( pucBlk, uiLoop); + pucEntry += 4; + uiCount += FB2UD( pucEntry); + uiLoop++; + } + } + else + { + uiCount = uiUntilOffset; + } + + return( uiCount); +} + +/*************************************************************************** +Desc: Method to estimate the average number of keys, based on the anticipated + average block usage (passed in) and the actual block usage. +****************************************************************************/ +FINLINE FLMUINT F_Btree::getAvgKeyCount( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT uiAvgBlkFullness) +{ + FLMUINT uiFromUsed; + FLMUINT uiUntilUsed; + FLMUINT uiTotalUsed; + FLMUINT uiFromKeys; + FLMUINT uiUntilKeys; + FLMUINT uiTotalKeys; + + uiFromUsed = m_uiBlockSize - + ((F_BLK_HDR *)pFromStack->pBlkHdr)->ui16BlkBytesAvail; + + uiUntilUsed = m_uiBlockSize - + ((F_BLK_HDR *)pUntilStack->pBlkHdr)->ui16BlkBytesAvail; + + uiTotalUsed = uiFromUsed + uiUntilUsed; + + uiFromKeys = pFromStack->pBlkHdr->ui16NumKeys; + uiUntilKeys = pUntilStack->pBlkHdr->ui16NumKeys; + uiTotalKeys = uiFromKeys + uiUntilKeys; + + return( (uiAvgBlkFullness * uiTotalKeys) / uiTotalUsed); +} + +/*************************************************************************** +Desc: Method to test if two blocks can be merged together to make a single + block. This is done only after a remove operation and is intended to + try to consolidate space as much as possible. If we can consolidate + two blocks, we will do it, then update the tree. +****************************************************************************/ +RCODE F_Btree::mergeBlocks( + FLMBOOL bLastEntry, + FLMBOOL * pbMergedWithPrev, + FLMBOOL * pbMergedWithNext, + F_ELM_UPD_ACTION * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32PrevBlkAddr; + F_CachedBlock * pPrevSCache = NULL; + FLMUINT32 ui32NextBlkAddr; + F_CachedBlock * pNextSCache = NULL; + + *pbMergedWithPrev = FALSE; + *pbMergedWithNext = FALSE; + + // Our first check is to see if we can merge the current block with its + // previous block. + + ui32PrevBlkAddr = m_pStack->pSCache->m_pBlkHdr->ui32PrevBlkInChain; + if( ui32PrevBlkAddr) + { + // Get the block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, ui32PrevBlkAddr, NULL, &pPrevSCache))) + { + goto Exit; + } + + // Is there room to merge? + + if( (FLMUINT)(pPrevSCache->m_pBlkHdr->ui16BlkBytesAvail + + m_pStack->pSCache->m_pBlkHdr->ui16BlkBytesAvail) >= + (FLMUINT)(m_uiBlockSize - sizeofBTreeBlkHdr( + (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr))) + { + // Looks like we can merge these two. We will move the content + // of the previous block into this one. + + if( RC_BAD( rc = merge( &pPrevSCache, &m_pStack->pSCache))) + { + goto Exit; + } + + // Save the changed block header address + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + + // Update the counts for the current block before releasing it. + + if( m_bCounts) + { + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + } + + if( bLastEntry) + { + // Need to save the replace information for the last entry in + // the block before we move to the previous block. This will + // allow us to do the replace later. + + FLMBYTE * pucEntry; + const FLMBYTE * pucKey; + FLMUINT uiKeyLen; + + pucEntry = BtEntry( + (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->pBlkHdr->ui16NumKeys - 1); + + uiKeyLen = getEntryKeyLength( + pucEntry, getBlkType( (FLMBYTE *)m_pStack->pBlkHdr), &pucKey); + + if( RC_BAD( rc = saveReplaceInfo( pucKey, uiKeyLen))) + { + goto Exit; + } + } + + // Move the stack to the previous entry + + if( RC_BAD( rc = moveStackToPrev( pPrevSCache))) + { + goto Exit; + } + pPrevSCache = NULL; + + flmAssert( m_pStack->pBlkHdr->ui16NumKeys == 0); + + // Free the empty block. + + if( RC_BAD( rc = deleteEmptyBlock())) + { + goto Exit; + } + + // Now we want to remove the parent entry for the block that was + // freed. + + m_pStack++; + *peAction = ELM_REMOVE; + *pbMergedWithPrev = TRUE; + + goto Exit; + } + else + { + // No room here so release the block. + + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + } + + // Can we merge with the next block? + + ui32NextBlkAddr = m_pStack->pSCache->m_pBlkHdr->ui32NextBlkInChain; + if( ui32NextBlkAddr) + { + // Get the block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, ui32NextBlkAddr, NULL, &pNextSCache))) + { + goto Exit; + } + + // Is there room to merge? + + if( (FLMUINT)(pNextSCache->m_pBlkHdr->ui16BlkBytesAvail + + m_pStack->pSCache->m_pBlkHdr->ui16BlkBytesAvail) >= + (FLMUINT)(m_uiBlockSize - sizeofBTreeBlkHdr( + (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr))) + { + // Looks like we can merge these two. + + if( RC_BAD( rc = merge( &m_pStack->pSCache, &pNextSCache))) + { + goto Exit; + } + + // Save the changed block header address. + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + + // Update the counts for the current block and the next block. + + if( m_bCounts) + { + pPrevSCache = m_pStack->pSCache; + + // Need to move the stack to the next entry. Don't let the current + // block get released because we still need it. + + if( RC_BAD( rc = moveStackToNext( pNextSCache, FALSE))) + { + goto Exit; + } + pNextSCache = NULL; + + if( RC_BAD( rc = updateCounts())) + { + goto Exit; + } + + // Move back to the original stack again. It's okay to release the + // now current block. + + if( RC_BAD( rc = moveStackToPrev( pPrevSCache))) + { + goto Exit; + } + + pPrevSCache = NULL; + } + + flmAssert( m_pStack->pBlkHdr->ui16NumKeys == 0); + + // Free the empty block. + + if( RC_BAD( rc = deleteEmptyBlock())) + { + goto Exit; + } + + // Now we want to remove the parent entry for the block that was freed. + + m_pStack++; + *peAction = ELM_REMOVE; + *pbMergedWithNext = TRUE; + goto Exit; + } + else + { + // No room here so release the block. + + ScaReleaseCache( pNextSCache, FALSE); + pNextSCache = NULL; + } + } + +Exit: + + if( *pbMergedWithPrev || *pbMergedWithNext) + { + if( m_pDb->m_pDbStats != NULL) + { + SFLM_LFILE_STATS * pLFileStats; + + if( (pLFileStats = m_pDb->getLFileStatPtr( m_pLFile)) != NULL) + { + pLFileStats->bHaveStats = TRUE; + pLFileStats->ui64BlockCombines++; + } + } + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + if( pNextSCache) + { + ScaReleaseCache( pNextSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Method to move the contents of the ppFromSCache block into the + ppToSCache block. Note that all merges are a move to next operation. +****************************************************************************/ +RCODE F_Btree::merge( + F_CachedBlock ** ppFromSCache, + F_CachedBlock ** ppToSCache) +{ + RCODE rc = NE_SFLM_OK; + F_BTSK tempStack; + F_BTSK * pStack = NULL; + F_CachedBlock * pSCache; + F_BTREE_BLK_HDR * pBlkHdr; + + // May need to defragment the blocks first. + + pBlkHdr = (F_BTREE_BLK_HDR *)(*ppToSCache)->m_pBlkHdr; + if( pBlkHdr->stdBlkHdr.ui16BlkBytesAvail != pBlkHdr->ui16HeapSize) + { + if( RC_BAD( rc = defragmentBlock( ppToSCache))) + { + goto Exit; + } + } + + // Make a temporary stack entry so we can "fool" the moveToNext + // function into moving the entries for us. + + pSCache = *ppFromSCache; + tempStack.pBlkHdr = (F_BTREE_BLK_HDR *)pSCache->m_pBlkHdr; + tempStack.ui32BlkAddr = pSCache->m_pBlkHdr->ui32BlkAddr; + tempStack.pSCache = pSCache; + tempStack.uiCurOffset = 0; + tempStack.uiLevel = m_pStack->uiLevel; + tempStack.pui16OffsetArray = BtOffsetArray( (FLMBYTE *)pSCache->m_pBlkHdr, 0); + + // Save the current m_pStack. + + pStack = m_pStack; + m_pStack = &tempStack; + + // Now do the move + + if( RC_BAD( rc = moveToNext( tempStack.pBlkHdr->ui16NumKeys - 1, + 0, ppToSCache))) + { + goto Exit; + } + + // Return the changed block structure + + *ppFromSCache = tempStack.pSCache; + +Exit: + + // Must always restore the stack. + + m_pStack = pStack; + return( rc); +} + +/*************************************************************************** +Desc: Method to test if the src and dst entries can be combined into one + entry. If they can, then they will be combined and stored in the + m_pucTempBlk buffer. +****************************************************************************/ +RCODE F_Btree::combineEntries( + F_BTREE_BLK_HDR * pSrcBlkHdr, + FLMUINT uiSrcOffset, + F_BTREE_BLK_HDR * pDstBlkHdr, + FLMUINT uiDstOffset, + FLMBOOL * pbEntriesCombined, + FLMUINT * puiEntrySize) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucSrcEntry; + FLMBYTE * pucDstEntry; + FLMUINT uiSrcKeyLen; + FLMUINT uiDstKeyLen; + const FLMBYTE * pucSrcKey; + const FLMBYTE * pucDstKey; + FLMUINT uiFlags = 0; + FLMBYTE * pucTmp; + FLMUINT uiSrcOADataLen; + FLMUINT uiDstOADataLen; + const FLMBYTE * pucSrcData; + const FLMBYTE * pucDstData; + FLMUINT uiSrcDataLen; + FLMUINT uiDstDataLen; + FLMUINT uiEntrySize; + + *pbEntriesCombined = FALSE; + *puiEntrySize = 0; + + if( pDstBlkHdr->ui16NumKeys == 0) + { + goto Exit; + } + + if( pSrcBlkHdr->ui16NumKeys == 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( getBlkType( (FLMBYTE *)pSrcBlkHdr) != BT_LEAF_DATA) + { + goto Exit; + } + + pucSrcEntry = BtEntry( (FLMBYTE *)pSrcBlkHdr, uiSrcOffset); + pucDstEntry = BtEntry( (FLMBYTE *)pDstBlkHdr, uiDstOffset); + + // Do we have the same key? + + uiSrcKeyLen = getEntryKeyLength( pucSrcEntry, BT_LEAF_DATA, &pucSrcKey); + uiDstKeyLen = getEntryKeyLength( pucDstEntry, BT_LEAF_DATA, &pucDstKey); + + if( uiSrcKeyLen != uiDstKeyLen) + { + // Not the same key. + + goto Exit; + } + + if( f_memcmp( pucSrcKey, pucDstKey, uiSrcKeyLen) != 0) + { + // Not the same key. + + goto Exit; + } + + // They match, so we can combine them. + + pucTmp = &m_pucTempBlk[ 1]; // Key length position + uiFlags = (pucDstEntry[0] & (BTE_FLAG_FIRST_ELEMENT | BTE_FLAG_LAST_ELEMENT)) | + (pucSrcEntry[0] & (BTE_FLAG_FIRST_ELEMENT | BTE_FLAG_LAST_ELEMENT)); + uiEntrySize = 1; + + if( uiSrcKeyLen > ONE_BYTE_SIZE) + { + uiFlags |= BTE_FLAG_KEY_LEN; + UW2FBA( (FLMUINT16)uiSrcKeyLen, pucTmp); + pucTmp += 2; + uiEntrySize += 2; + } + else + { + *pucTmp = (FLMBYTE)uiSrcKeyLen; + pucTmp++; + uiEntrySize++; + } + + uiSrcDataLen = btGetEntryDataLength( + pucSrcEntry, &pucSrcData, &uiSrcOADataLen, NULL); + + uiDstDataLen = btGetEntryDataLength( + pucDstEntry, &pucDstData, &uiDstOADataLen, NULL); + + if( (uiSrcDataLen + uiDstDataLen) > ONE_BYTE_SIZE) + { + uiFlags |= BTE_FLAG_DATA_LEN; + UW2FBA( (FLMUINT16)(uiSrcDataLen + uiDstDataLen), pucTmp); + pucTmp += 2; + uiEntrySize += 2; + } + else + { + *pucTmp = (FLMBYTE)(uiSrcDataLen + uiDstDataLen); + pucTmp++; + uiEntrySize++; + } + + // Verify the OA Data length + + if( (*pucSrcEntry & BTE_FLAG_OA_DATA_LEN) && + (uiSrcOADataLen > (uiSrcDataLen + uiDstDataLen))) + { + uiFlags |= BTE_FLAG_OA_DATA_LEN; + UD2FBA( (FLMUINT32)uiSrcOADataLen, pucTmp); + pucTmp += 4; + uiEntrySize += 4; + } + else if( (*pucDstEntry & BTE_FLAG_OA_DATA_LEN) && + (uiDstOADataLen > (uiSrcDataLen + uiDstDataLen))) + { + uiFlags |= BTE_FLAG_OA_DATA_LEN; + UD2FBA( (FLMUINT32)uiDstOADataLen, pucTmp); + pucTmp += 4; + uiEntrySize += 4; + } + + f_memcpy( pucTmp, pucSrcKey, uiSrcKeyLen); + pucTmp += uiSrcKeyLen; + uiEntrySize += uiSrcKeyLen; + + // Need to put the entry together in the right order. If the Src block is + // before the Dst block, then we will put down the Src data first. + + if( pSrcBlkHdr->stdBlkHdr.ui32NextBlkInChain == + pDstBlkHdr->stdBlkHdr.ui32BlkAddr) + { + f_memcpy( pucTmp, pucSrcData, uiSrcDataLen); + pucTmp += uiSrcDataLen; + uiEntrySize += uiSrcDataLen; + + f_memcpy( pucTmp, pucDstData, uiDstDataLen); + uiEntrySize += uiDstDataLen; + } + else + { + f_memcpy( pucTmp, pucDstData, uiDstDataLen); + uiEntrySize += uiDstDataLen; + pucTmp += uiDstDataLen; + + f_memcpy( pucTmp, pucSrcData, uiSrcDataLen); + uiEntrySize += uiSrcDataLen; + } + + m_pucTempBlk[ 0] = (FLMBYTE)uiFlags; + *puiEntrySize = uiEntrySize; + *pbEntriesCombined = TRUE; + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Method to move a block from one location to another. +****************************************************************************/ +RCODE F_Btree::btMoveBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiType; + + if( !m_bOpened || m_bSetupForRead || m_bSetupForReplace || + (m_bSetupForWrite)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + flmAssert( m_uiSearchLevel >= BH_MAX_LEVELS); + + // Verify the Txn type + + if( m_pDb->m_eTransType != SFLM_UPDATE_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( m_pDb->m_eTransType == SFLM_NO_TRANS + ? NE_SFLM_NO_TRANS_ACTIVE + : NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + + // Get the From block and retrieve the last key in the block. Make note + // of the level of the block. We will need this to make sure we get the + // right block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32FromBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + // Find out if this is a Btree block or a DO block. + + uiType = getBlkType((FLMBYTE *)m_pSCache->m_pBlkHdr); + + if( uiType == BT_FREE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( uiType == BT_DATA_ONLY) + { + if( RC_BAD( rc = moveDOBlock( ui32FromBlkAddr, ui32ToBlkAddr))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = moveBtreeBlock( ui32FromBlkAddr, ui32ToBlkAddr))) + { + goto Exit; + } + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + return( rc); +} + +/*************************************************************************** +Desc: Move a Btree block from one address to another, updating its parent. +****************************************************************************/ +RCODE F_Btree::moveBtreeBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr) +{ + RCODE rc = NE_SFLM_OK; + F_BTREE_BLK_HDR * pBlkHdr = NULL; + F_BTREE_BLK_HDR * pNewBlkHdr = NULL; + FLMBYTE * pucEntry; + const FLMBYTE * pucKeyRV = NULL; + FLMBYTE * pucKey = NULL; + FLMUINT uiBlkLevel; + FLMBYTE * pucSrc; + FLMBYTE * pucDest; + F_CachedBlock * pSCache = NULL; + FLMUINT uiKeyLen; + + // m_pSCache has already been retrieved. + + flmAssert( m_pSCache); + + pBlkHdr = (F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr; + uiBlkLevel = pBlkHdr->ui8BlkLevel; + + pucEntry = BtLastEntry( (FLMBYTE *)pBlkHdr); + + uiKeyLen = getEntryKeyLength( pucEntry, getBlkType((FLMBYTE *)pBlkHdr), + &pucKeyRV); + + if( RC_BAD( rc = f_calloc( uiKeyLen, &pucKey))) + { + goto Exit; + } + + f_memcpy( pucKey, pucKeyRV, uiKeyLen); + + // Release the block and search for the key. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + if( RC_BAD( rc = findEntry( pucKey, uiKeyLen, FLM_EXACT))) + { + // We must find it! + + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + + // Verify that we found the right block. + + m_pStack = &m_Stack[ uiBlkLevel]; + + if( ui32FromBlkAddr != m_pStack->ui32BlkAddr) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + m_pStack->pui16OffsetArray = BtOffsetArray( (FLMBYTE *)m_pStack->pBlkHdr, 0); + pBlkHdr = m_pStack->pBlkHdr; + + // Get the new block and verify that it is a free block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32ToBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( getBlkType( (FLMBYTE *)m_pSCache->m_pBlkHdr) != BT_FREE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Update the header of the new block to point to the prev and next + // blocks etc ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + + pNewBlkHdr = (F_BTREE_BLK_HDR *)m_pSCache->m_pBlkHdr; + + pNewBlkHdr->stdBlkHdr.ui32PrevBlkInChain = + pBlkHdr->stdBlkHdr.ui32PrevBlkInChain; + pNewBlkHdr->stdBlkHdr.ui32NextBlkInChain = + pBlkHdr->stdBlkHdr.ui32NextBlkInChain; + pNewBlkHdr->stdBlkHdr.ui16BlkBytesAvail = + pBlkHdr->stdBlkHdr.ui16BlkBytesAvail; + pNewBlkHdr->stdBlkHdr.ui8BlkType = pBlkHdr->stdBlkHdr.ui8BlkType; + pNewBlkHdr->stdBlkHdr.ui8BlkFlags = pBlkHdr->stdBlkHdr.ui8BlkFlags; + + pNewBlkHdr->ui16LogicalFile = pBlkHdr->ui16LogicalFile; + pNewBlkHdr->ui16NumKeys = pBlkHdr->ui16NumKeys; + pNewBlkHdr->ui8BlkLevel = pBlkHdr->ui8BlkLevel; + pNewBlkHdr->ui8BTreeFlags = pBlkHdr->ui8BTreeFlags; + pNewBlkHdr->ui16HeapSize = pBlkHdr->ui16HeapSize; + + // Get the previous and next blocks and set their next and prev addresses. + + if( pBlkHdr->stdBlkHdr.ui32PrevBlkInChain) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pBlkHdr->stdBlkHdr.ui32PrevBlkInChain, NULL, &pSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pSCache))) + { + goto Exit; + } + + flmAssert( pSCache->m_pBlkHdr->ui32NextBlkInChain == ui32FromBlkAddr); + pSCache->m_pBlkHdr->ui32NextBlkInChain = ui32ToBlkAddr; + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + + if( pBlkHdr->stdBlkHdr.ui32NextBlkInChain) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pBlkHdr->stdBlkHdr.ui32NextBlkInChain, NULL, &pSCache))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pSCache))) + { + goto Exit; + } + + flmAssert( pSCache->m_pBlkHdr->ui32PrevBlkInChain == ui32FromBlkAddr); + pSCache->m_pBlkHdr->ui32PrevBlkInChain = ui32ToBlkAddr; + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + } + + // Copy the content of the old block into the new block. + + pucSrc = (FLMBYTE *)pBlkHdr + sizeofBTreeBlkHdr( pBlkHdr); + pucDest = (FLMBYTE *)pNewBlkHdr + sizeofBTreeBlkHdr( pNewBlkHdr); + + f_memcpy( pucDest, pucSrc, m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr)); + + if( isRootBlk( pBlkHdr)) + { + m_pLFile->uiRootBlk = ui32ToBlkAddr; + rc = m_pDb->m_pDatabase->lFileWrite( m_pDb, m_pLFile); + goto Exit; + } + + // Move up one level to the parent entry. + + m_pStack++; + flmAssert( m_pStack->pSCache); + + // Log that we are making a change to the block. + + if( RC_BAD( rc = m_pDb->m_pDatabase->logPhysBlk( + m_pDb, &m_pStack->pSCache))) + { + goto Exit; + } + + m_pStack->pBlkHdr = (F_BTREE_BLK_HDR *)m_pStack->pSCache->m_pBlkHdr; + + // Update the parent block with a new address for the new block. + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + UD2FBA( ui32ToBlkAddr, pucEntry); + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + f_free( &pucKey); + releaseBlocks( TRUE); + return( rc); +} + +/*************************************************************************** +Desc: Move a DO block from one address to another, updating its reference + btree entry. +****************************************************************************/ +RCODE F_Btree::moveDOBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr) +{ + RCODE rc = NE_SFLM_OK; + F_BLK_HDR * pBlkHdr = NULL; + F_BLK_HDR * pNewBlkHdr = NULL; + FLMBYTE * pucEntry; + FLMBYTE * pucKey = NULL; + FLMBYTE * pucSrc; + FLMBYTE * pucDest; + F_CachedBlock * pSCache = NULL; + F_CachedBlock * pPrevSCache = NULL; + F_CachedBlock * pNextSCache = NULL; + FLMUINT uiKeyLen; + FLMUINT uiOADataLen; + const FLMBYTE * pucData; + FLMUINT32 ui32DOBlkAddr; + FLMUINT uiDataLen; + FLMBYTE ucDataBuffer[ sizeof(FLMUINT32)]; + FLMUINT uiBlkHdrSize; + + // m_pSCache has already been retrieved. + + flmAssert( m_pSCache); + + // Log that we are changing this block. + + if( RC_BAD( m_pDb->m_pDatabase->logPhysBlk( m_pDb, &m_pSCache))) + { + goto Exit; + } + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Get the new block and verify that it is a free block. + + if( RC_BAD( m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32ToBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + if( getBlkType( (FLMBYTE *)pSCache->m_pBlkHdr) != BT_FREE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Update the header of the new block to point to the prev and next + // blocks etc.. + + if( RC_BAD( m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pSCache))) + { + goto Exit; + } + + pNewBlkHdr = pSCache->m_pBlkHdr; + pNewBlkHdr->ui32PrevBlkInChain = pBlkHdr->ui32PrevBlkInChain; + pNewBlkHdr->ui32NextBlkInChain = pBlkHdr->ui32NextBlkInChain; + pNewBlkHdr->ui16BlkBytesAvail = pBlkHdr->ui16BlkBytesAvail; + pNewBlkHdr->ui8BlkType = pBlkHdr->ui8BlkType; + pNewBlkHdr->ui8BlkFlags = pBlkHdr->ui8BlkFlags; + + // Get the previous and next blocks and set their next and prev addresses. + + if( pBlkHdr->ui32PrevBlkInChain) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pBlkHdr->ui32PrevBlkInChain, NULL, &pPrevSCache))) + { + goto Exit; + } + + if( RC_BAD( m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pPrevSCache))) + { + goto Exit; + } + + flmAssert( pPrevSCache->m_pBlkHdr->ui32NextBlkInChain == ui32FromBlkAddr); + pPrevSCache->m_pBlkHdr->ui32NextBlkInChain = ui32ToBlkAddr; + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + + if( pBlkHdr->ui32NextBlkInChain) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + pBlkHdr->ui32NextBlkInChain, NULL, &pNextSCache))) + { + goto Exit; + } + + if( RC_BAD( m_pDb->m_pDatabase->logPhysBlk( m_pDb, &pNextSCache))) + { + goto Exit; + } + + flmAssert( pNextSCache->m_pBlkHdr->ui32PrevBlkInChain == ui32FromBlkAddr); + pNextSCache->m_pBlkHdr->ui32PrevBlkInChain = ui32ToBlkAddr; + ScaReleaseCache( pNextSCache, FALSE); + pNextSCache = NULL; + } + + // Copy the content of the old block into the new block. + + uiBlkHdrSize = sizeofDOBlkHdr((F_BLK_HDR *)pBlkHdr); + pucSrc = (FLMBYTE *)pBlkHdr + uiBlkHdrSize; + pucDest = (FLMBYTE *)pNewBlkHdr + uiBlkHdrSize; + f_memcpy( pucDest, pucSrc, m_uiBlockSize - uiBlkHdrSize); + + // Do we need to update the reference btree entry. + + if( pBlkHdr->ui32PrevBlkInChain == 0) + { + // Get the key from the beginning of the block. + + uiKeyLen = FB2UW( pucDest); + pucKey = pucDest + sizeof( FLMUINT16); + + if( RC_BAD( rc = findEntry( pucKey, uiKeyLen, FLM_EXACT))) + { + // We must find it! + + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + + // Verify that we found the right block. + + pucEntry = BtEntry( (FLMBYTE *)m_pStack->pBlkHdr, m_pStack->uiCurOffset); + + if( !bteDataBlockFlag( pucEntry)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + uiDataLen = btGetEntryDataLength( pucEntry, &pucData, + &uiOADataLen, NULL); + + ui32DOBlkAddr = bteGetBlkAddr( pucData); + + if( ui32DOBlkAddr != ui32FromBlkAddr) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( uiDataLen != sizeof( ucDataBuffer)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Make the data entry with the new block address + + UD2FBA( ui32ToBlkAddr, ucDataBuffer); + + if( RC_BAD( rc = updateEntry( + pucKey, uiKeyLen, ucDataBuffer, uiOADataLen, ELM_REPLACE_DO))) + { + goto Exit; + } + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + if( pNextSCache) + { + ScaReleaseCache( pNextSCache, FALSE); + } + + releaseBlocks( TRUE); + return( rc); +} + +/*************************************************************************** +Desc: Method to move the read point in an entry to a particular position + within the entry. This method will move to a previous or a later + position. +****************************************************************************/ +RCODE F_Btree::btSetReadPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT uiPosition) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry; + F_BLK_HDR * pBlkHdr = NULL; + FLMUINT32 ui32BlkAddr; + FLMBOOL bLastElement = FALSE; + + if( !m_bOpened || !m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + // We cannot position to a point beyond the end of the current entry. + + if( uiPosition >= m_uiOADataLength) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // If the transaction Id or the Block Change Count has changed, + // we must re-sync ourselves. + + if( (m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt)) + { + + // Test to see if we really need to re-synch... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + ( m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // We must call btLocateEntry so we can re-initialize the read. + + if( !m_bFirstRead) + { + if( RC_BAD( rc = btLocateEntry( + pucKey, uiKeyLen, &uiKeyLen, FLM_EXACT))) + { + goto Exit; + } + + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + else + { + rc = RC_SET(NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + } + } + + // The easiest case to handle is when we want to position within the + // current entry. We should not have to worry about the data only blocks + // because the m_uiDataLength and m_uiDataRemaining are being set correctly + // in setupReadState (via btLocateEntry, btNextEntry, btPrevEntry, + // btFirstEntry and btLastEntry) which is always called before this method is + // called. + + if( (uiPosition < (m_uiOffsetAtStart + m_uiDataLength)) && + (uiPosition >= m_uiOffsetAtStart)) + { + m_uiDataRemaining = m_uiDataLength - (uiPosition - m_uiOffsetAtStart); + m_uiOADataRemaining = m_uiOADataLength - uiPosition; + goto Exit; + } + + // Get the current block. It is either a DO or a Btree block. + + if( m_pSCache == NULL) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + } + + // The next case is when the new position is in a *previous* entry, possibly + // a previous block. + + while( uiPosition < m_uiOffsetAtStart) + { + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Are we dealing with DataOnly blocks? + + if( m_bDataOnlyBlock) + { + ui32BlkAddr = pBlkHdr->ui32PrevBlkInChain; + flmAssert( ui32BlkAddr); + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + m_ui32CurBlkAddr = ui32BlkAddr; + pBlkHdr = m_pSCache->m_pBlkHdr; + + m_uiDataLength = m_uiBlockSize - pBlkHdr->ui16BlkBytesAvail - + sizeofDOBlkHdr((F_BLK_HDR *)pBlkHdr); + + if( !pBlkHdr->ui32PrevBlkInChain) + { + FLMBYTE * pucPtr = (FLMBYTE *)pBlkHdr + sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr); + FLMUINT16 ui16KeyLen = FB2UW( pucPtr); + + // We need to adjust for the key in the first block. + + m_uiDataLength -= ui16KeyLen; + } + + // Decrement by the size of the current data + + m_uiOffsetAtStart -= m_uiDataLength; + } + else + { + // Backup to the previous element. This may or may not get + // another block + + if( RC_BAD( rc = backupToPrevElement( FALSE))) + { + goto Exit; + } + + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, m_uiCurOffset); + + // Make sure we are still looking at the same key etc. + + if( !checkContinuedEntry( + pucKey, uiKeyLen, &bLastElement, pucEntry, + getBlkType( (FLMBYTE *)m_pSCache->m_pBlkHdr))) + { + // Should always match at this point! + + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + m_uiDataLength = btGetEntryDataLength( pucEntry, NULL, NULL, NULL); + m_uiOffsetAtStart -= m_uiDataLength; + } + } + + // Did we find the block? + + if( (uiPosition < (m_uiOffsetAtStart + m_uiDataLength)) && + (uiPosition >= m_uiOffsetAtStart)) + { + m_uiDataRemaining = m_uiDataLength - (uiPosition - m_uiOffsetAtStart); + m_uiOADataRemaining = m_uiOADataLength - uiPosition; + goto Exit; + } + + // Finally, we realize that the new position is beyond the current entry. + + while( uiPosition >= (m_uiOffsetAtStart + m_uiDataLength)) + { + flmAssert( m_uiDataLength + m_uiOffsetAtStart <= m_uiOADataLength); + + // Get the next entry. + + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Are we dealing with DataOnly blocks? + + if( m_bDataOnlyBlock) + { + ui32BlkAddr = pBlkHdr->ui32NextBlkInChain; + flmAssert( ui32BlkAddr); + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32BlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + m_ui32CurBlkAddr = ui32BlkAddr; + pBlkHdr = m_pSCache->m_pBlkHdr; + + // Increment by the size of the previous data. Note that in this + // case, we do not have to be concerned about the key in the first + // DO block since we will never move forward to it. + + m_uiOffsetAtStart += m_uiDataLength; + m_uiDataLength = m_uiBlockSize - pBlkHdr->ui16BlkBytesAvail - + sizeofDOBlkHdr((F_BLK_HDR *)pBlkHdr); + } + else + { + // Advance to the next element. This may or may not get another block. + // Be sure we do not advance the stack since we do not have one. + + if( RC_BAD( rc = advanceToNextElement( FALSE))) + { + goto Exit; + } + + pucEntry = BtEntry( (FLMBYTE *)m_pSCache->m_pBlkHdr, m_uiCurOffset); + + // Make sure we are still looking at the same key etc. + + if( !checkContinuedEntry( + pucKey, uiKeyLen, &bLastElement, pucEntry, + getBlkType( (FLMBYTE *)m_pSCache->m_pBlkHdr))) + { + // Should always match at this point! + + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Get the data length of the current entry. + + m_uiOffsetAtStart += m_uiDataLength; + m_uiDataLength = btGetEntryDataLength( pucEntry, NULL, NULL, NULL); + } + } + + // Did we find the block? If we still don't find it, then we + // have a big problem. + + if( (uiPosition >= (m_uiOffsetAtStart + m_uiDataLength)) || + (uiPosition < m_uiOffsetAtStart)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + m_uiDataRemaining = m_uiDataLength - (uiPosition - m_uiOffsetAtStart); + m_uiOADataRemaining = m_uiOADataLength - uiPosition; + updateTransInfo( m_pSCache->m_pBlkHdr->ui64TransID, + m_pSCache->m_ui64HighTransID); + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: +****************************************************************************/ +RCODE F_Btree::btGetReadPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT * puiPosition) +{ + RCODE rc = NE_SFLM_OK; + + if( !m_bOpened || !m_bSetupForRead) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + + flmAssert( puiPosition); + + // If the transaction ID or the block change count has changed, + // we must re-sync ourselves. + + if( !m_bTempDb && + ((m_ui64CurrTransID != m_pDb->m_ui64CurrTransID) || + (m_uiBlkChangeCnt != m_pDb->m_uiBlkChangeCnt))) + { + // Test to see if we really need to re-sync ... + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + m_ui32CurBlkAddr, NULL, &m_pSCache))) + { + goto Exit; + } + + if( m_pSCache->m_pBlkHdr->ui64TransID != m_ui64LastBlkTransId || + (m_pDb->m_eTransType == SFLM_UPDATE_TRANS && + m_pDb->m_ui64CurrTransID == m_pSCache->m_pBlkHdr->ui64TransID)) + { + // We must call btLocateEntry so we can re-initialize the read + + if( !m_bFirstRead) + { + if( RC_BAD( rc = btLocateEntry( + pucKey, uiKeyLen, &uiKeyLen, FLM_EXACT))) + { + goto Exit; + } + + // Will need a new version of this block. + + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + else + { + rc = RC_SET(NE_SFLM_BTREE_BAD_STATE); + goto Exit; + } + } + } + + *puiPosition = m_uiOffsetAtStart + (m_uiDataLength - m_uiDataRemaining); + + if( m_pSCache) + { + updateTransInfo( m_pSCache->m_pBlkHdr->ui64TransID, + m_pSCache->m_ui64HighTransID); + } + +Exit: + + if( m_pSCache) + { + ScaReleaseCache( m_pSCache, FALSE); + m_pSCache = NULL; + } + + releaseBlocks( FALSE); + return( rc); +} + +/*************************************************************************** +Desc: Performs a consistancy check on the BTree + NOTE: Must be performed inside of a read transaction! +****************************************************************************/ +RCODE F_Btree::btCheck( + BTREE_ERR_STRUCT * pErrStruct) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32NextBlkAddr = 0; + FLMUINT32 ui32NextLevelBlkAddr = 0; + FLMUINT32 ui32ChildBlkAddr = 0; + FLMUINT32 ui32DOBlkAddr = 0; + FLMUINT uiNumKeys; + const FLMBYTE * pucPrevKey; + FLMUINT uiPrevKeySize; + const FLMBYTE * pucCurKey; + FLMUINT uiCurKeySize; + F_CachedBlock * pCurrentBlk = NULL; + F_CachedBlock * pPrevSCache = NULL; + FLMBYTE * pBlk = NULL; + FLMBYTE * pucEntry = NULL; + FLMBYTE * pucPrevEntry = NULL; + F_CachedBlock * pChildBlk = NULL; + FLMUINT16 * puiOffsetArray; + BTREE_ERR_STRUCT localErrStruct; + FLMINT iCmpResult; + FLMUINT uiOADataLength = 0; + + // Verify the Txn type + + if( m_pDb->m_eTransType == SFLM_NO_TRANS && !m_bTempDb) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // Initial setup... + + ui32NextLevelBlkAddr = (FLMUINT32)m_pLFile->uiRootBlk; + f_memset( &localErrStruct, 0, sizeof( localErrStruct)); + localErrStruct.uiBlockSize = m_uiBlockSize; + + // While there's a next level.... + + while( ui32NextLevelBlkAddr) + { + localErrStruct.uiLevels++; + ui32NextBlkAddr = ui32NextLevelBlkAddr; + + // Update uiNextLevelBlkAddr + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &pCurrentBlk))) + { + localErrStruct.type = SCA_GET_BLOCK_FAILED; + f_sprintf( localErrStruct.szMsg, + "Failed to get block at %X", ui32NextBlkAddr); + goto Exit; + } + + pBlk = (FLMBYTE *)pCurrentBlk->m_pBlkHdr; + puiOffsetArray = BtOffsetArray( pBlk, 0); + + if( (getBlkType( pBlk) == BT_LEAF) || (getBlkType( pBlk) == BT_LEAF_DATA)) + { + ui32NextLevelBlkAddr = 0; + } + else + { + pucEntry = BtEntry( pBlk, 0); + + // The child block address is the first part of the entry + + ui32NextLevelBlkAddr = bteGetBlkAddr( pucEntry); + } + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + + // While there's another block on this level... + + while( ui32NextBlkAddr) + { + // This loop assumes that pCurrentBlk and pBlk are already initialized. + + localErrStruct.uiBlocksChecked++; + localErrStruct.uiAvgFreeSpace = + (localErrStruct.uiAvgFreeSpace * (localErrStruct.uiBlocksChecked - 1) / + localErrStruct.uiBlocksChecked) + + (getBlkAvailSpace(pBlk) / localErrStruct.uiBlocksChecked); + localErrStruct.ui64FreeSpace += getBlkAvailSpace(pBlk); + + localErrStruct.LevelStats[ localErrStruct.uiLevels - 1].uiBlkCnt++; + localErrStruct.LevelStats[ localErrStruct.uiLevels - 1].uiBytesUsed += + (m_uiBlockSize - getBlkAvailSpace(pBlk)); + + uiNumKeys = ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys; + + // VISIT: Verify the block header fields + /* + ui32PrevBlkInChain = + ui32BlkCRC = + ui16BlkBytesAvail < ? + ui8BlkLevel?? + ui8BlkIsRoot?? + */ + + // Verify that the keys are in order... + // Make sure that we check the keys between blocks as well. + + if( pPrevSCache) + { + pucEntry = BtLastEntry( (FLMBYTE *)pPrevSCache->m_pBlkHdr); + uiPrevKeySize = getEntryKeyLength( pucEntry, getBlkType( + (FLMBYTE *)pPrevSCache->m_pBlkHdr), &pucPrevKey); + } + else + { + pucEntry = BtEntry( pBlk, 0); + uiPrevKeySize = getEntryKeyLength( pucEntry, getBlkType( pBlk), + &pucPrevKey); + if( getBlkType(pBlk) == BT_LEAF_DATA) + { + if( bteFirstElementFlag( pucEntry)) + { + localErrStruct.LevelStats[ + localErrStruct.uiLevels - 1].uiFirstKeyCnt++; + } + } + else + { + // Everything else is a first key. + + localErrStruct.LevelStats[ + localErrStruct.uiLevels - 1].uiFirstKeyCnt++; + } + } + + for( FLMUINT uiLoop = (pPrevSCache ? 0: 1); + uiLoop < uiNumKeys; uiLoop++) + { + pucPrevEntry = pucEntry; + pucEntry = BtEntry( pBlk, uiLoop); + + if( getBlkType(pBlk) == BT_LEAF_DATA) + { + if( bteFirstElementFlag( pucEntry)) + { + localErrStruct.LevelStats[ + localErrStruct.uiLevels - 1].uiFirstKeyCnt++; + } + } + else + { + // Everything else is a first key. + + localErrStruct.LevelStats[ + localErrStruct.uiLevels - 1].uiFirstKeyCnt++; + } + + uiCurKeySize = getEntryKeyLength( pucEntry, + getBlkType( pBlk), &pucCurKey); + + // The last key in the last block of each level is an infinity marker + // It must have a 0 keylength and if it's a leaf node, a 0 datalength. + + if( (uiLoop == uiNumKeys - 1) && + (((F_BLK_HDR *)pBlk)->ui32NextBlkInChain == 0)) + { + // If the key size is not 0, or we're a leaf block, and the + // data size is not 0 ... + + if( (uiCurKeySize != 0) || + (((getBlkType( pBlk) == BT_LEAF_DATA)) && + (btGetEntryDataLength( pucEntry, NULL, NULL, NULL) > 0))) + { + localErrStruct.type = INFINITY_MARKER; + localErrStruct.uiBlkAddr = ((F_BLK_HDR *)pBlk)->ui32BlkAddr; + f_sprintf( localErrStruct.szMsg, "Invalid Infinity Marker %ul", uiLoop); + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + else + { + // Do a comparison of the previous and current keys ... + + if( RC_BAD( rc = compareKeys( pucPrevKey, uiPrevKeySize, + pucCurKey, uiCurKeySize, &iCmpResult))) + { + goto Exit; + } + + if( iCmpResult > 0) + { + localErrStruct.type = KEY_ORDER; + localErrStruct.uiBlkAddr = ((F_BLK_HDR *)pBlk)->ui32BlkAddr; + f_sprintf( localErrStruct.szMsg, "Key Number %ul", uiLoop); + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + if( getBlkType(pBlk) == BT_LEAF_DATA) + { + if( iCmpResult < 0) + { + flmAssert( *pucEntry & BTE_FLAG_FIRST_ELEMENT); + } + else if( iCmpResult == 0) + { + flmAssert( (*pucEntry & BTE_FLAG_FIRST_ELEMENT) == 0); + flmAssert( (*pucPrevEntry & BTE_FLAG_LAST_ELEMENT) == 0); + } + } + } + + pucPrevKey = pucCurKey; + uiPrevKeySize = uiCurKeySize; + } + + localErrStruct.uiNumKeys += uiNumKeys; + localErrStruct.LevelStats[ + localErrStruct.uiLevels - 1].uiKeyCnt += uiNumKeys; + + // If this is a leaf block, check for any pointers to data-only + // blocks. Verify the blocks... + + if( getBlkType( pBlk) == BT_LEAF || + getBlkType( pBlk) == BT_LEAF_DATA) + { + if( getBlkType( pBlk) == BT_LEAF_DATA) + { + for( FLMUINT uiLoop = 0; uiLoop < uiNumKeys; uiLoop++) + { + pucEntry = BtEntry( pBlk, uiLoop); + + if( bteDataBlockFlag( pucEntry)) + { + FLMBYTE ucDOBlkAddr[ 4]; + + if( RC_BAD( rc = btGetEntryData( pucEntry, + &ucDOBlkAddr[ 0], 4, NULL))) + { + RC_UNEXPECTED_ASSERT( rc); + localErrStruct.type = CATASTROPHIC_FAILURE; + localErrStruct.uiBlkAddr = ((F_BLK_HDR *)pBlk)->ui32BlkAddr; + f_sprintf( localErrStruct.szMsg, + "getEntryData couldn't get the DO blk addr."); + goto Exit; + } + + ui32DOBlkAddr = bteGetBlkAddr( (FLMBYTE *)&ucDOBlkAddr[ 0]); + + // Verify that there is an OverallDataLength field + + if( bteOADataLenFlag( pucEntry) == 0) + { + localErrStruct.type = MISSING_OVERALL_DATA_LENGTH; + localErrStruct.uiBlkAddr = ((F_BLK_HDR *)pBlk)->ui32BlkAddr; + f_sprintf( localErrStruct.szMsg, + "OverallDataLength field is missing"); + } + else + { + if( bteKeyLenFlag( pucEntry)) + { + uiOADataLength = FB2UD( pucEntry + 4); + } + else + { + uiOADataLength = FB2UD( pucEntry + 3); + } + } + + if( RC_BAD( rc = verifyDOBlkChain( ui32DOBlkAddr, + uiOADataLength , &localErrStruct))) + { + goto Exit; + } + } + } + } + } + else + { + // This is a non-leaf block, verify that blocks exist for all + // the child block addresses + + // NOTE: Also need to somehow verify that no two elements have the + // same child block address... + + for( FLMUINT uiLoop = 0; uiLoop < uiNumKeys; uiLoop++) + { + pucEntry = BtEntry( pBlk, uiLoop); + ui32ChildBlkAddr = bteGetBlkAddr( pucEntry); + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32ChildBlkAddr, NULL, &pChildBlk))) + { + localErrStruct.type = SCA_GET_BLOCK_FAILED; + f_sprintf( localErrStruct.szMsg, "Failed to get block at %X", + ui32ChildBlkAddr); + goto Exit; + } + + ScaReleaseCache( pChildBlk, FALSE); + } + } + + // Release the current block and get the next one + + ui32NextBlkAddr = ((F_BLK_HDR *)pBlk)->ui32NextBlkInChain; + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + pPrevSCache = NULL; + } + + pPrevSCache = pCurrentBlk; + pCurrentBlk = NULL; + + if( ui32NextBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextBlkAddr, NULL, &pCurrentBlk))) + { + localErrStruct.type = SCA_GET_BLOCK_FAILED; + f_sprintf( localErrStruct.szMsg, + "Failed to get block at %X", ui32ChildBlkAddr); + goto Exit; + } + + pBlk = (FLMBYTE *)pCurrentBlk->m_pBlkHdr; + } + } + } + + if( m_bCounts) + { + if( RC_BAD( rc = verifyCounts( &localErrStruct))) + { + goto Exit; + } + } + +Exit: + + if( pPrevSCache) + { + ScaReleaseCache( pPrevSCache, FALSE); + } + + if( pCurrentBlk) + { + ScaReleaseCache( pCurrentBlk, FALSE); + } + + f_memcpy( pErrStruct, &localErrStruct, sizeof( localErrStruct)); + return( rc); +} + +/*************************************************************************** +Desc: Performs an integrity check on a chain of data-only blocks. Should + only be called from btCheck(). Note that unlike btCheck(), + errStruct CANNOT be NULL here. +****************************************************************************/ +RCODE F_Btree::verifyDOBlkChain( + FLMUINT uiDOAddr, // Address of first block in chain + FLMUINT uiDataLength, // The length of the entire entry + BTREE_ERR_STRUCT * errStruct) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiRunningLength = 0; // A running total of the DataLength fields + // for all of the blocks in this chain + F_CachedBlock * pCurrentBlk = NULL; + FLMUINT32 ui32NextAddr = (FLMUINT32)uiDOAddr; + FLMBYTE * pBlk; + FLMUINT uiDataSize; + + while( ui32NextAddr) + { + errStruct->LevelStats[ errStruct->uiLevels - 1].uiDOBlkCnt++; + + // Get the next block + + if( RC_BAD( m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + ui32NextAddr, NULL, &pCurrentBlk))) + { + errStruct->type = SCA_GET_BLOCK_FAILED; + f_sprintf( errStruct->szMsg, "Failed to get block at %X", uiDOAddr); + goto Exit; + } + + pBlk = (FLMBYTE *)pCurrentBlk->m_pBlkHdr; + + // Verify that it's really a DO Block + + if( getBlkType( pBlk) != BT_DATA_ONLY) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + errStruct->type = NOT_DATA_ONLY_BLOCK; + goto Exit; + } + + // Update counts info in errStruct + + errStruct->LevelStats[ errStruct->uiLevels - 1].uiDOBytesUsed += + m_uiBlockSize - pCurrentBlk->m_pBlkHdr->ui16BlkBytesAvail; + + // Update the data length running total + + uiDataSize = m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlk) - + ((F_BLK_HDR *)pBlk)->ui16BlkBytesAvail; + + if( ((F_BLK_HDR *)pBlk)->ui32PrevBlkInChain == 0) + { + FLMBYTE * pucPtr = pBlk + sizeofDOBlkHdr( (F_BLK_HDR *)pBlk); + FLMUINT16 ui16KeyLen = FB2UW( pucPtr); + + uiDataSize -= (ui16KeyLen + 2); + } + + uiRunningLength += uiDataSize; + + // Update ui32nextAddr + + ui32NextAddr = ((F_BLK_HDR *)pBlk)->ui32NextBlkInChain; + + // Release it when we no longer need it. + + ScaReleaseCache( pCurrentBlk, FALSE); + pCurrentBlk = NULL; + } + + // Check the calculated overall length vs. uiDataLength + + if( uiRunningLength != uiDataLength) + { + errStruct->type = BAD_DO_BLOCK_LENGTHS; + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + +Exit: + + if( pCurrentBlk) + { + ScaReleaseCache( pCurrentBlk, FALSE); + } + + if( rc == NE_SFLM_BTREE_ERROR) + { + f_sprintf( errStruct->szMsg, "Corrupt DO chain starting at %X", uiDOAddr); + } + + return( NE_SFLM_OK); +} + +/*************************************************************************** +Desc: Method to check the counts in a database with counts. +****************************************************************************/ +RCODE F_Btree::verifyCounts( + BTREE_ERR_STRUCT * pErrStruct) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNextLevelBlkAddr; + FLMUINT uiNextBlkAddr; + FLMUINT uiChildBlkAddr; + F_CachedBlock * pCurrentBlk = NULL; + F_CachedBlock * pChildBlk = NULL; + FLMBYTE * pucEntry; + FLMUINT uiNumKeys; + FLMUINT uiEntryNum; + FLMUINT uiParentCounts; + FLMUINT uiChildCounts; + FLMBYTE * pBlk; + FLMBOOL bDone = FALSE; + + flmAssert( m_bCounts); + + // Repeat at each level, starting at the root. + + uiNextLevelBlkAddr = m_pLFile->uiRootBlk; + + while( uiNextLevelBlkAddr) + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, uiNextLevelBlkAddr, NULL, &pCurrentBlk))) + { + goto Exit; + } + + if( pCurrentBlk->m_pBlkHdr->ui8BlkType != BT_NON_LEAF_COUNTS) + { + ScaReleaseCache( pCurrentBlk, FALSE); + pCurrentBlk = NULL; + break; + } + + pucEntry = BtEntry( (FLMBYTE *)pCurrentBlk->m_pBlkHdr, 0); + uiNextLevelBlkAddr = bteGetBlkAddr( pucEntry); + + // For every entry in the block, and for every block on this level, + // check that the counts match the actual counts in the corresponding + // child block. + + bDone = FALSE; + while( !bDone) + { + uiNumKeys = ((F_BTREE_BLK_HDR *)pCurrentBlk->m_pBlkHdr)->ui16NumKeys; + pBlk = (FLMBYTE *)pCurrentBlk->m_pBlkHdr; + + // Now check every entry in this block. + + for( uiEntryNum = 0; uiEntryNum < uiNumKeys; uiEntryNum++) + { + pucEntry = BtEntry( pBlk, uiEntryNum); + uiChildBlkAddr = bteGetBlkAddr( pucEntry); + + pucEntry += 4; + uiParentCounts = FB2UD( pucEntry); + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, m_pLFile, + uiChildBlkAddr, NULL, &pChildBlk))) + { + goto Exit; + } + + uiChildCounts = countKeys( (FLMBYTE *)pChildBlk->m_pBlkHdr); + + if( uiChildCounts != uiParentCounts) + { + pErrStruct->type = BAD_COUNTS; + pErrStruct->uiBlkAddr = pChildBlk->m_pBlkHdr->ui32BlkAddr; + f_sprintf( + pErrStruct->szMsg, + "Counts do not match. Expected %d, got %d", + uiParentCounts, uiChildCounts); + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + ScaReleaseCache( pChildBlk, FALSE); + pChildBlk = NULL; + } + + // Now get the next block at this level. + + uiNextBlkAddr = pCurrentBlk->m_pBlkHdr->ui32NextBlkInChain; + ScaReleaseCache( pCurrentBlk, FALSE); + pCurrentBlk = NULL; + + if( uiNextBlkAddr == 0) + { + bDone = TRUE; + } + else + { + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( + m_pDb, m_pLFile, uiNextBlkAddr, NULL, &pCurrentBlk))) + { + goto Exit; + } + } + } + } + +Exit: + + if( pCurrentBlk) + { + ScaReleaseCache( pCurrentBlk, FALSE); + } + + if( pChildBlk) + { + ScaReleaseCache( pChildBlk, FALSE); + } + + return( rc); +} diff --git a/sql/src/f_btree.h b/sql/src/f_btree.h new file mode 100644 index 0000000..066c0bb --- /dev/null +++ b/sql/src/f_btree.h @@ -0,0 +1,1058 @@ +//------------------------------------------------------------------------------ +// Desc: Header file for the B-Tree class definitions +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_btree.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef F_BTREE_H +#define F_BTREE_H + +// Represent the maximum size for data & key before needing two bytes to +// store the length. + +#define ONE_BYTE_SIZE 0xFF + +// Flag definitions - BT_LEAF_DATA + +#define BTE_LEAF_DATA_OVHD 7 // Offset (2) Flags (1) OA Data (4) + +#define BTE_FLAG 0 // Offset to the FLAGS field +#define BTE_FLAG_LAST_ELEMENT 0x04 +#define BTE_FLAG_FIRST_ELEMENT 0x08 +#define BTE_FLAG_DATA_BLOCK 0x10 // Data is stored in a Data-only Block +#define BTE_FLAG_OA_DATA_LEN 0x20 // Overall data length +#define BTE_FLAG_DATA_LEN 0x40 +#define BTE_FLAG_KEY_LEN 0x80 + +// BT_LEAF (no data) + +#define BTE_LEAF_OVHD 4 // Offset (2) KeyLen (2) +#define BTE_KEY_LEN 0 +#define BTE_KEY_START 2 + +// BT_NON_LEAF_DATA + +#define BTE_NON_LEAF_OVHD 8 // Offset (2) Child Blk Addr (4) KeyLen (2) +#define BTE_NL_CHILD_BLOCK_ADDR 0 +#define BTE_NL_KEY_LEN 4 +#define BTE_NL_KEY_START 6 + +// BT_NON_LEAF_COUNTS + +#define BTE_NON_LEAF_COUNTS_OVHD 12 // Offset (2) Child Blk Addr (4) Counts (4) KeyLen (2) +#define BTE_NLC_CHILD_BLOCK_ADDR 0 +#define BTE_NLC_COUNTS 4 +#define BTE_NLC_KEY_LEN 8 +#define BTE_NLC_KEY_START 10 + +// Low water mark for coalescing blocks (as a percentage) + +#define BT_LOW_WATER_MARK 65 + +FINLINE FLMBOOL bteKeyLenFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_KEY_LEN) ? TRUE : FALSE); +} + +FINLINE FLMBOOL bteDataLenFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_DATA_LEN) ? TRUE : FALSE); +} + +FINLINE FLMBOOL bteOADataLenFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_OA_DATA_LEN) ? TRUE : FALSE); +} + +FINLINE FLMBOOL bteDataBlockFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_DATA_BLOCK) ? TRUE : FALSE); +} + +FINLINE FLMBOOL bteFirstElementFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_FIRST_ELEMENT) ? TRUE : FALSE); +} + +FINLINE FLMBOOL bteLastElementFlag( + FLMBYTE * pucEntry) +{ + return( (pucEntry[ BTE_FLAG] & BTE_FLAG_LAST_ELEMENT) ? TRUE : FALSE); +} + +FINLINE FLMUINT32 bteGetBlkAddr( + const FLMBYTE * pucEntry) +{ + return( FB2UD( pucEntry)); +} + +FINLINE void bteSetEntryOffset( + FLMUINT16 * pui16OffsetArray, + FLMUINT uiOffsetIndex, + FLMUINT16 ui16Offset) +{ + UW2FBA( ui16Offset, (FLMBYTE *)&pui16OffsetArray[ uiOffsetIndex]); +} + +FINLINE FLMUINT16 bteGetEntryOffset( + const FLMUINT16 * pui16OffsetArray, + FLMUINT uiOffsetIndex) +{ + return( FB2UW( (FLMBYTE *)&pui16OffsetArray[ uiOffsetIndex])); +} + +typedef struct F_BTSK +{ + F_BTREE_BLK_HDR * pBlkHdr; + F_CachedBlock * pSCache; + const FLMBYTE * pucKeyBuf; + FLMUINT uiKeyLen; + FLMUINT uiCurOffset; + FLMUINT uiLevel; + FLMUINT16 * pui16OffsetArray; + FLMUINT32 ui32BlkAddr; +} F_BTSK; + +typedef enum +{ + ELM_INSERT_DO, + ELM_INSERT, + ELM_REPLACE_DO, + ELM_REPLACE, + ELM_REMOVE, + ELM_BLK_MERGE, + ELM_DONE +} F_ELM_UPD_ACTION; + + +FINLINE FLMBYTE * BtEntry( + FLMBYTE * pBlk, + FLMUINT uiIndex) +{ + FLMBYTE * pucOffsetArray = + pBlk + sizeofBTreeBlkHdr( (F_BTREE_BLK_HDR *)pBlk) + + (uiIndex * 2); // 2 byte offset entries. + + return( pBlk + FB2UW( pucOffsetArray)); +} + +FINLINE FLMBYTE * BtLastEntry( + FLMBYTE * pBlk + ) +{ + return BtEntry( pBlk, ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys - 1); +} + +FINLINE FLMUINT getBlkType( + FLMBYTE * pBlk) +{ + return (FLMUINT)((F_BLK_HDR *)pBlk)->ui8BlkType; +} + +FINLINE FLMUINT16 * BtOffsetArray( + FLMBYTE * pBlk, + FLMUINT uiIndex) +{ + return (FLMUINT16 *) + (pBlk + sizeofBTreeBlkHdr( (F_BTREE_BLK_HDR *)pBlk) + (uiIndex * sizeof( FLMUINT16))); +} + +// Returns the address of the first entry in the block. i.e. the first non-blank +// address after the offset array. + +FINLINE FLMBYTE * getBlockEnd( + F_BTREE_BLK_HDR * pBlkHdr) +{ + return ((FLMBYTE *)pBlkHdr + sizeofBTreeBlkHdr( pBlkHdr) + + (pBlkHdr->ui16NumKeys * 2) + + pBlkHdr->ui16HeapSize); +} + +// This inline function takes the parameter returned from getEntrySize which +// adds 2 for the offset. + +FINLINE FLMUINT actualEntrySize( + FLMUINT uiEntrySize) +{ + return uiEntrySize - 2; +} + +// Error information returned by btCheck() + +enum BTREE_ERR_TYPE +{ + NO_ERR = 0, // FYI: Visual Studio already defines NOERROR + BT_HEADER, + KEY_ORDER, + DUPLICATE_KEYS, + INFINITY_MARKER, + CHILD_BLOCK_ADDRESS, + SCA_GET_BLOCK_FAILED, + MISSING_OVERALL_DATA_LENGTH, + NOT_DATA_ONLY_BLOCK, + BAD_DO_BLOCK_LENGTHS, + BAD_COUNTS, + CATASTROPHIC_FAILURE = 999 +}; + +typedef struct +{ + FLMUINT uiKeyCnt; + FLMUINT uiFirstKeyCnt; + FLMUINT uiBlkCnt; + FLMUINT uiBytesUsed; + FLMUINT uiDOBlkCnt; + FLMUINT uiDOBytesUsed; +} BTREE_LEVEL_STATS; + +typedef struct +{ + FLMUINT uiBlkAddr; + FLMUINT uiBlockSize; + FLMUINT uiBlocksChecked; + FLMUINT uiAvgFreeSpace; + FLMUINT uiLevels; + FLMUINT uiNumKeys; + FLMUINT64 ui64FreeSpace; + BTREE_LEVEL_STATS LevelStats[ BH_MAX_LEVELS]; + char szMsg[ 64]; + BTREE_ERR_TYPE type; +} BTREE_ERR_STRUCT; + +typedef struct +{ + FLMUINT uiParentLevel; + FLMUINT uiParentKeyLen; + FLMUINT uiParentChildBlkAddr; + FLMUINT uiNewKeyLen; + FLMUINT uiChildBlkAddr; + FLMUINT uiCounts; + void * pPrev; + FLMBYTE pucParentKey[ SFLM_MAX_KEY_SIZE]; + FLMBYTE pucNewKey[ SFLM_MAX_KEY_SIZE]; +} BTREE_REPLACE_STRUCT; + +class F_BtPool; + +class F_Btree : public F_Object +{ +public: + + + F_Btree( void); + ~F_Btree( void); + + RCODE btCreate( + F_Db * pDb, + LFILE * pLFile, + FLMBOOL bCounts, + FLMBOOL bData); + + RCODE btOpen( + F_Db * pDb, + LFILE * pLFile, + FLMBOOL bCounts, + FLMBOOL bData, + IF_ResultSetCompare * pCompare = NULL); + + void btClose( void); + + RCODE btDeleteTree( + IF_DeleteStatus * ifpDeleteStatus); + + RCODE btGetBlockChains( + FLMUINT * puiBlockChains, + FLMUINT * puiNumLevels); + + RCODE btRemoveEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen); + + RCODE btInsertEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bFirst, + FLMBOOL bLast, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btReplaceEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bFirst, + FLMBOOL bLast, + FLMBOOL bTruncate = TRUE, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btLocateEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT uiMatch, + FLMUINT * puiPosition = NULL, + FLMUINT * puiDataLength = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btGetEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT uiKeyLen, + FLMBYTE * pucData, + FLMUINT uiDataBufSize, + FLMUINT * puiDataLen); + + RCODE btNextEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + + RCODE btPrevEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btFirstEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btLastEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT * puiDataLength = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE btSetReadPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT uiPosition); + + RCODE btGetReadPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT * puiPosition); + + RCODE btPositionTo( + FLMUINT uiPosition, + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen); + + RCODE btGetPosition( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiPosition); + + RCODE btCheck( + BTREE_ERR_STRUCT * pErrStruct); + + RCODE btRewind( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen); + + FINLINE void btGetTransInfo( + FLMUINT64 * pui64LowTransId, + FLMBOOL * pbMostCurrent) + { + *pui64LowTransId = m_ui64LowTransId; + *pbMostCurrent = m_bMostCurrent; + } + + FINLINE void btRelease( void) + { + releaseBlocks( TRUE); + } + + FINLINE void btResetBtree( void) + { + releaseBlocks( TRUE); + m_bSetupForRead = FALSE; + m_bSetupForWrite = FALSE; + m_bSetupForReplace = FALSE; + m_bOrigInDOBlocks = FALSE; + m_bDataOnlyBlock = FALSE; + m_ui32PrimaryBlkAddr = 0; + m_ui32CurBlkAddr = 0; + m_uiPrimaryOffset = 0; + m_uiCurOffset = 0; + m_uiDataLength = 0; + m_uiPrimaryDataLen = 0; + m_uiOADataLength = 0; + m_uiDataRemaining = 0; + m_uiOADataRemaining = 0; + m_uiOffsetAtStart = 0; + m_ui64CurrTransID = 0; + m_ui64LastBlkTransId = 0; + m_ui64PrimaryBlkTransId = 0; + m_uiBlkChangeCnt = 0; + m_uiSearchLevel = BH_MAX_LEVELS; + } + + RCODE btComputeCounts( + F_Btree * pUntilBtree, + FLMUINT64 * pui64BlkCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness); + + FINLINE void btSetSearchLevel( + FLMUINT uiSearchLevel) + { + flmAssert( uiSearchLevel <= BH_MAX_LEVELS); + + btResetBtree(); + + m_uiSearchLevel = uiSearchLevel; + } + + RCODE btMoveBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr); + + FINLINE FLMBOOL btHasCounts( void) + { + return m_bCounts; + } + + FINLINE FLMBOOL btHasData( void) + { + return m_bData; + } + + FINLINE FLMBOOL btDbIsOpen( void) + { + return m_bOpened; + } + + FINLINE FLMBOOL btIsSetupForRead( void) + { + return m_bSetupForRead; + } + + FINLINE FLMBOOL btIsSetupForWrite( void) + { + return m_bSetupForWrite; + } + + FINLINE FLMBOOL btIsSetupForReplace( void) + { + return m_bSetupForReplace; + } + +private: + + FINLINE FLMUINT calcEntrySize( + FLMUINT uiBlkType, + FLMUINT uiFlags, + FLMUINT uiKeyLen, + FLMUINT uiDataLen, + FLMUINT uiOADataLen) + { + switch( uiBlkType) + { + case BT_LEAF: + { + return( uiKeyLen + 2); + } + + case BT_LEAF_DATA: + { + return( 1 + // Flags + (uiKeyLen > ONE_BYTE_SIZE ? 2 : 1) + // KeyLen + (uiDataLen > ONE_BYTE_SIZE ? 2 : 1) + // DataLen + (uiOADataLen && // OA DataLen + (uiFlags & BTE_FLAG_FIRST_ELEMENT) ? 4 : 0) + + uiKeyLen + uiDataLen); + } + + case BT_NON_LEAF: + case BT_NON_LEAF_COUNTS: + { + return( 4 + // Child block address + (uiBlkType == BT_NON_LEAF_COUNTS ? 4 : 0) + // Counts + 2 + // Key length + uiKeyLen); + } + } + + return( 0); + } + + RCODE computeCounts( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT64 * pui64BlockCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness); + + RCODE blockCounts( + F_BTSK * pStack, + FLMUINT uiFirstOffset, + FLMUINT uiLastOffset, + FLMUINT * puiKeyCount, + FLMUINT * puiElementCount); + + RCODE getStoredCounts( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT64 * pui64BlockCount, + FLMUINT64 * pui64KeyCount, + FLMBOOL * pbTotalsEstimated, + FLMUINT uiAvgBlkFullness); + + RCODE getCacheBlocks( + F_BTSK * pStack1, + F_BTSK * pStack2); + + FINLINE FLMUINT getAvgKeyCount( + F_BTSK * pFromStack, + F_BTSK * pUntilStack, + FLMUINT uiAvgBlkFullness); + + FINLINE void updateTransInfo( + FLMUINT64 ui64LowTransID, + FLMUINT64 ui64HighTransID + ) + { + if (m_ui64LowTransId > ui64LowTransID) + { + m_ui64LowTransId = ui64LowTransID; + } + + if (!m_bMostCurrent) + { + m_bMostCurrent = (ui64HighTransID == FLM_MAX_UINT64) + ? TRUE + : FALSE; + } + } + + FINLINE FLMUINT getBlkEntryCount( + FLMBYTE * pBlk + ) + { + return ((F_BTREE_BLK_HDR *)pBlk)->ui16NumKeys; + } + + FINLINE FLMUINT getBlkAvailSpace( + FLMBYTE * pBlk + ) + { + return ((F_BLK_HDR *)pBlk)->ui16BlkBytesAvail; + } + + FLMUINT getEntryKeyLength( + FLMBYTE * pucEntry, + FLMUINT uiBlockType, + const FLMBYTE ** ppucKeyRV); + + FLMUINT getEntrySize( + FLMBYTE * pBlk, + FLMUINT uiOffset, + FLMBYTE ** ppucEntry = NULL); + + RCODE calcNewEntrySize( + FLMUINT uiKeyLen, + FLMUINT uiDataLen, + FLMUINT * puiEntrySize, + FLMBOOL * pbHaveRoom, + FLMBOOL * pbDefragBlk); + + RCODE extractEntryData( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBYTE * pucBuffer, + FLMUINT uiBufSiz, + FLMUINT * puiDataLen); + + RCODE updateEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + F_ELM_UPD_ACTION eAction, + FLMBOOL bTruncate = TRUE); + + RCODE insertEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction); + + RCODE storeEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + FLMUINT uiEntrySize, + FLMBOOL * pbLastEntry); + + RCODE removeEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + FLMBOOL * pbMoreToRemove, + F_ELM_UPD_ACTION * peAction); + + RCODE remove( + FLMBOOL bDeleteDOBlocks); + + RCODE removeRange( + FLMUINT uiStartElm, + FLMUINT uiEndElm, + FLMBOOL bDeleteDOBlocks); + + RCODE findEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT uiMatch, + FLMUINT * puiPosition = NULL, + FLMUINT32 * pui32BlkAddr = NULL, + FLMUINT * puiOffsetIndex = NULL); + + RCODE findInBlock( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMUINT uiMatch, + FLMUINT * uiPosition, + FLMUINT32 * ui32BlkAddr, + FLMUINT * uiOffsetIndex); + + RCODE scanBlock( + F_BTSK * pStack, + FLMUINT uiMatch); + + RCODE compareKeys( + const FLMBYTE * pucKey1, + FLMUINT uiKeyLen1, + const FLMBYTE * pucKey2, + FLMUINT uiKeyLen2, + FLMINT * piCompare); + + FINLINE RCODE compareBlkKeys( + const FLMBYTE * pucBlockKey, + FLMUINT uiBlockKeyLen, + const FLMBYTE * pucTargetKey, + FLMUINT uiTargetKeyLen, + FLMINT * piCompare) + { + flmAssert( uiBlockKeyLen); + + if( !m_pCompare && uiBlockKeyLen == uiTargetKeyLen) + { + *piCompare = f_memcmp( pucBlockKey, pucTargetKey, uiBlockKeyLen); + + return( NE_SFLM_OK); + } + + return( compareKeys( pucBlockKey, uiBlockKeyLen, + pucTargetKey, uiTargetKeyLen, piCompare)); + } + + RCODE positionToEntry( + FLMUINT uiPosition); + + RCODE searchBlock( + F_BTREE_BLK_HDR * pBlkHdr, + FLMUINT * puiPrevCounts, + FLMUINT uiPosition, + FLMUINT * puiOffset); + + RCODE defragmentBlock( + F_CachedBlock ** ppSCache); + + RCODE advanceToNextElement( + FLMBOOL bAdvanceStack); + + RCODE backupToPrevElement( + FLMBOOL bBackupStack); + + RCODE replaceEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction, + FLMBOOL bTruncate = TRUE); + + RCODE replaceOldEntry( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction, + FLMBOOL bTruncate = TRUE); + + RCODE replaceByInsert( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiDataLen, + FLMUINT uiOADataLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction); + + RCODE replace( + FLMBYTE * pucEntry, + FLMUINT uiEntrySize, + FLMBOOL * pbLastEntry); + + RCODE buildAndStoreEntry( + FLMUINT uiBlkType, + FLMUINT uiFlags, + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMUINT uiOADataLen, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + FLMBYTE * pucBuffer, + FLMUINT uiBufferSize, + FLMUINT * puiEntrySize); + + RCODE moveEntriesToPrevBlk( + FLMUINT uiNewEntrySize, + F_CachedBlock ** ppPrevSCache, + FLMBOOL * pbEntriesWereMoved); + + RCODE moveEntriesToNextBlk( + FLMUINT uiEntrySize, + FLMBOOL * pbEntriesWereMoved); + + RCODE splitBlock( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiOADataLen, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + FLMBOOL * pbBlockSplit); + + RCODE createNewLevel( void); + + RCODE storeDataOnlyBlocks( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL bSaveKey, + const FLMBYTE * pucData, + FLMUINT uiDataLen); + + RCODE replaceDataOnlyBlocks( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL bSaveKey, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBOOL bLast, + FLMBOOL bTruncate = TRUE); + + RCODE moveToPrev( + FLMUINT uiStart, + FLMUINT uiFinish, + F_CachedBlock ** ppPrevSCache); + + RCODE moveToNext( + FLMUINT uiStart, + FLMUINT uiFinish, + F_CachedBlock ** ppNextSCache); + + RCODE updateParentCounts( + F_CachedBlock * pChildSCache, + F_CachedBlock ** ppParentSCache, + FLMUINT uiParentElm); + + FLMUINT countKeys( + FLMBYTE * pBlk); + + FLMUINT countRangeOfKeys( + F_BTSK * pFromStack, + FLMUINT uiFromOffset, + FLMUINT uiUntilOffset); + + RCODE moveStackToPrev( + F_CachedBlock * pPrevSCache); + + RCODE moveStackToNext( + F_CachedBlock * pSCache, + FLMBOOL bReleaseCurrent = TRUE); + + RCODE calcOptimalDataLength( + FLMUINT uiKeyLen, + FLMUINT uiDataLen, + FLMUINT uiBytesAvail, + FLMUINT * puiNewDataLen); + + // Performs an integrity check on a chain of data-only blocks + + RCODE verifyDOBlkChain( + FLMUINT uiDOAddr, + FLMUINT uiDataLength, + BTREE_ERR_STRUCT * localErrStruct); + + // Performs a check to verify that the counts in the DB match. + RCODE verifyCounts( + BTREE_ERR_STRUCT * pErrStruct); + + void releaseBlocks( + FLMBOOL bResetStack); + + void releaseBtree( void); + + RCODE saveReplaceInfo( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen); + + RCODE restoreReplaceInfo( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts); + + FINLINE RCODE setReturnKey( + FLMBYTE * pucEntry, + FLMUINT uiBlockType, + FLMBYTE * pucKey, + FLMUINT * puiKeyLen, + FLMUINT uiKeyBufSize); + + RCODE setupReadState( + F_BLK_HDR * pBlkHdr, + FLMBYTE * pucEntry); + + RCODE removeRemainingEntries( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen); + + RCODE deleteEmptyBlock( void); + + RCODE removeDOBlocks( + FLMUINT32 ui32OrigDOAddr); + + RCODE replaceMultiples( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction); + + RCODE replaceMultiNoTruncate( + const FLMBYTE ** ppucKey, + FLMUINT * puiKeyLen, + const FLMBYTE * pucDataValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT * puiChildBlkAddr, + FLMUINT * puiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + F_ELM_UPD_ACTION * peAction); + + FINLINE RCODE getNextBlock( + F_CachedBlock ** ppSCache); + + FINLINE RCODE getPrevBlock( + F_CachedBlock ** ppSCache); + + FLMBOOL checkContinuedEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL * pbLastElement, + FLMBYTE * pucEntry, + FLMUINT uiBlkType); + + RCODE updateCounts( void); + + RCODE storePartialEntry( + const FLMBYTE * pucKey, + FLMUINT uiKeyLen, + const FLMBYTE * pucValue, + FLMUINT uiLen, + FLMUINT uiFlags, + FLMUINT uiChildBlkAddr, + FLMUINT uiCounts, + const FLMBYTE ** ppucRemainingValue, + FLMUINT * puiRemainingLen, + FLMBOOL bNewBlock = FALSE); + + RCODE mergeBlocks( + FLMBOOL bLastEntry, + FLMBOOL * pbMergedWithPrev, + FLMBOOL * pbMergedWithNext, + F_ELM_UPD_ACTION * peAction); + + RCODE merge( + F_CachedBlock ** ppFromSCache, + F_CachedBlock ** ppToSCache); + + RCODE checkDownLinks( void); + + RCODE verifyChildLinks( + F_CachedBlock * pParentSCache); + + RCODE combineEntries( + F_BTREE_BLK_HDR * pSrcBlkHdr, + FLMUINT uiSrcOffset, + F_BTREE_BLK_HDR * pDstBlkHdr, + FLMUINT uiDstOffset, + FLMBOOL * pbEntriesCombined, + FLMUINT * puiEntrySize); + + RCODE moveBtreeBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr); + + RCODE moveDOBlock( + FLMUINT32 ui32FromBlkAddr, + FLMUINT32 ui32ToBlkAddr); + +// Member variables + FLMBOOL m_bCounts; // BT_NON_LEAF_COUNTS + FLMBOOL m_bData; // BT_LEAF_DATA + FLMBOOL m_bSetupForRead; + FLMBOOL m_bSetupForWrite; + FLMBOOL m_bSetupForReplace; + FLMBOOL m_bOpened; + FLMBOOL m_bMostCurrent; + FLMBOOL m_bDataOnlyBlock; + FLMBOOL m_bOrigInDOBlocks; + FLMBOOL m_bFirstRead; + FLMBOOL m_bStackSetup; + LFILE * m_pLFile; + F_Db * m_pDb; + FLMBOOL m_bTempDb; + F_BTSK * m_pStack; // Used for traversing the B-Tree + FLMBYTE * m_pucTempBlk; + FLMBYTE * m_pucTempDefragBlk; + BTREE_REPLACE_STRUCT * m_pReplaceInfo; + BTREE_REPLACE_STRUCT * m_pReplaceStruct; + const FLMBYTE * m_pucDataPtr; + F_CachedBlock * m_pSCache; + F_BTREE_BLK_HDR * m_pBlkHdr; + FLMBYTE * m_pucBuffer; // Buffer used during moves + FLMUINT m_uiBufferSize; // Size of the buffer + FLMUINT m_uiBlockSize; + FLMUINT m_uiDefragThreshold; + FLMUINT m_uiOverflowThreshold; + FLMUINT m_uiStackLevels; + FLMUINT m_uiRootLevel; + FLMUINT m_uiReplaceLevels; + FLMUINT m_uiBlkChangeCnt; + FLMUINT m_uiDataLength; + FLMUINT m_uiPrimaryDataLen; + FLMUINT m_uiOADataLength; + FLMUINT m_uiDataRemaining; + FLMUINT m_uiOADataRemaining; + FLMUINT m_uiPrimaryOffset; // Offset into primary block + FLMUINT m_uiCurOffset; // Offset into current block + FLMUINT m_uiSearchLevel; + FLMUINT m_uiOffsetAtStart; // Offset into the current + // element at the beginning of + // the entry. An element may + // span multiple entries. + FLMUINT32 m_ui32PrimaryBlkAddr;// Primary block address + FLMUINT32 m_ui32DOBlkAddr; // Address of first DO Block + FLMUINT32 m_ui32CurBlkAddr; // Current block being read + FLMUINT64 m_ui64LowTransId; + FLMUINT64 m_ui64LastBlkTransId; + FLMUINT64 m_ui64PrimaryBlkTransId; + FLMUINT64 m_ui64CurrTransID; + F_BTSK m_Stack[ BH_MAX_LEVELS]; + // The m_pNext field is only used by the btPool object. + F_Btree * m_pNext; + IF_ResultSetCompare * m_pCompare; + +friend class F_BtPool; +friend class F_Db; +friend class F_Rfl; +}; + +RCODE btFreeBlockChain( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiStartAddr, + FLMUINT uiBlocksToFree, + FLMUINT * puiBlocksFreed, + FLMUINT * puiEndAddr, + IF_DeleteStatus * ifpDeleteStatus); + +#endif diff --git a/sql/src/f_nici.cpp b/sql/src/f_nici.cpp new file mode 100644 index 0000000..2b2687f --- /dev/null +++ b/sql/src/f_nici.cpp @@ -0,0 +1,2991 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the functions needed for the NICI interface +// functions. Adapted from ss_crypto.c written by Cameron Mashayekhi. +// +// Tabs: 3 +// +// Copyright (c) 2004-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_nici.cpp 3111 2006-01-19 13:10:50 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#ifdef FLM_USE_NICI +FSTATIC void GetIV( + FLMBYTE * pucIV, + FLMUINT uiLen); +#endif + +/*----------------------------------------------------------------------------- + * Desc: DTOR - Destroy an F_CCS object. + *---------------------------------------------------------------------------*/ +F_CCS::~F_CCS() +{ +#ifdef FLM_USE_NICI + if( m_keyHandle) + { + if( !m_hContext) + { + if( RC_BAD( CCS_CreateContext(0, &m_hContext))) + { + flmAssert( 0); + } + } + + // Get rid of the key handle. + + if ( m_hContext) + { + CCS_DestroyObject( m_hContext, + m_keyHandle); + + CCS_DestroyContext( m_hContext); + } + } + + if (m_hMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hMutex); + } +#endif +} + +/*----------------------------------------------------------------------------- + * Desc: wrapNiciKey - Save the wrapped key in m_pKey. NOTE: Make sure + * there is a buffer allocated for the wrapped key (m_pucWrappedKey). + *---------------------------------------------------------------------------*/ +RCODE F_CCS::wrapKey( + FLMBYTE ** ppucWrappedKey, + FLMUINT32 * pui32Length, + NICI_OBJECT_HANDLE masterWrappingKey) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( ppucWrappedKey); + F_UNREFERENCED_PARM( pui32Length); + F_UNREFERENCED_PARM( masterWrappingKey); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ATTRIBUTE wKey[2]; + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_aes128[] = {IDV_NOV_AES128CBCPad}; + FLMBYTE oid_aes192[] = {IDV_NOV_AES192CBCPad}; + FLMBYTE oid_aes256[] = {IDV_NOV_AES256CBCPad}; + FLMBYTE oid_3des[] = {IDV_DES_EDE3_CBCPadIV8}; + NICI_OBJECT_HANDLE wrappingKeyHandle; + FLMBOOL bLocked = FALSE; + + if (masterWrappingKey) + { + wrappingKeyHandle = masterWrappingKey; + } + else + { + if (RC_BAD( rc = getWrappingKey( &wrappingKeyHandle))) + { + goto Exit; + } + } + + f_mutexLock( m_hMutex); + bLocked = TRUE; + + /* Create NICI Context */ + if ( !m_hContext) + { + + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + f_memset( &wKey, 0, sizeof(NICI_ATTRIBUTE) * 2); + + wKey[0].type = NICI_A_KEY_TYPE; + wKey[1].type = NICI_A_KEY_SIZE; + + if (RC_BAD( rc = CCS_GetAttributeValue( + m_hContext, + wrappingKeyHandle, + &wKey[0], + 2))) + { + rc = RC_SET( NE_SFLM_NICI_ATTRIBUTE_VALUE); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (!wKey[0].u.f.hasValue || !wKey[1].u.f.hasValue) + { + rc = RC_SET( NE_SFLM_NICI_BAD_ATTRIBUTE); + goto Exit; + } + + switch (wKey[0].u.f.value) + { + case NICI_K_AES: + { + switch (wKey[1].u.f.value) + { + case SFLM_AES128_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes128; + break; + } + case SFLM_AES192_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes192; + break; + } + case SFLM_AES256_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes256; + break; + } + default: + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + } + algorithm.parameter = parm; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + algorithm.parameter->parms[0].u.b.len = IV_SZ; /* 16-byte IV */ + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + break; + } + + case NICI_K_DES3X: + { + algorithm.algorithm = (nuint8 *)oid_3des; + algorithm.parameter = parm; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + algorithm.parameter->parms[0].u.b.len = IV_SZ8; /* 8-byte IV */ + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + break; + } + + default: + { + rc = RC_SET( NE_SFLM_NICI_WRAPKEY_FAILED); + goto Exit; + } + } + + // We should be able to call this with NULL for the wrapped key, to get the length. + + if (RC_BAD( rc = CCS_WrapKey( + m_hContext, + &algorithm, + NICI_KM_UNSPECIFIED, + 0, + wrappingKeyHandle, + m_keyHandle, + (nuint8 *)NULL, + (pnuint32)pui32Length))) + { + rc = RC_SET( NE_SFLM_NICI_WRAPKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = f_calloc( *pui32Length, ppucWrappedKey))) + { + goto Exit; + } + + + if (RC_BAD( rc = CCS_WrapKey( + m_hContext, + &algorithm, + NICI_KM_UNSPECIFIED, + 0, + wrappingKeyHandle, + m_keyHandle, + (nuint8 *)*ppucWrappedKey, + (pnuint32)pui32Length))) + { + rc = RC_SET( NE_SFLM_NICI_WRAPKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (bLocked) + { + f_mutexUnlock( m_hMutex); + } + +#endif + + return(rc); +} + + +/*----------------------------------------------------------------------------- + * Desc: - unwrapKey + *---------------------------------------------------------------------------*/ +RCODE F_CCS::unwrapKey( + FLMBYTE * pucWrappedKey, + FLMUINT32 ui32WrappedKeyLength, + NICI_OBJECT_HANDLE masterWrappingKey) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucWrappedKey); + F_UNREFERENCED_PARM( ui32WrappedKeyLength); + F_UNREFERENCED_PARM( masterWrappingKey); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ATTRIBUTE wKey; + NICI_OBJECT_HANDLE wrappingKeyHandle; + FLMBOOL bLocked = FALSE; + + if (masterWrappingKey) + { + wrappingKeyHandle = masterWrappingKey; + } + else + { + if (RC_BAD( rc = getWrappingKey( &wrappingKeyHandle))) + { + goto Exit; + } + } + + f_mutexLock( m_hMutex); + bLocked = TRUE; + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + if (RC_BAD( rc = CCS_UnwrapKey( + m_hContext, + wrappingKeyHandle, + (nuint8 *)pucWrappedKey, + ui32WrappedKeyLength, + &m_keyHandle))) + { + rc = RC_SET( NE_SFLM_NICI_UNWRAPKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // We need to get the key size... + + f_memset( &wKey, 0, sizeof(NICI_ATTRIBUTE)); + + wKey.type = NICI_A_KEY_SIZE; + + if (RC_BAD( rc = CCS_GetAttributeValue( + m_hContext, + m_keyHandle, + &wKey, + 1))) + { + rc = RC_SET( NE_SFLM_NICI_ATTRIBUTE_VALUE); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (!wKey.u.f.hasValue) + { + rc = RC_SET( NE_SFLM_NICI_BAD_ATTRIBUTE); + goto Exit; + } + + m_uiEncKeySize = wKey.u.f.value; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (bLocked) + { + f_mutexUnlock( m_hMutex); + } + +#endif + + return(rc); +} + +/*----------------------------------------------------------------------------- + * Desc: generateEncryptionKey + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateEncryptionKey( + FLMUINT uiEncKeySize + ) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + + switch( m_eEncAlgorithm) + { + case SFLM_AES_ENCRYPTION: + { + rc = generateEncryptionKeyAES( uiEncKeySize); + break; + } + case SFLM_DES3_ENCRYPTION: + { + rc = generateEncryptionKeyDES3( uiEncKeySize); + break; + } + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + +#endif + +Exit: + + return rc; + +} + +/*----------------------------------------------------------------------------- + * Desc: generateEncryptionKey + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateEncryptionKeyAES( + FLMUINT uiEncKeySize + ) +{ + RCODE rc = NE_SFLM_OK; + + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[3]; + nbool8 keySizeChanged; + FLMBYTE oid_aes128[] = {IDV_AES128CBC}; + FLMBYTE oid_aes192[] = {IDV_AES192CBC}; + FLMBYTE oid_aes256[] = {IDV_AES256CBC}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + /* Set up AES Algorithm*/ + switch (uiEncKeySize) + { + case SFLM_AES128_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes128; + break; + } + case SFLM_AES192_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes192; + break; + } + case SFLM_AES256_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes256; + break; + } + default: + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + } + algorithm.parameterLen = 0; + algorithm.parameter = NULL; + + /* Set up key attributes */ + keyAttr[0].type = NICI_A_KEY_USAGE; + keyAttr[0].u.f.hasValue = 1; + keyAttr[0].u.f.value = NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT | NICI_F_EXTRACT; + keyAttr[0].u.f.valueInfo = 0; + + keyAttr[1].type = NICI_A_KEY_SIZE; + keyAttr[1].u.f.hasValue = 1; + keyAttr[1].u.f.value = uiEncKeySize; + keyAttr[1].u.f.valueInfo = 0; + + keyAttr[2].type = NICI_A_GLOBAL; + keyAttr[2].u.f.hasValue = 1; + keyAttr[2].u.f.value = N_TRUE; + keyAttr[2].u.f.valueInfo = 0; + + /*Generate a AES key */ + + if (RC_BAD( rc = CCS_GenerateKey( + m_hContext, + &algorithm, + keyAttr, + 3, + &keySizeChanged, + &m_keyHandle, + NICI_H_INVALID))) + { + rc = RC_SET( NE_SFLM_NICI_GENKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate some IV to use with this key. + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)m_ucIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + m_uiEncKeySize = uiEncKeySize; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + +/*----------------------------------------------------------------------------- + * Desc: generateEncryptionKey - DES3 + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateEncryptionKeyDES3( + FLMUINT uiEncKeySize + ) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[3]; + nbool8 keySizeChanged; + FLMBYTE oid_des3[] = {IDV_DES_EDE3_CBC_IV8}; + + f_mutexLock( m_hMutex); + + // Only one DES3 key size supported. + + if (uiEncKeySize != SFLM_DES3_168_KEY_SIZE) + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + + /* Create NICI Context */ + if ( !m_hContext) + { + + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + /* Set up AES Algorithm*/ + algorithm.algorithm = (nuint8 *)oid_des3; + algorithm.parameterLen = 0; + algorithm.parameter = NULL; + + /* Set up key attributes */ + keyAttr[0].type = NICI_A_KEY_USAGE; + keyAttr[0].u.f.hasValue = 1; + keyAttr[0].u.f.value = NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT | NICI_F_EXTRACT; + keyAttr[0].u.f.valueInfo = 0; + + keyAttr[1].type = NICI_A_KEY_SIZE; + keyAttr[1].u.f.hasValue = 1; + keyAttr[1].u.f.value = uiEncKeySize; + keyAttr[1].u.f.valueInfo = 0; + + keyAttr[2].type = NICI_A_GLOBAL; + keyAttr[2].u.f.hasValue = 1; + keyAttr[2].u.f.value = N_TRUE; + keyAttr[2].u.f.valueInfo = 0; + + /*Generate a AES key */ + + if (RC_BAD( rc = CCS_GenerateKey( + m_hContext, + &algorithm, + keyAttr, + 3, + &keySizeChanged, + &m_keyHandle, + NICI_H_INVALID))) + { + rc = RC_SET( NE_SFLM_NICI_GENKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate some IV to use with this key. + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)m_ucIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + m_uiEncKeySize = uiEncKeySize; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + +/*----------------------------------------------------------------------------- + * Desc: generateWrappingKey + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateWrappingKey( + FLMUINT uiEncKeySize + ) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + + switch( m_eEncAlgorithm) + { + case SFLM_AES_ENCRYPTION: + { + rc = generateWrappingKeyAES( uiEncKeySize); + break; + } + case SFLM_DES3_ENCRYPTION: + { + rc = generateWrappingKeyDES3( uiEncKeySize); + break; + } + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + +#endif + +Exit: + + return rc; + +} + +/*----------------------------------------------------------------------------- + * Desc: generateWrappingKeyAES - generates an AES wrapping key + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateWrappingKeyAES( + FLMUINT uiEncKeySize) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[6]; + nbool8 keySizeChanged; + FLMBYTE oid_aes128[] = {IDV_AES128CBC}; + FLMBYTE oid_aes192[] = {IDV_AES192CBC}; + FLMBYTE oid_aes256[] = {IDV_AES256CBC}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + /* Set up AES Algorithm*/ + switch (uiEncKeySize) + { + case SFLM_AES128_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes128; + keyAttr[1].u.v.valuePtr = oid_aes128; + keyAttr[1].u.v.valueLen = (nuint32)sizeof( oid_aes128); + break; + } + case SFLM_AES192_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes192; + keyAttr[1].u.v.valuePtr = oid_aes192; + keyAttr[1].u.v.valueLen = (nuint32)sizeof( oid_aes192); + break; + } + case SFLM_AES256_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes256; + keyAttr[1].u.v.valuePtr = oid_aes256; + keyAttr[1].u.v.valueLen = (nuint32)sizeof( oid_aes256); + break; + } + default: + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + } + algorithm.parameterLen = 0; + algorithm.parameter = NULL; + + /* Set up key attributes */ + keyAttr[0].type = NICI_A_KEY_TYPE; + keyAttr[0].u.f.hasValue = 1; + keyAttr[0].u.f.value = NICI_K_AES; + keyAttr[0].u.f.valueInfo = 0; + + keyAttr[1].type = NICI_A_KEY_FORMAT; + keyAttr[1].u.v.valueInfo = 0; + + keyAttr[2].type = NICI_A_KEY_USAGE; + keyAttr[2].u.f.hasValue = 1; + keyAttr[2].u.f.value = NICI_F_WRAP | NICI_F_UNWRAP | NICI_F_KM_ENCRYPT | NICI_F_KM_DECRYPT | NICI_F_EXTRACT | NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + keyAttr[2].u.f.valueInfo = 0; + + keyAttr[3].type = NICI_A_KEY_SIZE; + keyAttr[3].u.f.hasValue = 1; + keyAttr[3].u.f.value = uiEncKeySize; + keyAttr[3].u.f.valueInfo = 0; + + keyAttr[4].type = NICI_A_GLOBAL; + keyAttr[4].u.f.hasValue = 1; + keyAttr[4].u.f.value = N_TRUE; + keyAttr[4].u.f.valueInfo = 0; + + keyAttr[5].type = NICI_A_CLASS; + keyAttr[5].u.f.hasValue = 1; + keyAttr[5].u.f.value = NICI_O_SECRET_KEY; + keyAttr[5].u.f.valueInfo = 0; + + /*Generate an AES wrapping key */ + + if (RC_BAD( rc = CCS_GenerateKey( + m_hContext, + &algorithm, + keyAttr, + 6, + &keySizeChanged, + &m_keyHandle, + NICI_H_INVALID))) + { + rc = RC_SET( NE_SFLM_NICI_GENKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate some IV to use with this key. + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)m_ucIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // If we generated a wrapping key, then this object's key handle is actually a + // wrapping key. This means that we will use it to wrap the other keys in the + // system. + + m_bKeyIsWrappingKey = TRUE; + m_uiEncKeySize = uiEncKeySize; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + + +/*----------------------------------------------------------------------------- + * Desc: generateWrappingKeyDES3 - generates a triple DES (DES3) wrapping key + *---------------------------------------------------------------------------*/ +RCODE F_CCS::generateWrappingKeyDES3( + FLMUINT uiEncKeySize) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiEncKeySize); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[6]; + nbool8 keySizeChanged; + FLMBYTE oid_des3[] = {IDV_DES_EDE3_CBC_IV8}; + + f_mutexLock( m_hMutex); + + if (uiEncKeySize != SFLM_DES3_168_KEY_SIZE) + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + + /* Create NICI Context */ + if ( !m_hContext) + { + + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + /* Set up AES Algorithm*/ + algorithm.algorithm = (nuint8 *)oid_des3; + algorithm.parameterLen = 0; + algorithm.parameter = NULL; + + /* Set up key attributes */ + + keyAttr[0].type = NICI_A_KEY_TYPE; + keyAttr[0].u.f.hasValue = 1; + keyAttr[0].u.f.value = NICI_K_DES3X; + keyAttr[0].u.f.valueInfo = 0; + + keyAttr[1].type = NICI_A_KEY_FORMAT; + keyAttr[1].u.v.valuePtr = oid_des3; + keyAttr[1].u.v.valueLen = (nuint32)sizeof( oid_des3); + keyAttr[1].u.v.valueInfo = 0; + + keyAttr[2].type = NICI_A_KEY_USAGE; + keyAttr[2].u.f.hasValue = 1; + keyAttr[2].u.f.value = NICI_F_WRAP | NICI_F_UNWRAP | NICI_F_KM_ENCRYPT | NICI_F_KM_DECRYPT | NICI_F_EXTRACT | NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + keyAttr[2].u.f.valueInfo = 0; + + keyAttr[3].type = NICI_A_KEY_SIZE; + keyAttr[3].u.f.hasValue = 1; + keyAttr[3].u.f.value = uiEncKeySize; + keyAttr[3].u.f.valueInfo = 0; + + keyAttr[4].type = NICI_A_GLOBAL; + keyAttr[4].u.f.hasValue = 1; + keyAttr[4].u.f.value = N_TRUE; + keyAttr[4].u.f.valueInfo = 0; + + keyAttr[5].type = NICI_A_CLASS; + keyAttr[5].u.f.hasValue = 1; + keyAttr[5].u.f.value = NICI_O_SECRET_KEY; + keyAttr[5].u.f.valueInfo = 0; + + /*Generate an AES wrapping key */ + + if (RC_BAD( rc = CCS_GenerateKey( + m_hContext, + &algorithm, + keyAttr, + 6, + &keySizeChanged, + &m_keyHandle, + NICI_H_INVALID))) + { + rc = RC_SET( NE_SFLM_NICI_GENKEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate some IV to use with this key. + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)m_ucIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // If we generated a wrapping key, then this object's key handle is actually a + // wrapping key. This means that we will use it to wrap the other keys in the + // system. + m_bKeyIsWrappingKey = TRUE; + m_uiEncKeySize = uiEncKeySize; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + +/*----------------------------------------------------------------------------- + * Desc: encryptToStore (public) + *---------------------------------------------------------------------------*/ +RCODE F_CCS::encryptToStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + + switch (m_eEncAlgorithm) + { + case SFLM_AES_ENCRYPTION: + { + rc = encryptToStoreAES( + pucIn, + uiInLen, + pucOut, + puiOutLen, + pucIV); + break; + } + case SFLM_DES3_ENCRYPTION: + { + rc = encryptToStoreDES3( + pucIn, + uiInLen, + pucOut, + puiOutLen, + pucIV); + break; + } + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + +#endif + +Exit: + + return rc; + +} + +/*----------------------------------------------------------------------------- + * Desc: decryptFromStore (public) + *---------------------------------------------------------------------------*/ +RCODE F_CCS::decryptFromStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + + switch( m_eEncAlgorithm) + { + case SFLM_AES_ENCRYPTION: + { + rc = decryptFromStoreAES( + pucIn, + uiInLen, + pucOut, + puiOutLen, + pucIV); + break; + } + case SFLM_DES3_ENCRYPTION: + { + rc = decryptFromStoreDES3( + pucIn, + uiInLen, + pucOut, + puiOutLen, + pucIV); + break; + } + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + +#endif + +Exit: + + return rc; + +} + +/*----------------------------------------------------------------------------- + * Desc: encryptToStore - Using AES + *---------------------------------------------------------------------------*/ +RCODE F_CCS::encryptToStoreAES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_aes128[] = {IDV_AES128CBC}; + FLMBYTE oid_aes192[] = {IDV_AES192CBC}; + FLMBYTE oid_aes256[] = {IDV_AES256CBC}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context*/ + if ( !m_hContext) + { + + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + switch (m_uiEncKeySize) + { + case SFLM_AES128_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes128; + break; + } + case SFLM_AES192_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes192; + break; + } + case SFLM_AES256_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes256; + break; + } + default: + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + } + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + + if (pucIV) + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + } + else + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + } + + algorithm.parameter->parms[0].u.b.len = IV_SZ; + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataEncryptInit( m_hContext, &algorithm, m_keyHandle))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NICI_ENC_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Encrypt( + m_hContext, + (nuint8 *)pucIn, + uiInLen, + (nuint8 *)pucOut, + puiOutLen))) + { + rc = RC_SET( NE_SFLM_NICI_ENCRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + + +/*----------------------------------------------------------------------------- + * Desc: decryptFromStore - using the AES algorithm + *---------------------------------------------------------------------------*/ +RCODE F_CCS::decryptFromStoreAES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_aes128[] = {IDV_AES128CBC}; + FLMBYTE oid_aes192[] = {IDV_AES192CBC}; + FLMBYTE oid_aes256[] = {IDV_AES256CBC}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + switch (m_uiEncKeySize) + { + case SFLM_AES128_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes128; + break; + } + case SFLM_AES192_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes192; + break; + } + case SFLM_AES256_KEY_SIZE: + { + algorithm.algorithm = (nuint8 *)oid_aes256; + break; + } + default: + { + rc = RC_SET( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + } + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + + if (pucIV) + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + } + else + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + } + + algorithm.parameter->parms[0].u.b.len = IV_SZ; /* 16-byte IV */ + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataDecryptInit( + m_hContext, + &algorithm, + m_keyHandle))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Decrypt( + m_hContext, + (nuint8 *)pucIn, + uiInLen, + (nuint8 *)pucOut, + puiOutLen))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + + +/*----------------------------------------------------------------------------- + * Desc: encryptToStore - Using DES3 + *---------------------------------------------------------------------------*/ +RCODE F_CCS::encryptToStoreDES3( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_des3[] = {IDV_DES_EDE3_CBC_IV8}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context*/ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + algorithm.algorithm = (nuint8 *)oid_des3; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + + if (pucIV) + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + } + else + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + } + + algorithm.parameter->parms[0].u.b.len = IV_SZ8; /* 8-byte IV */ + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataEncryptInit(m_hContext, &algorithm, m_keyHandle))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NICI_ENC_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Encrypt( + m_hContext, + (nuint8 *)pucIn, + uiInLen, + (nuint8 *)pucOut, + puiOutLen))) + { + rc = RC_SET( NE_SFLM_NICI_ENCRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + + +/*----------------------------------------------------------------------------- + * Desc: decryptFromStore - using the Triple DES (DES3) algorithm + *---------------------------------------------------------------------------*/ +RCODE F_CCS::decryptFromStoreDES3( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucIn); + F_UNREFERENCED_PARM( uiInLen); + F_UNREFERENCED_PARM( pucOut); + F_UNREFERENCED_PARM( puiOutLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_des3[] = {IDV_DES_EDE3_CBC_IV8}; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + /*Set up alogrithm now to do triple des decryption */ + algorithm.algorithm = (nuint8 *)oid_des3; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + + if (pucIV) + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + } + else + { + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)m_ucIV; + } + + algorithm.parameter->parms[0].u.b.len = IV_SZ8; /* 8-byte IV */ + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataDecryptInit( + m_hContext, + &algorithm, + m_keyHandle))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Decrypt( + m_hContext, + (nuint8 *)pucIn, + uiInLen, + (nuint8 *)pucOut, + puiOutLen))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); + +} + +/*----------------------------------------------------------------------------- + * Desc: init - Initialize the context. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::init( + FLMBOOL bKeyIsWrappingKey, + eEncAlgorithm eEncAlg) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( bKeyIsWrappingKey); + F_UNREFERENCED_PARM( eEncAlg); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + FLMBOOL bLocked = FALSE; + + if (m_bInitCalled) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + m_bKeyIsWrappingKey = bKeyIsWrappingKey; + + if (eEncAlg != SFLM_AES_ENCRYPTION && + eEncAlg != SFLM_DES3_ENCRYPTION) + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_INVALID_ENC_ALGORITHM); + goto Exit; + } + + m_eEncAlgorithm = eEncAlg; + + // Create a mutex to control access to the nici operations. + if (RC_BAD( rc = f_mutexCreate( &m_hMutex))) + { + goto Exit; + } + + f_mutexLock( m_hMutex); + bLocked = TRUE; + + // Create NICI Context + + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + else + { + flmAssert( 0); // Should not have a context yet! + } + + // Generate the Random IV + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)&m_ucRndIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate an adjustment factor for the IV + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)&m_uiIVFactor, + sizeof(FLMUINT)))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + + m_bInitCalled = TRUE; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (bLocked) + { + f_mutexUnlock( m_hMutex); + } +#endif + + return rc; + +} + +/*----------------------------------------------------------------------------- + * Desc: selectWrappingKey - pick a wrapping key that we can use to wrap & + * unwrap the encryption key with. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::getWrappingKey( + NICI_OBJECT_HANDLE * pWrappingKeyHandle) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pWrappingKeyHandle); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ATTRIBUTE find[2]; + FLMUINT uiCount; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + find[0].type = NICI_A_GLOBAL; + find[0].u.f.hasValue = 1; + find[0].u.f.value = 1; + find[0].u.f.valueInfo = 0; + + find[1].type = NICI_A_FEATURE; + find[1].u.f.hasValue = 1; + find[1].u.f.value = NICI_AV_STORAGE; + find[1].u.f.valueInfo = 0; + + if (RC_BAD( rc = CCS_FindObjectsInit(m_hContext, find, 2))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_INIT); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + uiCount = 1; + + if (RC_BAD( rc = CCS_FindObjects( + m_hContext, + pWrappingKeyHandle, + &uiCount))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_OBJECT); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (uiCount < 1) + { + rc = RC_SET( NE_SFLM_NICI_WRAPKEY_NOT_FOUND); + goto Exit; + } + +#endif + +Exit: + +#ifdef FLM_USE_NICI + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); +} + + +/*----------------------------------------------------------------------------- + * Desc: getKeyToStore - Function used to obtain the key information in the + * format that will be stored on disk. A buffer will be allocated by this + * function that **MUST** be freed when no longer needed. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::getKeyToStore( + FLMBYTE ** ppucKeyInfo, + FLMUINT32 * pui32BufLen, + FLMBYTE * pszEncKeyPasswd, + F_CCS * pWrappingCcs) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( ppucKeyInfo); + F_UNREFERENCED_PARM( pui32BufLen); + F_UNREFERENCED_PARM( pszEncKeyPasswd); + F_UNREFERENCED_PARM( pWrappingCcs); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + FLMBYTE * pucTmp = NULL; + FLMBYTE * pucPtr = NULL; + FLMUINT32 ui32PaddedLength; + FLMBYTE * pucWrappedKey = NULL; + FLMUINT32 ui32WrappedKeyLen = 0; + FLMBYTE * pszFormattedEncKeyPasswd = NULL; + NICI_OBJECT_HANDLE wrappingKeyHandle = 0; + + *ppucKeyInfo = NULL; + + *pui32BufLen = 0; + + if (pWrappingCcs) + { + flmAssert(m_bKeyIsWrappingKey == FALSE); + wrappingKeyHandle = pWrappingCcs->m_keyHandle; + } + else if (!pszEncKeyPasswd) + { + flmAssert( m_bKeyIsWrappingKey); + } + + // Either extract the key or wrap the key. + if (pszEncKeyPasswd && pszEncKeyPasswd[0]) + { + // The password that is passed in to CCS_pbeEncrypt is NOT actually + // unicode. It must be treated as a sequence of bytes that that is + // terminated with 2 nulls and has an even length. If we treat it + // as unicode, then we'll have endian issues if we move the database + // to machines with different byte ordering. + if (RC_BAD( rc = f_calloc( f_strlen(pszEncKeyPasswd) + + (f_strlen(pszEncKeyPasswd) % 2) + 2, + &pszFormattedEncKeyPasswd))) + { + goto Exit; + } + f_strcpy( pszFormattedEncKeyPasswd, pszEncKeyPasswd); + + if (RC_BAD( rc = extractKey( &pucWrappedKey, + &ui32WrappedKeyLen, + (FLMUNICODE *)pszFormattedEncKeyPasswd))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = wrapKey( &pucWrappedKey, + &ui32WrappedKeyLen, + wrappingKeyHandle))) + { + goto Exit; + } + } + + // The shrouded or wrapped key will be stored in m_pKey. + ui32PaddedLength = (ui32WrappedKeyLen + + sizeof( FLMBOOL) + + sizeof (FLMUINT32) + + IV_SZ ); + + // Make sure our buffer size is padded to a 16 byte boundary. + if ((ui32PaddedLength % 16) != 0) + { + ui32PaddedLength += (16 - (ui32PaddedLength % 16)); + } + + // Add one extra byte for a NULL terminator + if (RC_BAD(rc = f_alloc( ui32PaddedLength + 1, &pucTmp))) + { + goto Exit; + } + + if ( !m_hContext) + { + if (CCS_CreateContext( 0, &m_hContext)) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + pucPtr = pucTmp; + + // Save a flag indicating whether the key is wrapped or encoded in + // a password. + UD2FBA( (pszEncKeyPasswd && pszEncKeyPasswd[0]) ? (FLMUINT)TRUE : (FLMUINT)FALSE, pucPtr); + pucPtr += sizeof(FLMBOOL); + + // Copy the key length. + UD2FBA(ui32WrappedKeyLen, pucPtr); + pucPtr += sizeof(FLMUINT32); + + // Copy the IV too. + f_memcpy( pucPtr, m_ucIV, IV_SZ); + pucPtr += IV_SZ; + + // Copy the wrapped key value + f_memcpy( pucPtr, pucWrappedKey, ui32WrappedKeyLen); + pucPtr += ui32WrappedKeyLen; + + // Fill the remainder of the buffer with random data. + if (CCS_GetRandom(m_hContext, + (nuint8 *)pucPtr, + ((FLMUINT)pucTmp + ui32PaddedLength) - (FLMUINT)pucPtr)) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + pucTmp[ ui32PaddedLength] = '\0'; + *ppucKeyInfo = pucTmp; + *pui32BufLen = ui32PaddedLength; + pucTmp = NULL; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (pucTmp) + { + f_free(&pucTmp); + } + + if (pucWrappedKey) + { + f_free( &pucWrappedKey); + } + + if (pszFormattedEncKeyPasswd) + { + f_free( &pszFormattedEncKeyPasswd); + } +#endif + + return rc; +} + +/*----------------------------------------------------------------------------- + * Desc: setKeyFromStore - Function used to set the key info using the binary + * key stored on the disk. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::setKeyFromStore( + FLMBYTE * pucKeyInfo, + FLMBYTE * pszEncKeyPasswd, + F_CCS * pWrappingCcs) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucKeyInfo); + F_UNREFERENCED_PARM( pszEncKeyPasswd); + F_UNREFERENCED_PARM( pWrappingCcs); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + FLMBYTE * pucTmp = pucKeyInfo; + FLMBYTE * pucBuffer = NULL; + FLMBOOL bShrouded = FALSE; + FLMUINT32 ui32Length; + FLMBYTE * pucKeyBuf = NULL; + FLMBYTE * pszFormattedEncKeyPasswd = NULL; + NICI_OBJECT_HANDLE wrappingKeyHandle = 0; + + if (pWrappingCcs) + { + flmAssert(m_bKeyIsWrappingKey == FALSE); + wrappingKeyHandle = pWrappingCcs->m_keyHandle; + } + + // Extract the fields from the buffer + // Is the key shrouded? + bShrouded = FB2UD( pucTmp); + pucTmp += sizeof(FLMUINT); + + // Actual length - note that the passed buffer is padded to 16 byte boundary. + ui32Length = FB2UD( pucTmp); + pucTmp += sizeof(FLMUINT32); + + // Get the IV + f_memcpy( m_ucIV, pucTmp, IV_SZ); + pucTmp += IV_SZ; + + // Need another temporary buffer to hold the encrypted / shrouded key. + if (RC_BAD( rc = f_alloc( ui32Length, &pucBuffer))) + { + goto Exit; + } + + f_memcpy( pucBuffer, pucTmp, ui32Length); + + if (bShrouded) + { + if (pszEncKeyPasswd == NULL || pszEncKeyPasswd[0] == '\0') + { + rc = RC_SET( NE_SFLM_EXPECTING_PASSWORD); + goto Exit; + } + + // The password that is passed in to CCS_pbeDecrypt is NOT actually + // unicode. It must be treated as a sequence of bytes that that is + // terminated with 2 nulls and has an even length. If we treat it + // as unicode, then we'll have endian issues if we move the database + // to machines with different byte ordering. + if (RC_BAD( rc = f_calloc( f_strlen(pszEncKeyPasswd) + + (f_strlen(pszEncKeyPasswd) % 2) + 2, + &pszFormattedEncKeyPasswd))) + { + goto Exit; + } + f_strcpy( pszFormattedEncKeyPasswd, pszEncKeyPasswd); + + // Unshroud the key using the password. + // Key handle is always kept in m_keyHandle. + if (RC_BAD( rc = injectKey( pucBuffer, + ui32Length, + (FLMUNICODE *)pszFormattedEncKeyPasswd))) + { + goto Exit; + } + } + else + { + if (pszEncKeyPasswd) + { + if ( pszEncKeyPasswd[0] != '\0') + { + rc = RC_SET( NE_SFLM_NOT_EXPECTING_PASSWORD); + goto Exit; + } + } + + // Unwrap the key. The Key handle is always store in m_keyHandle. + if (RC_BAD( rc = unwrapKey( pucBuffer, + ui32Length, + wrappingKeyHandle))) + { + goto Exit; + } + } + + m_bKeyVerified = TRUE; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (pucBuffer) + { + f_free( &pucBuffer); + } + + if (pucKeyBuf) + { + f_free( &pucKeyBuf); + } + + if (pszFormattedEncKeyPasswd) + { + f_free( &pszFormattedEncKeyPasswd); + } + +#endif + + return rc; + +} + +typedef struct +{ + FLMUINT uiKeyType; + FLMUINT uiFormatLen; + FLMUINT uiKeyLen; + FLMUINT uiKeySize; +} EXTRACTED_KEY; + +/*----------------------------------------------------------------------------- + * Desc: extractKey - Extract the key by encrypting it in a supplied password. The + * buffer ppucExtractedKey buffer is allocated and returned, thus *MUST* be released + * after it is no longer needed. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::extractKey( + FLMBYTE ** ppucExtractedKey, + FLMUINT32 * pui32Length, + FLMUNICODE * puzEncKeyPasswd) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( ppucExtractedKey); + F_UNREFERENCED_PARM( pui32Length); + F_UNREFERENCED_PARM( puzEncKeyPasswd); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[2]; + NICI_ATTRIBUTE attr[2]; + FLMBYTE oid_sha1[] = {IDV_SHA1}; + FLMBYTE oid_pbe[] = {IDV_pbeWithSHA1And3Key3xDES_CBC}; + FLMBYTE ucDigest[ 20]; + FLMUINT uiDigestLen = sizeof(ucDigest); + FLMUINT uiBufferSize; + FLMBYTE * pucKey = NULL; + FLMBYTE * pucFormat = NULL; + EXTRACTED_KEY * pExtractedKey = NULL; + FLMUINT uiEncLen; + FLMBYTE * pTemp = NULL; + NICI_PARAMETER_INFO * pParmInfo; + FLMBYTE * pucSalt; + FLMUINT uiAllocSize; + FLMUINT uiIndx; + FLMBYTE * pucTempPtr; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + f_memset( &attr[0], 0, sizeof(NICI_ATTRIBUTE) * 2); + attr[0].type = NICI_A_KEY_TYPE; + attr[1].type = NICI_A_KEY_FORMAT; + + if (RC_BAD( rc = CCS_GetAttributeValue( + m_hContext, + m_keyHandle, + &attr[0], + 2))) + { + rc = RC_SET( NE_SFLM_NICI_ATTRIBUTE_VALUE); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (!attr[0].u.f.hasValue) + { + rc = RC_SET( NE_SFLM_NICI_BAD_ATTRIBUTE); + goto Exit; + } + + f_memset( &keyAttr[0], 0, sizeof(NICI_ATTRIBUTE) * 2); + + switch (attr[0].u.f.value) + { + case NICI_K_AES: + { + uiIndx = 0; + keyAttr[uiIndx].type = NICI_A_KEY_VALUE; + switch (m_uiEncKeySize) + { + case SFLM_AES128_KEY_SIZE: + { + keyAttr[uiIndx].u.v.valueLen = 16; + break; + } + case SFLM_AES192_KEY_SIZE: + { + keyAttr[uiIndx].u.v.valueLen = 24; + break; + } + case SFLM_AES256_KEY_SIZE: + { + keyAttr[uiIndx].u.v.valueLen = 32; + break; + } + } + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_FORMAT; + keyAttr[uiIndx].u.v.valueLen = attr[1].u.v.valueLen; + + break; + } + case NICI_K_DES3X: + { + uiIndx = 0; + keyAttr[uiIndx].type = NICI_A_KEY_VALUE; + keyAttr[uiIndx].u.v.valueLen = 24; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_FORMAT; + keyAttr[uiIndx].u.v.valueLen = attr[1].u.v.valueLen; + break; + + } + case NICI_K_DES: + { + uiIndx = 0; + keyAttr[uiIndx].type = NICI_A_KEY_VALUE; + keyAttr[uiIndx].u.v.valueLen = 8; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_FORMAT; + keyAttr[uiIndx].u.v.valueLen = attr[1].u.v.valueLen; + break; + + } + + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + + // Make one allocation that we can then use to hold several different things. + + uiBufferSize = sizeof( EXTRACTED_KEY) + // pExtractedKey + attr[1].u.v.valueLen + // pucFormat + keyAttr[0].u.v.valueLen + // pucKey + sizeof (ucDigest); // pucDigest + uiAllocSize = uiBufferSize + + SALT_SZ + // Salt (not encrypted) + (sizeof(NICI_PARAMETER_DATA) * 2) + sizeof(FLMUINT32); // Parameter data (not encrypted) + + // Make sure the allocation size is on a 8 byte boundary + + if( (uiAllocSize % 8) != 0) + { + uiAllocSize += (8 - (uiAllocSize % 8)); + } + + if (RC_BAD( rc = f_calloc( uiAllocSize, &pExtractedKey))) + { + goto Exit; + } + + keyAttr[1].u.v.valuePtr = &pExtractedKey[1]; + pucFormat = (FLMBYTE *)keyAttr[1].u.v.valuePtr; + keyAttr[0].u.v.valuePtr = pucFormat + attr[1].u.v.valueLen; + pucKey = (FLMBYTE *)keyAttr[0].u.v.valuePtr; + + pucSalt = (FLMBYTE *)pExtractedKey + uiBufferSize; + + pParmInfo = (NICI_PARAMETER_INFO *)(pucSalt + SALT_SZ); + + // Make sure that pParmInfo is 8 byte alligned. + + if ((FLMUINT)pParmInfo % 8) + { + FLMBYTE * pucTemp = (FLMBYTE *)pParmInfo + + (8 - ((FLMUINT)pParmInfo % 8)); + pParmInfo = (NICI_PARAMETER_INFO *)pucTemp; + } + + // Extracted the key value now + + if (RC_BAD( rc = CCS_ExtractKey( + m_hContext, + m_keyHandle, + &keyAttr[0], + 2))) + { + rc = RC_SET( NE_SFLM_EXTRACT_KEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Calculate a SHA1 checksum. + + algorithm.algorithm = (nuint8 *)oid_sha1; + algorithm.parameter = NULL; + algorithm.parameterLen = 0; + + if (RC_BAD( rc = CCS_DigestInit( + m_hContext, + &algorithm))) + { + rc = RC_SET( NE_SFLM_DIGEST_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Digest( + m_hContext, + (nuint8 *)pucFormat, + keyAttr[0].u.v.valueLen + attr[1].u.v.valueLen, + (nuint8 *)ucDigest, + &uiDigestLen))) + { + rc = RC_SET( NE_SFLM_DIGEST_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + flmAssert( uiDigestLen == sizeof( ucDigest)); + + pucTempPtr = (FLMBYTE *)pExtractedKey; + + UD2FBA( attr[0].u.f.value, pucTempPtr); //pExtractedKey->uiKeyType = attr[0].u.f.value; + pucTempPtr += 4; + + UD2FBA( attr[1].u.v.valueLen, pucTempPtr); //pExtractedKey->uiFormatLen = attr[1].u.v.valueLen; + pucTempPtr += 4; + + UD2FBA( keyAttr[0].u.v.valueLen, pucTempPtr); //pExtractedKey->uiKeyLen = keyAttr[0].u.v.valueLen; + pucTempPtr += 4; + + UD2FBA( m_uiEncKeySize, pucTempPtr); // pEncKey->uiKeySize = m_uiEncKeySize; + + // Point to the Digest... + + pTemp = (FLMBYTE *)&pExtractedKey[1] + + attr[1].u.v.valueLen + // Format length + keyAttr[0].u.v.valueLen; // Key length + f_memcpy( pTemp, ucDigest, uiDigestLen); + + // Generate some salt. + + if (RC_BAD( rc = CCS_GetRandom( m_hContext, + (nuint8 *)pucSalt, + SALT_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + pTemp = NULL; // don't want this to be freed + goto Exit; + } + + // This buffer needs to be a separate allocation because it is returned to the caller. We will + // be returning the value of the SALT with the encrypted key. The call to CCS_pbeEncrypt + // may return an extra 8 bytes. + + if (RC_BAD( rc = f_alloc( uiBufferSize + SALT_SZ + 8, &pTemp))) + { + goto Exit; + } + + // Now to encrypt the buffer. + + algorithm.algorithm = (nuint8 *)oid_pbe; + + pParmInfo->count = 2; // Two parameters + + pParmInfo->parms[0].parmType = NICI_P_SALT; + pParmInfo->parms[0].u.b.len = SALT_SZ; + pParmInfo->parms[0].u.b.ptr = (nuint8 *)pucSalt; + + pParmInfo->parms[1].parmType = NICI_P_COUNT; + pParmInfo->parms[1].u.value = SALT_COUNT; + + algorithm.parameter = pParmInfo; + algorithm.parameterLen = sizeof(NICI_PARAMETER_DATA) * 2 + sizeof(FLMUINT32); + + uiEncLen = uiBufferSize + 8; + + if (RC_BAD( rc = CCS_pbeEncrypt( + m_hContext, + &algorithm, + puzEncKeyPasswd, + (nuint8 *)pExtractedKey, + uiBufferSize, + (nuint8 *)pTemp, + &uiEncLen))) + { + rc = RC_SET( NE_SFLM_PBE_ENCRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + *ppucExtractedKey = pTemp; + + // Now add the salt to the end of the buffer. + + pTemp += uiEncLen; + + f_memcpy( pTemp, pucSalt, SALT_SZ); + + pTemp = NULL; + + *pui32Length = uiEncLen + SALT_SZ; + +#endif + +Exit: +#ifdef FLM_USE_NICI + if (pTemp) + { + f_free( &pTemp); + } + + if (pucKey) + { + f_free( &pExtractedKey); + } + + f_mutexUnlock( m_hMutex); + +#endif + return(rc); +} + + +/*----------------------------------------------------------------------------- + * Desc: injectKey - Inject the encrypting key using the supplied password. + *---------------------------------------------------------------------------*/ +RCODE F_CCS::injectKey( + FLMBYTE * pszExtractedKey, + FLMUINT32 ui32Length, + FLMUNICODE * puzEncKeyPasswd) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pszExtractedKey); + F_UNREFERENCED_PARM( ui32Length); + F_UNREFERENCED_PARM( puzEncKeyPasswd); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_ALGORITHM algorithm; + NICI_ATTRIBUTE keyAttr[7]; + FLMBYTE oid_sha1[] = {IDV_SHA1}; + FLMBYTE oid_pbe[] = {IDV_pbeWithSHA1And3Key3xDES_CBC}; + FLMUINT uiIndx; + FLMBYTE ucDigest[ 20]; + FLMUINT uiDigestLen = sizeof(ucDigest); + FLMBYTE * pKey; + FLMBYTE * pucFormat; + EXTRACTED_KEY * pExtractedKey; + FLMUINT uiEncLen; + FLMBYTE * pTemp; + FLMBYTE * pucBuffer = NULL; + FLMBYTE * pucSalt; + FLMUINT uiAllocSize; + NICI_PARAMETER_INFO * pParmInfo = NULL; + FLMBYTE * pucTempPtr; + + f_mutexLock( m_hMutex); + + /* Create NICI Context */ + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + // Extract the SALT from the key buffer. + pucSalt = pszExtractedKey + (ui32Length - SALT_SZ); + ui32Length -= SALT_SZ; + + // Make one allocation and point into it for the different buffers we need. + + uiAllocSize = ui32Length + + sizeof(NICI_PARAMETER_DATA) * 2 + sizeof(FLMUINT32); + + if (RC_BAD( rc = f_calloc( uiAllocSize, &pucBuffer))) + { + goto Exit; + } + + pParmInfo = (NICI_PARAMETER_INFO *)(pucBuffer + ui32Length); + + + // Now to decrypt the buffer. + + algorithm.algorithm = (nuint8 *)oid_pbe; + + pParmInfo->count = 2; // Two parameters + + pParmInfo->parms[0].parmType = NICI_P_SALT; + pParmInfo->parms[0].u.b.len = SALT_SZ; + pParmInfo->parms[0].u.b.ptr = (nuint8 *)pucSalt; + + pParmInfo->parms[1].parmType = NICI_P_COUNT; + pParmInfo->parms[1].u.value = SALT_COUNT; + + algorithm.parameter = pParmInfo; + algorithm.parameterLen = sizeof(NICI_PARAMETER_DATA) * 2 + sizeof(FLMUINT32); + + uiEncLen = ui32Length; + + if (RC_BAD( rc = CCS_pbeDecrypt( + m_hContext, + &algorithm, + puzEncKeyPasswd, + (nuint8 *)pszExtractedKey, + ui32Length, + (nuint8 *)pucBuffer, + &uiEncLen))) + { + rc = RC_SET( NE_SFLM_PBE_DECRYPT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // For cross platform compatibility, we need to first extract the KeyType, + // FormatLen and KeyLen values then we will set them back again. They are + // stored in a specific byte order, which may not match the native order for + // referencing integers on the local platform. + + pExtractedKey = (EXTRACTED_KEY *)pucBuffer; + pucTempPtr = pucBuffer; + + pExtractedKey->uiKeyType = FB2UD( pucTempPtr); + pucTempPtr += 4; + + pExtractedKey->uiFormatLen = FB2UD( pucTempPtr); + pucTempPtr += 4; + + pExtractedKey->uiKeyLen = FB2UD( pucTempPtr); + pucTempPtr += 4; + + m_uiEncKeySize = FB2UD( pucTempPtr); + + // Calculate a SHA1 checksum. + + algorithm.algorithm = (nuint8 *)oid_sha1; + algorithm.parameter = NULL; + algorithm.parameterLen = 0; + + if (RC_BAD( rc = CCS_DigestInit( + m_hContext, + &algorithm))) + { + rc = RC_SET( NE_SFLM_DIGEST_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + pTemp = (FLMBYTE *)&pExtractedKey[ 1]; + + if (RC_BAD( rc = CCS_Digest( + m_hContext, + (nuint8 *)pTemp, + pExtractedKey->uiFormatLen + + pExtractedKey->uiKeyLen, + (nuint8 *)ucDigest, + &uiDigestLen))) + { + rc = RC_SET( NE_SFLM_DIGEST_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + flmAssert( uiDigestLen == sizeof( ucDigest)); + + // Now compare the two digests. They must be equal! + pTemp += pExtractedKey->uiKeyLen + pExtractedKey->uiFormatLen; + + if (f_memcmp( pTemp, ucDigest, uiDigestLen)) + { + rc = RC_SET( NE_SFLM_INVALID_ENCKEY_CRC); + goto Exit; + } + + pucFormat = (FLMBYTE *)&pExtractedKey[1]; // Point to the format + pKey = pucFormat + pExtractedKey->uiFormatLen; // Point to the key. + + uiIndx = 0; + f_memset( &keyAttr[0], 0, sizeof(NICI_ATTRIBUTE) * 7); + + switch (pExtractedKey->uiKeyType) + { + case NICI_K_AES: + { + /* Set key attributes */ + uiIndx = 0; + keyAttr[uiIndx].type = NICI_A_KEY_TYPE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_K_AES; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_FORMAT; + keyAttr[uiIndx].u.v.valuePtr = pucFormat; + keyAttr[uiIndx].u.v.valueLen = pExtractedKey->uiFormatLen; + keyAttr[uiIndx].u.v.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_USAGE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_F_WRAP | NICI_F_UNWRAP | NICI_F_KM_ENCRYPT | NICI_F_KM_DECRYPT | NICI_F_EXTRACT | NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_SIZE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = m_uiEncKeySize; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_VALUE; + keyAttr[uiIndx].u.v.valuePtr = pKey; + keyAttr[uiIndx].u.v.valueLen = pExtractedKey->uiKeyLen; + keyAttr[uiIndx].u.v.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_CLASS; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_O_SECRET_KEY; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_GLOBAL; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = N_TRUE; + keyAttr[uiIndx].u.f.valueInfo = 0; + break; + } + case NICI_K_DES3X: + { + /* Set key attributes */ + uiIndx = 0; + keyAttr[uiIndx].type = NICI_A_KEY_TYPE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_K_DES3X; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_FORMAT; + keyAttr[uiIndx].u.v.valuePtr = pucFormat; + keyAttr[uiIndx].u.v.valueLen = pExtractedKey->uiFormatLen; + keyAttr[uiIndx].u.v.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_USAGE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_F_WRAP | NICI_F_UNWRAP | NICI_F_KM_ENCRYPT | NICI_F_KM_DECRYPT | NICI_F_EXTRACT | NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_SIZE; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = m_uiEncKeySize; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_KEY_VALUE; + keyAttr[uiIndx].u.v.valuePtr = pKey; + keyAttr[uiIndx].u.v.valueLen = pExtractedKey->uiKeyLen; + keyAttr[uiIndx].u.v.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_CLASS; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = NICI_O_SECRET_KEY; + keyAttr[uiIndx].u.f.valueInfo = 0; + + uiIndx++; + keyAttr[uiIndx].type = NICI_A_GLOBAL; + keyAttr[uiIndx].u.f.hasValue = 1; + keyAttr[uiIndx].u.f.value = N_TRUE; + keyAttr[uiIndx].u.f.valueInfo = 0; + break; + } + default: + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_NICI_INVALID_ALGORITHM); + goto Exit; + } + } + + if (RC_BAD( rc = CCS_InjectKey( + m_hContext, + &keyAttr[0], + 7, + &m_keyHandle))) + { + rc = RC_SET( NE_SFLM_INJECT_KEY_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + +#endif + +Exit: +#ifdef FLM_USE_NICI + if (pucBuffer) + { + f_free( &pucBuffer); + } + + f_mutexUnlock( m_hMutex); + +#endif + + return(rc); +} + + +/**************************************************************************** +Desc: getIVLen returns the correct length of the IV for the type of + algorithm. +****************************************************************************/ +FLMUINT F_CCS::getIVLen() +{ + +#ifndef FLM_USE_NICI + return 0; +#else + switch (m_eEncAlgorithm) + { + case SFLM_AES_ENCRYPTION: + return IV_SZ; + case SFLM_DES3_ENCRYPTION: + return IV_SZ8; + default: + return 0; + } +#endif +} + +/**************************************************************************** +Desc: generateIV will generate a random set of bytes to be used as IV. +****************************************************************************/ +RCODE F_CCS::generateIV( + FLMUINT uiIVLen, + FLMBYTE * pucIV) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( uiIVLen); + F_UNREFERENCED_PARM( pucIV); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + FLMUINT uiLoop; + NICI_ALGORITHM algorithm; + FLMBYTE oid_sha1[] = {IDV_SHA1}; + FLMBOOL bLocked = FALSE; + FLMBYTE * pucIVPtr = m_ucRndIV; + FLMBYTE ucIVBuffer[ IV_SZ * 2]; + FLMUINT uiIVBufferLen = sizeof(ucIVBuffer); + + if (!uiIVLen) + { + goto Exit; + } + + f_mutexLock( m_hMutex); + bLocked = TRUE; + + /* Create NICI Context */ + + if ( !m_hContext) + { + if (RC_BAD( rc = CCS_CreateContext(0, &m_hContext))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + m_hContext = 0; + goto Exit; + } + } + + // See if it is time to reinitialize the Random IV. + + if ((m_uiIVFactor & 0x07FF) == 0) + { + // Generate the Random IV + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)&m_ucRndIV, + IV_SZ))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Generate an adjustment factor for the IV + + if (RC_BAD( rc = CCS_GetRandom( + m_hContext, + (nuint8 *)&m_uiIVFactor, + sizeof(FLMUINT)))) + { + rc = RC_SET( NE_SFLM_NICI_BAD_RANDOM); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + } + + + // Increment each byte of the IV by the IV Factor + + for( uiLoop = 0; uiLoop < IV_SZ; uiLoop++) + { + (*pucIVPtr) += (FLMBYTE)m_uiIVFactor; + pucIVPtr++; + } + + // Now run the resulting IV through a SHA1 digest. + + algorithm.algorithm = (nuint8 *)oid_sha1; + algorithm.parameter = NULL; + algorithm.parameterLen = 0; + + if (RC_BAD( rc = CCS_DigestInit( + m_hContext, + &algorithm))) + { + rc = RC_SET( NE_SFLM_DIGEST_INIT_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + if (RC_BAD( rc = CCS_Digest( + m_hContext, + (nuint8 *)m_ucRndIV, + uiIVLen, + (nuint8 *)ucIVBuffer, + &uiIVBufferLen))) + { + rc = RC_SET( NE_SFLM_DIGEST_FAILED); + m_hContext = 0; // Context has been destroyed + goto Exit; + } + + // Return the new IV! + + f_memcpy( pucIV, ucIVBuffer, uiIVLen); + + m_uiIVFactor++; + +#endif + +Exit: + +#ifdef FLM_USE_NICI + if (bLocked) + { + f_mutexUnlock( m_hMutex); + } +#endif + + return rc; +} + + +/**************************************************************************** +Desc: flmDecryptBuffer - assumes aes +****************************************************************************/ +RCODE flmDecryptBuffer( + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen) +{ + RCODE rc = NE_SFLM_OK; + + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucBuffer); + F_UNREFERENCED_PARM( puiBufLen); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_CC_HANDLE context = 0; + NICI_ATTRIBUTE find[2]; + NICI_OBJECT_HANDLE serverKeyHdl = 0; + FLMUINT uiCount; + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_aes[] = {IDV_AES128CBC}; + FLMBYTE pucIV[ IV_SZ]; + + /* Create NICI Context */ + + if (RC_BAD( rc = CCS_CreateContext(0, &context))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + goto Exit; + } + + find[0].type = NICI_A_GLOBAL; + find[0].u.f.hasValue = 1; + find[0].u.f.value = 1; + find[0].u.f.valueInfo = 0; + + find[1].type = NICI_A_FEATURE; + find[1].u.f.hasValue = 1; + find[1].u.f.value = NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + find[1].u.f.valueInfo = 0; + + if (RC_BAD( rc = CCS_FindObjectsInit(context, find, 2))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_INIT); + goto Exit; + } + + uiCount = 1; + + if (RC_BAD( rc = CCS_FindObjects( + context, + &serverKeyHdl, + &uiCount))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_OBJECT); + goto Exit; + } + + if (uiCount < 1) + { + rc = RC_SET( NE_SFLM_NICI_KEY_NOT_FOUND); + goto ExitCtx; + } + + /*Set up alogrithm now to do AES and pading for encryption */ + algorithm.algorithm = (nuint8 *)oid_aes; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + algorithm.parameter->parms[0].u.b.len = IV_SZ; /* 16-byte IV */ + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataDecryptInit( + context, + &algorithm, + serverKeyHdl))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_INIT_FAILED); + goto Exit; + } + + if (RC_BAD( rc = CCS_Decrypt( + context, + (nuint8 *)pucBuffer, + *puiBufLen, + (nuint8 *)pucBuffer, + puiBufLen))) + { + rc = RC_SET( NE_SFLM_NICI_DECRYPT_FAILED); + goto Exit; + } + +ExitCtx: + + CCS_DestroyContext( context); + +#endif + +Exit: + + return rc; + +} + +/**************************************************************************** +Desc: flmEncryptBuffer - assumes aes +****************************************************************************/ +RCODE flmEncryptBuffer( + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen) +{ + RCODE rc = NE_SFLM_OK; + +#ifndef FLM_USE_NICI + F_UNREFERENCED_PARM( pucBuffer); + F_UNREFERENCED_PARM( puiBufLen); + rc = RC_SET( NE_SFLM_UNSUPPORTED_FEATURE); + goto Exit; +#else + NICI_CC_HANDLE context = 0; + NICI_ATTRIBUTE find[2]; + NICI_OBJECT_HANDLE serverKeyHdl = 0; + FLMUINT uiCount; + NICI_ALGORITHM algorithm; + NICI_PARAMETER_INFO parm[1]; + FLMBYTE oid_aes[] = {IDV_AES128CBC}; + FLMBYTE pucIV[ IV_SZ]; + + /* Create NICI Context */ + if (RC_BAD( rc = CCS_CreateContext(0, &context))) + { + rc = RC_SET( NE_SFLM_NICI_CONTEXT); + goto Exit; + } + + find[0].type = NICI_A_GLOBAL; + find[0].u.f.hasValue = 1; + find[0].u.f.value = 1; + find[0].u.f.valueInfo = 0; + + find[1].type = NICI_A_FEATURE; + find[1].u.f.hasValue = 1; + find[1].u.f.value = NICI_F_DATA_ENCRYPT | NICI_F_DATA_DECRYPT; + find[1].u.f.valueInfo = 0; + + if (RC_BAD( rc = CCS_FindObjectsInit(context, find, 2))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_INIT); + goto Exit; + } + + uiCount = 1; + + if (RC_BAD( rc = CCS_FindObjects( + context, + &serverKeyHdl, + &uiCount))) + { + rc = RC_SET( NE_SFLM_NICI_FIND_OBJECT); + goto Exit; + } + + if (uiCount < 1) + { + rc = RC_SET( NE_SFLM_NICI_KEY_NOT_FOUND); + goto ExitCtx; + } + + + algorithm.algorithm = (nuint8 *)oid_aes; + algorithm.parameterLen = sizeof(algorithm.parameter->parms[0])+ + sizeof(algorithm.parameter->count); + algorithm.parameter = parm; + algorithm.parameter->count = 1; + algorithm.parameter->parms[0].parmType = NICI_P_IV; + algorithm.parameter->parms[0].u.b.len = IV_SZ; /* 16-byte IV */ + algorithm.parameter->parms[0].u.b.ptr = (nuint8 *)pucIV; + + GetIV(pucIV, IV_SZ); + + /* init encryption */ + + if (RC_BAD( rc = CCS_DataEncryptInit( + context, + &algorithm, + serverKeyHdl))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NICI_ENC_INIT_FAILED); + goto Exit; + } + + if (RC_BAD( rc = CCS_Encrypt( + context, + (nuint8 *)pucBuffer, + *puiBufLen, + (nuint8 *)pucBuffer, + puiBufLen))) + { + rc = RC_SET( NE_SFLM_NICI_ENCRYPT_FAILED); + goto Exit; + } + +ExitCtx: + + CCS_DestroyContext( context); + +#endif + +Exit: + + return rc; + +} + +#ifdef FLM_USE_NICI +FSTATIC void GetIV( + FLMBYTE * pucIV, + FLMUINT //uiLen + ) +{ + FLMUINT uiLoop; + FLMUINT uiLoop2; + + f_sprintf( (char *)pucIV, "3587903781145935"); + + for (uiLoop = 0; uiLoop < 100; uiLoop++) + { + for ( uiLoop2 = 0; uiLoop2 < IV_SZ; uiLoop2++) + { + pucIV[IV_SZ - uiLoop2] ^= pucIV[ uiLoop2]; + pucIV[IV_SZ - uiLoop2] += pucIV[ uiLoop2]; + pucIV[IV_SZ - uiLoop2] ^= pucIV[ uiLoop2]; + } + + } + +} +#endif + + +#ifdef FLM_USE_NICI +#ifndef FLM_UNIX +int CCSX_SetNewIV( + int ,//MODULEID, + FLMUINT32 ,//hContext, + pnuint8 ,//IV, + nuint32 //IVLen + ) +{ + return(NICI_E_FUNCTION_NOT_SUPPORTED); +} +#endif +#endif + diff --git a/sql/src/f_nici.h b/sql/src/f_nici.h new file mode 100644 index 0000000..e23c555 --- /dev/null +++ b/sql/src/f_nici.h @@ -0,0 +1,268 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the definitions needed for the NICI interface +// functions. +// +// Tabs: 3 +// +// Copyright (c) 2004-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: f_nici.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef _F_NICI_HPP +#define _F_NICI_HPP + +#ifdef FLM_USE_NICI + #ifdef FLM_NLM + #define N_PLAT_NLM + #endif + + #include "nwccs.h" + + #ifndef IDV_NOV_AES128CBCPad + #define IDV_NOV_AES128CBCPad NICI_AlgorithmPrefix(1), 97 /* 0x61 */ + #endif +#else + #define NICI_OBJECT_HANDLE void * + #define NICI_CC_HANDLE FLMUINT32 +#endif + +/*-------------------------------------------------------------------------- + * Definitions + *------------------------------------------------------------------------*/ + +#define IV_SZ 16 +#define IV_SZ8 8 +#define SALT_SZ 8 +#define SALT_COUNT 895 + +/*----------------------------------------------------------------------- + * CCS Interface. + *-----------------------------------------------------------------------*/ +class IF_CCS : public F_Object +{ +public: + + virtual ~IF_CCS() + { + } + + virtual RCODE generateEncryptionKey( + FLMUINT uiEncKeySize) = 0; + + virtual RCODE generateWrappingKey( + FLMUINT uiEncKeySize) = 0; + + virtual RCODE encryptToStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV = NULL) = 0; + + virtual RCODE decryptFromStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV = NULL) = 0; + +}; // IF_CCS + + + +class F_CCS : public IF_CCS +{ +public: + + // Constructor & destructor + F_CCS() + { + m_bInitCalled = FALSE; + m_bKeyVerified = FALSE; + f_memset( m_ucIV, 0, IV_SZ); + //m_bKeyIsWrappingKey = FALSE; + //m_uiAlgType = 0xFF; + m_keyHandle = 0; + m_hContext = 0; + m_uiEncKeySize = 0; + m_hMutex = F_MUTEX_NULL; + + } + + ~F_CCS(); + + RCODE init( + FLMBOOL bKeyIsWrappingKey, + eEncAlgorithm eEncAlg); + + RCODE generateEncryptionKey( + FLMUINT uiEncKeySize); + + RCODE generateWrappingKey( + FLMUINT uiEncKeySize); + + RCODE encryptToStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV = NULL); + + RCODE decryptFromStore( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV = NULL); + + RCODE getKeyToStore( + FLMBYTE ** ppucKeyInfo, + FLMUINT32 * pui32BufLen, + FLMBYTE * pzEncKeyPasswd = NULL, + F_CCS * pWrappingCcs = NULL); + + RCODE setKeyFromStore( + FLMBYTE * pucKeyInfo, + FLMBYTE * pszEncKeyPasswd = NULL, + F_CCS * pWrappingCcs = NULL); + + FINLINE FLMBOOL keyVerified() + { + return m_bKeyVerified; + } + + FINLINE FLMUINT getEncType( void) + { + return m_uiAlgType; + } + + FLMUINT getIVLen(); + + RCODE generateIV( + FLMUINT uiIVLen, + FLMBYTE * pucIV); + +private: + + RCODE getWrappingKey( + NICI_OBJECT_HANDLE * pWrappingKeyHandle); + + RCODE wrapKey( + FLMBYTE ** ppucWrappedKey, + FLMUINT32 * pui32Length, + NICI_OBJECT_HANDLE masterWrappingKey = 0 ); + + RCODE unwrapKey( + FLMBYTE * pucWrappedKey, + FLMUINT32 ui32WrappedKeyLength, + NICI_OBJECT_HANDLE masterWrappingKey = 0); + + RCODE extractKey( + FLMBYTE ** ppucShroudedKey, + FLMUINT32 * pui32Length, + FLMUNICODE * puzEncKeyPasswd ); + + RCODE injectKey( + FLMBYTE * pucBuffer, + FLMUINT32 ui32Length, + FLMUNICODE * puzEncKeyPasswd ); + + RCODE encryptToStoreAES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE encryptToStoreDES3( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE encryptToStoreDES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE decryptFromStoreAES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE decryptFromStoreDES3( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE decryptFromStoreDES( + FLMBYTE * pucIn, + FLMUINT uiInLen, + FLMBYTE * pucOut, + FLMUINT * puiOutLen, + FLMBYTE * pucIV); + + RCODE generateEncryptionKeyAES( + FLMUINT uiEncKeySize); + + RCODE generateEncryptionKeyDES3( + FLMUINT uiEncKeySize); + + RCODE generateEncryptionKeyDES( + FLMUINT uiEncKeySize); + + RCODE generateWrappingKeyAES( + FLMUINT uiEncKeySize); + + RCODE generateWrappingKeyDES3( + FLMUINT uiEncKeySize); + + RCODE generateWrappingKeyDES( + FLMUINT uiEncKeySize); + + FLMUINT m_uiAlgType; + FLMBOOL m_bInitCalled; + FLMBOOL m_bKeyIsWrappingKey; + FLMBOOL m_bKeyVerified; + NICI_OBJECT_HANDLE m_keyHandle; // Handle to the clear key - we don't ever get the actual key. + FLMBYTE m_ucIV[ IV_SZ]; // Used when the algorithm type is DES, 3DES or AES + FLMBYTE m_ucRndIV[ IV_SZ]; // Used when the IV is stored with the data. + FLMUINT m_uiIVFactor; + NICI_CC_HANDLE m_hContext; + FLMUINT m_uiEncKeySize; + F_MUTEX m_hMutex; + +}; // F_CCS + +RCODE flmDecryptBuffer( + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen); + +RCODE flmEncryptBuffer( + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen); + +#endif /* _F_NICI_HPP */ diff --git a/sql/src/fbtrset.cpp b/sql/src/fbtrset.cpp new file mode 100644 index 0000000..c43a6f2 --- /dev/null +++ b/sql/src/fbtrset.cpp @@ -0,0 +1,583 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains routines that implement a result set using +// a temporary XFLAIM database. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fbtrset.cpp 3111 2006-01-19 13:10:50 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: +****************************************************************************/ +F_BtResultSet::~F_BtResultSet() +{ + // Free the collection table if it was ever created. + + if (m_ppTableTbl) + { + FLMUINT uiLoop; + + for (uiLoop = 0; uiLoop < BT_MAX_TABLE_TBL_SIZ; uiLoop++) + { + if (m_ppTableTbl[ uiLoop] != NULL) + { + BT_TABLE_XREF * pTmp; + + while (m_ppTableTbl[ uiLoop] != NULL) + { + pTmp = m_ppTableTbl[ uiLoop]; + m_ppTableTbl[ uiLoop] = pTmp->pNext; + if (pTmp && pTmp->pCompare) + { + pTmp->pCompare->Release(); + } + f_free( &pTmp); + } + } + } + f_free( &m_ppTableTbl); + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getBTree( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree ** ppBTree) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCollHash; + BT_TABLE_XREF * pCollPtr = NULL; + F_Database * pDatabase; + + if (RC_BAD( rc = m_pBtPool->btpReserveBtree( ppBTree))) + { + goto Exit; + } + + if (pSrcIndex) + { + if (!m_ppTableTbl) + { + if (RC_BAD( rc = f_calloc( + BT_MAX_TABLE_TBL_SIZ * sizeof(BT_TABLE_XREF), + &m_ppTableTbl))) + { + goto Exit; + } + } + + uiCollHash = pSrcIndex->uiIndexNum % BT_MAX_TABLE_TBL_SIZ; + + pCollPtr = m_ppTableTbl[ uiCollHash]; + + // Verify that we have the right collection + while (pCollPtr && pCollPtr->uiKeyNum != pSrcIndex->uiIndexNum) + { + pCollPtr = pCollPtr->pNext; + } + + if (!pCollPtr) + { + pDatabase = m_pResultSetDb->m_pDatabase; + + // Allocate a new collection key context and create a new + // collection for it. + + if (RC_BAD( rc = f_calloc( sizeof(BT_TABLE_XREF), &pCollPtr))) + { + goto Exit; + } + + // Insert into the table at the head of the list. + + pCollPtr->pCompare = NULL; + pCollPtr->pNext = m_ppTableTbl[ uiCollHash]; + m_ppTableTbl[ uiCollHash] = pCollPtr; + + // Check to see if it already exists. + if (RC_BAD( rc = pDatabase->lFileCreate( m_pResultSetDb, + &pCollPtr->table.lfInfo, 101, SFLM_LF_TABLE, FALSE, TRUE, + pSrcIndex->lfInfo.uiEncDefNum))) + { + goto Exit; + } + + pCollPtr->uiKeyNum = pSrcIndex->uiIndexNum; + + // Set up the comparison object. + + if ((pCollPtr->pCompare = f_new IXKeyCompare) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + } + pCollPtr->pCompare->setIxInfo( pSrcDb, pSrcIndex); + + // Open the btree and use the specified collection. + + if (RC_BAD( rc = (*ppBTree)->btOpen( m_pResultSetDb, + &pCollPtr->table.lfInfo, + FALSE, TRUE, pCollPtr->pCompare))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = (*ppBTree)->btOpen( m_pResultSetDb, + &m_table.lfInfo, + FALSE, TRUE, NULL))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::addEntry( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + FLMBYTE * pucKey, + FLMUINT uiKeyLength, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pBTree = NULL; + + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + + flmAssert( uiKeyLength <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btInsertEntry( pucKey, + uiKeyLength, pucEntry, uiEntryLength, TRUE, TRUE))) + { + if (rc == NE_SFLM_NOT_UNIQUE) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + +Exit: + + if (pBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::modifyEntry( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + FLMBYTE * pucKey, + FLMUINT uiKeyLength, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pBTree = NULL; + + if (RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + + flmAssert( uiKeyLength <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btReplaceEntry( pucKey, uiKeyLength, + pucEntry, uiEntryLength, TRUE, TRUE))) + { + goto Exit; + } + +Exit: + + if( pBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::deleteEntry( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + FLMBYTE * pucKey, + FLMUINT uiKeyLength) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pBTree = NULL; + + if (RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + + flmAssert( uiKeyLength <= SFLM_MAX_KEY_SIZE); + + if (RC_BAD( rc = pBTree->btRemoveEntry( pucKey, uiKeyLength))) + { + goto Exit; + } + +Exit: + + if (pBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + return rc; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::findEntry( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeyLen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLengthRV; + F_Btree * pBTree = NULL; + + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + + flmAssert( uiKeyBufLen <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btLocateEntry( pucKey, uiKeyBufLen, puiKeyLen, + FLM_EXACT, NULL, &uiLengthRV))) + { + goto Exit; + } + + if( pucBuffer) + { + // Get the entry ... + + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, uiKeyBufLen, *puiKeyLen, + pucBuffer, uiBufferLength, puiReturnLength))) + { + goto Exit; + } + } + else if( puiReturnLength) + { + *puiReturnLength = uiLengthRV; + } + +Exit: + + if( pBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getCurrent( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + FLMBYTE * pucKey, + FLMUINT uiKeyLength, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pBTree = NULL; + + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + + flmAssert( uiKeyLength <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, uiKeyLength, uiKeyLength, + pucEntry, uiEntryLength, puiReturnLength))) + { + goto Exit; + } + +Exit: + + if( pBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getNext( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree * pBTree, + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeyLen, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bFreeBTree = FALSE; + + if( !pBTree) + { + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + bFreeBTree = TRUE; + } + + flmAssert( uiKeyBufLen <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btNextEntry( pucKey, uiKeyBufLen, puiKeyLen, + puiReturnLength))) + { + goto Exit; + } + + if( pucEntry) + { + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, *puiKeyLen, *puiKeyLen, + pucEntry, uiEntryLength, puiReturnLength))) + { + goto Exit; + } + } + +Exit: + + if( bFreeBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getPrev( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree * pBTree, + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeyLen, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bFreeBTree = FALSE; + + if( !pBTree) + { + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + bFreeBTree = TRUE; + } + + flmAssert( uiKeyBufLen <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btPrevEntry( pucKey, uiKeyBufLen, puiKeyLen, + puiReturnLength))) + { + goto Exit; + } + + if( pucEntry) + { + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, *puiKeyLen, *puiKeyLen, + pucEntry, uiEntryLength, puiReturnLength))) + { + goto Exit; + } + } + +Exit: + + if( bFreeBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getFirst( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree * pBTree, + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeyLen, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bFreeBTree = FALSE; + + if( !pBTree) + { + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + bFreeBTree = TRUE; + } + + flmAssert( uiKeyBufLen <= SFLM_MAX_KEY_SIZE); + + pBTree->btResetBtree(); + + if( RC_BAD( rc = pBTree->btFirstEntry( pucKey, uiKeyBufLen, puiKeyLen, + puiReturnLength))) + { + goto Exit; + } + + if( pucEntry) + { + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, *puiKeyLen, *puiKeyLen, + pucEntry, uiEntryLength, puiReturnLength))) + { + goto Exit; + } + } + +Exit: + + if( bFreeBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_BtResultSet::getLast( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree * pBTree, + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeyLen, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, + FLMUINT * puiReturnLength) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bFreeBTree = FALSE; + + if( !pBTree) + { + if( RC_BAD( rc = getBTree( pSrcDb, pSrcIndex, &pBTree))) + { + goto Exit; + } + bFreeBTree = TRUE; + } + + flmAssert( uiKeyBufLen <= SFLM_MAX_KEY_SIZE); + + if( RC_BAD( rc = pBTree->btLastEntry( pucKey, uiKeyBufLen, puiKeyLen, + puiReturnLength))) + { + goto Exit; + } + + if( pucEntry) + { + if( RC_BAD( rc = pBTree->btGetEntry( pucKey, *puiKeyLen, *puiKeyLen, + pucEntry, uiEntryLength, puiReturnLength))) + { + goto Exit; + } + } + +Exit: + + if( bFreeBTree) + { + m_pBtPool->btpReturnBtree( &pBTree); + } + + return( rc); +} diff --git a/sql/src/fbtrset.h b/sql/src/fbtrset.h new file mode 100644 index 0000000..22c90c2 --- /dev/null +++ b/sql/src/fbtrset.h @@ -0,0 +1,184 @@ +//------------------------------------------------------------------------------ +// Desc: This File contains routines which do certain types of verifications +// on objects in a FLAIM database. +// +// Tabs: 3 +// +// Copyright (c) 2003-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fbtrset.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef BTRSET_H +#define BTRSET_H + +#include "f_btpool.h" +#include "f_btree.h" + +class IXKeyCompare; + +typedef struct BT_TABLE_XREF +{ + FLMUINT uiKeyNum; + F_TABLE table; + BT_TABLE_XREF * pNext; + IXKeyCompare * pCompare; +} BT_TABLE_XREF; + +#define BT_MAX_TABLE_TBL_SIZ 256 + +/*============================================================================= +Desc: Result set class that uses an independant database. The name is randomly + generated. +=============================================================================*/ +class F_BtResultSet : public F_Object +{ +public: + + F_BtResultSet( + F_Db * pResultSetDb, + F_BtPool * pBtPool) + { + m_pBtPool = pBtPool; + m_pResultSetDb = pResultSetDb; + f_memset( &m_table, 0, sizeof( m_table)); + m_ppTableTbl = NULL; + } + + ~F_BtResultSet(); + + // Entry Add and Sort Methods + + RCODE addEntry( // Variable or fixed length entry coming in + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + FLMBYTE * pucKey, // key for sorting. + FLMUINT uiKeyLength, + FLMBYTE * pEntry, + FLMUINT uiEntryLength); // If length is zero then ignore entry. + + RCODE modifyEntry( // Modify current entry. + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + FLMBYTE * pucKey, + FLMUINT uiKeyLength, + FLMBYTE * pEntry, // Points to entry buffer + FLMUINT uiEntryLength); + + // Methods to read entries. + + RCODE getCurrent( // Return current entry + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + FLMBYTE * pucKey, + FLMUINT uiKeyLength, + FLMBYTE * pucEntry, + FLMUINT uiEntryLength, // Size of Entry buffer. + FLMUINT * puiReturnLength); + + RCODE getNext( + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + F_Btree * pBTree, // Preserves the context from one call to + // the next. May be null if not needed. + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeylen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength); + + RCODE getPrev( // Position to previous entry and return + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + F_Btree * pBTree, // Preserves the context from one call to + // the next. May be null if not needed. + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeylen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength); + + RCODE getFirst( // Position to the first entry and return + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + F_Btree * pBTree, // Preserves the context from one call to + // the next. May be null if not needed. + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeylen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength); + + RCODE getLast( // Position to the last entry and return + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + F_Btree * pBTree, // Preserves the context from one call to + // the next. May be null if not needed. + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeylen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength); + + RCODE findEntry( // Locate an entry + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + FLMBYTE * pucKey, + FLMUINT uiKeyBufLen, + FLMUINT * puiKeylen, + FLMBYTE * pucBuffer, + FLMUINT uiBufferLength, + FLMUINT * puiReturnLength); + + RCODE deleteEntry( + F_Db * pSrcDb, // Set for when we are keeping index keys + F_INDEX * pSrcIndex, // Set for when we are keeping index keys + FLMBYTE * pucKey, + FLMUINT uiKeyLength); + + // Methods for managing context + + RCODE getBTree( + F_Db * pSrcDb, + F_INDEX * pSrcIndex, + F_Btree ** ppBtree); + + FINLINE void freeBTree( + F_Btree ** ppBTree) + { + flmAssert( *ppBTree); + + m_pBtPool->btpReturnBtree( ppBTree); + + *ppBTree = NULL; + } + +private: + + F_BtPool * m_pBtPool; + F_Db * m_pResultSetDb; + F_TABLE m_table; + BT_TABLE_XREF ** m_ppTableTbl; + + friend class F_DbCheck; +}; +#endif diff --git a/sql/src/fcache.h b/sql/src/fcache.h new file mode 100644 index 0000000..940f7e5 --- /dev/null +++ b/sql/src/fcache.h @@ -0,0 +1,3320 @@ +//------------------------------------------------------------------------------ +// Desc: Various classes used to manage cache. +// +// Tabs: 3 +// +// Copyright (c) 2004-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fcache.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FCACHE_H +#define FCACHE_H + +class F_Rfl; +class F_Db; +class F_DbSystem; +class F_Database; +class F_MultiAlloc; +class F_Row; +class F_CachedBlock; +class F_BlockCacheMgr; +class F_RowCacheMgr; +class F_GlobalCacheMgr; +class F_CacheList; +class F_CachedItem; +class F_Btree; +class F_BTreeIStream; +class F_BTreeInfo; +class F_RowRelocator; +class F_ColumnDataRelocator; +class F_ColumnListRelocator; +class F_BlockRelocator; + +#define MIN_HASH_BUCKETS 0x10000 // 65536 buckets - multiple of 2. +#define MAX_HASH_BUCKETS 0x20000000 // roughly 500,000,000 buckets. + +FLMUINT caGetBestHashTblSize( // scache.cpp + FLMUINT uiCurrItemCount); + +FINLINE FLMUINT minItemCount( + FLMUINT uiNumHashBuckets) +{ + return( uiNumHashBuckets / 4); +} + +FINLINE FLMUINT maxItemCount( + FLMUINT uiNumHashBuckets) +{ + return( uiNumHashBuckets * 4); +} + +FINLINE FLMBOOL shouldRehash( + FLMUINT uiItemCount, + FLMUINT uiNumBuckets) +{ + return( ((uiItemCount > maxItemCount( uiNumBuckets) && + uiNumBuckets < MAX_HASH_BUCKETS) || + (uiItemCount < minItemCount( uiNumBuckets) && + uiNumBuckets > MIN_HASH_BUCKETS)) + ? TRUE + : FALSE); +} + +/*************************************************************************** +Desc: See if enough time has passed since we last tried to rehash. We + don't want to be continually trying to rehash. +***************************************************************************/ +FINLINE FLMBOOL checkHashFailTime( + FLMUINT * puiHashFailTime) +{ + if (*puiHashFailTime) + { + FLMUINT uiCurrTime = FLM_GET_TIMER(); + + if (FLM_ELAPSED_TIME( uiCurrTime, (*puiHashFailTime)) >= + gv_SFlmSysData.uiRehashAfterFailureBackoffTime) + { + *puiHashFailTime = 0; + return( TRUE); + } + else + { + return( FALSE); + } + } + else + { + return( TRUE); + } +} + +/***************************************************************************** +Desc: Cached item +******************************************************************************/ +class F_CachedItem +{ +public: + + F_CachedItem() + { + m_pNextInGlobal = NULL; + m_pPrevInGlobal = NULL; + } + + virtual ~F_CachedItem() + { + } + +#ifdef FLM_CACHE_PROTECT + virtual void protectCachedItem( void) = 0; + + virtual void unprotectCachedItem( void) = 0; +#endif + +private: + + F_CachedItem * m_pPrevInGlobal; + F_CachedItem * m_pNextInGlobal; + +friend class F_CacheList; +friend class F_CachedBlock; +friend class F_Row; +friend class F_GlobalCacheMgr; +friend class F_BlockCacheMgr; +friend class F_RowCacheMgr; +friend class F_Database; +friend class F_Db; +friend class F_RowRelocator; +friend class F_ColumnDataRelocator; +friend class F_ColumnListRelocator; +friend class F_BlockRelocator; +}; + +/*************************************************************************** +Desc: Object for keeping track of an MRU/LRU list of cached items (rows + or blocks) +***************************************************************************/ +class F_CacheList +{ +public: + + F_CacheList() + { + m_pMRUItem = NULL; + m_pLRUItem = NULL; + m_pLastMRUItem = NULL; + } + + ~F_CacheList() + { + flmAssert( !m_pMRUItem); + flmAssert( !m_pLRUItem); + flmAssert( !m_pLastMRUItem); + } + + // Link a cached item into the global list as the MRU item. This routine + // assumes that the cache mutex for managing this list + // has already been locked. + + FINLINE void linkGlobalAsMRU( + F_CachedItem * pItem) + { +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + + if ((pItem->m_pNextInGlobal = m_pMRUItem) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->unprotectCachedItem(); +#endif + pItem->m_pNextInGlobal->m_pPrevInGlobal = pItem; +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->protectCachedItem(); +#endif + } + else + { + m_pLRUItem = pItem; + m_pLastMRUItem = pItem; + } + + pItem->m_pPrevInGlobal = NULL; +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif + m_pMRUItem = pItem; + flmAssert( pItem != pItem->m_pPrevInGlobal); + flmAssert( pItem != pItem->m_pNextInGlobal); + } + + // Link a cached item into the global list as the last MRU item. + // This routine assumes that the cache mutex for managing this list + // has already been locked. + + FINLINE void linkGlobalAsLastMRU( + F_CachedItem * pItem) + { + if( !m_pLastMRUItem) + { + flmAssert( !m_pMRUItem); + linkGlobalAsMRU( pItem); + return; + } + + flmAssert( m_pLastMRUItem); + +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + + if( m_pLastMRUItem->m_pNextInGlobal) + { +#ifdef FLM_CACHE_PROTECT + m_pLastMRUItem->m_pNextInGlobal->unprotectCachedItem(); +#endif + m_pLastMRUItem->m_pNextInGlobal->m_pPrevInGlobal = pItem; +#ifdef FLM_CACHE_PROTECT + m_pLastMRUItem->m_pNextInGlobal->protectCachedItem(); +#endif + pItem->m_pNextInGlobal = m_pLastMRUItem->m_pNextInGlobal; + } + else + { + flmAssert( m_pLRUItem == m_pLastMRUItem); + m_pLRUItem = pItem; + } + +#ifdef FLM_CACHE_PROTECT + m_pLastMRUItem->unprotectCachedItem(); +#endif + m_pLastMRUItem->m_pNextInGlobal = pItem; +#ifdef FLM_CACHE_PROTECT + m_pLastMRUItem->protectCachedItem(); +#endif + + pItem->m_pPrevInGlobal = m_pLastMRUItem; + m_pLastMRUItem = pItem; + +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif + + flmAssert( pItem != pItem->m_pPrevInGlobal); + flmAssert( pItem != pItem->m_pNextInGlobal); + } + + // Link a cached item into the global list as the LRU item. This routine + // assumes that the cache mutex for managing this list + // has already been locked. + + FINLINE void linkGlobalAsLRU( + F_CachedItem * pItem) + { +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + + if ((pItem->m_pPrevInGlobal = m_pLRUItem) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pItem->m_pPrevInGlobal->unprotectCachedItem(); +#endif + pItem->m_pPrevInGlobal->m_pNextInGlobal = pItem; +#ifdef FLM_CACHE_PROTECT + pItem->m_pPrevInGlobal->protectCachedItem(); +#endif + } + else + { + flmAssert( !m_pMRUItem); + flmAssert( !m_pLastMRUItem); + + m_pMRUItem = pItem; + m_pLastMRUItem = pItem; + } + + pItem->m_pNextInGlobal = NULL; +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif + m_pLRUItem = pItem; + + flmAssert( pItem != pItem->m_pPrevInGlobal); + flmAssert( pItem != pItem->m_pNextInGlobal); + } + + // Unlink a cached item from the global list. This routine + // assumes that the cache mutex for managing this list + // has already been locked. + + FINLINE void unlinkGlobal( + F_CachedItem * pItem) + { + if( pItem == m_pLastMRUItem) + { + if( m_pLastMRUItem->m_pPrevInGlobal) + { + m_pLastMRUItem = m_pLastMRUItem->m_pPrevInGlobal; + } + else + { + m_pLastMRUItem = m_pLastMRUItem->m_pNextInGlobal; + } + } + + if (pItem->m_pNextInGlobal) + { + flmAssert( pItem != m_pLRUItem); + +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->unprotectCachedItem(); +#endif + pItem->m_pNextInGlobal->m_pPrevInGlobal = pItem->m_pPrevInGlobal; +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->protectCachedItem(); +#endif + } + else + { + m_pLRUItem = pItem->m_pPrevInGlobal; + } + + if (pItem->m_pPrevInGlobal) + { + flmAssert( pItem != m_pMRUItem); + +#ifdef FLM_CACHE_PROTECT + pItem->m_pPrevInGlobal->unprotectCachedItem(); +#endif + pItem->m_pPrevInGlobal->m_pNextInGlobal = pItem->m_pNextInGlobal; +#ifdef FLM_CACHE_PROTECT + pItem->m_pPrevInGlobal->protectCachedItem(); +#endif + } + else + { + m_pMRUItem = pItem->m_pNextInGlobal; + } + +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + pItem->m_pNextInGlobal = NULL; + pItem->m_pPrevInGlobal = NULL; +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif + } + + // Moves a cached item one step closer to the MRU slot in the global list. + // This routine assumes that the cache mutex for managing this list + // has already been locked. + + FINLINE void stepUpInGlobal( + F_CachedItem * pItem) + { + F_CachedItem * pPrevItem; + + if ((pPrevItem = pItem->m_pPrevInGlobal) != NULL) + { + if( pItem == m_pLastMRUItem) + { + m_pLastMRUItem = m_pLastMRUItem->m_pPrevInGlobal; + } + + if (pPrevItem->m_pPrevInGlobal) + { +#ifdef FLM_CACHE_PROTECT + pPrevItem->m_pPrevInGlobal->unprotectCachedItem(); +#endif + pPrevItem->m_pPrevInGlobal->m_pNextInGlobal = pItem; +#ifdef FLM_CACHE_PROTECT + pPrevItem->m_pPrevInGlobal->protectCachedItem(); +#endif + } + else + { + m_pMRUItem = pItem; + } + +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + pItem->m_pPrevInGlobal = pPrevItem->m_pPrevInGlobal; +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif +#ifdef FLM_CACHE_PROTECT + pPrevItem->unprotectCachedItem(); +#endif + pPrevItem->m_pPrevInGlobal = pItem; + pPrevItem->m_pNextInGlobal = pItem->m_pNextInGlobal; +#ifdef FLM_CACHE_PROTECT + pPrevItem->protectCachedItem(); +#endif + + if (pItem->m_pNextInGlobal) + { +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->unprotectCachedItem(); +#endif + pItem->m_pNextInGlobal->m_pPrevInGlobal = pPrevItem; +#ifdef FLM_CACHE_PROTECT + pItem->m_pNextInGlobal->protectCachedItem(); +#endif + } + else + { + m_pLRUItem = pPrevItem; + } + +#ifdef FLM_CACHE_PROTECT + pItem->unprotectCachedItem(); +#endif + pItem->m_pNextInGlobal = pPrevItem; +#ifdef FLM_CACHE_PROTECT + pItem->protectCachedItem(); +#endif + } + } + +private: + + F_CachedItem * m_pMRUItem; + F_CachedItem * m_pLRUItem; + F_CachedItem * m_pLastMRUItem; + +friend class F_CachedItem; +friend class F_RowCacheMgr; +friend class F_BlockCacheMgr; +friend class F_CachedBlock; +friend class F_RowRelocator; +friend class F_ColumnDataRelocator; +friend class F_ColumnListRelocator; +friend class F_BlockRelocator; +}; + +/*************************************************************************** +Desc: Global cache manager for FLAIM-SQL. +***************************************************************************/ +class F_GlobalCacheMgr : public F_Object +{ +public: + F_GlobalCacheMgr(); + + ~F_GlobalCacheMgr(); + + RCODE setup( void); + + FINLINE void incrTotalBytes( + FLMUINT uiIncrAmount) + { + m_pSlabManager->incrementTotalBytesAllocated( uiIncrAmount, FALSE); + } + + FINLINE void decrTotalBytes( + FLMUINT uiDecrAmount) + { + m_pSlabManager->decrementTotalBytesAllocated( uiDecrAmount, FALSE); + } + + FINLINE FLMUINT totalBytes( void) + { + return( m_pSlabManager->totalBytesAllocated()); + } + + FINLINE FLMUINT availSlabs( void) + { + return( m_pSlabManager->availSlabs()); + } + + FINLINE FLMUINT allocatedSlabs( void) + { + return( m_pSlabManager->getTotalSlabs()); + } + + FINLINE FLMBOOL cacheOverLimit( void) + { + if( allocatedSlabs() > m_uiMaxSlabs) + { + return( TRUE); + } + + return( FALSE); + } + + RCODE setCacheLimit( + FLMUINT uiMaxCache, + FLMBOOL bPreallocateCache); + + RCODE setDynamicMemoryLimit( + FLMUINT uiCacheAdjustPercent, + FLMUINT uiCacheAdjustMin, + FLMUINT uiCacheAdjustMax, + FLMUINT uiCacheAdjustMinToLeave); + + RCODE setHardMemoryLimit( + FLMUINT uiPercent, + FLMBOOL bPercentOfAvail, + FLMUINT uiMin, + FLMUINT uiMax, + FLMUINT uiMinToLeave, + FLMBOOL bPreallocate); + + void getCacheInfo( + SFLM_CACHE_INFO * pMemInfo); + + RCODE adjustCache( + FLMUINT * puiCurrTime, + FLMUINT * puiLastCacheAdjustTime); + + RCODE clearCache( + F_Db * pDb); + + FINLINE void lockMutex( void) + { + f_mutexLock( m_hMutex); + } + + FINLINE void unlockMutex( void) + { + f_mutexUnlock( m_hMutex); + } + +private: + + IF_SlabManager * m_pSlabManager; + FLMUINT m_uiMaxBytes; + FLMUINT m_uiMaxSlabs; + FLMBOOL m_bCachePreallocated; + FLMBOOL m_bDynamicCacheAdjust; + // Is cache to be dynamically adjusted? + FLMUINT m_uiCacheAdjustPercent; + // Percent of available memory to adjust to. + FLMUINT m_uiCacheAdjustMin; + // Minimum limit to adjust cache to. + FLMUINT m_uiCacheAdjustMax; + // Maximum limit to adjust cache to. + FLMUINT m_uiCacheAdjustMinToLeave; + // Minimum bytes to leave when adjusting cache. + FLMUINT m_uiCacheAdjustInterval; + // Interval for adjusting cache limit. + FLMUINT m_uiCacheCleanupInterval; + // Interval for cleaning up old things out of + // cache. + FLMUINT m_uiUnusedCleanupInterval; + // Interval for cleaning up unused structures + F_MUTEX m_hMutex; // Mutex to control access to global cache + // manager object. + +friend class F_CachedItem; +friend class F_Row; +friend class F_CachedBlock; +friend class F_BlockCacheMgr; +friend class F_RowCacheMgr; +friend class F_Database; +friend class F_DbSystem; +}; + +/**************************************************************************** +Desc: Class for moving cache blocks in cache. +****************************************************************************/ +class F_BlockRelocator : public IF_Relocator +{ +public: + + F_BlockRelocator() + { + } + + virtual ~F_BlockRelocator() + { + } + + void relocate( + void * pvOldAlloc, + void * pvNewAlloc); + + FLMBOOL canRelocate( + void * pvOldAlloc); +}; + +/**************************************************************************** +Desc: This class manages block cache. +****************************************************************************/ +class F_BlockCacheMgr : public F_Object +{ +public: + F_BlockCacheMgr(); + + ~F_BlockCacheMgr(); + + RCODE initCache( void); + + void cleanupLRUCache( void); + + void cleanupReplaceList( void); + + void cleanupFreeCache( void); + + void reduceReuseList( void); + + RCODE reduceCache( + F_Db * pDb); + + RCODE rehash( void); + + RCODE allocBlock( + F_Db * pDb, + F_CachedBlock ** ppSCache); + + // Returns a pointer to the correct entry in the block cache hash table for + // the given block address + + FINLINE F_CachedBlock ** blockHash( + FLMUINT uiSigBitsInBlkSize, + FLMUINT uiBlkAddress + ) + { + return( &m_ppHashBuckets[ (uiBlkAddress >> + uiSigBitsInBlkSize) & m_uiHashMask]); + } + + FINLINE void defragmentMemory( + FLMBOOL bMutexLocked = FALSE) + { + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + } + + m_pBlockAllocator->defragmentMemory(); + + if( !bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + } + + FINLINE void getUsageStats( + FLM_SLAB_USAGE * pUsage) + { + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->lockMutex(); + f_memcpy( pUsage, &m_Usage, sizeof( FLM_SLAB_USAGE)); + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->unlockMutex(); + } + +private: + + RCODE initHashTbl( void); + + F_CacheList m_MRUList; // List of all block objects in MRU order. + F_CachedBlock * m_pMRUReplace; // Pointer to the MRU end of the list + // of cache items with no flags set. + F_CachedBlock * m_pLRUReplace; // Pointer to the LRU end of the list + // of cache items with no flags set. + F_CachedBlock * m_pFirstFree; + // Pointer to a linked list of cache + // blocks that need to be freed. + // Cache blocks in this list are no + // longer associated with a file and can + // be freed or re-used as needed. They + // are linked using the pNextInFile and + // pPrevInFile pointers. + F_CachedBlock * m_pLastFree; + // Pointer to a linked list of cache + // blocks that need to be freed. + // Cache blocks in this list are no + // longer associated with a file and can + // be freed or re-used as needed. They + // are linked using the pNextInFile and + // pPrevInFile pointers. + SFLM_CACHE_USAGE m_Usage; // Contains usage information. + FLMUINT m_uiFreeBytes; // Number of free bytes + FLMUINT m_uiFreeCount; // Number of free blocks + FLMUINT m_uiReplaceableCount; + // Number of blocks whose flags are 0 + FLMUINT m_uiReplaceableBytes; + // Number of bytes belonging to blocks whose + // flags are 0 + FLMBOOL m_bAutoCalcMaxDirty; + // Flag indicating we should automatically + // calculate maximum dirty cache. + FLMUINT m_uiMaxDirtyCache; + // Maximum cache that can be dirty. + FLMUINT m_uiLowDirtyCache; + // When maximum dirty cache is exceeded, + // threshhold it should be brought back + // under + FLMUINT m_uiTotalUses; // Total number of uses currently held + // on blocks in cache. + FLMUINT m_uiBlocksUsed;// Total number of blocks in cache that + // are being used. + FLMUINT m_uiPendingReads; + // Total reads currently pending. + FLMUINT m_uiIoWaits; // Number of times multiple threads + // were reading the same block from + // disk at the same time. + F_CachedBlock ** m_ppHashBuckets;// This is a pointer to a hash table that + // is used to find cache blocks. Each + // element in the table points to a + // linked list of F_CachedBlock objects that + // all hash to the same hash bucket. + FLMUINT m_uiNumBuckets;// This contains the number of buckets + // in the hash table. + FLMUINT m_uiHashFailTime; + // Last time we tried to rehash and + // failed. Want to wait before we + // retry again. + FLMUINT m_uiHashMask; // Bits that are significant + // for the number of hash buckets. + IF_MultiAlloc * m_pBlockAllocator; + // Fixed size allocators for cache blocks + F_BlockRelocator m_blockRelocator; + // Relocator for cache blocks + FLMBOOL m_bReduceInProgress; +#ifdef FLM_DEBUG + FLMBOOL m_bDebug; // Enables checksumming and cache use + // monitoring. Only available when + // debug is compiled in. +#endif + +friend class F_CachedBlock; +friend class F_GlobalCacheMgr; +friend class F_Database; +friend class F_Db; +friend class F_DbSystem; +friend class F_BlockRelocator; +}; + +#ifdef FLM_DEBUG +/**************************************************************************** +Struct: SCACHE_USE (Cache Block Use) +Desc: This is a debug only structure that is used to keep track of the + threads that are currently using a block. +****************************************************************************/ +typedef struct SCache_Use +{ + SCache_Use * pNext; // Pointer to next SCACHE_USE structure in + // the list. + FLMUINT uiThreadId; // Thread ID of thread using the block. + FLMUINT uiUseCount; // Use count for this particular thread. +} SCACHE_USE; +#endif + +// Flags for m_ui16Flags field in F_CachedBlock + +#define CA_DIRTY 0x0001 + // This bit indicates that the block is + // dirty and needs to be flushed to disk. + // NOTE: For 3.x files, this bit may remain + // set on prior versions of blocks until the + // current transaction commits. +#define CA_WRITE_INHIBIT 0x0002 + // Must not write block until use count + // goes to zero. NOTE: Can ignore when + // in the checkpoint thread. +#define CA_READ_PENDING 0x0004 + // This bit indicates that the block is + // currently being read in from disk. +#define CA_WRITE_TO_LOG 0x0008 + // This bit indicates that this version of + // the block should be written to the + // rollback log before being replaced. + // During an update transaction, the first + // time a block is updated, FLAIM will + // create a new version of the block and + // insert it into cache. The prior version + // of the block is marked with this flag + // to indicate that it needs to be written + // to the log before it can be replaced. +#define CA_LOG_FOR_CP 0x0010 + // This bit indicates that this version of + // the block needs to be logged to the + // physical rollback in order to restore + // the last checkpoint. This is only + // applicable to 3.x files. +#define CA_WAS_DIRTY 0x0020 + // This bit indicates that this version of + // the block was dirty before the newer + // version of the block was created. + // Its dirty state should be restored if + // the current transaction aborts. This + // flag is only used for 3.x files. +#define CA_WRITE_PENDING 0x0040 + // This bit indicates that a block is in + // the process of being written out to + // disk. +#define CA_IN_WRITE_PENDING_LIST 0x0080 + // This bit indicates that a block is in + // the write pending list. +#define CA_FREE 0x0100 + // The block has been linked to the free + // list (and unlinked from all other lists) +#define CA_IN_FILE_LOG_LIST 0x0200 + // Block is in the list of blocks that may + // have one or more versions that need to + // be logged +#define CA_IN_NEW_LIST 0x0400 + // Dirty block that is beyond the last CP EOF +#define CA_DUMMY_FLAG 0x0800 + // Used to prevent blocks from being linked + // into the replace list in cases where + // they will be removed immediately (because + // a bit is going to being set) + +/**************************************************************************** +Desc: This is the header structure for a cached data block. +****************************************************************************/ +class F_CachedBlock : public F_CachedItem +{ +public: + F_CachedBlock( + FLMUINT uiBlockSize); + + ~F_CachedBlock(); + +#ifdef FLM_CACHE_PROTECT + FINLINE void protectCachedItem( void) + { + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->protectBuffer( this); + } +#endif + +#ifdef FLM_CACHE_PROTECT + FINLINE void unprotectCachedItem( void) + { + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->unprotectBuffer( this); + } +#endif + + FINLINE FLMUINT memSize( void) + { + return( gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->getTrueSize( + (FLMBYTE *)this)); + } + + FINLINE FLMUINT blkAddress( void) + { + return( m_uiBlkAddress); + } + + FINLINE F_BLK_HDR * getBlockPtr( void) + { + return( m_pBlkHdr); + } + + FINLINE F_Database * getDatabase( void) + { + return( m_pDatabase); + } + + FINLINE FLMUINT16 getModeFlags( void) + { + return( m_ui16Flags); + } + + FINLINE FLMUINT getUseCount( void) + { + return( m_uiUseCount); + } + + // Gets the prior image block address from the block header. + // NOTE: This function assumes that the block cache mutex is locked. + + FINLINE FLMUINT getPriorImageAddress( void) + { + return( (FLMUINT)m_pBlkHdr->ui32PriorBlkImgAddr); + } + + // Gets the transaction ID from the block header. NOTE: This function + // assumes that the block cache mutex is locked. + + FINLINE FLMUINT64 getLowTransID( void) + { + return( m_pBlkHdr->ui64TransID); + } + + // Set the high transaction ID for a cache block. + // NOTE: This function assumes that the block cache mutex is locked. + + FINLINE void setTransID( + FLMUINT64 ui64NewTransID) + { + if (m_ui64HighTransID == FLM_MAX_UINT64 && ui64NewTransID != FLM_MAX_UINT64) + { + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerBytes += memSize(); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerCount++; + } + else if (m_ui64HighTransID != FLM_MAX_UINT64 && ui64NewTransID == FLM_MAX_UINT64) + { + FLMUINT uiSize = memSize(); + + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerBytes >= uiSize); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerBytes -= uiSize; + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerCount); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerCount--; + } + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_ui64HighTransID = ui64NewTransID; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Determines if a cache block is needed by a read transaction. + FINLINE FLMBOOL neededByReadTrans( void) + { + return( m_pDatabase->neededByReadTrans( getLowTransID(), + m_ui64HighTransID)); + } + + // Link a cache block into the replace list as the MRU item. This routine + // assumes that the block cache mutex has already been locked. + FINLINE void linkToReplaceListAsMRU( void) + { + flmAssert( !m_ui16Flags); + + if ((m_pNextInReplaceList = + gv_SFlmSysData.pBlockCacheMgr->m_pMRUReplace) != NULL) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInReplaceList->unprotectCachedItem(); +#endif + m_pNextInReplaceList->m_pPrevInReplaceList = this; +#ifdef FLM_CACHE_PROTECT + m_pNextInReplaceList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pLRUReplace = this; + } + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_pPrevInReplaceList = NULL; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + + gv_SFlmSysData.pBlockCacheMgr->m_pMRUReplace = this; + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableCount++; + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableBytes += memSize(); + } + + // Link a cache block into the replace list as the LRU item. This routine + // assumes that the block cache mutex has already been locked. + FINLINE void linkToReplaceListAsLRU( void) + { + flmAssert( !m_ui16Flags); + + if ((m_pPrevInReplaceList = gv_SFlmSysData.pBlockCacheMgr->m_pLRUReplace) != NULL) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInReplaceList->unprotectCachedItem(); +#endif + m_pPrevInReplaceList->m_pNextInReplaceList = this; +#ifdef FLM_CACHE_PROTECT + m_pPrevInReplaceList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pMRUReplace = this; + } + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_pNextInReplaceList = NULL; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + gv_SFlmSysData.pBlockCacheMgr->m_pLRUReplace = this; + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableCount++; + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableBytes += memSize(); + } + + // Moves a block one step closer to the MRU slot in the replace list. + // This routine assumes that the block cache mutex has already been locked. + FINLINE void stepUpInReplaceList( void) + { + F_CachedBlock * pPrevSCache; + + flmAssert( !m_ui16Flags); + + if( (pPrevSCache = m_pPrevInReplaceList) != NULL) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if( pPrevSCache->m_pPrevInReplaceList) + { +#ifdef FLM_CACHE_PROTECT + pPrevSCache->m_pPrevInReplaceList->unprotectCachedItem(); +#endif + pPrevSCache->m_pPrevInReplaceList->m_pNextInReplaceList = this; +#ifdef FLM_CACHE_PROTECT + pPrevSCache->m_pPrevInReplaceList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pMRUReplace = this; + } + + m_pPrevInReplaceList = pPrevSCache->m_pPrevInReplaceList; + +#ifdef FLM_CACHE_PROTECT + pPrevSCache->unprotectCachedItem(); +#endif + pPrevSCache->m_pPrevInReplaceList = this; + pPrevSCache->m_pNextInReplaceList = m_pNextInReplaceList; +#ifdef FLM_CACHE_PROTECT + pPrevSCache->protectCachedItem(); +#endif + + if( m_pNextInReplaceList) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInReplaceList->unprotectCachedItem(); +#endif + m_pNextInReplaceList->m_pPrevInReplaceList = pPrevSCache; +#ifdef FLM_CACHE_PROTECT + m_pNextInReplaceList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pLRUReplace = pPrevSCache; + } + + m_pNextInReplaceList = pPrevSCache; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + } + + // Clears the passed-in flags from the F_CachedBlock object + // This routine assumes that the block cache mutex is locked. + FINLINE void clearFlags( + FLMUINT16 ui16FlagsToClear) + { + if( m_ui16Flags) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + if( (m_ui16Flags &= ~ui16FlagsToClear) == 0) + { + if( !m_pPrevInGlobal || + m_ui64HighTransID == ~((FLMUINT64)0) || + neededByReadTrans()) + { + linkToReplaceListAsMRU(); + } + else + { + linkToReplaceListAsLRU(); + } + } +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + } + + // Sets the passed-in flags on the object + // This routine assumes that the block cache mutex is locked. + + FINLINE void setFlags( + FLMUINT16 ui16FlagsToSet) + { + flmAssert( ui16FlagsToSet); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + if( !m_ui16Flags) + { + unlinkFromReplaceList(); + } + + m_ui16Flags |= ui16FlagsToSet; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Set the dirty flag on a cache block. + // This routine assumes that the block cache mutex is locked. + + FINLINE void setDirtyFlag( + F_Database * pDatabase) + { + flmAssert( !(m_ui16Flags & + (CA_DIRTY | CA_WRITE_PENDING | CA_IN_FILE_LOG_LIST | CA_IN_NEW_LIST))); + setFlags( CA_DIRTY); + pDatabase->incrementDirtyCacheCount(); + } + + // Unset the dirty flag on a cache block. + // This routine assumes that the block cache mutex is locked. + FINLINE void unsetDirtyFlag( void) + { + flmAssert( m_ui16Flags & CA_DIRTY); + flmAssert( m_pDatabase->getDirtyCacheCount()); + + if (m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + unlinkFromLogList(); + } + else if (m_ui16Flags & CA_IN_NEW_LIST) + { + unlinkFromNewList(); + } + + clearFlags( CA_DIRTY); + m_pDatabase->decrementDirtyCacheCount(); + } + + FINLINE FLMUINT getBlkSize( void) + { + return( (FLMUINT)m_ui16BlkSize); + } + +#ifdef FLM_DBG_LOG + void logFlgChange( + FLMUINT16 ui16OldFlags, + char cPlace); +#endif + + void linkToLogList( void); + + void unlinkFromLogList( void); + + void linkToNewList( void); + + void unlinkFromNewList( void); + + void linkToDatabase( + F_Database * pDatabase); + + void unlinkFromDatabase( void); + + void unlinkFromTransLogList( void); + + // Increment the use count on a cache block for a particular + // thread. NOTE: This routine assumes that the block cache mutex + // is locked. + FINLINE void useForThread( + #ifdef FLM_DEBUG + FLMUINT uiThreadId) + #else + FLMUINT) // uiThreadId) + #endif + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + #ifdef FLM_DEBUG + if (m_pUseList || + (gv_SFlmSysData.pBlockCacheMgr->m_bDebug && !m_uiUseCount)) + { + dbgUseForThread( uiThreadId); + } + else + #endif + { + if (!m_uiUseCount) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiBlocksUsed++; + } + m_uiUseCount++; + gv_SFlmSysData.pBlockCacheMgr->m_uiTotalUses++; + } + } + + // Decrement the use count on a cache block for a particular + // thread. NOTE: This routine assumes that the block cache mutex + // is locked. + FINLINE void releaseForThread( void) + { + if (!m_uiUseCount) + { + return; + } + + #ifdef FLM_DEBUG + if (m_pUseList) + { + dbgReleaseForThread(); + } + else + #endif + { + + #ifdef FLM_DEBUG + + // If count is one, it will be decremented to zero. + + if (m_uiUseCount == 1) + { + m_uiChecksum = computeChecksum(); + } + #endif + + m_uiUseCount--; + gv_SFlmSysData.pBlockCacheMgr->m_uiTotalUses--; + if (!m_uiUseCount) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiBlocksUsed--; + } + } + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Tests if a block can be freed from cache. + // NOTE: This routine assumes that the block cache mutex is locked. + FINLINE FLMBOOL canBeFreed( void) + { + if (!m_uiUseCount && !m_ui16Flags) + { + F_CachedBlock * pNewerSCache = m_pPrevInVersionList; + + // The following code is attempting to ensure that newer + // versions of the block have had the prior block address + // properly transferred to them from an older version of + // the block. If not, we cannot remove the current version + // of the block (pointed to by pSCache), because it is + // the older version that needs to be logged in order for + // the prior block address to be properly transferred to + // the newer version of the block. + + // If there is no newer version of the block, we can remove + // this block, because it means that there was at one point + // in time a newer version, the prior block address was + // safely transferred - otherwise, the newer version would + // still be in cache. + + // If there is a newer version of the block, but it is in the + // process of being read in from disk (CA_READ_PENDING bit is + // set), we can know that the prior block address has been + // properly transferred to the block being read in. + // Explanation: If the CA_READ_PENDING bit it set, the block + // had to have been written out to disk at some prior time. The + // rules for writing out a block to disk are such that it is + // impossible for a block to be written out without having a + // pointer to some prior version of the block. The only + // exception to this is a newly created block - but in that + // case, the block does not need to have a prior version pointer + // - because there are none! + // This assertion is obvious for a version of a block that is + // being read from the rollback log - it would be impossible + // to be reading such a block from the rollback log if it hadn't + // been part of a version chain! As for the current version of a + // block, it cannot be written out and removed from cache without + // having a pointer to the chain of older versions that may still + // be needed (by a read transactions, for rollback, or to recover + // a checkpoint). + + // NOTE: Although we know that a block being read in from disk + // has to already have a prior block address, we cannot just + // look at the block header, because it is being read in from + // disk and the prior block address is not yet there. Usually, + // it will still be zeroes - making it look as though the block + // does not have a prior block address when, in fact, it does. + // Thus, we look at the CA_READ_PENDING bit first. If that + // is not set, we can safely look at the prior block address. + + // Note also that even if there is a newer block that doesn't + // have a prior block address, we may still be able to remove + // the current block (pSCache) if it is not needed by any + // read transactions. + + if (!pNewerSCache || + (pNewerSCache->m_ui16Flags & CA_READ_PENDING) || + pNewerSCache->getPriorImageAddress() != 0 || + !m_pDatabase || + !neededByReadTrans()) + { + return( TRUE); + } + } + return( FALSE); + } + + void linkToFreeList( + FLMUINT uiFreeTime); + + void unlinkFromFreeList( void); + + void * operator new( + FLMSIZET uiSize, + FLMUINT uiBlockSize, + FLMBOOL bAllocMutexLocked = FALSE) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new( + FLMSIZET uiSize) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new( + FLMSIZET uiSize, + const char * pszFile, + int iLine) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new[]( + FLMSIZET uiSize) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new[]( + FLMSIZET uiSize, + const char * pszFile, + int iLine) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void operator delete( + void * ptr); + +#if !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete( + void * ptr, + FLMSIZET uiSize, + const char * pszFileName, + int iLine); +#endif + + void operator delete[]( + void * ptr); + +#if !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete( + void * ptr, + FLMUINT uiBlockSize, + FLMBOOL bAllocMutexLocked); +#endif + + +#if !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete( + void * ptr, + const char * file, + int line); +#endif + +#if !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete[]( + void * ptr, + const char * pszFileName, + int iLineNum); +#endif + +private: + + void unlinkFromReplaceList( void); + +#ifdef FLM_DEBUG + void dbgUseForThread( + FLMUINT uiThreadId); + + void dbgReleaseForThread( void); + + FLMUINT computeChecksum( void); +#endif + +#ifdef SCACHE_LINK_CHECKING + void verifyCache( + int iPlace); +#else + FINLINE void verifyCache( + int) + { + } +#endif + + // Link a cached block into the global list as the MRU item. This routine + // assumes that the block cache mutex has already been locked. + + FINLINE void linkToGlobalListAsMRU( void) + { + if( (m_pBlkHdr->ui8BlkType & BT_FREE) || + (m_pBlkHdr->ui8BlkType & BT_LEAF) || + (m_pBlkHdr->ui8BlkType & BT_LEAF_DATA) || + (m_pBlkHdr->ui8BlkType & BT_DATA_ONLY)) + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.linkGlobalAsLastMRU( + (F_CachedItem *)this); + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.linkGlobalAsMRU( + (F_CachedItem *)this); + } + + if( !m_ui16Flags) + { + linkToReplaceListAsMRU(); + } + } + + // Link a cached block into the global list as the LRU item. This routine + // assumes that the block cache mutex has already been locked. + + FINLINE void linkToGlobalListAsLRU( void) + { + if( (m_pBlkHdr->ui8BlkType & BT_FREE) || + (m_pBlkHdr->ui8BlkType & BT_LEAF) || + (m_pBlkHdr->ui8BlkType & BT_LEAF_DATA) || + (m_pBlkHdr->ui8BlkType & BT_DATA_ONLY)) + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.linkGlobalAsLRU( + (F_CachedItem *)this); + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.linkGlobalAsLastMRU( + (F_CachedItem *)this); + } + + if( !m_ui16Flags) + { + linkToReplaceListAsLRU(); + } + } + + // Unlink a cache block from the global list. This routine + // assumes that the block cache mutex has already been locked. + + FINLINE void unlinkFromGlobalList( void) + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.unlinkGlobal( + (F_CachedItem *)this); + if( !m_ui16Flags) + { + unlinkFromReplaceList(); + } + } + + // Moves a block one step closer to the MRU slot in the global list. This + // routine assumes that the block cache mutex has already been locked. + + FINLINE void stepUpInGlobalList( void) + { + gv_SFlmSysData.pBlockCacheMgr->m_MRUList.stepUpInGlobal( + (F_CachedItem *)this); + if( !m_ui16Flags) + { + stepUpInReplaceList(); + } + } + + #ifdef SCACHE_LINK_CHECKING + void checkHashLinks( + F_CachedBlock ** ppSCacheBucket); + + void checkHashUnlinks( + F_CachedBlock ** ppSCacheBucket); + #endif + + // Link a cache block to its hash bucket. This routine assumes + // that the block cache mutex has already been locked. + FINLINE void linkToHashBucket( + F_CachedBlock ** ppSCacheBucket) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + #ifdef SCACHE_LINK_CHECKING + checkHashLinks( ppSCacheBucket); + #endif + m_pPrevInHashBucket = NULL; + if ((m_pNextInHashBucket = *ppSCacheBucket) != NULL) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInHashBucket->unprotectCachedItem(); +#endif + m_pNextInHashBucket->m_pPrevInHashBucket = this; +#ifdef FLM_CACHE_PROTECT + m_pNextInHashBucket->protectCachedItem(); +#endif + } + *ppSCacheBucket = this; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Unlink a cache block from its hash bucket. This routine assumes + // that the block cache mutex has already been locked. + FINLINE void unlinkFromHashBucket( + F_CachedBlock ** ppSCacheBucket) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + #ifdef SCACHE_LINK_CHECKING + checkHashUnlinks( ppSCacheBucket); + #endif + + // Make sure it is not in the list of log blocks. + + flmAssert( !(m_ui16Flags & CA_WRITE_TO_LOG)); + + if (m_pNextInHashBucket) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInHashBucket->unprotectCachedItem(); +#endif + m_pNextInHashBucket->m_pPrevInHashBucket = m_pPrevInHashBucket; +#ifdef FLM_CACHE_PROTECT + m_pNextInHashBucket->protectCachedItem(); +#endif + } + + if (m_pPrevInHashBucket) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInHashBucket->unprotectCachedItem(); +#endif + m_pPrevInHashBucket->m_pNextInHashBucket = m_pNextInHashBucket; +#ifdef FLM_CACHE_PROTECT + m_pPrevInHashBucket->protectCachedItem(); +#endif + } + else + { + *ppSCacheBucket = m_pNextInHashBucket; + } + + m_pNextInHashBucket = NULL; + m_pPrevInHashBucket = NULL; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + void unlinkCache( + FLMBOOL bFreeIt, + RCODE NotifyRc); + + void savePrevBlkAddress( void); + + F_CachedBlock * m_pPrevInDatabase; // This is a pointer to the previous block + // in the linked list of blocks that are + // in the same database. + F_CachedBlock * m_pNextInDatabase; // This is a pointer to the next block in + // the linked list of blocks that are in + // the same database. + F_BLK_HDR * m_pBlkHdr; // Pointer to this block's header and data. + F_Database * m_pDatabase; // Pointer to the database this data block + // belongs to. + FLMUINT m_uiBlkAddress; // Block address. + F_CachedBlock * m_pNextInReplaceList;// This is a pointer to the next block in + // the global linked list of cache blocks + // that have a flags value of zero. + F_CachedBlock * m_pPrevInReplaceList;// This is a pointer to the previous block in + // the global linked list of cache blocks + // that have a flags value of zero. + F_CachedBlock * m_pPrevInHashBucket; // This is a pointer to the previous block + // in the linked list of blocks that are + // in the same hash bucket. + F_CachedBlock * m_pNextInHashBucket; // This is a pointer to the next block in + // the linked list of blocks that are in + // the same hash bucket. + F_CachedBlock * m_pPrevInVersionList;// This is a pointer to the previous block + // in the linked list of blocks that are + // all just different versions of the + // same block. The previous block is a + // more recent version of the block. + F_CachedBlock * m_pNextInVersionList;// This is a pointer to the next block in + // the linked list of blocks that are all + // just different versions of the same + // block. The next block is an older + // version of the block. + FNOTIFY * m_pNotifyList; // This is a pointer to a list of threads + // that want to be notified when a pending + // I/O is complete. This pointer is only + // non-null if the block is currently being + // read from disk and there are multiple + // threads all waiting for the block to + // be read in. + FLMUINT64 m_ui64HighTransID; // This indicates the highest known moment + // in the file's update history when this + // version of the block was the active + // block. + // A block's low transaction ID and high + // transaction ID indicate a span of + // transactions where this version of the + // block was the active version of the + // block. + FLMUINT m_uiUseCount; // Number of times this block has been + // retrieved for use by threads. A use + // count of zero indicates that no thread + // is currently using the block. Note that + // a block cannot be replaced when its use + // count is non-zero. + FLMUINT16 m_ui16Flags; // This is a set of flags for the block + // that indicate various things about the + // block's current state. + FLMUINT16 m_ui16BlkSize; // Block size + +// NOTE: Keep debug items at the END of the structure. + +#ifdef FLM_DEBUG + FLMUINT m_uiChecksum; // Checksum for the block and header. + SCACHE_USE * m_pUseList; // This is a pointer to a list of threads + // that are currently using the block. +#endif +friend class F_BlockCacheMgr; +friend class F_GlobalCacheMgr; +friend class F_Database; +friend class F_DbSystem; +friend class F_Db; +friend class F_Btree; +friend class F_BTreeInfo; +friend class F_BlockRelocator; +}; + +/**************************************************************************** +Desc: Class for moving rows in cache. +****************************************************************************/ +class F_RowRelocator : public IF_Relocator +{ +public: + + F_RowRelocator() + { + } + + virtual ~F_RowRelocator() + { + } + + void relocate( + void * pvOldAlloc, + void * pvNewAlloc); + + FLMBOOL canRelocate( + void * pvOldAlloc); +}; + +/**************************************************************************** +Desc: Class for moving row data buffers in cache. +****************************************************************************/ +class F_ColumnDataRelocator : public IF_Relocator +{ +public: + + F_ColumnDataRelocator() + { + } + + virtual ~F_ColumnDataRelocator() + { + } + + void relocate( + void * pvOldAlloc, + void * pvNewAlloc); + + FLMBOOL canRelocate( + void * pvOldAlloc); +}; + +/**************************************************************************** +Desc: Class for moving row column lists in cache. +****************************************************************************/ +class F_ColumnListRelocator : public IF_Relocator +{ +public: + + F_ColumnListRelocator() + { + } + + virtual ~F_ColumnListRelocator() + { + } + + void relocate( + void * pvOldAlloc, + void * pvNewAlloc); + + FLMBOOL canRelocate( + void * pvOldAlloc); +}; + +/**************************************************************************** +Desc: This class is used to control the row cache. +****************************************************************************/ +class F_RowCacheMgr : public F_Object +{ +public: + F_RowCacheMgr(); + + ~F_RowCacheMgr(); + + RCODE initCache( void); + + void cleanupOldCache( void); + + void cleanupPurgedCache( void); + + void reduceCache( void); + + FINLINE void setDebugMode( + FLMBOOL bDebug) + { +#ifdef FLM_DEBUG + m_bDebug = bDebug; +#else + (void)bDebug; +#endif + } + + RCODE allocRow( + F_Row ** ppRow, + FLMBOOL bMutexLocked); + + RCODE retrieveRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + F_Row ** ppRow); + + RCODE createRow( + F_Db * pDb, + FLMUINT uiTableNum, + F_Row ** ppRow); + + RCODE makeWriteCopy( + F_Db * pDb, + F_Row ** ppRow); + + void removeRow( + F_Db * pDb, + F_Row * pRow, + FLMBOOL bDecrementUseCount, + FLMBOOL bMutexLocked = FALSE); + + void removeRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId); + + FINLINE void defragmentMemory( + FLMBOOL bMutexLocked = FALSE) + { + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + } + + m_pRowAllocator->defragmentMemory(); + m_pBufAllocator->defragmentMemory(); + + if( !bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + } + + FINLINE void getUsageStats( + FLM_SLAB_USAGE * pUsage) + { + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->lockMutex(); + f_memcpy( pUsage, &m_Usage, sizeof( FLM_SLAB_USAGE)); + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->unlockMutex(); + } + +private: + + // Hash function for hashing to rows in row cache. + // Assumes that the row cache mutex has already been locked. + FINLINE F_Row ** rowHash( + FLMUINT64 ui64RowId) + { + return( &m_ppHashBuckets[(FLMUINT)ui64RowId & m_uiHashMask]); + } + + RCODE rehash( void); + + RCODE waitNotify( + F_Db * pDb, + F_Row ** ppRow); + + void notifyWaiters( + FNOTIFY * pNotify, + F_Row * pUseRow, + RCODE NotifyRc); + + void linkIntoRowCache( + F_Row * pNewerRow, + F_Row * pOlderRow, + F_Row * pRow, + FLMBOOL bLinkAsMRU); + + void findRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT64 ui64VersionNeeded, + FLMBOOL bDontPoisonCache, + FLMUINT * puiNumLooks, + F_Row ** ppRow, + F_Row ** ppNewerRow, + F_Row ** ppOlderRow); + + RCODE readRowFromDisk( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + F_Row * pRow, + FLMUINT64 * pui64LowTransId, + FLMBOOL * pbMostCurrent); + + RCODE _makeWriteCopy( + F_Db * pDb, + F_Row ** ppRow); + + // Private Data + + F_CacheList m_MRUList; // List of all row objects in MRU order. + F_Row * m_pPurgeList; // List of F_Row objects that + // should be deleted when the use count + // goes to zero. + F_Row * m_pHeapList; // List of rows with heap allocations + F_Row * m_pOldList; // List of old versions + SFLM_CACHE_USAGE m_Usage; // Contains maximum, bytes used, etc. + F_Row ** m_ppHashBuckets; // Array of hash buckets. + FLMUINT m_uiNumBuckets; // Total number of hash buckets. + // must be an exponent of 2. + FLMUINT m_uiHashFailTime; + // Last time we tried to rehash and + // failed. Want to wait before we + // retry again. + FLMUINT m_uiHashMask; // Hash mask mask for hashing a + // row id to a hash bucket. + FLMUINT m_uiPendingReads; // Total reads currently pending. + FLMUINT m_uiIoWaits; // Number of times multiple threads + // were reading the same row from + // disk at the same time. + IF_FixedAlloc * m_pRowAllocator; // Fixed size allocator for F_Row + // objects + IF_BufferAlloc * m_pBufAllocator; // Buffer allocator for buffers in + // F_Row objects + + F_RowRelocator m_rowRelocator; + F_ColumnDataRelocator m_columnDataRelocator; + F_ColumnListRelocator m_columnListRelocator; + FLMBOOL m_bReduceInProgress; + +#ifdef FLM_DEBUG + FLMBOOL m_bDebug; // Debug mode? +#endif +friend class F_Row; +friend class F_GlobalCacheMgr; +friend class F_Database; +friend class F_DbSystem; +friend class F_RowRelocator; +friend class F_ColumnDataRelocator; +friend class F_ColumnListRelocator; +}; + +// Flags kept in F_Row::m_uiCacheFlags. Mutex needs to be locked to +// access these. + +#define NCA_READING_IN 0x80000000 +#define NCA_UNCOMMITTED 0x40000000 +#define NCA_LATEST_VER 0x20000000 +#define NCA_PURGED 0x10000000 +#define NCA_LINKED_TO_DATABASE 0x08000000 + +#define NCA_COUNTER_BITS (~(NCA_READING_IN | NCA_UNCOMMITTED | \ + NCA_LATEST_VER | NCA_PURGED | \ + NCA_LINKED_TO_DATABASE)) + +// Flags kept in F_Row::m_uiFlags. Mutex does not have to be locked to +// access these. + +#define FROW_DIRTY 0x00000010 +#define FROW_NEW 0x00000020 +#define FROW_HEAP_ALLOC 0x00000040 + +/***************************************************************************** +Desc: Header for each column +******************************************************************************/ +typedef struct F_COLUMN_ITEM +{ + FLMUINT uiDataLen; + FLMUINT uiDataOffset; +} F_COLUMN_ITEM; + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE FLMUINT allocOverhead( void) +{ + // Round sizeof( F_Row *) + 1 to nearest 8 byte boundary. + + return( (sizeof( F_Row *) + 9) & (~((FLMUINT)7))); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE FLMBYTE * getActualPointer( + void * pvPtr) +{ + if (pvPtr) + { + return( (FLMBYTE *)pvPtr - allocOverhead()); + } + else + { + return( NULL); + } +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE FLMUINT calcColumnListBufSize( + FLMUINT uiColumnCount) +{ + return( uiColumnCount * sizeof( F_COLUMN_ITEM) + allocOverhead()); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE FLMUINT calcDataBufSize( + FLMUINT uiDataSize) +{ + return( uiDataSize + allocOverhead()); +} + +/***************************************************************************** +Desc: Cached Row +******************************************************************************/ +class F_Row : public F_CachedItem +{ +public: + + F_Row(); + + ~F_Row(); + +#ifdef FLM_CACHE_PROTECT + FINLINE void protectCachedItem( void) + { + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->protectCell( this); + } + + FINLINE void unprotectCachedItem( void) + { + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->unprotectCell( this); + } +#endif + + // This method assumes that the row cache mutex has been locked. + + FINLINE FLMBOOL canBeFreed( void) + { + return( (!rowInUse() && !readingInRow() && !rowIsDirty()) ? TRUE : FALSE); + } + + // This method assumes that the row cache mutex has been locked. + + FINLINE void freeRow( void) + { + if (rowPurged()) + { + freePurged(); + } + else + { + freeCache( FALSE); + } + } + + FINLINE FLMUINT64 getLowTransId( void) + { + return( m_ui64LowTransId); + } + + FINLINE FLMUINT64 getHighTransId( void) + { + return( m_ui64HighTransId); + } + + RCODE resizeDataBuffer( + FLMUINT uiSize, + FLMBOOL bMutexAlreadyLocked); + + RCODE resizeColumnList( + FLMUINT uiColumnCount, + FLMBOOL bMutexAlreadyLocked); + + RCODE flushRow( + F_Db * pDb, + F_Btree * pBTree); + + RCODE flushRow( + F_Db * pDb); + + RCODE copyColumnList( + F_Db * pDb, + F_Row * pSourceRow, + FLMBOOL bMutexAlreadyLocked); + + RCODE readRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + IF_IStream * pIStream, + FLMUINT uiRowDataLength); + + FINLINE void setRowAndDataPtr( + FLMBYTE * pucActualAlloc) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + *((F_Row **)(pucActualAlloc)) = this; + m_pucColumnData = pucActualAlloc + allocOverhead(); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + FINLINE void setColumnListPtr( + FLMBYTE * pucActualAlloc) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + *((F_Row **)(pucActualAlloc)) = this; + m_pColumns = (F_COLUMN_ITEM *)(pucActualAlloc + allocOverhead()); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + FINLINE FLMUINT64 getRowId( void) + { + return( m_ui64RowId); + } + + FINLINE F_Database * getDatabase( void) + { + return( m_pDatabase); + } + + FINLINE FLMUINT getTableNum( void) + { + return( m_uiTableNum); + } + + FINLINE FLMUINT getOffsetIndex( void) + { + return( m_uiOffsetIndex); + } + + FINLINE void setOffsetIndex( + FLMUINT uiOffsetIndex) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiOffsetIndex = uiOffsetIndex; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + FINLINE FLMUINT32 getBlkAddr( void) + { + return( m_ui32BlkAddr); + } + + FINLINE void setBlkAddr( + FLMUINT32 ui32BlkAddr) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_ui32BlkAddr = ui32BlkAddr; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + FINLINE FLMBOOL isRightVersion( + FLMUINT64 ui64TransId) + { + return( (ui64TransId >= m_ui64LowTransId && + ui64TransId <= m_ui64HighTransId) + ? TRUE + : FALSE); + } + + // Generally, assumes that the row cache mutex has already been locked. + // There is one case where it is not locked, but it is not + // critical that it be locked - inside syncFromDb. + + FINLINE FLMBOOL rowPurged( void) + { + return( (m_uiCacheFlags & NCA_PURGED ) ? TRUE : FALSE); + } + +#ifdef FLM_DEBUG + void checkReadFromDisk( + F_Db * pDb); +#endif + + void * operator new( + FLMSIZET uiSize) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new( + FLMSIZET uiSize, + const char * pszFile, + int iLine) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new[]( + FLMSIZET uiSize) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void * operator new[]( + FLMSIZET uiSize, + const char * pszFile, + int iLine) + #if !defined( FLM_NLM) + throw() + #endif + ; + + void operator delete( + void * ptr); + + void operator delete[]( + void * ptr); + +#if defined( FLM_DEBUG) && !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete( + void * ptr, + const char * file, + int line); +#endif + +#if defined( FLM_DEBUG) && !defined( __WATCOMC__) && !defined( FLM_SOLARIS) + void operator delete[]( + void * ptr, + const char * pszFileName, + int iLineNum); +#endif + + // Assumes that the row cache mutex has already been locked. + FINLINE void incrRowUseCount( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags = (m_uiCacheFlags & (~(NCA_COUNTER_BITS))) | + (((m_uiCacheFlags & NCA_COUNTER_BITS) + 1)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void decrRowUseCount( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags = (m_uiCacheFlags & (~(NCA_COUNTER_BITS))) | + (((m_uiCacheFlags & NCA_COUNTER_BITS) - 1)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + void setRowDirty( + F_Db * pDb, + FLMBOOL bNew); + + void unsetRowDirtyAndNew( + F_Db * pDb, + FLMBOOL bMutexAlreadyLocked = FALSE); + + FINLINE FLMBOOL rowIsDirty( void) + { + return( (m_uiFlags & FROW_DIRTY) ? TRUE : FALSE); + } + + FINLINE FLMBOOL rowIsNew( void) + { + return( (m_uiFlags & FROW_NEW) ? TRUE : FALSE); + } + + // Assumes that the row cache mutex has already been locked. + FINLINE FLMBOOL rowUncommitted( void) + { + return( (m_uiCacheFlags & NCA_UNCOMMITTED ) ? TRUE : FALSE); + } + + void freeCache( + FLMBOOL bPutInPurgeList); + + // Column functions + + FINLINE FLMBYTE * getColumnDataPtr( + FLMUINT uiColumnNum) + { + F_COLUMN_ITEM * pColumnItem = getColumn( uiColumnNum); + return( (FLMBYTE *)(pColumnItem->uiDataLen <= sizeof( FLMUINT) + ? (FLMBYTE *)(&pColumnItem->uiDataOffset) + : (FLMBYTE *)(m_pucColumnData + pColumnItem->uiDataOffset))); + } + + RCODE allocColumnDataSpace( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT uiSizeNeeded, + FLMBOOL bMutexAlreadyLocked); + + RCODE setNumber64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 ui64Value, + FLMBOOL bNeg); + + RCODE getNumber64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 * pui64Value, + FLMBOOL * pbNeg, + FLMBOOL * pbIsNull); + + FINLINE RCODE getUINT32( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT32 * pui32Num, + FLMBOOL * pbIsNull) + { + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMBOOL bNeg; + + if( RC_OK( rc = getNumber64( pDb, uiColumnNum, &ui64Num, + &bNeg, pbIsNull))) + { + if (!(*pbIsNull)) + { + rc = convertToUINT32( ui64Num, bNeg, pui32Num); + } + } + return( rc); + } + + FINLINE RCODE getUINT( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT * puiNum, + FLMBOOL * pbIsNull) + { + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMBOOL bNeg; + + if( RC_OK( rc = getNumber64( pDb, uiColumnNum, &ui64Num, + &bNeg, pbIsNull))) + { + if (!(*pbIsNull)) + { + rc = convertToUINT( ui64Num, bNeg, puiNum); + } + } + return( rc); + } + + FINLINE RCODE getUINT64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 * pui64Num, + FLMBOOL * pbIsNull) + { + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMBOOL bNeg; + + if( RC_OK( rc = getNumber64( pDb, uiColumnNum, &ui64Num, + &bNeg, pbIsNull))) + { + if (!(*pbIsNull)) + { + rc = convertToUINT64( ui64Num, bNeg, pui64Num); + } + } + + return( rc); + } + + FINLINE RCODE getINT( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMINT * piNum, + FLMBOOL * pbIsNull) + { + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMBOOL bNeg; + + if (RC_OK( rc = getNumber64( pDb, uiColumnNum, &ui64Num, + &bNeg, pbIsNull))) + { + if (!(*pbIsNull)) + { + rc = convertToINT( ui64Num, bNeg, piNum); + } + } + + return( rc); + } + + FINLINE RCODE getINT64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMINT64 * pi64Num, + FLMBOOL * pbIsNull) + { + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMBOOL bNeg; + + if (RC_OK( rc = getNumber64( pDb, uiColumnNum, &ui64Num, + &bNeg, pbIsNull))) + { + if (!(*pbIsNull)) + { + rc = convertToINT64( ui64Num, bNeg, pi64Num); + } + } + + return( rc); + } + + FINLINE RCODE setUINT( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT uiValue) + { + return( setNumber64( pDb, uiColumnNum, (FLMUINT64)uiValue, FALSE)); + } + + FINLINE RCODE setUINT64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 ui64Value) + { + return( setNumber64( pDb, uiColumnNum, ui64Value, FALSE)); + } + + FINLINE RCODE setINT( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMINT iValue) + { + FLMBOOL bNeg; + FLMUINT ui64Value; + + if (iValue < 0) + { + bNeg = TRUE; + ui64Value = (FLMUINT64)((FLMINT64)(-iValue)); + } + else + { + bNeg = FALSE; + ui64Value = (FLMUINT64)iValue; + } + return( setNumber64( pDb, uiColumnNum, ui64Value, bNeg)); + } + + FINLINE RCODE setINT64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMINT64 i64Value) + { + FLMBOOL bNeg; + FLMUINT ui64Value; + + if (i64Value < 0) + { + bNeg = TRUE; + ui64Value = (FLMUINT64)(-i64Value); + } + else + { + bNeg = FALSE; + ui64Value = (FLMUINT64)i64Value; + } + return( setNumber64( pDb, uiColumnNum, ui64Value, bNeg)); + } + + RCODE setUTF8( + F_Db * pDb, + FLMUINT uiColumnNum, + const char * pszValue, + FLMUINT uiNumBytesInValue, + FLMUINT uiNumCharsInValue); + + RCODE getUTF8( + F_Db * pDb, + FLMUINT uiColumnNum, + char * pszValueBuffer, + FLMUINT uiBufferSize, + FLMBOOL * pbIsNull, + FLMUINT * puiCharsReturned, + FLMUINT * puiBufferBytesUsed); + + RCODE setBinary( + F_Db * pDb, + FLMUINT uiColumnNum, + const void * pvValue, + FLMUINT uiNumBytesInBuffer); + + RCODE getBinary( + F_Db * pDb, + FLMUINT uiColumnNum, + void * pvBuffer, + FLMUINT uiBufferLen, + FLMUINT * puiDataLen, + FLMBOOL * pbIsNull); + + void setToNull( + F_Db * pDb, + FLMUINT uiColumnNum); + + void getDataLen( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT * puiDataLen, + FLMBOOL * pbIsNull); + + FINLINE F_COLUMN_ITEM * getColumn( + FLMUINT uiColumnNum) + { + return( (F_COLUMN_ITEM *)((uiColumnNum && uiColumnNum <= m_uiNumColumns && + m_pColumns [uiColumnNum - 1].uiDataLen) + ? &m_pColumns [uiColumnNum - 1] + : (F_COLUMN_ITEM *)NULL)); + } + + RCODE getIStream( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMBOOL * pbIsNull, + F_BufferIStream * pBufferIStream, + eDataType * peDataType, + FLMUINT * puiDataLength); + + RCODE getTextIStream( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMBOOL * pbIsNull, + F_BufferIStream * pBufferIStream, + FLMUINT * puiNumChars); + + FINLINE void ReleaseRow( void) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + decrRowUseCount(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + +private: + + // Assumes that the row cache mutex has already been locked. + FINLINE FLMBOOL readingInRow( void) + { + return( (m_uiCacheFlags & NCA_READING_IN ) ? TRUE : FALSE); + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void setReadingIn( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags |= NCA_READING_IN; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unsetReadingIn( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags &= (~(NCA_READING_IN)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void setUncommitted( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags |= NCA_UNCOMMITTED; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unsetUncommitted( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags &= (~(NCA_UNCOMMITTED)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE FLMBOOL rowIsLatestVer( void) + { + return( (m_uiCacheFlags & NCA_LATEST_VER ) ? TRUE : FALSE); + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void setLatestVer( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags |= NCA_LATEST_VER; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unsetLatestVer( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags &= (~(NCA_LATEST_VER)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void setPurged( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags |= NCA_PURGED; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unsetPurged( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags &= (~(NCA_PURGED)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE FLMBOOL rowLinkedToDatabase( void) + { + return( (m_uiCacheFlags & NCA_LINKED_TO_DATABASE ) ? TRUE : FALSE); + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void setLinkedToDatabase( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags |= NCA_LINKED_TO_DATABASE; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unsetLinkedToDatabase( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiCacheFlags &= (~(NCA_LINKED_TO_DATABASE)); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE FLMBOOL rowInUse( void) + { + return( (m_uiCacheFlags & NCA_COUNTER_BITS ) ? TRUE : FALSE); + } + + // Unlink a row from the global purged list. + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromPurged( void) + { + if (m_pNextInGlobal) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInGlobal->unprotectCachedItem(); +#endif + m_pNextInGlobal->m_pPrevInGlobal = m_pPrevInGlobal; +#ifdef FLM_CACHE_PROTECT + m_pNextInGlobal->protectCachedItem(); +#endif + } + + if (m_pPrevInGlobal) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInGlobal->unprotectCachedItem(); +#endif + m_pPrevInGlobal->m_pNextInGlobal = m_pNextInGlobal; +#ifdef FLM_CACHE_PROTECT + m_pPrevInGlobal->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pRowCacheMgr->m_pPurgeList = + (F_Row *)m_pNextInGlobal; + } + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_pPrevInGlobal = NULL; + m_pNextInGlobal = NULL; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Link a row to an F_Database list at the head of the list. + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToDatabaseAtHead( + F_Database * pDatabase) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (!pDatabase->m_pLastDirtyRow || rowIsDirty()) + { + m_pPrevInDatabase = NULL; + if ((m_pNextInDatabase = pDatabase->m_pFirstRow) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pDatabase->m_pFirstRow->unprotectCachedItem(); +#endif + pDatabase->m_pFirstRow->m_pPrevInDatabase = this; +#ifdef FLM_CACHE_PROTECT + pDatabase->m_pFirstRow->protectCachedItem(); +#endif + } + else + { + pDatabase->m_pLastRow = this; + } + + pDatabase->m_pFirstRow = this; + if (rowIsDirty() && !pDatabase->m_pLastDirtyRow) + { + pDatabase->m_pLastDirtyRow = this; + } + } + else + { + // pDatabase->m_pLastDirtyRow is guaranteed to be non-NULL, + // Hence, m_pPrevInDatabase will be non-NULL. + // We are also guaranteed that the row is not dirty. + + m_pPrevInDatabase = pDatabase->m_pLastDirtyRow; + m_pNextInDatabase = m_pPrevInDatabase->m_pNextInDatabase; + +#ifdef FLM_CACHE_PROTECT + m_pPrevInDatabase->unprotectCachedItem(); +#endif + m_pPrevInDatabase->m_pNextInDatabase = this; +#ifdef FLM_CACHE_PROTECT + m_pPrevInDatabase->protectCachedItem(); +#endif + if (m_pNextInDatabase) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInDatabase->unprotectCachedItem(); +#endif + m_pNextInDatabase->m_pPrevInDatabase = this; +#ifdef FLM_CACHE_PROTECT + m_pNextInDatabase->protectCachedItem(); +#endif + } + else + { + pDatabase->m_pLastRow = this; + } + } + + m_pDatabase = pDatabase; + setLinkedToDatabase(); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Link a row to an F_Database list at the end of the list. + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToDatabaseAtEnd( + F_Database * pDatabase) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + // Row cannot be a dirty row. + + flmAssert( !rowIsDirty()); + m_pNextInDatabase = NULL; + if( (m_pPrevInDatabase = pDatabase->m_pLastRow) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pDatabase->m_pLastRow->unprotectCachedItem(); +#endif + pDatabase->m_pLastRow->m_pNextInDatabase = this; +#ifdef FLM_CACHE_PROTECT + pDatabase->m_pLastRow->protectCachedItem(); +#endif + } + else + { + pDatabase->m_pFirstRow = this; + } + + pDatabase->m_pLastRow = this; + m_pDatabase = pDatabase; + setLinkedToDatabase(); + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Unlink a row from its F_Database list. + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromDatabase( void) + { + if( rowLinkedToDatabase()) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + // If this is the last dirty row, change the database's + // last dirty pointer to point to the previous row, if any. + + if (m_pDatabase->m_pLastDirtyRow == this) + { + flmAssert( rowIsDirty()); + m_pDatabase->m_pLastDirtyRow = m_pPrevInDatabase; + } + + // Remove the row from the database's list. + + if( m_pNextInDatabase) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInDatabase->unprotectCachedItem(); +#endif + m_pNextInDatabase->m_pPrevInDatabase = m_pPrevInDatabase; +#ifdef FLM_CACHE_PROTECT + m_pNextInDatabase->protectCachedItem(); +#endif + } + else + { + m_pDatabase->m_pLastRow = m_pPrevInDatabase; + } + + if( m_pPrevInDatabase) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInDatabase->unprotectCachedItem(); +#endif + m_pPrevInDatabase->m_pNextInDatabase = m_pNextInDatabase; +#ifdef FLM_CACHE_PROTECT + m_pPrevInDatabase->protectCachedItem(); +#endif + } + else + { + m_pDatabase->m_pFirstRow = m_pNextInDatabase; + } + + m_pPrevInDatabase = NULL; + m_pNextInDatabase = NULL; + m_pDatabase = NULL; + unsetLinkedToDatabase(); + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + } + + // Link a row into its hash bucket. + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToHashBucket( void) + { + F_Row ** ppHashBucket = gv_SFlmSysData.pRowCacheMgr->rowHash( + m_ui64RowId); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + flmAssert( m_pNewerVersion == NULL); + + m_pPrevInBucket = NULL; + + if ((m_pNextInBucket = *ppHashBucket) != NULL) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInBucket->unprotectCachedItem(); +#endif + m_pNextInBucket->m_pPrevInBucket = this; +#ifdef FLM_CACHE_PROTECT + m_pNextInBucket->protectCachedItem(); +#endif + } + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + *ppHashBucket = this; + } + + // Unlink a row from its hash bucket. + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromHashBucket( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + flmAssert( m_pNewerVersion == NULL); + + if (m_pNextInBucket) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInBucket->unprotectCachedItem(); +#endif + m_pNextInBucket->m_pPrevInBucket = m_pPrevInBucket; +#ifdef FLM_CACHE_PROTECT + m_pNextInBucket->protectCachedItem(); +#endif + } + + if (m_pPrevInBucket) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInBucket->unprotectCachedItem(); +#endif + m_pPrevInBucket->m_pNextInBucket = m_pNextInBucket; +#ifdef FLM_CACHE_PROTECT + m_pPrevInBucket->protectCachedItem(); +#endif + } + else + { + F_Row ** ppHashBucket = + gv_SFlmSysData.pRowCacheMgr->rowHash( + m_ui64RowId); + + *ppHashBucket = m_pNextInBucket; + } + + m_pPrevInBucket = NULL; + m_pNextInBucket = NULL; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Unlink a row from its version list. + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToVerList( + F_Row * pNewerVer, + F_Row * pOlderVer) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if( (m_pNewerVersion = pNewerVer) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pNewerVer->unprotectCachedItem(); +#endif + pNewerVer->m_pOlderVersion = this; +#ifdef FLM_CACHE_PROTECT + pNewerVer->protectCachedItem(); +#endif + } + + if ((m_pOlderVersion = pOlderVer) != NULL) + { +#ifdef FLM_CACHE_PROTECT + pOlderVer->unprotectCachedItem(); +#endif + pOlderVer->m_pNewerVersion = this; +#ifdef FLM_CACHE_PROTECT + pOlderVer->protectCachedItem(); +#endif + } + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Unlink a row from its version list. This routine + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromVerList( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (m_pNewerVersion) + { +#ifdef FLM_CACHE_PROTECT + m_pNewerVersion->unprotectCachedItem(); +#endif + m_pNewerVersion->m_pOlderVersion = m_pOlderVersion; +#ifdef FLM_CACHE_PROTECT + m_pNewerVersion->protectCachedItem(); +#endif + } + + if (m_pOlderVersion) + { +#ifdef FLM_CACHE_PROTECT + m_pOlderVersion->unprotectCachedItem(); +#endif + m_pOlderVersion->m_pNewerVersion = m_pNewerVersion; +#ifdef FLM_CACHE_PROTECT + m_pOlderVersion->protectCachedItem(); +#endif + } + + m_pNewerVersion = NULL; + m_pOlderVersion = NULL; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Link a row into the heap list + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToHeapList( void) + { + flmAssert( !m_pPrevInHeapList); + flmAssert( (m_uiFlags & FROW_HEAP_ALLOC) == 0); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if( (m_pNextInHeapList = + gv_SFlmSysData.pRowCacheMgr->m_pHeapList) != NULL) + { +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pHeapList->unprotectCachedItem(); +#endif + gv_SFlmSysData.pRowCacheMgr->m_pHeapList->m_pPrevInHeapList = this; +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pHeapList->protectCachedItem(); +#endif + } + + gv_SFlmSysData.pRowCacheMgr->m_pHeapList = this; + m_uiFlags |= FROW_HEAP_ALLOC; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Unlink a row from the heap list + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromHeapList( void) + { + flmAssert( m_uiFlags & FROW_HEAP_ALLOC); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (m_pNextInHeapList) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInHeapList->unprotectCachedItem(); +#endif + m_pNextInHeapList->m_pPrevInHeapList = m_pPrevInHeapList; +#ifdef FLM_CACHE_PROTECT + m_pNextInHeapList->protectCachedItem(); +#endif + } + + if (m_pPrevInHeapList) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInHeapList->unprotectCachedItem(); +#endif + m_pPrevInHeapList->m_pNextInHeapList = m_pNextInHeapList; +#ifdef FLM_CACHE_PROTECT + m_pPrevInHeapList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pRowCacheMgr->m_pHeapList = m_pNextInHeapList; + } + + m_pPrevInHeapList = NULL; + m_pNextInHeapList = NULL; + m_uiFlags &= ~FROW_HEAP_ALLOC; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void linkToOldList( void) + { + flmAssert( !m_pPrevInOldList); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if( (m_pNextInOldList = + gv_SFlmSysData.pRowCacheMgr->m_pOldList) != NULL) + { +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pOldList->unprotectCachedItem(); +#endif + gv_SFlmSysData.pRowCacheMgr->m_pOldList->m_pPrevInOldList = this; +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pOldList->protectCachedItem(); +#endif + } + + gv_SFlmSysData.pRowCacheMgr->m_pOldList = this; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + // Assumes that the row cache mutex has already been locked. + FINLINE void unlinkFromOldList( void) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (m_pNextInOldList) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInOldList->unprotectCachedItem(); +#endif + m_pNextInOldList->m_pPrevInOldList = m_pPrevInOldList; +#ifdef FLM_CACHE_PROTECT + m_pNextInOldList->protectCachedItem(); +#endif + } + + if (m_pPrevInOldList) + { +#ifdef FLM_CACHE_PROTECT + m_pPrevInOldList->unprotectCachedItem(); +#endif + m_pPrevInOldList->m_pNextInOldList = m_pNextInOldList; +#ifdef FLM_CACHE_PROTECT + m_pPrevInOldList->protectCachedItem(); +#endif + } + else + { + gv_SFlmSysData.pRowCacheMgr->m_pOldList = m_pNextInOldList; + } + + m_pPrevInOldList = NULL; + m_pNextInOldList = NULL; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + FINLINE FLMUINT memSize( void) + { + FLMUINT uiSize = gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->getCellSize(); + + if (m_pucColumnData) + { + uiSize += gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->getTrueSize( + calcDataBufSize( m_uiColumnDataBufSize), + getActualPointer( m_pucColumnData)); + } + + if (m_pColumns) + { + uiSize += gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->getTrueSize( + calcColumnListBufSize( m_uiNumColumns), + getActualPointer( m_pColumns)); + } + + return( uiSize); + } + + // Assumes that the row cache mutex is locked, because + // it potentially updates the cache usage statistics. + + FINLINE void setTransID( + FLMUINT64 ui64NewTransID) + { + FLMUINT uiSize; + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (m_ui64HighTransId == FLM_MAX_UINT64 && + ui64NewTransID != FLM_MAX_UINT64) + { + uiSize = memSize(); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes += uiSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount++; + linkToOldList(); + } + else if (m_ui64HighTransId != FLM_MAX_UINT64 && + ui64NewTransID == FLM_MAX_UINT64) + { + uiSize = memSize(); + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes >= uiSize); + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes -= uiSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount--; + unlinkFromOldList(); + } + + m_ui64HighTransId = ui64NewTransID; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + } + + void freePurged( void); + + void linkToDatabase( + F_Database * pDatabase, + F_Db * pDb, + FLMUINT64 ui64LowTransId, + FLMBOOL bMostCurrent); + + void resetRow( void); + + // Things that manage the row's place in row cache. These are + // always initialized in the constructor + + F_Row * m_pPrevInBucket; + F_Row * m_pNextInBucket; + F_Row * m_pPrevInDatabase; + F_Row * m_pNextInDatabase; + F_Row * m_pOlderVersion; + F_Row * m_pNewerVersion; + F_Row * m_pPrevInHeapList; + F_Row * m_pNextInHeapList; + F_Row * m_pPrevInOldList; + F_Row * m_pNextInOldList; + + FLMUINT64 m_ui64LowTransId; + FLMUINT64 m_ui64HighTransId; + FNOTIFY * m_pNotifyList; + FLMUINT m_uiCacheFlags; + FLMUINT m_uiStreamUseCount; + + // Things we hash on - initialized by caller of constructor + + F_Database * m_pDatabase; + + // Items initialized in constructor + + FLMUINT m_uiTableNum; + FLMUINT64 m_ui64RowId; + FLMUINT m_uiFlags; + F_COLUMN_ITEM * m_pColumns; + FLMUINT m_uiNumColumns; + FLMBYTE * m_pucColumnData; + FLMUINT m_uiColumnDataBufSize; + + // Items initialized by caller of constructor, but not + // in constructor - for performance reasons - so we don't + // end up setting them twice. + + FLMUINT m_uiOffsetIndex; + FLMUINT32 m_ui32BlkAddr; + + // Items that m_uiFlags indicates whether they are present + +friend class F_RowCacheMgr; +friend class F_GlobalCacheMgr; +friend class F_Database; +friend class F_Db; +friend class F_DbSystem; +friend class F_BTreeIStream; +friend class F_Rfl; +friend class F_RebuildRowIStream; +friend class F_DbRebuild; +friend class F_RowRelocator; +friend class F_ColumnDataRelocator; +friend class F_ColumnListRelocator; +}; + +#endif // FCACHE_H diff --git a/sql/src/fcollate.cpp b/sql/src/fcollate.cpp new file mode 100644 index 0000000..10f63d5 --- /dev/null +++ b/sql/src/fcollate.cpp @@ -0,0 +1,174 @@ +//------------------------------------------------------------------------------ +// Desc: Routines for building collation keys +// +// Tabs: 3 +// +// Copyright (c) 1993-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fcollate.cpp 3111 2006-01-19 13:10:50 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/************************************************************************** +Desc: Get the Flaim collating string and convert back to a text string +Ret: Length of new wpStr +Notes: Allocates the area for the word string buffer if will be over 256. +***************************************************************************/ +RCODE flmColText2StorageText( + const FLMBYTE * pucColStr, // Points to the collated string + FLMUINT uiColStrLen, // Length of the collated string + FLMBYTE * pucStorageBuf, // Output string to build - TEXT string + FLMUINT * puiStorageLen, // In: Size of buffer, Out: Bytes used + FLMUINT uiLang, + FLMBOOL * pbDataTruncated, // Sets to TRUE if data had been truncated + FLMBOOL * pbFirstSubstring) // Sets to TRUE if first substring +{ +#define LOCAL_CHARS 150 + FLMBYTE ucWPStr[ LOCAL_CHARS * 2 + LOCAL_CHARS / 5 ]; // Sample + 20% + FLMBYTE * pucWPPtr = NULL; + FLMBYTE * pucAllocatedWSPtr = NULL; + FLMUINT uiWPStrLen; + FLMBYTE * pucStoragePtr; + FLMUINT uiUnconvChars; + FLMUINT uiTmp; + FLMUINT uiMaxStorageBytes = *puiStorageLen; + FLMUINT uiMaxWPBytes; + FLMUINT uiStorageOffset; + FLMBYTE ucTmpSen[ 5]; + FLMBYTE * pucTmpSen = &ucTmpSen[ 0]; + RCODE rc = NE_FLM_OK; + + if( uiColStrLen > LOCAL_CHARS) + { + // If it won't fit, allocate a new buffer + + if( RC_BAD( rc = f_alloc( SFLM_MAX_KEY_SIZE * 2, &pucWPPtr))) + { + goto Exit; + } + + pucAllocatedWSPtr = pucWPPtr; + uiMaxWPBytes = uiWPStrLen = SFLM_MAX_KEY_SIZE * 2; + } + else + { + pucWPPtr = &ucWPStr[ 0]; + uiMaxWPBytes = uiWPStrLen = sizeof( ucWPStr); + } + + if( (uiLang >= FLM_FIRST_DBCS_LANG) && + (uiLang <= FLM_LAST_DBCS_LANG)) + { + if( RC_BAD( rc = f_asiaColStr2WPStr( pucColStr, uiColStrLen, + pucWPPtr, &uiWPStrLen, &uiUnconvChars, + pbDataTruncated, pbFirstSubstring))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = f_colStr2WPStr( pucColStr, uiColStrLen, + pucWPPtr, &uiWPStrLen, uiLang, &uiUnconvChars, + pbDataTruncated, pbFirstSubstring))) + { + goto Exit; + } + } + + // Copy word string to the storage string area + + uiWPStrLen >>= 1; // Convert # of bytes to # of words + pucStoragePtr = pucStorageBuf; + uiStorageOffset = 0; + + // Encode the number of characters as a SEN. If pucEncPtr is + // NULL, the caller is only interested in the length of the encoded + // string, so a temporary buffer is used to call f_encodeSEN. + + uiTmp = f_encodeSEN( uiWPStrLen - uiUnconvChars, &pucTmpSen); + if( (uiStorageOffset + uiTmp) >= uiMaxStorageBytes) + { + rc = RC_SET( NE_FLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucStoragePtr, &ucTmpSen[ 0], uiTmp); + uiStorageOffset += uiTmp; + + // Encode each of the WP characters into UTF-8 + + while( uiWPStrLen--) + { + FLMBYTE ucChar; + FLMBYTE ucCharSet; + FLMUNICODE uChar; + + // Put the character in a local variable for speed + + ucChar = *pucWPPtr++; + ucCharSet = *pucWPPtr++; + + if( ucCharSet == 0xFF && ucChar == 0xFF) + { + uChar = (((FLMUNICODE)*(pucWPPtr + 1)) << 8) | *pucWPPtr; + pucWPPtr += 2; + uiWPStrLen--; // Skip past 4 bytes for UNICODE + } + else + { + if( RC_BAD( rc = f_wpToUnicode( + (((FLMUINT16)ucCharSet) << 8) + ucChar, &uChar))) + { + goto Exit; + } + } + + uiTmp = uiMaxStorageBytes - uiStorageOffset; + if( RC_BAD( rc = f_uni2UTF8( uChar, + &pucStorageBuf[ uiStorageOffset], &uiTmp))) + { + goto Exit; + } + uiStorageOffset += uiTmp; + } + + if( uiStorageOffset >= uiMaxStorageBytes) + { + rc = RC_SET( NE_FLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + // Tack on a trailing NULL byte + + pucStorageBuf[ uiStorageOffset++] = 0; + + // Return the length of the storage buffer + + *puiStorageLen = uiStorageOffset; + +Exit: + + if( pucAllocatedWSPtr) + { + f_free( &pucAllocatedWSPtr); + } + + return( rc); +} + diff --git a/sql/src/fcollate.h b/sql/src/fcollate.h new file mode 100644 index 0000000..8df0750 --- /dev/null +++ b/sql/src/fcollate.h @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +// Desc: Header for collation routines +// +// Tabs: 3 +// +// Copyright (c) 1991-1992, 1994-2000, 2002-2006 Novell, Inc. +// All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fcollate.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FCOLLATE_H +#define FCOLLATE_H + +#define TRUNCATED_FLAG 0x8000 +#define EXCLUSIVE_LT_FLAG 0x4000 +#define EXCLUSIVE_GT_FLAG 0x2000 +#define SEARCH_KEY_FLAG 0x1000 + +#define KEY_COMPONENT_LENGTH_MASK 0x0FFF +#define KEY_LOW_VALUE 0x0FFE +#define KEY_HIGH_VALUE 0x0FFF + +FINLINE FLMBOOL isKeyComponentLTExclusive( + const FLMBYTE * pucKeyComponent) +{ + return( (FB2UW( pucKeyComponent) & EXCLUSIVE_LT_FLAG) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isKeyComponentGTExclusive( + const FLMBYTE * pucKeyComponent) +{ + return( (FB2UW( pucKeyComponent) & EXCLUSIVE_GT_FLAG) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isKeyComponentTruncated( + const FLMBYTE * pucKeyComponent) +{ + return( (FB2UW( pucKeyComponent) & TRUNCATED_FLAG) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isSearchKeyComponent( + const FLMBYTE * pucKeyComponent) +{ + return( (FB2UW( pucKeyComponent) & SEARCH_KEY_FLAG) ? TRUE : FALSE); +} + +FINLINE FLMUINT getKeyComponentLength( + const FLMBYTE * pucKeyComponent) +{ + return( (FLMUINT)(FB2UW( pucKeyComponent)) & KEY_COMPONENT_LENGTH_MASK); +} + +RCODE flmColText2StorageText( + const FLMBYTE * pucColStr, + FLMUINT uiColStrLen, + FLMBYTE * pucStorageBuf, + FLMUINT * puiStorageLen, + FLMUINT uiLang, + FLMBOOL * pbDataTruncated, + FLMBOOL * pbFirstSubstring); + +RCODE flmUTF8ToColText( + IF_PosIStream * pIStream, + FLMBYTE * pucCollatedStr, + FLMUINT * puiCollatedStrLen, + FLMBOOL bCaseInsensitive, + FLMUINT * puiCollationLen, + FLMUINT * puiCaseLen, + FLMUINT uiLanguage, + FLMUINT uiCharLimit, + FLMBOOL bFirstSubstring, + FLMBOOL bDataTruncated, + FLMBOOL * pbOriginalCharsLost, + FLMBOOL * pbDataTruncated); + +#endif diff --git a/sql/src/fdbcnfig.cpp b/sql/src/fdbcnfig.cpp new file mode 100644 index 0000000..14cfd49 --- /dev/null +++ b/sql/src/fdbcnfig.cpp @@ -0,0 +1,1307 @@ +//------------------------------------------------------------------------------ +// Desc: Database config get/set functions +// +// Tabs: 3 +// +// Copyright (c) 1996-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdbcnfig.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Set the RFL keep files flag. +****************************************************************************/ +RCODE F_Db::setRflKeepFilesFlag( + FLMBOOL bKeepFiles) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bDbLocked = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Make sure we don't have a transaction going + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // Make sure there is no active backup running + + m_pDatabase->lockMutex(); + if (m_pDatabase->m_bBackupActive) + { + m_pDatabase->unlockMutex(); + rc = RC_SET( NE_SFLM_BACKUP_ACTIVE); + goto Exit; + } + m_pDatabase->unlockMutex(); + + // Need to lock the database but not start a transaction yet. + + if (!(m_uiFlags & (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_SHARED))) + { + if (RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, + SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bDbLocked = TRUE; + } + + // If we aren't changing the keep flag, jump to exit without doing + // anything. + + if ((bKeepFiles && + m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles) || + (!bKeepFiles && + !m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles)) + { + goto Exit; // Will return NE_SFLM_OK; + } + + // Force a checkpoint and roll to the next RFL file numbers. + // When changing from keep to no-keep or vice versa, we need to + // go to a new RFL file so that the new RFL file gets new + // serial numbers and a new keep or no-keep flag. + + if (RC_BAD( rc = doCheckpoint( SFLM_NO_TIMEOUT))) + { + goto Exit; + } + + f_memcpy( &m_pDatabase->m_uncommittedDbHdr, + &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + m_pDatabase->m_uncommittedDbHdr.ui8RflKeepFiles = + (FLMUINT8)(bKeepFiles + ? (FLMUINT8)1 + : (FLMUINT8)0); + + // Force a new RFL file - this will also write out the entire + // log header - including the changes we made above. + + if (RC_BAD( rc = m_pDatabase->m_pRfl->finishCurrFile( this, TRUE))) + { + goto Exit; + } + +Exit: + + if (bDbLocked) + { + dbUnlock(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Set the RFL directory for a database. +****************************************************************************/ +RCODE F_Db::setRflDir( + const char * pszNewRflDir) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bDbLocked = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Make sure we don't have a transaction going + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // Make sure there is no active backup running + + m_pDatabase->lockMutex(); + if (m_pDatabase->m_bBackupActive) + { + m_pDatabase->unlockMutex(); + rc = RC_SET( NE_SFLM_BACKUP_ACTIVE); + goto Exit; + } + m_pDatabase->unlockMutex(); + + // Make sure the path exists and that it is a directory + // rather than a file. + + if (pszNewRflDir && *pszNewRflDir) + { + if (!gv_SFlmSysData.pFileSystem->isDir( pszNewRflDir)) + { + rc = RC_SET( NE_FLM_IO_INVALID_FILENAME); + goto Exit; + } + } + + // Need to lock the database because we can't change the RFL + // directory until after the checkpoint has completed. The + // checkpoint code will unlock the transaction, but not the + // file if we have an explicit lock. We need to do this to + // prevent another transaction from beginning before we have + // changed the RFL directory. + + if (!(m_uiFlags & (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_SHARED))) + { + if( RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, + SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bDbLocked = TRUE; + } + + // Force a checkpoint and roll to the next RFL file numbers. Both + // of these steps are necessary to ensure that we won't have to do + // any recovery using the current RFL file - because we do not + // move the current RFL file to the new directory. Forcing the + // checkpoint ensures that we have no transactions that will need + // to be recovered if we were to crash. Rolling the RFL file number + // ensures that no more transactions will be logged to the current + // RFL file. + + if (RC_BAD( rc = doCheckpoint( SFLM_NO_TIMEOUT))) + { + goto Exit; + } + + // Force a new RFL file. + + if (RC_BAD( rc = m_pDatabase->m_pRfl->finishCurrFile( this, FALSE))) + { + goto Exit; + } + + // Set the RFL directory to the new value now that we have + // finished the checkpoint and rolled to the next RFL file. + + m_pDatabase->lockMutex(); + rc = m_pDatabase->m_pRfl->setRflDir( pszNewRflDir); + m_pDatabase->unlockMutex(); + +Exit: + + if (bDbLocked) + { + dbUnlock(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Set the RFL file size limits for a database. +****************************************************************************/ +RCODE F_Db::setRflFileSizeLimits( + FLMUINT uiMinRflSize, + FLMUINT uiMaxRflSize) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Make sure the limits are valid. + + // Maximum must be enough to hold at least one packet plus + // the RFL header. Minimum must not be greater than the + // maximum. NOTE: Minimum and maximum are allowed to be + // equal, but in all cases, maximum takes precedence over + // minimum. We will first NOT exceed the maximum. Then, + // if possible, we will go above the minimum. + + if (uiMaxRflSize < RFL_MAX_PACKET_SIZE + 512) + { + uiMaxRflSize = RFL_MAX_PACKET_SIZE + 512; + } + if (uiMaxRflSize > gv_SFlmSysData.uiMaxFileSize) + { + uiMaxRflSize = gv_SFlmSysData.uiMaxFileSize; + } + if (uiMinRflSize > uiMaxRflSize) + { + uiMinRflSize = uiMaxRflSize; + } + + // Start an update transaction. Must not already be one going. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Commit the transaction. + + m_pDatabase->m_uncommittedDbHdr.ui32RflMinFileSize = + (FLMUINT32)uiMinRflSize; + m_pDatabase->m_uncommittedDbHdr.ui32RflMaxFileSize = + (FLMUINT32)uiMaxRflSize; + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, FALSE))) + { + goto Exit; + } + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Roll to the next RFL file for this database +****************************************************************************/ +RCODE F_Db::rflRollToNextFile( void) +{ + RCODE rc = NE_SFLM_OK; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // NOTE: finishCurrFile will not roll to the next file if the current + // file has not been created. + + if (RC_BAD( rc = m_pDatabase->m_pRfl->finishCurrFile( this, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set keep aborted transactions in RFL flag. +****************************************************************************/ +RCODE F_Db::setKeepAbortedTransInRflFlag( + FLMBOOL bKeep + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Start an update transaction. Must not already be one going. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Change the uncommitted log header + + m_pDatabase->m_uncommittedDbHdr.ui8RflKeepAbortedTrans = + (FLMUINT8)(bKeep + ? (FLMUINT8)1 + : (FLMUINT8)0); + + // Commit the transaction. + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, FALSE))) + { + goto Exit; + } + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Set auto turn off keep RFL flag. +****************************************************************************/ +RCODE F_Db::setAutoTurnOffKeepRflFlag( + FLMBOOL bAutoTurnOff + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Start an update transaction. Must not already be one going. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Change the uncommitted log header + + m_pDatabase->m_uncommittedDbHdr.ui8RflAutoTurnOffKeep = + (FLMUINT8)(bAutoTurnOff + ? (FLMUINT8)1 + : (FLMUINT8)0); + + // Commit the transaction. + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, FALSE))) + { + goto Exit; + } + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Retrieves the Checkpoint info for the database passed in. This assumes + global mutex has already been locked. +*****************************************************************************/ +void F_Database::getCPInfo( + SFLM_CHECKPOINT_INFO * pCheckpointInfo) +{ + FLMUINT uiElapTime; + FLMUINT uiCurrTime; + + flmAssert( pCheckpointInfo); + + f_memset( pCheckpointInfo, 0, sizeof( SFLM_CHECKPOINT_INFO)); + if (m_pCPInfo) + { + pCheckpointInfo->bRunning = m_pCPInfo->bDoingCheckpoint; + if (pCheckpointInfo->bRunning) + { + if (m_pCPInfo->uiStartTime) + { + uiCurrTime = FLM_GET_TIMER(); + + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, + m_pCPInfo->uiStartTime); + pCheckpointInfo->uiRunningTime = FLM_TIMER_UNITS_TO_MILLI( uiElapTime); + } + else + { + pCheckpointInfo->uiRunningTime = 0; + } + pCheckpointInfo->bForcingCheckpoint = + m_pCPInfo->bForcingCheckpoint; + if (m_pCPInfo->uiForceCheckpointStartTime) + { + uiCurrTime = FLM_GET_TIMER(); + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, + m_pCPInfo->uiForceCheckpointStartTime); + pCheckpointInfo->uiForceCheckpointRunningTime = + FLM_TIMER_UNITS_TO_MILLI( uiElapTime); + } + else + { + pCheckpointInfo->uiForceCheckpointRunningTime = 0; + } + pCheckpointInfo->eForceCheckpointReason = + m_pCPInfo->eForceCheckpointReason; + pCheckpointInfo->bWritingDataBlocks = + m_pCPInfo->bWritingDataBlocks; + pCheckpointInfo->uiLogBlocksWritten = + m_pCPInfo->uiLogBlocksWritten; + pCheckpointInfo->uiDataBlocksWritten = + m_pCPInfo->uiDataBlocksWritten; + } + pCheckpointInfo->uiBlockSize = m_uiBlockSize; + pCheckpointInfo->uiDirtyCacheBytes = + m_uiDirtyCacheCount * m_uiBlockSize; + if (m_pCPInfo->uiStartWaitTruncateTime) + { + uiCurrTime = FLM_GET_TIMER(); + + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, + m_pCPInfo->uiStartWaitTruncateTime); + pCheckpointInfo->uiWaitTruncateTime = + FLM_TIMER_UNITS_TO_MILLI( uiElapTime); + } + else + { + pCheckpointInfo->uiWaitTruncateTime = 0; + } + } +} + +/**************************************************************************** +Desc: Retrieves the Checkpoint info for the database. +*****************************************************************************/ +void F_Db::getCheckpointInfo( + SFLM_CHECKPOINT_INFO * pCheckpointInfo) +{ + m_pDatabase->lockMutex(); + m_pDatabase->getCPInfo( pCheckpointInfo); + m_pDatabase->unlockMutex(); +} + +/**************************************************************************** +Desc: Returns current RFL file number +****************************************************************************/ +RCODE F_Db::getRflFileNum( + FLMUINT * puiRflFileNum + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + FLMUINT uiLastCPFile; + FLMUINT uiLastTransFile; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + // Get the CP and last trans RFL file numbers. Need to + // return the higher of the two. No need to lock the + // mutex because we are in an update transaction. + + uiLastCPFile = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflLastCPFileNum; + + uiLastTransFile = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflCurrFileNum; + + *puiRflFileNum = uiLastCPFile > uiLastTransFile + ? uiLastCPFile + : uiLastTransFile; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns highest not used RFL file number +****************************************************************************/ +RCODE F_Db::getHighestNotUsedRflFileNum( + FLMUINT * puiHighestNotUsedRflFileNum + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + FLMUINT uiLastCPFile; + FLMUINT uiLastTransFile; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + // Get the CP and last trans RFL file numbers. Need to + // return the lower of the two minus 1. + + uiLastCPFile = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflLastCPFileNum; + + uiLastTransFile = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflCurrFileNum; + + *puiHighestNotUsedRflFileNum = + (FLMUINT)((uiLastCPFile < uiLastTransFile + ? uiLastCPFile + : uiLastTransFile) - 1); +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns RFL file size limits for the database +****************************************************************************/ +RCODE F_Db::getRflFileSizeLimits( + FLMUINT * puiRflMinFileSize, + FLMUINT * puiRflMaxFileSize + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + if (puiRflMinFileSize) + { + *puiRflMinFileSize = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflMinFileSize; + } + if (puiRflMaxFileSize) + { + *puiRflMaxFileSize = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflMaxFileSize; + } + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns RFL keep flag for the database +****************************************************************************/ +RCODE F_Db::getRflKeepFlag( + FLMBOOL * pbKeep + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *pbKeep = m_pDatabase->m_uncommittedDbHdr.ui8RflKeepFiles + ? TRUE + : FALSE; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns last backup transaction ID for the database +****************************************************************************/ +RCODE F_Db::getLastBackupTransID( + FLMUINT64 * pui64LastBackupTransID + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *pui64LastBackupTransID = + m_pDatabase->m_uncommittedDbHdr.ui64LastBackupTransID; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns blocks changed since the last backup for the database +****************************************************************************/ +RCODE F_Db::getBlocksChangedSinceBackup( + FLMUINT * puiBlocksChangedSinceBackup + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *puiBlocksChangedSinceBackup = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32BlksChangedSinceBackup; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns the auto-turn-off-keep-RFL flag for the database +****************************************************************************/ +RCODE F_Db::getAutoTurnOffKeepRflFlag( + FLMBOOL * pbAutoTurnOff + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *pbAutoTurnOff = m_pDatabase->m_uncommittedDbHdr.ui8RflAutoTurnOffKeep + ? TRUE + : FALSE; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns the keep aborted transactions in RFL flag for the database +****************************************************************************/ +RCODE F_Db::getKeepAbortedTransInRflFlag( + FLMBOOL * pbKeep + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *pbKeep = m_pDatabase->m_uncommittedDbHdr.ui8RflKeepAbortedTrans + ? TRUE + : FALSE; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns disk space usage for the database +****************************************************************************/ +RCODE F_Db::getDiskSpaceUsage( + FLMUINT64 * pui64DataSize, + FLMUINT64 * pui64RollbackSize, + FLMUINT64 * pui64RflSize) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + FLMUINT uiEndAddress; + FLMUINT uiLastFileNumber; + FLMUINT64 ui64LastFileSize; + char szTmpName [F_PATH_MAX_SIZE]; + char szRflDir [F_PATH_MAX_SIZE]; + IF_FileHdl * pFileHdl = NULL; + IF_DirHdl * pDirHdl = NULL; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + // See if they want the database files sizes. + + if (pui64DataSize) + { + uiEndAddress = m_uiLogicalEOF; + uiLastFileNumber = FSGetFileNumber( uiEndAddress); + + // Last file number better be in the proper range. + + flmAssert( uiLastFileNumber >= 1 && + uiLastFileNumber <= MAX_DATA_BLOCK_FILE_NUMBER); + + // Get the actual size of the last file. + + if (RC_BAD( rc = m_pSFileHdl->getFileSize( uiLastFileNumber, + &ui64LastFileSize))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME) + { + if (uiLastFileNumber > 1) + { + rc = NE_SFLM_OK; + ui64LastFileSize = 0; + } + else + { + + // Should always be a data file #1 + + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + } + else + { + goto Exit; + } + } + + // One of two situations exists with respect to the last + // file: 1) it has not been fully written out yet (blocks + // are still cached, or 2) it has been written out and + // extended beyond what the logical EOF shows. We want + // the larger of these two possibilities. + + if (FSGetFileOffset( uiEndAddress) > ui64LastFileSize) + { + ui64LastFileSize = FSGetFileOffset( uiEndAddress); + } + + if (uiLastFileNumber == 1) + { + + // Only one file - use last file's size. + + *pui64DataSize = ui64LastFileSize; + } + else + { + + // Size is the sum of full size for all files except the last one, + // plus the calculated (or actual) size of the last one. + + (*pui64DataSize) = (FLMUINT64)(uiLastFileNumber - 1) * + (FLMUINT64)m_pDatabase->m_uiMaxFileSize + + ui64LastFileSize; + } + } + + // See if they want the rollback files sizes. + + if (pui64RollbackSize) + { + uiEndAddress = (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RblEOF; + uiLastFileNumber = FSGetFileNumber( uiEndAddress); + + // Last file number better be in the proper range. + + flmAssert( !uiLastFileNumber || + (uiLastFileNumber >= + FIRST_LOG_BLOCK_FILE_NUMBER && + uiLastFileNumber <= + MAX_LOG_BLOCK_FILE_NUMBER)); + + // Get the size of the last file number. + + if (RC_BAD( rc = m_pSFileHdl->getFileSize( uiLastFileNumber, + &ui64LastFileSize))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME) + { + if (uiLastFileNumber) + { + rc = NE_SFLM_OK; + ui64LastFileSize = 0; + } + else + { + + // Should always have rollback file #0 + + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + } + else + { + goto Exit; + } + } + + // If the EOF offset for the last file is greater than the + // actual file size, use it instead of the actual file size. + + if (FSGetFileOffset( uiEndAddress) > ui64LastFileSize) + { + ui64LastFileSize = FSGetFileOffset( uiEndAddress); + } + + // Special case handling here because rollback file numbers start with + // zero and then skip to a file number that is one beyond the + // highest data file number - so the calculation for file size needs + // to account for this. + + if (!uiLastFileNumber) + { + *pui64RollbackSize = ui64LastFileSize; + } + else + { + FLMUINT uiFirstLogFileNum = FIRST_LOG_BLOCK_FILE_NUMBER; + + // Add full size of file zero plus a full size for every file + // except the last one. + + (*pui64RollbackSize) = (FLMUINT64)(uiLastFileNumber - + uiFirstLogFileNum + 1) * + (FLMUINT64)m_pDatabase->m_uiMaxFileSize + + ui64LastFileSize; + } + } + + // See if they want the roll-forward log file sizes. + + if (pui64RflSize) + { + char * pszDbFileName = m_pDatabase->m_pszDbPath; + + *pui64RflSize = 0; + + // Scan the RFL directory for + // RFL files. The call below to rflGetDirAndPrefix is done + // to get the prefix. It will not return the correct + // RFL directory name, because we are passing in a NULL + // RFL directory path (which may or may not be correct). + // That's OK, because we get the RFL directory directly + // from the F_Rfl object anyway. + + if (RC_BAD( rc = rflGetDirAndPrefix( pszDbFileName, + NULL, szRflDir))) + { + goto Exit; + } + + // We need to get the RFL directory from the F_Rfl object. + + m_pDatabase->lockMutex(); + f_strcpy( szRflDir, m_pDatabase->m_pRfl->getRflDirPtr()); + m_pDatabase->unlockMutex(); + + // See if the directory exists. If not, we are done. + + if (gv_SFlmSysData.pFileSystem->isDir( szRflDir)) + { + + // Open the directory and scan for RFL files. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openDir( szRflDir, + "*", &pDirHdl))) + { + goto Exit; + } + for (;;) + { + if (RC_BAD( rc = pDirHdl->next())) + { + if (rc == NE_FLM_IO_NO_MORE_FILES) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + pDirHdl->currentItemPath( szTmpName); + + // If the item looks like an RFL file name, get + // its size. + + if (!pDirHdl->currentItemIsDir() && + rflGetFileNum( szTmpName, &uiLastFileNumber)) + { + + // Open the file and get its size. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openBlockFile( + szTmpName, FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + 512, &pFileHdl))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + ui64LastFileSize = 0; + } + else + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = pFileHdl->size( &ui64LastFileSize))) + { + goto Exit; + } + } + if (pFileHdl) + { + pFileHdl->Release(); + pFileHdl = NULL; + } + (*pui64RflSize) += ui64LastFileSize; + } + } + } + } + +Exit: + + if (pFileHdl) + { + pFileHdl->Release(); + } + + if (pDirHdl) + { + pDirHdl->Release(); + } + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns the next incremental backup sequence number for the database +****************************************************************************/ +RCODE F_Db::getNextIncBackupSequenceNum( + FLMUINT * puiNextIncBackupSequenceNum + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + else if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + *puiNextIncBackupSequenceNum = + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32IncBackupSeqNum; + +Exit: + + if (bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns list of lock waiters in an object that allows caller to + iterate through the list. +****************************************************************************/ +RCODE F_Db::getLockWaiters( + IF_LockInfoClient * pLockInfo + ) +{ + RCODE rc = NE_SFLM_OK; + + if (m_pDatabase->m_pDatabaseLockObj) + { + rc = m_pDatabase->m_pDatabaseLockObj->GetLockInfo( pLockInfo); + } + else + { + pLockInfo->setLockCount( 0); + } + return( rc); +} + +/**************************************************************************** +Desc: Returns RFL directory for the database +****************************************************************************/ +void F_Db::getRflDir( + char * pszRflDir + ) +{ + m_pDatabase->lockMutex(); + f_strcpy( pszRflDir, m_pDatabase->m_pRfl->getRflDirPtr()); + m_pDatabase->unlockMutex(); +} + +/**************************************************************************** +Desc: Returns database serial number +****************************************************************************/ +void F_Db::getSerialNumber( + char * pucSerialNumber) +{ + m_pDatabase->lockMutex(); + f_memcpy( pucSerialNumber, m_pDatabase->m_lastCommittedDbHdr.ucDbSerialNum, + SFLM_SERIAL_NUM_SIZE); + m_pDatabase->unlockMutex(); +} diff --git a/sql/src/fdbcopy.cpp b/sql/src/fdbcopy.cpp new file mode 100644 index 0000000..0735756 --- /dev/null +++ b/sql/src/fdbcopy.cpp @@ -0,0 +1,884 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the F_DbSystem::dbCopy method. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdbcopy.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +// Local prototypes + +typedef struct Copied_Name * COPIED_NAME_p; + +typedef struct Copied_Name +{ + char szPath[ F_PATH_MAX_SIZE]; + COPIED_NAME_p pNext; +} COPIED_NAME; + +typedef struct +{ + FLMUINT64 ui64BytesToCopy; + FLMUINT64 ui64BytesCopied; + FLMBOOL bNewSrcFile; + char szSrcFileName[ F_PATH_MAX_SIZE]; + char szDestFileName[ F_PATH_MAX_SIZE]; +} DB_COPY_INFO, * DB_COPY_INFO_p; + + +FSTATIC RCODE flmCopyFile( + DB_COPY_INFO * pDbCopyInfo, + COPIED_NAME ** ppCopiedListRV, + FLMBOOL bOkToTruncate, + IF_DbCopyStatus * ifpStatus); + +/**************************************************************************** +Desc: Copies a database, including roll-forward log files. +****************************************************************************/ +RCODE F_DbSystem::dbCopy( + const char * pszSrcDbName, + // [IN] Name of source database to be copied. + const char * pszSrcDataDir, + // [IN] Name of source data directory. + const char * pszSrcRflDir, + // [IN] RFL directory of source database. NULL can be + // passed to indicate that the log files are located + // in the same directory as the other database files. + const char * pszDestDbName, + // [IN] Destination name of database - will be overwritten if it + // already exists. + const char * pszDestDataDir, + // [IN] Name of destination data directory. + const char * pszDestRflDir, + // [IN] RFL directory of destination database. NULL can be + // passed to indicate that the log files are to be located + // in the same directory as the other database files. + IF_DbCopyStatus * ifpStatus) + // [IN] Status callback interface. +{ + RCODE rc = NE_SFLM_OK; + F_Db * pDb = NULL; + FLMBOOL bDbLocked = FALSE; + + // Make sure the destination database is closed + + if (RC_BAD( rc = checkDatabaseClosed( pszDestDbName, pszDestDataDir))) + { + goto Exit; + } + + // Open the source database so we can force a checkpoint. + + if (RC_BAD( rc = openDb( pszSrcDbName, pszSrcDataDir, pszSrcRflDir, + NULL, 0, &pDb))) + { + goto Exit; + } + + // Need to lock the database, because we want to do a checkpoint + // and then the copy immediately after without letting other + // threads have the opportunity to get in and update the + // database. + + if (RC_BAD( rc = pDb->dbLock( SFLM_LOCK_EXCLUSIVE, 0, SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bDbLocked = TRUE; + + // Force a checkpoint + + if (RC_BAD( rc = pDb->doCheckpoint( SFLM_NO_TIMEOUT))) + { + goto Exit; + } + + // Once we get this far, we have exclusive access to the database + // and we have forced a checkpoint. The database's contents are + // guaranteed to be on disk at this point, and they will not + // change. + + rc = copyDb( pszSrcDbName, pszSrcDataDir, pszSrcRflDir, + pszDestDbName, pszDestDataDir, pszDestRflDir, + ifpStatus); + +Exit: + + // Unlock and close the database + + if (bDbLocked) + { + pDb->dbUnlock(); + } + + if (pDb) + { + pDb->Release(); + } + return( rc); +} + +/**************************************************************************** +Desc: Copy a database's files, including roll-forward log files. +*****************************************************************************/ +RCODE F_DbSystem::copyDb( + const char * pszSrcDbName, + const char * pszSrcDataDir, + const char * pszSrcRflDir, + const char * pszDestDbName, + const char * pszDestDataDir, + const char * pszDestRflDir, + IF_DbCopyStatus * ifpStatus) +{ + RCODE rc = NE_SFLM_OK; + DB_COPY_INFO DbCopyInfo; + F_SuperFileHdl SrcSFileHdl; + F_SuperFileHdl DestSFileHdl; + FLMUINT uiFileNumber; + FLMUINT uiHighFileNumber; + FLMUINT uiHighLogFileNumber; + FLMUINT64 ui64FileSize; + F_Database * pDatabase = NULL; + FLMBOOL bMutexLocked = FALSE; + IF_FileHdl * pLockFileHdl = NULL; + IF_FileHdl * pTmpFileHdl = NULL; + IF_DirHdl * pDirHdl = NULL; + FLMBOOL bDatabaseLocked = FALSE; + FLMBOOL bWriteLocked = FALSE; + ServerLockObject * pWriteLockObj = NULL; + ServerLockObject * pDatabaseLockObj = NULL; + COPIED_NAME * pCopiedList = NULL; + FLMBOOL bUsedDatabase = FALSE; + eDbLockType eCurrLockType; + FLMUINT uiThreadId; + FLMUINT uiNumExclQueued; + FLMUINT uiNumSharedQueued; + FLMUINT uiPriorityCount; + char * pszActualSrcRflPath = NULL; + char * pszActualDestRflPath = NULL; + FLMBOOL bCreatedDestRflDir = FALSE; + FLMBOOL bWaited; + F_SEM hWaitSem = F_SEM_NULL; + + f_memset( &DbCopyInfo, 0, sizeof( DbCopyInfo)); + + // Should not do anything if the source and destination names + // are the same. + + if (f_stricmp( pszSrcDbName, pszDestDbName) == 0) + { + goto Exit; + } + + // Create a "wait" semaphore + + if( RC_BAD( rc = f_semCreate( &hWaitSem))) + { + goto Exit; + } + + // Allocate memory for paths we don't want to push onto the stack. + + if (RC_BAD( rc = f_calloc( F_PATH_MAX_SIZE * 2, + &pszActualSrcRflPath))) + { + goto Exit; + } + + pszActualDestRflPath = &pszActualSrcRflPath[ F_PATH_MAX_SIZE]; + + // Set up the super file object for the source database. + // Must at least open the control file. + + if (RC_BAD( rc = SrcSFileHdl.setup( pszSrcDbName, pszSrcDataDir))) + { + goto Exit; + } + + // Lock the destination database, if not already locked. + // This is so we can overwrite it without necessarily + // deleting it. May unlock and re-lock the global mutex. + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + +retry: + + if (RC_BAD( rc = F_DbSystem::findDatabase( pszDestDbName, + pszDestDataDir, &pDatabase))) + { + goto Exit; + } + + // If we didn't find an FFILE structure, get an + // exclusive lock on the file. + + if (!pDatabase) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + // Attempt to get an exclusive lock on the file. + + if (RC_BAD( rc = flmCreateLckFile( pszDestDbName, &pLockFileHdl))) + { + goto Exit; + } + } + else + { + // The call to verifyOkToUse will wait if the database is in + // the process of being opened by another thread. + + if (RC_BAD( rc = pDatabase->verifyOkToUse( &bWaited))) + { + goto Exit; + } + + if (bWaited) + { + goto retry; + } + + // Increment the open count on the F_Database object so it will not + // disappear while we are copying the database. + + pDatabase->incrOpenCount(); + bUsedDatabase = TRUE; + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + // Lock the destination file object and transaction + // object, if not already locked. + + pDatabase->m_pDatabaseLockObj->GetLockInfo( (FLMINT)0, + &eCurrLockType, + &uiThreadId, &uiNumExclQueued, + &uiNumSharedQueued, + &uiPriorityCount); + if (eCurrLockType != SFLM_LOCK_EXCLUSIVE || + uiThreadId != f_threadId()) + { + pDatabaseLockObj = pDatabase->m_pDatabaseLockObj; + pDatabaseLockObj->AddRef(); + + if (RC_BAD( rc = pDatabaseLockObj->Lock( + NULL, hWaitSem, TRUE, FALSE, TRUE, 15, 0))) + { + goto Exit; + } + bDatabaseLocked = TRUE; + } + + // Lock the write object, if not already locked + + pDatabase->m_pWriteLockObj->GetLockInfo( (FLMINT)0, + &eCurrLockType, + &uiThreadId, &uiNumExclQueued, + &uiNumSharedQueued, + &uiPriorityCount); + if (eCurrLockType != SFLM_LOCK_EXCLUSIVE || + uiThreadId != (FLMUINT)f_threadId()) + { + pWriteLockObj = pDatabase->m_pWriteLockObj; + pWriteLockObj->AddRef(); + + // Only contention here is with the checkpoint thread - wait + // forever until the checkpoint thread gives it up. + + if (RC_BAD( rc = pDatabase->dbWriteLock( hWaitSem))) + { + goto Exit; + } + bWriteLocked = TRUE; + } + } + + // Set up the super file object for the destination database. + + if (RC_BAD( rc = DestSFileHdl.setup( pszDestDbName, pszDestDataDir))) + { + goto Exit; + } + + // See how many files we have and calculate the total size. + + uiHighFileNumber = 0; + for (;;) + { + if ((RC_BAD( rc = SrcSFileHdl.getFileSize( + uiHighFileNumber, &ui64FileSize))) || !ui64FileSize ) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME || + !ui64FileSize) + { + // If the control file doesn't exist, we will return + // path not found. + + if (!uiHighFileNumber) + { + goto Exit; + } + uiHighFileNumber--; + rc = NE_SFLM_OK; + break; + } + goto Exit; + } + + DbCopyInfo.ui64BytesToCopy += ui64FileSize; + if (uiHighFileNumber == MAX_DATA_BLOCK_FILE_NUMBER) + { + break; + } + uiHighFileNumber++; + } + + // See how many rollback log files we have, and calculate + // their total size. + + uiHighLogFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + for (;;) + { + if ((RC_BAD( rc = SrcSFileHdl.getFileSize( + uiHighLogFileNumber, &ui64FileSize))) || !ui64FileSize) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME || + !ui64FileSize) + { + if (uiHighLogFileNumber == + FIRST_LOG_BLOCK_FILE_NUMBER) + { + uiHighLogFileNumber = 0; + } + else + { + uiHighLogFileNumber--; + } + rc = NE_SFLM_OK; + break; + } + goto Exit; + } + + DbCopyInfo.ui64BytesToCopy += ui64FileSize; + if (uiHighLogFileNumber == MAX_LOG_BLOCK_FILE_NUMBER) + { + break; + } + uiHighLogFileNumber++; + } + + // Get the sizes of the roll-forward log files + + if (RC_BAD( rc = rflGetDirAndPrefix( pszSrcDbName, + pszSrcRflDir, pszActualSrcRflPath))) + { + goto Exit; + } + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openDir( + pszActualSrcRflPath, (char *)"*", &pDirHdl))) + { + goto Exit; + } + + for (;;) + { + if (RC_BAD( rc = pDirHdl->next())) + { + if (rc == NE_FLM_IO_NO_MORE_FILES) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + + // If the current file is an RFL file, increment ui64BytesToCopy + + if (rflGetFileNum( pDirHdl->currentItemName(), &uiFileNumber)) + { + DbCopyInfo.ui64BytesToCopy += (FLMUINT64)pDirHdl->currentItemSize(); + } + } + + pDirHdl->Release(); + pDirHdl = NULL; + + // Close all file handles in the source and destination + + SrcSFileHdl.releaseFiles( TRUE); + DestSFileHdl.releaseFiles( TRUE); + + // Copy the database files. + + for (uiFileNumber = 0; uiFileNumber <= uiHighFileNumber; uiFileNumber++) + { + + // Get the source file path and destination file path. + + if( RC_BAD( rc = SrcSFileHdl.getFilePath( + uiFileNumber, DbCopyInfo.szSrcFileName))) + { + goto Exit; + } + if( RC_BAD( rc = DestSFileHdl.getFilePath( + uiFileNumber, DbCopyInfo.szDestFileName))) + { + goto Exit; + } + + DbCopyInfo.bNewSrcFile = TRUE; + if (RC_BAD( rc = flmCopyFile( + &DbCopyInfo, &pCopiedList, TRUE, ifpStatus))) + { + goto Exit; + } + } + + // Copy the additional rollback log files, if any. + + for (uiFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + uiFileNumber <= uiHighLogFileNumber; uiFileNumber++) + { + + // Get the source file path and destination file path. + + if (RC_BAD( rc = SrcSFileHdl.getFilePath( uiFileNumber, + DbCopyInfo.szSrcFileName))) + { + goto Exit; + } + + if (RC_BAD( rc = DestSFileHdl.getFilePath( uiFileNumber, + DbCopyInfo.szDestFileName))) + { + goto Exit; + } + + DbCopyInfo.bNewSrcFile = TRUE; + if (RC_BAD( rc = flmCopyFile( + &DbCopyInfo, &pCopiedList, TRUE, ifpStatus))) + { + goto Exit; + } + } + + // Copy the RFL files + + // Create the destination RFL directory, if needed. The purpose of this + // code is two-fold: 1) We want to keep track of the fact that we tried + // to create the destination RFL directory so we can try to remove it + // if the copy fails; 2) If the destination RFL directory path specifies + // a directory with existing files, we want to remove them. + + if( RC_BAD( rc = rflGetDirAndPrefix( pszDestDbName, + pszDestRflDir, pszActualDestRflPath))) + { + goto Exit; + } + + if( RC_OK( gv_SFlmSysData.pFileSystem->doesFileExist( pszActualDestRflPath))) + { + if( gv_SFlmSysData.pFileSystem->isDir( pszActualDestRflPath)) + { + // Remove the existing directory and all files, etc. + + (void)gv_SFlmSysData.pFileSystem->removeDir( + pszActualDestRflPath, TRUE); + } + else + { + (void)gv_SFlmSysData.pFileSystem->deleteFile( pszActualDestRflPath); + } + } + + // Try to create the destination RFL directory. This might fail if + // another process was accessing the directory for some reason + // (i.e., from a command prompt), when we tried to remove it above. + // We really don't care if the call to CreateDir is sucessful, because + // when we try to create the destination files (below), the FLAIM file + // file system code will try to create any necessary directories. + + (void)gv_SFlmSysData.pFileSystem->createDir( pszActualDestRflPath); + bCreatedDestRflDir = TRUE; + + // Copy the RFL files. NOTE: We need to copy all of the RFL files + // in the source RFL directory so that they will be available + // when performing a database restore operation. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openDir( + pszActualSrcRflPath, (char *)"*", &pDirHdl))) + { + goto Exit; + } + + for (;;) + { + if( RC_BAD( rc = pDirHdl->next())) + { + if (rc == NE_FLM_IO_NO_MORE_FILES) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + + // If the current file is an RFL file, copy it to the destination + + if( rflGetFileNum( pDirHdl->currentItemName(), &uiFileNumber)) + { + // Get the source file path and the destination file path. + + if (RC_BAD( rc = rflGetFileName( pszSrcDbName, + pszSrcRflDir, uiFileNumber, + DbCopyInfo.szSrcFileName))) + { + goto Exit; + } + + if (RC_BAD( rc = rflGetFileName( pszDestDbName, + pszDestRflDir, uiFileNumber, + DbCopyInfo.szDestFileName))) + { + goto Exit; + } + + DbCopyInfo.bNewSrcFile = TRUE; + if (RC_BAD( rc = flmCopyFile( + &DbCopyInfo, &pCopiedList, TRUE, ifpStatus))) + { + goto Exit; + } + } + } + + pDirHdl->Release(); + pDirHdl = NULL; + +Exit: + + if (bUsedDatabase) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + } + pDatabase->decrOpenCount(); + } + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + } + + // Unlock the database, if it is locked. + + if (bWriteLocked) + { + pDatabase->dbWriteUnlock(); + bWriteLocked = FALSE; + } + + if (bDatabaseLocked) + { + RCODE rc3; + + if (RC_BAD( rc3 = pDatabaseLockObj->Unlock( TRUE, NULL))) + { + if (RC_OK( rc)) + rc = rc3; + } + bDatabaseLocked = FALSE; + } + + if (pWriteLockObj) + { + pWriteLockObj->Release( FALSE); + pWriteLockObj = NULL; + } + + if (pDatabaseLockObj) + { + pDatabaseLockObj->Release( FALSE); + pDatabaseLockObj = NULL; + } + + if (pLockFileHdl) + { + (void)pLockFileHdl->close(); + pLockFileHdl->Release(); + pLockFileHdl = NULL; + } + + if (pTmpFileHdl) + { + pTmpFileHdl->Release(); + } + + if (pDirHdl) + { + pDirHdl->Release(); + } + + // Free all the names of files that were copied. + // If the copy didn't finish, try to delete any files + // that were copied. + + while (pCopiedList) + { + COPIED_NAME_p pNext = pCopiedList->pNext; + + // If the overall copy failed, delete the copied file. + + if (RC_BAD( rc)) + { + (void)gv_SFlmSysData.pFileSystem->deleteFile( pCopiedList->szPath); + } + + f_free( &pCopiedList); + pCopiedList = pNext; + } + + if (RC_BAD( rc) && bCreatedDestRflDir) + { + (void)gv_SFlmSysData.pFileSystem->removeDir( pszActualDestRflPath); + } + + if (pszActualSrcRflPath) + { + f_free( &pszActualSrcRflPath); + } + + if( hWaitSem != F_SEM_NULL) + { + f_semDestroy( &hWaitSem); + } + + return( rc); +} + +/**************************************************************************** +Desc: Copy a file that is one of the files of the database. +*****************************************************************************/ +FSTATIC RCODE flmCopyFile( + DB_COPY_INFO * pDbCopyInfo, + COPIED_NAME ** ppCopiedListRV, + FLMBOOL bOkToTruncate, + IF_DbCopyStatus * ifpStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucBuffer = NULL; + IF_FileHdl * pSrcFileHdl = NULL; + IF_FileHdl * pDestFileHdl = NULL; + FLMUINT uiBufferSize = 32768; + FLMUINT uiBytesToRead; + FLMUINT uiBytesRead; + FLMUINT uiBytesWritten; + FLMUINT uiOffset; + FLMBOOL bCreatedDestFile = FALSE; + + // Open the source file. + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->openFile( + pDbCopyInfo->szSrcFileName, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + &pSrcFileHdl))) + { + goto Exit; + } + + // Get a file handle for the destination file. + // First attempt to open the destination file. If it does + // not exist, attempt to create it. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openFile( pDbCopyInfo->szDestFileName, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + &pDestFileHdl))) + { + if (rc != NE_FLM_IO_PATH_NOT_FOUND && + rc != NE_FLM_IO_INVALID_FILENAME) + { + goto Exit; + } + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->createFile( + pDbCopyInfo->szDestFileName, + FLM_IO_RDWR | FLM_IO_EXCL | FLM_IO_SH_DENYNONE | + FLM_IO_CREATE_DIR | FLM_IO_DIRECT, + &pDestFileHdl))) + { + goto Exit; + } + bCreatedDestFile = TRUE; + } + + // Allocate a buffer for reading and writing. + + if (RC_BAD( rc = f_alloc( uiBufferSize, &pucBuffer))) + { + goto Exit; + } + + // Read from source file until we hit EOF in the file or + // we hit the end offset. + + uiOffset = 0; + for (;;) + { + uiBytesToRead = (FLMUINT)((0xFFFFFFFF - uiOffset >= + uiBufferSize) + ? uiBufferSize + : (FLMUINT)(0xFFFFFFFF - uiOffset)); + + // Read data from source file. + + if (RC_BAD( rc = pSrcFileHdl->sectorRead( uiOffset, uiBytesToRead, + pucBuffer, &uiBytesRead))) + { + if (rc == NE_FLM_IO_END_OF_FILE) + { + rc = NE_SFLM_OK; + if (!uiBytesRead) + { + break; + } + } + else + { + goto Exit; + } + } + + // Write data to destination file. + + if (RC_BAD( rc = pDestFileHdl->write( uiOffset, + uiBytesRead, pucBuffer, &uiBytesWritten))) + { + goto Exit; + } + + // Do callback to report progress. + + if (ifpStatus) + { + pDbCopyInfo->ui64BytesCopied += (FLMUINT64)uiBytesWritten; + if (RC_BAD( rc = ifpStatus->dbCopyStatus( + pDbCopyInfo->ui64BytesToCopy, + pDbCopyInfo->ui64BytesCopied, + pDbCopyInfo->bNewSrcFile, + pDbCopyInfo->szSrcFileName, + pDbCopyInfo->szDestFileName))) + { + goto Exit; + } + pDbCopyInfo->bNewSrcFile = FALSE; + } + + if (0xFFFFFFFF - uiBytesWritten < uiOffset) + { + uiOffset = 0xFFFFFFFF; + break; + } + + uiOffset += uiBytesWritten; + + // Quit once we reach the end offset or we read fewer bytes + // than we asked for. + + if (uiBytesRead < uiBytesToRead) + { + break; + } + } + + // If we overwrote the destination file, as opposed to creating + // it, truncate it in case it was larger than the number of + // bytes we actually copied. + + if (!bCreatedDestFile && bOkToTruncate) + { + if (RC_BAD( rc = pDestFileHdl->truncate( uiOffset))) + { + goto Exit; + } + } + + // If the copy succeeded, add the destination name to a list + // of destination files. This is done so we can clean up + // copied files if we fail somewhere in the overall database + // copy. + + if (ppCopiedListRV) + { + COPIED_NAME_p pCopyName; + + if( RC_BAD( rc = f_alloc( (FLMUINT)sizeof( COPIED_NAME), &pCopyName))) + { + goto Exit; + } + f_strcpy( pCopyName->szPath, pDbCopyInfo->szDestFileName); + pCopyName->pNext = *ppCopiedListRV; + *ppCopiedListRV = pCopyName; + } + +Exit: + + if (pucBuffer) + { + f_free( &pucBuffer); + } + + if (pSrcFileHdl) + { + pSrcFileHdl->Release(); + } + + if (pDestFileHdl) + { + pDestFileHdl->flush(); + pDestFileHdl->Release(); + } + + // Attempt to delete the destination file if + // we didn't successfully copy it. + + if (RC_BAD( rc)) + { + (void)gv_SFlmSysData.pFileSystem->deleteFile( pDbCopyInfo->szDestFileName); + } + + return( rc); +} diff --git a/sql/src/fdbremov.cpp b/sql/src/fdbremov.cpp new file mode 100644 index 0000000..ac508cc --- /dev/null +++ b/sql/src/fdbremov.cpp @@ -0,0 +1,319 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the F_DbSystem::dbRemove method. +// +// Tabs: 3 +// +// Copyright (c) 2001-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdbremov.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Deletes all files of a database +****************************************************************************/ +RCODE F_DbSystem::dbRemove( + const char * pszDbName, + // [IN] Name of source database to be deleted. + const char * pszDataDir, + // [IN] Directory where data files are located. + const char * pszRflDir, + // [IN] RFL directory of database. NULL can be + // passed to indicate that the log files are located + // in the same directory as the other database files. + FLMBOOL bRemoveRflFiles) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNumber; + char * pszTmpName = NULL; + char * pszRflDirName; + char * pszDataName; + char * pszBaseName; + char * pszExt; + char * pszDataExt; + IF_DirHdl * pDirHdl = NULL; + + // Cannot handle empty database name. + + if( !pszDbName || !(*pszDbName)) + { + rc = RC_SET( NE_FLM_IO_INVALID_FILENAME); + goto Exit; + } + + // Allocate memory, so as to not consume stack. + + if( RC_BAD( rc = f_alloc( F_PATH_MAX_SIZE * 3 + F_FILENAME_SIZE, + &pszTmpName))) + { + goto Exit; + } + + pszRflDirName = pszTmpName + F_PATH_MAX_SIZE; + pszDataName = pszRflDirName + F_PATH_MAX_SIZE; + pszBaseName = pszDataName + F_PATH_MAX_SIZE; + + // First make sure we have closed this database and gotten rid of + // it from our internal memory tables - in case it had been open. + + if (RC_BAD( rc = checkDatabaseClosed( pszDbName, pszDataDir))) + { + goto Exit; + } + + if (pszDataDir && *pszDataDir) + { + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + pszDbName, pszDataName, pszBaseName))) + { + goto Exit; + } + f_strcpy( pszDataName, pszDataDir); + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + pszDataName, pszBaseName))) + { + goto Exit; + } + } + else + { + f_strcpy( pszDataName, pszDbName); + } + f_strcpy( pszTmpName, pszDbName); + + // Start deleting files, beginning with the main DB file. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->deleteFile( pszDbName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + + // Find where the extension of the database name is + + pszExt = pszTmpName + f_strlen( pszTmpName) - 1; + pszDataExt = pszDataName + f_strlen( pszDataName) - 1; + while (pszExt != pszTmpName && *pszExt != '.') + { + pszExt--; + + // Both the db name and data name have the same + // base name, so we can decrement pszDataExt + // at the same time we decrement pszExt. + + pszDataExt--; + } + if (*pszExt != '.') + { + pszExt = pszTmpName + f_strlen( pszTmpName); + pszDataExt = pszDataName + f_strlen( pszDataName); + } + + // Delete the .lck file, if any + + f_strcpy( pszExt, ".lck"); + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->deleteFile( pszTmpName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + + // Delete block (data) files. + + uiFileNumber = 1; + for (;;) + { + bldSuperFileExtension( uiFileNumber, pszDataExt); + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->deleteFile( pszDataName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + if (uiFileNumber == MAX_DATA_BLOCK_FILE_NUMBER) + { + break; + } + uiFileNumber++; + } + + // Delete rollback log files. + + uiFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + for (;;) + { + bldSuperFileExtension( uiFileNumber, pszExt); + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->deleteFile( pszTmpName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + if (uiFileNumber == MAX_LOG_BLOCK_FILE_NUMBER) + { + break; + } + uiFileNumber++; + } + + if (bRemoveRflFiles) + { + + // Delete roll-forward log files. + + FLMBOOL bCanDeleteDir; + + // Scan the RFL directory for RFL files. + + if (RC_BAD( rc = rflGetDirAndPrefix( pszDbName, pszRflDir, pszRflDirName))) + { + goto Exit; + } + + // See if the directory exists. If not, we are done. + + if (!gv_SFlmSysData.pFileSystem->isDir( pszRflDirName)) + { + goto Exit; // Should return NE_SFLM_OK + } + + // Open the directory and scan for RFL files. + // NOTE: DO NOT just call removeDir. There may be other + // things in the directory that we do not want to delete. + // Look specifically for files that match our expected + // name format for RFL files. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openDir( pszRflDirName, + "*", &pDirHdl))) + { + goto Exit; + } + + // Assume that we can delete the directory. This will only + // be set to FALSE if we can't delete all of the files in + // the directory - i.e., some don't look like RFL log files. + + bCanDeleteDir = TRUE; + for (;;) + { + if (RC_BAD( rc = pDirHdl->next())) + { + if (rc == NE_FLM_IO_NO_MORE_FILES) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + pDirHdl->currentItemPath( pszTmpName); + if (pDirHdl->currentItemIsDir()) + { + bCanDeleteDir = FALSE; + } + else if (!rflGetFileNum( pszTmpName, &uiFileNumber)) + { + bCanDeleteDir = FALSE; + } + else + { + if( RC_BAD( rc = + gv_SFlmSysData.pFileSystem->deleteFile( pszTmpName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + } + } + + // Attempt to delete the directory - if allowed. + + if (bCanDeleteDir) + { + + // Need to release the directory handle so the + // directory will be closed when we try to delete it + // below. + + if (pDirHdl) + { + pDirHdl->Release(); + pDirHdl = NULL; + } + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->removeDir( pszRflDirName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + } + } + +Exit: + + if( pszTmpName) + { + f_free( &pszTmpName); + } + + if( pDirHdl) + { + pDirHdl->Release(); + } + + return( rc); +} diff --git a/sql/src/fdbrenam.cpp b/sql/src/fdbrenam.cpp new file mode 100644 index 0000000..2d0ff7f --- /dev/null +++ b/sql/src/fdbrenam.cpp @@ -0,0 +1,446 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the F_DbSystem::dbRename method. +// +// Tabs: 3 +// +// Copyright (c) 2001-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdbrenam.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +typedef struct +{ + char szSrcFileName [F_PATH_MAX_SIZE]; + char szDstFileName [F_PATH_MAX_SIZE]; +} DB_RENAME_INFO, * DB_RENAME_INFO_p; + +typedef struct DBRenameInfoTag +{ + DB_RENAME_INFO Info; + DBRenameInfoTag * pNext; +} DBRenameInfo; + +FSTATIC RCODE flmRenameFile( + const char * pszSrcFileName, + const char * pszDstFileName, + FLMBOOL bOverwriteDestOk, + FLMBOOL bPathNotFoundOk, + DBRenameInfo ** ppRenameList, + FLMBOOL * pbFileFound, + IF_DbRenameStatus * ifpStatus); + +/**************************************************************************** +Desc: Renames all files of a database +****************************************************************************/ +RCODE F_DbSystem::dbRename( + const char * pszDbName, + // [IN] Database to be renamed. + const char * pszDataDir, + // [IN] Directory for data files. + const char * pszRflDir, + // [IN] RFL directory of database. NULL can be + // passed to indicate that the log files are located + // in the same directory as the other database files. + const char * pszNewDbName, + // [IN] New name to be given to the database. May be + // the short name only, or include a directory. If it + // includes a directory, it must be the same directory + // as the directory given in pszDbName. + FLMBOOL bOverwriteDestOk, + // [IN] Ok to overwrite existing file with rename? + IF_DbRenameStatus * ifpStatus) + // [IN] Status callback function. +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNumber; + DBRenameInfo * pRenameList = NULL; + FLMBOOL bFileFound; + char * pszOldName = NULL; + char * pszNewName; + char * pszOldDataName; + char * pszNewDataName; + char * pszFullNewName; + char szOldBase [F_FILENAME_SIZE]; + char szNewBase [F_FILENAME_SIZE]; + char * pszExtOld; + char * pszExtNew; + char * pszDataExtOld; + char * pszDataExtNew; + + // Cannot handle empty database name. + + flmAssert( pszDbName && *pszDbName); + flmAssert( pszNewDbName && *pszNewDbName); + + // Allocate memory for a read buffer, the log header, and various + // file names. + + if (RC_BAD( rc = f_alloc( F_PATH_MAX_SIZE * 5, &pszOldName))) + { + goto Exit; + } + pszNewName = pszOldName + F_PATH_MAX_SIZE; + pszOldDataName = pszNewName + F_PATH_MAX_SIZE; + pszNewDataName = pszOldDataName + F_PATH_MAX_SIZE; + pszFullNewName = pszNewDataName + F_PATH_MAX_SIZE; + + // There must be either no directory specified for the new name, or + // it must be identical to the old directory. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + pszDbName, pszOldName, szOldBase))) + { + goto Exit; + } + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + pszNewDbName, pszNewName, szNewBase))) + { + goto Exit; + } + + // Directories must be the same. + + if (*pszNewName && f_stricmp( pszOldName, pszNewName) != 0) + { + rc = RC_SET( NE_SFLM_INVALID_PARM); + goto Exit; + } + f_strcpy( pszNewName, pszOldName); + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + pszNewName, szNewBase))) + { + goto Exit; + } + + f_strcpy( pszFullNewName, pszNewName); + f_strcpy( pszOldName, pszDbName); + + if (pszDataDir && *pszDataDir) + { + f_strcpy( pszOldDataName, pszDataDir); + f_strcpy( pszNewDataName, pszDataDir); + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + pszOldDataName, szOldBase))) + { + goto Exit; + } + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + pszNewDataName, szNewBase))) + { + goto Exit; + } + } + else + { + f_strcpy( pszNewDataName, pszNewName); + f_strcpy( pszOldDataName, pszOldName); + } + + // First make sure we have closed the databases and gotten rid of + // them from our internal memory tables - in case they had been open. + + if (RC_BAD( rc = checkDatabaseClosed( pszDbName, pszDataDir))) + { + goto Exit; + } + if (RC_BAD( rc = checkDatabaseClosed( pszFullNewName, pszDataDir))) + { + goto Exit; + } + + // Start renaming files, beginning with the main DB file. + + if (RC_BAD( rc = flmRenameFile( pszDbName, pszFullNewName, + bOverwriteDestOk, FALSE, + &pRenameList, &bFileFound, + ifpStatus))) + { + goto Exit; + } + + // Find where the extension of the old and new database names are + + pszExtOld = pszOldName + f_strlen( pszOldName) - 1; + pszDataExtOld = pszOldDataName + f_strlen( pszOldDataName) - 1; + while (pszExtOld != pszOldName && *pszExtOld != '.') + { + pszExtOld--; + + // Both the old db name and old data name have the same + // base name, so we can decrement pszDataExtOld + // at the same time we decrement pszExtOld. + + pszDataExtOld--; + } + if (*pszExtOld != '.') + { + pszExtOld = pszOldName + f_strlen( pszOldName); + pszDataExtOld = pszOldDataName + f_strlen( pszOldDataName); + } + + pszExtNew = pszNewName + f_strlen( pszNewName) - 1; + pszDataExtNew = pszNewDataName + f_strlen( pszNewDataName) - 1; + while (pszExtNew != pszOldName && *pszExtNew != '.') + { + pszExtNew--; + + // Both the new db name and new data name have the same + // base name, so we can decrement pszDataExtNew + // at the same time we decrement pszExtNew. + + pszDataExtNew--; + } + if (*pszExtNew != '.') + { + pszExtNew = pszNewName + f_strlen( pszNewName); + pszDataExtNew = pszNewDataName + f_strlen( pszNewDataName); + } + + // Rename the .lck file, if any. This is necessary for UNIX. + + f_strcpy( pszExtOld, ".lck"); + f_strcpy( pszExtNew, ".lck"); + if (RC_BAD( rc = flmRenameFile( pszOldName, pszNewName, + bOverwriteDestOk, TRUE, + &pRenameList, &bFileFound, + ifpStatus))) + { + goto Exit; + } + + // Rename block (data) files. + + uiFileNumber = 1; + for (;;) + { + bldSuperFileExtension( uiFileNumber, pszDataExtOld); + bldSuperFileExtension( uiFileNumber, pszDataExtNew); + + if (RC_BAD( rc = flmRenameFile( pszOldDataName, pszNewDataName, + bOverwriteDestOk, TRUE, + &pRenameList, &bFileFound, + ifpStatus))) + { + goto Exit; + } + if (!bFileFound) + { + break; + } + if (uiFileNumber == MAX_DATA_BLOCK_FILE_NUMBER) + { + break; + } + uiFileNumber++; + } + + // Rename rollback log files. + + uiFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + for (;;) + { + bldSuperFileExtension( uiFileNumber, pszExtOld); + bldSuperFileExtension( uiFileNumber, pszExtNew); + + if (RC_BAD( rc = flmRenameFile( pszOldName, pszNewName, + bOverwriteDestOk, TRUE, + &pRenameList, &bFileFound, + ifpStatus))) + { + goto Exit; + } + if (!bFileFound) + { + break; + } + if (uiFileNumber == MAX_LOG_BLOCK_FILE_NUMBER) + { + break; + } + uiFileNumber++; + } + + // Rename the RFL directory. + + if (RC_BAD( rc = rflGetDirAndPrefix( pszDbName, pszRflDir, pszOldName))) + { + goto Exit; + } + + if (RC_BAD( rc = rflGetDirAndPrefix( pszFullNewName, pszRflDir, + pszNewName))) + { + goto Exit; + } + + if (RC_BAD( rc = flmRenameFile( pszOldName, pszNewName, + bOverwriteDestOk, TRUE, + &pRenameList, &bFileFound, + ifpStatus))) + { + goto Exit; + } + +Exit: + if (pszOldName) + { + f_free( &pszOldName); + } + + // Free the list of renamed files. + + while (pRenameList) + { + DBRenameInfo * pRenameFile; + + pRenameFile = pRenameList; + pRenameList = pRenameList->pNext; + + // If we had an error of some sort, attempt to un-rename + // the file that had been renamed. + + if (RC_BAD( rc)) + { + gv_SFlmSysData.pFileSystem->renameFile( + pRenameFile->Info.szDstFileName, pRenameFile->Info.szSrcFileName); + } + f_free( &pRenameFile); + } + return( rc); +} + +/**************************************************************************** +Desc: Rename a database file and add to list of renamed files. +****************************************************************************/ +FSTATIC RCODE flmRenameFile( + const char * pszSrcFileName, + const char * pszDstFileName, + FLMBOOL bOverwriteDestOk, + FLMBOOL bPathNotFoundOk, + DBRenameInfo ** ppRenameList, + FLMBOOL * pbFileFound, + IF_DbRenameStatus * ifpStatus) +{ + RCODE rc = NE_SFLM_OK; + DBRenameInfo * pRenameFile = NULL; + + *pbFileFound = FALSE; + + // Should not do anything if the source and destination names + // are the same. + + if (f_stricmp( pszSrcFileName, pszDstFileName) == 0) + { + if (gv_SFlmSysData.pFileSystem->doesFileExist( + pszSrcFileName) == NE_SFLM_OK) + { + *pbFileFound = TRUE; + } + goto Exit; + } + + if (RC_BAD( rc = f_alloc( sizeof( DBRenameInfo), &pRenameFile))) + { + goto Exit; + } + + // If a destination file exists, and it is OK to overwrite + // it, it must be deleted. + + if (bOverwriteDestOk) + { + if (gv_SFlmSysData.pFileSystem->isDir( pszDstFileName)) + { + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->removeDir( + pszDstFileName, TRUE))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->deleteFile( + pszDstFileName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + } + } + + // If names are the same, no need to actually do the + // rename. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->renameFile( + pszSrcFileName, pszDstFileName))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + if (bPathNotFoundOk) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + else + { + goto Exit; + } + } + else + { + *pbFileFound = TRUE; + pRenameFile->pNext = *ppRenameList; + *ppRenameList = pRenameFile; + + // Do user callback. User could choose to stop the rename + // from continuing. + if (ifpStatus) + { + f_strcpy( pRenameFile->Info.szSrcFileName, pszSrcFileName); + f_strcpy( pRenameFile->Info.szDstFileName, pszDstFileName); + if (RC_BAD( rc = ifpStatus->dbRenameStatus( + pRenameFile->Info.szSrcFileName, + pRenameFile->Info.szDstFileName))) + { + goto Exit; + } + } + + // So it won't get deallocated at exit. + + pRenameFile = NULL; + } +Exit: + if (pRenameFile) + { + f_free( &pRenameFile); + } + return( rc); +} diff --git a/sql/src/fdict.cpp b/sql/src/fdict.cpp new file mode 100644 index 0000000..95bcb9b --- /dev/null +++ b/sql/src/fdict.cpp @@ -0,0 +1,3776 @@ +//------------------------------------------------------------------------------ +// Desc: Routines to access anything in the dictionary +// +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +// Local function prototypes + +FSTATIC FLMBOOL validBooleanValue( + const char * pszValue, + FLMBOOL * pbTrue); + +FSTATIC FLMBOOL validDataType( + const char * pszDataType, + eDataType * peDataTyp); + +FSTATIC FLMBOOL validEncAlgorithm( + const char * pszEncAlgorithm, + eEncAlgorithm * peEncAlg); + +FSTATIC FLMBOOL validEncKeySize( + eEncAlgorithm eEncAlg, + FLMUINT uiKeySize); + +FSTATIC void sortNameTbl( + NAME_INFO * pNameInfoTbl, + FLMUINT uiLowerBounds, + FLMUINT uiUpperBounds, + FLMBOOL * pbDuplicateNames); + +FSTATIC FLMBOOL validIndexState( + const char * pszIndexState, + FLMUINT * puiFlags); + +FSTATIC FLMBOOL validIndexOnValue( + const char * pszIndexOn, + FLMUINT * puiFlags); + +FSTATIC FLMBOOL validCompareRulesValue( + char * pszCompareRules, + FLMUINT * puiCompareRules); + +/*************************************************************************** +Desc: Constructor +***************************************************************************/ +F_Dict::F_Dict() +{ + m_pNext = NULL; + m_pPrev = NULL; + m_pDatabase = NULL; + m_uiDictSeq = 0; + m_dictPool.poolInit( 1024); + + m_pTableTbl = NULL; + m_uiTableTblSize = 0; + m_uiHighestTableNum = 0; + m_pTableNames = NULL; + + m_pIndexTbl = NULL; + m_uiIndexTblSize = 0; + m_uiHighestIndexNum = 0; + m_pIndexNames = NULL; + + m_pEncDefTbl = NULL; + m_uiEncDefTblSize = 0; + m_uiHighestEncDefNum = 0; + m_pEncDefNames = NULL; + + // Whenever an F_Dict is allocated, it is always immediately + // used by an F_Db. + + m_uiUseCount = 1; +} + +/*************************************************************************** +Desc: Destructor +***************************************************************************/ +F_Dict::~F_Dict() +{ + resetDict(); +} + +/*************************************************************************** +Desc: Clear the dictionary object so it can be reused. NOTE: This function + also needs to do all the freeing up that the destructor would do + because it is called by the destructor. +***************************************************************************/ +void F_Dict::resetDict( void) +{ + FLMUINT uiLoop; + + for (uiLoop = 0; uiLoop < m_uiHighestTableNum; uiLoop++) + { + if (m_pTableTbl [uiLoop].uiTableNum && + m_pTableTbl [uiLoop].pColumnNames) + { + m_pTableTbl [uiLoop].pColumnNames->Release(); + } + } + + f_free( &m_pTableTbl); + m_uiTableTblSize = 0; + m_uiHighestTableNum = 0; + if (m_pTableNames) + { + m_pTableNames->Release(); + m_pTableNames = NULL; + } + + f_free( &m_pIndexTbl); + m_uiIndexTblSize = 0; + m_uiHighestIndexNum = 0; + if (m_pIndexNames) + { + m_pIndexNames->Release(); + m_pIndexNames = NULL; + } + + for (uiLoop = 0; uiLoop < m_uiHighestEncDefNum; uiLoop++) + { + if (m_pEncDefTbl [uiLoop].uiEncDefNum && + m_pEncDefTbl [uiLoop].pCcs) + { + m_pEncDefTbl[ uiLoop].pCcs->Release(); + } + } + f_free( &m_pEncDefTbl); + m_uiEncDefTblSize = 0; + m_uiHighestEncDefNum = 0; + if (m_pEncDefNames) + { + m_pEncDefNames->Release(); + m_pEncDefNames = NULL; + } + + m_dictPool.poolFree(); + m_dictPool.poolInit( 1024); +} + +/*************************************************************************** +Desc: Get the table given a table name. +***************************************************************************/ +RCODE F_Dict::getTable( + const char * pszTableName, + F_TABLE ** ppTable, + FLMBOOL bOfflineOk) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + + if ((pTable = findTable( pszTableName)) == NULL) + { + rc = RC_SET( NE_SFLM_BAD_TABLE); + goto Exit; + } + + // If the table is encrypted, and we are in limited mode, then we must + // treat is as an offline table. + + if (pTable->lfInfo.uiEncDefNum && m_pDatabase && + m_pDatabase->inLimitedMode() && !bOfflineOk) + { + rc = RC_SET( NE_SFLM_TABLE_OFFLINE); + } + +Exit: + + if (ppTable) + { + *ppTable = pTable; + } + + return( rc); +} + +/*************************************************************************** +Desc: Get the index given an index name. +***************************************************************************/ +RCODE F_Dict::getIndex( + const char * pszIndexName, + F_INDEX ** ppIndex, + FLMBOOL bOfflineOk) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + + if ((pIndex = findIndex( pszIndexName)) == NULL) + { + rc = RC_SET( NE_SFLM_BAD_IX); + goto Exit; + } + + // If the index is suspended the IXD_OFFLINE flag + // will be set, so it is sufficient to just test + // the IXD_OFFLINE for both suspended and offline + // conditions. + + if ((pIndex->uiFlags & IXD_OFFLINE) && !bOfflineOk) + { + rc = RC_SET( NE_SFLM_INDEX_OFFLINE); + goto Exit; + } + + // An encrypted index is offline if we are in limited mode. + + if (pIndex->lfInfo.uiEncDefNum && m_pDatabase && + m_pDatabase->inLimitedMode() && !bOfflineOk) + { + rc = RC_SET( NE_SFLM_INDEX_OFFLINE); + goto Exit; + } + +Exit: + + if (ppIndex) + { + *ppIndex = pIndex; + } + + return( rc); +} + +/**************************************************************************** +Desc: Link the dictionary to an F_Database object + NOTE: This routine assumes the global mutex is locked. +****************************************************************************/ +void F_Dict::linkToDatabase( + F_Database * pDatabase) +{ + if ((m_pNext = pDatabase->m_pDictList) != NULL) + { + m_uiDictSeq = m_pNext->m_uiDictSeq + 1; + m_pNext->m_pPrev = this; + } + else + { + m_uiDictSeq = 1; + } + pDatabase->m_pDictList = this; + m_pDatabase = pDatabase; +} + +/**************************************************************************** +Desc: Unlink the dictionary from its F_Database object + NOTE: This routine assumes the database mutex is locked. +****************************************************************************/ +void F_Dict::unlinkFromDatabase( void) +{ + + // Unlink the local dictionary from its database - if it is connected + // to one. + + if (m_pDatabase) + { + if (m_pPrev) + { + m_pPrev->m_pNext = m_pNext; + } + else + { + m_pDatabase->m_pDictList = m_pNext; + } + if (m_pNext) + { + m_pNext->m_pPrev = m_pPrev; + } + } + + // Free the local dictionary and its associated tables. + + Release(); +} + +/**************************************************************************** +Desc: Copies an encryption def (F_ENCDEF) +****************************************************************************/ +RCODE F_Dict::copyEncDef( + F_ENCDEF * pDestEncDef, + F_ENCDEF * pSrcEncDef) +{ + RCODE rc = NE_SFLM_OK; + + f_memcpy( pDestEncDef, pSrcEncDef, sizeof( F_ENCDEF)); + if (RC_BAD( rc = m_pEncDefNames->copyName( pSrcEncDef->pszEncDefName, + pDestEncDef->uiEncDefNum, &pDestEncDef->pszEncDefName, + &m_dictPool))) + { + goto Exit; + } + pDestEncDef->pCcs->AddRef(); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Copies a table column. NOTE: This routine assumes that + the encryption definitions have already been copied. +****************************************************************************/ +RCODE F_Dict::copyColumn( + F_NameTable * pDestColumnNameTable, + F_COLUMN * pDestColumn, + F_COLUMN * pSrcColumn) +{ + RCODE rc = NE_SFLM_OK; + + f_memcpy( pDestColumn, pSrcColumn, sizeof( F_COLUMN)); + if (RC_BAD( rc = pDestColumnNameTable->copyName( pSrcColumn->pszColumnName, + pDestColumn->uiColumnNum, &pDestColumn->pszColumnName, + &m_dictPool))) + { + goto Exit; + } + + // ICDs will be fixed up and set when the indexes are copied later on. + + pDestColumn->pFirstIcd = NULL; + pDestColumn->pFirstDataIcd = NULL; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Copies a table and all of its columns. NOTE: This routine assumes that + the encryption definitions have already been copied. +****************************************************************************/ +RCODE F_Dict::copyTable( + F_TABLE * pDestTable, + F_TABLE * pSrcTable) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + F_COLUMN * pSrcColumn; + F_COLUMN * pDestColumn; + + f_memcpy( pDestTable, pSrcTable, sizeof( F_TABLE)); + + // Add the table name to the table name table. + + if (RC_BAD( rc = m_pTableNames->copyName( pSrcTable->pszTableName, + pDestTable->uiTableNum, &pDestTable->pszTableName, + &m_dictPool))) + { + goto Exit; + } + + // Allocate a name table for the column names. + + if ((pDestTable->pColumnNames = f_new F_NameTable) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = f_alloc( sizeof( NAME_INFO) * pSrcTable->pColumnNames->m_uiNumNames, + &pDestTable->pColumnNames->m_pNames))) + { + goto Exit; + } + pDestTable->pColumnNames->m_uiTblSize = pSrcTable->pColumnNames->m_uiNumNames; + + // Copy the columns. - There is always at least one column, so no + // need to check for a column count. + + if (RC_BAD( rc = m_dictPool.poolAlloc( sizeof( F_COLUMN) * pSrcTable->uiNumColumns, + (void **)&pDestTable->pColumns))) + { + goto Exit; + } + + for (uiLoop = 0, pSrcColumn = pSrcTable->pColumns, pDestColumn = pDestTable->pColumns; + uiLoop < pSrcTable->uiNumColumns; + uiLoop++, pSrcColumn++, pDestColumn++) + { + if (RC_BAD( rc = copyColumn( pDestTable->pColumnNames, pDestColumn, pSrcColumn))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Copies an index and all of its ICDs. NOTE: This routine assumes that + the tables and columns have already been copied. +****************************************************************************/ +RCODE F_Dict::copyIndex( + F_INDEX * pDestIndex, + F_INDEX * pSrcIndex) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTotalIcds; + ICD * pIcd; + F_COLUMN * pColumn; + F_TABLE * pDestTable; + FLMUINT uiLoop; + + f_memcpy( pDestIndex, pSrcIndex, sizeof( F_INDEX)); + + // Tables should already have been set up, and pSrcIndex->uiTableNum + // better be referencing a valid table! + + pDestTable = getTable( pDestIndex->uiTableNum); + flmAssert( pDestTable); + + // Add the index name to the index name table. + + if (RC_BAD( rc = m_pIndexNames->copyName( pSrcIndex->pszIndexName, + pDestIndex->uiIndexNum, &pDestIndex->pszIndexName, + &m_dictPool))) + { + goto Exit; + } + + // Allocate space for the ICDs. + + uiTotalIcds = pDestIndex->uiNumKeyComponents + pDestIndex->uiNumDataComponents; + if (RC_BAD( rc = m_dictPool.poolAlloc( sizeof( ICD) * uiTotalIcds, + (void **)&pDestIndex->pKeyIcds))) + { + goto Exit; + } + f_memcpy( pDestIndex->pKeyIcds, pSrcIndex->pKeyIcds, + sizeof( ICD) * pDestIndex->uiNumKeyComponents); + if (pDestIndex->pDataIcds) + { + pDestIndex->pDataIcds = pDestIndex->pKeyIcds + pDestIndex->uiNumKeyComponents; + f_memcpy( pDestIndex->pDataIcds, pSrcIndex->pDataIcds, + sizeof( ICD) * pDestIndex->uiNumDataComponents); + } + + // Fixup the index and column pointers in the destination ICD. + + for (uiLoop = 0, pIcd = pDestIndex->pKeyIcds; + uiLoop < pDestIndex->uiNumKeyComponents; + uiLoop++, pIcd++) + { + + // Columns should already be set up and column number better be + // referencing a valid column! + + pColumn = getColumn( pDestTable, pIcd->uiColumnNum); + flmAssert( pColumn); + + pIcd->pNextInChain = pColumn->pFirstIcd; + pColumn->pFirstIcd = pIcd->pNextInChain; + } + for (uiLoop = 0, pIcd = pDestIndex->pDataIcds; + uiLoop < pDestIndex->uiNumDataComponents; + uiLoop++, pIcd++) + { + + // Columns should already be set up and column number better be + // referencing a valid column! + + pColumn = getColumn( pDestTable, pIcd->uiColumnNum); + flmAssert( pColumn); + pIcd->pNextInDataChain = pColumn->pFirstDataIcd; + pColumn->pFirstDataIcd = pIcd->pNextInDataChain; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Clones a dictionary from another one. +****************************************************************************/ +RCODE F_Dict::cloneDict( + F_Dict * pSrcDict) +{ + RCODE rc = NE_SFLM_OK; + F_ENCDEF * pSrcEncDef; + F_ENCDEF * pDestEncDef; + F_TABLE * pSrcTable; + F_TABLE * pDestTable; + F_INDEX * pSrcIndex; + F_INDEX * pDestIndex; + FLMUINT uiLoop; + + resetDict(); + + // Set up all of the name tables to be large enough to hold the + // names in the source dictionary. + + if (pSrcDict->m_pTableNames) + { + if ((m_pTableNames = f_new F_NameTable) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = f_alloc( sizeof( NAME_INFO) * pSrcDict->m_pTableNames->m_uiNumNames, + &m_pTableNames->m_pNames))) + { + goto Exit; + } + m_pTableNames->m_uiTblSize = pSrcDict->m_pTableNames->m_uiNumNames; + } + + if (pSrcDict->m_pIndexNames) + { + if ((m_pIndexNames = f_new F_NameTable) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = f_alloc( sizeof( NAME_INFO) * pSrcDict->m_pIndexNames->m_uiNumNames, + &m_pIndexNames->m_pNames))) + { + goto Exit; + } + m_pIndexNames->m_uiTblSize = pSrcDict->m_pIndexNames->m_uiNumNames; + } + + if (pSrcDict->m_pEncDefNames) + { + if ((m_pEncDefNames = f_new F_NameTable) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = f_alloc( sizeof( NAME_INFO) * pSrcDict->m_pEncDefNames->m_uiNumNames, + &m_pEncDefNames->m_pNames))) + { + goto Exit; + } + m_pEncDefNames->m_uiTblSize = pSrcDict->m_pEncDefNames->m_uiNumNames; + } + + // Copy encryption definitions first. + + if (pSrcDict->m_uiHighestEncDefNum) + { + if (RC_BAD( rc = f_alloc( sizeof( F_ENCDEF) * pSrcDict->m_uiHighestEncDefNum, + &m_pEncDefTbl))) + { + goto Exit; + } + m_uiEncDefTblSize = m_uiHighestEncDefNum = pSrcDict->m_uiHighestEncDefNum; + + for (uiLoop = 0, pSrcEncDef = pSrcDict->m_pEncDefTbl, pDestEncDef = m_pEncDefTbl; + uiLoop < m_uiHighestEncDefNum; + uiLoop++, pSrcEncDef++, pDestEncDef++) + { + if (RC_BAD( rc = copyEncDef( pDestEncDef, pSrcEncDef))) + { + goto Exit; + } + } + } + else + { + m_uiEncDefTblSize = 0; + m_uiHighestEncDefNum = 0; + } + + // Copy tables and columns. - There is always at least one table, so no + // need to check for a table count. + + if (RC_BAD( rc = f_alloc( sizeof( F_TABLE) * pSrcDict->m_uiHighestTableNum, + &m_pTableTbl))) + { + goto Exit; + } + m_uiTableTblSize = m_uiHighestTableNum = pSrcDict->m_uiHighestTableNum; + + for (uiLoop = 0, pSrcTable = pSrcDict->m_pTableTbl, pDestTable = m_pTableTbl; + uiLoop < m_uiHighestTableNum; + uiLoop++, pSrcTable++, pDestTable++) + { + if (RC_BAD( rc = copyTable( pDestTable, pSrcTable))) + { + goto Exit; + } + } + + // Copy indexes. - there is always at least one index, because we have + // internal indexes - so no need to check for an index count. + + if (RC_BAD( rc = f_alloc( sizeof( F_ENCDEF) * pSrcDict->m_uiHighestIndexNum, + &m_pIndexTbl))) + { + goto Exit; + } + m_uiIndexTblSize = m_uiHighestIndexNum = pSrcDict->m_uiHighestIndexNum; + + for (uiLoop = 0, pSrcIndex = pSrcDict->m_pIndexTbl, pDestIndex = m_pIndexTbl; + uiLoop < m_uiHighestIndexNum; + uiLoop++, pSrcIndex++, pDestIndex++) + { + if (RC_BAD( rc = copyIndex( pDestIndex, pSrcIndex))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: See if the read-only value is valid. +****************************************************************************/ +FSTATIC FLMBOOL validBooleanValue( + const char * pszValue, + FLMBOOL * pbTrue) +{ + if (f_stricmp( pszValue, "yes") == 0 || + f_stricmp( pszValue, "1") == 0 || + f_stricmp( pszValue, "enabled") == 0 || + f_stricmp( pszValue, "on") == 0) + { + *pbTrue = TRUE; + return( TRUE); + } + else if (f_stricmp( pszValue, "no") == 0 || + f_stricmp( pszValue, "0") == 0 || + f_stricmp( pszValue, "disabled") == 0 || + f_stricmp( pszValue, "off") == 0) + { + *pbTrue = FALSE; + return( TRUE); + } + return( FALSE); +} + +/*************************************************************************** +Desc: Maps a string to an element or attribute data type. +***************************************************************************/ +FSTATIC FLMBOOL validDataType( + const char * pszDataType, + eDataType * peDataTyp) +{ + if (f_stricmp( pszDataType, SFLM_STRING_OPTION_STR) == 0) + { + *peDataTyp = SFLM_STRING_TYPE; + return( TRUE); + } + else if (f_stricmp( pszDataType, SFLM_INTEGER_OPTION_STR) == 0) + { + *peDataTyp = SFLM_NUMBER_TYPE; + return( TRUE); + } + else if (f_stricmp( pszDataType, SFLM_BINARY_OPTION_STR) == 0) + { + *peDataTyp = SFLM_BINARY_TYPE; + return( TRUE); + } + return( FALSE); +} + +/*************************************************************************** +Desc: Determine if an encryption algorithm is valid. +***************************************************************************/ +FSTATIC FLMBOOL validEncAlgorithm( + const char * pszEncAlgorithm, + eEncAlgorithm * peEncAlg) +{ + if (f_stricmp( pszEncAlgorithm, SFLM_ENC_AES_OPTION_STR) == 0) + { + *peEncAlg = SFLM_AES_ENCRYPTION; + return( TRUE); + } + else if (f_stricmp( pszEncAlgorithm, SFLM_ENC_DES3_OPTION_STR)) + { + *peEncAlg = SFLM_DES3_ENCRYPTION; + return( TRUE); + } + return( FALSE); +} + +/*************************************************************************** +Desc: Determine if the key size for an encryption algorithm is valid. +***************************************************************************/ +FSTATIC FLMBOOL validEncKeySize( + eEncAlgorithm eEncAlg, + FLMUINT uiKeySize) +{ + switch (eEncAlg) + { + case SFLM_AES_ENCRYPTION: + if (uiKeySize == SFLM_AES128_KEY_SIZE || + uiKeySize == SFLM_AES192_KEY_SIZE || + uiKeySize == SFLM_AES256_KEY_SIZE) + { + return( TRUE); + } + break; + case SFLM_DES3_ENCRYPTION: + if (uiKeySize == SFLM_DES3_168_KEY_SIZE) + { + return( TRUE); + } + default: + // Should never hit this case. + flmAssert( 0); + break; + } + return( FALSE); +} + +/**************************************************************************** +Desc: Add an encryption definition to the dictionary. +****************************************************************************/ +RCODE F_Dict::addEncDef( + FLMUINT uiEncDefNum, + FLMUINT64 ui64DefRowId, + const char * pszEncDefName, + eEncAlgorithm eEncAlg, + FLMUINT uiEncKeySize, + FLMBYTE * pucEncKey, + FLMUINT uiEncKeyLen) +{ + RCODE rc = NE_SFLM_OK; + F_ENCDEF * pEncDef; + F_CCS * pCcs = NULL; + + if (!uiEncDefNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + + // Add a table entry to the array of tables. + // See if there is room in the table for another table. + + if (uiEncDefNum >= m_uiEncDefTblSize) + { + FLMUINT uiNewTblSize = uiEncDefNum + 10; + + if (RC_BAD( rc = f_realloc( sizeof( F_ENCDEF) * uiNewTblSize, + &m_pEncDefTbl))) + { + goto Exit; + } + + // Memset the new part of the table to all zeroes - so we can tell + // which slots are empty. + + f_memset( &m_pEncDefTbl [m_uiEncDefTblSize], 0, + sizeof( F_ENCDEF) * (uiNewTblSize - m_uiEncDefTblSize)); + m_uiEncDefTblSize = uiNewTblSize; + } + pEncDef = &m_pEncDefTbl [uiEncDefNum - 1]; + + // Make sure we have not already defined this slot. + + if (pEncDef->uiEncDefNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_ENCDEF_NUM); + goto Exit; + } + + // Add the table name to the table name table. + + if (RC_BAD( rc = m_pEncDefNames->copyName( pszEncDefName, + uiEncDefNum, &pEncDef->pszEncDefName, &m_dictPool))) + { + goto Exit; + } + pEncDef->eEncAlg = eEncAlg; + pEncDef->uiEncKeySize = uiEncKeySize; + + // Create the CCS object + + if ((pCcs = f_new( F_CCS)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pCcs->init( FALSE, eEncAlg))) + { + goto Exit; + } + + if( RC_BAD( rc = pCcs->setKeyFromStore( + pucEncKey, NULL, m_pDatabase->m_pWrappingKey))) + { + goto Exit; + } + + pEncDef->pCcs = pCcs; + pEncDef->pCcs->AddRef(); + + pEncDef->uiEncDefNum = uiEncDefNum; + pEncDef->ui64DefRowId = ui64DefRowId; + if (uiEncDefNum > m_uiHighestEncDefNum) + { + m_uiHighestEncDefNum = uiEncDefNum; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add a table to the dictionary, reserving room for its columns. +****************************************************************************/ +RCODE F_Dict::addTable( + FLMUINT uiTableNum, + FLMUINT64 ui64DefRowId, + const char * pszTableName, + FLMBOOL bSystemTable, + FLMUINT uiNumColumns, + FLMUINT uiEncDefNum) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + + if (!uiTableNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_TABLE_NUM); + goto Exit; + } + + // Verify that the encryption definition is valid. + + if (uiEncDefNum && getEncDef( uiEncDefNum) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + + // Add a table entry to the array of tables. + // See if there is room in the table for another table. + + if (uiTableNum >= m_uiTableTblSize) + { + FLMUINT uiNewTblSize = uiTableNum + 20; + + if (RC_BAD( rc = f_realloc( sizeof( F_TABLE) * uiNewTblSize, + &m_pTableTbl))) + { + goto Exit; + } + + // Memset the new part of the table to all zeroes - so we can tell + // which slots are empty. + + f_memset( &m_pTableTbl [m_uiTableTblSize], 0, + sizeof( F_TABLE) * (uiNewTblSize - m_uiTableTblSize)); + m_uiTableTblSize = uiNewTblSize; + } + pTable = &m_pTableTbl [uiTableNum - 1]; + + // Make sure we have not already defined this slot. + + if (pTable->uiTableNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_TABLE_NUM); + goto Exit; + } + + // Add the table name to the table name table. + + if (RC_BAD( rc = m_pTableNames->copyName( pszTableName, + uiTableNum, &pTable->pszTableName, &m_dictPool))) + { + goto Exit; + } + + // Allocate a name table for the column names. + + if ((pTable->pColumnNames = f_new F_NameTable) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = f_calloc( sizeof( NAME_INFO) * uiNumColumns, + &pTable->pColumnNames->m_pNames))) + { + goto Exit; + } + pTable->pColumnNames->m_uiTblSize = uiNumColumns; + + // Make space for the columns. + + if (RC_BAD( rc = m_dictPool.poolAlloc( sizeof( F_COLUMN) * uiNumColumns, + (void **)&pTable->pColumns))) + { + goto Exit; + } + pTable->uiNumColumns = uiNumColumns; + pTable->bSystemTable = bSystemTable; + f_memset( &pTable->lfInfo, 0, sizeof( LFILE)); + pTable->lfInfo.uiLfNum = uiTableNum; + pTable->lfInfo.uiEncDefNum = uiEncDefNum; + pTable->uiTableNum = uiTableNum; + pTable->ui64DefRowId = ui64DefRowId; + pTable->uiFirstIndexNum = 0; + + if (uiTableNum > m_uiHighestTableNum) + { + m_uiHighestTableNum = uiTableNum; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add a column to a table in the dictionary. +****************************************************************************/ +RCODE F_Dict::addColumn( + FLMUINT uiTableNum, + FLMUINT64 ui64DefRowId, + FLMUINT uiColumnNum, + const char * pszColumnName, + FLMUINT uiFlags, + eDataType eDataTyp, + FLMUINT uiEncDefNum) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + F_COLUMN * pColumn; + + // Verify the table number + + if ((pTable = getTable( uiTableNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_TABLE_NUM); + goto Exit; + } + + // Verify that the encryption definition is valid. + + if (uiEncDefNum && getEncDef( uiEncDefNum) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + + // Make sure the column number is valid. + + if (!uiColumnNum || uiColumnNum > pTable->uiNumColumns) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_COLUMN_NUM); + goto Exit; + } + pColumn = &pTable->pColumns [uiColumnNum - 1]; + + // Column number should not yet be set up. + + if (pColumn->uiColumnNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_COLUMN_NUM); + goto Exit; + } + + // Add the column name to the table's column name table. + + if (RC_BAD( rc = pTable->pColumnNames->copyName( pszColumnName, + uiColumnNum, &pColumn->pszColumnName, &m_dictPool))) + { + goto Exit; + } + pColumn->uiColumnNum = uiColumnNum; + pColumn->ui64DefRowId = ui64DefRowId; + pColumn->uiFlags = uiFlags; + pColumn->eDataTyp = eDataTyp; + pColumn->uiEncDefNum = uiEncDefNum; + pColumn->pFirstIcd = NULL; + pColumn->pFirstDataIcd = NULL; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add an index to the dictionary, reserving room for its key and data + components. +****************************************************************************/ +RCODE F_Dict::addIndex( + FLMUINT uiIndexNum, + FLMUINT64 ui64DefRowId, + const char * pszIndexName, + FLMUINT uiTableNum, + FLMUINT uiEncDefNum, + FLMUINT uiFlags, + FLMUINT uiNumKeyComponents, + FLMUINT uiNumDataComponents, + FLMUINT uiLanguage, + FLMUINT64 ui64LastRowIndexed) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + F_TABLE * pTable; + + if (!uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_INDEX_NUM); + goto Exit; + } + + // Verify the table number + + if ((pTable = getTable( uiTableNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_TABLE_NUM); + goto Exit; + } + + // Verify that the encryption definition is valid. + + if (uiEncDefNum && getEncDef( uiEncDefNum) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + + // Add an index entry to the array of indexes. + // See if there is room in the table for another index. + + if (uiIndexNum >= m_uiIndexTblSize) + { + FLMUINT uiNewTblSize = uiIndexNum + 20; + + if (RC_BAD( rc = f_realloc( sizeof( F_INDEX) * uiNewTblSize, + &m_pIndexTbl))) + { + goto Exit; + } + + // Memset the new part of the table to all zeroes - so we can tell + // which slots are empty. + + f_memset( &m_pIndexTbl [m_uiIndexTblSize], 0, + sizeof( F_INDEX) * (uiNewTblSize - m_uiIndexTblSize)); + m_uiIndexTblSize = uiNewTblSize; + } + pIndex = &m_pIndexTbl [uiIndexNum - 1]; + + // Make sure we have not already defined this slot. + + if (pIndex->uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_INDEX_NUM); + goto Exit; + } + + // Add the index name to the index name table. + + if (RC_BAD( rc = m_pIndexNames->copyName( pszIndexName, + uiIndexNum, &pIndex->pszIndexName, &m_dictPool))) + { + goto Exit; + } + + // Allocate space for the key components and data components + // Must be at least one key component. + + flmAssert( uiNumKeyComponents); + if (RC_BAD( rc = m_dictPool.poolCalloc( + sizeof( ICD) * (uiNumKeyComponents + uiNumDataComponents), + (void **)&pIndex->pKeyIcds))) + { + goto Exit; + } + if (uiNumDataComponents) + { + pIndex->pDataIcds = pIndex->pKeyIcds + uiNumKeyComponents; + } + else + { + pIndex->pDataIcds = NULL; + } + pIndex->uiNumKeyComponents = uiNumKeyComponents; + pIndex->uiNumDataComponents = uiNumDataComponents; + + // Set other members of the index structure. + + pIndex->uiTableNum = uiTableNum; + pIndex->uiFlags = uiFlags; + pIndex->uiLanguage = uiLanguage; + pIndex->ui64LastRowIndexed = ui64LastRowIndexed; + f_memset( &pIndex->lfInfo, 0, sizeof( LFILE)); + pIndex->lfInfo.uiLfNum = uiIndexNum; + pIndex->lfInfo.uiEncDefNum = uiEncDefNum; + pIndex->uiIndexNum = uiIndexNum; + pIndex->ui64DefRowId = ui64DefRowId; + + // Link the index into the list of indexes for the table + + pIndex->uiNextIndexNum = pTable->uiFirstIndexNum; + pTable->uiFirstIndexNum = uiIndexNum; + + if (uiIndexNum > m_uiHighestIndexNum) + { + m_uiHighestIndexNum = uiIndexNum; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add an index column to an index's key component ICD array and/or to + the index's data component ICD array. +****************************************************************************/ +RCODE F_Dict::addIndexComponent( + FLMUINT uiIndexNum, + FLMUINT64 ui64DefRowId, + FLMUINT uiColumnNum, + FLMUINT uiFlags, + FLMUINT uiCompareRules, + FLMUINT uiLimit, + FLMUINT uiKeyComponent, + FLMUINT uiDataComponent) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + F_TABLE * pTable; + ICD * pIcd; + F_COLUMN * pColumn; + + // Verify the index number + + if ((pIndex = getIndex( uiIndexNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_INDEX_NUM); + goto Exit; + } + + // If we have a valid pIndex, it's table number should have already + // been validated. + + pTable = getTable( pIndex->uiTableNum); + flmAssert( pTable); + + // Verify the column number. + + if ((pColumn = getColumn( pTable, uiColumnNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_COLUMN_NUM); + goto Exit; + } + + // See if it is a key component. + // NOTE: It is possible for the column to be both a key component and + // a data component. + + if (uiKeyComponent) + { + if (uiKeyComponent > pIndex->uiNumKeyComponents) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_KEY_COMPONENT); + goto Exit; + } + pIcd = &pIndex->pKeyIcds [uiKeyComponent - 1]; + + // Make sure this component hasn't already been defined. + + if (pIcd->uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_KEY_COMPONENT); + goto Exit; + } + + pIcd->uiIndexNum = uiIndexNum; + pIcd->uiColumnNum = uiColumnNum; + pIcd->ui64DefRowId = ui64DefRowId; + pIcd->uiFlags = uiFlags; + pIcd->uiCompareRules = uiCompareRules; + pIcd->uiLimit = uiLimit; + pIcd->pNextInChain = pColumn->pFirstIcd; + pColumn->pFirstIcd = pIcd; + } + + // See if it is a data component. + + if (uiDataComponent) + { + if (uiDataComponent > pIndex->uiNumDataComponents) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_DATA_COMPONENT); + goto Exit; + } + + pIcd = &pIndex->pDataIcds [uiDataComponent - 1]; + + // Make sure this component hasn't already been defined. + + if (pIcd->uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_DATA_COMPONENT); + goto Exit; + } + + pIcd->uiIndexNum = uiIndexNum; + pIcd->uiColumnNum = uiColumnNum; + pIcd->ui64DefRowId = ui64DefRowId; + pIcd->uiFlags = 0; + pIcd->uiCompareRules = 0; + pIcd->pNextInChain = NULL; + pIcd->uiLimit = 0; + pIcd->pNextInDataChain = pColumn->pFirstDataIcd; + pColumn->pFirstDataIcd = pIcd; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the encryption definition table and its associated indexes. +****************************************************************************/ +RCODE F_Dict::setupEncDefTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create the table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_TBLNAM_ENCDEFS, TRUE, + SFLM_ENCDEFS_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add Columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_COLNUM_ENCDEFS_ENCDEF_NAME, + SFLM_COLNAM_ENCDEFS_ENCDEF_NAME, + 0, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_COLNUM_ENCDEFS_ENCDEF_NUM, + SFLM_COLNAM_ENCDEFS_ENCDEF_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_COLNUM_ENCDEFS_ENC_ALGORITHM, + SFLM_COLNAM_ENCDEFS_ENC_ALGORITHM, + COL_READ_ONLY, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_COLNUM_ENCDEFS_ENC_KEY_SIZE, + SFLM_COLNAM_ENCDEFS_ENC_KEY_SIZE, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_ENCDEFS, 0, + SFLM_COLNUM_ENCDEFS_ENC_KEY, + SFLM_COLNAM_ENCDEFS_ENC_KEY, + COL_READ_ONLY, SFLM_BINARY_TYPE, 0))) + { + goto Exit; + } + + // Add an index on the encdef table, encdef name column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_ENCDEF_NAME, 0, + SFLM_IXNAM_ENCDEF_NAME, + SFLM_TBLNUM_ENCDEFS, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_ENCDEF_NAME, 0, + SFLM_COLNUM_ENCDEFS_ENCDEF_NAME, + ICD_VALUE, FLM_COMP_CASE_INSENSITIVE, 0, + 1, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the table table and its associated indexes. +****************************************************************************/ +RCODE F_Dict::setupTableTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_TABLES, 0, + SFLM_TBLNAM_TABLES, TRUE, + SFLM_TABLES_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add Columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_TABLES, 0, + SFLM_COLNUM_TABLES_TABLE_NAME, + SFLM_COLNAM_TABLES_TABLE_NAME, + 0, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_TABLES, 0, + SFLM_COLNUM_TABLES_TABLE_NUM, + SFLM_COLNAM_TABLES_TABLE_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_TABLES, 0, + SFLM_COLNUM_TABLES_ENCDEF_NUM, + SFLM_COLNAM_TABLES_ENCDEF_NUM, + 0, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_TABLES, 0, + SFLM_COLNUM_TABLES_NUM_COLUMNS, + SFLM_COLNAM_TABLES_NUM_COLUMNS, + 0, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + + // Add an index on the table name column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_TABLE_NAME, 0, + SFLM_IXNAM_TABLE_NAME, + SFLM_TBLNUM_TABLES, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_TABLE_NAME, 0, + SFLM_COLNUM_TABLES_TABLE_NAME, + ICD_VALUE, FLM_COMP_CASE_INSENSITIVE, 0, + 1, 0))) + { + goto Exit; + } + + // Add an index on the encdef number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_TABLE_ENCDEF_NUM, 0, + SFLM_IXNAM_TABLE_ENCDEF_NUM, + SFLM_TBLNUM_TABLES, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_TABLE_ENCDEF_NUM, 0, + SFLM_COLNUM_TABLES_ENCDEF_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the column table and its associated indexes. +****************************************************************************/ +RCODE F_Dict::setupColumnTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_COLUMNS, 0, + SFLM_TBLNAM_COLUMNS, TRUE, + SFLM_COLUMNS_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_TABLE_NUM, + SFLM_COLNAM_COLUMNS_TABLE_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_COLUMN_NAME, + SFLM_COLNAM_COLUMNS_COLUMN_NAME, + 0, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_COLUMN_NUM, + SFLM_COLNAM_COLUMNS_COLUMN_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_DATA_TYPE, + SFLM_COLNAM_COLUMNS_DATA_TYPE, + COL_READ_ONLY, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_ENCDEF_NUM, + SFLM_COLNAM_COLUMNS_ENCDEF_NUM, + COL_READ_ONLY | COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_READ_ONLY, + SFLM_COLNAM_COLUMNS_READ_ONLY, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_COLUMNS, 0, + SFLM_COLNUM_COLUMNS_NULL_ALLOWED, + SFLM_COLNAM_COLUMNS_NULL_ALLOWED, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + + // Add an index on the table number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_COLUMN_TABLE_NUM, 0, + SFLM_IXNAM_COLUMN_TABLE_NUM, + SFLM_TBLNUM_COLUMNS, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_COLUMN_TABLE_NUM, 0, + SFLM_COLNUM_COLUMNS_TABLE_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + + // Add an index on the encdef number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_COLUMN_ENCDEF_NUM, 0, + SFLM_IXNAM_COLUMN_ENCDEF_NUM, + SFLM_TBLNUM_COLUMNS, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_COLUMN_ENCDEF_NUM, 0, + SFLM_COLNUM_COLUMNS_ENCDEF_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the index table and its associated indexes. +****************************************************************************/ +RCODE F_Dict::setupIndexTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create the table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_INDEXES, 0, + SFLM_TBLNAM_INDEXES, TRUE, + SFLM_INDEXES_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_INDEX_NAME, + SFLM_COLNAM_INDEXES_INDEX_NAME, + 0, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_INDEX_NUM, + SFLM_COLNAM_INDEXES_INDEX_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_TABLE_NUM, + SFLM_COLNAM_INDEXES_TABLE_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_ENCDEF_NUM, + SFLM_COLNAM_INDEXES_ENCDEF_NUM, + COL_READ_ONLY | COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_LANGUAGE, + SFLM_COLNAM_INDEXES_LANGUAGE, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_NUM_KEY_COMPONENTS, + SFLM_COLNAM_INDEXES_NUM_KEY_COMPONENTS, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_NUM_DATA_COMPONENTS, + SFLM_COLNAM_INDEXES_NUM_DATA_COMPONENTS, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_LAST_ROW_INDEXED, + SFLM_COLNAM_INDEXES_LAST_ROW_INDEXED, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEXES, 0, + SFLM_COLNUM_INDEXES_INDEX_STATE, + SFLM_COLNAM_INDEXES_INDEX_STATE, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + + // Add an index on the index name column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_INDEX_NAME, 0, + SFLM_IXNAM_INDEX_NAME, + SFLM_TBLNUM_INDEXES, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_INDEX_NAME, 0, + SFLM_COLNUM_INDEXES_INDEX_NAME, + ICD_VALUE, FLM_COMP_CASE_INSENSITIVE, 0, + 1, 0))) + { + goto Exit; + } + + // Add an index on the table number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_INDEX_TABLE_NUM, 0, + SFLM_IXNAM_INDEX_TABLE_NUM, + SFLM_TBLNUM_INDEXES, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_INDEX_TABLE_NUM, 0, + SFLM_COLNUM_INDEXES_TABLE_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + + // Add an index on the encdef number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_INDEX_ENCDEF_NUM, 0, + SFLM_IXNAM_INDEX_ENCDEF_NUM, + SFLM_TBLNUM_INDEXES, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_INDEX_ENCDEF_NUM, 0, + SFLM_COLNUM_INDEXES_ENCDEF_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the index component table and its associated indexes. +****************************************************************************/ +RCODE F_Dict::setupIndexComponentTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create the table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_TBLNAM_INDEX_COMPONENTS, TRUE, + SFLM_INDEX_COMP_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add Columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_INDEX_NUM, + SFLM_COLNAM_INDEX_COMP_INDEX_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_COLUMN_NUM, + SFLM_COLNAM_INDEX_COMP_COLUMN_NUM, + COL_READ_ONLY, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_KEY_COMPONENT, + SFLM_COLNAM_INDEX_COMP_KEY_COMPONENT, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_DATA_COMPONENT, + SFLM_COLNAM_INDEX_COMP_DATA_COMPONENT, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_INDEX_ON, + SFLM_COLNAM_INDEX_COMP_INDEX_ON, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_COMPARE_RULES, + SFLM_COLNAM_INDEX_COMP_COMPARE_RULES, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_SORT_DESCENDING, + SFLM_COLNAM_INDEX_COMP_SORT_DESCENDING, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_SORT_MISSING_HIGH, + SFLM_COLNAM_INDEX_COMP_SORT_MISSING_HIGH, + COL_NULL_ALLOWED, SFLM_STRING_TYPE, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_INDEX_COMPONENTS, 0, + SFLM_COLNUM_INDEX_COMP_LIMIT, + SFLM_COLNAM_INDEX_COMP_LIMIT, + COL_NULL_ALLOWED, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + + // Add an index on the index number column + + if (RC_BAD( rc = addIndex( SFLM_IXNUM_INDEX_COMP_INDEX_NUM, 0, + SFLM_IXNAM_INDEX_COMP_INDEX_NUM, + SFLM_TBLNUM_INDEX_COMPONENTS, 0, + IXD_SYSTEM, 1, 0, FLM_US_LANG, 0))) + { + goto Exit; + } + if (RC_BAD( rc = addIndexComponent( SFLM_IXNUM_INDEX_COMP_INDEX_NUM, 0, + SFLM_COLNUM_INDEX_COMP_INDEX_NUM, + ICD_VALUE, 0, 0, + 1, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the block chain table. +****************************************************************************/ +RCODE F_Dict::setupBlockChainTable( void) +{ + RCODE rc = NE_SFLM_OK; + + // Create the table + + if (RC_BAD( rc = addTable( SFLM_TBLNUM_BLOCK_CHAINS, 0, + SFLM_TBLNAM_BLOCK_CHAINS, TRUE, + SFLM_BLOCK_CHAINS_NUM_COLUMNS, 0))) + { + goto Exit; + } + + // Add columns + + if (RC_BAD( rc = addColumn( SFLM_TBLNUM_BLOCK_CHAINS, 0, + SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS, + SFLM_COLNAM_BLOCK_CHAINS_BLOCK_ADDRESS, + 0, SFLM_NUMBER_TYPE, 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the predefined tables and indexes. +****************************************************************************/ +RCODE F_Dict::setupPredefined( void) +{ + RCODE rc = NE_SFLM_OK; + + // Set up the encryption definition table + + if (RC_BAD( rc = setupEncDefTable())) + { + goto Exit; + } + + // Set up the table table + + if (RC_BAD( rc = setupTableTable())) + { + goto Exit; + } + + // Set up the column table + + if (RC_BAD( rc = setupColumnTable())) + { + goto Exit; + } + + // Set up the index table + + if (RC_BAD( rc = setupIndexTable())) + { + goto Exit; + } + + // Set up the index component table + + if (RC_BAD( rc = setupIndexComponentTable())) + { + goto Exit; + } + + // Set up the block chain deletion table + + if (RC_BAD( rc= setupBlockChainTable())) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Verify that a dictionary is good after having read it in. These are + things that didn't get verified as we were reading in. +****************************************************************************/ +RCODE F_Dict::verifyDict( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTableNum; + F_TABLE * pTable; + FLMUINT uiColumnNum; + F_COLUMN * pColumn; + FLMUINT uiIndexNum; + F_INDEX * pIndex; + ICD * pIcd; + FLMUINT uiKeyComponent; + FLMUINT uiDataComponent; + + // Make sure we have no duplicate table names, index names, or encryption + // definition names. + + if (m_pTableNames) + { + m_pTableNames->sortNames(); + if (m_pTableNames->m_bDuplicateNames) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_TABLE_NAME); + goto Exit; + } + } + if (m_pIndexNames) + { + m_pIndexNames->sortNames(); + if (m_pTableNames->m_bDuplicateNames) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_INDEX_NAME); + goto Exit; + } + } + if (m_pEncDefNames) + { + m_pEncDefNames->sortNames(); + if (m_pTableNames->m_bDuplicateNames) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_ENCDEF_NAME); + goto Exit; + } + } + + // Loop through all of the tables. Verify that all of the column numbers + // are defined, and that for each column name table there are no duplicate + // column names. + + for (uiTableNum = 0, pTable = m_pTableTbl; + uiTableNum < m_uiHighestTableNum; + uiTableNum++, pTable++) + { + if (!pTable->uiTableNum) + { + continue; + } + + // Sort the column names - make sure there are no duplicate column names. + + pTable->pColumnNames->sortNames(); + if (pTable->pColumnNames->m_bDuplicateNames) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DUPLICATE_COLUMN_NAME); + goto Exit; + } + + // Make sure each column is defined. + + for (uiColumnNum = 0, pColumn = pTable->pColumns; + uiColumnNum < pTable->uiNumColumns; + uiColumnNum++, pColumn++) + { + + // uiColumnNum will be zero if it never got defined. + + if (!pColumn->uiColumnNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_UNDEFINED_COLUMN_NUM); + goto Exit; + } + } + } + + // Verify that for each index all of the key components and data components + // are defined. + + for (uiIndexNum = 0, pIndex = m_pIndexTbl; + uiIndexNum < m_uiHighestIndexNum; + uiIndexNum++, pIndex++) + { + if (!pIndex->uiIndexNum) + { + continue; + } + + // Make sure all key components are defined. + + for (uiKeyComponent = 0, pIcd = pIndex->pKeyIcds; + uiKeyComponent < pIndex->uiNumKeyComponents; + uiKeyComponent++, pIcd++) + { + + // If the ICD is not defined, the uiIndexNum field will be 0 + + if (!pIcd->uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_UNDEFINED_KEY_COMPONENT); + goto Exit; + } + } + + // Make sure all data components are defined. + + for (uiDataComponent = 0, pIcd = pIndex->pDataIcds; + uiDataComponent < pIndex->uiNumDataComponents; + uiDataComponent++, pIcd++) + { + + // If the ICD is not defined, the uiIndexNum field will be 0 + + if (!pIcd->uiIndexNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_UNDEFINED_DATA_COMPONENT); + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Read in LFH headers. +****************************************************************************/ +RCODE F_Db::dictReadLFH( void) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + F_INDEX * pIndex; + F_CachedBlock * pSCache; + FLMBOOL bReleaseCache = FALSE; + F_BLK_HDR * pBlkHdr; + FLMUINT uiBlkAddress; + FLMUINT uiPos; + FLMUINT uiEndPos; + FLMUINT uiBlkSize = m_pDatabase->m_uiBlockSize; + LFILE TmpLFile; + + f_memset( &TmpLFile, 0, sizeof( LFILE)); + + // Read through all of the LFILE blocks. + + uiBlkAddress = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32FirstLFBlkAddr; + while (uiBlkAddress) + { + if (RC_BAD( rc = m_pDatabase->getBlock( this, NULL, + uiBlkAddress, NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + pBlkHdr = pSCache->m_pBlkHdr; + uiPos = SIZEOF_STD_BLK_HDR; + uiEndPos = blkGetEnd( uiBlkSize, SIZEOF_STD_BLK_HDR, pBlkHdr); + + // Read through all of the logical file definitions in the block + + for( ; uiPos + sizeof( F_LF_HDR) <= uiEndPos; uiPos += sizeof( F_LF_HDR)) + { + F_LF_HDR * pLfHdr = (F_LF_HDR *)((FLMBYTE *)(pBlkHdr) + uiPos); + eLFileType eLfType = (eLFileType)pLfHdr->ui32LfType; + + // Have to fix up the offsets later when they are read in + + if (eLfType == SFLM_LF_INVALID) + { + continue; + } + + // Populate the LFILE in the dictionary, if one has been set up. + + if (eLfType == SFLM_LF_INDEX) + { + FSLFileIn( (FLMBYTE *)pLfHdr, &TmpLFile, uiBlkAddress, uiPos); + + if ((pIndex = m_pDict->getIndex( TmpLFile.uiLfNum)) != NULL) + { + f_memcpy( &pIndex->lfInfo, &TmpLFile, sizeof( LFILE)); + } + + // LFILE better have a non-zero root block. + + if (!TmpLFile.uiRootBlk) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + } + else + { + + // Better be a container + + flmAssert( eLfType == SFLM_LF_TABLE); + + FSLFileIn( (FLMBYTE *)pLfHdr, &TmpLFile, uiBlkAddress, uiPos); + + if ((pTable = m_pDict->getTable( TmpLFile.uiLfNum)) != NULL) + { + f_memcpy( &pTable->lfInfo, &TmpLFile, sizeof( LFILE)); + } + + // LFILE better have a non-zero root block. + + if (!TmpLFile.uiRootBlk) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + } + } + + // Get the next block in the chain + + uiBlkAddress = (FLMUINT)pBlkHdr->ui32NextBlkInChain; + ScaReleaseCache( pSCache, FALSE); + bReleaseCache = FALSE; + } + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc ); +} + +/**************************************************************************** +Desc: Read in all rows from the encryption definition table. + Create the in-memory structures for each encryption definition. +****************************************************************************/ +RCODE F_Db::dictReadEncDefs( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow = NULL; + FLMUINT64 ui64DefRowId; + FSTableCursor * pTableCursor = NULL; + char szEncDefName [MAX_ENCDEF_NAME_LEN + 1]; + FLMUINT uiEncDefNameLen; + FLMUINT uiEncDefNum; + char szEncAlgorithm [20]; + eEncAlgorithm eEncAlg; + FLMUINT uiEncKeySize; + FLMBYTE * pucEncKey = NULL; + FLMUINT uiEncKeyLen; + FLMBOOL bIsNull; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_ENCDEFS, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the table. Each row defines a table in + // the database, so we add it to the in-memory dictionary. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, &ui64DefRowId))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the encryption definition name - required. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_ENCDEFS_ENCDEF_NAME, + szEncDefName, sizeof( szEncDefName), + &bIsNull, &uiEncDefNameLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiEncDefNameLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENCDEF_NAME); + goto Exit; + } + + // Get the encryption definition number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_ENCDEFS_ENCDEF_NUM, + &uiEncDefNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENCDEF_NUM); + goto Exit; + } + + // Get the encryption algorithm - required. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_ENCDEFS_ENC_ALGORITHM, + szEncAlgorithm, sizeof( szEncAlgorithm), + &bIsNull, NULL, NULL))) + { + goto Exit; + } + if (bIsNull || !szEncAlgorithm [0]) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENC_ALGORITHM); + goto Exit; + } + + // Make sure the encryption algorithm and key size are valid. + + if (!validEncAlgorithm( szEncAlgorithm, &eEncAlg)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENC_ALGORITHM); + goto Exit; + } + + // Get the encryption key size - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_ENCDEFS_ENC_KEY_SIZE, + &uiEncKeySize, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENC_KEY_SIZE); + goto Exit; + } + if (!validEncKeySize( eEncAlg, uiEncKeySize)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENC_KEY_SIZE); + goto Exit; + } + + // Get the encryption key. There better be one at this point. + + pRow->getDataLen( this, SFLM_COLNUM_ENCDEFS_ENC_KEY, + &uiEncKeyLen, &bIsNull); + if (bIsNull || !uiEncKeyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENC_KEY); + goto Exit; + } + if (RC_BAD( rc = f_alloc( uiEncKeyLen, &pucEncKey))) + { + goto Exit; + } + if (RC_BAD( rc = pRow->getBinary( this, SFLM_COLNUM_ENCDEFS_ENC_KEY, + pucEncKey, uiEncKeyLen, &uiEncKeyLen, + &bIsNull))) + { + goto Exit; + } + flmAssert( !bIsNull); // If encryption key length > 0, this better never be TRUE! + + if (RC_BAD( rc = m_pDict->addEncDef( uiEncDefNum, ui64DefRowId, + szEncDefName, eEncAlg, + uiEncKeySize, pucEncKey, uiEncKeyLen))) + { + goto Exit; + } + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + if (pucEncKey) + { + f_free( &pucEncKey); + } + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Read in all rows from the table table. Create the in-memory structures + for each defined table. +****************************************************************************/ +RCODE F_Db::dictReadTables( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow = NULL; + FLMUINT64 ui64DefRowId; + FSTableCursor * pTableCursor = NULL; + char szTableName [MAX_TABLE_NAME_LEN + 1]; + FLMUINT uiTableNameLen; + FLMUINT uiTableNum; + FLMUINT uiEncDefNum; + FLMUINT uiNumColumns; + FLMBOOL bIsNull; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_TABLES, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the table. Each row defines a table in + // the database, so we add it to the in-memory dictionary. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, &ui64DefRowId))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the table name - required. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_TABLES_TABLE_NAME, + szTableName, sizeof( szTableName), + &bIsNull, &uiTableNameLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiTableNameLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_TABLE_NAME); + goto Exit; + } + + // Get the table number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_TABLES_TABLE_NUM, + &uiTableNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_TABLE_NUM); + goto Exit; + } + + // Get the encryption name - optional. But if present, it must be + // defined in the dictionary already. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_TABLES_ENCDEF_NUM, + &uiEncDefNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiEncDefNum = 0; + } + + // Get the number of columns. Must be non-zero. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_TABLES_NUM_COLUMNS, + &uiNumColumns, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_NUM_COLUMNS); + goto Exit; + } + if (!uiNumColumns) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_NUM_COLUMNS); + goto Exit; + } + + // Add an entry to the in-memory dictionary for the table. + + if (RC_BAD( rc = m_pDict->addTable( uiTableNum, ui64DefRowId, + szTableName, FALSE, + uiNumColumns, uiEncDefNum))) + { + goto Exit; + } + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Read in all rows from the column table. Create the in-memory structures + for each defined column. +****************************************************************************/ +RCODE F_Db::dictReadColumns( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow = NULL; + FLMUINT64 ui64DefRowId; + FSTableCursor * pTableCursor = NULL; + FLMUINT uiEncDefNum; + FLMUINT uiTableNum; + char szColumnName [MAX_COLUMN_NAME_LEN + 1]; + FLMUINT uiColumnNameLen; + FLMUINT uiColumnNum; + eDataType eDataTyp; + FLMUINT uiFlags; + char szTmp [100]; + FLMUINT uiTmpLen; + FLMBOOL bTmp; + FLMBOOL bIsNull; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_COLUMNS, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the table. Each row defines a column in + // a table in the database, so we add it to the in-memory dictionary. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, &ui64DefRowId))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the table number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_COLUMNS_TABLE_NUM, + &uiTableNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_TABLE_NUM); + goto Exit; + } + + // Get the column name. There must be a column name. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_COLUMNS_COLUMN_NAME, + szColumnName, sizeof( szColumnName), + &bIsNull, &uiColumnNameLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiColumnNameLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_COLUMN_NAME); + goto Exit; + } + + // Get column number - must be non-zero. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_COLUMNS_COLUMN_NUM, + &uiColumnNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_COLUMN_NUM); + goto Exit; + } + flmAssert( uiColumnNum); + + // Get the encryption definition number - optional. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_COLUMNS_ENCDEF_NUM, + &uiEncDefNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiEncDefNum = 0; + } + + // Get the data type - required. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_COLUMNS_DATA_TYPE, + szTmp, sizeof( szTmp), + &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiTmpLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_DATA_TYPE); + goto Exit; + } + if (!validDataType( szTmp, &eDataTyp)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_DATA_TYPE); + goto Exit; + } + + uiFlags = 0; + + // Get the read-only flag - optional. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_COLUMNS_READ_ONLY, + szTmp, sizeof( szTmp), + &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (!validBooleanValue( szTmp, &bTmp)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_READ_ONLY_VALUE); + goto Exit; + } + if (bTmp) + { + uiFlags |= COL_READ_ONLY; + } + } + + // Get the null-allowed flag + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_COLUMNS_NULL_ALLOWED, + szTmp, sizeof( szTmp), + &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (!validBooleanValue( szTmp, &bTmp)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_NULL_ALLOWED_VALUE); + goto Exit; + } + if (bTmp) + { + uiFlags |= COL_NULL_ALLOWED; + } + } + + // Add the column to the table entry in the in-memory dictionary. + + if (RC_BAD( rc = m_pDict->addColumn( uiTableNum, ui64DefRowId, + uiColumnNum, szColumnName, uiFlags, + eDataTyp, uiEncDefNum))) + { + goto Exit; + } + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Test the index state string to see if it is valid. Set index flags + accordingly. +****************************************************************************/ +FSTATIC FLMBOOL validIndexState( + const char * pszIndexState, + FLMUINT * puiFlags) +{ + if (f_stricmp( pszIndexState, SFLM_INDEX_SUSPENDED_STR) == 0) + { + (*puiFlags) |= IXD_SUSPENDED; + (*puiFlags) &= (~(IXD_OFFLINE)); + return( TRUE); + } + else if (f_stricmp( pszIndexState, SFLM_INDEX_OFFLINE_STR) == 0) + { + (*puiFlags) |= IXD_OFFLINE; + (*puiFlags) &= (~(IXD_SUSPENDED)); + return( TRUE); + } + else if (f_stricmp( pszIndexState, SFLM_INDEX_ONLINE_STR) == 0) + { + (*puiFlags) &= (~(IXD_SUSPENDED | IXD_OFFLINE)); + return( TRUE); + } + return( FALSE); +} + +/**************************************************************************** +Desc: Read in all rows from the index table. Create the in-memory structures + for each index. +****************************************************************************/ +RCODE F_Db::dictReadIndexes( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow = NULL; + FLMUINT64 ui64DefRowId; + FSTableCursor * pTableCursor = NULL; + FLMUINT uiEncDefNum; + FLMUINT uiTableNum; + char szIndexName [MAX_INDEX_NAME_LEN + 1]; + FLMUINT uiIndexNameLen; + FLMUINT uiIndexNum; + char szTmp [50]; + FLMUINT uiTmpLen; + FLMUINT uiLanguage; + FLMUINT64 ui64LastRowIndexed; + FLMUINT uiNumKeyComponents; + FLMUINT uiNumDataComponents; + FLMUINT uiFlags; + FLMBOOL bIsNull; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_INDEXES, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the table. Each row defines an index on + // a table in the database, so we add it to the in-memory dictionary. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, &ui64DefRowId))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the index name - required. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEXES_INDEX_NAME, + szIndexName, sizeof( szIndexName), + &bIsNull, &uiIndexNameLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiIndexNameLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_INDEX_NAME); + goto Exit; + } + + // Get the index number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEXES_INDEX_NUM, + &uiIndexNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_INDEX_NUM); + goto Exit; + } + + // Get the table number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEXES_TABLE_NUM, + &uiTableNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_TABLE_NUM); + goto Exit; + } + + // Get the encryption definition number - optional. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEXES_ENCDEF_NUM, + &uiEncDefNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiEncDefNum = 0; + } + + // Get the number of key components - optional, defaults to zero. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEXES_NUM_KEY_COMPONENTS, + &uiNumKeyComponents, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiNumKeyComponents = 0; + } + + // Get the number of data components - optional, defaults to zero + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEXES_NUM_DATA_COMPONENTS, + &uiNumDataComponents, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiNumDataComponents = 0; + } + + // Get index language - optional. + + uiLanguage = FLM_US_LANG; + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEXES_LANGUAGE, + szTmp, sizeof( szTmp), + &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (uiTmpLen != 2) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_LANGUAGE); + goto Exit; + } + + uiLanguage = f_languageToNum( szTmp); + + // f_languageToNum returns FLM_US_LANG for all strings it doesn't + // recognize. If that is what is returned make sure that the string + // is "US". + + if (uiLanguage == FLM_US_LANG && f_stricmp( szTmp, "US") != 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_LANGUAGE); + goto Exit; + } + } + + // Get the last row indexed, optional - defaults to zero. + + if (RC_BAD( rc = pRow->getUINT64( this, SFLM_COLNUM_INDEXES_LAST_ROW_INDEXED, + &ui64LastRowIndexed, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + ui64LastRowIndexed = 0; + } + + // Get the index state - optional. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEXES_INDEX_STATE, + szTmp, sizeof( szTmp), &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + uiFlags = 0; + if (!bIsNull && uiTmpLen) + { + if (!validIndexState( szTmp, &uiFlags)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_INDEX_STATE); + goto Exit; + } + } + + // Add the index to the the in-memory dictionary. + + if (RC_BAD( rc = m_pDict->addIndex( uiIndexNum, ui64DefRowId, + szIndexName, uiTableNum, uiEncDefNum, + uiFlags, uiNumKeyComponents, + uiNumDataComponents, uiLanguage, + ui64LastRowIndexed))) + { + goto Exit; + } + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Determine if the "index on" value for an index component is valid. +****************************************************************************/ +FSTATIC FLMBOOL validIndexOnValue( + const char * pszIndexOn, + FLMUINT * puiFlags) +{ + if (f_stricmp( pszIndexOn, SFLM_VALUE_OPTION_STR) == 0) + { + (*puiFlags) |= ICD_VALUE; + return( TRUE); + } + else if (f_stricmp( pszIndexOn, SFLM_PRESENCE_OPTION_STR) == 0) + { + (*puiFlags) |= ICD_PRESENCE; + return( TRUE); + } + else if (f_stricmp( pszIndexOn, SFLM_SUBSTRING_OPTION_STR) == 0) + { + (*puiFlags) |= ICD_SUBSTRING; + return( TRUE); + } + else if (f_stricmp( pszIndexOn, SFLM_EACHWORD_OPTION_STR) == 0) + { + (*puiFlags) |= ICD_EACHWORD; + return( TRUE); + } + else if (f_stricmp( pszIndexOn, SFLM_METAPHONE_OPTION_STR) == 0) + { + (*puiFlags) |= ICD_METAPHONE; + return( TRUE); + } + return( FALSE); +} + +/**************************************************************************** +Desc: Determine if the "compare rules" value for an index component is valid. + NOTE: pszCompareRules will be modified, but it doesn't matter because + the caller should have passed a temporary buffer. +****************************************************************************/ +FSTATIC FLMBOOL validCompareRulesValue( + char * pszCompareRules, + FLMUINT * puiCompareRules) +{ + char * pszRuleStart; + char * pszRuleEnd; + + pszRuleStart = pszCompareRules; + while (*pszRuleStart) + { + // Skip leading spaces, tabs, newlines, and commas. + + while (*pszRuleStart == ' ' || *pszRuleStart == ',' || + *pszRuleStart == '\t' || *pszRuleStart == '\n') + { + pszRuleStart++; + } + + // If we are at the end, there are no more rules to look at. + + if (*pszRuleStart == 0) + { + break; + } + + // Go until we hit a comma or whitespace. + + pszRuleEnd = pszRuleStart; + while (*pszRuleEnd && *pszRuleEnd != ' ' && *pszRuleEnd != ',' && + *pszRuleEnd != '\t' && *pszRuleEnd != '\n') + { + pszRuleEnd++; + } + if (*pszRuleEnd) + { + *pszRuleEnd = 0; + pszRuleEnd++; + } + + // See if the rule is valid. + + if (f_stricmp( SFLM_CASE_INSENSITIVE_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_CASE_INSENSITIVE; + } + else if (f_stricmp( SFLM_MINSPACES_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_COMPRESS_WHITESPACE; + } + else if (f_stricmp( SFLM_WHITESPACE_AS_SPACE_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_WHITESPACE_AS_SPACE; + } + else if (f_stricmp( SFLM_IGNORE_LEADINGSPACES_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_IGNORE_LEADING_SPACE; + } + else if (f_stricmp( SFLM_IGNORE_TRAILINGSPACES_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_IGNORE_TRAILING_SPACE; + } + else if (f_stricmp( SFLM_NOUNDERSCORE_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_NO_UNDERSCORES; + } + else if (f_stricmp( SFLM_NOSPACE_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_NO_WHITESPACE; + } + else if (f_stricmp( SFLM_NODASH_OPTION_STR, pszRuleStart) == 0) + { + (*puiCompareRules) |= FLM_COMP_NO_DASHES; + } + else + { + return( FALSE); + } + } + return( TRUE); +} + +/**************************************************************************** +Desc: Read in all rows from the index component table. Create the in-memory + structure for each index component. +****************************************************************************/ +RCODE F_Db::dictReadIndexComponents( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow = NULL; + FLMUINT64 ui64DefRowId; + FSTableCursor * pTableCursor = NULL; + FLMUINT uiColumnNum; + FLMUINT uiIndexNum; + char szTmp [200]; + FLMUINT uiTmpLen; + FLMBOOL bTmp; + FLMUINT uiKeyComponent; + FLMUINT uiDataComponent; + FLMUINT uiFlags; + FLMUINT uiCompareRules; + FLMUINT uiLimit; + FLMBOOL bIsNull; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_INDEX_COMPONENTS, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the table. Each row defines an index component + // for an index in the database, so we add it to the in-memory dictionary. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, &ui64DefRowId))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the index number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEX_COMP_INDEX_NUM, + &uiIndexNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_INDEX_NUM); + goto Exit; + } + + // Get the column number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEX_COMP_COLUMN_NUM, + &uiColumnNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_COLUMN_NUM); + goto Exit; + } + + // Get the key component number - optional, defaults to zero. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEX_COMP_KEY_COMPONENT, + &uiKeyComponent, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiKeyComponent = 0; + } + + // Get the data component number - optional, defaults to zero. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEX_COMP_DATA_COMPONENT, + &uiDataComponent, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiDataComponent = 0; + } + + // We must have at least one of the components non-zero. + + if (!uiKeyComponent && !uiDataComponent) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NO_COMP_NUM_FOR_INDEX_COLUMN); + goto Exit; + } + + uiFlags = 0; + uiCompareRules = 0; + uiLimit = 0; + + // For data components we just ignore the index-on, compare-rules, + // sort-descending, sort-missing-high, and limit columns. + + if (uiKeyComponent) + { + + // Get what we are indexing on - optional, defaults to value. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEX_COMP_INDEX_ON, + szTmp, sizeof( szTmp), + &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (bIsNull || !uiTmpLen) + { + uiFlags |= ICD_VALUE; + } + else + { + if (!validIndexOnValue( szTmp, &uiFlags)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_INDEX_ON_VALUE); + goto Exit; + } + } + + // Get compare rules - optional, defaults to none. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEX_COMP_COMPARE_RULES, + szTmp, sizeof( szTmp), &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (!validCompareRulesValue( szTmp, &uiCompareRules)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_COMPARE_RULES_VALUE); + goto Exit; + } + } + + // Get the sort-descending flag - optional, defaults to sort ascending. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEX_COMP_SORT_DESCENDING, + szTmp, sizeof( szTmp), &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (!validBooleanValue( szTmp, &bTmp)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_SORT_DESCENDING_VALUE); + goto Exit; + } + if (bTmp) + { + uiFlags |= ICD_DESCENDING; + } + } + + // Get the sort-missing-high flag - optional, defaults to sort missing low. + + if (RC_BAD( rc = pRow->getUTF8( this, SFLM_COLNUM_INDEX_COMP_SORT_MISSING_HIGH, + szTmp, sizeof( szTmp), &bIsNull, &uiTmpLen, NULL))) + { + goto Exit; + } + if (!bIsNull && uiTmpLen) + { + if (!validBooleanValue( szTmp, &bTmp)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_SORT_MISSING_HIGH_VALUE); + goto Exit; + } + if (bTmp) + { + uiFlags |= ICD_MISSING_HIGH; + } + } + + // Get the limit - defaults to different things depending on + // what we are indexing. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_INDEX_COMP_LIMIT, + &uiLimit, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + uiLimit = (FLMUINT)((uiFlags & ICD_SUBSTRING) + ? (FLMUINT)ICD_DEFAULT_SUBSTRING_LIMIT + : (FLMUINT)ICD_DEFAULT_LIMIT); + } + } + + // Add the index component to the the in-memory dictionary. + + if (RC_BAD( rc = m_pDict->addIndexComponent( uiIndexNum, ui64DefRowId, + uiColumnNum, + uiFlags, uiCompareRules, uiLimit, + uiKeyComponent, uiDataComponent))) + { + goto Exit; + } + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Open a dictionary by reading in all of the dictionary tables + from the dictionaries. +****************************************************************************/ +RCODE F_Db::dictOpen( void) +{ + RCODE rc = NE_SFLM_OK; + + // At this point, better not be pointing to a dictionary. + + flmAssert( !m_pDict); + + // Should never get here for a temporary database. + + flmAssert( !m_pDatabase->m_bTempDb); + + // Allocate a new F_Dict object for reading the dictionary + // into memory. + + if ((m_pDict = f_new F_Dict) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Allocate the fixed collections and indexes and set them up + + if (RC_BAD( rc = m_pDict->setupPredefined())) + { + goto Exit; + } + + // Read in the LFH's for the predefined stuff. + + if (RC_BAD( rc = dictReadLFH())) + { + goto Exit; + } + + // If dictionary tables are not yet set up, do nothing. + + if (m_pDict->m_pTableTbl [SFLM_TBLNUM_TABLES - 1].lfInfo.uiBlkAddress) + { + + // Read in definitions in the following order: + // 1) encryption definitions + // 2) table definitions + // 3) column definitions + // 4) index definitions + // 5) index components + // This guarantees that things will be defined by the + // time they are referenced. + + if (RC_BAD( rc = dictReadEncDefs())) + { + goto Exit; + } + if (RC_BAD( rc = dictReadTables())) + { + goto Exit; + } + if (RC_BAD( rc = dictReadColumns())) + { + goto Exit; + } + if (RC_BAD( rc = dictReadIndexes())) + { + goto Exit; + } + if (RC_BAD( rc = dictReadIndexComponents())) + { + goto Exit; + } + + // Must read LFHs to get the LFILE information for the + // tables and indexes we have just added. + + if (RC_BAD( rc = dictReadLFH())) + { + goto Exit; + } + } + + // Verify the dictionary after it is all read into memory. + + if (RC_BAD( rc = m_pDict->verifyDict())) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc) && m_pDict) + { + m_pDict->Release(); + m_pDict = NULL; + } + + return( rc); +} + +/**************************************************************************** +Desc: Creates a new dictionary for a database. + This occurs on database create and on a dictionary change. +****************************************************************************/ +RCODE F_Db::createNewDict( void) +{ + RCODE rc = NE_SFLM_OK; + + // Unlink the DB from the current F_Dict object, if any. + + if (m_pDict) + { + m_pDatabase->lockMutex(); + unlinkFromDict(); + m_pDatabase->unlockMutex(); + } + + // Allocate a new F_Dict object for the new dictionary we + // are going to create. + + if (RC_BAD( rc = dictOpen())) + { + goto Exit; + } + + // Update the F_Db flags to indicate that the dictionary + // was updated. + + m_uiFlags |= FDB_UPDATED_DICTIONARY; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Create the tables and indexes needed for storing dictionary + definitions. +****************************************************************************/ +RCODE F_Db::dictCreate( void) +{ + RCODE rc = NE_SFLM_OK; + LFILE TempLFile; + + // This should never be called for a temporary database. + + flmAssert( !m_pDatabase->m_bTempDb); + + // Create the dictionary tables + + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_TBLNUM_ENCDEFS, SFLM_LF_TABLE, FALSE, TRUE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_TBLNUM_TABLES, SFLM_LF_TABLE, FALSE, TRUE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_TBLNUM_COLUMNS, SFLM_LF_TABLE, FALSE, TRUE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_TBLNUM_INDEXES, SFLM_LF_TABLE, FALSE, TRUE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_TBLNUM_INDEX_COMPONENTS, SFLM_LF_TABLE, FALSE, TRUE, 0))) + { + goto Exit; + } + + // Create the dictionary indexes + + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_ENCDEF_NAME, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_TABLE_NAME, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_TABLE_ENCDEF_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_COLUMN_TABLE_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_COLUMN_ENCDEF_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_INDEX_NAME, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_INDEX_TABLE_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_INDEX_ENCDEF_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + if (RC_BAD(rc = m_pDatabase->lFileCreate( this, &TempLFile, + SFLM_IXNUM_INDEX_COMP_INDEX_NUM, SFLM_LF_INDEX, FALSE, FALSE, 0))) + { + goto Exit; + } + +//visit - create maintenance tables and indexes + + // Create a new dictionary we can work with. + + if (RC_BAD( rc = createNewDict())) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Lookup a name in the name table +****************************************************************************/ +NAME_INFO * F_NameTable::findName( + const char * pszName, + FLMUINT * puiInsertPos) +{ + NAME_INFO * pNameInfo = NULL; + const char * pszTblName; + FLMUINT uiTblSize; + FLMUINT uiLow; + FLMUINT uiMid; + FLMUINT uiHigh; + FLMINT iCmp; + + if (!m_bTableSorted) + { + sortNames(); + } + + // Do binary search in the table + + if ((uiTblSize = m_uiNumNames) == 0) + { + if (puiInsertPos) + { + *puiInsertPos = 0; + } + goto Exit; + } + + uiHigh = --uiTblSize; + uiLow = 0; + for (;;) + { + uiMid = (uiLow + uiHigh) / 2; + + pszTblName = m_pNames [uiMid].pszName; + if ((iCmp = (FLMINT)f_strcmp( pszName, pszTblName)) == 0) + { + + // Found Match + + pNameInfo = &m_pNames [uiMid]; + if (puiInsertPos) + { + *puiInsertPos = uiMid; + } + goto Exit; + } + + // Check if we are done + + if (uiLow >= uiHigh) + { + + // Done, item not found + + if (puiInsertPos) + { + *puiInsertPos = (iCmp < 0) + ? uiMid + : uiMid + 1; + } + goto Exit; + } + + if (iCmp < 0) + { + if (uiMid == 0) + { + if (puiInsertPos) + { + *puiInsertPos = 0; + } + goto Exit; + } + uiHigh = uiMid - 1; + } + else + { + if (uiMid == uiTblSize) + { + if (puiInsertPos) + { + *puiInsertPos = uiMid + 1; + } + goto Exit; + } + uiLow = uiMid + 1; + } + } + +Exit: + + return( pNameInfo); +} + +/*************************************************************************** +Desc: Swap two entries in tag info table during sort. +*****************************************************************************/ +FINLINE void nameInfoSwap( + NAME_INFO * pNameInfoTbl, + FLMUINT uiPos1, + FLMUINT uiPos2) +{ + NAME_INFO tmpNameInfo; + + tmpNameInfo.pszName = pNameInfoTbl [uiPos1].pszName; + tmpNameInfo.uiItemNum = pNameInfoTbl [uiPos1].uiItemNum; + pNameInfoTbl [uiPos1].pszName = pNameInfoTbl [uiPos2].pszName; + pNameInfoTbl [uiPos1].uiItemNum = pNameInfoTbl [uiPos2].uiItemNum; + pNameInfoTbl [uiPos2].pszName = tmpNameInfo.pszName; + pNameInfoTbl [uiPos2].uiItemNum = tmpNameInfo.uiItemNum; +} + +/*************************************************************************** +Desc: Comparison function for sorting name table +****************************************************************************/ +FINLINE FLMINT compareNameInfo( + NAME_INFO * pNameInfo1, + NAME_INFO * pNameInfo2, + FLMBOOL * pbDuplicateNames) +{ + FLMINT iCmp = (FLMINT)f_strcmp( pNameInfo1->pszName, pNameInfo2->pszName); + + if (iCmp == 0) + { + *pbDuplicateNames = TRUE; + return( 0); + } + return( (iCmp < 0) ? -1 : 1); +} + +/*************************************************************************** +Desc: Sort a name table. +****************************************************************************/ +FSTATIC void sortNameTbl( + NAME_INFO * pNameInfoTbl, + FLMUINT uiLowerBounds, + FLMUINT uiUpperBounds, + FLMBOOL * pbDuplicateNames) +{ + FLMUINT uiLBPos; + FLMUINT uiUBPos; + FLMUINT uiMIDPos; + FLMUINT uiLeftItems; + FLMUINT uiRightItems; + NAME_INFO * pCurNameInfo; + FLMINT iCompare; + +Iterate_Larger_Half: + + uiUBPos = uiUpperBounds; + uiLBPos = uiLowerBounds; + uiMIDPos = (uiUpperBounds + uiLowerBounds + 1) / 2; + pCurNameInfo = &pNameInfoTbl [uiMIDPos]; + for (;;) + { + while (uiLBPos == uiMIDPos || // Don't compare with target + ((iCompare = + compareNameInfo( &pNameInfoTbl [uiLBPos], pCurNameInfo, + pbDuplicateNames)) < 0)) + { + if (uiLBPos >= uiUpperBounds) + { + break; + } + uiLBPos++; + } + + while (uiUBPos == uiMIDPos || // Don't compare with target + (((iCompare = + compareNameInfo( pCurNameInfo, &pNameInfoTbl [uiUBPos], + pbDuplicateNames)) < 0))) + { + if (!uiUBPos) + { + break; + } + uiUBPos--; + } + + if (uiLBPos < uiUBPos ) // Interchange and continue loop. + { + + // Exchange [uiLBPos] with [uiUBPos]. + + nameInfoSwap( pNameInfoTbl, uiLBPos, uiUBPos); + uiLBPos++; // Scan from left to right. + uiUBPos--; // Scan from right to left. + } + else // Past each other - done + { + break; + } + } + + // Check for swap( LB, MID ) - cases 3 and 4 + + if( uiLBPos < uiMIDPos ) + { + + // Exchange [uiLBPos] with [uiMIDPos] + + nameInfoSwap( pNameInfoTbl, uiMIDPos, uiLBPos); + uiMIDPos = uiLBPos; + } + else if( uiMIDPos < uiUBPos ) + { + + // Exchange [uUBPos] with [uiMIDPos] + + nameInfoSwap( pNameInfoTbl, uiMIDPos, uiUBPos); + uiMIDPos = uiUBPos; + } + + // Check the left piece. + + uiLeftItems = (uiLowerBounds + 1 < uiMIDPos ) + ? uiMIDPos - uiLowerBounds // 2 or more + : 0; + uiRightItems = (uiMIDPos + 1 < uiUpperBounds ) + ? uiUpperBounds - uiMIDPos // 2 or more + : 0; + + if( uiLeftItems < uiRightItems ) + { + + // Recurse on the LEFT side and goto the top on the RIGHT side. + + if (uiLeftItems ) + { + sortNameTbl( pNameInfoTbl, uiLowerBounds, uiMIDPos - 1, pbDuplicateNames); + } + uiLowerBounds = uiMIDPos + 1; + goto Iterate_Larger_Half; + } + else if (uiLeftItems ) // Compute a truth table to figure out this check. + { + + // Recurse on the RIGHT side and goto the top for the LEFT side. + + if (uiRightItems ) + { + sortNameTbl( pNameInfoTbl, uiMIDPos + 1, uiUpperBounds, pbDuplicateNames); + } + uiUpperBounds = uiMIDPos - 1; + goto Iterate_Larger_Half; + } +} + +/**************************************************************************** +Desc: Add a name to the table. +****************************************************************************/ +RCODE F_NameTable::addName( + const char * pszName, + FLMUINT uiItemNum, + FLMBOOL bCheckDuplicates, + RCODE rcDuplicateError, + FLMUINT uiTableGrowSize) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiInsertPos; + + if (!pszName || !uiItemNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + if (bCheckDuplicates) + { + + // Make sure that the name is not already used. + + if (findName( pszName, &uiInsertPos)) + { + rc = RC_SET( rcDuplicateError); + goto Exit; + } + } + else + { + uiInsertPos = m_uiNumNames; + m_bTableSorted = FALSE; + } + + // See if we need to grow the table. + + if (m_uiNumNames == m_uiTblSize) + { + FLMUINT uiNewSize = m_uiTblSize + uiTableGrowSize; + + if (RC_BAD( rc = f_realloc( sizeof( NAME_INFO) * uiNewSize, + &m_pNames))) + { + goto Exit; + } + m_uiTblSize = uiNewSize; + } + + // If necessary, move names up in the table to make room for the + // new name. + + if (uiInsertPos < m_uiNumNames) + { + f_memmove( &m_pNames [uiInsertPos + 1], + &m_pNames [uiInsertPos], + sizeof( NAME_INFO) * (m_uiNumNames - uiInsertPos)); + } + m_pNames [uiInsertPos].pszName = pszName; + m_pNames [uiInsertPos].uiItemNum = uiItemNum; + m_uiNumNames++; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Copy a name into the name table. +****************************************************************************/ +RCODE F_NameTable::copyName( + const char * pszName, + FLMUINT uiItemNum, + const char ** ppszDestName, + F_Pool * pPool) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNameLen; + char * pszDestName; + + uiNameLen = f_strlen( pszName) + 1; + if (RC_BAD( rc = pPool->poolAlloc( uiNameLen, (void **)&pszDestName))) + { + goto Exit; + } + f_memcpy( pszDestName, pszName, uiNameLen); + *ppszDestName = pszDestName; + + if (RC_BAD( rc = addName( pszDestName, uiItemNum, + FALSE, NE_SFLM_OK, 0))) + { + goto Exit; + } +Exit: + return( rc); +} + +/**************************************************************************** +Desc: Sort the name table. +****************************************************************************/ +void F_NameTable::sortNames( void) +{ + if (!m_bTableSorted) + { + m_bDuplicateNames = FALSE; + if (m_uiNumNames > 1) + { + sortNameTbl( m_pNames, 0, m_uiNumNames - 1, &m_bDuplicateNames); + } + m_bTableSorted = TRUE; + } +} + +/**************************************************************************** +Desc: Remove a name from the table +****************************************************************************/ +void F_NameTable::removeName( + const char * pszName) +{ + FLMUINT uiPos; + + if (findName( pszName, &uiPos) != NULL) + { + if (uiPos < m_uiNumNames - 1) + { + f_memmove( &m_pNames [uiPos], &m_pNames [uiPos + 1], + sizeof( NAME_INFO) * (m_uiNumNames - uiPos - 1)); + } + m_uiNumNames--; + } +} + diff --git a/sql/src/fdict.h b/sql/src/fdict.h new file mode 100644 index 0000000..9391466 --- /dev/null +++ b/sql/src/fdict.h @@ -0,0 +1,509 @@ +//------------------------------------------------------------------------------ +// Desc: F_Dict class definitions - internal object for database's +// dictionary. +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdict.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FDICT_H +#define FDICT_H + +struct ICD; +struct F_INDEX; +class F_Dict; +class F_NameTable; +class F_Database; +class F_CCS; + +/**************************************************************************** +Desc: Encryption definition +****************************************************************************/ +typedef struct F_ENCDEF +{ + FLMUINT uiEncDefNum; // Encryption definition number. This is + // also the place in the encryption definition + // table minus one. + FLMUINT64 ui64DefRowId; // Definition row ID. + const char * pszEncDefName; // Encryption definition name. + eEncAlgorithm eEncAlg; // AES or DES3 + FLMUINT uiEncKeySize; // Key size + F_CCS * pCcs; // Encryption object +} F_ENCDEF; + +/**************************************************************************** +Desc: Table column definition. +****************************************************************************/ +typedef struct F_COLUMN +{ + FLMUINT uiColumnNum; // Column number. This is also the place in + // the table's column table minus one. + FLMUINT64 ui64DefRowId; // Definition row ID. + const char * pszColumnName; // Column name. + FLMUINT uiFlags; // Column flags +#define COL_READ_ONLY 0x0001 +#define COL_NULL_ALLOWED 0x0002 + eDataType eDataTyp; // Column data type + FLMUINT uiEncDefNum; // If column is encrypted, this is the + // encryption definition number. Zero if + // column is not encrypted. + ICD * pFirstIcd; // First index component where this column + // is a key component. + ICD * pFirstDataIcd; // First index component where this column + // is a data component. +} F_COLUMN; + +/**************************************************************************** +Desc: Logical File +****************************************************************************/ +typedef struct LFILE +{ + FLMUINT uiLfNum; // Index number or table number. + FLMUINT uiEncDefNum; // If index or table is encrypted, this is + // the table's encryption definition number. + // Zero if the index or table is not encrypted. + eLFileType eLfType; // Type of logical file + FLMUINT uiRootBlk; // Address of root block. + FLMUINT uiBlkAddress; // Block address of LFile entry. + FLMUINT uiOffsetInBlk; // Offset within block of entry. + FLMUINT64 ui64NextRowId; // Next row ID in table, if eLfType is a table + // Not used for indexes. + FLMBOOL bNeedToWriteOut; // If TRUE, this LFILE needs to be written + // to disk. NOTE: This is only used for + // tables - so that we can update ui64NextRowId + // many times in memory, but only write it out + // at commit time. +} LFILE; + +/***************************************************************************** +Desc: Table definition +*****************************************************************************/ +typedef struct F_TABLE +{ + FLMUINT uiTableNum; // Table number. This is also this entry's place + // in the table table, minus one. + FLMUINT64 ui64DefRowId; // Definition row ID. + const char * pszTableName; // Table name. + FLMBOOL bSystemTable; // System table - cannot be dropped or altered by + // an application. + FLMUINT uiNumColumns; // Number of columns in the table. + F_COLUMN * pColumns; // Contains all columns for the table. + F_NameTable * pColumnNames; // Contains all column names for the table + LFILE lfInfo; // Logical file information. + FLMUINT uiFirstIndexNum; // Number of first index on this table +} F_TABLE; + +/***************************************************************************** +Desc: Index definition +*****************************************************************************/ +typedef struct F_INDEX +{ + FLMUINT uiIndexNum; // Index number. This also corresponds + // to the position of this entry in + // the index table minus one. + FLMUINT64 ui64DefRowId; // Definition row ID. + FLMUINT uiNextIndexNum; // Next index on this same table. + FLMUINT uiTableNum; // Table this index is on. + const char * pszIndexName; // Index name. + ICD * pKeyIcds; // Array of key components for the index. + FLMUINT uiNumKeyComponents; // Number of key components. + ICD * pDataIcds; // Array of data components for the index. + FLMUINT uiNumDataComponents; // Number of data components. + FLMUINT uiFlags; // Index flags. + #define IXD_ABS_POS 0x00001 // Maintain absolute positioning info. + #define IXD_HAS_SUBSTRING 0x00002 // At least one key component is substring. + #define IXD_OFFLINE 0x00004 // Index is offline - may or may + // not be suspended. + #define IXD_SUSPENDED 0x00008 // IXD_OFFLINE should also be set + #define IXD_SYSTEM 0x00010 // Index is an internal system index + FLMUINT uiLanguage; // Language for the index. + LFILE lfInfo; // Logical file information. + FLMUINT64 ui64LastRowIndexed; // Last row indexed, if indexing in + // the background. +} F_INDEX; + +/***************************************************************************** +Desc: Index component +*****************************************************************************/ +typedef struct ICD +{ + FLMUINT uiIndexNum; // Index this component belongs to + FLMUINT uiColumnNum; // Column in table this component refers to + FLMUINT64 ui64DefRowId; // Definition row ID. + FLMUINT uiFlags; // Flags for component +#define ICD_VALUE 0x00000010 // Value agrees with parsing syntax +#define ICD_EACHWORD 0x00000020 // Index each and every word in the field +#define ICD_PRESENCE 0x00000040 // Index the tag and NOT the value +#define ICD_METAPHONE 0x00000080 // Index words of text strings using + // metaphone values +#define ICD_SUBSTRING 0x00000100 // Index all substrings pieces +#define ICD_DESCENDING 0x00000200 // Sort in descending order. +#define ICD_MISSING_HIGH 0x00000400 // Sort missing components high instead of low. +#define ICD_ESC_CHAR 0x00001000 // Placehold so that a query can parse the input + // string and find a literal '*' or '\\'. + // Not specified in dictionary def. or kept in ICD. + // Only a temporary flag. + FLMUINT uiCompareRules; // Comparison rules for this component. + FLMUINT uiLimit; // Limit for this component. +#define ICD_DEFAULT_LIMIT 128 +#define ICD_DEFAULT_SUBSTRING_LIMIT 48 + ICD * pNextInChain; + ICD * pNextInDataChain; +} ICD; + +/************************************************************************** +Desc: Structure kept in name table for sorting names. +**************************************************************************/ +typedef struct NAME_INFO +{ + const char * pszName; + FLMUINT uiItemNum; +}; + +/************************************************************************** +Desc: This class is the name table class. +**************************************************************************/ +class F_NameTable : public F_Object +{ +public: + + // Constructor and destructor + + F_NameTable() + { + m_pNames = NULL; + m_uiTblSize = 0; + m_uiNumNames = 0; + m_bTableSorted = FALSE; + m_bDuplicateNames = FALSE; + } + + ~F_NameTable() + { + f_free( &m_pNames); + } + + NAME_INFO * findName( + const char * pszName, + FLMUINT * puiInsertPos); + + RCODE addName( + const char * pszName, + FLMUINT uiItemNum, + FLMBOOL bCheckDuplicates, + RCODE rcDuplicateError, + FLMUINT uiTableGrowSize); + + RCODE copyName( + const char * pszName, + FLMUINT uiItemNum, + const char ** ppszDestName, + F_Pool * pPool); + + void sortNames( void); + + void removeName( + const char * pszName); + +private: + + NAME_INFO * m_pNames; + FLMUINT m_uiTblSize; + FLMUINT m_uiNumNames; + FLMBOOL m_bTableSorted; + FLMBOOL m_bDuplicateNames; + +friend class F_Db; +friend class F_Dict; +}; + +/************************************************************************** +Desc: This class is the FLAIM dictionary class. +**************************************************************************/ +class F_Dict : public F_Object +{ +public: + + // Constructor and destructor + + F_Dict(); + + ~F_Dict(); + + void resetDict( void); + + RCODE getTable( + const char * pszTableName, + F_TABLE ** ppTable, + FLMBOOL bOfflineOk); + + RCODE getIndex( + const char * pszIndexName, + F_INDEX ** ppIndex, + FLMBOOL bOfflineOk); + + FINLINE F_ENCDEF * getEncDef( + FLMUINT uiEncDefNum) + { + return( (F_ENCDEF *)((uiEncDefNum && uiEncDefNum <= m_uiHighestEncDefNum && + m_pEncDefTbl [uiEncDefNum - 1].uiEncDefNum) + ? &m_pEncDefTbl [uiEncDefNum - 1] + : (F_ENCDEF *)NULL)); + } + + FINLINE F_TABLE * getTable( + FLMUINT uiTableNum) + { + return( (F_TABLE *)((uiTableNum && uiTableNum <= m_uiHighestTableNum && + m_pTableTbl [uiTableNum - 1].uiTableNum) + ? &m_pTableTbl [uiTableNum - 1] + : (F_TABLE *)NULL)); + } + + FINLINE F_INDEX * getIndex( + FLMUINT uiIndexNum) + { + return( (F_INDEX *)((uiIndexNum && uiIndexNum <= m_uiHighestIndexNum && + m_pIndexTbl [uiIndexNum - 1].uiIndexNum) + ? &m_pIndexTbl [uiIndexNum - 1] + : (F_INDEX *)NULL)); + } + + FINLINE F_COLUMN * getColumn( + F_TABLE * pTable, + FLMUINT uiColumnNum) + { + return( (F_COLUMN *)((uiColumnNum && uiColumnNum <= pTable->uiNumColumns) + ? &pTable->pColumns [uiColumnNum - 1] + : (F_COLUMN *)NULL)); + } + + FINLINE F_TABLE * findTable( + const char * pszTableName) + { + NAME_INFO * pNameInfo; + + if (m_pTableNames) + { + if ((pNameInfo = m_pTableNames->findName( pszTableName, NULL)) != NULL) + { + return( getTable( pNameInfo->uiItemNum)); + } + } + return( NULL); + } + + FINLINE F_INDEX * findIndex( + const char * pszIndexName) + { + NAME_INFO * pNameInfo; + + if (m_pIndexNames) + { + if ((pNameInfo = m_pIndexNames->findName( pszIndexName, NULL)) != NULL) + { + return( getIndex( pNameInfo->uiItemNum)); + } + } + return( NULL); + } + + FINLINE F_ENCDEF * findEncDef( + const char * pszEncDefName) + { + NAME_INFO * pNameInfo; + + if (m_pEncDefNames) + { + if ((pNameInfo = m_pEncDefNames->findName( pszEncDefName, NULL)) != NULL) + { + return( getEncDef( pNameInfo->uiItemNum)); + } + } + return( NULL); + } + + void linkToDatabase( + F_Database * pDatabase); + + void unlinkFromDatabase( void); + + FINLINE FLMUINT getUseCount( void) + { + return( m_uiUseCount); + } + + FINLINE FLMUINT decrUseCount( void) + { + return( --m_uiUseCount); + } + + FINLINE void incrUseCount( void) + { + m_uiUseCount++; + } + + FINLINE F_Dict * getPrev( void) + { + return( m_pPrev); + } + + FINLINE F_Dict * getNext( void) + { + return( m_pNext); + } + + FINLINE F_Database * getDatabase( void) + { + return( m_pDatabase); + } + + RCODE copyEncDef( + F_ENCDEF * pDestEncDef, + F_ENCDEF * pSrcEncDef); + + RCODE copyColumn( + F_NameTable * pDestColumnNameTable, + F_COLUMN * pDestColumn, + F_COLUMN * pSrcColumn); + + RCODE copyTable( + F_TABLE * pDestTable, + F_TABLE * pSrcTable); + + RCODE copyIndex( + F_INDEX * pDestIndex, + F_INDEX * pSrcIndex); + + RCODE cloneDict( + F_Dict * pSrcDict); + + RCODE addEncDef( + FLMUINT uiEncDefNum, + FLMUINT64 ui64DefRowId, + const char * pszEncDefName, + eEncAlgorithm eEncAlg, + FLMUINT uiEncKeySize, + FLMBYTE * pucEncKey, + FLMUINT uiEncKeyLen); + + RCODE addTable( + FLMUINT uiTableNum, + FLMUINT64 ui64DefRowId, + const char * pszTableName, + FLMBOOL bSystemTable, + FLMUINT uiNumColumns, + FLMUINT uiEncDefNum); + + RCODE addColumn( + FLMUINT uiTableNum, + FLMUINT64 ui64DefRowId, + FLMUINT uiColumnNum, + const char * pszColumnName, + FLMUINT uiFlags, + eDataType eDataTyp, + FLMUINT uiEncDefNum); + + RCODE addIndex( + FLMUINT uiIndexNum, + FLMUINT64 ui64DefRowId, + const char * pszIndexName, + FLMUINT uiTableNum, + FLMUINT uiEncDefNum, + FLMUINT uiFlags, + FLMUINT uiNumKeyComponents, + FLMUINT uiNumDataComponents, + FLMUINT uiLanguage, + FLMUINT64 ui64LastRowIndexed); + + RCODE addIndexComponent( + FLMUINT uiIndexNum, + FLMUINT64 ui64DefRowId, + FLMUINT uiColumnNum, + FLMUINT uiFlags, + FLMUINT uiCompareRules, + FLMUINT uiLimit, + FLMUINT uiKeyComponent, + FLMUINT uiDataComponent); + + RCODE setupEncDefTable( void); + + RCODE setupTableTable( void); + + RCODE setupColumnTable( void); + + RCODE setupIndexTable( void); + + RCODE setupIndexComponentTable( void); + + RCODE setupBlockChainTable( void); + + RCODE setupPredefined( void); + + RCODE verifyDict( void); + +private: + + F_Dict * m_pNext; // Pointer to next F_Dict object in the list, + // if any. All versions of a dictionary that + // are currently in use are linked together. + // Usually, there will be only one local + // dictionary in the list. + F_Dict * m_pPrev; // Previous F_Dict object in the list. + F_Database * m_pDatabase; // database this dictionary is associated with. + // A null value means it is not yet linked + // to a database. + FLMUINT m_uiDictSeq; // This is the sequence number of the + // dictionary + F_Pool m_dictPool; // Pool for all allocations except tables. + + // Encryption definition table + + F_ENCDEF * m_pEncDefTbl; + FLMUINT m_uiEncDefTblSize; + FLMUINT m_uiHighestEncDefNum; + F_NameTable * m_pEncDefNames; + + // Table table + + F_TABLE * m_pTableTbl; + FLMUINT m_uiTableTblSize; + FLMUINT m_uiHighestTableNum; + F_NameTable * m_pTableNames; + + // Index table + + F_INDEX * m_pIndexTbl; + FLMUINT m_uiIndexTblSize; + FLMUINT m_uiHighestIndexNum; + F_NameTable * m_pIndexNames; + + FLMUINT m_uiUseCount; // Number of F_Db structures currently + // pointing to this dictionary. + +friend class F_Database; +friend class F_Db; +friend class F_Query; +friend class F_DbCheck; +friend class F_BTreeInfo; +}; + +#endif // #ifndef FDICT_H diff --git a/sql/src/fdllmain.cpp b/sql/src/fdllmain.cpp new file mode 100644 index 0000000..baa28c7 --- /dev/null +++ b/sql/src/fdllmain.cpp @@ -0,0 +1,176 @@ +//------------------------------------------------------------------------------ +// Desc: This is the standard functionality that all com servers must export +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdllmain.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +static F_DbSystem * gv_pDbSystem = NULL; +static FLMATOMIC gv_lockCount = 0; + +FLMEXTC RCODE DllCanUnloadNow( void); +FLMEXTC RCODE DllStart( void); +FLMEXTC RCODE DllStop( void); + +#if defined( FLM_UNIX) + +#elif defined( FLM_WIN) + + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + + #pragma comment(linker, "/export:DllCanUnloadNow=_DllCanUnloadNow@0,PRIVATE") + #pragma comment(linker, "/export:DllStart=_DllStart@0,PRIVATE") + #pragma comment(linker, "/export:DllStop=_DllStop@0,PRIVATE") + +#elif !defined( FLM_NLM) + #error platform not supported. +#endif + +/****************************************************************************** +Desc: +******************************************************************************/ +void LockModule(void) +{ + f_atomicInc( &gv_lockCount); +} + +/****************************************************************************** +Desc: +******************************************************************************/ +void UnlockModule(void) +{ + f_atomicDec( &gv_lockCount); +} + +/****************************************************************************** +Desc: Returns 0 if it's okay to unload, or a non-zero status + code if not. +******************************************************************************/ +FLMEXTC RCODE DllCanUnloadNow( void) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( gv_pDbSystem); + + if( gv_lockCount > 1) + { + rc = RC_SET( NE_SFLM_FAILURE); + } + else + { + flmAssert( gv_lockCount == 1); + + f_mutexLock( gv_SFlmSysData.hShareMutex); + + if (gv_SFlmSysData.pDatabaseHashTbl) + { + FBUCKET * pDatabaseHashTbl; + FLMUINT uiCnt; + + for (uiCnt = 0, pDatabaseHashTbl = gv_SFlmSysData.pDatabaseHashTbl; + uiCnt < FILE_HASH_ENTRIES; + uiCnt++, pDatabaseHashTbl++) + { + if (pDatabaseHashTbl->pFirstInBucket != NULL) + { + rc = RC_SET( NE_SFLM_FAILURE); + break; + } + } + } + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + return( rc); +} + +/****************************************************************************** +Desc: Called by PSA when it loads the library. Must return 0 for + success, or a non-zero error code. +******************************************************************************/ +FLMEXTC RCODE DllStart( void) +{ + RCODE rc = NE_SFLM_OK; + + if( (gv_pDbSystem = f_new F_DbSystem) == NULL) + { + rc = NE_SFLM_MEM; + goto Exit; + } + + if( RC_BAD( rc = gv_pDbSystem->init())) + { + goto Exit; + } + +Exit: + + if( RC_BAD( rc)) + { + if( gv_pDbSystem) + { + gv_pDbSystem->Release(); + gv_pDbSystem = NULL; + } + } + + return( rc); +} + +/****************************************************************************** +Desc: Called by PSA when it unloads the library. The return value + is ignored. +******************************************************************************/ +FLMEXTC RCODE DllStop( void) +{ + if( gv_pDbSystem) + { + flmAssert( gv_lockCount == 1); + + gv_pDbSystem->exit(); + gv_pDbSystem->Release(); + gv_pDbSystem = NULL; + } + + return( NE_SFLM_OK); +} + +/****************************************************************************** +Desc: +******************************************************************************/ +FLMEXTC RCODE DllRegisterServer( + const char *) +{ + return( NE_SFLM_OK); +} + +/****************************************************************************** +Desc: +******************************************************************************/ +FLMEXTC RCODE DllUnregisterServer( void) +{ + return( NE_SFLM_OK); +} diff --git a/sql/src/fdynbtre.cpp b/sql/src/fdynbtre.cpp new file mode 100644 index 0000000..6539f1c --- /dev/null +++ b/sql/src/fdynbtre.cpp @@ -0,0 +1,1198 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the methods for the FFixedBlk, FBtreeRoot, FBtreeLeaf, +// FBtreeNonLeaf, and FBtreeBlk classes. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdynbtre.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" +#include "fdynsset.h" + +// Make sure that the extension is in lower case characters. + +#define FRSET_FILENAME_EXTENSION "frs" + +/**************************************************************************** + +Organization: + + These modules are orgianized by function and NOT by class. + For example, all of the searchEntry modules are together. + All of the split and getFirst/next/last... modules are together. + +****************************************************************************/ + + +/**************************************************************************** + Constructors and Setup Methods +****************************************************************************/ + +/**************************************************************************** +Desc: Set common variables +****************************************************************************/ +FFixedBlk::FFixedBlk() +{ + m_fnCompare = NULL; + m_pvUserData = NULL; + m_uiPosition = DYNSSET_POSITION_NOT_SET; + m_bDirty = FALSE; + m_pucBlkBuf = NULL; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FBtreeRoot::FBtreeRoot() +{ + int i; + + m_pFileHdl = NULL; + m_pszFileName = NULL; + m_eBlkType = ACCESS_BTREE_ROOT; + m_uiEntryOvhd = 4; + m_uiLRUCount = 1; + m_uiLevels = 1; + m_uiNewBlkAddr = 0; + m_uiHighestWrittenBlkAddr = 0; + m_uiTotalEntries = 0; + + // Initialize the cache blocks. + for( i = 0; i < FBTREE_CACHE_BLKS; i++) + { + m_CacheBlks[i].uiBlkAddr = 0xFFFFFFFF; + m_CacheBlks[i].uiLRUValue = 0; + m_CacheBlks[i].pBlk = NULL; + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FBtreeRoot::~FBtreeRoot() +{ + int i; + + closeFile(); + + for( i = 0; i < FBTREE_CACHE_BLKS; i++) + { + if( m_CacheBlks[i].pBlk) + { + m_CacheBlks[i].pBlk->Release(); + } + } +} + +/**************************************************************************** +Desc: Allocate structures and set entry size. +****************************************************************************/ +RCODE FBtreeLeaf::setup( + FLMUINT uiEntrySize) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = f_calloc( DYNSSET_BLOCK_SIZE, &m_pucBlkBuf))) + { + goto Exit; + } + + m_uiEntrySize = uiEntrySize; + m_pvUserData = (void *) uiEntrySize; + reset( ACCESS_BTREE_LEAF); + nextBlk( FBTREE_END); + prevBlk( FBTREE_END); + lemBlk( FBTREE_END); + reset( ACCESS_BTREE_LEAF); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Allocate structures and set entry size. +****************************************************************************/ +RCODE FBtreeNonLeaf::setup( + FLMUINT uiEntrySize) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = f_calloc( DYNSSET_BLOCK_SIZE, &m_pucBlkBuf))) + { + goto Exit; + } + + m_uiEntrySize = uiEntrySize; + m_pvUserData = (void *) uiEntrySize; + reset( ACCESS_BTREE_NON_LEAF); + nextBlk( FBTREE_END); + prevBlk( FBTREE_END); + lemBlk( FBTREE_END); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Allocate structures and set entry size. +****************************************************************************/ +RCODE FBtreeRoot::setup( + FLMUINT uiEntrySize, + char * pszFileName) +{ + RCODE rc; + + if (RC_BAD( rc = f_calloc( DYNSSET_BLOCK_SIZE, &m_pucBlkBuf))) + { + goto Exit; + } + + m_uiEntrySize = uiEntrySize; + m_pvUserData = (void *) uiEntrySize; + reset( ACCESS_BTREE_ROOT); + m_pszFileName = pszFileName; + nextBlk( FBTREE_END); + prevBlk( FBTREE_END); + lemBlk( FBTREE_END); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the block as a new block +****************************************************************************/ +void FBtreeBlk::reset( + FBlkTypes eBlkType) +{ + m_eBlkType = eBlkType; + + if (eBlkType == ACCESS_BTREE_ROOT || eBlkType == ACCESS_BTREE_NON_LEAF) + { + m_uiEntryOvhd = 4; + } + else + { + m_uiEntryOvhd = 0; + } + m_uiNumSlots = (DYNSSET_BLOCK_SIZE - sizeof( FixedBlkHdr)) / + ( m_uiEntrySize + m_uiEntryOvhd); + entryCount( 0); + m_uiPosition = DYNSSET_POSITION_NOT_SET; + m_bDirty = FALSE; +} + +/**************************************************************************** +Desc: Return the next entry in the result set. If the result set + is not positioned then the first entry will be returned. +****************************************************************************/ +RCODE FBtreeBlk::getNext( + void * pvEntryBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPos = m_uiPosition; + + // Position to the next/first entry. + + if (uiPos == DYNSSET_POSITION_NOT_SET) + { + uiPos = 0; + } + else + { + if (++uiPos > entryCount()) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + } + f_memcpy( pvEntryBuffer, ENTRY_POS(uiPos), m_uiEntrySize); + m_uiPosition = uiPos; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Return the last entry in the result set. +****************************************************************************/ +RCODE FBtreeBlk::getLast( + void * pvEntryBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPos = entryCount(); + + // Position to the next/first entry. + + if (uiPos == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + uiPos--; + f_memcpy( pvEntryBuffer, ENTRY_POS(uiPos), m_uiEntrySize); + m_uiPosition = uiPos; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Search a btree. Position for get* or for insert. +****************************************************************************/ +RCODE FBtreeRoot::search( + void * pvEntry, + void * pvFoundEntry) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCurLevel = m_uiLevels - 1; // Min 2 levels + FLMUINT uiBlkAddr; + + // Reset the stack - only needed for debugging. + //f_memset( m_BTStack, 0, sizeof(FBtreeBlk *) * FBTREE_MAX_LEVELS); + + // Search this root block. + + m_BTStack[ uiCurLevel] = this; + (void) searchEntry( pvEntry, &uiBlkAddr); + + while( uiCurLevel--) + { + // Read the next block and place at uiCurLevel (backwards from FS). + + if( RC_BAD( rc = readBlk( uiBlkAddr, + uiCurLevel ? ACCESS_BTREE_NON_LEAF : ACCESS_BTREE_LEAF, + &m_BTStack[ uiCurLevel] ))) + { + goto Exit; + } + + // Set the rc - only for the leaf block, otherwise rc should be ignored. + + rc = m_BTStack[ uiCurLevel]->searchEntry( pvEntry, &uiBlkAddr, + uiCurLevel ? NULL : pvFoundEntry); + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Search a single block tree. Position for get* or for insert. + Do a binary search on all of the entries to find a match. + If no match then position to the entry where an insert + will take place. +****************************************************************************/ +RCODE FBtreeBlk::searchEntry( + void * pvEntry, + FLMUINT * puiChildAddr, + void * pvFoundEntry) +{ + RCODE rc = RC_SET( NE_SFLM_NOT_FOUND); + FLMUINT uiLow; + FLMUINT uiMid; + FLMUINT uiHigh; + FLMUINT uiTblSize; + FLMINT iCompare; + + // check for zero entries. + + if (!entryCount()) + { + uiMid = 0; + goto Exit; + } + uiHigh = uiTblSize = entryCount() - 1; + uiLow = 0; + for(;;) + { + uiMid = (uiLow + uiHigh) >> 1; // (uiLow + uiHigh) / 2 + + // Use compare routine + + if (m_fnCompare) + { + iCompare = m_fnCompare( pvEntry, ENTRY_POS( uiMid), m_pvUserData); + } + else + { + iCompare = f_memcmp( pvEntry, ENTRY_POS( uiMid), m_uiEntrySize); + } + + if (iCompare == 0) + { + if (pvFoundEntry) + { + f_memcpy( pvFoundEntry, ENTRY_POS( uiMid), m_uiEntrySize); + } + rc = NE_SFLM_OK; + goto Exit; + } + + // Check if we are done - where wLow equals uiHigh or mid is at end. + + if (iCompare < 0) + { + if (uiMid == uiLow || uiLow == uiHigh) + { + break; + } + uiHigh = uiMid - 1; // Too high + } + else + { + if (uiMid == uiHigh || uiLow == uiHigh) + { + + // Go up one for the correct position? + + uiMid++; + break; + } + uiLow = uiMid + 1; // Too low + } + } + +Exit: + + m_uiPosition = uiMid; + if (puiChildAddr && blkType() != ACCESS_BTREE_LEAF) + { + if (uiMid == entryCount()) + { + *puiChildAddr = lemBlk(); + } + else + { + FLMBYTE * pucChildAddr = ENTRY_POS(uiMid) + m_uiEntrySize; + *puiChildAddr = (FLMUINT)FB2UD( pucChildAddr); + } + } + return( rc); +} + +/**************************************************************************** +Desc: Insert the entry into the btree - should be positioned +****************************************************************************/ +RCODE FBtreeRoot::insert( + void * pvEntry) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCurLevel; + FLMBYTE ucEntryBuf[FBTREE_MAX_LEVELS][DYNSSET_MAX_FIXED_ENTRY_SIZE]; + FLMUINT uiNewBlkAddr; + + if (RC_OK( rc = m_BTStack[0]->insert( pvEntry))) + { + goto Exit; + } + + // Failed to input at the left level. Do block split(s). + // This is an iterative and NOT a recursive split algorithm. + // The debugging, and cases to test should be lots easier this way. + + f_memcpy( ucEntryBuf[0], pvEntry, m_uiEntrySize); + uiCurLevel = 0; + uiNewBlkAddr = FBTREE_END; + for(;;) + { + + // Split while adding the element. + + if (RC_BAD( rc = (m_BTStack[uiCurLevel])->split( + this, + ucEntryBuf[ uiCurLevel], + uiNewBlkAddr, + ucEntryBuf[ uiCurLevel+1], + &uiNewBlkAddr))) + { + goto Exit; + } + + uiCurLevel++; + flmAssert( uiCurLevel < m_uiLevels); + + if (RC_OK( rc = m_BTStack[uiCurLevel]->insertEntry( + ucEntryBuf[uiCurLevel], uiNewBlkAddr))) + { + goto Exit; + } + + // Only returns NE_SFLM_OK or FAILURE. + + // Root split? + + if (uiCurLevel + 1 == m_uiLevels) + { + flmAssert( m_uiLevels + 1 <= FBTREE_MAX_LEVELS); + if (m_uiLevels + 1 > FBTREE_MAX_LEVELS) + { + rc = RC_SET( NE_SFLM_BTREE_FULL); + goto Exit; + } + + // Need to split the root block. + rc = ((FBtreeRoot *)m_BTStack[uiCurLevel])->split( + ucEntryBuf[uiCurLevel], uiNewBlkAddr ); + break; + } + } + +Exit: + + if (RC_OK(rc)) + { + m_uiTotalEntries++; + } + return( rc); +} + +/**************************************************************************** +Desc: Insert the entry into the buffer. +****************************************************************************/ +RCODE FBtreeBlk::insertEntry( + void * pvEntry, + FLMUINT uiChildAddr) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucCurEntry; + FLMUINT uiShiftBytes; // Always shift down + + if( entryCount() >= m_uiNumSlots) + { + rc = RC_SET( NE_SFLM_FAILURE); + goto Exit; + } + flmAssert( m_uiPosition != DYNSSET_POSITION_NOT_SET); + pucCurEntry = ENTRY_POS( m_uiPosition); + if ((uiShiftBytes = (entryCount() - m_uiPosition) * + (m_uiEntrySize + m_uiEntryOvhd)) != 0) + { + + // Big hairy assert. Finds coding bugs and corruptions. + + flmAssert( m_uiPosition * (m_uiEntrySize + m_uiEntryOvhd) + + uiShiftBytes < DYNSSET_BLOCK_SIZE - sizeof( FixedBlkHdr)); + + f_memmove( pucCurEntry + m_uiEntrySize + m_uiEntryOvhd, + pucCurEntry, uiShiftBytes); + } + f_memcpy( pucCurEntry, pvEntry, m_uiEntrySize); + if( m_uiEntryOvhd) + { + UD2FBA( (FLMUINT32)uiChildAddr, &pucCurEntry[m_uiEntrySize]); + } + entryCount( entryCount() + 1); + m_uiPosition++; + m_bDirty = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Move first half of entries into new block. Reset previous block + to point to new block. Add new last entry in new block to parent. + Fixup prev/next linkages. +****************************************************************************/ +RCODE FBtreeBlk::split( + FBtreeRoot * pRoot, + FLMBYTE * pucCurEntry, // (in) Contains entry to insert + FLMUINT uiCurBlkAddr, // (in) Blk addr if non-leaf + FLMBYTE * pucParentEntry, // (out) Entry to insert into parent. + FLMUINT * puiNewBlkAddr) // (out) New blk addr to insert into parent. +{ + RCODE rc = NE_SFLM_OK; + FBtreeBlk * pPrevBlk; + FBtreeBlk * pNewBlk = NULL; + FLMBYTE * pucEntry = NULL; + FLMBYTE * pucMidEntry; + FLMBYTE * pucChildAddr; + FLMUINT uiChildAddr; + FLMUINT uiPrevBlkAddr; + FLMUINT uiMid; + FLMUINT uiPos; + FLMUINT uiMoveBytes; + FLMBOOL bInserted = FALSE; + + // Allocate a new block for the split. + + if (RC_BAD( rc = pRoot->newBlk( &pNewBlk, blkType() ))) + { + goto Exit; + } + pNewBlk->AddRef(); // Pin the block - may get flushed out. + + + // the last half into the new block, but that would force the parent + // entry to be changed. This may take a little longer, but it is much + // more easier to code. + + // Call search entry once just to setup for insert. + + (void) pNewBlk->searchEntry( ENTRY_POS( 0)); + + // get the count and move more then half into the new block. + + uiMid = (entryCount() + 5) >> 1; + uiChildAddr = FBTREE_END; + + for (uiPos = 0; uiPos < uiMid; uiPos++) + { + pucEntry = ENTRY_POS( uiPos); + if (blkType() != ACCESS_BTREE_LEAF) + { + pucChildAddr = pucEntry + m_uiEntrySize; + uiChildAddr = (FLMUINT)FB2UD(pucChildAddr); + } + + // m_uiPosition automatically gets incremented. + + if (RC_BAD( rc = pNewBlk->insertEntry( pucEntry, uiChildAddr))) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + } + + if (m_uiPosition < uiMid) + { + + // Insert this entry now + + bInserted = TRUE; + (void) pNewBlk->searchEntry( pucCurEntry); + if (RC_BAD( rc = pNewBlk->insertEntry( pucCurEntry, uiCurBlkAddr))) + { + goto Exit; + } + } + + // Let caller insert into parent entry. This rids us of recursion. + + f_memcpy( pucParentEntry, pucEntry, m_uiEntrySize); + + // Move the rest down + + pucEntry = ENTRY_POS( 0); + pucMidEntry = ENTRY_POS( uiMid); + + entryCount( entryCount() - uiMid); + uiMoveBytes = entryCount() * (m_uiEntrySize + m_uiEntryOvhd); + flmAssert( uiMoveBytes < DYNSSET_BLOCK_SIZE - sizeof( FixedBlkHdr)); + f_memmove( pucEntry, pucMidEntry, uiMoveBytes); + + if( !bInserted) + { + + // m_uiPosition -= uiMid; + + (void) searchEntry( pucCurEntry); + if (RC_BAD( rc = insertEntry( pucCurEntry, uiCurBlkAddr))) + { + goto Exit; + } + } + + // VISIT: Could position stack to point to current element to insert. + + // Fixup the prev/next block linkages. + + if (prevBlk() != FBTREE_END) + { + if (RC_BAD( rc = pRoot->readBlk( prevBlk(), blkType(), &pPrevBlk ))) + { + goto Exit; + } + + pPrevBlk->nextBlk( pNewBlk->blkAddr()); + uiPrevBlkAddr = pPrevBlk->blkAddr(); + } + else + { + uiPrevBlkAddr = FBTREE_END; + } + pNewBlk->prevBlk( uiPrevBlkAddr); + pNewBlk->nextBlk( blkAddr()); + prevBlk( pNewBlk->blkAddr()); + + *puiNewBlkAddr = pNewBlk->blkAddr(); + +Exit: + + if (pNewBlk) + { + pNewBlk->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Reinsert all entries given a new root block. + Caller will release 'this'. Used ONLY for building the first + ROOT and two leaves of the tree. +****************************************************************************/ +RCODE FBtreeLeaf::split( + FBtreeRoot * pNewRoot) // New Non-leaf root +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry; + FLMUINT uiPos; + FLMUINT uiEntryCount = entryCount(); + FLMUINT uiMid = (uiEntryCount + 1) >> 1; + + if (RC_BAD( rc = pNewRoot->setupTree( ENTRY_POS(uiMid), + ACCESS_BTREE_LEAF, NULL, NULL))) + { + goto Exit; + } + + for (uiPos = 0; uiPos < uiEntryCount; uiPos++) + { + pucEntry = ENTRY_POS( uiPos); + if ((rc = pNewRoot->search( pucEntry)) != NE_SFLM_NOT_FOUND) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + if (RC_BAD( rc = pNewRoot->insert( pucEntry))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Split the root block and make two new non-leaf blocks. + The secret here is that the root block never moves (cheers!). + This takes a little longer but is worth the work because the + root block never goes out to disk and is not in the cache. +****************************************************************************/ +RCODE FBtreeRoot::split( + void * pvCurEntry, + FLMUINT uiCurChildAddr) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEntry; + FLMBYTE * pucChildAddr; + FBtreeBlk * pLeftBlk; + FBtreeBlk * pRightBlk; + FBtreeBlk * pBlk; + FLMUINT uiChildAddr; + FLMUINT uiPos; + FLMUINT uiEntryCount = entryCount(); + FLMUINT uiMid = (uiEntryCount + 1) >> 1; + + if (RC_BAD( rc = setupTree( NULL, ACCESS_BTREE_NON_LEAF, + &pLeftBlk, &pRightBlk))) + { + goto Exit; + } + + // Call search entry once just to setup for insert. + + (void) pLeftBlk->searchEntry( ENTRY_POS( 0)); + + // Take the entries from the root block and move into leafs. + + for (uiPos = 0; uiPos <= uiMid; uiPos++) + { + pucEntry = ENTRY_POS( uiPos); + pucChildAddr = pucEntry + m_uiEntrySize; + uiChildAddr = (FLMUINT)FB2UD(pucChildAddr); + + if (RC_BAD( rc = pLeftBlk->insertEntry( pucEntry, uiChildAddr))) + { + goto Exit; + } + } + + // Call search entry once just to setup for insert. + + (void) pRightBlk->searchEntry( ENTRY_POS( 0)); + + for (uiPos = uiMid + 1; uiPos < uiEntryCount; uiPos++) + { + pucEntry = ENTRY_POS( uiPos); + pucChildAddr = pucEntry + m_uiEntrySize; + uiChildAddr = (FLMUINT)FB2UD(pucChildAddr); + + if ((rc = pRightBlk->searchEntry( pucEntry )) != NE_SFLM_NOT_FOUND) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + if (RC_BAD( rc = pRightBlk->insertEntry( pucEntry, uiChildAddr))) + { + goto Exit; + } + } + + // Reset the root block and insert new midpoint. + + entryCount( 0); + lemBlk( pRightBlk->blkAddr()); // Duplicated just in case. + pucEntry = ENTRY_POS( uiMid); + + if ((rc = searchEntry( pucEntry )) != NE_SFLM_NOT_FOUND) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + if (RC_BAD( rc = insertEntry( pucEntry, pLeftBlk->blkAddr() ))) + { + goto Exit; + } + + // Insert the current entry (parameters) into the left or right blk. + // This could be done a number of different ways. + (void) searchEntry( pvCurEntry, &uiChildAddr); + if (RC_BAD( rc = readBlk( uiChildAddr, ACCESS_BTREE_NON_LEAF, &pBlk))) + { + goto Exit; + } + (void) pBlk->searchEntry( pvCurEntry); + if (RC_BAD( rc = pBlk->insertEntry( pvCurEntry, uiCurChildAddr))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup two child blocks for a root block. +****************************************************************************/ +RCODE FBtreeRoot::setupTree( + FLMBYTE * pucMidEntry, // If !NULL entry to insert into root. + FBlkTypes eBlkType, // Leaf or non-leaf + FBtreeBlk ** ppLeftBlk, // (out) + FBtreeBlk ** ppRightBlk) // (out) +{ + RCODE rc = NE_SFLM_OK; + FBtreeBlk * pLeftBlk = NULL; + FBtreeBlk * pRightBlk = NULL; + + if (RC_BAD( rc = newBlk( &pLeftBlk, eBlkType))) + { + goto Exit; + } + + if (RC_BAD( rc = newBlk( &pRightBlk, eBlkType))) + { + goto Exit; + } + + if (eBlkType == ACCESS_BTREE_NON_LEAF) + { + ((FBtreeNonLeaf *)pRightBlk)->lemBlk( lemBlk()); + } + + // Fix up the linkages + + pLeftBlk->nextBlk( pRightBlk->blkAddr()); + pRightBlk->prevBlk( pLeftBlk->blkAddr()); + lemBlk( pRightBlk->blkAddr()); + + if (pucMidEntry) + { + + // Add the midentry to the root block. Search to position and insert. + + searchEntry( pucMidEntry); + insertEntry( pucMidEntry, pLeftBlk->blkAddr()); + } + m_uiLevels++; + + if (ppLeftBlk) + { + *ppLeftBlk = pLeftBlk; + } + if (ppRightBlk) + { + *ppRightBlk = pRightBlk; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Read in the block or get it from the cache. +****************************************************************************/ +RCODE FBtreeRoot::readBlk( + FLMUINT uiBlkAddr, // Blk address to read + FBlkTypes eBlkType, // Expected access type to read + FBtreeBlk ** ppBlk) // (out) Return block +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPos; + FLMUINT uiLRUValue = (FLMUINT)~0; + FLMUINT uiLRUPos = 0; + FBtreeBlk * pNewBlk; + + for (uiPos = 0; uiPos < FBTREE_CACHE_BLKS; uiPos++) + { + if (m_CacheBlks[uiPos].uiBlkAddr == uiBlkAddr) + { + goto Exit; + } + + // The ref count is used for pinning the block. + + if (m_CacheBlks[uiPos].pBlk && + m_CacheBlks[uiPos].pBlk->getRefCount() == 1 && + uiLRUValue > m_CacheBlks[uiPos].uiLRUValue) + { + uiLRUValue = m_CacheBlks[uiPos].uiLRUValue; + uiLRUPos = uiPos; + } + + // There better not be a hole by this point. + + flmAssert( m_CacheBlks[uiPos].pBlk != NULL); + } + uiPos = uiLRUPos; + + // Read from disk? + + flmAssert( m_pFileHdl != NULL); + + if (RC_BAD( rc = newCacheBlk( uiPos, &pNewBlk, eBlkType))) + { + goto Exit; + } + + // Pick the LRU block and make that object do the reading + // so it can reset all internals and get used to being a different blk. + + pNewBlk->blkAddr( uiBlkAddr); + m_CacheBlks[uiPos].uiBlkAddr = uiBlkAddr; + m_CacheBlks[uiPos].uiLRUValue = m_uiLRUCount++; + + if (RC_BAD( rc = pNewBlk->readBlk( m_pFileHdl, uiBlkAddr))) + { + + // Release the block because the reset() changed the object type. + // May hit the assert above. + + m_CacheBlks[uiPos].pBlk->Release(); + m_CacheBlks[uiPos].pBlk = NULL; + goto Exit; + } + +Exit: + + if (RC_OK(rc)) + { + *ppBlk = m_CacheBlks[uiPos].pBlk; + m_CacheBlks[uiPos].uiLRUValue = m_uiLRUCount++; + } + return( rc); +} + +/**************************************************************************** +Desc: Get a new block using an exising or newly allocated block from + the cache. Initializes the block. May be leaf or non-leaf + but NOT the root block. +****************************************************************************/ +RCODE FBtreeRoot::newBlk( + FBtreeBlk ** ppBlk, + FBlkTypes eBlkType) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLRUValue = (FLMUINT)~0; + FLMUINT uiPos; + FLMUINT uiLRUPos = 0; + FBtreeBlk * pNewBlk; + + for (uiPos = 0; uiPos < FBTREE_CACHE_BLKS; uiPos++) + { + + // The ref count is used for pinning the block. + + if (getRefCount() == 1 && + uiLRUValue > m_CacheBlks[uiPos].uiLRUValue) + { + uiLRUValue = m_CacheBlks[uiPos].uiLRUValue; + uiLRUPos = uiPos; + } + + // use the first hole. + + if (m_CacheBlks[uiPos].pBlk == NULL) + { + uiLRUPos = uiPos; + break; + } + } + uiPos = uiLRUPos; + if (RC_BAD( rc = newCacheBlk( uiPos, &pNewBlk, eBlkType))) + { + goto Exit; + } + + pNewBlk->blkAddr( newBlkAddr()); + m_CacheBlks[uiPos].uiBlkAddr = pNewBlk->blkAddr(); + m_CacheBlks[uiPos].uiLRUValue = m_uiLRUCount++; + + pNewBlk->entryCount(0); + pNewBlk->lemBlk( FBTREE_END); + pNewBlk->nextBlk( FBTREE_END); + pNewBlk->prevBlk( FBTREE_END); + *ppBlk = pNewBlk; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Release the existing cache block and setup and alloc new blk. +****************************************************************************/ +RCODE FBtreeRoot::newCacheBlk( + FLMUINT uiCachePos, + FBtreeBlk ** ppBlk, + FBlkTypes eBlkType) +{ + RCODE rc = NE_SFLM_OK; + FBtreeBlk * pNewBlk = NULL; + + if (m_CacheBlks[uiCachePos].pBlk) + { + if (m_CacheBlks[uiCachePos].pBlk->isDirty()) + { + if (RC_BAD( rc = writeBlk( uiCachePos))) + { + goto Exit; + } + } + } + + if (m_CacheBlks[uiCachePos].pBlk != NULL && + m_CacheBlks[uiCachePos].pBlk->blkType() == eBlkType) + { + + // If block is of the same type then reset it and use it. + + pNewBlk = m_CacheBlks[uiCachePos].pBlk; + pNewBlk->reset( eBlkType); + *ppBlk = pNewBlk; + goto Exit; + } + + if (m_CacheBlks[uiCachePos].pBlk) + { + m_CacheBlks[uiCachePos].pBlk->Release(); + } + if (eBlkType == ACCESS_BTREE_LEAF) + { + FBtreeLeaf * pLeafBlk; + if ((pLeafBlk = f_new FBtreeLeaf) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = pLeafBlk->setup( m_uiEntrySize))) + { + pLeafBlk->Release(); + goto Exit; + } + pLeafBlk->setCompareFunc( m_fnCompare, m_pvUserData); + pNewBlk = (FBtreeBlk *) pLeafBlk; + } + else + { + FBtreeNonLeaf * pNonLeafBlk; + if ((pNonLeafBlk = f_new FBtreeNonLeaf) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = pNonLeafBlk->setup( m_uiEntrySize))) + { + pNonLeafBlk->Release(); + goto Exit; + } + pNonLeafBlk->setCompareFunc( m_fnCompare, m_pvUserData); + pNewBlk = (FBtreeBlk *) pNonLeafBlk; + } + m_CacheBlks[uiCachePos].pBlk = pNewBlk; + *ppBlk = pNewBlk; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Read the block from disk. +****************************************************************************/ +RCODE FBtreeBlk::readBlk( + IF_FileHdl * pFileHdl, + FLMUINT uiBlkAddr) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesRead; + + if (RC_BAD( rc = pFileHdl->read( + uiBlkAddr * DYNSSET_BLOCK_SIZE, DYNSSET_BLOCK_SIZE, + m_pucBlkBuf, &uiBytesRead))) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + + if (blkAddr() != uiBlkAddr) + { + + // Most likely a coding error rather than an I/O error. + + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Write the block to disk. +****************************************************************************/ +RCODE FBtreeBlk::writeBlk( + IF_FileHdl * pFileHdl) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesWritten; + FLMUINT uiBlkAddr = blkAddr(); + + if (RC_BAD( rc = pFileHdl->write( + uiBlkAddr * DYNSSET_BLOCK_SIZE, + DYNSSET_BLOCK_SIZE, + m_pucBlkBuf, + &uiBytesWritten))) + { + goto Exit; + } + + m_bDirty = FALSE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Write all blocks that are dirty and have an addrees lower + than the input block address and then write this block. + Write in order so that + we don't have to write zeros for any block. +****************************************************************************/ +RCODE FBtreeRoot::writeBlk( + FLMUINT uiWritePos) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPos; + FLMUINT uiHighBlkAddr = m_CacheBlks[uiWritePos].uiBlkAddr; + + if (!m_pFileHdl) + { + if (RC_BAD( rc = openFile())) + { + goto Exit; + } + } + for (uiPos = 0; uiPos < FBTREE_CACHE_BLKS; uiPos++) + { + if( (uiWritePos != uiPos) && + (m_CacheBlks[uiPos].pBlk) && + (m_CacheBlks[uiPos].uiBlkAddr >= m_uiHighestWrittenBlkAddr) && + (m_CacheBlks[uiPos].uiBlkAddr < uiHighBlkAddr) && + (m_CacheBlks[uiPos].pBlk->isDirty()) ) + { + + // Recursive call to write out lower blocks if needed. + + if (RC_BAD( rc = writeBlk( uiPos))) + { + goto Exit; + } + } + } + m_CacheBlks[ uiWritePos].pBlk->writeBlk( m_pFileHdl); + if (m_CacheBlks[uiWritePos].uiBlkAddr > m_uiHighestWrittenBlkAddr) + { + m_uiHighestWrittenBlkAddr = m_CacheBlks[uiWritePos].uiBlkAddr; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Close the file if previously opened and creates the file. +****************************************************************************/ +RCODE FBtreeRoot::openFile( void) +{ + RCODE rc = NE_SFLM_OK; + + if (!m_pFileHdl) + { + rc = gv_SFlmSysData.pFileSystem->createUniqueFile( m_pszFileName, + FRSET_FILENAME_EXTENSION, + FLM_IO_RDWR | FLM_IO_CREATE_DIR, &m_pFileHdl); + } + return( rc); +} + +/**************************************************************************** +Desc: Closes and deletes the temporary file. +****************************************************************************/ +void FBtreeRoot::closeFile( void) +{ + if (m_pFileHdl) + { + m_pFileHdl->close(); + gv_SFlmSysData.pFileSystem->deleteFile( m_pszFileName); + m_pFileHdl->Release(); + m_pFileHdl = NULL; + } +} diff --git a/sql/src/fdynbuf.cpp b/sql/src/fdynbuf.cpp new file mode 100644 index 0000000..ae5a6d0 --- /dev/null +++ b/sql/src/fdynbuf.cpp @@ -0,0 +1,123 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the routines for the Flaim Dynamic Buffer Class. +// +// Tabs: 3 +// +// Copyright (c) 2003-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdynbuf.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" +#include "fdynbuf.h" + +/****************************************************************** +Desc: Implements the addChar function of the DynamicBuffer class +*******************************************************************/ +RCODE F_DynamicBuffer::addChar( + FLMBYTE ucCharacter) +{ + RCODE rc = NE_SFLM_OK; + + if (!m_bSetup) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + + } + + f_mutexLock( m_hMutex); + + // Is there room for just one more character plus a terminator? + if ((m_uiBuffSize - m_uiUsedChars) > 1) + { + m_psBuffer[m_uiUsedChars++] = ucCharacter; + m_psBuffer[m_uiUsedChars] = 0; + } + else + { + // Allocate a new buffer or increase the size of the existing one. + if( !m_uiBuffSize) + { + if( RC_BAD( rc = f_alloc( 50, &m_psBuffer))) + { + goto Exit; + } + m_uiBuffSize = 50; + } + else + { + if( RC_BAD( rc = f_realloc( m_uiBuffSize + 50, &m_psBuffer))) + { + goto Exit; + } + m_uiBuffSize += 50; + } + + + m_psBuffer[m_uiUsedChars++] = ucCharacter; + m_psBuffer[m_uiUsedChars] = 0; + } + +Exit: + + if ( m_bSetup) + { + f_mutexUnlock( m_hMutex); + } + + return rc; +} + +/****************************************************************** +Desc: Implements the addChar function of the DynamicBuffer class +*******************************************************************/ +RCODE F_DynamicBuffer::addString( const char * pszString) +{ + RCODE rc = NE_SFLM_OK; + const char * pTemp = pszString; + FLMUINT uiTmpPos = m_uiUsedChars; + + + while( *pTemp) + { + if (RC_BAD( rc = addChar( (FLMBYTE)*pTemp))) + { + // Reset the buffer to its state prior to this call. + m_uiUsedChars = uiTmpPos; + if (m_uiBuffSize > 0) + { + m_psBuffer[ m_uiUsedChars] = 0; + } + goto Exit; + } + pTemp++; + } + +Exit: + + return rc; +} + +/****************************************************************** +Desc: Implements the addChar function of the DynamicBuffer class +*******************************************************************/ +const char * F_DynamicBuffer::printBuffer() +{ + return (char *)m_psBuffer; +} diff --git a/sql/src/fdynbuf.h b/sql/src/fdynbuf.h new file mode 100644 index 0000000..e71ea96 --- /dev/null +++ b/sql/src/fdynbuf.h @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +// Desc: Dynamic buffer +// +// Tabs: 3 +// +// Copyright (c) 2003-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdynbuf.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +/**************************************************************************** +Desc: Utility class to help prepare a character buffer for printing when the + output buffer size is not known in advance. +*****************************************************************************/ +class F_DynamicBuffer : public F_Object +{ +public: + + F_DynamicBuffer() + { + m_bSetup = FALSE; + m_psBuffer = NULL; + m_uiBuffSize = 0; + m_uiUsedChars = 0; + if (RC_OK( f_mutexCreate( &m_hMutex))) + { + m_bSetup = TRUE; + } + } + + ~F_DynamicBuffer() + { + f_free( &m_psBuffer); + m_psBuffer = NULL; + m_uiBuffSize = 0; + m_uiUsedChars = 0; + if (m_bSetup) + { + f_mutexDestroy( &m_hMutex); + m_bSetup = FALSE; + } + } + + RCODE addChar( FLMBYTE ucCharacter); + + RCODE addString( const char * pszString); + + const char * printBuffer(); + + FLMUINT getBufferSize( void) + { + return m_uiUsedChars; + } + + void reset( void) + { + f_free( &m_psBuffer); + m_psBuffer = NULL; + m_uiBuffSize = 0; + m_uiUsedChars = 0; + } + +private: + + FLMBOOL m_bSetup; + FLMBYTE * m_psBuffer; + FLMUINT m_uiBuffSize; + FLMUINT m_uiUsedChars; + F_MUTEX m_hMutex; +}; diff --git a/sql/src/fdynsset.cpp b/sql/src/fdynsset.cpp new file mode 100644 index 0000000..d0c3a35 --- /dev/null +++ b/sql/src/fdynsset.cpp @@ -0,0 +1,328 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the methods for FDynSearchSet class. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdynsset.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" +#include "fdynsset.h" + +#define HASH_POS(vp) \ + (((FLMUINT)(FB2UD((FLMBYTE*)vp)) % m_uiNumSlots) * m_uiEntrySize) + +static const FLMBYTE ucZeros [ DYNSSET_MAX_FIXED_ENTRY_SIZE ] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/**************************************************************************** +Desc: Setup the result set with input values. This method must be + called and only called once. +****************************************************************************/ +RCODE FDynSearchSet::setup( + char * pszTmpDir, + FLMUINT uiEntrySize) +{ + RCODE rc = NE_SFLM_OK; + FHashBlk * pHashBlk; + + // Set the input variables. + + if( pszTmpDir ) + { + f_strcpy( m_szFileName, pszTmpDir); // Dest <- src + } + else + { + f_memset( m_szFileName, 0, F_PATH_MAX_SIZE); + } + m_uiEntrySize = uiEntrySize; + + if ((pHashBlk = f_new FHashBlk) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + pHashBlk->setup( uiEntrySize); + m_pAccess = (FFixedBlk *) pHashBlk; + m_pvUserData = (void *) uiEntrySize; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add a fixed length entry to the dynamic search result set. +****************************************************************************/ +RCODE FDynSearchSet::addEntry( + void * pvEntry) +{ + RCODE rc = NE_SFLM_OK; + +Add_Again: + + if (RC_OK( rc = m_pAccess->search( pvEntry))) + { + rc = RC_SET( NE_SFLM_EXISTS); + } + else if (rc == NE_SFLM_NOT_FOUND) + { + + // Insert the entry. + + if ((rc = m_pAccess->insert( pvEntry)) == NE_SFLM_FAILURE) + { + // Find the type of access method implemented + + if (m_pAccess->blkType() == ACCESS_HASH) + { + FBtreeLeaf * pBtreeBlk; + FLMBYTE ucEntryBuffer[ DYNSSET_MAX_FIXED_ENTRY_SIZE]; + + // Go from a hash to a b-tree object. Alloc and move stuff over. + + if ((pBtreeBlk = f_new FBtreeLeaf) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + pBtreeBlk->setup( m_uiEntrySize); + pBtreeBlk->setCompareFunc( m_fnCompare, m_pvUserData); + for( rc = m_pAccess->getFirst( ucEntryBuffer ); + RC_OK(rc); + rc = m_pAccess->getNext( ucEntryBuffer) ) + { + // Call search to setup for insert. + (void) pBtreeBlk->search( ucEntryBuffer); + if (RC_BAD( rc = pBtreeBlk->insert( ucEntryBuffer))) + { + pBtreeBlk->Release(); + goto Exit; + } + } + rc = NE_SFLM_OK; + m_pAccess->Release(); + m_pAccess = pBtreeBlk; + goto Add_Again; + } + else if( m_pAccess->blkType() == ACCESS_BTREE_LEAF) + { + FBtreeRoot * pFullBtree; + + // Go from 1 block to 3 changing root blocks and free m_pAccess + // All new splits will be taken care of automatically. + + if ((pFullBtree = f_new FBtreeRoot) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if( RC_BAD( rc = pFullBtree->setup( m_uiEntrySize, m_szFileName))) + { + pFullBtree->Release(); + goto Exit; + } + pFullBtree->setCompareFunc( m_fnCompare, m_pvUserData); + if (RC_BAD( rc = ((FBtreeLeaf *)m_pAccess)->split( pFullBtree))) + { + goto Exit; + } + m_pAccess->Release(); + m_pAccess = pFullBtree; + goto Add_Again; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Find matching entry. Position for Get* or for insert. +****************************************************************************/ +RCODE FHashBlk::search( + void * pvEntry, + void * pvFoundEntry) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiHashPos = HASH_POS( pvEntry); + FLMINT iCompare; + + for (;;) + { + + // If all zeros then setup to insert at this position. + + if (!f_memcmp( &m_pucBlkBuf[ uiHashPos], ucZeros, m_uiEntrySize)) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + + if (m_fnCompare) + { + iCompare = m_fnCompare( pvEntry, &m_pucBlkBuf[ uiHashPos], + m_pvUserData); + } + else + { + iCompare = f_memcmp( pvEntry, &m_pucBlkBuf[ uiHashPos], + m_uiEntrySize); + } + + if (iCompare == 0) + { + + // Found match. + + if (pvFoundEntry) + { + f_memcpy( pvFoundEntry, &m_pucBlkBuf[ uiHashPos], m_uiEntrySize); + } + break; + } + + // Go to the next entry + + uiHashPos += m_uiEntrySize; + if (uiHashPos >= DYNSSET_HASH_BUFFER_SIZE) + { + uiHashPos = 0; + } + } + +Exit: + + m_uiPosition = uiHashPos; + return( rc); +} + +/**************************************************************************** +Desc: Insert the entry into the buffer. +****************************************************************************/ +RCODE FHashBlk::insert( + void * pvEntry) +{ + RCODE rc = NE_SFLM_OK; + + if( getTotalEntries() > ((m_uiNumSlots * 7) / 10)) + { + rc = RC_SET( NE_SFLM_FAILURE); + goto Exit; + } + + f_memcpy( &m_pucBlkBuf[ m_uiPosition], pvEntry, m_uiEntrySize); + m_uiTotalEntries++; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Return the next entry in the result set. If the result set + is not positioned then the first entry will be returned. +****************************************************************************/ +RCODE FHashBlk::getNext( + void * pvEntryBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiHashPos; + + // Position to the next/first entry. + + if (m_uiPosition == DYNSSET_POSITION_NOT_SET) + { + uiHashPos = 0; + } + else + { + uiHashPos = m_uiPosition + m_uiEntrySize; + } + + for ( ; ; uiHashPos += m_uiEntrySize) + { + if (uiHashPos >= DYNSSET_HASH_BUFFER_SIZE) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + // If all zeros then setup to insert at this position. + + if (f_memcmp( &m_pucBlkBuf[ uiHashPos], ucZeros, m_uiEntrySize)) + { + f_memcpy( pvEntryBuffer, &m_pucBlkBuf[ uiHashPos], m_uiEntrySize); + m_uiPosition = uiHashPos; + goto Exit; + } + } + +Exit: + + return rc; +} + +/**************************************************************************** +Desc: Returns the last entry in the result set. +****************************************************************************/ +RCODE FHashBlk::getLast( + void * pvEntryBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiHashPos; + + // Position to the next/first entry. + + uiHashPos = DYNSSET_HASH_BUFFER_SIZE; + + for( ; ; ) + { + uiHashPos -= m_uiEntrySize; + + // If all zeros then setup to insert at this position. + + if (f_memcmp( &m_pucBlkBuf[ uiHashPos], ucZeros, m_uiEntrySize)) + { + f_memcpy( pvEntryBuffer, &m_pucBlkBuf[ uiHashPos], m_uiEntrySize); + m_uiPosition = uiHashPos; + goto Exit; + } + if (uiHashPos == 0) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + } + +Exit: + + return( rc); +} diff --git a/sql/src/fdynsset.h b/sql/src/fdynsset.h new file mode 100644 index 0000000..5b58ad3 --- /dev/null +++ b/sql/src/fdynsset.h @@ -0,0 +1,672 @@ +//------------------------------------------------------------------------------ +// Desc: FLAIM Dynamic search result set class. +// +// Tabs: 3 +// +// Copyright (c) 1998-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fdynsset.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FDYNSSET_H +#define FDYNSSET_H + +/***************************************************************************** +***** +** Definitions +***** +*****************************************************************************/ + +/* +A block size of 8K will perform well in minimizing the number of reads +to obtain a block. A 6K may perform better if the file is located +across the network. +*/ + +#define DYNSSET_BLOCK_SIZE 0x4000 +#define DYNSSET_HASH_BUFFER_SIZE 0x2000 +#define DYNSSET_MIN_FIXED_ENTRY_SIZE 4 +// Change ucZeros in fdynsset.cpp if this changes. +#define DYNSSET_MAX_FIXED_ENTRY_SIZE 32 +#define DYNSSET_POSITION_NOT_SET 0xFFFFFFFF + +#define FBTREE_CACHE_BLKS 32 +#define FBTREE_END 0xFFFFFFFF +#define FBTREE_MAX_LEVELS 4 + +typedef int (* FDYNSET_COMPARE_FUNC)( + void * pvData1, + void * pvData2, + void * pvUserData); + + +/***************************************************************************** +***** +** Forward References +***** +*****************************************************************************/ + +class FDynSearchSet; +class FHashBlk; +class FBtreeBlk; +class FBtreeRoot; +class FBtreeNonLeaf; +class FBtreeLeaf; + +enum eBlkTypes +{ + ACCESS_HASH, + ACCESS_BTREE_LEAF, + ACCESS_BTREE_ROOT, + ACCESS_BTREE_NON_LEAF +}; +typedef enum eBlkTypes FBlkTypes; + + +/*=========================================================================== + Virtual FFixedBlk +===========================================================================*/ +class FFixedBlk : public F_Object +{ +public: + + FFixedBlk(); + ~FFixedBlk() + { + } + + /* virtual methods that must be implemented */ + + FBlkTypes blkType() + { + return m_eBlkType; + } + + virtual RCODE getCurrent( + void * pvEntryBuffer) = 0; + + virtual RCODE getFirst( + void * pvEntryBuffer) = 0; + + virtual RCODE getLast( + void * pvEntryBuffer) = 0; + + virtual RCODE getNext( + void * pvEntryBuffer) = 0; + + virtual FLMUINT getTotalEntries() = 0; + + virtual RCODE insert( + void * pvEntry) = 0; + + FINLINE FLMBOOL isDirty( void) + { + return( m_bDirty); + } + + virtual RCODE search( + void * pvEntry, + void * pvFoundEntry = NULL) = 0; + + void setCompareFunc( + FDYNSET_COMPARE_FUNC fnCompare, + void * pvUserData) + { + m_fnCompare = fnCompare; + m_pvUserData = pvUserData; + } + +protected: + + // Variables + + FDYNSET_COMPARE_FUNC m_fnCompare; + void * m_pvUserData; + FBlkTypes m_eBlkType; + FLMUINT m_uiEntrySize; + FLMUINT m_uiNumSlots; + FLMUINT m_uiPosition; + FLMBOOL m_bDirty; + FLMBYTE * m_pucBlkBuf; +}; + + +/***************************************************************************** +***** +** Result Set Class Definitions +** Source: fdynsset.cpp +***** +*****************************************************************************/ +class FDynSearchSet : public F_Object +{ + +public: + + FDynSearchSet() + { + m_fnCompare = NULL; + m_pvUserData = NULL; + m_uiEntrySize = 0; + m_pAccess = NULL; + } + + ~FDynSearchSet() + { + if (m_pAccess) + { + m_pAccess->Release(); + } + } + + RCODE setup( + char * pszTmpDir, + FLMUINT uiEntrySize); + + FINLINE void setCompareFunc( + FDYNSET_COMPARE_FUNC fnCompare, + void * pvUserData + ) + { + m_fnCompare = fnCompare; + m_pvUserData = pvUserData; + m_pAccess->setCompareFunc( fnCompare, pvUserData); + } + + RCODE addEntry( + void * pvEntry); + + FINLINE RCODE findMatch( + void * pvMatchEntry, + void * pvFoundEntry + ) + { + return m_pAccess->search( pvMatchEntry, pvFoundEntry); + } + + FINLINE FLMUINT getEntrySize( void) + { + return m_uiEntrySize; + } + + FINLINE FLMUINT getTotalEntries( void) + { + return m_pAccess->getTotalEntries(); + } + +private: + + // Variables + + FDYNSET_COMPARE_FUNC m_fnCompare; + void * m_pvUserData; + FLMUINT m_uiEntrySize; + FFixedBlk * m_pAccess; + char m_szFileName [F_PATH_MAX_SIZE]; +}; + +/*=========================================================================== + Block Header Definition + +Desc: Actually stored as the first section of each block. + We can write this structure because the same process will + read the block header i.e. portability is not a problem. +===========================================================================*/ + +typedef struct +{ + FLMUINT uiBlkAddr; + FLMUINT uiPrevBlkAddr; + FLMUINT uiNextBlkAddr; + FLMUINT uiLEMAddr; + FLMUINT uiNumEntries; +} FixedBlkHdr; + +/*=========================================================================== + Fixed Length HASH Access Method +===========================================================================*/ +class FHashBlk : public FFixedBlk +{ +public: + + FHashBlk() + { + // Base class constructors are called before this constructor is. + + m_eBlkType = ACCESS_HASH; + m_pucBlkBuf = m_ucHashBlk; + f_memset( m_ucHashBlk, 0, sizeof( m_ucHashBlk)); + m_uiTotalEntries = 0; + } + + ~FHashBlk() + { + // Set to NULL so we don't free the block + m_pucBlkBuf = NULL; + } + + FINLINE RCODE setup( + FLMUINT uiEntrySize + ) + { + m_uiEntrySize = uiEntrySize; + m_uiNumSlots = DYNSSET_HASH_BUFFER_SIZE / uiEntrySize; + return( NE_SFLM_OK); + } + + FINLINE RCODE getCurrent( + void * pvEntryBuffer + ) + { + + // Position to the next/first entry. + + if (m_uiPosition == DYNSSET_POSITION_NOT_SET) + { + return RC_SET( NE_SFLM_NOT_FOUND); + } + + f_memcpy( pvEntryBuffer, &m_pucBlkBuf[ m_uiPosition], m_uiEntrySize); + return( NE_SFLM_OK); + } + + FINLINE RCODE getFirst( + void * pvEntryBuffer + ) + { + m_uiPosition = DYNSSET_POSITION_NOT_SET; + return( getNext( pvEntryBuffer)); + } + + RCODE getLast( + void * pvEntryBuffer); + + RCODE getNext( + void * pvEntryBuffer); + + FINLINE FLMUINT getTotalEntries( void) + { + return m_uiTotalEntries; + } + + RCODE insert( + void * pvEntry); + + RCODE search( + void * pvEntry, + void * pvFoundEntry = NULL); + +private: + + FLMUINT m_uiTotalEntries; + // Allocate the hash block to save 1 allocation. + // We need to make the hash as fast as possible. + FLMBYTE m_ucHashBlk[ DYNSSET_HASH_BUFFER_SIZE]; +}; + + +/*=========================================================================== + Virtual FBtreeBlk +===========================================================================*/ + +// Leaf and non-leaf entry position. Don't do any ++ or -- ! ! ! ! + +#define ENTRY_POS(uiPos) (m_pucBlkBuf + sizeof( FixedBlkHdr) + \ + (uiPos * (m_uiEntrySize+m_uiEntryOvhd))) + +class FBtreeBlk : public FFixedBlk +{ +public: + + FBtreeBlk() + { + } + + ~FBtreeBlk() + { + if (m_pucBlkBuf) + { + f_free( &m_pucBlkBuf); + } + } + + // virtual methods that must be implemented + + FINLINE RCODE getCurrent( + void * pvEntryBuffer + ) + { + // Position to the next/first entry. + + if (m_uiPosition == DYNSSET_POSITION_NOT_SET) + { + return RC_SET( NE_SFLM_NOT_FOUND); + } + + f_memcpy( pvEntryBuffer, ENTRY_POS( m_uiPosition), m_uiEntrySize); + return( NE_SFLM_OK); + } + + FINLINE RCODE getFirst( + void * pvEntryBuffer + ) + { + m_uiPosition = DYNSSET_POSITION_NOT_SET; + return( getNext( pvEntryBuffer)); + } + + RCODE getLast( + void * pvEntryBuffer); + + RCODE getNext( + void * pvEntryBuffer); + + RCODE readBlk( + IF_FileHdl * pFileHdl, + FLMUINT uiBlkAddr); + + void reset( + FBlkTypes eBlkType); + + RCODE split( + FBtreeRoot * pParent, + FLMBYTE * pucCurEntry, + FLMUINT uiCurBlkAddr, + FLMBYTE * pucParentEntry, + FLMUINT * puiNewBlkAddr); + + RCODE writeBlk( + IF_FileHdl * pFileHdl); + + // Virtual methods + + virtual FLMUINT getTotalEntries( void) = 0; + + virtual RCODE insert( + void * pvEntry) = 0; + + virtual RCODE search( + void * pvEntry, + void * pvFoundEntry = NULL) = 0; + + virtual RCODE searchEntry( + void * pvEntry, + FLMUINT * puiChildAddr = NULL, + void * pvFoundEntry = NULL); + + // Implemented as inline functions. + // Even though these are b-tree specific - keep them here to + // avoid having a b-tree block class. Most are not used for hash. + + FINLINE FLMUINT blkAddr( void) + { + return( ((FixedBlkHdr *)m_pucBlkBuf)->uiBlkAddr); + } + + FINLINE void blkAddr( + FLMUINT uiBlkAddr) + { + ((FixedBlkHdr *)m_pucBlkBuf)->uiBlkAddr = uiBlkAddr; + m_bDirty = TRUE; + } + + // Get and set the number of entries in the block + + FINLINE FLMUINT entryCount( void) + { + return( ((FixedBlkHdr *)m_pucBlkBuf)->uiNumEntries); + } + + FINLINE void entryCount( + FLMUINT uiNumEntries) + { + ((FixedBlkHdr *)m_pucBlkBuf)->uiNumEntries = uiNumEntries; + m_bDirty = TRUE; + } + + RCODE insertEntry( + void * pvEntry, + FLMUINT uiChildAddr = FBTREE_END); + + + // Get and set the last element marker in the block. + + FINLINE FLMUINT lemBlk( void) + { + return( ((FixedBlkHdr *)m_pucBlkBuf)->uiLEMAddr); + } + + FINLINE void lemBlk( + FLMUINT uiLEMAddr) + { + ((FixedBlkHdr *)m_pucBlkBuf)->uiLEMAddr = uiLEMAddr; + m_bDirty = TRUE; + } + + // Get and set the next block address element + + FINLINE FLMUINT nextBlk( void) + { + return( ((FixedBlkHdr *)m_pucBlkBuf)->uiNextBlkAddr); + } + + FINLINE void nextBlk( + FLMUINT uiBlkAddr) + { + ((FixedBlkHdr *)m_pucBlkBuf)->uiNextBlkAddr = uiBlkAddr; + m_bDirty = TRUE; + } + + // Get and set the previous block address element + + FINLINE FLMUINT prevBlk( void) + { + return( ((FixedBlkHdr *)m_pucBlkBuf)->uiPrevBlkAddr); + } + + FINLINE void prevBlk( + FLMUINT uiBlkAddr) + { + ((FixedBlkHdr *)m_pucBlkBuf)->uiPrevBlkAddr = uiBlkAddr; + m_bDirty = TRUE; + } + +protected: + + // Variables + + FLMUINT m_uiEntryOvhd; // Overhead in the entry. +}; + + +/*=========================================================================== + Fixed Length B-tree Leaf - may be a root +===========================================================================*/ + +class FBtreeLeaf : public FBtreeBlk +{ +public: + + FBtreeLeaf() + { + m_eBlkType = ACCESS_BTREE_LEAF; + m_uiEntryOvhd = 0; + } + + ~FBtreeLeaf() + { + } + + RCODE setup( + FLMUINT uiEntrySize); + + FINLINE FLMUINT getTotalEntries( void) + { + return( (FLMUINT)entryCount()); + } + + FINLINE RCODE insert( + void * pvEntry) + { + return( insertEntry( pvEntry, FBTREE_END)); + } + + FINLINE RCODE search( + void * pvEntry, + void * pvFoundEntry = NULL) + { + return( searchEntry( pvEntry, NULL, pvFoundEntry)); + } + + RCODE split( + FBtreeRoot * pNewRoot); + +}; + +typedef struct +{ + FLMUINT uiBlkAddr; + FLMUINT uiLRUValue; + FBtreeBlk * pBlk; // Points to leaf or non-leaf block +} FBTREE_CACHE; + +/*=========================================================================== + Fixed Length B-tree non-leaf block +===========================================================================*/ + +class FBtreeNonLeaf : public FBtreeBlk +{ +public: + + FBtreeNonLeaf() + { + m_eBlkType = ACCESS_BTREE_NON_LEAF; + m_uiEntryOvhd = sizeof( FLMUINT32); + } + + ~FBtreeNonLeaf() + { + } + + RCODE setup( + FLMUINT uiEntrySize); + + FINLINE FLMUINT getTotalEntries( void) + { + return( (FLMUINT) entryCount()); + } + + FINLINE RCODE insert( + void * // pvEntry + ) + { + return( NE_SFLM_OK); + } + + FINLINE RCODE search( + void *, // pvEntry, + void * pvFoundEntry = NULL + ) + { + F_UNREFERENCED_PARM( pvFoundEntry); + flmAssert( 0); + return( NE_SFLM_OK); + } +}; + +/*=========================================================================== + Fixed Length B-tree Non-Leaf Root +===========================================================================*/ +class FBtreeRoot : public FBtreeNonLeaf +{ +public: + FBtreeRoot(); + ~FBtreeRoot(); + + RCODE setup( + FLMUINT uiEntrySize, + char * pszFileName); + + + void closeFile( void); + + FINLINE FLMUINT getTotalEntries( void) + { + return( m_uiTotalEntries); + } + + RCODE insert( + void * pvEntry); + + RCODE newBlk( + FBtreeBlk ** ppBlk, + FBlkTypes eBlkType); + + FINLINE FLMUINT newBlkAddr( void) + { + return( m_uiNewBlkAddr++); + } + + RCODE newCacheBlk( + FLMUINT uiCachePos, + FBtreeBlk ** ppBlk, + FBlkTypes eBlkType); + + RCODE openFile( void); + + RCODE readBlk( + FLMUINT uiBlkAddr, + FBlkTypes eBlkType, + FBtreeBlk ** ppBlk); + + RCODE search( + void * pvEntry, + void * pvFoundEntry = NULL); + + RCODE setupTree( + FLMBYTE * pucMidEntry, // If !NULL entry to insert into root. + FBlkTypes eBlkType, // Leaf or non-leaf + FBtreeBlk ** ppLeftBlk, // (out) + FBtreeBlk ** ppRightBlk); // (out) + + RCODE split( + void * pvCurEntry, + FLMUINT uiCurChildAddr); + + RCODE writeBlk( + FLMUINT uiWritePos); + +private: + + FLMUINT m_uiLevels; // Number of levels in the b-tree + FLMUINT m_uiTotalEntries; // Count of total entries + FLMUINT m_uiNewBlkAddr; // Next new blk addr. + FLMUINT m_uiHighestWrittenBlkAddr; + + IF_FileHdl * m_pFileHdl; // File handle or NULL if not open. + char * m_pszFileName; // File created for result set or default. + + // Cache of 'n' blocks to apply LRU algorithm. + + FLMUINT m_uiLRUCount; // Count to find least rec. used. + FBTREE_CACHE m_CacheBlks[ FBTREE_CACHE_BLKS]; + + // B-tree stack. + FBtreeBlk * m_BTStack[ FBTREE_MAX_LEVELS]; +}; + +#endif // ifndef FRSET_H diff --git a/sql/src/ffilehdr.cpp b/sql/src/ffilehdr.cpp new file mode 100644 index 0000000..2bb481e --- /dev/null +++ b/sql/src/ffilehdr.cpp @@ -0,0 +1,454 @@ +//------------------------------------------------------------------------------ +// Desc: Routines for accessing information in the database header. +// +// Tabs: 3 +// +// Copyright (c) 1995-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: ffilehdr.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC RCODE verifyDbHdr( + SFLM_DB_HDR * pDbHdr); + +/******************************************************************** +Desc: This routine adjusts the block size that is passe in (wBlkSize) + to the nearest valid block size. +*********************************************************************/ +FLMUINT flmAdjustBlkSize( + FLMUINT uiBlkSize) +{ + FLMUINT uiTmpBlkSize; + + uiTmpBlkSize = SFLM_MIN_BLOCK_SIZE; + while (uiBlkSize > uiTmpBlkSize && uiTmpBlkSize < SFLM_MAX_BLOCK_SIZE) + { + uiTmpBlkSize <<= 1; + } + + return( uiTmpBlkSize); +} + +/******************************************************************** +Desc: This routine initializes a SFLM_DB_HDR structure. +*********************************************************************/ +void flmInitDbHdr( + SFLM_CREATE_OPTS * pCreateOpts, + FLMBOOL bCreatingDatabase, + FLMBOOL bTempDb, + SFLM_DB_HDR * pDbHdr) +{ + FLMUINT uiMinRflFileSize; + FLMUINT uiMaxRflFileSize; + + if (bCreatingDatabase) + { + f_memset( pDbHdr, 0, sizeof( SFLM_DB_HDR)); + } + + // If pCreateOpts is non-NULL, copy it into the file header. + + f_strcpy( (char *)pDbHdr->szSignature, SFLM_DB_SIGNATURE); + pDbHdr->ui8IsLittleEndian = SFLM_NATIVE_IS_LITTLE_ENDIAN; + + if (pCreateOpts) + { + pDbHdr->ui16BlockSize = (FLMUINT16)pCreateOpts->uiBlockSize; + pDbHdr->ui8DefaultLanguage = (FLMUINT8)pCreateOpts->uiDefaultLanguage; + if (pCreateOpts->bKeepRflFiles) + { + pDbHdr->ui8RflKeepFiles = 1; + } + if (pCreateOpts->bLogAbortedTransToRfl) + { + pDbHdr->ui8RflKeepAbortedTrans = 1; + } + + if( (uiMinRflFileSize = pCreateOpts->uiMinRflFileSize) == 0) + { + uiMinRflFileSize = SFLM_DEFAULT_MIN_RFL_FILE_SIZE; + } + + if( (uiMaxRflFileSize = pCreateOpts->uiMaxRflFileSize) == 0) + { + uiMaxRflFileSize = SFLM_DEFAULT_MAX_RFL_FILE_SIZE; + } + } + else + { + + // If pCreateOpts is NULL, initialize some default values. + + pDbHdr->ui16BlockSize = SFLM_DEFAULT_BLKSIZ; + pDbHdr->ui8DefaultLanguage = SFLM_DEFAULT_LANG; + uiMinRflFileSize = SFLM_DEFAULT_MIN_RFL_FILE_SIZE; + uiMaxRflFileSize = SFLM_DEFAULT_MAX_RFL_FILE_SIZE; + } + + // Make sure the RFL size limits are valid. + // Maximum must be enough to hold at least one packet plus + // the RFL header. Minimum must not be greater than the + // maximum. NOTE: Minimum and maximum are allowed to be + // equal, but in all cases, maximum takes precedence over + // minimum. We will first NOT exceed the maximum. Then, + // if possible, we will go above the minimum. + + if (uiMaxRflFileSize < RFL_MAX_PACKET_SIZE + 512) + { + uiMaxRflFileSize = RFL_MAX_PACKET_SIZE + 512; + } + if (uiMaxRflFileSize > gv_SFlmSysData.uiMaxFileSize) + { + uiMaxRflFileSize = gv_SFlmSysData.uiMaxFileSize; + } + if (uiMinRflFileSize > uiMaxRflFileSize) + { + uiMinRflFileSize = uiMaxRflFileSize; + } + pDbHdr->ui32RflMinFileSize = (FLMUINT32)uiMinRflFileSize; + pDbHdr->ui32RflMaxFileSize = (FLMUINT32)uiMaxRflFileSize; + + // Only allow database to be created with current version number + + pDbHdr->ui32DbVersion = SFLM_CURRENT_VERSION_NUM; + pDbHdr->ui8BlkChkSummingEnabled = 1; + + // Round block size up to nearest legal block size. + + pDbHdr->ui16BlockSize = + (FLMUINT16)flmAdjustBlkSize( (FLMUINT)pDbHdr->ui16BlockSize); + + if (!bTempDb) + { + pDbHdr->ui32FirstLFBlkAddr = (FLMUINT32)FSBlkAddress(1, 0); + } + + // If creating a database, initialize some more items. + + if (bCreatingDatabase) + { + + // Set the logical EOF. + + if (!bTempDb) + { + pDbHdr->ui32LogicalEOF = pDbHdr->ui32FirstLFBlkAddr + + (FLMUINT32)pDbHdr->ui16BlockSize; + } + else + { + pDbHdr->ui32LogicalEOF = (FLMUINT32)FSBlkAddress(1, 0); + } + pDbHdr->ui64CurrTransID = (FLMUINT64)0; + pDbHdr->ui32RflCurrFileNum = 1; + + // Putting a zero in this value tells the RFL code that the + // RFL file should be created - overwriting it if it already + // exists. + + pDbHdr->ui32RflLastCPFileNum = 1; + pDbHdr->ui32RflLastCPOffset = 512; + pDbHdr->ui32RblEOF = pDbHdr->ui16BlockSize; + + // Set the database serial number + + f_createSerialNumber( pDbHdr->ucDbSerialNum); + + // Set the "current" RFL serial number - will be stamped into the RFL + // file when it is first created. + + f_createSerialNumber( pDbHdr->ucLastTransRflSerialNum); + + // Set the "next" RFL serial number + + f_createSerialNumber( pDbHdr->ucNextRflSerialNum); + + // Set the incremental backup serial number and sequence number + + f_createSerialNumber( pDbHdr->ucIncBackupSerialNum); + pDbHdr->ui32IncBackupSeqNum = 1; + + // Set the file size limits + + pDbHdr->ui32MaxFileSize = (FLMUINT32)gv_SFlmSysData.uiMaxFileSize; + + // Need at least two blocks to create a database! + + flmAssert( (FLMUINT)pDbHdr->ui32MaxFileSize >= + (FLMUINT)pDbHdr->ui16BlockSize * 2); + } +} + +/*************************************************************************** +Desc: This routine changes the endian-ness of the passed in 64 bit number. + If it was big-endian, it will be converted to little-endian and + vice versa. +*****************************************************************************/ +void convert64( + FLMUINT64 * pui64Num + ) +{ + FLMBYTE * pucTmp = (FLMBYTE *)pui64Num; + FLMBYTE ucTmp; + + // Swap bytes 0 and 7 + + ucTmp = pucTmp [0]; + pucTmp [0] = pucTmp [7]; + pucTmp [7] = ucTmp; + + // Swap bytes 1 and 6 + + ucTmp = pucTmp [1]; + pucTmp [1] = pucTmp [6]; + pucTmp [6] = ucTmp; + + // Swap bytes 2 and 5 + + ucTmp = pucTmp [2]; + pucTmp [2] = pucTmp [5]; + pucTmp [5] = ucTmp; + + // Swap bytes 3 and 4 + + ucTmp = pucTmp [3]; + pucTmp [3] = pucTmp [4]; + pucTmp [4] = ucTmp; +} + +/*************************************************************************** +Desc: This routine changes the endian-ness of the passed in 32 bit number. + If it was big-endian, it will be converted to little-endian and + vice versa. +*****************************************************************************/ +void convert32( + FLMUINT32 * pui32Num + ) +{ + FLMBYTE * pucTmp = (FLMBYTE *)pui32Num; + FLMBYTE ucTmp; + + // Swap bytes 0 and 3 + + ucTmp = pucTmp [0]; + pucTmp [0] = pucTmp [3]; + pucTmp [3] = ucTmp; + + // Swap bytes 1 and 2 + + ucTmp = pucTmp [1]; + pucTmp [1] = pucTmp [2]; + pucTmp [2] = ucTmp; +} + +/*************************************************************************** +Desc: This routine changes the endian-ness of the passed in 16 bit number. + If it was big-endian, it will be converted to little-endian and + vice versa. +*****************************************************************************/ +void convert16( + FLMUINT16 * pui16Num + ) +{ + FLMBYTE * pucTmp = (FLMBYTE *)pui16Num; + FLMBYTE ucTmp; + + // Swap bytes 0 and 1 + + ucTmp = pucTmp [0]; + pucTmp [0] = pucTmp [1]; + pucTmp [1] = ucTmp; +} + +/*************************************************************************** +Desc: This routine changed a database header to native platform format. +*****************************************************************************/ +void convertDbHdr( + SFLM_DB_HDR * pDbHdr + ) +{ + + // This routine should only be called to convert a header to native + // format. + + flmAssert( hdrIsNonNativeFormat( pDbHdr)); + convert16( &pDbHdr->ui16BlockSize); + convert32( &pDbHdr->ui32DbVersion); + convert64( &pDbHdr->ui64LastRflCommitID); + convert32( &pDbHdr->ui32RflLastFileNumDeleted); + convert32( &pDbHdr->ui32RflCurrFileNum); + convert32( &pDbHdr->ui32RflLastTransOffset); + convert32( &pDbHdr->ui32RflLastCPFileNum); + convert32( &pDbHdr->ui32RflLastCPOffset); + convert64( &pDbHdr->ui64RflLastCPTransID); + convert32( &pDbHdr->ui32RflMinFileSize); + convert32( &pDbHdr->ui32RflMaxFileSize); + convert64( &pDbHdr->ui64CurrTransID); + convert64( &pDbHdr->ui64TransCommitCnt); + convert32( &pDbHdr->ui32RblEOF); + convert32( &pDbHdr->ui32RblFirstCPBlkAddr); + convert32( &pDbHdr->ui32FirstAvailBlkAddr); + convert32( &pDbHdr->ui32FirstLFBlkAddr); + convert32( &pDbHdr->ui32LogicalEOF); + convert32( &pDbHdr->ui32MaxFileSize); + convert64( &pDbHdr->ui64LastBackupTransID); + convert32( &pDbHdr->ui32IncBackupSeqNum); + convert32( &pDbHdr->ui32BlksChangedSinceBackup); + convert32( &pDbHdr->ui32HdrCRC); + pDbHdr->ui8IsLittleEndian = SFLM_NATIVE_IS_LITTLE_ENDIAN; +} + +/*************************************************************************** +Desc: This routine verifies that the header is a real FLAIM header. + It will also change the endian-ness if need be. This should always + be called immediately after reading a header from disk. +*****************************************************************************/ +FSTATIC RCODE verifyDbHdr( + SFLM_DB_HDR * pDbHdr + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLen; + FLMUINT32 ui32CRC; + + // Calculate the checksum before doing any conversions. + + ui32CRC = calcDbHdrCRC( pDbHdr); + + // Convert the header to native platform format if necessary. + + if (hdrIsNonNativeFormat( pDbHdr)) + { + convertDbHdr( pDbHdr); + } + + // Check the signature. + + uiLen = f_strlen( SFLM_DB_SIGNATURE); + if (f_memcmp( pDbHdr->szSignature, SFLM_DB_SIGNATURE, uiLen) != 0) + { + rc = RC_SET( NE_SFLM_NOT_FLAIM); + goto Exit; + } + + // See if the database version is OK. + + switch (pDbHdr->ui32DbVersion) + { + default: + if (pDbHdr->ui32DbVersion > SFLM_CURRENT_VERSION_NUM) + { + rc = RC_SET( NE_SFLM_NEWER_FLAIM); + } + else + { + rc = RC_SET( NE_SFLM_UNSUPPORTED_VERSION); + } + goto Exit; + } + + // Validate the checksum + + if (ui32CRC != pDbHdr->ui32HdrCRC) + { + rc = RC_SET( NE_SFLM_HDR_CRC); + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: This routine reads and verifies the information contained in the + file header and log header of a FLAIM database. +*****************************************************************************/ +RCODE flmReadAndVerifyHdrInfo( + SFLM_DB_STATS * pDbStats, + IF_FileHdl * pFileHdl, + SFLM_DB_HDR * pDbHdr, + FLMUINT32 * pui32CalcCRC) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesRead; + + // Read the database header. + + f_memset( pDbHdr, 0, sizeof( SFLM_DB_HDR)); + if (RC_BAD( rc = pFileHdl->read( (FLMUINT)0, sizeof( SFLM_DB_HDR), + pDbHdr, &uiBytesRead))) + { + if (rc != NE_FLM_IO_END_OF_FILE) + { + if (pDbStats) + { + pDbStats->uiReadErrors++; + } + } + else + { + if (pui32CalcCRC) + { + *pui32CalcCRC = calcDbHdrCRC( pDbHdr); + } + + // Get what we can out of the header. + + if (hdrIsNonNativeFormat( pDbHdr)) + { + convertDbHdr( pDbHdr); + } + } + goto Exit; + } + + if (pui32CalcCRC) + { + *pui32CalcCRC = calcDbHdrCRC( pDbHdr); + } + if (uiBytesRead < sizeof( SFLM_DB_HDR)) + { + + // Still get what we can out of the header. + + if (hdrIsNonNativeFormat( pDbHdr)) + { + convertDbHdr( pDbHdr); + } + rc = RC_SET( NE_SFLM_NOT_FLAIM); + goto Exit; + } + else + { + + // This routine will convert to native format if it is not + // in native format. + + if (RC_BAD( rc = verifyDbHdr( pDbHdr))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} diff --git a/sql/src/filesys.h b/sql/src/filesys.h new file mode 100644 index 0000000..217eec2 --- /dev/null +++ b/sql/src/filesys.h @@ -0,0 +1,411 @@ +//------------------------------------------------------------------------------ +// Desc: Various macros, prototypes, structures. +// +// Tabs: 3 +// +// Copyright (c) 1990-1993, 1995-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: filesys.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FILESYS_H +#define FILESYS_H + +/*************************************************************** +** +** Defined Constants that the File system cares about +** +****************************************************************/ + +#define MAX_DATA_BLOCK_FILE_NUMBER 0x7FF +#define FIRST_LOG_BLOCK_FILE_NUMBER (MAX_DATA_BLOCK_FILE_NUMBER + 1) +#define MAX_LOG_BLOCK_FILE_NUMBER 0xFFF + +#define FSGetFileNumber( uiBlkAddr) ((uiBlkAddr) & MAX_LOG_BLOCK_FILE_NUMBER) +#define FSGetFileOffset( udBlkAddr) ((udBlkAddr) & 0xFFFFF000) +#define FSBlkAddress( iFileNum, udFileOfs) ((udFileOfs) + (iFileNum)) + +// Max file size and log threshold. + +#define LOG_THRESHOLD_SIZE ((FLMUINT) 0x40000) + +// very large threshhold is the size we will allow the physical +// log to grow to before we force a truncation. At the low end, +// it is about 10 megabytes. At the high end it is about +// 1 gigabyte. + +#define LOW_VERY_LARGE_LOG_THRESHOLD_SIZE ((FLMUINT)0xA00000) +#define HIGH_VERY_LARGE_LOG_THRESHOLD_SIZE ((FLMUINT) 0x40000000) + +// RFL_TRUNCATE_SIZE is the size we will let an RFL file grow to +// before we truncate it back. RFL files are only truncated if +// we are configured to delete old RFL files. + +#define RFL_TRUNCATE_SIZE ((FLMUINT)1024 * (FLMUINT)1024 * (FLMUINT)10) + +/*============================================================================ + Shared Cache Routines +============================================================================*/ + +RCODE flmPrepareBlockForUse( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr); + +RCODE flmPrepareBlockToWrite( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr); + +void ScaUseCache( + F_CachedBlock * pSCache, + FLMBOOL bMutexAlreadyLocked); + +void ScaReleaseCache( + F_CachedBlock * pSCache, + FLMBOOL bMutexAlreadyLocked); + +/*============================================================================ + File system Btree Cache Routines +============================================================================*/ + +FLMUINT SENNextVal( + FLMBYTE ** senPtrRV ); + +FLMUINT FSGetDomain( + FLMBYTE ** curElmRV, + FLMUINT uiElmOvhd); + +RCODE dbLock( + F_Db * pDb, + FLMUINT uiMaxLockWait ); + +RCODE dbUnlock( + F_Db * pDb); + +FINLINE void FSLFileIn( + FLMBYTE * pucBuf, + LFILE * pLFile, + FLMUINT uiBlkAddress, + FLMUINT uiOffsetInBlk) +{ + F_LF_HDR * pLfHdr = (F_LF_HDR *)pucBuf; + + pLFile->uiBlkAddress = uiBlkAddress; + pLFile->uiOffsetInBlk= uiOffsetInBlk; + + if ((pLFile->eLfType = (eLFileType)pLfHdr->ui32LfType) != SFLM_LF_INVALID) + { + pLFile->uiLfNum = (FLMUINT)pLfHdr->ui32LfNum; + pLFile->uiEncDefNum = (FLMUINT)pLfHdr->ui32EncDefNum; + pLFile->uiRootBlk = (FLMUINT)pLfHdr->ui32RootBlkAddr; + + if (pLFile->eLfType == SFLM_LF_TABLE) + { + pLFile->ui64NextRowId = pLfHdr->ui64NextRowId; + } + else + { + flmAssert( pLFile->eLfType == SFLM_LF_INDEX); + pLFile->ui64NextRowId = 0; + } + } + pLFile->bNeedToWriteOut = FALSE; +} + +void lgSetSyncCheckpoint( + F_Database * pDatabase, + FLMUINT uiCheckpoint, + FLMUINT uiBlkAddress); + +FLMUINT32 calcBlkCRC( + F_BLK_HDR * pBlkHdr, + FLMUINT uiBlkEnd); + +FLMUINT32 calcDbHdrCRC( + SFLM_DB_HDR * pDbHdr); + +FLMUINT flmAdjustBlkSize( + FLMUINT uiBlkSize); + +void flmInitDbHdr( + SFLM_CREATE_OPTS * pCreateOpts, + FLMBOOL bCreatingDatabase, + FLMBOOL bTempDb, + SFLM_DB_HDR * pDbHdr); + +RCODE flmCreateLckFile( + const char * pszFilePath, + IF_FileHdl ** ppLockFileHdl); + +RCODE flmAllocHashTbl( + FLMUINT uiHashTblSize, + FBUCKET ** ppHashTblRV); + +RCODE flmWaitNotifyReq( + F_MUTEX hMutex, + F_SEM hSem, + FNOTIFY ** ppNotifyListRV, + void * pvUserData); + +RCODE flmReadAndVerifyHdrInfo( + SFLM_DB_STATS * pDbStats, + IF_FileHdl * pFileHdl, + SFLM_DB_HDR * pDbHdr, + FLMUINT32 * pui32CalcCRC = NULL); + +void flmDoEventCallback( + eEventCategory eCategory, + eEventType eEvent, + F_Db * pDb, + FLMUINT uiThreadId, + FLMUINT64 ui64TransID, + FLMUINT uiIndexOrTable, + FLMUINT64 ui64RowId, + RCODE rc); + +void flmLogError( + RCODE rc, + const char * pszDoing, + const char * pszFileName = NULL, + FLMINT iLineNumber = 0); + +RCODE flmCollation2Number( + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT64 * pui64Num, + FLMBOOL * pbNeg, + FLMUINT * puiBytesProcessed); + +RCODE flmStorageNum2CollationNum( + const FLMBYTE * pucStorageBuf, + FLMUINT uiStorageLen, + FLMBYTE * pucCollBuf, + FLMUINT * puiCollLen); + +RCODE flmCollationNum2StorageNum( + const FLMBYTE * pucCollBuf, + FLMUINT uiCollLen, + FLMBYTE * pucStorageBuf, + FLMUINT * puiStorageLen); + +RCODE flmStorageNum2StorageText( + const FLMBYTE * pucNum, + FLMUINT uiNumLen, + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen); + +RCODE flmStorageNumberToNumber( + const FLMBYTE * pucNumBuf, + FLMUINT uiNumBufLen, + FLMUINT64 * pui64Number, + FLMBOOL * pbNeg); + +RCODE KYCollateValue( + FLMBYTE * pucDest, + FLMUINT * puiDestLen, + IF_PosIStream * pIStream, + FLMUINT uiDataType, + FLMUINT uiFlags, + FLMUINT uiCompareRules, + FLMUINT uiLimit, + FLMUINT * puiCollationLen, + FLMUINT * puiLuLen, + FLMUINT uiLanguage, + FLMBOOL bFirstSubstring, + FLMBOOL bDataTruncated, + FLMBOOL * pbDataTruncated, + FLMBOOL * pbOriginalCharsLost); + +#define UNDF_CHR 0x0000 // Undefined char - ignore for now +#define IGNR_CHR 0x0001 // Ignore this char +#define SDWD_CHR 0x0002 // Space delimited word chr +#define DELI_CHR 0x0040 // Delimiter +#define WDJN_CHR 0x0080 // Word Joining chr ".,/-_" + +// Implement later + +#define KATA_CHR 0x0004 // Katakana word chr +#define HANG_CHR 0x0008 // Hangul word chr +#define CJK_CHR 0x0010 // CJK word chr + +RCODE KYSubstringParse( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUINT uiLimit, + FLMBYTE * pucSubstrBuf, + FLMUINT * puiSubstrBytes, + FLMUINT * puiSubstrChars); + +RCODE KYEachWordParse( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUINT uiLimit, + FLMBYTE * pucWordBuf, + FLMUINT * puiWordLen); + +RCODE fdictGetState( + const char * pszState, + FLMUINT * puiState); + +RCODE fdictGetIndexState( + const char * pszState, + FLMUINT * puiState); + +#define FLM_BACKGROUND_LOCK_PRIORITY -100 + +void flmLogIndexingProgress( + FLMUINT uiIndexNum, + FLMUINT64 ui64LastDocumentId); + +F_BKGND_IX * flmBackgroundIndexGet( + F_Database * pDatabase, + FLMUINT uiValue, + FLMBOOL bMutexLocked, + FLMUINT * puiThreadId = NULL); + +RCODE flmGetHdrInfo( + F_SuperFileHdl * pSFileHdl, + SFLM_DB_HDR * pDbHdr, + FLMUINT32 * pui32CalcCRC = NULL); + +void convert64( + FLMUINT64 * pui64Num); + +void convert32( + FLMUINT32 * pui32Num); + +void convert16( + FLMUINT16 * pui16Num); + +void convertDbHdr( + SFLM_DB_HDR * pDbHdr); + +void convertBlkHdr( + F_BLK_HDR * pBlkHdr); + +void convertBlk( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr); + +void convertLfHdr( + F_LF_HDR * pLfHdr); + +/*-------------------------------------------------------- +** Inline Functions +**-------------------------------------------------------*/ + +/************************************************************************** +Desc: Returns TRUE if a block address is less than another block address. +**************************************************************************/ +FINLINE FLMBOOL FSAddrIsBelow( + FLMUINT uiAddress1, + FLMUINT uiAddress2) +{ + if( FSGetFileNumber( uiAddress1) == FSGetFileNumber( uiAddress2)) + { + if( FSGetFileOffset( uiAddress1) >= FSGetFileOffset( uiAddress2)) + { + return FALSE; + } + } + else if( FSGetFileNumber( uiAddress1) > FSGetFileNumber( uiAddress2)) + { + return FALSE; + } + + return TRUE; +} + +/************************************************************************** +Desc: Returns TRUE if a block address is less than or equal another + block address. +**************************************************************************/ +FINLINE FLMBOOL FSAddrIsAtOrBelow( + FLMUINT uiAddress1, + FLMUINT uiAddress2) +{ + if( FSGetFileNumber( uiAddress1) == FSGetFileNumber( uiAddress2)) + { + if( FSGetFileOffset( uiAddress1) > FSGetFileOffset( uiAddress2)) + { + return FALSE; + } + } + else if( FSGetFileNumber( uiAddress1) > FSGetFileNumber( uiAddress2)) + { + return FALSE; + } + return TRUE; +} + +/**************************************************************************** +Desc: Get the total bytes represented by a particular block address. +****************************************************************************/ +FINLINE FLMUINT64 FSGetSizeInBytes( + FLMUINT uiMaxFileSize, + FLMUINT uiBlkAddress) +{ + FLMUINT uiFileNum; + FLMUINT uiFileOffset; + FLMUINT64 ui64Size; + + uiFileNum = FSGetFileNumber( uiBlkAddress); + uiFileOffset = FSGetFileOffset( uiBlkAddress); + if( uiFileNum > 1) + { + ui64Size = (FLMUINT64)(((FLMUINT64)uiFileNum - (FLMUINT64)1) * + (FLMUINT64)uiMaxFileSize + + (FLMUINT64)uiFileOffset); + } + else + { + ui64Size = (FLMUINT64)uiFileOffset; + } + return( ui64Size); +} + +/************************************************************************** +Desc: Calculate the significant bits in a block size. +**************************************************************************/ +FINLINE FLMUINT calcSigBits( + FLMUINT uiBlockSize + ) +{ + FLMUINT uiSigBitsInBlkSize = 0; + while (!(uiBlockSize & 1)) + { + uiSigBitsInBlkSize++; + uiBlockSize >>= 1; + } + return( uiSigBitsInBlkSize); +} + +/************************************************************************** +Desc: Outputs an update event callback. +**************************************************************************/ +FINLINE void flmTransEventCallback( + eEventType eEventType, + F_Db * pDb, + RCODE rc, + FLMUINT64 ui64TransId) +{ + flmDoEventCallback( SFLM_EVENT_UPDATES, eEventType, pDb, + f_threadId(), ui64TransId, 0, 0, rc); +} + +#endif // FILESYS_H + diff --git a/sql/src/flaimodbc.h b/sql/src/flaimodbc.h new file mode 100644 index 0000000..1349511 --- /dev/null +++ b/sql/src/flaimodbc.h @@ -0,0 +1,442 @@ +//------------------------------------------------------------------------- +// Desc: Structures, classes, prototypes, and defines needed to support +// ODBC in FLAIM. +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------- + +#ifndef FLAIMODBC_H +#define FLAIMODBC_H + +#include "fdynsset.h" + +// Forward declarations + +struct SQL_NODE; +struct SQL_VALUE; +struct SQL_TABLE; +struct SQL_INDEX; +struct SQL_KEY; +struct SQL_COLUMN; +class FSTableCursor; + +//------------------------------------------------------------------------- +// Desc: Types of nodes in SQL query. +//------------------------------------------------------------------------- +typedef enum +{ + SQL_OPERATOR_NODE = 0, + SQL_VALUE_NODE, + SQL_COLUMN_NODE, + SQL_PRED_NODE +} eSQLNodeTypes; + +#define SQL_FIRST_ARITH_OP SQL_BITAND_OP +#define SQL_LAST_ARITH_OP SQL_NEG_OP + +FINLINE FLMBOOL isLegalSQLOperator( + eSQLQueryOperators eOperator) +{ + return( (eOperator >= SQL_AND_OP && eOperator <= SQL_NEG_OP) + ? TRUE + : FALSE); +} + +FINLINE FLMBOOL isSQLLogicalOp( + eSQLQueryOperators eOperator) +{ + return( (eOperator >= SQL_AND_OP && eOperator <= SQL_NOT_OP) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isSQLCompareOp( + eSQLQueryOperators eOperator) +{ + return( (eOperator >= SQL_EQ_OP && eOperator <= SQL_GE_OP) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isSQLArithOp( + eSQLQueryOperators eOperator) +{ + return( (eOperator >= SQL_FIRST_ARITH_OP && + eOperator <= SQL_LAST_ARITH_OP) ? TRUE : FALSE); +} + +FINLINE FLMBOOL isSQLValUnsigned( + eSQLValTypes eValType) +{ + return( eValType == SQL_UINT_VAL || eValType == SQL_UINT64_VAL + ? TRUE + : FALSE); +} + +FINLINE FLMBOOL isSQLValSigned( + eSQLValTypes eValType) +{ + return( eValType == SQL_INT_VAL || eValType == SQL_INT64_VAL + ? TRUE + : FALSE); +} + +FINLINE FLMBOOL isSQLVal64Bit( + eSQLValTypes eValType) +{ + return( eValType == SQL_UINT64_VAL || eValType == SQL_INT64_VAL + ? TRUE + : FALSE); +} + +FINLINE FLMBOOL isSQLValNativeNum( + eSQLValTypes eValType) +{ + return( eValType == SQL_UINT_VAL || eValType == SQL_INT_VAL + ? TRUE + : FALSE); +} + +typedef struct SQL_PRED +{ + SQL_TABLE * pTable; + FLMUINT uiColumnNum; + eSQLQueryOperators eOperator; // Operator of the predicate + FLMUINT uiCompareRules;// Comparison rules + FLMBOOL bNotted; // Has operator been notted? + SQL_VALUE * pFromValue; // Points to SQL_VALUE that has the FROM value for + // this predicate. Will be NULL for unary + // operators such as exists + FLMBOOL bInclFrom; // Flag indicating if the from value is + // inclusive. + SQL_VALUE * pUntilValue; // Points to SQL_VALUE that has the UNTIL value + // for this predicate. + FLMBOOL bInclUntil; // Flag indicating if until value is + // inclusive. + SQL_PRED * pNext; +} SQL_PRED; + +typedef struct SQL_OP +{ + eSQLQueryOperators eOperator; + FLMUINT uiCompareRules; +} SQL_OP; + +typedef struct SQL_VALUE +{ + eSQLValTypes eValType; + FLMUINT uiFlags; +#define SQL_VAL_IS_STREAM 0x0001 +#define SQL_VAL_IS_CONSTANT 0x0002 // During query evaluation, this indicates + // that this value is a constant. If it + // is a FLM_UTF8_VAL, then asterisks will + // be treated as a wildcard, unless + // escaped (\*). If the value is NOT + // a constant, the asterisk is NEVER + // treated as a wildcard, and the + // backslash is NEVER treated as an + // escape character. +#define SQL_VAL_HAS_WILDCARDS 0x0004 // This is only set if the value is a + // constant, FLM_UTF8_VAL, that has + // wildcards. + FLMUINT uiDataLen; // Length in bytes if the type is text + // or binary + union + { + SQLBoolType eBool; + FLMUINT uiVal; + FLMUINT64 ui64Val; + FLMINT iVal; + FLMINT64 i64Val; + FLMBYTE * pucBuf; + IF_PosIStream * pIStream; + } val; // Holds or points to the atom value. +} SQL_VALUE; + +/*************************************************************************** +Desc: Can two values be compared? +***************************************************************************/ +FINLINE FLMBOOL sqlCanCompare( + SQL_VALUE * pValue1, + SQL_VALUE * pValue2 + ) +{ + if (!pValue1 || !pValue2 || + pValue1->eValType == pValue2->eValType) + { + return( TRUE); + } + else + { + switch (pValue1->eValType) + { + case SQL_UINT_VAL: + case SQL_UINT64_VAL: + case SQL_INT_VAL: + case SQL_INT64_VAL: + return( (FLMBOOL)(pValue2->eValType == SQL_UINT_VAL || + pValue2->eValType == SQL_UINT64_VAL || + pValue2->eValType == SQL_INT_VAL || + pValue2->eValType == SQL_INT64_VAL + ? TRUE + : FALSE)); + default: + return( FALSE); + } + } +} + +typedef struct SQL_KEY +{ + SQL_INDEX * pIndex; + FLMUINT uiComponentsUsed; + SQL_PRED ** ppKeyComponents; + SQL_KEY * pNext; + SQL_KEY * pPrev; +} SQL_KEY; + +typedef struct SQL_INDEX +{ + FLMUINT uiIndexNum; + FLMUINT uiNumComponents; + SQL_TABLE * pTable; + SQL_KEY * pFirstKey; + SQL_KEY * pLastKey; + SQL_INDEX * pNext; + SQL_INDEX * pPrev; +} SQL_INDEX; + +typedef struct SQL_TABLE +{ + FLMUINT uiTableNum; + FSTableCursor * pFSTableCursor; + FLMUINT uiCost; + FLMBOOL bScan; + FLMBOOL bScanIndex; + FLMUINT uiIndex; + FLMBOOL bIndexSet; + SQL_INDEX * pFirstIndex; + SQL_INDEX * pLastIndex; + SQL_TABLE * pNext; + SQL_TABLE * pPrev; +} SQL_TABLE; + +typedef struct SQL_COLUMN +{ + SQL_TABLE * pTable; + FLMUINT uiColumnNum; +} SQL_COLUMN; + +typedef struct SQL_NODE +{ + eSQLNodeTypes eNodeType; // Type of node this is + FLMUINT uiNestLevel; // Nesting level of node - only used when + // setting up the query + FLMBOOL bUsedValue; // Used during evaluation + FLMBOOL bLastValue; // Used during evaluation + FLMBOOL bNotted; + SQL_NODE * pParent; // Parent of this query node + SQL_NODE * pPrevSib; // Previous sibling of this query node + SQL_NODE * pNextSib; // Next sibling of this query node + SQL_NODE * pFirstChild; // First child of this query node + SQL_NODE * pLastChild; // Last child of this query node + union + { + SQL_OP op; + SQL_COLUMN column; + SQL_VALUE value; + SQL_PRED pred; + } nd; +} SQL_NODE; + +FINLINE FLMBOOL isSQLNodeBool( + SQL_NODE * pNode + ) +{ + return( (pNode->eNodeType == SQL_VALUE_NODE && + pNode->nd.value.eValType == SQL_BOOL_VAL) ? TRUE : FALSE); +} + +typedef struct SQL_PARSE_STATE +{ + SQL_NODE * pRootNode; + SQL_NODE * pCurOperatorNode; + SQL_NODE * pLastNode; + FLMUINT uiNestLevel; + FLMBOOL bExpectingOperator; + FLMBOOL bExpectingLParen; + SQL_PARSE_STATE * pPrev; + SQL_PARSE_STATE * pNext; +} SQL_PARSE_STATE; + +typedef struct SQL_SUBQUERY +{ + FLMUINT uiOperandCount; + SQL_NODE ** ppOperands; + SQL_SUBQUERY * pNext; + SQL_SUBQUERY * pPrev; +} SQL_SUBQUERY; + +typedef struct SQL_DNF_NODE +{ + SQL_DNF_NODE * pParent; + SQL_DNF_NODE * pFirstChild; + SQL_DNF_NODE * pLastChild; + SQL_DNF_NODE * pNextSib; + SQL_DNF_NODE * pPrevSib; + SQL_NODE * pNode; // If NULL, bAndOp is used to tell if it is an OR or AND operator + FLMBOOL bAndOp; // Only set if pNode is NULL. +} SQL_DNF_NODE; + +//------------------------------------------------------------------------- +// Desc: SQLQuery class - for building up an SQL query. +//------------------------------------------------------------------------- +class SQLQuery +{ +public: + + SQLQuery(); + + ~SQLQuery(); + +private: + + FINLINE FLMBOOL expectingOperand( void) + { + return( !m_pCurrParseState->bExpectingOperator); + } + + FINLINE FLMBOOL expectingOperator( void) + { + return( m_pCurrParseState->bExpectingOperator); + } + RCODE allocParseState( void); + + RCODE allocValueNode( + FLMUINT uiValLen, + eSQLValTypes eValType, + SQL_NODE ** ppSQLNode); + + RCODE addOperator( + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules); + + RCODE addColumn( + FLMUINT uiTableNum, + FLMUINT uiColumnNum); + + RCODE intersectPredicates( + SQL_PRED * pPred, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue, + FLMBOOL * pbAlwaysFalse, + FLMBOOL * pbIntersected); + + RCODE unionPredicates( + SQL_PRED * pPred, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue, + FLMBOOL * pbUnioned); + + RCODE addPredicate( + SQL_SUBQUERY * pSubQuery, + FLMUINT * puiOperand, + SQL_TABLE * pTable, + FLMUINT uiColumnNum, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue); + + RCODE convertOperandsToPredicates( void); + + RCODE flattenTree( void); + + RCODE convertToDNF( void); + + RCODE getPredKeys( + SQL_PRED * pPred, + SQL_TABLE * pTable); + + RCODE chooseBestIndex( + SQL_TABLE * pTable, + FLMUINT * puiCost); + + RCODE calcTableScanCost( + SQL_TABLE * pTable, + FLMUINT64 * pui64Cost, + FSTableCursor ** ppFSTableCursor); + + RCODE mergeKeys( + SQL_TABLE * pDestTable, + SQL_TABLE * pSrcTable); + + RCODE optimizeTable( + SQL_SUBQUERY * pSubQuery, + SQL_TABLE * pTable); + + RCODE optimizeSubQueries( void); + + RCODE optimize( void); + + F_Pool m_pool; + FLMUINT m_uiLanguage; + SQL_PARSE_STATE * m_pCurrParseState; + SQL_SUBQUERY * m_pFirstSubQuery; + SQL_SUBQUERY * m_pLastSubQuery; + SQL_TABLE * m_pFirstTable; + SQL_TABLE * m_pLastTable; + FLMBOOL m_bOptimized; + SQL_NODE * m_pQuery; + F_Database * m_pDatabase; + F_Db * m_pDb; + FLMBOOL m_bScan; + FLMBOOL m_bScanIndex; + FLMUINT m_uiIndexNum; + FLMBOOL m_bIndexSet; + FLMBOOL m_bEmpty; + SQLQuery * m_pNext; + SQLQuery * m_pPrev; +friend class F_Db; +friend class F_Database; +}; + +typedef struct SQL_KEYPOS +{ + FLMBYTE ucKey [SFLM_MAX_KEY_SIZE]; + FLMUINT uiKeyLen; +} SQL_KEYPOS; + + +RCODE sqlCompare( // sqleval.cpp + SQL_VALUE * pValue1, + SQL_VALUE * pValue2, + FLMUINT uiCompareRules, + FLMUINT uiLanguage, + FLMINT * piCmp); + +RCODE sqlEvalArithOperator( // sqleval.cpp + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + eSQLQueryOperators eOperator, + SQL_VALUE * pResult); + +#endif // #ifndef FLAIMODBC_H diff --git a/sql/src/flaimsql.h b/sql/src/flaimsql.h new file mode 100644 index 0000000..9dba324 --- /dev/null +++ b/sql/src/flaimsql.h @@ -0,0 +1,1710 @@ +//------------------------------------------------------------------------------ +// Desc: FLAIM SQL public definitions and interfaces +// +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +/// \file + +#ifndef FLAIMSQL_H +#define FLAIMSQL_H + +/// \defgroup errors Error Codes + + /// \defgroup ftk_errors Toolkit Error Codes + /// \ingroup errors + + /// \defgroup flaim_errors FLAIM Error Codes + /// \ingroup errors + + +#ifdef HAVE_CONFIG_H + #include "config.h" +#endif + +#include "ftk.h" + +// Forward declarations + +class F_Db; +class F_Row; + +/// Create options for creating a database. +typedef struct SFLM_CREATE_OPTS +{ + FLMUINT uiBlockSize; ///< Size of blocks in the database. +#define SFLM_DEFAULT_BLKSIZ 4096 + + FLMUINT uiVersionNum; ///< Database version number. +#define SFLM_VER_6_00 600 +#define SFLM_CURRENT_VERSION_NUM SFLM_VER_6_00 +#define SFLM_CURRENT_VER_STR "6.00" + + FLMUINT uiMinRflFileSize; ///< Minimum bytes per RFL file. +#define SFLM_DEFAULT_MIN_RFL_FILE_SIZE ((FLMUINT)100 * (FLMUINT)1024 * (FLMUINT)1024) + FLMUINT uiMaxRflFileSize; ///< Maximum bytes per RFL file. +#define SFLM_DEFAULT_MAX_RFL_FILE_SIZE SFLM_MAXIMUM_FILE_SIZE + FLMBOOL bKeepRflFiles; ///< Keep RFL files? +#define SFLM_DEFAULT_KEEP_RFL_FILES_FLAG FALSE + FLMBOOL bLogAbortedTransToRfl; ///< Log aborted transactions to RFL? +#define SFLM_DEFAULT_LOG_ABORTED_TRANS_FLAG FALSE + + FLMUINT uiDefaultLanguage; ///< Default language for database.\ See \ref flm_languages "here" for + ///< valid languages. +#define SFLM_DEFAULT_LANG FLM_US_LANG + +} SFLM_CREATE_OPTS; + +/// Database header. +typedef struct SFLM_DB_HDR +{ + // The following items can only be changed by a convert operation, which + // always converts out to a different file. A convert is NEVER done + // "in-place." Hence, once a database is opened, it can be assumed that + // these items are stable - they will not change. This is an important + // thing for the read transaction, because we do not want to be copying + // this entire structure on read transactions. We only want to do that on + // update transactions. It will be safe for the read transaction to assume + // that its copy of these items inside the FDB structure are current. + + FLMBYTE szSignature[ 8]; // Contains the string "SFLMDB " +#define SFLM_DB_SIGNATURE "SFLMDB " + FLMUINT8 ui8IsLittleEndian; // Non-zero if DB is little-endian +#ifdef FLM_BIG_ENDIAN + #define SFLM_NATIVE_IS_LITTLE_ENDIAN 0 +#else + #define SFLM_NATIVE_IS_LITTLE_ENDIAN 1 +#endif + FLMUINT8 ui8DefaultLanguage; ///< Default language for the database. + FLMUINT16 ui16BlockSize; ///< Database block size. + + // The following items can be changed by an update transaction. + // NOTE: The following items should always be accessed from the database + // object. The database object should have a copy of them that it takes + // at the beginning of the transaction: + // ui64CurrTransID + // ui32LogicalEOF + // ui32FirstAvailBlkAddr + // IMPORTANT NOTE: The following items cannot currently be changed + // during an update transaction: + // ui32DbVersion + // ui16BlockSize + // ui8DefaultLanguage + // ui32MaxFileSize + // ui32FirstLFBlkAddr + // This is because they are always accessed from the "last committed" + // database header, regardless of transaction type. If we ever want to + // change them in an update transaction, we will need to copy them + // into the database object at the beginning of a transaction, and then + // only access them from within the database object. + + FLMUINT32 ui32DbVersion; ///< Database version. + FLMUINT8 ui8BlkChkSummingEnabled; ///< Is block checksumming enabled? + FLMUINT8 ui8RflKeepFiles; ///< Keep RFL files flag - 1=Keep, 0=Dont Keep. + FLMUINT8 ui8RflAutoTurnOffKeep; ///< Automatically turn off keeping of RFL files when RFL disk is full.\ 1=Turn off + ///< 0=Don't automatically turn off. + FLMUINT8 ui8RflKeepAbortedTrans; ///< Keep aborted transactions in RFL.\ 1=Keep, 0=Don't keep. + FLMUINT32 ui32RflCurrFileNum; ///< Current RFL log file number. + FLMUINT64 ui64LastRflCommitID; ///< Transaction ID of last transaction that was committed in RFL. + FLMUINT32 ui32RflLastFileNumDeleted; ///< Last RFL file that was deleted. + FLMUINT32 ui32RflLastTransOffset; ///< Offset in RFL file where the last committed transaction ends. + FLMUINT32 ui32RflLastCPFileNum; ///< RFL file number where last check-pointed transaction ends. + FLMUINT32 ui32RflLastCPOffset; ///< Offset in RFL file where last check-pointed transaction ends. + FLMUINT64 ui64RflLastCPTransID; ///< Transaction Id of the last check-pointed transaction. + FLMUINT32 ui32RflMinFileSize; ///< Minimum RFL file size. + FLMUINT32 ui32RflMaxFileSize; ///< Maximum RFL file size. + FLMUINT64 ui64CurrTransID; ///< Current update transaction ID. + FLMUINT64 ui64TransCommitCnt; ///< Total update transactions that have committed. + FLMUINT32 ui32RblEOF; ///< Offset where the rollback log ends. + FLMUINT32 ui32RblFirstCPBlkAddr; ///< Block address in rollback log of the first block modified after the last checkpoint. + FLMUINT32 ui32FirstAvailBlkAddr; ///< Block address of the first block in the avail list. + FLMUINT32 ui32FirstLFBlkAddr; ///< Block address of the first block in the list of logical file blocks. + FLMUINT32 ui32LogicalEOF; ///< Address where the last block in the database ends. + FLMUINT32 ui32MaxFileSize; ///< Maximum file size for database files. + FLMUINT64 ui64LastBackupTransID; ///< Transaction ID when last backup was taken. + FLMUINT32 ui32IncBackupSeqNum; ///< Sequence number of last incremental backup that was taken. + FLMUINT32 ui32BlksChangedSinceBackup;///< Total blocks that have changed since last backup was taken. + +#define SFLM_SERIAL_NUM_SIZE 16 + + FLMBYTE ucDbSerialNum[ SFLM_SERIAL_NUM_SIZE]; + ///< Database serial number. + FLMBYTE ucLastTransRflSerialNum[ SFLM_SERIAL_NUM_SIZE]; + ///< Serial number of RFL file that holds the last committed transaction. + FLMBYTE ucNextRflSerialNum[ SFLM_SERIAL_NUM_SIZE]; + ///< Serial number for the next RFL file that will be created. + FLMBYTE ucIncBackupSerialNum[ SFLM_SERIAL_NUM_SIZE]; + ///< Serial number for the next incremental backup to be taken. + FLMUINT32 ui32DbKeyLen; ///< Length of database encryption key. + + // IMPORTANT NOTE: If anything is changed in here, need to make + // corresponding changes to convertDbHdr routine and + // flmVerifyDiskStructOffsets routine. + + FLMBYTE ucReserved[ 64]; // Reserved for future + // Always initialized to zero + + // Checksum should ALWAYS be last + + FLMUINT32 ui32HdrCRC; ///< Checksum for database header. + + // Encryption Key stuff + + #define SFLM_MAX_ENC_KEY_SIZE 256 + + FLMBYTE ucDbKey[ SFLM_MAX_ENC_KEY_SIZE]; + ///< Database encryption key. + + // Offsets of variables in the structure + + #define SFLM_DB_HDR_szSignature_OFFSET 0 + #define SFLM_DB_HDR_ui8IsLittleEndian_OFFSET 8 + #define SFLM_DB_HDR_ui8DefaultLanguage_OFFSET 9 + #define SFLM_DB_HDR_ui16BlockSize_OFFSET 10 + #define SFLM_DB_HDR_ui32DbVersion_OFFSET 12 + #define SFLM_DB_HDR_ui8BlkChkSummingEnabled_OFFSET 16 + #define SFLM_DB_HDR_ui8RflKeepFiles_OFFSET 17 + #define SFLM_DB_HDR_ui8RflAutoTurnOffKeep_OFFSET 18 + #define SFLM_DB_HDR_ui8RflKeepAbortedTrans_OFFSET 19 + #define SFLM_DB_HDR_ui32RflCurrFileNum_OFFSET 20 + #define SFLM_DB_HDR_ui64LastRflCommitID_OFFSET 24 + #define SFLM_DB_HDR_ui32RflLastFileNumDeleted_OFFSET 32 + #define SFLM_DB_HDR_ui32RflLastTransOffset_OFFSET 36 + #define SFLM_DB_HDR_ui32RflLastCPFileNum_OFFSET 40 + #define SFLM_DB_HDR_ui32RflLastCPOffset_OFFSET 44 + #define SFLM_DB_HDR_ui64RflLastCPTransID_OFFSET 48 + #define SFLM_DB_HDR_ui32RflMinFileSize_OFFSET 56 + #define SFLM_DB_HDR_ui32RflMaxFileSize_OFFSET 60 + #define SFLM_DB_HDR_ui64CurrTransID_OFFSET 64 + #define SFLM_DB_HDR_ui64TransCommitCnt_OFFSET 72 + #define SFLM_DB_HDR_ui32RblEOF_OFFSET 80 + #define SFLM_DB_HDR_ui32RblFirstCPBlkAddr_OFFSET 84 + #define SFLM_DB_HDR_ui32FirstAvailBlkAddr_OFFSET 88 + #define SFLM_DB_HDR_ui32FirstLFBlkAddr_OFFSET 92 + #define SFLM_DB_HDR_ui32LogicalEOF_OFFSET 96 + #define SFLM_DB_HDR_ui32MaxFileSize_OFFSET 100 + #define SFLM_DB_HDR_ui64LastBackupTransID_OFFSET 104 + #define SFLM_DB_HDR_ui32IncBackupSeqNum_OFFSET 112 + #define SFLM_DB_HDR_ui32BlksChangedSinceBackup_OFFSET 116 + #define SFLM_DB_HDR_ucDbSerialNum_OFFSET 120 + #define SFLM_DB_HDR_ucLastTransRflSerialNum_OFFSET 136 + #define SFLM_DB_HDR_ucNextRflSerialNum_OFFSET 152 + #define SFLM_DB_HDR_ucIncBackupSerialNum_OFFSET 168 + #define SFLM_DB_HDR_ui32DbKeyLen 184 + #define SFLM_DB_HDR_ucReserved_OFFSET 188 + #define SFLM_DB_HDR_ui32HdrCRC_OFFSET 252 + #define SFLM_DB_HDR_ucDbKey 256 +} SFLM_DB_HDR; + +// Flags used by openDb method + +#define SFLM_DONT_REDO_LOG 0x0001 +#define SFLM_DONT_RESUME_THREADS 0x0002 +#define SFLM_ALLOW_LIMITED_MODE 0x0004 + +// Maximum file size + +#define SFLM_MAXIMUM_FILE_SIZE 0xFFFC0000 + +// Maximum key size + +#define SFLM_MAX_KEY_SIZE 1024 + +// Extra retrieval flag - others are defined in ftk.h. Make sure this bit +// doesn't collide with FLM_EXCL, FLM_INCL, FLM_FIRST, FLM_LAST, FLM_EXACT, +// FLM_KEY_EXACT, etc. + +#define FLM_MATCH_ROW_ID 0x0800 + +/// Transaction types +typedef enum +{ + SFLM_NO_TRANS = 0, + SFLM_READ_TRANS, ///< 1 = Read transaction + SFLM_UPDATE_TRANS ///< 2 = Update transaction +} eDbTransType; + +/// Database lock types. +typedef enum +{ + SFLM_LOCK_NONE = 0, + SFLM_LOCK_EXCLUSIVE, ///< 1 = Exclusive lock. + SFLM_LOCK_SHARED ///< 2 = Shared lock. +} eDbLockType; + +// Transaction flags + +#define SFLM_DONT_KILL_TRANS 0x1 +#define SFLM_DONT_POISON_CACHE 0x2 + +// Defines used for uiMaxLockWait parameter + +#define SFLM_NO_TIMEOUT 0xFF + +/// Backup types. +typedef enum +{ + // These values are stored in the header of the + // backup, so do not change their values. + SFLM_FULL_BACKUP = 0, ///< 0 = Full backup. + SFLM_INCREMENTAL_BACKUP ///< 1 = Incremental backup. +} eDbBackupType; + +/// Data types. +typedef enum +{ + SFLM_UNKNOWN_TYPE = 0, + SFLM_STRING_TYPE, ///< 1 = UTF8 string + SFLM_NUMBER_TYPE, ///< 2 = 64 bit integer number + SFLM_BINARY_TYPE, ///< 3 = Binary + SFLM_NUM_OF_TYPES +} eDataType; + +/// Encryption algorithms. +typedef enum +{ + SFLM_AES_ENCRYPTION = 0, ///< 0 = AES encryption + SFLM_DES3_ENCRYPTION ///< 1 = DES3 encryption +} eEncAlgorithm; + +// Supported encryption key sizes + +#define SFLM_AES128_KEY_SIZE 128 +#define SFLM_AES192_KEY_SIZE 192 +#define SFLM_AES256_KEY_SIZE 256 +#define SFLM_DES3_168_KEY_SIZE 168 + +// Maximums for table, index, column, and encryption definition names. + +#define MAX_TABLE_NAME_LEN 256 +#define MAX_INDEX_NAME_LEN 256 +#define MAX_COLUMN_NAME_LEN 256 +#define MAX_ENCDEF_NAME_LEN 256 + +// Predefined table names and numbers + +#define SFLM_TBLNUM_ENCDEFS 1 +#define SFLM_TBLNAM_ENCDEFS "flmtbl_encdefs" +#define SFLM_TBLNUM_TABLES 2 +#define SFLM_TBLNAM_TABLES "flmtbl_tables" +#define SFLM_TBLNUM_COLUMNS 3 +#define SFLM_TBLNAM_COLUMNS "flmtbl_columns" +#define SFLM_TBLNUM_INDEXES 4 +#define SFLM_TBLNAM_INDEXES "flmtbl_indexes" +#define SFLM_TBLNUM_INDEX_COMPONENTS 5 +#define SFLM_TBLNAM_INDEX_COMPONENTS "flmtbl_index_components" +#define SFLM_TBLNUM_BLOCK_CHAINS 6 +#define SFLM_TBLNAM_BLOCK_CHAINS "flmtbl_block_chains" + +// Predefined index names and numbers + +#define SFLM_IXNUM_ENCDEF_NAME 1 +#define SFLM_IXNAM_ENCDEF_NAME "flmix_encdef_name" +#define SFLM_IXNUM_TABLE_NAME 2 +#define SFLM_IXNAM_TABLE_NAME "flmix_table_name" +#define SFLM_IXNUM_TABLE_ENCDEF_NUM 3 +#define SFLM_IXNAM_TABLE_ENCDEF_NUM "flmix_table_encdef_num" +#define SFLM_IXNUM_COLUMN_TABLE_NUM 4 +#define SFLM_IXNAM_COLUMN_TABLE_NUM "flmix_column_table_num" +#define SFLM_IXNUM_COLUMN_ENCDEF_NUM 5 +#define SFLM_IXNAM_COLUMN_ENCDEF_NUM "flmix_column_encdef_num" +#define SFLM_IXNUM_INDEX_NAME 6 +#define SFLM_IXNAM_INDEX_NAME "flmix_index_name" +#define SFLM_IXNUM_INDEX_TABLE_NUM 7 +#define SFLM_IXNAM_INDEX_TABLE_NUM "flmix_index_table_num" +#define SFLM_IXNUM_INDEX_ENCDEF_NUM 8 +#define SFLM_IXNAM_INDEX_ENCDEF_NUM "flmix_index_encdef_num" +#define SFLM_IXNUM_INDEX_COMP_INDEX_NUM 9 +#define SFLM_IXNAM_INDEX_COMP_INDEX_NUM "flmix_index_comp_index_num" + +// Column names and numbers for the encryption definitions table + +#define SFLM_COLNUM_ENCDEFS_ENCDEF_NAME 1 +#define SFLM_COLNAM_ENCDEFS_ENCDEF_NAME "encdef_name" +#define SFLM_COLNUM_ENCDEFS_ENCDEF_NUM 2 +#define SFLM_COLNAM_ENCDEFS_ENCDEF_NUM "encdef_num" +#define SFLM_COLNUM_ENCDEFS_ENC_ALGORITHM 3 +#define SFLM_COLNAM_ENCDEFS_ENC_ALGORITHM "enc_algorithm" +#define SFLM_COLNUM_ENCDEFS_ENC_KEY_SIZE 4 +#define SFLM_COLNAM_ENCDEFS_ENC_KEY_SIZE "enc_key_size" +#define SFLM_COLNUM_ENCDEFS_ENC_KEY 5 +#define SFLM_COLNAM_ENCDEFS_ENC_KEY "enc_key" +#define SFLM_ENCDEFS_NUM_COLUMNS 5 + +// Column names and numbers for the tables table + +#define SFLM_COLNUM_TABLES_TABLE_NAME 1 +#define SFLM_COLNAM_TABLES_TABLE_NAME "table_name" +#define SFLM_COLNUM_TABLES_TABLE_NUM 2 +#define SFLM_COLNAM_TABLES_TABLE_NUM "table_num" +#define SFLM_COLNUM_TABLES_ENCDEF_NUM 3 +#define SFLM_COLNAM_TABLES_ENCDEF_NUM "encdef_num" +#define SFLM_COLNUM_TABLES_NUM_COLUMNS 4 +#define SFLM_COLNAM_TABLES_NUM_COLUMNS "num_columns" +#define SFLM_TABLES_NUM_COLUMNS 4 + +// Column names and numbers for the columns table + +#define SFLM_COLNUM_COLUMNS_TABLE_NUM 1 +#define SFLM_COLNAM_COLUMNS_TABLE_NUM "table_num" +#define SFLM_COLNUM_COLUMNS_COLUMN_NAME 2 +#define SFLM_COLNAM_COLUMNS_COLUMN_NAME "column_name" +#define SFLM_COLNUM_COLUMNS_COLUMN_NUM 3 +#define SFLM_COLNAM_COLUMNS_COLUMN_NUM "column_num" +#define SFLM_COLNUM_COLUMNS_DATA_TYPE 4 +#define SFLM_COLNAM_COLUMNS_DATA_TYPE "data_type" +#define SFLM_COLNUM_COLUMNS_ENCDEF_NUM 5 +#define SFLM_COLNAM_COLUMNS_ENCDEF_NUM "encdef_num" +#define SFLM_COLNUM_COLUMNS_READ_ONLY 6 +#define SFLM_COLNAM_COLUMNS_READ_ONLY "read_only" +#define SFLM_COLNUM_COLUMNS_NULL_ALLOWED 7 +#define SFLM_COLNAM_COLUMNS_NULL_ALLOWED "null_allowed" +#define SFLM_COLUMNS_NUM_COLUMNS 7 + +// Column names and numbers for the indexes table + +#define SFLM_COLNUM_INDEXES_INDEX_NAME 1 +#define SFLM_COLNAM_INDEXES_INDEX_NAME "index_name" +#define SFLM_COLNUM_INDEXES_INDEX_NUM 2 +#define SFLM_COLNAM_INDEXES_INDEX_NUM "index_num" +#define SFLM_COLNUM_INDEXES_TABLE_NUM 3 +#define SFLM_COLNAM_INDEXES_TABLE_NUM "table_num" +#define SFLM_COLNUM_INDEXES_ENCDEF_NUM 4 +#define SFLM_COLNAM_INDEXES_ENCDEF_NUM "encdef_num" +#define SFLM_COLNUM_INDEXES_LANGUAGE 5 +#define SFLM_COLNAM_INDEXES_LANGUAGE "language" +#define SFLM_COLNUM_INDEXES_NUM_KEY_COMPONENTS 6 +#define SFLM_COLNAM_INDEXES_NUM_KEY_COMPONENTS "num_key_components" +#define SFLM_COLNUM_INDEXES_NUM_DATA_COMPONENTS 7 +#define SFLM_COLNAM_INDEXES_NUM_DATA_COMPONENTS "num_data_components" +#define SFLM_COLNUM_INDEXES_LAST_ROW_INDEXED 8 +#define SFLM_COLNAM_INDEXES_LAST_ROW_INDEXED "last_row_indexed" +#define SFLM_COLNUM_INDEXES_INDEX_STATE 9 +#define SFLM_COLNAM_INDEXES_INDEX_STATE "index_state" +#define SFLM_INDEXES_NUM_COLUMNS 9 + +// Column names and numbers for the index components table + +#define SFLM_COLNUM_INDEX_COMP_INDEX_NUM 1 +#define SFLM_COLNAM_INDEX_COMP_INDEX_NUM "index_num" +#define SFLM_COLNUM_INDEX_COMP_COLUMN_NUM 2 +#define SFLM_COLNAM_INDEX_COMP_COLUMN_NUM "column_num" +#define SFLM_COLNUM_INDEX_COMP_KEY_COMPONENT 3 +#define SFLM_COLNAM_INDEX_COMP_KEY_COMPONENT "key_component" +#define SFLM_COLNUM_INDEX_COMP_DATA_COMPONENT 4 +#define SFLM_COLNAM_INDEX_COMP_DATA_COMPONENT "data_component" +#define SFLM_COLNUM_INDEX_COMP_INDEX_ON 5 +#define SFLM_COLNAM_INDEX_COMP_INDEX_ON "index_on" +#define SFLM_COLNUM_INDEX_COMP_COMPARE_RULES 6 +#define SFLM_COLNAM_INDEX_COMP_COMPARE_RULES "compare_rules" +#define SFLM_COLNUM_INDEX_COMP_SORT_DESCENDING 7 +#define SFLM_COLNAM_INDEX_COMP_SORT_DESCENDING "sort_descending" +#define SFLM_COLNUM_INDEX_COMP_SORT_MISSING_HIGH 8 +#define SFLM_COLNAM_INDEX_COMP_SORT_MISSING_HIGH "sort_missing_high" +#define SFLM_COLNUM_INDEX_COMP_LIMIT 9 +#define SFLM_COLNAM_INDEX_COMP_LIMIT "limit" +#define SFLM_INDEX_COMP_NUM_COLUMNS 9 + +// Column names and numbers for the block chains table + +#define SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS 1 +#define SFLM_COLNAM_BLOCK_CHAINS_BLOCK_ADDRESS "block_address" +#define SFLM_BLOCK_CHAINS_NUM_COLUMNS 1 + +#define SFLM_STRING_OPTION_STR "string" +#define SFLM_INTEGER_OPTION_STR "integer" +#define SFLM_BINARY_OPTION_STR "binary" +#define SFLM_CHECKING_OPTION_STR "checking" +#define SFLM_PURGE_OPTION_STR "purge" +#define SFLM_ACTIVE_OPTION_STR "active" +#define SFLM_INDEX_SUSPENDED_STR "suspended" +#define SFLM_INDEX_OFFLINE_STR "offline" +#define SFLM_INDEX_ONLINE_STR "online" +#define SFLM_CASE_INSENSITIVE_OPTION_STR "caseinsensitive" +#define SFLM_MINSPACES_OPTION_STR "minspaces" +#define SFLM_WHITESPACE_AS_SPACE_STR "whitespaceasspace" +#define SFLM_IGNORE_LEADINGSPACES_OPTION_STR "ignoreleadingspaces" +#define SFLM_IGNORE_TRAILINGSPACES_OPTION_STR "ignoretrailingspaces" +#define SFLM_NOUNDERSCORE_OPTION_STR "nounderscores" +#define SFLM_NOSPACE_OPTION_STR "nospaces" +#define SFLM_NODASH_OPTION_STR "nodashes" +#define SFLM_VALUE_OPTION_STR "value" +#define SFLM_PRESENCE_OPTION_STR "presence" +#define SFLM_SUBSTRING_OPTION_STR "substring" +#define SFLM_EACHWORD_OPTION_STR "eachword" +#define SFLM_ABS_POS_OPTION_STR "abspos" +#define SFLM_METAPHONE_OPTION_STR "metaphone" + +// Encryption Schemes + +#define SFLM_ENC_AES_OPTION_STR "aes" +#define SFLM_ENC_DES3_OPTION_STR "des3" + +/// Actions that can be returned from an application during database +/// restore operations. +typedef enum +{ + SFLM_RESTORE_ACTION_CONTINUE = 0, ///< 0 = Continue recovery + SFLM_RESTORE_ACTION_STOP, ///< 1 = Stop recovery + SFLM_RESTORE_ACTION_SKIP, ///< 2 = Skip operation (future) + SFLM_RESTORE_ACTION_RETRY ///< 3 = Retry the operation +} eRestoreAction; + +/// Event categories an application may subscribe to. +typedef enum +{ + SFLM_EVENT_LOCKS, ///< Lock events. + SFLM_EVENT_UPDATES, ///< Update events. + SFLM_MAX_EVENT_CATEGORIES // Should always be last. +} eEventCategory; + +/// Events an application may receive. +typedef enum +{ + SFLM_EVENT_LOCK_WAITING, ///< 0 = Thread is waiting to obtain database lock. + SFLM_EVENT_LOCK_GRANTED, ///< 1 = Thread was granted database lock. + SFLM_EVENT_LOCK_SUSPENDED, ///< 2 = Thread is suspended waiting to obtain database lock. + SFLM_EVENT_LOCK_RESUMED, ///< 3 = Thread was granted database lock and resumed. + SFLM_EVENT_LOCK_RELEASED, ///< 4 = Thread that had the database lock released it. + SFLM_EVENT_LOCK_TIMEOUT, ///< 5 = Thread that was waiting for a database lock timed out. + SFLM_EVENT_BEGIN_TRANS, ///< 6 = Transaction was started. + SFLM_EVENT_COMMIT_TRANS, ///< 7 = Transaction was committed. + SFLM_EVENT_ABORT_TRANS, ///< 8 = Transaction was aborted. + SFLM_EVENT_CREATE_TABLE, ///< 9 = New table created. + SFLM_EVENT_CREATE_INDEX, ///< 10 = New index created. + SFLM_EVENT_DROP_TABLE, ///< 11 = Table dropped. + SFLM_EVENT_DROP_INDEX, ///< 12 = Index dropped. + SFLM_EVENT_INSERT_ROW, ///< 13 = Row inserted into table. + SFLM_EVENT_DELETE_ROW, ///< 14 = Row deleted from table. + SFLM_EVENT_MODIFY_ROW, ///< 15 = Row modified in table. + SFLM_EVENT_INDEXING_PROGRESS, ///< 16 = Background indexing progress. + SFLM_MAX_EVENT_TYPES // Should always be last. +} eEventType; + +/// Types of logical files in the database. +typedef enum +{ + SFLM_LF_INVALID = 0, ///< 0 = Invalid + SFLM_LF_TABLE, ///< 1 = Table + SFLM_LF_INDEX ///< 2 = Index +} eLFileType; + +/// Types of messages logged by the database system. +typedef enum +{ + SFLM_QUERY_MESSAGE, ///< Query + SFLM_TRANSACTION_MESSAGE, ///< Transaction + SFLM_GENERAL_MESSAGE, ///< General messages + SFLM_NUM_MESSAGE_TYPES +} eLogMessageType; + +/// Reasons the checkpoint thread forces a checkpoint. +typedef enum +{ + SFLM_CP_NO_REASON = 0, + SFLM_CP_TIME_INTERVAL_REASON = 1, ///< Maximum time interval since last completed checkpoint reached. + SFLM_CP_SHUTTING_DOWN_REASON, ///< Database is shutting down. + SFLM_CP_RFL_VOLUME_PROBLEM ///< Had a problem writing to the RFL disk volume. +} eForceCPReason; + +/// Structure that returns information about the current activity of the +/// checkpoint thread. +typedef struct +{ + FLMBOOL bRunning; ///< Is checkpoint thread currently running? + FLMUINT uiRunningTime; ///< Milliseconds checkpoint thread has been running. + FLMBOOL bForcingCheckpoint; ///< Is the checkpoint currently forcing a checkpoint? + FLMUINT uiForceCheckpointRunningTime; ///< Milliseconds checkpoint thread has been in the forcing checkpoint state. + eForceCPReason eForceCheckpointReason; ///< Reason checkpoint is being forced. + FLMBOOL bWritingDataBlocks; ///< Is the checkpoint thread currently writing out data blocks? + FLMUINT uiLogBlocksWritten; ///< Total blocks written to the rollback log. + FLMUINT uiDataBlocksWritten; ///< Total data blocks written. + FLMUINT uiDirtyCacheBytes; ///< Total bytes in cache that are dirty data blocks. + FLMUINT uiBlockSize; ///< Database block size. + FLMUINT uiWaitTruncateTime; ///< Milliseconds checkpoint thread has been waiting to truncate the rollback log. +} SFLM_CHECKPOINT_INFO; + +/// Structure for reporting cache usage and statistics. +typedef struct +{ + FLMUINT uiByteCount; ///< Total bytes used in cache. + FLMUINT uiCount; ///< Total objects (blocks or rows) cached. + FLMUINT uiOldVerBytes; ///< Total bytes allocated to old versions of objects (blocks or rows). + FLMUINT uiOldVerCount; ///< Total objects (blocks or rows) in cache that are old versions. + FLMUINT uiCacheHits; ///< Total cache hits. + FLMUINT uiCacheHitLooks; ///< Total objects (blocks or rows) examined to get cache hits. + FLMUINT uiCacheFaults; ///< Total cache faults. + FLMUINT uiCacheFaultLooks; ///< Total objects (blocks or rows) examined to get cache faults. + FLM_SLAB_USAGE slabUsage; ///< Slab usage for this particular cache. +} SFLM_CACHE_USAGE; + +/// Structure used to return cache information and statistics. +typedef struct +{ + FLMUINT uiMaxBytes; ///< Current cache maximum (in bytes). + FLMUINT uiTotalBytesAllocated; ///< Total cache bytes allocated. + FLMBOOL bDynamicCacheAdjust; ///< Is cache maximum being dynamically adjusted? + FLMUINT uiCacheAdjustPercent; ///< Cache adjust percent - only valid if bDynamicCacheAdjust is TRUE. + FLMUINT uiCacheAdjustMin; ///< Cache adjust minimum (bytes) - only valid if bDynamicCacheAdjust is TRUE. + FLMUINT uiCacheAdjustMax; ///< Cache adjust maximum (bytes) - only valid if bDynamicCacheAdjust is TRUE. + FLMUINT uiCacheAdjustMinToLeave; ///< Cache adjust minimum to leave (bytes) - only valid if bDynamicCacheAdjust is TRUE. + FLMUINT uiDirtyCount; ///< Blocks in block cache that are dirty. + FLMUINT uiDirtyBytes; ///< Total bytes in block cache that are dirty. + FLMUINT uiNewCount; ///< Blocks in block cache that are "new" - meaning they are newly created blocks that + ///< are beyond what was the database EOF at the beginning of the current update + ///< transaction. + FLMUINT uiNewBytes; ///< Total bytes in block cache that are "new" blocks (see uiNewCount). + FLMUINT uiLogCount; ///< Total blocks in cache that need to be written to the rollback log. + FLMUINT uiLogBytes; ///< Total bytes in block cache that need to be written to the rollback log. + FLMUINT uiFreeCount; ///< Total blocks in block cache that are free blocks - not associated with any + ///< database. + FLMUINT uiFreeBytes; ///< Total bytes in block cache that are free blocks - not associated with any database. + FLMUINT uiReplaceableCount; ///< Total blocks in block cache that can be replaced.\ These are blocks that aren't + ///< flagged in some way that would prevent them from being replaced in cache. + FLMUINT uiReplaceableBytes; ///< Total bytes in replaceable blocks (see uiReplaceableCount). + FLMBOOL bPreallocatedCache; ///< Was cache pre-allocated? + SFLM_CACHE_USAGE BlockCache; ///< Information about block cache usage. + SFLM_CACHE_USAGE RowCache; ///< Information about row cache usage. +} SFLM_CACHE_INFO; + +/// Structure used in gathering statistics to hold an operation count and an elapsed time. +typedef struct +{ + FLMUINT64 ui64Count; ///< Number of times operation was performed + FLMUINT64 ui64ElapMilli; ///< Total elapsed time (milliseconds) for the operations. +} SFLM_COUNT_TIME_STAT; + +/// Structure used in gathering statistics to hold a operation count, a byte count, and an elapsed time. This +/// is typically used for I/O operations where it is useful to know the number of bytes that were read or +/// written by the operation. +typedef struct +{ + FLMUINT64 ui64Count; ///< Number of times operation was performed. + FLMUINT64 ui64TotalBytes; ///< Total number of bytes involved in the operations.\ This usually represents + ///< bytes read from or written to disk. + FLMUINT64 ui64ElapMilli; ///< Total elapsed time (milliseconds) for the operations. +} SFLM_DISKIO_STAT; + +/// Statistics for read transactions. +typedef struct +{ + SFLM_COUNT_TIME_STAT CommittedTrans; ///< Statistics for read transactions committed. + SFLM_COUNT_TIME_STAT AbortedTrans; ///< Statistics for read transactions aborted. +} SFLM_RTRANS_STATS; + +/// Statistics for update transactions. +typedef struct +{ + SFLM_COUNT_TIME_STAT CommittedTrans; ///< Statistics for update transactions committed. + SFLM_COUNT_TIME_STAT GroupCompletes; ///< Statistics for number of times multiple transactions were committed together. + FLMUINT64 ui64GroupFinished;///< Total update transactions that were committed in a group. + SFLM_COUNT_TIME_STAT AbortedTrans; ///< Statistics for update transactions aborted. +} SFLM_UTRANS_STATS; + +/// Statistics for block reads and writes. +typedef struct +{ + SFLM_DISKIO_STAT BlockReads; ///< Statistics on block reads. + SFLM_DISKIO_STAT OldViewBlockReads; ///< Statistics on block reads that resulted in an old view error. + FLMUINT uiBlockChkErrs; ///< Number of times we had errors checking the block after it + ///< was read in - either checksum errors or other problems + ///< validating data in the block. + FLMUINT uiOldViewBlockChkErrs; ///< Number of times we had errors checking the block after it + ///< was read in - either checksum errors or other problems + ///< validating data in the block.\ This statistic is for + ///< older versions of a block as opposed to the current version. + FLMUINT uiOldViewErrors; ///< Number of times we had an old view error when reading blocks. + SFLM_DISKIO_STAT BlockWrites; ///< Statistics on Block writes. +} SFLM_BLOCKIO_STATS; + +/// Statistics gathered for a particular logical file (index or table). +typedef struct +{ + FLMBOOL bHaveStats; ///< Flag indicating whether or not there are statistics + ///< for this logical file. + FLMUINT uiLFileNum; ///< Logical file number. + eLFileType eLfType; ///< Type of logical file. + SFLM_BLOCKIO_STATS RootBlockStats; ///< Block I/O statistics for the logical file's root blocks. + SFLM_BLOCKIO_STATS MiddleBlockStats; ///< Block I/O statistics for for the blocks in the logical file that are not + ///< root blocks or leaf blocks. + SFLM_BLOCKIO_STATS LeafBlockStats; ///< Block I/O statistics for the logical file's leaf blocks. + FLMUINT64 ui64BlockSplits; ///< Number of block splits that have occurred in this logical file. + FLMUINT64 ui64BlockCombines; ///< Number of block combines that have occurred in this logical file. +} SFLM_LFILE_STATS; + +/// Database statistics. +typedef struct +{ + const char * pszDbName; ///< Name of database these statistics are for. + FLMBOOL bHaveStats; ///< Flag indicating whether or not there are statistics for this database. + SFLM_RTRANS_STATS ReadTransStats; ///< Read transaction statistics for the database. + SFLM_UTRANS_STATS UpdateTransStats; ///< Update transaction statistics for the database. + FLMUINT uiLFileAllocSeq; ///< Allocation sequence number for pLFileStats array - used internally. + SFLM_LFILE_STATS * pLFileStats; ///< Logical file statistics for this database. + FLMUINT uiLFileStatArraySize; ///< Number of logical files in the pLFileStats array - used internally. + FLMUINT uiNumLFileStats; ///< Number of elements in the pLFileStats array currently in use. + SFLM_BLOCKIO_STATS LFHBlockStats; ///< Block I/O statistics for LFH blocks. + SFLM_BLOCKIO_STATS AvailBlockStats; ///< Block I/O statistics for AVAIL blocks. + + // Write statistics + + SFLM_DISKIO_STAT DbHdrWrites; ///< Statistics for writes to the database header. + SFLM_DISKIO_STAT LogBlockWrites; ///< Statistics for writes of blocks to the rollback log. + SFLM_DISKIO_STAT LogBlockRestores; ///< Statistics for writing of blocks from the rollback log back into data files (done + ///< during database recovery or when aborting a transaction). + // Read statistics + + SFLM_DISKIO_STAT LogBlockReads; ///< Statistics on reading blocks from the rollback log. + FLMUINT uiLogBlockChkErrs; ///< Number of times we had checksum errors reading blocks from the rollback log. + FLMUINT uiReadErrors; ///< Number of times we got read errors. + FLMUINT uiWriteErrors; ///< Number of times we got write errors. + + // Lock statistics + + SFLM_COUNT_TIME_STAT NoLocks; ///< Statistics on times when nobody was holding a lock on the database. + SFLM_COUNT_TIME_STAT WaitingForLock; ///< Statistics on times threads were waiting to obtain a database lock. + SFLM_COUNT_TIME_STAT HeldLock; ///< Statistics on times when a thread was holding a lock on the database. + + // Update statistics + + SFLM_COUNT_TIME_STAT RowInserts; ///< Number of row insert operations that have been performed on + ///< this database. + SFLM_COUNT_TIME_STAT RowDeletes; ///< Number of row delete operations that have been performed + ///< on this database. + SFLM_COUNT_TIME_STAT RowModifies; ///< Number of row modify operations that have been performed + ///< on this database. +} SFLM_DB_STATS; + +/// Structure for return various database statistics. +typedef struct +{ + SFLM_DB_STATS * pDbStats; ///< Pointer to array of database statistics.\ There will be an ::SFLM_DB_STATS + ///< structure for every database currently open. + FLMUINT uiDBAllocSeq; ///< Allocation sequence number for the pDbStats array - only used internally. + FLMUINT uiDbStatArraySize; ///< Size of the pDbStats array. + FLMUINT uiNumDbStats; ///< Number of elements in the pDbStats array that are currently in use. + FLMBOOL bCollectingStats; ///< Flag indicating whether or not we are currently collecting statistics. + FLMUINT uiStartTime; ///< Time we started collecting statistics.\ This is GMT time - seconds since + ///< January 1, 1970 midnight. + FLMUINT uiStopTime; ///< Time we stopped collecting statistics.\ This is GMT time - seconds since + ///< January 1, 1970 midnight. +} SFLM_STATS; + +/// Index states. +typedef enum +{ + SFLM_INDEX_ONLINE = 0, ///< 0 = Index is on-line + SFLM_INDEX_BRINGING_ONLINE, ///< 1 = Index is currently being brought on-line + SFLM_INDEX_SUSPENDED ///< 2 = Index is suspended +} eSFlmIndexState; + +/// Structure for returning information about an index. +typedef struct +{ + FLMUINT uiIndexNum; ///< Index number + eSFlmIndexState eState; ///< Current index state. + + // Statistics when eState is SFLM_INDEX_BRINGING_ONLINE + + FLMUINT uiStartTime; ///< Time when index started to be brought on-line.\ Only valid if the state is + ///< eSflmIndexState::SFLM_INDEX_BRINGING_ONLINE. + FLMUINT64 ui64LastRowIndexed; ///< If ~0 then index is online, otherwises this is the row ID of the last row + ///< that was indexed. + FLMUINT64 ui64KeysProcessed; ///< Total keys processed by the background indexing thread. + FLMUINT64 ui64RowsProcessed; ///< total rows indexed by the background indexing thread. + FLMUINT64 ui64Transactions; ///< Total transactions performed by the background indexing thread. +} SFLM_INDEX_STATUS; + +/// Actions that the database rebuild operation performs. +typedef enum +{ + REBUILD_GET_BLK_SIZ = 0, ///< 0 = Rebuild is making a pass through the database to try and figure out the block size. + REBUILD_RECOVER_DICT, ///< 1 = Rebuild is recovering dictionary information. + REBUILD_RECOVER_DATA ///< 2 = Rebuild is recovering table rows. +} eRebuildActions; + +/// Structure passed to the application during database rebuild operations. +typedef struct +{ + eRebuildActions eAction; ///< Current action being taken by the database rebuild operation. + FLMBOOL bStartFlag; ///< Flag indicating if we are just starting the current operation. + FLMUINT64 ui64DatabaseSize; ///< Total database size (bytes). + FLMUINT64 ui64BytesExamined; ///< Total bytes examined so far. + FLMUINT64 ui64TotalRows; ///< Total rows examined so far. + FLMUINT64 ui64RowsRecov; ///< Total rows recovered so far. +} SFLM_REBUILD_INFO; + +/// Types of corruption +typedef enum +{ + + // WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE + // REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN flerrstr.cpp + + SFLM_NO_CORRUPTION = 0, + SFLM_BAD_CHAR, ///< 1 + SFLM_BAD_ASIAN_CHAR, ///< 2 + SFLM_BAD_CHAR_SET, ///< 3 + SFLM_BAD_TEXT_FIELD, ///< 4 + SFLM_BAD_NUMBER_FIELD, ///< 5 + SFLM_BAD_FIELD_TYPE, ///< 6 + SFLM_BAD_IX_DEF, ///< 7 + SFLM_BAD_NUMBER_KEY, ///< 8 + SFLM_BAD_BINARY_KEY, ///< 9 + SFLM_BAD_KEY_FIELD_TYPE, ///< 10 + SFLM_BAD_KEY_LEN, ///< 11 + SFLM_BAD_LFH_LIST_PTR, ///< 12 + SFLM_BAD_LFH_LIST_END, ///< 13 + SFLM_BAD_BLK_END, ///< 14 + SFLM_KEY_COUNT_MISMATCH, ///< 15 + SFLM_REF_COUNT_MISMATCH, ///< 16 + SFLM_BAD_BLK_HDR_ADDR, ///< 17 + SFLM_BAD_BLK_HDR_LEVEL, ///< 18 + SFLM_BAD_BLK_HDR_PREV, ///< 19 + + // WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE + // REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN flerrstr.cpp + + SFLM_BAD_BLK_HDR_NEXT, ///< 20 + SFLM_BAD_BLK_HDR_TYPE, ///< 21 + SFLM_BAD_BLK_HDR_ROOT_BIT, ///< 22 + SFLM_BAD_BLK_HDR_BLK_END, ///< 23 + SFLM_BAD_BLK_HDR_LF_NUM, ///< 24 + SFLM_BAD_AVAIL_LIST_END, ///< 25 + SFLM_BAD_PREV_BLK_NEXT, ///< 26 + SFLM_BAD_FIRST_ELM_FLAG, ///< 27 + SFLM_BAD_LEM, ///< 28 + SFLM_BAD_ELM_LEN, ///< 29 + SFLM_BAD_ELM_KEY_SIZE, ///< 30 + SFLM_BAD_ELM_KEY, ///< 31 + SFLM_BAD_ELM_KEY_ORDER, ///< 32 + SFLM_BAD_CONT_ELM_KEY, ///< 34 + SFLM_NON_UNIQUE_FIRST_ELM_KEY, ///< 35 + SFLM_BAD_ELM_OFFSET, ///< 36 + SFLM_BAD_ELM_INVALID_LEVEL, ///< 37 + SFLM_BAD_ELM_FLD_NUM, ///< 38 + SFLM_BAD_ELM_FLD_LEN, ///< 39 + + // WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE + // REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN flerrstr.cpp + + SFLM_BAD_ELM_FLD_TYPE, ///< 40 + SFLM_BAD_ELM_END, ///< 41 + SFLM_BAD_PARENT_KEY, ///< 42 + SFLM_BAD_ELM_IX_REF, ///< 43 + SFLM_BAD_LAST_BLK_NEXT, ///< 44 + SFLM_REBUILD_REC_EXISTS, ///< 45 + SFLM_REBUILD_KEY_NOT_UNIQUE, ///< 46 + SFLM_NON_UNIQUE_ELM_KEY_REF, ///< 47 + SFLM_OLD_VIEW, ///< 48 + SFLM_COULD_NOT_SYNC_BLK, ///< 49 + SFLM_IX_KEY_NOT_FOUND_IN_REC, ///< 50 + SFLM_KEY_NOT_IN_KEY_REFSET, ///< 51 + SFLM_BAD_BLK_CHECKSUM, ///< 52 + SFLM_BAD_FILE_SIZE, ///< 53 + SFLM_BAD_BLK_TYPE, ///< 54 + SFLM_BAD_ELEMENT_CHAIN, ///< 55 + SFLM_BAD_ELM_EXTRA_DATA, ///< 56 + SFLM_BAD_BLOCK_STRUCTURE, ///< 57 + SFLM_BAD_DATA_BLOCK_COUNT, ///< 58 + SFLM_BAD_AVAIL_SIZE, ///< 59 + SFLM_BAD_CHILD_ELM_COUNT, ///< 60 + SFLM_NUM_CORRUPT_ERRORS +} eCorruptionType; + +/// Locations in the database where corruption are reported. +typedef enum +{ + SFLM_LOCALE_NONE = 0, ///< 0 = None + SFLM_LOCALE_LFH_LIST, ///< 1 = Corruption was found in logical file block list. + SFLM_LOCALE_AVAIL_LIST, ///< 2 = Corruption was found in the avail block list. + SFLM_LOCALE_B_TREE, ///< 3 = Corruption was found in an index or table b-tree. + SFLM_LOCALE_INDEX ///< 4 = Corruption is a logical corruption of an index. +} eErrorLocation; + +/// Structure for reporting a corruption when checking a database. +typedef struct +{ + eCorruptionType eCorruption; ///< Type of corruption that was found. + eErrorLocation eErrLocale; ///< Location of the corruption. + FLMUINT uiErrLfNumber; ///< Logical file number where corruption occurred, if eErrLocal == eErrorLocation::SFLM_LOCALE_B_TREE. + eLFileType eErrLfType; ///< Logical file type, if eErrLocal == eErrorLocation::SFLM_LOCALE_B_TREE. + FLMUINT uiErrBTreeLevel; ///< Level in b-tree where corruption occurred.\ Zero if not applicable. + FLMUINT uiErrBlkAddress; ///< Address of block that had the corruption.\ Zero if not applicable. + FLMUINT uiErrParentBlkAddress; ///< Address of the parent block to the block that had the corruption. + FLMUINT uiErrElmOffset; ///< Offset of element in the block where the corruption was found.\ Zero if not applicable. + FLMUINT64 ui64ErrRowId; ///< Row ID where corruption was detected.\ Zero if not applicable. +} SFLM_CORRUPT_INFO; + +/// Phases that the database check operation goes through. +typedef enum +{ + SFLM_CHECK_LFH_BLOCKS = 1, ///< 1 = Checking logical file block list. + SFLM_CHECK_B_TREE, ///< 2 = Checking physical structure of b-trees. + SFLM_CHECK_AVAIL_BLOCKS, ///< 3 = Checking the available block list. + SFLM_CHECK_RS_SORT ///< 4 = Sorting index keys result set. +} eCheckPhase; + +/// Structure for reporting the status and progress of a database check operation. +typedef struct +{ + eCheckPhase ePhase; ///< Current phase the database check operation is currently in. + FLMBOOL bStartFlag; ///< Is the current check phase just starting? + FLMUINT64 ui64DatabaseSize; ///< Database size (in bytes). + FLMUINT64 ui64BytesExamined; ///< Total bytes checked so far in the database. + FLMUINT uiNumLFs; ///< Total logical files (tables and indexes) in the database. + FLMUINT uiCurrLF; ///< Count of the logical file (table or index) currently being checked. + FLMUINT uiLfNumber; ///< Logical file number (table or index) currently being checked. + eLFileType eLfType; ///< Type of logical file currently being checked. + FLMUINT uiNumProblemsFixed; ///< Total number of corruptions fixed. + + // Index check progress + + FLMUINT64 ui64NumKeys; ///< Number of index keys found in rows. + FLMUINT64 ui64NumDuplicateKeys; ///< Number of duplicate index keys found. + FLMUINT64 ui64NumKeysExamined; ///< Number of index keys checked. + FLMUINT64 ui64NumKeysNotFound; ///< Number of keys found in index, but not found in the row they reference. + FLMUINT64 ui64NumRowKeysNotFound; ///< Number of keys found in rows, but not found in the index. + FLMUINT64 ui64NumConflicts; ///< Number of corruptions that turned out not to be corruptions when an attempt was made to fix them. +} SFLM_PROGRESS_CHECK_INFO; + +/// Query operators. +typedef enum +{ + SQL_UNKNOWN_OP = 0, + + // NOTE: These operators MUST stay in this order - this order is assumed + // by the precedence table - see sqlparse.cpp + + SQL_AND_OP = 1, ///< 1 = and + SQL_OR_OP = 2, ///< 2 = or + SQL_NOT_OP = 3, ///< 3 = ! (not) + SQL_EQ_OP = 4, ///< 4 = equals + SQL_NE_OP = 5, ///< 5 = not equal + SQL_APPROX_EQ_OP = 6, ///< 6 = approximate equal + SQL_LT_OP = 7, ///< 7 = less than + SQL_LE_OP = 8, ///< 8 = less than or equal + SQL_GT_OP = 9, ///< 9 = greater than + SQL_GE_OP = 10, ///< 10 = greater than or equal + SQL_BITAND_OP = 11, ///< 11 = bit and + SQL_BITOR_OP = 12, ///< 12 = bit or + SQL_BITXOR_OP = 13, ///< 13 = bit exclusive or + SQL_MULT_OP = 14, ///< 14 = multiply + SQL_DIV_OP = 15, ///< 15 = divide + SQL_MOD_OP = 16, ///< 16 = modulo + SQL_PLUS_OP = 17, ///< 17 = plus + SQL_MINUS_OP = 18, ///< 18 = minus + SQL_NEG_OP = 19, ///< 19 = negative + SQL_LPAREN_OP = 20, ///< 20 = left parenthesis + SQL_RPAREN_OP = 21, ///< 21 = right parenthesis + + // IMPORTANT NOTE: If operators are added after this point, + // modify the isLegalOperator method below. + + // The following operators are only used internally. They + // may NOT be passed into the addOperator method. + + SQL_EXISTS_OP = 25, ///< 25 = exists + SQL_RANGE_OP = 26, ///< 26 = range + SQL_MATCH_OP = 27 ///< 27 = match +} eSQLQueryOperators; + +#define SQL_FIRST_ARITH_OP SQL_BITAND_OP +#define SQL_LAST_ARITH_OP SQL_NEG_OP + +/// States of a query. +typedef enum +{ + SQL_QUERY_NOT_POSITIONED, ///< Query has not been positioned yet + SQL_QUERY_AT_BOF, ///< Positioned at the beginning of result set (before the first row). + SQL_QUERY_AT_FIRST, ///< Positioned on the first row in result set. + SQL_QUERY_AT_FIRST_AND_LAST, ///< Positioned on the first and last row in the result set. + SQL_QUERY_POSITIONED, ///< Positioned in the middle of the result set - not on first or last row. + SQL_QUERY_AT_LAST, ///< Positioned on the last row in the result set. + SQL_QUERY_AT_EOF ///< Positioned at the end of the result set (after the last row). +} eQueryStates; + +/// Boolean values - tri-state. +typedef enum +{ + SQL_FALSE = 0, ///< 0 = FALSE + SQL_TRUE, ///< 1 = TRUE + SQL_UNKNOWN ///< 2 = UNKNOWN +} SQLBoolType; + +/// Value types +typedef enum +{ + SQL_MISSING_VAL = 0, + + // WARNING: Don't renumber below _VAL enums without + // re-doing gv_DoValAndDictTypesMatch table + + SQL_BOOL_VAL, ///< 1 = Boolean (::SQLBoolType) + SQL_UINT_VAL, ///< 2 = Unsigned integer (FLMUINT) + SQL_UINT64_VAL, ///< 3 = Unsigned 64 bit integer (FLMUINT64) + SQL_INT_VAL, ///< 4 = Signed integer (FLMINT) + SQL_INT64_VAL, ///< 5 = Signed 64 bit integer (FLMINT64) + SQL_BINARY_VAL, ///< 6 = Binary + SQL_UTF8_VAL, ///< 7 = UTF8 string +} eSQLValTypes; + +/// Optimization types. +typedef enum +{ + SQL_OPT_NONE = 0, ///< 0 = None + SQL_OPT_USING_INDEX, ///< 1 = Using an index. + SQL_OPT_FULL_TABLE_SCAN, ///< 2 = Full table scan. + SQL_OPT_SINGLE_ROW_ID, ///< 3 = Fetch single row using row ID. + SQL_OPT_ROW_ID_RANGE ///< 4 = Range of row IDs. +} eSQLOptTypes; + +/// Query optimization structure. +typedef struct +{ + eSQLOptTypes eOptType; ///< Type of optimization done. + FLMUINT uiCost; ///< Cost calculated for sub-query. + FLMUINT64 ui64RowId; ///< Only valid if eOptType is + ///< eSQLOptTypes::SQL_OPT_SINGLE_ROW_ID or + ///< eSQLOptTypes::SQL_OPT_ROW_ID_RANGE + FLMUINT64 ui64EndRowId; ///< Only valid if eOptType is + ///< eSQLOptTypes::SQL_OPT_ROW_ID_RANGE + FLMUINT uiIxNum; ///< Index used to execute query if + ///< eOptType == SQL_OPT_USING_INDEX + FLMBYTE szIxName[ 80]; ///< Name of index corresponding to uiIxNum. + FLMBOOL bDoRowMatch; ///< Node must be retrieved to exe + ///< query. Only valid if eOptType + ///< is eSQLOptTypes::SQL_OPT_USING_INDEX. + FLMUINT bCanCompareOnKey; ///< Can we compare using only the data from index keys?\ Only + ///< valid if eOptType == eSQLOptTypes::SQL_OPT_USING_INDEX. + FLMUINT64 ui64KeysRead; ///< Total index keys read. + FLMUINT64 ui64KeyHadDupRow; ///< Total index keys that pointed to a row we had already passed. + FLMUINT64 ui64KeysPassed; ///< Total keys that passed query criteria. + FLMUINT64 ui64RowsRead; ///< Total rows read. + FLMUINT64 ui64RowsPassed; ///< Total rows that passed the query criteria. + FLMUINT64 ui64DupRowsEliminated; ///< Total rows that were duplicates that we eliminated. + FLMUINT64 ui64RowsFailedValidation; ///< Total rows that failed validation callback. +} SQL_OPT_INFO; + +//************************************************************************* +// Dictionary Tag Numbers +// +// NOTES: +// 1) These numbers cannot be changed for backward compatibility reasons. +// 2) IF ANY NEW TAGS ARE INSERTED - Then you MUST change the database +// version number, because old databases will become invalid..... +// +//************************************************************************** + +// Special purpose tables +// NOTE: Do not change the order of these definitions. The +// getNextTable routine assumes they are in this order. + +// We have reserved from 65501 to 65535 for internal collections +// These should be allocated starting from 65535 and going down. + +#define SFLM_MAX_TABLE_NUM 65500 + +#define SFLM_MAINT_TABLE 65530 +#define SFLM_ENCDEF_TABLE 65531 +#define SFLM_INDEX_COLUMNS_TABLE 65532 +#define SFLM_INDEX_TABLE 65533 +#define SFLM_TABLE_COLUMNS_TABLE 65534 +#define SFLM_TABLE_TABLE 65535 + +// Special purpose indexes +// NOTE: Do not change the order of these definitions. The +// getNextIndex routine assumes they are in this order. + +// We have reserved from 65501 to 65535 for internal indexes +// These should be allocated starting from 65535 and going down. + +#define SFLM_MAX_INDEX_NUM 65500 +#define SFLM_INDEX_COLUMN_NAME_INDEX 65530 +#define SFLM_INDEX_NUMBER_INDEX 65531 +#define SFLM_INDEX_NAME_INDEX 65532 +#define SFLM_TABLE_COLUMN_NAME_INDEX 65533 +#define SFLM_TABLE_NUMBER_INDEX 65534 +#define SFLM_TABLE_NAME_INDEX 65535 + +#define SFLM_MAX_ENCDEF_NUM 65500 + +//*************************************************************************** +// Dictionary Identifiers +//*************************************************************************** + +#define SFLM_INI_CACHE "cache" +#define SFLM_INI_CACHE_ADJUST_INTERVAL "cacheadjustinterval" +#define SFLM_INI_CACHE_CLEANUP_INTERVAL "cachecleanupinterval" +#define SFLM_INI_MAX_DIRTY_CACHE "maxdirtycache" +#define SFLM_INI_LOW_DIRTY_CACHE "lowdirtycache" + +// Defaults for certain settable items in the database system + +#define SFLM_DEFAULT_MAX_CP_INTERVAL 180 +#define SFLM_DEFAULT_CACHE_ADJUST_PERCENT 70 +#define SFLM_DEFAULT_CACHE_ADJUST_MIN (16 * 1024 * 1024) +#define SFLM_DEFAULT_CACHE_ADJUST_MAX 0xE0000000 +#define SFLM_DEFAULT_CACHE_ADJUST_MIN_TO_LEAVE 0 +#define SFLM_DEFAULT_CACHE_ADJUST_INTERVAL 15 +#define SFLM_DEFAULT_CACHE_CLEANUP_INTERVAL 15 +#define SFLM_DEFAULT_UNUSED_CLEANUP_INTERVAL 2 +#define SFLM_DEFAULT_MAX_UNUSED_TIME 120 +#define SFLM_DEFAULT_FILE_EXTEND_SIZE (8192 * 1024) +#define SFLM_MIN_BLOCK_SIZE 4096 +#define SFLM_MAX_BLOCK_SIZE 8192 +#define SFLM_DEFAULT_OPEN_THRESHOLD 100 // 100 file handles to cache +#define SFLM_DEFAULT_MAX_AVAIL_TIME 900 // 15 minutes +#define SFLM_DEFAULT_REHASH_BACKOFF_INTERVAL 60 // 1 minute + + +// Error codes + +/// \addtogroup flaim_errors +/// @{ +#define NE_SFLM_OK NE_FLM_OK +#define NE_SFLM_NOT_IMPLEMENTED 0xE000 ///< 0xE000 = Attempt was made to use a feature that is not implemented. +#define NE_SFLM_MEM 0xE001 ///< 0xE001 = Attempt to allocate memory failed. +#define NE_SFLM_INVALID_PARM 0xE002 ///< 0xD002 = Invalid parameter passed into a function. +#define NE_SFLM_TIMEOUT 0xE003 ///< 0xE003 = Operation timed out (usually a query operation). +#define NE_SFLM_NOT_FOUND 0xE004 ///< 0xE004 = An object was not found. +#define NE_SFLM_EXISTS 0xE005 ///< 0xE005 = Object already exists. +#define NE_SFLM_USER_ABORT 0xE006 ///< 0xE006 = User or application aborted (canceled) the operation. +#define NE_SFLM_FAILURE 0xE007 ///< 0xE007 = Internal failure. +#define NE_SFLM_BOF_HIT 0xE008 ///< 0xE008 = Beginning of results encountered.\ This error is may be returned when reading query results in reverse order (from last to first). +#define NE_SFLM_EOF_HIT 0xE009 ///< 0xE009 = End of results encountered.\ This error may be returned when reading query results in forward order (first to last). +#define NE_SFLM_END 0xE00A ///< 0xE00A = End of roll-forward log packets encountered.\ NOTE: This error code should never be returned to an application. +#define NE_SFLM_BAD_TABLE 0xE00B ///< 0xE00B = Invalid table number specified.\ Table is not defined. +#define NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT 0xE00C ///< 0xE00C = Request to lock the database timed out. +#define NE_SFLM_BAD_DATA_TYPE 0xE00D ///< 0xE00D = Attempt to set/get data on a column using a data type that is incompatible with the data type specified in the dictionary. +#define NE_SFLM_BAD_IX 0xE00E ///< 0xE00E = Invalid index number specified.\ Index is not defined. +#define NE_SFLM_BACKUP_ACTIVE 0xE00F ///< 0xE00F = Operation could not be performed because a backup is currently in progress. +#define NE_SFLM_SERIAL_NUM_MISMATCH 0xE010 ///< 0xE010 = Serial number on backup file does not match the serial number that is expected. +#define NE_SFLM_BAD_RFL_DB_SERIAL_NUM 0xE011 ///< 0xE011 = Bad database serial number in roll-forward log file header. +#define NE_SFLM_BTREE_ERROR 0xE012 ///< 0xE012 = A B-Tree in the database has a corruption. +#define NE_SFLM_BTREE_FULL 0xE013 ///< 0xE013 = A B-tree in the database is full, or a b-tree being used for a temporary result set is full. +#define NE_SFLM_BAD_RFL_FILE_NUMBER 0xE014 ///< 0xE014 = Bad roll-forward log file number in roll-forward log file header. +#define NE_SFLM_CANNOT_MOD_DATA_TYPE 0xE015 ///< 0xE015 = Cannot modify the data type for a column definition in the dictionary. +#define NE_SFLM_CANNOT_INDEX_DATA_TYPE 0xE016 ///< 0xE016 = Data type of column is not one that can be indexed. +#define NE_SFLM_CONV_BAD_DIGIT 0xE017 ///< 0xE017 = Non-numeric digit found in text to numeric conversion. +#define NE_SFLM_CONV_DEST_OVERFLOW 0xE018 ///< 0xE018 = Destination buffer not large enough to hold data. +#define NE_SFLM_CONV_ILLEGAL 0xE019 ///< 0xE019 = Attempt to convert between data types is an unsupported conversion. +#define NE_SFLM_CONV_NULL_SRC 0xE01A ///< 0xE01A = Data source cannot be NULL when doing data conversion. +#define NE_SFLM_CONV_NUM_OVERFLOW 0xE01B ///< 0xE01B = Numeric overflow (> upper bound) converting to numeric type. +#define NE_SFLM_CONV_NUM_UNDERFLOW 0xE01C ///< 0xE01C = Numeric underflow (< lower bound) converting to numeric type. +#define NE_SFLM_BAD_COLUMN_NUM 0xE01D ///< 0xE01D = Bad column number specified - column not defined in dictionary. +#define NE_SFLM_DATA_ERROR 0xE01E ///< 0xE01E = Encountered data in the database that was corrupted. +#define NE_SFLM_INVALID_FILE_SEQUENCE 0xE01F ///< 0xE01F = Incremental backup file number provided during a restore is invalid. +#define NE_SFLM_ILLEGAL_OP 0xE020 ///< 0xE020 = Attempt to perform an illegal operation. +#define NE_SFLM_ILLEGAL_TRANS_TYPE 0xE021 ///< 0xE021 = Illegal transaction type specified for transaction begin operation. +#define NE_SFLM_UNSUPPORTED_VERSION 0xE022 ///< 0xE022 = Version of database found in database header is not supported. +#define NE_SFLM_ILLEGAL_TRANS_OP 0xE023 ///< 0xE023 = Illegal operation for transaction type. +#define NE_SFLM_INCOMPLETE_LOG 0xE024 ///< 0xE024 = Incomplete rollback log. +#define NE_SFLM_BAD_RFL_SERIAL_NUM 0xE025 ///< 0xE025 = Serial number in roll-forward log file header does not match expected serial number. +#define NE_SFLM_NEWER_FLAIM 0xE026 ///< 0xE026 = Running old code on a newer version of database.\ Newer code must be used. +#define NE_SFLM_NO_TRANS_ACTIVE 0xE027 ///< 0xE027 = Operation must be performed inside a database transaction. +#define NE_SFLM_NOT_UNIQUE 0xE028 ///< 0xE028 = Attempt was made to insert a key into a b-tree that was already in the b-tree. +#define NE_SFLM_NOT_FLAIM 0xE029 ///< 0xE029 = The file specified is not a FLAIM database. +#define NE_SFLM_OLD_VIEW 0xE02A ///< 0xE02A = Unable to maintain read transaction's view of the database. +#define NE_SFLM_SHARED_LOCK 0xE02B ///< 0xE02B = Attempted to perform an operation on the database that requires exclusive access, but cannot because there is a shared lock. +#define NE_SFLM_TRANS_ACTIVE 0xE02C ///< 0xE02C = Operation cannot be performed while a transaction is active. +#define NE_SFLM_RFL_TRANS_GAP 0xE02D ///< 0xE02D = A gap was found in the transaction sequence in the roll-forward log. +#define NE_SFLM_BAD_COLLATED_KEY 0xE02E ///< 0xE02E = Something in collated key is bad. +#define NE_SFLM_UNSUPPORTED_FEATURE 0xE02F ///< 0xE02F = Attempting to use a feature for which full support has been disabled. +#define NE_SFLM_MUST_DELETE_INDEXES 0xE030 ///< 0xE030 = Attempting to drop a table that has indexes defined for it.\ Associated indexes must be deleted before the table can be dropped. +#define NE_SFLM_RFL_INCOMPLETE 0xE031 ///< 0xE031 = Roll-forward log file is incomplete. +#define NE_SFLM_CANNOT_RESTORE_RFL_FILES 0xE032 ///< 0xE032 = Cannot restore roll-forward log files - not using multiple roll-forward log files. +#define NE_SFLM_INCONSISTENT_BACKUP 0xE033 ///< 0xE033 = A problem (corruption, etc.\ ) was detected in a backup set. +#define NE_SFLM_BLOCK_CRC 0xE034 ///< 0xE034 = CRC for database block was invalid.\ May indicate problems in reading from or writing to disk. +#define NE_SFLM_ABORT_TRANS 0xE035 ///< 0xE035 = Attempted operation after a critical error - transaction should be aborted. +#define NE_SFLM_NOT_RFL 0xE036 ///< 0xE036 = File was not a roll-forward log file as expected. +#define NE_SFLM_BAD_RFL_PACKET 0xE037 ///< 0xE037 = Roll-forward log file packet was bad. +#define NE_SFLM_DATA_PATH_MISMATCH 0xE038 ///< 0xE038 = Bad data path specified to open database.\ Does not match data path specified for prior opens of the database. +#define NE_SFLM_DATABASE_EXISTS 0xE039 ///< 0xE039 = Attempt to create a database, but the database already exists. +#define NE_SFLM_COULD_NOT_CREATE_SEMAPHORE 0xE03A ///< 0xE03A = Could not create a semaphore. +#define NE_SFLM_MUST_CLOSE_DATABASE 0xE03B ///< 0xE03B = Database must be closed due to a critical error. +#define NE_SFLM_INVALID_ENCKEY_CRC 0xE03C ///< 0xE03C = Encryption key CRC could not be verified. +#define NE_SFLM_BAD_UTF8 0xE03D ///< 0xE03D = An invalid byte sequence was found in a UTF-8 string. +#define NE_SFLM_COULD_NOT_CREATE_MUTEX 0xE03E ///< 0xE03E = Could not create a mutex. +#define NE_SFLM_ERROR_WAITING_ON_SEMPAHORE 0xE03F ///< 0xE03F = Error occurred while waiting on a sempahore. +#define NE_SFLM_BAD_PLATFORM_FORMAT 0xE040 ///< 0xE040 = Cannot support platform format.\ NOTE: This error is internal only. +#define NE_SFLM_HDR_CRC 0xE041 ///< 0xE041 = Database header has a bad CRC. +#define NE_SFLM_NO_NAME_TABLE 0xE042 ///< 0xE042 = No name table was set up for the database. +#define NE_SFLM_UNALLOWED_UPGRADE 0xE043 ///< 0xE043 = Cannot upgrade database from one version to another. +#define NE_SFLM_BTREE_BAD_STATE 0xE044 ///< 0xE044 = Btree function called before proper setup steps taken. +#define NE_SFLM_TOO_MANY_OPEN_DATABASES 0xE045 ///< 0xE045 = Too many open databases, cannot open another one. +#define NE_SFLM_DATABASE_OPEN 0xE046 ///< 0xE046 = Operation cannot be performed because the database is currently open. +#define NE_SFLM_CACHE_ERROR 0xE047 ///< 0xE047 = Cached database block has been compromised while in cache. +#define NE_SFLM_BTREE_KEY_SIZE 0xE048 ///< 0xE048 = Key too large to insert/lookup in a b-tree. +#define NE_SFLM_DB_FULL 0xE049 ///< 0xE049 = Database is full, cannot create more blocks. +#define NE_SFLM_QUERY_SYNTAX 0xE04A ///< 0xE04A = Query expression had improper syntax. +#define NE_SFLM_COULD_NOT_START_THREAD 0xE04B ///< 0xE04B = Error occurred while attempting to start a thread. +#define NE_SFLM_INDEX_OFFLINE 0xE04C ///< 0xE04C = Index is offline, cannot be used in a query. +#define NE_SFLM_RFL_DISK_FULL 0xE04D ///< 0xE04D = Disk which contains roll-forward log is full. +#define NE_SFLM_MUST_WAIT_CHECKPOINT 0xE04E ///< 0xE04E = Must wait for a checkpoint before starting transaction - due to disk problems - usually in disk containing roll-forward log files. +#define NE_SFLM_BAD_SEN 0xE04F ///< 0xE04F = Invalid simple encoded number. +#define NE_SFLM_RFL_FILE_NOT_FOUND 0xE050 ///< 0xE050 = Could not open a roll-forward log file - was not found in the roll-forward log directory. +#define NE_SFLM_BAD_RCODE_TABLE 0xE051 ///< 0xE051 = The error code tables are incorrect.\ NOTE: This is an internal error only. +#define NE_SFLM_BUFFER_OVERFLOW 0xE052 ///< 0xE052 = Buffer overflow. +#define NE_SFLM_KEY_OVERFLOW 0xE053 ///< 0xE053 = Generated index key too large. +#define NE_SFLM_NO_MORE_ENCDEF_NUMS 0xE054 ///< 0xE054 = The highest encryption definition number has already been used cannot create more. +#define NE_SFLM_TABLE_OFFLINE 0xE055 ///< 0xE055 = Table is encrypted, cannot be accessed while in operating in limited mode. +#define NE_SFLM_READ_ONLY 0xE056 ///< 0xE056 = Column is read-only and cannot be updated. +#define NE_SFLM_DELETE_NOT_ALLOWED 0xE057 ///< 0xE057 = Row cannot be deleted. +#define NE_SFLM_RESET_NEEDED 0xE058 ///< 0xE058 = Used during check operations to indicate we need to reset the view.\ NOTE: This is an internal error code. +#define NE_SFLM_ILLEGAL_LANGUAGE 0xE059 ///< 0xE059 = Invalid language specified in an index definition. +#define NE_SFLM_ROW_DELETED 0xE05A ///< 0xE05A = Row being accessed has been deleted. +#define NE_SFLM_ROW_NOT_FOUND 0xE05B ///< 0xE05B - Row being retrieved does not exist. + +// Dictionary definition errors. + +#define NE_SFLM_ENCDEF_NAME_ALREADY_DEFINED 0xE100 ///< 0xE100 = Encryption definition name already defined.\ Cannot use again. +#define NE_SFLM_NULL_ENCDEF_NAME 0xE101 ///< 0xE101 = Encryption definition name may not be null. +#define NE_SFLM_NULL_ENCDEF_NUM 0xE102 ///< 0xE102 = Encryption definition number is null.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_ENCDEF_NAME 0xE103 ///< 0xE103 = Duplicate encryption definition name in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_ENCDEF_NUM 0xE104 ///< 0xE104 = Duplicate encryption definition number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_ENCDEF_NUM 0xE105 ///< 0xE105 = Invalid encryption definition number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_NULL_ENC_ALGORITHM 0xE106 ///< 0xE106 = Encryption algorithm may not be null. +#define NE_SFLM_INVALID_ENC_ALGORITHM 0xE107 ///< 0xE107 = Invalid encryption algorithm specified in encryption definition. +#define NE_SFLM_NULL_ENC_KEY_SIZE 0xE108 ///< 0xE108 = Encryption key size may not be null. +#define NE_SFLM_INVALID_ENC_KEY_SIZE 0xE109 ///< 0xE109 = Invalid key size specified in encryption definition. +#define NE_SFLM_NULL_ENC_KEY 0xE10A ///< 0xE10A = Encryption key is null.\ Dictionary is corrupt. + +#define NE_SFLM_TABLE_NAME_ALREADY_DEFINED 0xE10B ///< 0xE10B = Table name already defined.\ Cannot use again. +#define NE_SFLM_NULL_TABLE_NAME 0xE10C ///< 0xE10C = Table name may not be null. +#define NE_SFLM_NULL_TABLE_NUM 0xE10D ///< 0xE10D = Table number is null.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_TABLE_NAME 0xE10E ///< 0xE10E = Duplicate table name in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_TABLE_NUM 0xE10F ///< 0xE10F = Duplicate table number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_TABLE_NUM 0xE110 ///< 0xE110 = Invalid table number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_NULL_NUM_COLUMNS 0xE111 ///< 0xE111 = Number of columns for table is null.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_NUM_COLUMNS 0xE112 ///< 0xE112 = Number of columns for table is invalid.\ Dictionary is corrupt. + +#define NE_SFLM_COLUMN_NAME_ALREADY_DEFINED 0xE113 ///< 0xE113 = Column name already defined in table.\ Cannot use again. +#define NE_SFLM_NULL_COLUMN_NAME 0xE114 ///< 0xE114 = Column name may not be null. +#define NE_SFLM_NULL_COLUMN_NUM 0xE115 ///< 0xE115 = Column number is null.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_COLUMN_NAME 0xE116 ///< 0xE116 = Duplicate column name found for table in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_COLUMN_NUM 0xE117 ///< 0xE117 = Duplicate column number found for table in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_COLUMN_NUM 0xE118 ///< 0xE118 = Invalid column number for table in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_UNDEFINED_COLUMN_NUM 0xE119 ///< 0xE119 = Required column number for table not defined in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_NULL_DATA_TYPE 0xE11A ///< 0xE11A = Data type may not be null. +#define NE_SFLM_ILLEGAL_DATA_TYPE 0xE11B ///< 0xE11B = Data type specified for column definition is illegal. +#define NE_SFLM_INVALID_READ_ONLY_VALUE 0xE11C ///< 0xE11C = Read-only value specified for column definition is illegal. +#define NE_SFLM_INVALID_NULL_ALLOWED_VALUE 0xE11D ///< 0xE11D = Null-allowed value specified for colum definition is illegal. + +#define NE_SFLM_INDEX_NAME_ALREADY_DEFINED 0xE11E ///< 0xE11E = Index name already defined.\ Cannot use again. +#define NE_SFLM_NULL_INDEX_NAME 0xE11F ///< 0xE11F = Index name may not be null. +#define NE_SFLM_NULL_INDEX_NUM 0xE120 ///< 0xE120 = Index number is null.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_INDEX_NAME 0xE121 ///< 0xE121 = Duplicate index name in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_DUPLICATE_INDEX_NUM 0xE122 ///< 0xE122 = Duplicate index number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_INDEX_NUM 0xE123 ///< 0xE123 = Invalid index number in dictionary.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_COMPARE_RULE 0xE124 ///< 0xE124 = Invalid comparison rule in index definition.\ Dictionary is corrupt. +#define NE_SFLM_INVALID_INDEX_OPTION 0xE125 ///< 0xE125 = Invalid index option specified on index definition. +#define NE_SFLM_INVALID_INDEX_STATE 0xE126 ///< 0xE126 = Invalid index state specified for an index. +#define NE_SFLM_DUPLICATE_KEY_COMPONENT 0xE127 ///< 0xE127 = Duplicate key component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_INVALID_KEY_COMPONENT 0xE128 ///< 0xE128 = Invalid key component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_UNDEFINED_KEY_COMPONENT 0xE129 ///< 0xE129 = Undefined key component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_DUPLICATE_DATA_COMPONENT 0xE12A ///< 0xE12A = Duplicate data component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_INVALID_DATA_COMPONENT 0xE12B ///< 0xE12B = Invalid data component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_UNDEFINED_DATA_COMPONENT 0xE12C ///< 0xE12C = Undefined data component in internal index definition.\ Data dictionary is corrupt. +#define NE_SFLM_NO_COMP_NUM_FOR_INDEX_COLUMN 0xE12D ///< 0xE12D = Neither key component or data component specified for index column.\ Data dictionary is corrupt. +#define NE_SFLM_INVALID_INDEX_ON_VALUE 0xE12E ///< 0xE12E = Invalid value for the "index on" option in an index definition. +#define NE_SFLM_INVALID_COMPARE_RULES_VALUE 0xE12F ///< 0xE12F = Invalid value for the "compare rules" option in an index definition. +#define NE_SFLM_INVALID_SORT_DESCENDING_VALUE 0xE130 ///< 0xE130 = Invalid value for the "sort descending" option in an index definition. +#define NE_SFLM_INVALID_SORT_MISSING_HIGH_VALUE 0xE131 ///< 0xE131 = Invalid value for the "sort missing high" option in an index definition. + +// Query Errors + +#define NE_SFLM_Q_UNMATCHED_RPAREN 0xE200 ///< 0xE200 = Query syntax error: Unmatched right paren. +#define NE_SFLM_Q_UNEXPECTED_LPAREN 0xE201 ///< 0xE201 = Query syntax error: Unexpected left paren. +#define NE_SFLM_Q_UNEXPECTED_RPAREN 0xE202 ///< 0xE202 = Query syntax error: Unexpected right paren. +#define NE_SFLM_Q_EXPECTING_OPERAND 0xE203 ///< 0xE203 = Query syntax error: Expecting an operand. +#define NE_SFLM_Q_EXPECTING_OPERATOR 0xE204 ///< 0xE204 = Query syntax error: Expecting an operator. +#define NE_SFLM_Q_UNEXPECTED_VALUE 0xE205 ///< 0xE205 = Query syntax error: Unexpected value. +#define NE_SFLM_Q_ILLEGAL_OPERAND 0xE206 ///< 0xE206 = Query syntax error: Operand for some operator is not valid for that operator type. +#define NE_SFLM_Q_INCOMPLETE_QUERY_EXPR 0xE207 ///< 0xE207 = Query syntax error: Query expression is incomplete. +#define NE_SFLM_Q_INVALID_ROW_ID_VALUE 0xE208 ///< 0xE208 = Query syntax error: Invalid type of value constant used for row id value comparison. +#define NE_SFLM_Q_ALREADY_OPTIMIZED 0xE209 ///< 0xE209 = Operation is illegal, cannot change certain things after query has been optimized. +#define NE_SFLM_Q_MISMATCHED_DB 0xE20A ///< 0xE20A = Database handle passed in does not match database associated with query. +#define NE_SFLM_Q_NOT_POSITIONED 0xE20B ///< 0xE20B = Query not positioned due to previous error, cannot call getNext, getPrev, or getCurrent. +#define NE_SFLM_Q_BAD_SEARCH_STRING 0xE20C ///< 0xE20C = String in query criteria is bad. +#define NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH 0xE20D ///< 0xE20D = Types of operands in a comparison operator are incompatible, cannot be compared. +#define NE_SFLM_Q_INVALID_OPERATOR 0xE20E ///< 0xE20E = Invalid operator in query expression. +#define NE_Q_INVALID_ROW_ID_VALUE 0xE20F ///< 0xE20F = Could not get ROW ID from constant in query expression. +#define NE_SFLM_Q_EXPECTING_LPAREN 0xE210 ///< 0xE210 = Expecting a left parenthesis in query expression. +#define NE_SFLM_Q_ILLEGAL_OPERATOR 0xE211 ///< 0xE211 = Illegal operator specified in query expression. +#define NE_SFLM_Q_ILLEGAL_COMPARE_RULES 0xE212 ///< 0xE212 = Illegal combination comparison rules specified in query expression. +#define NE_SFLM_Q_UNEXPECTED_COLUMN 0xE213 ///< 0xE213 = Not expecting a column name in query expression. + +// Desc: NICI / Encryption Errors + +#define NE_SFLM_NICI_CONTEXT 0xE300 ///< 0xE300 = Error occurred while creating NICI context for encryption/decryption. +#define NE_SFLM_NICI_ATTRIBUTE_VALUE 0xE301 ///< 0xE301 = Error occurred while accessing an attribute on a NICI encryption key. +#define NE_SFLM_NICI_BAD_ATTRIBUTE 0xE302 ///< 0xE302 = Value retrieved from an attribute on a NICI encryption key was bad. +#define NE_SFLM_NICI_WRAPKEY_FAILED 0xE303 ///< 0xE303 = Error occurred while wrapping a NICI encryption key in another NICI encryption key. +#define NE_SFLM_NICI_UNWRAPKEY_FAILED 0xE304 ///< 0xE304 = Error occurred while unwrapping a NICI encryption key that is wrapped in another NICI encryption key. +#define NE_SFLM_NICI_INVALID_ALGORITHM 0xE305 ///< 0xE305 = Attempt to use invalid NICI encryption algorithm. +#define NE_SFLM_NICI_GENKEY_FAILED 0xE306 ///< 0xE306 = Error occurred while attempting to generate a NICI encryption key. +#define NE_SFLM_NICI_BAD_RANDOM 0xE307 ///< 0xE307 = Error occurred while generating random data using NICI. +#define NE_SFLM_PBE_ENCRYPT_FAILED 0xE308 ///< 0xE308 = Error occurred while attempting to wrap a NICI encryption key in a password. +#define NE_SFLM_PBE_DECRYPT_FAILED 0xE309 ///< 0xE309 = Error occurred while attempting to unwrap a NICI encryption key that was previously wrapped in a password. +#define NE_SFLM_DIGEST_INIT_FAILED 0xE30A ///< 0xE30A = Error occurred while attempting to initialize the NICI digest functionality. +#define NE_SFLM_DIGEST_FAILED 0xE30B ///< 0xE30B = Error occurred while attempting to create a NICI digest. +#define NE_SFLM_INJECT_KEY_FAILED 0xE30C ///< 0xE30C = Error occurred while attempting to inject an encryption key into NICI. +#define NE_SFLM_NICI_FIND_INIT 0xE30D ///< 0xE30D = Error occurred while attempting to initialize NICI to find information on a NICI encryption key. +#define NE_SFLM_NICI_FIND_OBJECT 0xE30E ///< 0xE30E = Error occurred while attempting to find information on a NICI encryption key. +#define NE_SFLM_NICI_KEY_NOT_FOUND 0xE30F ///< 0xE30F = Could not find the NICI encryption key or information on the NICI encryption key. +#define NE_SFLM_NICI_ENC_INIT_FAILED 0xE310 ///< 0xE310 = Error occurred while initializing NICI to encrypt data. +#define NE_SFLM_NICI_ENCRYPT_FAILED 0xE311 ///< 0xE311 = Error occurred while encrypting data. +#define NE_SFLM_NICI_DECRYPT_INIT_FAILED 0xE312 ///< 0xE312 = Error occurred while initializing NICI to decrypt data. +#define NE_SFLM_NICI_DECRYPT_FAILED 0xE313 ///< 0xE313 = Error occurred while decrypting data. +#define NE_SFLM_NICI_WRAPKEY_NOT_FOUND 0xE314 ///< 0xE314 = Could not find the NICI encryption key used to wrap another NICI encryption key. +#define NE_SFLM_NOT_EXPECTING_PASSWORD 0xE315 ///< 0xE315 = Password supplied when none was expected. +#define NE_SFLM_EXPECTING_PASSWORD 0xE316 ///< 0xE316 = No password supplied when one was required. +#define NE_SFLM_EXTRACT_KEY_FAILED 0xE317 ///< 0xE317 = Error occurred while attempting to extract a NICI encryption key. +#define NE_SFLM_NICI_INIT_FAILED 0xE318 ///< 0xE318 = Error occurred while initializing NICI. +#define NE_SFLM_BAD_ENCKEY_SIZE 0xE319 ///< 0xE319 = Bad encryption key size found in roll-forward log packet. +#define NE_SFLM_ENCRYPTION_UNAVAILABLE 0xE31A ///< 0xE31A = Attempt was made to encrypt data when NICI is unavailable. + + +// VARIOUS INTERFACES FOR CALLBACKS, ETC. + +// Note: Any interfaces ending in Client or Status are interfaces +// that FlaimSQL does not provide implementations of. They exist to +// allow FlaimSQL to pass data back to the client. Interfaces ending in +// Status are, generally, informational only, while interfaces ending +// in Client exist to allow the client to modify the data or take +// other action. + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_BackupClient : public F_Object +{ + virtual RCODE WriteData( + const void * pvBuffer, + FLMUINT uiBytesToWrite) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_BackupStatus : public F_Object +{ + virtual RCODE backupStatus( + FLMUINT64 ui64BytesToDo, + FLMUINT64 ui64BytesDone) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_RestoreStatus : public F_Object +{ + virtual RCODE reportProgress( + eRestoreAction * peAction, + FLMUINT64 ui64BytesToDo, + FLMUINT64 ui64BytesDone) = 0; + + virtual RCODE reportError( + eRestoreAction * peAction, + RCODE rcErr) = 0; + + virtual RCODE reportOpenRflFile( + eRestoreAction * peAction, + FLMUINT uiFileNum) = 0; + + virtual RCODE reportRflRead( + eRestoreAction * peAction, + FLMUINT uiFileNum, + FLMUINT uiBytesRead) = 0; + + virtual RCODE reportBeginTrans( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportCommitTrans( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportAbortTrans( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportBlockChainFree( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT64 ui64MaintDocNum, + FLMUINT uiStartBlkAddr, + FLMUINT uiEndBlkAddr, + FLMUINT uiCount) = 0; + + virtual RCODE reportIndexSuspend( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT uiIndexNum) = 0; + + virtual RCODE reportIndexResume( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT uiIndexNum) = 0; + + virtual RCODE reportReduce( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT uiCount) = 0; + + virtual RCODE reportUpgrade( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT uiOldDbVersion, + FLMUINT uiNewDbVersion) = 0; + + virtual RCODE reportEnableEncryption( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportWrapKey( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportRollOverDbKey( + eRestoreAction * peAction, + FLMUINT64 ui64TransId) = 0; + + virtual RCODE reportSetNextRowId( + eRestoreAction * peAction, + FLMUINT64 ui64TransId, + FLMUINT uiTableNum, + FLMUINT64 ui64NextRowId) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_RestoreClient : public F_Object +{ + virtual RCODE openBackupSet( void) = 0; + + virtual RCODE openRflFile( // Open an RFL file + FLMUINT uiFileNum) = 0; + + virtual RCODE openIncFile( // Open an incremental backup file + FLMUINT uiFileNum) = 0; + + virtual RCODE read( + FLMUINT uiLength, // Number of bytes to read + void * pvBuffer, // Buffer to place read bytes into + FLMUINT * puiBytesRead) = 0; // [out] Number of bytes read + + virtual RCODE close( void) = 0; // Close the current file + + virtual RCODE abortFile( void) = 0; // Abort processing the file + // and close file handles, etc. +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_UpgradeClient : public F_Object +{ +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_DbCopyStatus : public F_Object +{ + virtual RCODE dbCopyStatus( + FLMUINT64 ui64BytesToCopy, + FLMUINT64 ui64BytesCopied, + FLMBOOL bNewSrcFile, + const char * pszSrcFileName, + const char * pszDestFileName) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_DbRebuildStatus : public F_Object +{ + virtual RCODE reportRebuild( + SFLM_REBUILD_INFO * pRebuild) = 0; + + virtual RCODE reportRebuildErr( + SFLM_CORRUPT_INFO * pCorruptInfo) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_DbCheckStatus : public F_Object +{ + virtual RCODE reportProgress( + SFLM_PROGRESS_CHECK_INFO * pProgCheck) = 0; + + virtual RCODE reportCheckErr( + SFLM_CORRUPT_INFO * pCorruptInfo, + FLMBOOL * pbFix) = 0; + // [OUT] - If the client sets this to true, then FlaimSQL will + // attempt to fix the problem. NOTE: It is allowable for + // FlaimSQL to pass in NULL here!! (This means that the client + // doesn't have a choice regarding FlaimSQL's actions.) The + // client must check for NULL before attempting to assign a + // value to this parameter!! +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_DbRenameStatus : public F_Object +{ + virtual RCODE dbRenameStatus( + const char * pszSrcFileName, + const char * pszDstFileName) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_IxStatus : public F_Object +{ + virtual RCODE reportIndex( + FLMUINT64 ui64LastRowId) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_DbInfo : public F_Object +{ + virtual FLMUINT getNumTables( void) = 0; + + virtual FLMUINT getNumIndexes( void) = 0; + + virtual FLMUINT getNumLogicalFiles( void) = 0; + + virtual FLMUINT64 getFileSize( void) = 0; + + virtual const SFLM_DB_HDR * getDbHdr( void) = 0; + + virtual void getAvailBlockStats( + FLMUINT64 * pui64BytesUsed, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors) = 0; + + virtual void getLFHBlockStats( + FLMUINT64 * pui64BytesUsed, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors) = 0; + + virtual void getBTreeInfo( + FLMUINT uiNthLogicalFile, + FLMUINT * puiLfNum, + eLFileType * peLfType, + FLMUINT * puiRootBlkAddress, + FLMUINT * puiNumLevels) = 0; + + virtual void getBTreeBlockStats( + FLMUINT uiNthLogicalFile, + FLMUINT uiLevel, + FLMUINT64 * pui64KeyCount, + FLMUINT64 * pui64BytesUsed, + FLMUINT64 * pui64ElementCount, + FLMUINT64 * pui64ContElementCount, + FLMUINT64 * pui64ContElmBytes, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_QueryStatus : public F_Object +{ + virtual RCODE queryStatus( + SQL_OPT_INFO * pOptInfo) = 0; + + virtual RCODE newSource( + SQL_OPT_INFO * pOptInfo) = 0; + + virtual RCODE resultSetStatus( + FLMUINT64 ui64TotalDocsRead, + FLMUINT64 ui64TotalDocsPassed, + FLMBOOL bCanRetrieveDocs) = 0; + + virtual RCODE resultSetComplete( + FLMUINT64 ui64TotalDocsRead, + FLMUINT64 ui64TotalDocsPassed) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_CommitClient : public F_Object +{ + virtual void commit( + F_Db * pDb) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_EventClient : public F_Object +{ + virtual void catchEvent( + eEventType eEvent, + F_Db * pDb, + FLMUINT uiThreadId, + FLMUINT64 ui64TransID, + FLMUINT uiIndexOrTable, + FLMUINT64 ui64RowId, + RCODE rc) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_IxClient : public F_Object +{ + virtual RCODE doIndexing( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMUINT uiTableNum, + F_Row * pRow) = 0; +}; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_LockInfoClient : public F_Object +{ + virtual FLMBOOL setLockCount( // Return TRUE to continue, FALSE to stop + FLMUINT uiTotalLocks) = 0; + + virtual FLMBOOL addLockInfo( // Return TRUE to continue, FALSE to stop + FLMUINT uiLockNum, // Position in queue (0 = lock holder, + // 1 ... n = lock waiter) + FLMUINT uiThreadID, // Thread ID of the lock holder/waiter + FLMUINT uiTime) = 0; // For the lock holder, this is the + // time when the lock was obtained. + // For a lock waiter, this is the time + // that the waiter was placed in the queue. + // Both times are presented in milliseconds. +}; + +/**************************************************************************** +Desc: Types of information that can be gathered about a B-Tree. +****************************************************************************/ +typedef struct SFLM_BTREE_LEVEL_INFO +{ + FLMUINT64 ui64BlockCount; + FLMUINT64 ui64BlockLength; + FLMUINT64 ui64BlockFreeSpace; + FLMUINT64 ui64ElmOffsetOverhead; + FLMUINT64 ui64ElmCount; + FLMUINT64 ui64ContElmCount; + FLMUINT64 ui64ElmFlagOvhd; + FLMUINT64 ui64ElmKeyLengthOvhd; + FLMUINT64 ui64ElmCountsOvhd; + FLMUINT64 ui64ElmChildAddrsOvhd; + FLMUINT64 ui64ElmDataLenOvhd; + FLMUINT64 ui64ElmOADataLenOvhd; + FLMUINT64 ui64ElmKeyLength; + FLMUINT64 ui64ElmDataLength; + + // The following three are how ui64ElmKeyLength is subdivided. + // They are only applicable to index keys. + + FLMUINT64 ui64KeyDataSize; + FLMUINT64 ui64KeyIdSize; + FLMUINT64 ui64KeyComponentLengthsSize; + + // Data only blocks + + FLMUINT64 ui64DataOnlyBlockCount; + FLMUINT64 ui64DataOnlyBlockLength; + FLMUINT64 ui64DataOnlyBlockFreeSpace; +} SFLM_BTREE_LEVEL_INFO; + +/**************************************************************************** +Desc: +****************************************************************************/ +flminterface IF_BTreeInfoStatus : public F_Object +{ + virtual RCODE infoStatus( + FLMUINT uiCurrLfNum, + FLMBOOL bIsTable, + char * pszCurrLfName, + FLMUINT uiCurrLevel, + FLMUINT64 ui64CurrLfBlockCount, + FLMUINT64 ui64CurrLevelBlockCount, + FLMUINT64 ui64TotalBlockCount) = 0; +}; + +/**************************************************************************** +Desc: BTree Info. Gatherer +****************************************************************************/ +flminterface IF_BTreeInfo : public F_Object +{ + virtual void clearBTreeInfo( void) = 0; + + virtual RCODE collectIndexInfo( + F_Db * pDb, + FLMUINT uiIndexNum, + IF_BTreeInfoStatus * pInfoStatus) = 0; + + virtual RCODE collectTableInfo( + F_Db * pDb, + FLMUINT uiTableNum, + IF_BTreeInfoStatus * pInfoStatus) = 0; + + virtual FLMUINT getNumIndexes( void) = 0; + + virtual FLMUINT getNumTables( void) = 0; + + virtual FLMBOOL getIndexInfo( + FLMUINT uiNthIndex, + FLMUINT * puiIndexNum, + char ** ppszIndexName, + FLMUINT * puiNumLevels) = 0; + + virtual FLMBOOL getTableInfo( + FLMUINT uiNthCollection, + FLMUINT * puiTableNum, + char ** ppszTableName, + FLMUINT * puiNumLevels) = 0; + + virtual FLMBOOL getIndexLevelInfo( + FLMUINT uiNthIndex, + FLMUINT uiBTreeLevel, + SFLM_BTREE_LEVEL_INFO * pLevelInfo) = 0; + + virtual FLMBOOL getTableLevelInfo( + FLMUINT uiNthTable, + FLMUINT uiBTreeLevel, + SFLM_BTREE_LEVEL_INFO * pLevelInfo) = 0; +}; + +#endif // FLAIMSQL_H diff --git a/sql/src/flaimsys.h b/sql/src/flaimsys.h new file mode 100644 index 0000000..22b0b84 --- /dev/null +++ b/sql/src/flaimsys.h @@ -0,0 +1,4359 @@ +//------------------------------------------------------------------------------ +// Desc: This is the master header file for FLAIM. It is included by nearly +// all of the source files. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#ifndef FLAIMSYS_H +#define FLAIMSYS_H + +// Public includes + +#include "flaimsql.h" + +#if defined( FLM_WIN) + // Conversion from XXX to YYY, possible loss of data + #pragma warning( disable : 4244) + + // Local variable XXX may be used without having been initialized + #pragma warning( disable : 4701) + + // Function XXX not inlined + #pragma warning( disable : 4710) +#endif + +#if defined( FLM_WATCOM_NLM) + + // Disable "Warning! W549: col(XX) 'sizeof' operand contains + // compiler generated information" + + #pragma warning 549 9 +#endif + +// Put all forward references here + +class F_Database; +class F_Dict; +class F_Db; +class F_NameTable; +class F_Rfl; +class F_Btree; +class F_Query; +class F_DbRebuild; +class F_DbCheck; +class F_DbInfo; +class F_KeyCollector; +class FSIndexCursor; +class FSTableCursor; +class FDynSearchSet; +class F_CachedBlock; +class F_Row; +class F_GlobalCacheMgr; +class F_BlockCacheMgr; +class F_RowCacheMgr; +class F_BTreeIStream; +class F_BTreeIStreamPool; +class F_QueryResultSet; +class F_BTreeInfo; +class F_RebuildRowIStream; +class F_DbInfo; + +// Some in-line functions included by other header files. + +// Misc. global constants + +#ifdef DEFINE_NUMBER_MAXIMUMS + #define GV_EXTERN +#else + #define GV_EXTERN extern +#endif + +GV_EXTERN FLMBOOL gv_b32BitPlatform +#ifdef DEFINE_NUMBER_MAXIMUMS + = (FLMBOOL)(sizeof( FLMUINT) == 4 ? TRUE : FALSE) +#endif + ; + +GV_EXTERN FLMUINT gv_uiMaxUInt32Val +#ifdef DEFINE_NUMBER_MAXIMUMS + = (FLMUINT)0xFFFFFFFF +#endif + ; + +GV_EXTERN FLMUINT gv_uiMaxUIntVal +#ifdef DEFINE_NUMBER_MAXIMUMS + = (FLMUINT)(~(FLMUINT)0) +#endif + ; + +GV_EXTERN FLMUINT gv_uiMaxSignedIntVal +#ifdef DEFINE_NUMBER_MAXIMUMS + = (FLMUINT)((((~(FLMUINT)0) << 1) >> 1)) +#endif + ; + +GV_EXTERN FLMUINT64 gv_ui64MaxSignedIntVal +#ifdef DEFINE_NUMBER_MAXIMUMS + = (FLMUINT64)((((~(FLMUINT64)0) << 1) >> 1)) +#endif + ; + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToUINT( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMUINT * puiNum) +{ + if( bNeg) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + + if( gv_b32BitPlatform && ui64Num > 0xFFFFFFFF) + { + return( RC_SET( NE_SFLM_CONV_NUM_OVERFLOW)); + } + + *puiNum = (FLMUINT)ui64Num; + return( NE_SFLM_OK); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToUINT32( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMUINT32 * pui32Num) +{ + if( bNeg) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + + if( ui64Num > 0xFFFFFFFF) + { + return( RC_SET( NE_SFLM_CONV_NUM_OVERFLOW)); + } + + *pui32Num = (FLMUINT32)ui64Num; + return( NE_SFLM_OK); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToUINT64( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMUINT64 * pui64Num) +{ + if( bNeg) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + + *pui64Num = ui64Num; + return( NE_SFLM_OK); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToINT( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMINT * piNum) +{ + if( bNeg) + { + if (ui64Num == (FLMUINT64)(FLM_MAX_INT) + 1) + { + *piNum = FLM_MIN_INT; + } + else if( ui64Num > (FLMUINT64)(FLM_MAX_INT) + 1) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + else + { + *piNum = -((FLMINT)ui64Num); + } + } + else + { + if( ui64Num > (FLMUINT64)FLM_MAX_INT) + { + return( RC_SET( NE_SFLM_CONV_NUM_OVERFLOW)); + } + + *piNum = (FLMINT)ui64Num; + } + + return( NE_SFLM_OK); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToINT32( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMINT32 * pi32Num) +{ + if( bNeg) + { + if (ui64Num == (FLMUINT64)(FLM_MAX_INT32) + 1) + { + *pi32Num = FLM_MIN_INT32; + } + else if( ui64Num > (FLMUINT64)(FLM_MAX_INT32) + 1) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + else + { + *pi32Num = -((FLMINT32)ui64Num); + } + } + else + { + if( ui64Num > (FLMUINT64)FLM_MAX_INT32) + { + return( RC_SET( NE_SFLM_CONV_NUM_OVERFLOW)); + } + + *pi32Num = (FLMINT32)ui64Num; + } + + return( NE_SFLM_OK); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FINLINE RCODE convertToINT64( + FLMUINT64 ui64Num, + FLMBOOL bNeg, + FLMINT64 * pi64Num) +{ + if( bNeg) + { + if( ui64Num > gv_ui64MaxSignedIntVal + 1) + { + return( RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW)); + } + + *pi64Num = -(FLMINT64)ui64Num; + } + else + { + if( ui64Num > gv_ui64MaxSignedIntVal) + { + return( RC_SET( NE_SFLM_CONV_NUM_OVERFLOW)); + } + + *pi64Num = (FLMINT64)ui64Num; + } + + return( NE_SFLM_OK); +} + +// NOTE: ENCRYPT_MIN_CHUNK_SIZE should always be a power of 2 +// getEncLen supposes that it is. + +#define ENCRYPT_MIN_CHUNK_SIZE 16 +#define ENCRYPT_BOUNDARY_MASK (~(ENCRYPT_MIN_CHUNK_SIZE - 1)) + +/***************************************************************************** +Desc: Calculate the number of bytes extra there are beyond the closest + encryption boundary. +******************************************************************************/ +FINLINE FLMUINT extraEncBytes( + FLMUINT uiDataLen) +{ + // This works if ENCRYPT_MIN_CHUNK_SIZE is a power of 2 + // Otherwise, it needs to be changed to uiDataLen % ENCRYPT_MIN_CHUNK_SIZE + + return( uiDataLen & (ENCRYPT_MIN_CHUNK_SIZE - 1)); +} + +/***************************************************************************** +Desc: Calculate the encryption length for a piece of data. +******************************************************************************/ +FINLINE FLMUINT getEncLen( + FLMUINT uiDataLen) +{ + return( extraEncBytes( uiDataLen) + ? ((uiDataLen + ENCRYPT_MIN_CHUNK_SIZE) & ENCRYPT_BOUNDARY_MASK) + : uiDataLen); +} + +/**************************************************************************** +Storage conversion functions. +****************************************************************************/ + +#define FLM_MAX_NUM_BUF_SIZE 9 + +RCODE flmStorage2Number( + eDataType eDataTyp, + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT * puiNum, + FLMINT * piNum); + +RCODE flmStorage2Number64( + eDataType eDataTyp, + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT64 * pui64Num, + FLMINT64 * pi64Num); + +RCODE flmNumber64ToStorage( + FLMUINT64 ui64Num, + FLMUINT * puiBufLen, + FLMBYTE * pucBuf, + FLMBOOL bNegative, + FLMBOOL bCollation); + +RCODE FlmUINT2Storage( + FLMUINT uiNum, + FLMUINT * puiBufLen, + FLMBYTE * pucBuf); + +RCODE FlmINT2Storage( + FLMINT iNum, + FLMUINT * puiBufLen, + FLMBYTE * pucBuf); + +RCODE flmUTF8ToStorage( + const FLMBYTE * pucUTF8, + FLMUINT uiBytesInBuffer, + FLMBYTE * pucBuf, + FLMUINT * puiBufLength); + +RCODE flmGetCharCountFromStorageBuf( + const FLMBYTE ** ppucBuf, + FLMUINT uiBufSize, + FLMUINT * puiNumChars, + FLMUINT * puiSenLen = NULL); + +RCODE flmStorage2UTF8( + FLMUINT uiType, + FLMUINT uiBufLength, + const FLMBYTE * pucBuffer, + FLMUINT * puiOutBufLen, + FLMBYTE * pucOutBuf); + +RCODE flmStorage2UTF8( + FLMUINT uiType, + FLMUINT uiBufLength, + const FLMBYTE * pucBuffer, + F_DynaBuf * pBuffer); + +RCODE flmStorage2Unicode( + FLMUINT uiType, + FLMUINT uiBufLength, + const FLMBYTE * pucBuffer, + FLMUINT * puiOutBufLen, + void * pOutBuf); + +RCODE flmStorage2Unicode( + FLMUINT uiType, + FLMUINT uiStorageLength, + const FLMBYTE * pucStorageBuffer, + F_DynaBuf * pBuffer); + +RCODE flmUnicode2Storage( + const FLMUNICODE * puzStr, + FLMUINT uiStrLen, + FLMBYTE * pucBuf, + FLMUINT * puiBufLength, + FLMUINT * puiCharCount); + +RCODE flmStorageNum2StorageText( + const FLMBYTE * pucNum, + FLMUINT uiNumLen, + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen); + +/**************************************************************************** +Desc: Returns the size of buffer needed to hold the unicode string in + FLAIM's storage format. +****************************************************************************/ +FINLINE RCODE FlmGetUnicodeStorageLength( + FLMUNICODE * puzStr, + FLMUINT * puiByteCount) +{ + FLMUINT uiByteCount; + RCODE rc; + + if( RC_BAD( rc = flmUnicode2Storage( puzStr, 0, NULL, + &uiByteCount, NULL))) + { + return( rc); + } + + *puiByteCount = uiByteCount + sizeof( FLMUNICODE); + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: Copies and formats a Unicode string into FLAIM's storage format. + The Unicode string must be in little-endian byte order. + Unicode values that are not represented as WordPerfect 6.x characters + are preserved as non-WP characters. +****************************************************************************/ +FINLINE RCODE FlmUnicode2Storage( + FLMUNICODE * puzStr, + FLMUINT * puiBufLength, // [IN] size of pBuf, + // [OUT] amount of pBuf used to hold puzStr + FLMBYTE * pBuf) +{ + return( flmUnicode2Storage( puzStr, 0, pBuf, puiBufLength, NULL)); +} + +/**************************************************************************** +Desc: Converts from FLAIM's internal storage format to a Unicode string +****************************************************************************/ +FINLINE RCODE FlmStorage2Unicode( + FLMUINT uiType, + FLMUINT uiBufLength, + FLMBYTE * pBuffer, + FLMUINT * puiOutBufLen, + FLMUNICODE * puzOutBuf) +{ + return( flmStorage2Unicode( uiType, uiBufLength, pBuffer, + puiOutBufLen, puzOutBuf)); +} + +/**************************************************************************** +Desc: Convert storage text into a null-terminated UTF-8 string +****************************************************************************/ +FINLINE RCODE FlmStorage2UTF8( + FLMUINT uiType, + FLMUINT uiBufLength, + FLMBYTE * pBuffer, + FLMUINT * puiOutBufLenRV, + // [IN] Specified the number of bytes available in buffer. + // [OUT] Returns the number of bytes of UTF-8 text. This value + // does not include a terminating NULL byte in the buffer. + FLMBYTE * pOutBuffer) + // [OUT] The buffer that will hold the output UTF-8 text. + // If this value is NULL then only bufLenRV will computed so that + // the caller may know how many bytes to allocate for a buffer. +{ + return( flmStorage2UTF8( uiType, uiBufLength, + pBuffer, puiOutBufLenRV, pOutBuffer)); +} + +// Internal includes + +#include "fcollate.h" +#include "fdict.h" +#include "fstructs.h" +#include "fcache.h" +#include "flmstat.h" +#include "fbtrset.h" +#include "fsrvlock.h" +#include "fcollate.h" +#include "f_btree.h" +#include "f_btpool.h" +#include "rfl.h" +#include "fsuperfl.h" +#include "filesys.h" +#include "flog.h" +#include "f_nici.h" + +/**************************************************************************** +Desc: +****************************************************************************/ +typedef struct FlmVectorElementTag +{ + FLMUINT uiColumnNum; + FLMUINT uiFlags; +#define VECT_SLOT_HAS_DATA 0x01 +#define VECT_SLOT_HAS_ID 0x02 +#define VECT_SLOT_RIGHT_TRUNCATED 0x04 +#define VECT_SLOT_LEFT_TRUNCATED 0x08 +#define VECT_SLOT_HAS_COLUMN_NUM 0x10 +#define VECT_SLOT_IS_ATTR 0x20 +#define VECT_SLOT_IS_DATA 0x40 + eDataType eDataTyp; + FLMUINT uiDataLength; + FLMUINT uiDataOffset; +} F_VECTOR_ELEMENT; + +/***************************************************************************** +Desc: Used to build keys and data components +*****************************************************************************/ +class F_DataVector : public F_Object +{ +public: + + // Constructor/Destructor + + F_DataVector(); + virtual ~F_DataVector(); + + // Setter methods + + FINLINE void setRowID( + FLMUINT64 ui64RowId) + { + m_ui64RowId = ui64RowId; + } + + RCODE setColumnNum( + FLMUINT uiElementNumber, + FLMUINT uiColumnNum, + FLMBOOL bIsData); + + RCODE setINT( + FLMUINT uiElementNumber, + FLMINT iNum); + + RCODE setINT64( + FLMUINT uiElementNumber, + FLMINT64 i64Num); + + RCODE setUINT( + FLMUINT uiElementNumber, + FLMUINT uiNum); + + RCODE setUINT64( + FLMUINT uiElementNumber, + FLMUINT64 ui64Num); + + RCODE setUnicode( + FLMUINT uiElementNumber, + const FLMUNICODE * puzUnicode); + + RCODE setUTF8( + FLMUINT uiElementNumber, + const FLMBYTE * pszUtf8, + FLMUINT uiBytesInBuffer = 0); + + FINLINE RCODE setBinary( + FLMUINT uiElementNumber, + const void * pvBinary, + FLMUINT uiBinaryLen) + { + return( storeValue( uiElementNumber, + SFLM_BINARY_TYPE, (FLMBYTE *)pvBinary, uiBinaryLen)); + } + + FINLINE void setRightTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + setRightTruncated( pVector); + } + else + { + // Need to set some data value before setting + // truncated. + flmAssert( 0); + } + } + + FINLINE void setLeftTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + setLeftTruncated( pVector); + } + else + { + // Need to set some data value before setting + // truncated. + flmAssert( 0); + } + } + + FINLINE void clearRightTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + clearRightTruncated( pVector); + } + else + { + // Need to set some data value before clearing + // truncated. + flmAssert( 0); + } + } + + FINLINE void clearLeftTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + clearLeftTruncated( pVector); + } + else + { + // Need to set some data value before clearing + // truncated. + flmAssert( 0); + } + } + + FINLINE FLMBOOL isRightTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + return( isRightTruncated( pVector)); + } + + return( FALSE); + } + + FINLINE FLMBOOL isLeftTruncated( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + return( isLeftTruncated( pVector)); + } + + return( FALSE); + } + + // Getter methods + + FINLINE FLMUINT64 getRowId( void) + { + return( m_ui64RowId); + } + + FINLINE FLMUINT getColumnNum( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_COLUMN_NUM)) == NULL) + { + return( 0); + } + else + { + return( pVector->uiColumnNum); + } + } + + FINLINE FLMBOOL isDataComponent( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_COLUMN_NUM)) == NULL) + { + return( FALSE); + } + else + { + return( (pVector->uiFlags & VECT_SLOT_IS_DATA) ? TRUE : FALSE); + } + } + + FINLINE FLMBOOL isKeyComponent( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_COLUMN_NUM)) == NULL) + { + return( FALSE); + } + else + { + return( (pVector->uiFlags & VECT_SLOT_IS_DATA) ? FALSE : TRUE); + } + } + + FLMUINT getDataLength( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) == NULL) + { + return( 0); + } + else + { + return( pVector->uiDataLength); + } + } + + eDataType getDataType( + FLMUINT uiElementNumber) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) == NULL) + { + return( SFLM_UNKNOWN_TYPE); + } + else + { + return( pVector->eDataTyp); + } + } + + RCODE getUTF8Ptr( + FLMUINT uiElementNumber, + const FLMBYTE ** ppszUTF8, + FLMUINT * puiBufLen); + + FINLINE RCODE getINT( + FLMUINT uiElementNumber, + FLMINT * piNum) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Number( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + NULL, piNum) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getINT64( + FLMUINT uiElementNumber, + FLMINT64 * pi64Num) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Number64( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + NULL, pi64Num) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getUINT( + FLMUINT uiElementNumber, + FLMUINT * puiNum) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Number( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + puiNum, NULL) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getUINT64( + FLMUINT uiElementNumber, + FLMUINT64 * pui64Num) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Number64( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + pui64Num, NULL) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + RCODE getUnicode( + FLMUINT uiElementNumber, + FLMUNICODE ** ppuzUnicode); + + FINLINE RCODE getUnicode( + FLMUINT uiElementNumber, + F_DynaBuf * pBuffer) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Unicode( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + pBuffer) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getUnicode( + FLMUINT uiElementNumber, + FLMUNICODE * puzUnicode, + FLMUINT * puiBufLen) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2Unicode( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + puiBufLen, puzUnicode) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getUTF8( + FLMUINT uiElementNumber, + FLMBYTE * pszUTF8, + FLMUINT * puiBufLen) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2UTF8( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + puiBufLen, pszUTF8) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getUTF8( + FLMUINT uiElementNumber, + F_DynaBuf * pBuffer) + { + F_VECTOR_ELEMENT * pVector; + + return( (RCODE)((pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA)) != NULL + ? flmStorage2UTF8( pVector->eDataTyp, + pVector->uiDataLength, + (FLMBYTE *)getDataPtr( pVector), + pBuffer) + : RC_SET( NE_SFLM_NOT_FOUND))); + } + + FINLINE RCODE getBinary( + FLMUINT uiElementNumber, + void * pvBuffer, + FLMUINT * puiLength) + { + F_VECTOR_ELEMENT * pVector; + + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + *puiLength = f_min( (*puiLength), pVector->uiDataLength); + if (pvBuffer && *puiLength) + { + f_memcpy( pvBuffer, getDataPtr( pVector), *puiLength); + } + + return( NE_SFLM_OK); + } + else + { + *puiLength = 0; + } + + return( RC_SET( NE_SFLM_NOT_FOUND)); + } + + FINLINE RCODE getBinary( + FLMUINT uiElementNumber, + F_DynaBuf * pBuffer) + { + F_VECTOR_ELEMENT * pVector; + + pBuffer->truncateData( 0); + if ((pVector = getVector( uiElementNumber, VECT_SLOT_HAS_DATA)) != NULL) + { + return( pBuffer->appendData( getDataPtr( pVector), + pVector->uiDataLength)); + } + + return( RC_SET( NE_SFLM_NOT_FOUND)); + } + + RCODE outputKey( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMUINT uiMatchFlags, + FLMBYTE * pucKeyBuf, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT uiSearchKeyFlag); + + RCODE outputData( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMBYTE * pucDataBuf, + FLMUINT uiDataBufSize, + FLMUINT * puiDataLen); + + RCODE inputKey( + F_Db * pDb, + FLMUINT uiIndexNum, + const FLMBYTE * pucKey, + FLMUINT uiKeyLen); + + RCODE inputData( + F_Db * pDb, + FLMUINT uiIndexNum, + const FLMBYTE * pucData, + FLMUINT uiDataLen); + + // Miscellaneous methods + + void reset( void); + + FINLINE const void * getDataPtr( + FLMUINT uiElementNumber) + { + return( getDataPtr( getVector( uiElementNumber, VECT_SLOT_HAS_DATA))); + } + +private: + + RCODE allocVectorArray( + FLMUINT uiElementNumber); + + RCODE storeValue( + FLMINT uiElementNumber, + eDataType eDataTyp, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBYTE ** ppucDataPtr = NULL); + + FINLINE F_VECTOR_ELEMENT * getVector( + FLMUINT uiElementNumber, + FLMUINT uiTestFlags) + { + F_VECTOR_ELEMENT * pVector; + + if (uiElementNumber >= m_uiNumElements) + { + return( NULL); + } + pVector = &m_pVectorElements [uiElementNumber]; + if (!(pVector->uiFlags & uiTestFlags)) + { + return( NULL); + } + else + { + return( pVector); + } + } + + FINLINE FLMBOOL isRightTruncated( + F_VECTOR_ELEMENT * pVector) + { + return( (pVector->uiFlags & VECT_SLOT_RIGHT_TRUNCATED) + ? TRUE + : FALSE); + } + + FINLINE void setRightTruncated( + F_VECTOR_ELEMENT * pVector) + { + pVector->uiFlags |= VECT_SLOT_RIGHT_TRUNCATED; + } + + FINLINE void clearRightTruncated( + F_VECTOR_ELEMENT * pVector) + { + pVector->uiFlags &= (~(VECT_SLOT_RIGHT_TRUNCATED)); + } + + FINLINE FLMBOOL isLeftTruncated( + F_VECTOR_ELEMENT * pVector) + { + return( (pVector->uiFlags & VECT_SLOT_LEFT_TRUNCATED) + ? TRUE + : FALSE); + } + + FINLINE void setLeftTruncated( + F_VECTOR_ELEMENT * pVector) + { + pVector->uiFlags |= VECT_SLOT_LEFT_TRUNCATED; + } + + FINLINE void clearLeftTruncated( + F_VECTOR_ELEMENT * pVector) + { + pVector->uiFlags &= (~(VECT_SLOT_LEFT_TRUNCATED)); + } + + FINLINE void * getDataPtr( + F_VECTOR_ELEMENT * pVector) + { + if (!pVector || !pVector->uiDataLength) + { + return( NULL); + } + else if (pVector->uiDataLength <= sizeof( FLMUINT)) + { + return( (void *)&pVector->uiDataOffset); + } + else + { + return( (void *)(m_pucDataBuf + pVector->uiDataOffset)); + } + } + +#define MIN_VECTOR_ELEMENTS 6 + F_VECTOR_ELEMENT m_VectorArray [MIN_VECTOR_ELEMENTS]; + F_VECTOR_ELEMENT * m_pVectorElements; // Pointer to vector elements + FLMUINT m_uiVectorArraySize; // Size of vector array + FLMUINT m_uiNumElements; // Number of elements actually + // populated in the array. + + FLMBYTE m_ucIntDataBuf[ 32]; // Internal data buffer + FLMBYTE * m_pucDataBuf; // Values stored here + FLMUINT m_uiDataBufLength; // Bytes of data allocated + FLMUINT m_uiDataBufOffset; // Current offset into allocated + // data buffer. + FLMUINT64 m_ui64RowId; // Row ID; + +friend class F_Db; +friend class FSIndexCursor; +friend class FSTableCursor; +friend class F_QueryResultSet; +}; + +// Flags for the m_uiFlags member of the F_Db object + +#define FDB_UPDATED_DICTIONARY 0x0001 + // Flag indicating whether the file's + // local dictionary was updated + // during the transaction. +#define FDB_DO_TRUNCATE 0x0002 + // Truncate log extents at the end + // of a transaction. +#define FDB_HAS_FILE_LOCK 0x0004 + // FDB has a file lock. +#define FDB_FILE_LOCK_SHARED 0x0008 + // File lock is shared. Update + // transactions are not allowed when + // the lock is shared. +#define FDB_FILE_LOCK_IMPLICIT 0x0010 + // File lock is implicit - means file + // lock was obtained when the update + // transaction began and cannot be + // released by a call to FlmDbUnlock. +#define FDB_DONT_KILL_TRANS 0x0020 + // Do not attempt to kill an active + // read transaction on this database + // handle. This is used by FlmDbBackup. +#define FDB_INTERNAL_OPEN 0x0040 + // FDB is an internal one used by a + // background thread. +#define FDB_DONT_POISON_CACHE 0x0080 + // If blocks are read from disk during + // a transaction, release them at the LRU + // end of the cache chain. +#define FDB_UPGRADING 0x0100 + // Database is being upgraded. +#define FDB_REPLAYING_RFL 0x0200 + // Database is being recovered +#define FDB_REPLAYING_COMMIT 0x0400 + // During replay of the RFL, this + // is an actual call to FlmDbTransCommit. +#define FDB_BACKGROUND_INDEXING 0x0800 + // FDB is being used by a background indexing + // thread +#define FDB_HAS_WRITE_LOCK 0x1000 + // FDB has the write lock +#define FDB_REBUILDING_DATABASE 0x2000 + // Database is being rebuilt +#define FDB_SWEEP_SCHEDULED 0x4000 + // Sweep operation scheduled due to a + // dictionary change during the transaction + +/***************************************************************************** +Desc: Class for performing database backup. +*****************************************************************************/ +class F_Backup : public F_Object +{ +public: + + F_Backup(); + virtual ~F_Backup(); + + FINLINE FLMUINT64 getBackupTransId( void) + { + return( m_ui64TransId); + } + + FINLINE FLMUINT64 getLastBackupTransId( void) + { + return( m_ui64LastBackupTransId); + } + + RCODE backup( + const char * pszBackupPath, + const char * pszPassword, + IF_BackupClient * pClient, + IF_BackupStatus * pStatus, + FLMUINT * puiIncSeqNum); + + RCODE endBackup( void); + +private: + + void reset( void); + + F_Db * m_pDb; + eDbTransType m_eTransType; + FLMUINT64 m_ui64TransId; + FLMUINT64 m_ui64LastBackupTransId; + FLMUINT m_uiDbVersion; + FLMUINT m_uiBlkChgSinceLastBackup; + FLMBOOL m_bTransStarted; + FLMUINT m_uiBlockSize; + FLMUINT m_uiLogicalEOF; + FLMUINT m_uiFirstReqRfl; + FLMUINT m_uiIncSeqNum; + FLMBOOL m_bCompletedBackup; + eDbBackupType m_eBackupType; + RCODE m_backupRc; + FLMBYTE m_ucNextIncSerialNum[ SFLM_SERIAL_NUM_SIZE]; + char m_szDbPath[ F_PATH_MAX_SIZE]; + SFLM_DB_HDR m_dbHdr; + +friend class F_Db; +}; + + +/***************************************************************************** +Desc: An implementation of IF_Backup_Client that backs up to the + local hard disk. +*****************************************************************************/ +class F_DefaultBackupClient : public IF_BackupClient +{ +public: + + F_DefaultBackupClient( + const char * pszBackupPath); + + virtual ~F_DefaultBackupClient(); + + RCODE WriteData( + const void * pvBuffer, + FLMUINT uiBytesToWrite); + + virtual FLMINT getRefCount( void) + { + return( IF_BackupClient::getRefCount()); + } + + virtual FLMINT AddRef( void) + { + return( IF_BackupClient::AddRef()); + } + + virtual FLMINT Release( void) + { + return( IF_BackupClient::Release()); + } + +private: + + char m_szPath[ F_PATH_MAX_SIZE]; + IF_MultiFileHdl * m_pMultiFileHdl; + FLMUINT64 m_ui64Offset; + RCODE m_rc; +}; + +/***************************************************************************** +Desc: The F_FSRestore class is used to read backup and RFL files from + a disk file system. +*****************************************************************************/ +class F_FSRestore : public IF_RestoreClient +{ +public: + + virtual ~F_FSRestore(); + F_FSRestore(); + + RCODE setup( + const char * pszDbPath, + const char * pszBackupSetPath, + const char * pszRflDir); + + RCODE openBackupSet( void); + + RCODE openIncFile( + FLMUINT uiFileNum); + + RCODE openRflFile( + FLMUINT uiFileNum); + + RCODE read( + FLMUINT uiLength, + void * pvBuffer, + FLMUINT * puiBytesRead); + + RCODE close( void); + + RCODE abortFile( void); + + virtual FLMINT getRefCount( void) + { + return( IF_RestoreClient::getRefCount()); + } + + virtual FLMINT AddRef( void) + { + return( IF_RestoreClient::AddRef()); + } + + virtual FLMINT Release( void) + { + return( IF_RestoreClient::Release()); + } + +protected: + + IF_FileHdl * m_pFileHdl; + IF_MultiFileHdl * m_pMultiFileHdl; + FLMUINT64 m_ui64Offset; + FLMUINT m_uiDbVersion; + char m_szDbPath[ F_PATH_MAX_SIZE]; + char m_szBackupSetPath[ F_PATH_MAX_SIZE]; + char m_szRflDir[ F_PATH_MAX_SIZE]; + FLMBOOL m_bSetupCalled; + FLMBOOL m_bOpen; +}; + +/***************************************************************************** +Desc: Default implementation of a restore status object than can + be inherited by a user implementation. +*****************************************************************************/ +class F_DefaultRestoreStatus : public IF_RestoreStatus +{ +public: + + F_DefaultRestoreStatus() + { + } + + RCODE reportProgress( + eRestoreAction * peAction, + FLMUINT64, // ui64BytesToDo, + FLMUINT64) // ui64BytesDone + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportError( + eRestoreAction * peAction, + RCODE) // rcErr + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportBeginTrans( + eRestoreAction * peAction, + FLMUINT64) // ui64TransId + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportCommitTrans( + eRestoreAction * peAction, + FLMUINT64) // ui64TransId + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportAbortTrans( + eRestoreAction * peAction, + FLMUINT64) // ui64TransId + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportRemoveData( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiLfNum, + FLMUINT, // uiKeyLen, + FLMBYTE *) // pucKey + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportInsertData( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiLfNum, + FLMUINT, // uiKeyLen, + FLMBYTE *) // pucKey + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportReplaceData( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiLfNum, + FLMUINT, // uiKeyLen, + FLMBYTE *) // pucKey + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportLFileCreate( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT) // uiLfNum + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportLFileUpdate( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiLfNum, + FLMUINT, // uiRootBlk, + FLMUINT64, // ui64NextNodeId, + FLMUINT64, // ui64FirstDocId, + FLMUINT64 // ui64LastDocId + ) + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportUpdateDict( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiDictType, + FLMUINT, // uiDictNum, + FLMBOOL) // bDeleting + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportIndexSuspend( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT) // uiIndexNum + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportIndexResume( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT) // uiIndexNum + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportReduce( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT) // uiCount + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportUpgrade( + eRestoreAction * peAction, + FLMUINT64, // ui64TransId, + FLMUINT, // uiOldDbVersion, + FLMUINT) // uiNewDbVersion + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportEnableEncryption( + eRestoreAction * peAction, + FLMUINT64 // ui64TransId + ) + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportWrapKey( + eRestoreAction * peAction, + FLMUINT64) // ui64TransId + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportOpenRflFile( + eRestoreAction * peAction, + FLMUINT) // uiFileNum + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + RCODE reportRflRead( + eRestoreAction * peAction, + FLMUINT, // uiFileNum, + FLMUINT) // uiBytesRead + { + *peAction = SFLM_RESTORE_ACTION_CONTINUE; + return( NE_SFLM_OK); + } + + virtual FLMINT getRefCount( void) + { + return( IF_RestoreStatus::getRefCount()); + } + + virtual FLMINT AddRef( void) + { + return( IF_RestoreStatus::AddRef()); + } + + virtual FLMINT Release( void) + { + return( IF_RestoreStatus::Release()); + } +}; + +typedef struct KEY_GEN_INFO +{ + F_TABLE * pTable; + F_INDEX * pIndex; + F_Row * pRow; + FLMBOOL bIsAsia; + FLMBOOL bIsCompound; + FLMBOOL bAddKeys; + FLMBYTE * pucKeyBuf; + FLMBYTE * pucData; + FLMUINT uiDataBufSize; + FLMBOOL bDataBufAllocated; +} KEY_GEN_INFO; + +/***************************************************************************** +Desc: Thread's database object - returned by dbOpen, dbCreate in F_DbSystem class +*****************************************************************************/ +class F_Db : public F_Object +{ +public: + + F_Db( + FLMBOOL bInternalOpen); + + virtual ~F_Db(); + + RCODE transBegin( + eDbTransType eTransType, + FLMUINT uiMaxLockWait = SFLM_NO_TIMEOUT, + FLMUINT uiFlags = 0, + SFLM_DB_HDR * pDbHeader = NULL); + + RCODE transBegin( + F_Db * pDb); + + RCODE transCommit( + FLMBOOL * pbEmpty = NULL); + + RCODE transAbort( void); + + FINLINE eDbTransType getTransType( void) + { + return( m_eTransType); + } + + RCODE doCheckpoint( + FLMUINT uiTimeout); + + RCODE dbLock( + eDbLockType eLockType, + FLMINT iPriority, + FLMUINT uiTimeout); + + RCODE dbUnlock( void); + + RCODE getLockType( + eDbLockType * peLockType, + FLMBOOL * pbImplicit); + + RCODE getLockInfo( + FLMINT iPriority, + eDbLockType * peCurrLockType, + FLMUINT * puiThreadId, + FLMUINT * puiNumExclQueued, + FLMUINT * puiNumSharedQueued, + FLMUINT * puiPriorityCount); + + RCODE dupTrans( + FLMUINT64 ui64TransId); + + RCODE demoteTrans( void); + + RCODE cancelTrans( + FLMUINT64 ui64TransId); + + RCODE getCommitCnt( + FLMUINT64 * pui64CommitCount); + + // Index methods + + RCODE indexStatus( + FLMUINT uiIndexNum, + SFLM_INDEX_STATUS * pIndexStatus); + + RCODE indexGetNext( + FLMUINT * puiIndexNum); + + RCODE indexSuspend( + FLMUINT uiIndexNum); + + RCODE indexResume( + FLMUINT uiIndexNum); + + // Retrieval Functions + + RCODE keyRetrieve( + FLMUINT uiIndex, + F_DataVector * pSearchKey, + FLMUINT uiFlags, + F_DataVector * pFoundKey); + + RCODE enableEncryption( void); + + RCODE wrapKey( + const char * pszPassword = NULL); + + RCODE rollOverDbKey( void); + + RCODE reduceSize( + FLMUINT uiCount, + FLMUINT * puiCountRV); + + RCODE upgrade( + IF_UpgradeClient * pUpgradeClient); + + RCODE backupBegin( + eDbBackupType eBackupType, + eDbTransType eTransType, + FLMUINT uiMaxLockWait, + F_Backup ** ppBackup); + + void getRflFileName( + FLMUINT uiFileNum, + FLMBOOL bBaseOnly, + char * pszFileName, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated = NULL); + + // Configuration methods + + RCODE setRflKeepFilesFlag( + FLMBOOL bKeep); + + RCODE getRflKeepFlag( + FLMBOOL * pbKeep); + + RCODE setRflDir( + const char * pszNewRflDir); + + void getRflDir( + char * pszRflDir); + + RCODE getRflFileNum( + FLMUINT * puiRflFileNum); + + RCODE getHighestNotUsedRflFileNum( + FLMUINT * puiHighestNotUsedRflFileNum); + + RCODE setRflFileSizeLimits( + FLMUINT uiMinRflSize, + FLMUINT uiMaxRflSize); + + RCODE getRflFileSizeLimits( + FLMUINT * puiRflMinFileSize, + FLMUINT * puiRflMaxFileSize); + + RCODE rflRollToNextFile( void); + + RCODE setKeepAbortedTransInRflFlag( + FLMBOOL bKeep); + + RCODE getKeepAbortedTransInRflFlag( + FLMBOOL * pbKeep); + + RCODE setAutoTurnOffKeepRflFlag( + FLMBOOL bAutoTurnOff); + + RCODE getAutoTurnOffKeepRflFlag( + FLMBOOL * pbAutoTurnOff); + + FINLINE void setFileExtendSize( + FLMUINT uiFileExtendSize) + { + m_pDatabase->m_uiFileExtendSize = uiFileExtendSize; + } + + FINLINE FLMUINT getFileExtendSize( void) + { + return( m_pDatabase->m_uiFileExtendSize); + } + + FINLINE void setAppData( + void * pvAppData) + { + m_pvAppData = pvAppData; + } + + FINLINE void * getAppData( void) + { + return( m_pvAppData); + } + + FINLINE void setDeleteStatusObject( + IF_DeleteStatus * pDeleteStatus) + { + if (m_pDeleteStatus) + { + m_pDeleteStatus->Release(); + } + if ((m_pDeleteStatus = pDeleteStatus) != NULL) + { + m_pDeleteStatus->AddRef(); + } + } + + FINLINE void setCommitClientObject( + IF_CommitClient * pCommitClient) + { + if (m_pCommitClient) + { + m_pCommitClient->Release(); + } + + m_pCommitClient = pCommitClient; + + if (m_pCommitClient) + { + m_pCommitClient->AddRef(); + } + } + + FINLINE void setIndexingClientObject( + IF_IxClient * pIxClient) + { + if (m_pIxClient) + { + m_pIxClient->Release(); + } + m_pIxClient = pIxClient; + if (m_pIxClient) + { + m_pIxClient->AddRef(); + } + } + + FINLINE void setIndexingStatusObject( + IF_IxStatus * ifpIxStatus) + { + if (m_pIxStatus) + { + m_pIxStatus->Release(); + } + m_pIxStatus = ifpIxStatus; + if (m_pIxStatus) + { + m_pIxStatus->AddRef(); + } + } + + // Configuration information getting methods + + FINLINE FLMUINT getDbVersion( void) + { + return( (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32DbVersion); + } + + FINLINE FLMUINT getBlockSize( void) + { + return( m_pDatabase->m_uiBlockSize); + } + + FINLINE FLMUINT getDefaultLanguage( void) + { + return( m_pDatabase->m_uiDefaultLanguage); + } + + FINLINE FLMUINT64 getTransID( void) + { + if (m_eTransType != SFLM_NO_TRANS) + { + return( m_ui64CurrTransID); + } + else if (m_uiFlags & FDB_HAS_FILE_LOCK) + { + return( m_pDatabase->m_lastCommittedDbHdr.ui64CurrTransID); + } + + return( 0); + } + + void getCheckpointInfo( + SFLM_CHECKPOINT_INFO * pCheckpointInfo); + + RCODE getDbControlFileName( + char * pszControlFileName, + FLMUINT uiControlFileBufSize) + { + RCODE rc = NE_SFLM_OK; + FLMUINT uiLen = f_strlen( m_pDatabase->m_pszDbPath); + + if (uiLen + 1 > uiControlFileBufSize) + { + uiLen = uiControlFileBufSize - 1; + rc = RC_SET( NE_SFLM_BUFFER_OVERFLOW); + } + f_memcpy( pszControlFileName, m_pDatabase->m_pszDbPath, uiLen); + pszControlFileName [uiLen] = 0; + return( rc); + } + + FINLINE FLMBOOL threadWaitingLock( void) + { + return( m_pDatabase->m_pDatabaseLockObj->ThreadWaitingLock()); + } + + RCODE getLockWaiters( + IF_LockInfoClient * pLockInfo); + + RCODE getLastBackupTransID( + FLMUINT64 * pui64LastBackupTransID); + + RCODE getBlocksChangedSinceBackup( + FLMUINT * puiBlocksChangedSinceBackup); + + RCODE getNextIncBackupSequenceNum( + FLMUINT * puiNextIncBackupSequenceNum); + + void getSerialNumber( + char * pucSerialNumber); + + RCODE getDiskSpaceUsage( + FLMUINT64 * pui64DataSize, + FLMUINT64 * pui64RollbackSize, + FLMUINT64 * pui64RflSize); + + FINLINE RCODE getMustCloseRC( void) + { + return( m_pDatabase->m_rcMustClose); + } + + FINLINE RCODE getAbortRC( void) + { + return( m_AbortRc); + } + + FINLINE RCODE startTransaction( + eDbTransType eReqTransType, + FLMBOOL * pbStartedTrans) + { + RCODE rc; + + if( m_eTransType != SFLM_NO_TRANS) + { + return( RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_TRANS_OP)); + } + + if( !pbStartedTrans) + { + return( RC_SET( NE_SFLM_NO_TRANS_ACTIVE)); + } + + if( RC_BAD( rc = transBegin( eReqTransType))) + { + return( rc); + } + + *pbStartedTrans = TRUE; + return( NE_SFLM_OK); + } + + FINLINE RCODE checkTransaction( + eDbTransType eReqTransType, + FLMBOOL * pbStartedTrans) + { + if( m_AbortRc) + { + return( m_AbortRc); + } + else if( m_eTransType >= eReqTransType) + { + return( NE_SFLM_OK); + } + + return( startTransaction( eReqTransType, pbStartedTrans)); + } + + FINLINE void setMustAbortTrans( + RCODE rc) + { + if( RC_BAD( rc) && RC_OK( m_AbortRc)) + { + m_AbortRc = rc; + } + } + + FINLINE RCODE checkState( + const char * pszFileName, + FLMINT iLineNumber) + { + RCODE rc = NE_SFLM_OK; + + if (m_bMustClose) + { + m_pDatabase->logMustCloseReason( pszFileName, iLineNumber); + rc = RC_SET( NE_SFLM_MUST_CLOSE_DATABASE); + } + return( rc); + } + + FINLINE F_Database * getDatabase( void) + { + return m_pDatabase; + } + + RCODE backgroundIndexBuild( + IF_Thread * pThread, + FLMBOOL * pbShutdown, + FLMINT * piErrorLine); + + FINLINE FLMUINT getLogicalEOF( void) + { + return( m_uiLogicalEOF); + } + + // Key Collector object, used when checking indexes + + FINLINE void setKeyCollector( + F_KeyCollector * pKeyColl) + { + m_pKeyColl = pKeyColl; + } + + FINLINE F_KeyCollector * getKeyCollector( void) + { + return m_pKeyColl; + } + + RCODE waitForMaintenanceToComplete( void); + + FINLINE F_Dict * getDict( void) + { + return( m_pDict); + } + + void removeTableRows( + FLMUINT uiTableNum, + FLMUINT64 ui64TransId); + +private: + + // This routine assumes that the database mutex is locked + FINLINE void linkToDict( + F_Dict * pDict) + { + if (pDict != m_pDict) + { + if (m_pDict) + { + unlinkFromDict(); + } + if ((m_pDict = pDict) != NULL) + { + pDict->incrUseCount(); + } + } + } + + // This routine assumes the database mutex is locked. + FINLINE void unlinkFromDict( void) + { + if (m_pDict) + { + + // If the use count goes to zero and the F_Dict is not the first one + // in the file's list or it is not linked to a file, unlink the F_Dict + // object from its database and delete it. + + if (!m_pDict->decrUseCount() && + (m_pDict->getPrev() || !m_pDict->getDatabase())) + { + m_pDict->unlinkFromDatabase(); + } + m_pDict = NULL; + } + } + + RCODE linkToDatabase( + F_Database * pDatabase); + + void unlinkFromDatabase(); + + RCODE initDbFiles( + const char * pszRflDir, + SFLM_CREATE_OPTS * pCreateOpts); + + RCODE beginBackgroundTrans( + IF_Thread * pThread); + + RCODE beginTrans( + eDbTransType eTransType, + FLMUINT uiMaxLockWait = SFLM_NO_TIMEOUT, + FLMUINT uiFlags = 0, + SFLM_DB_HDR * pDbHdr = NULL); + + RCODE beginTrans( + F_Db * pDb); + + RCODE commitTrans( + FLMUINT uiNewLogicalEOF, + FLMBOOL bForceCheckpoint, + FLMBOOL * pbEmpty = NULL); + + RCODE abortTrans( + FLMBOOL bOkToLogAbort = TRUE); + + RCODE readRollbackLog( + FLMUINT uiLogEOF, + FLMUINT * puiCurrAddr, + F_BLK_HDR * pBlkHdr, + FLMBOOL * pbIsBeforeImageBlk); + + RCODE processBeforeImage( + FLMUINT uiLogEOF, + FLMUINT * puiCurrAddrRV, + F_BLK_HDR * pBlkHdr, + FLMBOOL bDoingRecovery, + FLMUINT64 ui64MaxTransID); + + RCODE physRollback( + FLMUINT uiLogEOF, + FLMUINT uiFirstLogBlkAddr, + FLMBOOL bDoingRecovery, + FLMUINT64 ui64MaxTransID); + + void completeOpenOrCreate( + RCODE rc, + FLMBOOL bNewDatabase); + + RCODE startBackgroundIndexing( void); + + void unlinkFromTransList( + FLMBOOL bCommitting); + + RCODE lockExclusive( + FLMUINT uiMaxLockWait); + + void unlockExclusive( void); + + RCODE readDictionary( void); + + RCODE dictCreate( void); + + RCODE dictReadLFH( void); + + RCODE dictReadEncDefs( void); + + RCODE dictReadTables( void); + + RCODE dictReadColumns( void); + + RCODE dictReadIndexes( void); + + RCODE dictReadIndexComponents( void); + + RCODE dictOpen( void); + + RCODE dictClone( void); + + RCODE createNewDict( void); + + FINLINE void getDbHdrInfo( + SFLM_DB_HDR * pDbHdr) + { + // IMPORTANT NOTE: Any changes to this method should also be + // mirrored with changes to the other getDbHdrInfo call - see below. + + m_ui64CurrTransID = pDbHdr->ui64CurrTransID; + m_uiLogicalEOF = (FLMUINT)pDbHdr->ui32LogicalEOF; + + // If we are doing a read transaction, this is only needed + // if we are checking the database. + + m_uiFirstAvailBlkAddr = (FLMUINT)pDbHdr->ui32FirstAvailBlkAddr; + } + + FINLINE void getDbHdrInfo( + F_Db * pDb) + { + m_ui64CurrTransID = pDb->m_ui64CurrTransID; + m_uiLogicalEOF = pDb->m_uiLogicalEOF; + + // If we are doing a read transaction, this is only needed + // if we are checking the database. + + m_uiFirstAvailBlkAddr = pDb->m_uiFirstAvailBlkAddr; + } + + FINLINE FLMBOOL okToCommitTrans( void) + { + return( m_eTransType == SFLM_READ_TRANS || + m_AbortRc == NE_SFLM_OK + ? TRUE + : FALSE); + } + + RCODE processDupKeys( + F_INDEX * pIndex); + + RCODE keysCommit( + FLMBOOL bCommittingTrans, + FLMBOOL bSortKeys = TRUE); + + RCODE refUpdate( + F_INDEX * pIndex, + KREF_ENTRY * pKrefEntry, + FLMBOOL bNormalUpdate); + + FINLINE RCODE flushKeys( void) + { + RCODE rc = NE_SFLM_OK; + + if( m_bKrefSetup) + { + if( m_uiKrefCount) + { + if (RC_BAD( rc = keysCommit( FALSE))) + { + goto Exit; + } + } + + m_pKrefReset = m_pKrefPool->poolMark(); + } + + Exit: + + return( rc); + } + + RCODE krefCntrlCheck( void); + + void krefCntrlFree( void); + + FINLINE FLMBOOL isKrefOverThreshold( void) + { + if( (((m_pKrefPool->getBlockSize() * 3) - 250) <= m_uiTotalKrefBytes) || + m_uiKrefCount > (m_uiKrefTblSize - 128)) + { + return( TRUE); + } + + return( FALSE); + } + + RCODE addToKrefTbl( + FLMUINT uiKeyLen, + FLMUINT uiDataLen); + + RCODE buildData( + ICD * pIcd, + FLMUINT uiDataComponent, + FLMUINT uiKeyLen, + FLMUINT uiDataLen); + + RCODE finishKeyComponent( + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen); + + RCODE genTextKeyComponents( + F_COLUMN * pColumn, + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen, + FLMBYTE ** ppucTmpBuf, + FLMUINT * puiTmpBufSize, + void ** ppvMark); + + RCODE genOtherKeyComponent( + F_COLUMN * pColumn, + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen); + + RCODE buildKeys( + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen); + + RCODE buildKeys( + F_INDEX * pIndex, + F_TABLE * pTable, + F_Row * pOldRow, + F_Row * pNewRow); + + RCODE updateIndexKeys( + FLMUINT uiTableNum, + F_Row * pOldRow, + F_Row * pNewRow); + + void indexingAfterCommit( void); + + void indexingAfterAbort( void); + + RCODE addToStopList( + FLMUINT uiIndexNum); + + RCODE addToStartList( + FLMUINT uiIndexNum); + + void stopBackgroundIndexThread( + FLMUINT uiIndexNum, + FLMBOOL bWait, + FLMBOOL * pbStopped); + + RCODE startIndexBuild( + FLMUINT uiIndexNum); + + RCODE setIxStateInfo( + FLMUINT uiIndexNum, + FLMUINT64 ui64LastRowIndexed, + FLMUINT uiState); + + RCODE indexSetOfRows( + FLMUINT uiIndexNum, + FLMUINT64 ui64StartRowId, + FLMUINT64 ui64EndRowId, + IF_IxStatus * pIxStatus, + IF_IxClient * pIxClient, + SFLM_INDEX_STATUS * pIndexStatus, + FLMBOOL * pbHitEnd, + IF_Thread * pThread); + + RCODE buildIndex( + FLMUINT uiIndexNum, + FLMUINT uiState); + + RCODE readBlkHdr( + FLMUINT uiBlkAddress, + F_BLK_HDR * pBlkHdr, + FLMINT * piType); + + SFLM_LFILE_STATS * getLFileStatPtr( + LFILE * pLFile); + +#define FLM_UPD_ADD 0x00001 +#define FLM_UPD_INTERNAL_CHANGE 0x00004 + + RCODE getCachedBTree( + FLMUINT uiTableNum, + F_Btree ** ppBTree); + + RCODE flushDirtyRows( void); + + RCODE maintBlockChainFree( + FLMUINT64 ui64MaintRowId, + FLMUINT uiStartBlkAddr, + FLMUINT uiBlocksToFree, + FLMUINT uiExpectedEndBlkAddr, + FLMUINT * puiBlocksFreed); + + RCODE encryptData( + FLMUINT uiEncDefNum, + FLMBYTE * pucIV, + FLMBYTE * pucBuffer, + FLMUINT uiBufferSize, + FLMUINT uiDataLen, + FLMUINT * puiEncryptedLength); + + RCODE decryptData( + FLMUINT uiEncDefNum, + FLMBYTE * pucIV, + void * pvInBuf, + FLMUINT uiInLen, + void * pvOutBuf, + FLMUINT uiOutBufLen); + + RCODE createDbKey(); + + // Private data members. + + F_Database * m_pDatabase; // Pointer to F_Database object + F_Dict * m_pDict; // Pointer to dictionary object + F_Db * m_pNextForDatabase; // Next F_Db associated with F_Database + // NOTE: gv_SFlmSysData.hShareMutex + // must be locked to set this + F_Db * m_pPrevForDatabase; // Prev F_Db associated with F_Database + // NOTE: gv_SFlmSysData.hShareMutex + // must be locked to set this + void * m_pvAppData; // Application data that is used + // to associate this F_Db with + // an object in the application + // space. + FLMUINT m_uiThreadId; // Thread that started the current + // transaction, if any. NOTE: + // Only set on transaction begin. + // Hence, if operations are performed + // by multiple threads, within the + // transaction, it will not necessarily + // reflect the thread that is currently + // using the F_Db. + FLMBOOL m_bMustClose; // An error has occurred that requires + // the application to stop using (close) + // this FDB + F_SuperFileHdl * m_pSFileHdl; // Pointer to the super file handle + FLMUINT m_uiFlags; // Flags for this F_Db. + + // TRANSACTION STATE STUFF + + FLMUINT m_uiTransCount; // Transaction counter for the F_Db. + // Incremented whenever a transaction + // is started on this F_Db. Used so + // that FLAIM can tell if an implicit + // transaction it started is still in + // effect. This should NOT be + // confused with update transaction + // IDs. + eDbTransType m_eTransType; // Type of transaction + RCODE m_AbortRc; // If not NE_SFLM_OK, transaction must be + // aborted. + FLMUINT64 m_ui64CurrTransID;// Current transaction ID. + FLMUINT m_uiFirstAvailBlkAddr; // Address of first block in avail list + FLMUINT m_uiLogicalEOF; // Current logical end of file. New + // blocks are allocated at this address. + FLMUINT m_uiUpgradeCPFileNum; + FLMUINT m_uiUpgradeCPOffset; + // RFL file number and offset to set + // RFL to during an upgrade operation + // that happens during a restore or + // recovery. + FLMUINT m_uiTransEOF; // Address of logical end of file + // when the last transaction + // committed. A block beyond this + // point in the file is going to be + // a new block and will not need to + // be logged. + F_TMSTAMP m_TransStartTime; // Transaction start time, for stats + + // KREF STUFF + + KEY_GEN_INFO m_keyGenInfo; // Information for generating index + // keys. + KREF_ENTRY ** m_pKrefTbl; // Pointer to KREF table, which is an array + // of KREF_ENTRY * pointers. + FLMUINT m_uiKrefTblSize; // KREF table size. + FLMUINT m_uiKrefCount; // Number of entries in KREF table that + // are currently used. + FLMUINT m_uiTotalKrefBytes; // Total number of entries allocated + // in the pool. + FLMBYTE * m_pucKrefKeyBuf; // Pointer to temporary key buffer. + FLMBOOL m_bKrefSetup; // True if the KRef table has been initialized. + F_Pool * m_pKrefPool; // Memory pool to use + FLMBOOL m_bReuseKrefPool; // Reuse pool instead of free it? + FLMBOOL m_bKrefCompoundKey; // True if a compound key has been processed. + void * m_pKrefReset; // Used to reset the Kref pool on + // indexing failures + F_Pool m_tmpKrefPool; // KREF pool to be used during + // read transactions - only used when + // checking indexes. + + // UPDATE TRANSACTION STUFF + + FLMBOOL m_bHadUpdOper; // Did this transaction have any + // updates? + FLMUINT m_uiBlkChangeCnt; // Number of times ScaLogPhysBlk has + // been called during this transaction. + // This is used by the cursor code to + // know when it is necessary to + // re-position in the B-Tree.0 + IXD_FIXUP * m_pIxdFixups; // List of indexes whose IXD needs + // to be restored to its prior + // state if the transaction aborts + // READ TRANSACTION STUFF + + F_Db * m_pNextReadTrans; // Next active read transaction for + // this database. + // NOTE: If uiKilledTime (see below) + // is non-zero, then transaction is + // in killed list. + F_Db * m_pPrevReadTrans; // Previous active read transaction + // for this database. + // NOTE: If m_uiKilledTime (see below) + // is non-zero, then transaction is + // in killed list. + FLMUINT m_uiInactiveTime; // If non-zero, this is the last time + // the checkpoint thread marked this + // transaction as inactive. If zero, + // it means that the transaction is + // active, or it has not been marked + // by the checkpoint thread as + // inactive. If it stays non-zero for + // five or more minutes, it will be + // killed. + FLMUINT m_uiKilledTime; // Time transaction was killed, if + // non-zero. + // Misc. DB Info. + + FLMBOOL m_bItemStateUpdOk;// This variable is used to ensure + // that FlmDbSweep / recovery are the + // only ways that: + // 1) an element or attribute's state + // can be changed to 'unused' + // 2) a 'purge' element or attribute + // can be deleted + + F_Pool m_tempPool; // Temporary memory pool. It + // is only used for the duration of + // a FLAIM operation and then reset. + // The first block in the pool is + // retained between operations to + // help performance. + + // Callback functions. + IF_DeleteStatus * m_pDeleteStatus; // Handles status info coming back + // from deleting a BTree + IF_IxClient * m_pIxClient; // Indexing callback + IF_IxStatus * m_pIxStatus; // Indexing status callback + IF_CommitClient * m_pCommitClient; // Commit callback + + SFLM_STATS * m_pStats; + SFLM_DB_STATS * m_pDbStats; // DB statistics pointer. + SFLM_LFILE_STATS * m_pLFileStats; // LFILE statistics pointer. + FLMUINT m_uiLFileAllocSeq;// Allocation sequence number for + // LFILE statistics array so we + // can tell if the array has been + // reallocated and we need to reset + // our pLFileStats pointer. + SFLM_STATS m_Stats; // Statistics kept here until end + // of transaction. + FLMBOOL m_bStatsInitialized;// Has statistics structure been + // initialized? + F_BKGND_IX * m_pIxStartList; // Indexing threads to start at + // the conclusion of the transaction. + F_BKGND_IX * m_pIxStopList; // Indexing threads to stop at + // the conclusion of the transaction. + F_Btree * m_pCachedBTree; // BTree object used for node operations + F_KeyCollector * m_pKeyColl; // Special purpose object used when checking + // indexes in the F_DbCheck class. + FLMUINT m_uiDirtyRowCount; + F_SEM m_hWaitSem; // Semaphore that is used when + // waiting for reads to complete + +friend class F_Database; +friend class F_Dict; +friend class F_DbSystem; +friend class F_Rfl; +friend class F_Btree; +friend class F_Backup; +friend class F_Row; +friend class F_BTreeIStream; +friend class F_DataVector; +friend class F_DbRebuild; +friend class F_DbCheck; +friend class F_Query; +friend class FSIndexCursor; +friend class FSTableCursor; +friend class F_BtRSFactory; +friend class F_BtResultSet; +friend class F_CachedBlock; +friend class F_BlockCacheMgr; +friend class F_RowCacheMgr; +friend class F_GlobalCacheMgr; +friend class F_QueryResultSet; +friend class ServerLockObject; +friend class F_BTreeInfo; +friend class SQLQuery; +}; + +typedef struct BTREE_INFO +{ + FLMUINT uiLfNum; + char * pszLfName; + FLMUINT uiNumLevels; + SFLM_BTREE_LEVEL_INFO levelInfo [MAX_LEVELS]; +} BTREE_INFO; + +/***************************************************************************** +Desc: Object for gathering B-Tree information. +*****************************************************************************/ +class F_BTreeInfo : public F_Object +{ +public: + F_BTreeInfo() + { + m_pIndexArray = NULL; + m_uiIndexArraySize = 0; + m_uiNumIndexes = 0; + m_pTableArray = NULL; + m_uiTableArraySize = 0; + m_uiNumTables = 0; + m_pool.poolInit( 512); + } + + virtual ~F_BTreeInfo() + { + if (m_pIndexArray) + { + f_free( &m_pIndexArray); + } + if (m_pTableArray) + { + f_free( &m_pTableArray); + } + m_pool.poolFree(); + } + + FINLINE void clearBTreeInfo( void) + { + m_uiNumIndexes = 0; + m_uiNumTables = 0; + } + + RCODE collectIndexInfo( + F_Db * pDb, + FLMUINT uiIndexNum, + IF_BTreeInfoStatus * pInfoStatus); + + RCODE collectTableInfo( + F_Db * pDb, + FLMUINT uiTableNum, + IF_BTreeInfoStatus * pInfoStatus); + + FINLINE FLMUINT getNumIndexes( void) + { + return( m_uiNumIndexes); + } + + FINLINE FLMUINT getNumTables( void) + { + return( m_uiNumTables); + } + + FINLINE FLMBOOL getIndexInfo( + FLMUINT uiNthIndex, + FLMUINT * puiIndexNum, + char ** ppszIndexName, + FLMUINT * puiNumLevels) + { + if (uiNthIndex < m_uiNumIndexes) + { + *puiIndexNum = m_pIndexArray [uiNthIndex].uiLfNum; + *puiNumLevels = m_pIndexArray [uiNthIndex].uiNumLevels; + *ppszIndexName = m_pIndexArray [uiNthIndex].pszLfName; + return( TRUE); + } + else + { + *puiIndexNum = 0; + *ppszIndexName = NULL; + *puiNumLevels = 0; + return( FALSE); + } + } + + FINLINE FLMBOOL getTableInfo( + FLMUINT uiNthTable, + FLMUINT * puiTableNum, + char ** ppszTableName, + FLMUINT * puiNumLevels) + { + if (uiNthTable < m_uiNumTables) + { + *puiTableNum = m_pTableArray [uiNthTable].uiLfNum; + *puiNumLevels = m_pTableArray [uiNthTable].uiNumLevels; + *ppszTableName = m_pTableArray [uiNthTable].pszLfName; + return( TRUE); + } + else + { + *puiTableNum = 0; + *puiNumLevels = 0; + *ppszTableName = NULL; + return( FALSE); + } + } + + FINLINE FLMBOOL getIndexLevelInfo( + FLMUINT uiNthIndex, + FLMUINT uiBTreeLevel, + SFLM_BTREE_LEVEL_INFO * pLevelInfo) + { + if (uiNthIndex < m_uiNumIndexes && + uiBTreeLevel < m_pIndexArray [uiNthIndex].uiNumLevels) + { + f_memcpy( pLevelInfo, + &(m_pIndexArray [uiNthIndex].levelInfo [uiBTreeLevel]), + sizeof( SFLM_BTREE_LEVEL_INFO)); + return( TRUE); + } + else + { + return( FALSE); + } + } + + FINLINE FLMBOOL getTableLevelInfo( + FLMUINT uiNthTable, + FLMUINT uiBTreeLevel, + SFLM_BTREE_LEVEL_INFO * pLevelInfo) + { + if (uiNthTable < m_uiNumTables && + uiBTreeLevel < m_pTableArray [uiNthTable].uiNumLevels) + { + f_memcpy( pLevelInfo, + &(m_pTableArray [uiNthTable].levelInfo [uiBTreeLevel]), + sizeof( SFLM_BTREE_LEVEL_INFO)); + return( TRUE); + } + else + { + return( FALSE); + } + } + +private: + + RCODE collectBlockInfo( + F_Db * pDb, + LFILE * pLFile, + BTREE_INFO * pBTreeInfo, + F_BTREE_BLK_HDR * pBlkHdr, + F_INDEX * pIndex); + + RCODE collectBTreeInfo( + F_Db * pDb, + LFILE * pLFile, + BTREE_INFO * pBTreeInfo, + const char * pszName, + F_INDEX * pIndex); + + FINLINE RCODE doCallback( void) + { + if (m_pInfoStatus) + { + return( m_pInfoStatus->infoStatus( m_uiCurrLfNum, m_bIsTable, + m_pszCurrLfName, m_uiCurrLevel, + m_ui64CurrLfBlockCount, m_ui64CurrLevelBlockCount, + m_ui64TotalBlockCount)); + } + else + { + return( NE_SFLM_OK); + } + } + + BTREE_INFO * m_pIndexArray; + FLMUINT m_uiIndexArraySize; + FLMUINT m_uiNumIndexes; + BTREE_INFO * m_pTableArray; + FLMUINT m_uiTableArraySize; + FLMUINT m_uiNumTables; + F_Pool m_pool; + + // Items for the callback function. + + IF_BTreeInfoStatus * m_pInfoStatus; + FLMUINT m_uiBlockSize; + FLMUINT m_uiCurrLfNum; + FLMBOOL m_bIsTable; + char * m_pszCurrLfName; + FLMUINT m_uiCurrLevel; + FLMUINT64 m_ui64CurrLfBlockCount; + FLMUINT64 m_ui64CurrLevelBlockCount; + FLMUINT64 m_ui64TotalBlockCount; +}; + +RCODE ixKeyCompare( + F_Db * pDb, + F_INDEX * pIndex, + F_DataVector * pSearchKey, + F_Row * pRow1, + F_Row * pRow2, + FLMBOOL bCompareRowId, + const void * pvKey1, + FLMUINT uiKeyLen1, + const void * pvKey2, + FLMUINT uiKeyLen2, + FLMINT * piCompare); + +/******************************************************************** +Desc: Class for comparing two keys in an index. +********************************************************************/ +class IXKeyCompare : public IF_ResultSetCompare +{ +public: + + IXKeyCompare() + { + + // m_pDb is used to sort truncated keys if necessary. + // m_pIxd is used for comparison + + m_pDb = NULL; + m_pIndex = NULL; + m_pSearchKey = NULL; + m_bCompareRowId = TRUE; + m_pOldRow = NULL; + } + + virtual ~IXKeyCompare() + { + } + + FINLINE RCODE compare( + const void * pvKey1, + FLMUINT uiKeyLen1, + const void * pvKey2, + FLMUINT uiKeyLen2, + FLMINT * piCompare) + { + return( ixKeyCompare( m_pDb, m_pIndex, m_pSearchKey, + m_pOldRow, m_pOldRow, m_bCompareRowId, + pvKey1, uiKeyLen1, pvKey2, uiKeyLen2, piCompare)); + } + + FINLINE void setIxInfo( + F_Db * pDb, + F_INDEX * pIndex) + { + m_pDb = pDb; + m_pIndex = pIndex; + } + + FINLINE void setSearchKey( + F_DataVector * pSearchKey) + { + m_pSearchKey = pSearchKey; + } + + FINLINE void setCompareRowId( + FLMBOOL bCompareRowId) + { + m_bCompareRowId = bCompareRowId; + } + + FINLINE void setOldRow( + F_Row * pOldRow) + { + m_pOldRow = pOldRow; + } + + virtual FLMINT getRefCount( void) + { + return( IF_ResultSetCompare::getRefCount()); + } + + virtual FLMINT AddRef( void) + { + return( IF_ResultSetCompare::AddRef()); + } + + virtual FLMINT Release( void) + { + return( IF_ResultSetCompare::Release()); + } + +private: + + F_Db * m_pDb; + F_INDEX * m_pIndex; + F_DataVector * m_pSearchKey; + FLMBOOL m_bCompareRowId; + F_Row * m_pOldRow; +}; + +/*============================================================================= +Desc: Result set class for queries that do sorting. +=============================================================================*/ +class F_QueryResultSet : public F_Object +{ +public: + + F_QueryResultSet() + { + m_pBTree = NULL; + m_pResultSetDb = NULL; + m_pSrcDb = NULL; + m_pIndex = NULL; + m_uiCurrPos = FLM_MAX_UINT; + m_uiCount = 0; + m_bPositioned = FALSE; + m_hMutex = F_MUTEX_NULL; + } + + ~F_QueryResultSet(); + + // Initialize the result set + + RCODE initResultSet( + FLMBOOL bUseIxCompareObj, + FLMBOOL bEnableEncryption); + + FINLINE void setIxInfo( + F_Db * pSrcDb, + F_INDEX * pIndex) + { + m_pSrcDb = pSrcDb; + m_pIndex = pIndex; + m_compareObj.setIxInfo( pSrcDb, pIndex); + } + + // Entry Add and Sort Methods + + RCODE addEntry( // Variable or fixed length entry coming in + FLMBYTE * pucKey, // key for sorting. + FLMUINT uiKeyLength, + FLMBOOL bLockMutex); + + // Methods to read entries. + + RCODE getFirst( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + RCODE getLast( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + RCODE getNext( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + RCODE getPrev( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + RCODE getCurrent( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + RCODE positionToEntry( + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + F_DataVector * pSearchKey, + FLMUINT uiFlags, + FLMBOOL bLockMutex); + + RCODE positionToEntry( + FLMUINT uiPosition, + FLMBYTE * pucKey, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMBOOL bLockMutex); + + FINLINE FLMUINT getCount( void) + { + return( m_uiCount); + } + + FINLINE FLMUINT getCurrPos( void) + { + return( m_uiCurrPos); + } + + FINLINE void lockMutex( void) + { + f_mutexLock( m_hMutex); + } + + FINLINE void unlockMutex( void) + { + f_mutexUnlock( m_hMutex); + } + +private: + + char m_szResultSetDibName [F_PATH_MAX_SIZE]; + F_Db * m_pResultSetDb; + F_Btree * m_pBTree; + LFILE m_LFile; + F_Db * m_pSrcDb; + F_INDEX * m_pIndex; + IXKeyCompare m_compareObj; + FLMUINT m_uiCurrPos; + FLMUINT m_uiCount; + FLMBOOL m_bPositioned; + F_MUTEX m_hMutex; +}; + +typedef struct RS_WAITER +{ + FLMUINT uiThreadId; // Thread of waiter + F_SEM hESem; // Semaphore to signal to wake up thread. + RCODE * pRc; // Pointer to return code that is to + // be set. + FLMUINT uiWaitStartTime; + // Time we started waiting. + FLMUINT uiTimeLimit; // Maximum time (milliseconds) to wait + // before timing out. + FLMUINT uiNumToWaitFor;// Wait until we get at least this many + // in the result set - or until the + // result set is complete. + RS_WAITER * pNext; // Next lock waiter in list. + RS_WAITER * pPrev; // Previous lock waiter in list. +} RS_WAITER; + +/***************************************************************************** +Desc: FLAIM database system object +******************************************************************************/ +class F_DbSystem : public F_Object +{ +public: + + F_DbSystem() + { + m_refCnt = 1; + } + + virtual ~F_DbSystem() + { + } + + virtual FLMINT AddRef( void); + + virtual FLMINT Release( void); + + RCODE init( void); + + RCODE updateIniFile( + const char * pszParamName, + const char * pszValue); + + void exit(); + + void getFileSystem( + IF_FileSystem ** ppFileSystem); + + RCODE dbCreate( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + SFLM_CREATE_OPTS * pCreateOpts, + FLMBOOL bTempDb, + F_Db ** ppDb); + + FINLINE RCODE dbCreate( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + SFLM_CREATE_OPTS * pCreateOpts, + F_Db ** ppDb) + { + return( dbCreate( pszDbFileName, pszDataDir, pszRflDir, + pCreateOpts, FALSE, ppDb)); + } + + FINLINE RCODE dbOpen( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMBOOL bAllowLimited, + F_Db ** ppDb) + { + FLMUINT uiOpenFlags = bAllowLimited ? SFLM_ALLOW_LIMITED_MODE : 0; + + return( openDb( pszDbFileName, pszDataDir, pszRflDir, + pszPassword, uiOpenFlags, ppDb)); + } + + RCODE dbRebuild( + const char * pszSourceDbPath, + const char * pszSourceDataDir, + const char * pszDestDbPath, + const char * pszDestDataDir, + const char * pszDestRflDir, + const char * pszDictPath, + const char * pszPassword, + SFLM_CREATE_OPTS * pCreateOpts, + FLMUINT64 * pui64TotRows, + FLMUINT64 * pui64RowsRecov, + IF_DbRebuildStatus * pRebuildStatus); + + RCODE dbCheck( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiFlags, + F_DbInfo ** ppDbInfo, + IF_DbCheckStatus * pDbCheck); + + FINLINE RCODE dbDup( + F_Db * pDb, + F_Db ** ppDb) + { + return( openDatabase( pDb->m_pDatabase, NULL, NULL, NULL, NULL, 0, + FALSE, NULL, NULL, NULL, ppDb)); + } + + FINLINE RCODE setDynamicMemoryLimit( + FLMUINT uiCacheAdjustPercent, + FLMUINT uiCacheAdjustMin, + FLMUINT uiCacheAdjustMax, + FLMUINT uiCacheAdjustMinToLeave) + { + return( gv_SFlmSysData.pGlobalCacheMgr->setDynamicMemoryLimit( + uiCacheAdjustPercent, uiCacheAdjustMin, + uiCacheAdjustMax, uiCacheAdjustMinToLeave)); + } + + FINLINE RCODE setHardMemoryLimit( + FLMUINT uiPercent, + FLMBOOL bPercentOfAvail, + FLMUINT uiMin, + FLMUINT uiMax, + FLMUINT uiMinToLeave, + FLMBOOL bPreallocate) + { + return( gv_SFlmSysData.pGlobalCacheMgr->setHardMemoryLimit( uiPercent, + bPercentOfAvail, uiMin, uiMax, uiMinToLeave, bPreallocate)); + } + + // Determine if dyamic cache adjusting is supported. + + FINLINE FLMBOOL getDynamicCacheSupported( void) + { +#ifdef FLM_CAN_GET_PHYS_MEM + return( TRUE); +#else + return( FALSE); +#endif + } + + FINLINE void getCacheInfo( + SFLM_CACHE_INFO * pCacheInfo) + { + gv_SFlmSysData.pGlobalCacheMgr->getCacheInfo( pCacheInfo); + } + + // Enable/disable cache debugging mode + + void enableCacheDebug( + FLMBOOL bDebug); + + FLMBOOL cacheDebugEnabled( void); + + // Clear cache + + FINLINE RCODE clearCache( + F_Db * pDb) + { + return( gv_SFlmSysData.pGlobalCacheMgr->clearCache( pDb)); + } + + // Close all files that have not been used for the specified number of + // seconds. + + RCODE closeUnusedFiles( + FLMUINT uiSeconds); + + // Start gathering statistics. + + void startStats( void); + + // Stop gathering statistics. + + void stopStats( void); + + // Reset statistics. + + void resetStats( void); + + RCODE getStats( + SFLM_STATS * pFlmStats); + + void freeStats( + SFLM_STATS * pFlmStats); + + // Set the maximum number of queries to save. + + void setQuerySaveMax( + FLMUINT uiMaxToSave); + + FLMUINT getQuerySaveMax( void); + + // Set temporary directory. + + RCODE setTempDir( + const char * pszPath); + + RCODE getTempDir( + char * pszPath); + + // Maximum seconds between checkpoints. + + void setCheckpointInterval( + FLMUINT uiSeconds); + + FLMUINT getCheckpointInterval( void); + + // Set interval for dynamically adjusting cache limit. + + void setCacheAdjustInterval( + FLMUINT uiSeconds); + + FLMUINT getCacheAdjustInterval( void); + + // Set interval for dynamically cleaning out old cache blocks and records. + + void setCacheCleanupInterval( + FLMUINT uiSeconds); + + FLMUINT getCacheCleanupInterval( void); + + // Set interval for cleaning up unused structures. + + void setUnusedCleanupInterval( + FLMUINT uiSeconds); + + FLMUINT getUnusedCleanupInterval( void); + + // Set maximum time for an item to be unused. + + void setMaxUnusedTime( + FLMUINT uiSeconds); + + FLMUINT getMaxUnusedTime( void); + + // Specify the logger object + + void setLogger( + IF_LoggerClient * pLogger); + + // Enable or disable use of ESM + + void enableExtendedServerMemory( + FLMBOOL bEnable); + + FLMBOOL extendedServerMemoryEnabled( void); + + void deactivateOpenDb( + const char * pszDbFileName, + const char * pszDataDir); + + // Maximum dirty cache. + + void setDirtyCacheLimits( + FLMUINT uiMaxDirty, + FLMUINT uiLowDirty); + + void getDirtyCacheLimits( + FLMUINT * puiMaxDirty, + FLMUINT * puiLowDirty); + + RCODE getThreadInfo( + IF_ThreadInfo ** ppThreadInfo); + + RCODE registerForEvent( + eEventCategory eCategory, + IF_EventClient * pEventClient); + + void deregisterForEvent( + eEventCategory eCategory, + IF_EventClient * pEventClient); + + RCODE getNextMetaphone( + IF_IStream * pIStream, + FLMUINT * puiMetaphone, + FLMUINT * puiAltMetaphone = NULL); + + RCODE dbCopy( + const char * pszSrcDbName, + const char * pszSrcDataDir, + const char * pszSrcRflDir, + const char * pszDestDbName, + const char * pszDestDataDir, + const char * pszDestRflDir, + IF_DbCopyStatus * ifpStatus); + + RCODE dbRemove( + const char * pszDbName, + const char * pszDataDir, + const char * pszRflDir, + FLMBOOL bRemoveRflFiles); + + RCODE dbRename( + const char * pszDbName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszNewDbName, + FLMBOOL bOverwriteDestOk, + IF_DbRenameStatus * ifpStatus); + + RCODE dbRestore( + const char * pszDbPath, + const char * pszDataDir, + const char * pszRflDir, + const char * pszBackupPath, + const char * pszPassword, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus); + + RCODE strCmp( + FLMUINT uiCompFlags, + FLMUINT uiLanguage, + FLMUNICODE * uzStr1, + FLMUNICODE * uzStr2, + FLMINT * piCmp); + + FLMBOOL errorIsFileCorrupt( + RCODE rc); + + static FLMBOOL _errorIsFileCorrupt( + RCODE rc) + { + F_DbSystem dbSystem; + + return( dbSystem.errorIsFileCorrupt( rc)); + } + + const char * checkErrorToStr( + eCorruptionType eCorruption); + + FINLINE void freeMem( + void ** ppMem) + { + f_free( ppMem); + } + + FINLINE RCODE internalDbOpen( + F_Database * pDatabase, + F_Db ** ppDb) + { + RCODE rc = NE_SFLM_OK; + F_Db * pDb; + + if (RC_OK( rc = openDatabase( pDatabase, NULL, NULL, NULL, + NULL, 0, TRUE, NULL, NULL, NULL, &pDb))) + { + *ppDb = (F_Db *)pDb; + } + return( rc); + } + + RCODE openDb( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + F_Db ** ppDb); + + static FINLINE FLMBOOL validBlockSize( + FLMUINT uiBlockSize) + { + if( uiBlockSize == 4096 || uiBlockSize == 8192) + { + return( TRUE); + } + + return( FALSE); + } + + static void getDbBasePath( + char * pszBaseDbName, + const char * pszDbName, + FLMUINT * puiBaseDbNameLen); + + RCODE waitToClose( + const char * pszDbPath); + +private: + + // Methods + + RCODE readIniFile( void); + + RCODE setCacheParams( + IF_IniFile * pIniFile); + + void cleanup( void); + + FINLINE RCODE internalDbDup( + F_Db * pDb, + F_Db ** ppDb) + { + return( openDatabase( pDb->m_pDatabase, NULL, NULL, + NULL, NULL, 0, TRUE, NULL, NULL, NULL, ppDb)); + } + + void lockSysData( void); + + void unlockSysData( void); + + void initFastBlockCheckSum( void); + + RCODE allocDb( + F_Db ** ppDb, + FLMBOOL bInternalOpen); + + RCODE findDatabase( + const char * pszDbPath, + const char * pszDataDir, + F_Database ** ppDatabase); + + RCODE checkDatabaseClosed( + const char * pszDbName, + const char * pszDataDir); + + RCODE allocDatabase( + const char * pszDbPath, + const char * pszDataDir, + FLMBOOL bTempDb, + F_Database ** ppDatabase); + + RCODE openDatabase( + F_Database * pDatabase, + const char * pszDbPath, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + FLMBOOL bInternalOpen, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus, + IF_FileHdl * pLockFileHdl, + F_Db ** pDb); + + RCODE copyDb( + const char * pszSrcDbName, + const char * pszSrcDataDir, + const char * pszSrcRflDir, + const char * pszDestDbName, + const char * pszDestDataDir, + const char * pszDestRflDir, + IF_DbCopyStatus * ifpStatus); + + static RCODE monitorThrd( + IF_Thread * pThread); + + static RCODE cacheCleanupThrd( + IF_Thread * pThread); + + static void checkNotUsedObjects( void); + + static FLMATOMIC m_flmSysSpinLock; + static FLMUINT m_uiFlmSysStartupCount; + +friend class F_Db; +friend class F_Database; +friend class F_DbRebuild; +friend class F_DbCheck; +}; + +// Supported text types + +typedef enum +{ + SFLM_UNICODE_TEXT = 1, + SFLM_UTF8_TEXT +} eSFlmTextType; + +/*------------------------------------------------------ + FLAIM Processing Hooks (call-backs) +-------------------------------------------------------*/ + +#define FLM_DATA_LEFT_TRUNCATED 0x10 // Data is left truncated +#define FLM_DATA_RIGHT_TRUNCATED 0x20 // Data is right truncated + +RCODE flmReadStorageAsText( + IF_IStream * pIStream, + FLMBYTE * pucStorageData, + FLMUINT uiDataLen, + eDataType eDataTyp, + void * pvBuffer, + FLMUINT uiBufLen, + eSFlmTextType eTextType, + FLMUINT uiMaxCharsToRead, + FLMUINT uiCharOffset, + FLMUINT * puiCharsRead, + FLMUINT * puiBufferBytesUsed); + +RCODE flmReadStorageAsBinary( + IF_IStream * pIStream, + void * pvBuffer, + FLMUINT uiBufLen, + FLMUINT uiByteOffset, + FLMUINT * puiBytesRead); + +RCODE flmReadStorageAsNumber( + IF_IStream * pIStream, + eDataType eDataTyp, + FLMUINT64 * pui64Number, + FLMBOOL * pbNeg); + +RCODE flmReadLine( + IF_IStream * pIStream, + FLMBYTE * pucBuffer, + FLMUINT * puiSize); + +#define FLM_ENCRYPT_CHUNK_SIZE 512 + +/***************************************************************************** +Desc: +******************************************************************************/ +class F_BTreeIStream : public IF_PosIStream +{ +public: + + F_BTreeIStream() + { + m_pucBuffer = NULL; + m_pBTree = NULL; + m_bReleaseBTree = FALSE; + reset(); + } + + virtual ~F_BTreeIStream() + { + reset(); + } + + FINLINE void reset( void) + { + m_pNextInPool = NULL; + if( m_pBTree && m_bReleaseBTree) + { + m_pBTree->btClose(); + gv_SFlmSysData.pBtPool->btpReturnBtree( &m_pBTree); + m_pBTree = NULL; + } + + if( m_pucBuffer != &m_ucBuffer [0]) + { + f_free( &m_pucBuffer); + } + + m_pDb = NULL; + m_uiTableNum = 0; + m_ui64RowId = 0; + m_pBTree = NULL; + m_bReleaseBTree = FALSE; + m_uiKeyLength = 0; + m_uiStreamSize = 0; + m_uiBufferBytes = 0; + m_uiBufferOffset = 0; + m_uiBufferStartOffset = 0; + m_uiBufferSize = sizeof( m_ucBuffer); + m_pucBuffer = &m_ucBuffer [0]; + m_ui32BlkAddr = 0; + m_uiOffsetIndex = 0; + m_uiDataLength = 0; + } + + RCODE open( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT32 ui32BlkAddr = 0, + FLMUINT uiOffsetIndex = 0); + + RCODE open( + F_Db * pDb, + F_Btree * pBTree, + FLMUINT uiFlags, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT32 ui32BlkAddr = 0, + FLMUINT uiOffsetIndex = 0); + + FINLINE FLMUINT64 totalSize( void) + { + return( m_uiStreamSize); + } + + FINLINE FLMUINT64 remainingSize( void) + { + return( m_uiStreamSize - (m_uiBufferStartOffset + m_uiBufferOffset)); + } + + FINLINE RCODE close( void) + { + reset(); + return( NE_SFLM_OK); + } + + RCODE positionTo( + FLMUINT64 ui64Position); + + FINLINE FLMUINT64 getCurrPosition( void) + { + return( m_uiBufferStartOffset + m_uiBufferOffset); + } + + RCODE read( + void * pvBuffer, + FLMUINT uiBytesToRead, + FLMUINT * puiBytesRead); + + FLMINT Release( void); + + FINLINE FLMUINT32 getBlkAddr( void) + { + return( m_ui32BlkAddr); + } + + FINLINE FLMUINT getOffsetIndex( void) + { + return( m_uiOffsetIndex); + } + +private: + + F_BTreeIStream * m_pNextInPool; + F_Db * m_pDb; + F_Btree * m_pBTree; + FLMUINT m_uiTableNum; + FLMUINT64 m_ui64RowId; + FLMUINT m_uiStreamSize; + FLMUINT m_uiKeyLength; + FLMUINT m_uiBufferBytes; + FLMUINT m_uiBufferSize; + FLMUINT m_uiBufferOffset; + FLMUINT m_uiBufferStartOffset; + FLMUINT m_uiDataLength; + FLMBYTE m_ucBuffer[ FLM_ENCRYPT_CHUNK_SIZE]; + FLMBYTE * m_pucBuffer; + FLMUINT m_uiOffsetIndex; + FLMUINT32 m_ui32BlkAddr; + FLMBOOL m_bReleaseBTree; + FLMBYTE m_ucKey[ FLM_MAX_NUM_BUF_SIZE]; +friend class F_Row; +friend class F_Db; +friend class F_BTreeIStreamPool; +}; + +/*=========================================================================== +Desc: Pool manager for b-tree istream objects +===========================================================================*/ +class F_BTreeIStreamPool : public F_Object +{ +public: + + F_BTreeIStreamPool() + { + m_pFirstBTreeIStream = NULL; + m_hMutex = F_MUTEX_NULL; + } + + ~F_BTreeIStreamPool(); + + RCODE setup( void); + + RCODE allocBTreeIStream( + F_BTreeIStream ** ppBTreeIStream); + + void insertBTreeIStream( + F_BTreeIStream * pBTreeIStream); + +private: + + F_BTreeIStream * m_pFirstBTreeIStream; + F_MUTEX m_hMutex; + +friend class F_BTreeIStream; +}; + +typedef struct RSIxKeyTag +{ + FLMBYTE pucRSKeyBuf[ SFLM_MAX_KEY_SIZE]; + FLMUINT uiRSKeyLen; + FLMBYTE pucRSDataBuf[ SFLM_MAX_KEY_SIZE]; + FLMUINT uiRSDataLen; +} RS_IX_KEY; + +/****************************************************************************** +Desc: +******************************************************************************/ +class F_KeyCollector : public F_Object +{ + +public: + + F_KeyCollector( + F_DbCheck * pDbCheck) + { + m_pDbCheck = pDbCheck; + m_ui64TotalKeys = 0; + } + + ~F_KeyCollector(){} + + RCODE addKey( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY * pKref); + + FLMUINT64 getTotalKeys() + { + return m_ui64TotalKeys; + } + +private: + + F_DbCheck * m_pDbCheck; + FLMUINT64 m_ui64TotalKeys; + +friend class F_DbCheck; +}; + +typedef struct +{ + FLMUINT64 ui64BytesUsed; + FLMUINT64 ui64ElementCount; + FLMUINT64 ui64ContElementCount; + FLMUINT64 ui64ContElmBytes; + FLMUINT uiBlockCount; + FLMINT iErrCode; + FLMUINT uiNumErrors; +} BLOCK_INFO; + +typedef struct +{ + FLMUINT64 ui64KeyCount; + BLOCK_INFO BlockInfo; +} LEVEL_INFO; + +typedef struct +{ + FLMUINT uiLfNum; + eLFileType eLfType; + FLMUINT uiRootBlk; + FLMUINT uiNumLevels; + LEVEL_INFO * pLevelInfo; +} LF_HDR; + +/****************************************************************************** +Desc: +******************************************************************************/ +typedef struct State_Info +{ + F_Db * pDb; + FLMUINT32 ui32BlkAddress; + FLMUINT32 ui32NextBlkAddr; + FLMUINT32 ui32PrevBlkAddr; + FLMUINT32 ui32LastChildAddr; + FLMUINT uiElmLastFlag; + FLMUINT64 ui64KeyCount; + FLMUINT64 ui64KeyRefs; + FLMUINT uiBlkType; + FLMUINT uiLevel; + FLMUINT uiRootLevel; + FLMUINT uiElmOffset; + FLMBYTE * pucElm; + FLMUINT uiElmLen; + FLMUINT uiElmKeyLenOffset; // New + FLMUINT uiElmKeyOffset; // New + FLMBYTE * pucElmKey; + FLMUINT uiElmKeyLen; + FLMUINT uiCurKeyLen; // Used in Rebuild... + FLMUINT uiElmDataLenOffset; // New + FLMUINT uiElmDataOffset; // uiElmRecOffset; + FLMUINT uiElmDataLen; // uiElmRecLen; + FLMBYTE * pucElmData; // pucElmRec; + FLMUINT uiElmCounts; // New + FLMUINT uiElmCountsOffset; // New + FLMUINT uiElmOADataLenOffset; + FLMUINT uiElmOADataLen; + FLMUINT64 ui64ElmNodeId; + FLMBOOL bValidKey; + F_BLK_HDR * pBlkHdr; + BLOCK_INFO BlkInfo; + F_BtResultSet * pNodeRS; + F_BtResultSet * pXRefRS; + FLMUINT uiCurrLf; +} STATE_INFO; + +/****************************************************************************** +Desc: +******************************************************************************/ +typedef struct +{ + FLMUINT uiStartOfEntry; + FLMUINT uiEndOfEntry; +} BlkStruct; + +/****************************************************************************** +Desc: DbCheck object for verifying the condition of the database. +******************************************************************************/ +class F_DbCheck : public F_Object +{ + +public: + F_DbCheck( void) + { + m_pXRefRS = NULL; + m_pDb = NULL; + m_pIndex = NULL; + m_pTable = NULL; + m_pLFile = NULL; + m_pDbInfo = NULL; + m_pBtPool = NULL; + m_pResultSetDb = NULL; + m_pRandGen = NULL; + m_pDbCheckStatus = NULL; + f_memset( m_szResultSetDibName, 0, sizeof( m_szResultSetDibName)); + m_bPhysicalCorrupt = FALSE; + m_bIndexCorrupt = FALSE; + m_LastStatusRc = NE_SFLM_OK; + m_uiFlags = 0; + m_bStartedUpdateTrans = FALSE; + m_puiIxArray = NULL; + m_pIxRSet = NULL; + m_bGetNextRSKey = FALSE; + f_memset( &m_IxKey1, 0, sizeof( m_IxKey1)); + f_memset( &m_IxKey2, 0, sizeof( m_IxKey2)); + m_pCurrRSKey = NULL; + m_pPrevRSKey = NULL; + m_pBlkEntries = NULL; + m_uiBlkEntryArraySize = 0; + } + + ~F_DbCheck(); + + RCODE dbCheck( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiFlags, + F_DbInfo ** ppDbInfo, + IF_DbCheckStatus * pDbCheck); + +private: + + FINLINE RCODE chkCallProgFunc( void) + { + if (m_pDbCheckStatus && RC_OK( m_LastStatusRc)) + { + m_LastStatusRc = m_pDbCheckStatus->reportProgress( &m_Progress); + } + return( m_LastStatusRc); + } + + RCODE chkReportError( + FLMINT iErrCode, + FLMUINT uiErrLocale, + FLMUINT uiErrLfNumber, + FLMUINT uiErrLfType, + FLMUINT uiErrBTreeLevel, + FLMUINT uiErrBlkAddress, + FLMUINT uiErrParentBlkAddress, + FLMUINT uiErrElmOffset, + FLMUINT64 ui64ErrNodeId); + + FINLINE SFLM_PROGRESS_CHECK_INFO * getProgress( void) + { + return( &m_Progress); + } + + RCODE getBtResultSet( + F_BtResultSet ** ppBtRSet); + + RCODE createAndOpenResultSetDb( void); + + RCODE closeAndDeleteResultSetDb( void); + + RCODE getDictInfo( void); + + RCODE verifyBlkChain( + BLOCK_INFO * pBlkInfo, + FLMUINT uiLocale, + FLMUINT uiFirstBlkAddr, + FLMUINT uiBlkType, + FLMBOOL * pbStartOverRV); + + RCODE verifyLFHBlocks( + FLMBOOL * pbStartOverRV); + + RCODE verifyAvailList( + FLMBOOL * pbStartOverRV); + + RCODE blkRead( + FLMUINT uiBlkAddress, + F_BLK_HDR ** ppBlkHdr, + F_CachedBlock ** ppSCache, + FLMINT * piBlkErrCodeRV); + + RCODE verifySubTree( + STATE_INFO * pParentState, + STATE_INFO * pStateInfo, + FLMUINT uiBlkAddress, + FLMBYTE ** ppucResetKey, + FLMUINT uiResetKeyLen, + FLMUINT64 ui64ResetNodeId); + + RCODE buildIndexKeyList( + FLMUINT64 * pui64TotalKeys); + + RCODE verifyBTrees( + FLMBOOL * pbStartOverRV); + + RCODE setupLfTable(); + + RCODE setupIxInfo( void); + + RCODE getLfInfo( + LF_HDR * pLogicalFile, + LFILE * pLFile); + + RCODE verifyNodePointers( + STATE_INFO * pStateInfo, + FLMINT * piErrCode); + + RCODE verifyDOChain( + STATE_INFO * pParentState, + FLMUINT uiBlkAddr, + FLMINT * piElmErrCode); + + RCODE chkGetNextRSKey( void); + + RCODE verifyIXRSet( + STATE_INFO * pStateInfo); + + RCODE resolveIXMissingKey( + STATE_INFO * pStateInfo); + + RCODE verifyComponentInDoc( + ICD * pIcd, + FLMUINT uiComponent, + F_DataVector * pKey, + FLMBOOL * pbInDoc); + + RCODE getKeySource( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL * pbKeyInDoc, + FLMBOOL * pbKeyInIndex); + + RCODE resolveRSetMissingKey( + STATE_INFO * pStateInfo); + + RCODE chkVerifyKeyExists( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL * pbFoundRV); + + RCODE addDelKeyRef( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + FLMBOOL bDelete); + + RCODE reportIxError( + STATE_INFO * pStateInfo, + FLMINT iErrCode, + FLMBYTE * pucErrKey, + FLMUINT uiErrKeyLen, + FLMBOOL * pbFixErrRV); + + RCODE startUpdate( void); + + RCODE keyToVector( + FLMBYTE * pucKey, + FLMUINT uiKeyLen, + F_DataVector ** ppKeyRV); + + RCODE verifyIXRefs( + STATE_INFO * pStateInfo, + FLMUINT64 ui64ResetNodeId); + + RCODE verifyBlockStructure( + FLMUINT uiBlockSize, + F_BTREE_BLK_HDR * pBlkHdr); + + RCODE chkEndUpdate( void); + + F_Db * m_pDb; + F_TABLE * m_pTable; + F_INDEX * m_pIndex; + LFILE * m_pLFile; + F_DbInfo * m_pDbInfo; + F_BtResultSet * m_pXRefRS; + F_BtPool * m_pBtPool; + IF_RandomGenerator * m_pRandGen; + char m_szResultSetDibName [F_PATH_MAX_SIZE]; + F_Db * m_pResultSetDb; + IF_DbCheckStatus * m_pDbCheckStatus; + FLMBOOL m_bPhysicalCorrupt; + FLMBOOL m_bIndexCorrupt; + SFLM_PROGRESS_CHECK_INFO m_Progress; + RCODE m_LastStatusRc; + FLMUINT m_uiFlags; + FLMBOOL m_bStartedUpdateTrans; + FLMUINT * m_puiIxArray; + F_BtResultSet * m_pIxRSet; + FLMBOOL m_bGetNextRSKey; + RS_IX_KEY m_IxKey1; + RS_IX_KEY m_IxKey2; + RS_IX_KEY * m_pCurrRSKey; + RS_IX_KEY * m_pPrevRSKey; + BlkStruct * m_pBlkEntries; + FLMUINT m_uiBlkEntryArraySize; +friend class F_DbInfo; +friend class F_KeyCollector; +}; + +// Check Error Codes + +/* +** WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE +** REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN FLERRSTR.CPP +*/ +#define FLM_BAD_CHAR 1 +#define FLM_BAD_ASIAN_CHAR 2 +#define FLM_BAD_CHAR_SET 3 +#define FLM_BAD_TEXT_FIELD 4 +#define FLM_BAD_NUMBER_FIELD 5 +#define FLM_BAD_FIELD_TYPE 6 +#define FLM_BAD_IX_DEF 7 +#define FLM_MISSING_REQ_KEY_FIELD 8 +#define FLM_BAD_TEXT_KEY_COLL_CHAR 9 +#define FLM_BAD_TEXT_KEY_CASE_MARKER 10 +#define FLM_BAD_NUMBER_KEY 11 +#define FLM_BAD_BINARY_KEY 12 +#define FLM_BAD_CONTEXT_KEY 13 +#define FLM_BAD_KEY_FIELD_TYPE 14 +//#define Not_Used_15 15 +//#define Not_Used_16 16 +//#define Not_Used_17 17 +#define FLM_BAD_KEY_LEN 18 +#define FLM_BAD_LFH_LIST_PTR 19 +#define FLM_BAD_LFH_LIST_END 20 +#define FLM_INCOMPLETE_NODE 21 // Not really an error. Part of a field has been split across entries +#define FLM_BAD_BLK_END 22 +#define FLM_KEY_COUNT_MISMATCH 23 +#define FLM_REF_COUNT_MISMATCH 24 +#define FLM_BAD_CONTAINER_IN_KEY 25 +#define FLM_BAD_BLK_HDR_ADDR 26 +#define FLM_BAD_BLK_HDR_LEVEL 27 +#define FLM_BAD_BLK_HDR_PREV 28 +/* +** WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE +** REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN FLERRSTR.CPP +*/ +#define FLM_BAD_BLK_HDR_NEXT 29 +#define FLM_BAD_BLK_HDR_TYPE 30 +#define FLM_BAD_BLK_HDR_ROOT_BIT 31 +#define FLM_BAD_BLK_HDR_BLK_END 32 +#define FLM_BAD_BLK_HDR_LF_NUM 33 +#define FLM_BAD_AVAIL_LIST_END 34 +#define FLM_BAD_PREV_BLK_NEXT 35 +#define FLM_BAD_FIRST_ELM_FLAG 36 // NOTE: This is only needed during rebuild +#define FLM_BAD_LAST_ELM_FLAG 37 // NOTE: This is only needed during rebuild +#define FLM_BAD_LEM 38 +#define FLM_BAD_ELM_LEN 39 +#define FLM_BAD_ELM_KEY_SIZE 40 +#define FLM_BAD_ELM_KEY 41 +#define FLM_BAD_ELM_KEY_ORDER 42 +#define FLM_BAD_ELM_KEY_COMPRESS 43 // NOTE: 5.x keys are not compressed +#define FLM_BAD_CONT_ELM_KEY 44 +#define FLM_NON_UNIQUE_FIRST_ELM_KEY 45 +#define FLM_BAD_ELM_OFFSET 46 +#define FLM_BAD_ELM_INVALID_LEVEL 47 +#define FLM_BAD_ELM_FLD_NUM 48 +#define FLM_BAD_ELM_FLD_LEN 49 +#define FLM_BAD_ELM_FLD_TYPE 50 +#define FLM_BAD_ELM_END 51 +#define FLM_BAD_PARENT_KEY 52 +#define FLM_BAD_ELM_DOMAIN_SEN 53 +#define FLM_BAD_ELM_BASE_SEN 54 +#define FLM_BAD_ELM_IX_REF 55 +#define FLM_BAD_ELM_ONE_RUN_SEN 56 +#define FLM_BAD_ELM_DELTA_SEN 57 +#define FLM_BAD_ELM_DOMAIN 58 +/* +** WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE +** REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN FLERRSTR.CPP +*/ +#define FLM_BAD_LAST_BLK_NEXT 59 +#define FLM_BAD_FIELD_PTR 60 +#define FLM_REBUILD_REC_EXISTS 61 +#define FLM_REBUILD_KEY_NOT_UNIQUE 62 +#define FLM_NON_UNIQUE_ELM_KEY_REF 63 +#define FLM_OLD_VIEW 64 +#define FLM_COULD_NOT_SYNC_BLK 65 +#define FLM_IX_REF_REC_NOT_FOUND 66 +#define FLM_IX_KEY_NOT_FOUND_IN_REC 67 +#define FLM_KEY_NOT_IN_KEY_REFSET 68 +#define FLM_BAD_BLK_CHECKSUM 69 +#define FLM_BAD_LAST_DRN 70 +#define FLM_BAD_FILE_SIZE 71 +#define FLM_BAD_FIRST_LAST_ELM_FLAG 72 +#define FLM_BAD_DATE_FIELD 73 +#define FLM_BAD_TIME_FIELD 74 +#define FLM_BAD_TMSTAMP_FIELD 75 +#define FLM_BAD_DATE_KEY 76 +#define FLM_BAD_TIME_KEY 77 +#define FLM_BAD_TMSTAMP_KEY 78 +#define FLM_BAD_BLOB_FIELD 79 + +/* +** WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE +** REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN FLERRSTR.CPP +*/ + +#define FLM_BAD_PCODE_IXD_TBL 80 +#define FLM_NODE_QUARANTINED 81 +#define FLM_BAD_BLK_TYPE 82 +#define FLM_BAD_ELEMENT_CHAIN 83 +#define FLM_BAD_ELM_EXTRA_DATA 84 +#define FLM_BAD_BLOCK_STRUCTURE 85 +#define FLM_BAD_ROOT_PARENT 86 +#define FLM_BAD_ROOT_LINK 87 +#define FLM_BAD_PARENT_LINK 88 +#define FLM_BAD_INVALID_ROOT 89 +#define FLM_BAD_FIRST_CHILD_LINK 90 +#define FLM_BAD_LAST_CHILD_LINK 91 +#define FLM_BAD_PREV_SIBLING_LINK 92 +#define FLM_BAD_NEXT_SIBLING_LINK 93 +#define FLM_BAD_ANNOTATION_LINK 94 +#define FLM_UNSUPPORTED_NODE_TYPE 95 +#define FLM_BAD_INVALID_NAME_ID 96 +#define FLM_BAD_INVALID_PREFIX_ID 97 +#define FLM_BAD_DATA_BLOCK_COUNT 98 +#define FLM_BAD_AVAIL_SIZE 99 +#define FLM_BAD_NODE_TYPE 100 +#define FLM_BAD_CHILD_ELM_COUNT 101 +#define FLM_NUM_CORRUPT_ERRORS 101 + +/* +** WARNING: ANY CHANGES MADE TO THE CHECK ERROR CODE DEFINES MUST BE +** REFLECTED IN THE FlmCorruptStrings TABLE FOUND IN FLERRSTR.CPP +*/ + +class F_DbInfo : public F_Object +{ +public: + + F_DbInfo() + { + m_uiLogicalCorruptions = 0; + m_uiLogicalRepairs = 0; + m_ui64FileSize = 0; + m_uiNumIndexes = 0; + m_uiNumTables = 0; + m_uiNumLogicalFiles = 0; + m_pLogicalFiles = NULL; + f_memset( &m_dbHdr, 0, sizeof( m_dbHdr)); + f_memset( &m_AvailBlocks, 0, sizeof( m_AvailBlocks)); + f_memset( &m_LFHBlocks, 0, sizeof( m_LFHBlocks)); + } + + virtual ~F_DbInfo() + { + freeLogicalFiles(); + } + + FINLINE void freeLogicalFiles( void) + { + FLMUINT uiLoop; + + if (m_pLogicalFiles) + { + for (uiLoop = 0; uiLoop < m_uiNumLogicalFiles; uiLoop++) + { + if (m_pLogicalFiles [uiLoop].pLevelInfo) + { + f_free( &m_pLogicalFiles [uiLoop].pLevelInfo); + } + } + f_free( &m_pLogicalFiles); + } + m_uiNumLogicalFiles = 0; + m_uiNumIndexes = 0; + m_uiNumTables = 0; + } + + FINLINE FLMUINT getNumTables( void) + { + return( m_uiNumTables); + } + + FINLINE FLMUINT getNumIndexes( void) + { + return( m_uiNumIndexes); + } + + FINLINE FLMUINT getNumLogicalFiles( void) + { + return( m_uiNumLogicalFiles); + } + + FINLINE FLMUINT64 getFileSize( void) + { + return( m_ui64FileSize); + } + + FINLINE SFLM_DB_HDR * getDbHdr( void) + { + return( &m_dbHdr); + } + + FINLINE void getAvailBlockStats( + FLMUINT64 * pui64BytesUsed, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors) + { + *pui64BytesUsed = m_LFHBlocks.ui64BytesUsed; + *puiBlockCount = m_AvailBlocks.uiBlockCount; + *piLastError = m_AvailBlocks.iErrCode; + *puiNumErrors = m_AvailBlocks.uiNumErrors; + } + + FINLINE void getLFHBlockStats( + FLMUINT64 * pui64BytesUsed, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors) + { + *pui64BytesUsed = m_LFHBlocks.ui64BytesUsed; + *puiBlockCount = m_LFHBlocks.uiBlockCount; + *piLastError = m_LFHBlocks.iErrCode; + *puiNumErrors = m_LFHBlocks.uiNumErrors; + } + + void getBTreeInfo( + FLMUINT uiNthLogicalFile, + FLMUINT * puiLfNum, + eLFileType * peLfType, + FLMUINT * puiRootBlkAddress, + FLMUINT * puiNumLevels); + + void getBTreeBlockStats( + FLMUINT uiNthLogicalFile, + FLMUINT uiLevel, + FLMUINT64 * pui64KeyCount, + FLMUINT64 * pui64BytesUsed, + FLMUINT64 * pui64ElementCount, + FLMUINT64 * pui64ContElementCount, + FLMUINT64 * pui64ContElmBytes, + FLMUINT * puiBlockCount, + FLMINT * piLastError, + FLMUINT * puiNumErrors); + +private: + + FLMUINT m_uiLogicalCorruptions; + FLMUINT m_uiLogicalRepairs; + FLMUINT64 m_ui64FileSize; + FLMUINT m_uiNumIndexes; + FLMUINT m_uiNumTables; + FLMUINT m_uiNumLogicalFiles; + LF_HDR * m_pLogicalFiles; + SFLM_DB_HDR m_dbHdr; + BLOCK_INFO m_AvailBlocks; + BLOCK_INFO m_LFHBlocks; +friend class F_DbCheck; +}; + +#define REBUILD_BLK_SIZE (1024 * 50) +#define REBUILD_RSET_ENTRY_SIZE 21 + +/*============================================================================= +Desc: Class to rebuild a broken database. This class is used by + F_DbSystem::dbRebuild() +=============================================================================*/ +class F_DbRebuild : public F_Object +{ +public: + + // Constructor and Destructor + + F_DbRebuild( void) + { + m_pDb = NULL; + m_pSFileHdl = NULL; + } + + ~F_DbRebuild() + { + } + + RCODE dbRebuild( + const char * pszSourceDbPath, + const char * pszSourceDataDir, + const char * pszDestDbPath, + const char * pszDestDataDir, + const char * pszDestRflDir, + const char * pDictPath, + const char * pszPassword, + SFLM_CREATE_OPTS * pCreateOpts, + FLMUINT64 * pui64TotNodes, + FLMUINT64 * pui64NodesRecov, + FLMUINT64 * pui64QuarantinedNodes, + IF_DbRebuildStatus * pRebuildStatus); + + FINLINE FLMUINT getBlockSize( void) + { + return( m_dbHdr.ui16BlockSize); + } + + FINLINE RCODE reportStatus( + FLMBOOL bForce = FALSE) + { + RCODE rc = NE_SFLM_OK; + + if( m_pRebuildStatus) + { + FLMUINT uiCurrentTime = FLM_GET_TIMER(); + FLMUINT uiElapTime = FLM_ELAPSED_TIME( uiCurrentTime, m_uiLastStatusTime); + + uiElapTime = FLM_TIMER_UNITS_TO_SECS( uiElapTime); + + if( bForce || uiElapTime >= 1) + { + m_uiLastStatusTime = uiCurrentTime; + m_callbackData.bStartFlag = FALSE; + if( RC_BAD( rc = m_pRebuildStatus->reportRebuild( &m_callbackData))) + { + m_cbrc = rc; + goto Exit; + } + } + } + + Exit: + + return( rc); + } + +private: + + RCODE rebuildDatabase( void); + + RCODE recoverNodes( + FLMBOOL bRecoverDictionary); + + FINLINE void buildRSetEntry( + FLMBYTE ucPrefix, + FLMUINT uiCollection, + FLMUINT64 ui64NodeId, + FLMUINT uiBlockAddr, + FLMUINT uiElmNumber, + FLMBYTE * pucBuffer) + { + pucBuffer[ 0] = ucPrefix; + + UD2FBA( (FLMUINT32)uiCollection, &pucBuffer[ 1]); + U642FBA( ui64NodeId, &pucBuffer[ 5]); + UD2FBA( (FLMUINT32)uiBlockAddr, &pucBuffer[ 13]); + UD2FBA( (FLMUINT32)uiElmNumber, &pucBuffer[ 17]); + } + + FINLINE void extractRSetEntry( + FLMBYTE * pucBuffer, + FLMUINT * puiCollection, + FLMUINT64 * pui64NodeId, + FLMUINT * puiBlockAddr, + FLMUINT * puiElmNumber) + { + if( puiCollection) + { + *puiCollection = FB2UD( &pucBuffer[ 1]); + } + + if( pui64NodeId) + { + *pui64NodeId = FB2U64( &pucBuffer[ 5]); + } + + if( puiBlockAddr) + { + *puiBlockAddr = FB2UD( &pucBuffer[ 13]); + } + + if( puiElmNumber) + { + *puiElmNumber = FB2UD( &pucBuffer[ 17]); + } + } + + RCODE recoverTree( + F_RebuildRowIStream * pIStream, + IF_ResultSet * pNonRootRSet, + F_Row * pRecovCachedNode, + FLMBYTE * pucNodeIV); + + FINLINE RCODE reportCorruption( + eCorruptionType eCorruption, + FLMUINT uiErrBlkAddress, + FLMUINT uiErrElmOffset, + FLMUINT64 ui64ErrRowId) + { + RCODE rc; + + if( m_pRebuildStatus) + { + m_corruptInfo.eCorruption = eCorruption; + m_corruptInfo.uiErrBlkAddress = uiErrBlkAddress; + m_corruptInfo.uiErrElmOffset = uiErrElmOffset; + m_corruptInfo.ui64ErrRowId = ui64ErrRowId; + rc = m_pRebuildStatus->reportRebuildErr( &m_corruptInfo); + m_corruptInfo.eCorruption = SFLM_NO_CORRUPTION; + return( rc); + } + + return( NE_SFLM_OK); + } + + RCODE determineBlkSize( + FLMUINT * puiBlkSizeRV); + + F_Db * m_pDb; + F_SuperFileHdl * m_pSFileHdl; + IF_DbRebuildStatus * m_pRebuildStatus; + FLMBOOL m_bBadHeader; + FLMUINT m_uiLastStatusTime; + SFLM_DB_HDR m_dbHdr; + SFLM_CREATE_OPTS m_createOpts; + SFLM_REBUILD_INFO m_callbackData; + SFLM_CORRUPT_INFO m_corruptInfo; + RCODE m_cbrc; + +friend class F_RebuildRowIStream; +}; + +RCODE chkBlkRead( + F_DbInfo * pDbInfo, + FLMUINT uiBlkAddress, + F_BLK_HDR ** ppBlkHdr, + F_CachedBlock ** ppSCache, + FLMINT * piBlkErrCodeRV); + +FLMINT flmCompareKeys( + FLMBYTE * pBuf1, + FLMUINT uiBuf1Len, + FLMBYTE * pBuf2, + FLMUINT uiBuf2Len); + +void flmInitReadState( + STATE_INFO * pStateInfo, + FLMBOOL * pbStateInitialized, + FLMUINT uiVersionNum, + F_Db * pDb, + LF_HDR * pLogicalFile, + FLMUINT uiLevel, + FLMUINT uiBlkType, + FLMBYTE * pucKeyBuffer); + +FLMINT flmVerifyBlockHeader( + STATE_INFO * pStateInfo, + BLOCK_INFO * pBlockInfoRV, + FLMUINT uiBlockSize, + FLMUINT uiExpNextBlkAddr, + FLMUINT uiExpPrevBlkAddr, + FLMBOOL bCheckEOF); + +RCODE flmVerifyElement( + STATE_INFO * pStateInfo, + LFILE * pLFile, + F_INDEX * pIndex, + FLMINT * piErrCode); + +void getEntryInfo( + F_BTREE_BLK_HDR * pBlkHdr, + FLMUINT uiOffset, + FLMBYTE ** ppucElm, + FLMUINT * puiElmLen, + FLMUINT * puiElmKeyLen, + FLMUINT * puiElmDataLen, + FLMBYTE ** ppucElmKey, + FLMBYTE ** ppucElmData); + +FINLINE RCODE F_RowCacheMgr::makeWriteCopy( + F_Db * pDb, + F_Row ** ppRow) +{ + if ((*ppRow)->getLowTransId() < pDb->m_ui64CurrTransID) + { + return( gv_SFlmSysData.pRowCacheMgr->_makeWriteCopy( pDb, ppRow)); + } + + return( NE_SFLM_OK); +} + +// More includes + +#include "flaimodbc.h" +#include "fscursor.h" + +#endif // FLAIMSYS_H diff --git a/sql/src/flbackup.cpp b/sql/src/flbackup.cpp new file mode 100644 index 0000000..ab73db5 --- /dev/null +++ b/sql/src/flbackup.cpp @@ -0,0 +1,2573 @@ +//------------------------------------------------------------------------------ +// Desc: Backup and restore Routines +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flbackup.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +// Typedefs + +typedef struct +{ + FLMUINT64 ui64BytesToDo; + FLMUINT64 ui64BytesDone; +} DB_BACKUP_INFO, * DB_BACKUP_INFO_p; + +// Local classes + +class F_BackerStream : public F_Object +{ +public: + + F_BackerStream( void); + ~F_BackerStream( void); + + RCODE setup( + FLMUINT uiMTUSize, + IF_RestoreClient * pRestoreObj); + + RCODE setup( + FLMUINT uiMTUSize, + IF_BackupClient * pClient); + + RCODE startThreads( void); + + void shutdownThreads( void); + + RCODE read( + FLMUINT uiLength, + FLMBYTE * pucData, + FLMUINT * puiBytesRead = NULL); + + RCODE write( + FLMUINT uiLength, + FLMBYTE * pucData, + FLMUINT * puiBytesWritten = NULL); + + RCODE flush( void); + + FINLINE FLMUINT64 getByteCount( void) + { + // Returns the total number of bytes read or written. + + return( m_ui64ByteCount); + } + + FINLINE FLMUINT getMTUSize( void) + { + return( m_uiMTUSize); + } + +private: + + // Methods + + RCODE signalThread( void); + + RCODE _setup( void); + + static RCODE readThread( + IF_Thread * pThread); + + static RCODE writeThread( + IF_Thread * pThread); + + // Data + + FLMBOOL m_bSetup; + FLMBOOL m_bFirstRead; + FLMBOOL m_bFinalRead; + FLMUINT m_uiBufOffset; + FLMUINT64 m_ui64ByteCount; + IF_RestoreClient * m_pRestoreObj; + F_SEM m_hDataSem; + F_SEM m_hIdleSem; + IF_Thread * m_pThread; + RCODE m_rc; + FLMBYTE * m_pucInBuf; + FLMUINT * m_puiInOffset; + FLMBYTE * m_pucOutBuf; + FLMUINT * m_puiOutOffset; + FLMBYTE * m_pucBufs[ 2]; + FLMUINT m_uiOffsets[ 2]; + FLMUINT m_uiMTUSize; + IF_BackupClient * m_pClient; +}; + +// Constants + +#define FLM_BACKER_SIGNATURE_OFFSET 0 +#define FLM_BACKER_SIGNATURE "!DB_BACKUP_FILE!" +#define FLM_BACKER_SIGNATURE_SIZE 16 +#define FLM_BACKER_VERSION_OFFSET 16 +#define FLM_BACKER_VERSION_5_0_0 500 +#define FLM_BACKER_VERSION FLM_BACKER_VERSION_5_0_0 +#define FLM_BACKER_DB_BLOCK_SIZE_OFFSET 20 +#define FLM_BACKER_BFMAX_OFFSET 24 +#define FLM_BACKER_MTU_OFFSET 28 +#define FLM_BACKER_TIME_OFFSET 32 +#define FLM_BACKER_DB_NAME_OFFSET 36 +#define FLM_BACKER_BACKUP_TYPE_OFFSET 40 +#define FLM_BACKER_NEXT_INC_SERIAL_NUM 44 +#define FLM_BACKER_DB_VERSION 60 + +// The backer MTU size must be a multiple of the largest +// supported block size. Additionally, it must be at least +// 2 * FLM_BACKER_MAX_DB_BLOCK_SIZE. DO NOT CHANGE THE MTU +// SIZE UNLESS YOU ALSO BUMP THE BACKUP VERSION. + +#define FLM_BACKER_MTU_SIZE ((FLMUINT) 1024 * 512) +#define FLM_BACKER_MAX_FILE_SIZE ((FLMUINT) 1024 * 1024 * 1024 * 2) // 2 Gigabytes +#define FLM_BACKER_MIN_DB_BLOCK_SIZE ((FLMUINT) 2 * 1024) +#define FLM_BACKER_MAX_DB_BLOCK_SIZE ((FLMUINT) 16 * 1024) + +// Backup block header + +#define FLM_BACKER_BLK_HDR_SIZE ((FLMUINT) 4) +#define FLM_BACKER_BLK_ADDR_OFFSET 0 + +// Local prototypes + +FSTATIC RCODE flmRestoreFile( + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus, + F_SuperFileHdl * pSFile, + FLMBOOL bIncremental, + FLMUINT * puiDbVersion, + FLMUINT * puiNextIncSeqNum, + FLMBOOL * pbRflPreserved, + eRestoreAction * peAction, + FLMBOOL * pbOKToRetry); + +// Functions + +/*************************************************************************** +Desc : Prepares FLAIM to backup a database. +Notes: Only one backup of a particular database can be active at any time +*END************************************************************************/ +RCODE F_Db::backupBegin( + eDbBackupType eBackupType, + eDbTransType eTransType, + FLMUINT uiMaxLockWait, + F_Backup ** ppBackup) +{ + F_Backup * pBackup = NULL; + FLMBOOL bBackupFlagSet = FALSE; + FLMUINT uiLastCPFileNum; + FLMUINT uiLastTransFileNum; + FLMUINT uiDbVersion; + SFLM_DB_HDR * pDbHdr; + RCODE rc = NE_SFLM_OK; + + // Initialize the handle + + *ppBackup = NULL; + + // Make sure we are not being called inside a transaction + + if( getTransType() != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // Verify that the application has specified a valid transaction type. + + if( eTransType != SFLM_READ_TRANS && eTransType != SFLM_UPDATE_TRANS) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + + // Make sure a valid backup type has been specified + + uiDbVersion = getDbVersion(); + + // See if a backup is currently running against the database. If so, + // return an error. Otherwise, set the backup flag on the FFILE. + + m_pDatabase->lockMutex(); + if( m_pDatabase->m_bBackupActive) + { + m_pDatabase->unlockMutex(); + rc = RC_SET( NE_SFLM_BACKUP_ACTIVE); + goto Exit; + } + else + { + bBackupFlagSet = TRUE; + m_pDatabase->m_bBackupActive = TRUE; + } + m_pDatabase->unlockMutex(); + + // Allocate the backup handle + + if( (pBackup = f_new F_Backup) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + pBackup->m_pDb = this; + pBackup->m_uiDbVersion = uiDbVersion; + + // Start a transaction + + if( RC_BAD( rc = beginTrans( eTransType, uiMaxLockWait, + SFLM_DONT_KILL_TRANS | SFLM_DONT_POISON_CACHE, &pBackup->m_dbHdr))) + { + goto Exit; + } + + pBackup->m_bTransStarted = TRUE; + pBackup->m_eTransType = eTransType; + pDbHdr = &pBackup->m_dbHdr; + + // Don't allow an incremental backup to be performed + // if a full backup has not yet been done. + + if( eBackupType == SFLM_INCREMENTAL_BACKUP && + pDbHdr->ui64LastBackupTransID == 0) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + pBackup->m_eBackupType = eBackupType; + + // Set the next incremental backup serial number. This is + // done regardless of the backup type to prevent the wrong + // set of incremental backup files from being applied + // to a database. + + if( RC_BAD( rc = f_createSerialNumber( + pBackup->m_ucNextIncSerialNum))) + { + goto Exit; + } + + // Get the incremental sequence number from the DB header + + pBackup->m_uiIncSeqNum = (FLMUINT)pDbHdr->ui32IncBackupSeqNum; + + // Determine the transaction ID of the last backup + + pBackup->m_ui64LastBackupTransId = pDbHdr->ui64LastBackupTransID; + + // Get the block change count + + pBackup->m_uiBlkChgSinceLastBackup = + (FLMUINT)pDbHdr->ui32BlksChangedSinceBackup; + + // Get the current transaction ID + + pBackup->m_ui64TransId = pBackup->m_pDb->getTransID(); + + // Get the logical end of file + + pBackup->m_uiLogicalEOF = (FLMUINT)pDbHdr->ui32LogicalEOF; + + // Get the first required RFL file needed by the restore. + + uiLastCPFileNum = (FLMUINT)pDbHdr->ui32RflLastCPFileNum; + uiLastTransFileNum = (FLMUINT)pDbHdr->ui32RflCurrFileNum; + + flmAssert( uiLastCPFileNum <= uiLastTransFileNum); + + pBackup->m_uiFirstReqRfl = uiLastCPFileNum < uiLastTransFileNum + ? uiLastCPFileNum + : uiLastTransFileNum; + + flmAssert( pBackup->m_uiFirstReqRfl); + + // Get the database block size + + pBackup->m_uiBlockSize = getBlockSize(); + + // Get the database path + + (void)getDbControlFileName( pBackup->m_szDbPath, + sizeof( pBackup->m_szDbPath)); + + // Done + + *ppBackup = pBackup; + pBackup = NULL; + +Exit: + + if( RC_BAD( rc)) + { + if( pBackup) + { + if( pBackup->m_bTransStarted) + { + abortTrans(); + } + + pBackup->Release(); + } + + if( bBackupFlagSet) + { + m_pDatabase->lockMutex(); + m_pDatabase->m_bBackupActive = FALSE; + m_pDatabase->unlockMutex(); + } + } + + return( rc); +} + +/**************************************************************************** +Desc : Constructor +****************************************************************************/ +F_Backup::F_Backup() +{ + m_pDb = NULL; + m_bTransStarted = FALSE; + reset(); +} + +/**************************************************************************** +Desc : Destructor +****************************************************************************/ +F_Backup::~F_Backup() +{ + endBackup(); +} + +/**************************************************************************** +Desc : Reset member variables to their initial state +****************************************************************************/ +void F_Backup::reset( void) +{ + if( m_bTransStarted) + { + m_pDb->abortTrans(); + m_bTransStarted = FALSE; + } + + m_pDb = NULL; + m_eTransType = SFLM_NO_TRANS; + m_ui64TransId = 0; + m_ui64LastBackupTransId = 0; + m_uiDbVersion = 0; + m_uiBlkChgSinceLastBackup = 0; + m_uiBlockSize = 0; + m_uiLogicalEOF = 0; + m_uiFirstReqRfl = 0; + m_uiIncSeqNum = 0; + m_bCompletedBackup = FALSE; + m_eBackupType = SFLM_FULL_BACKUP; + m_backupRc = NE_SFLM_OK; +} + +/**************************************************************************** +Desc : Streams the contents of a database to the write hook supplied by + the application. +Notes: This routine attempts to create a backup of a database without + excluding any readers or updaters. However, if the backup runs + too long in an environment where extensive updates are happening, + an old view error could be returned. +****************************************************************************/ +RCODE F_Backup::backup( + const char * pszBackupPath, + const char * pszPassword, + IF_BackupClient * ifpClient, + IF_BackupStatus * ifpStatus, + FLMUINT * puiIncSeqNum) +{ + FLMBOOL bFullBackup = TRUE; + FLMINT iFileNum; + FLMUINT uiBlkAddr; + FLMUINT uiTime; + F_CachedBlock * pSCache = NULL; + SFLM_DB_HDR * pDbHdr; + DB_BACKUP_INFO backupInfo; + FLMUINT uiBlockFileOffset; + FLMUINT uiCount; + FLMUINT uiBlockCount; + FLMUINT uiBlockCountLastCB = 0; + FLMUINT uiBytesToPad; + F_BackerStream * pBackerStream = NULL; + FLMBYTE * pucBlkBuf = NULL; + FLMUINT uiBlkBufOffset; + FLMUINT uiBlkBufSize; + FLMUINT uiMaxCSBlocks; + FLMUINT uiCPTransOffset; + FLMUINT uiMaxFileSize; + FLMBOOL bReleaseClient = FALSE; + FLMBOOL bMustUnlock = FALSE; + RCODE rc = NE_SFLM_OK; + + if( puiIncSeqNum) + { + *puiIncSeqNum = 0; + } + + // Setup the status callback info + + f_memset( &backupInfo, 0, sizeof( DB_BACKUP_INFO)); + + // Make sure a backup attempt has not been made with this + // backup handle. + + if( m_bCompletedBackup) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + if( RC_BAD( m_backupRc)) + { + rc = m_backupRc; + goto Exit; + } + + // Look at the backup type + + if( m_eBackupType == SFLM_INCREMENTAL_BACKUP) + { + if( puiIncSeqNum) + { + *puiIncSeqNum = m_uiIncSeqNum; + } + + bFullBackup = FALSE; + } + + // Set up the callback + + if( !ifpClient) + { + if( !pszBackupPath) + { + rc = RC_SET( NE_SFLM_INVALID_PARM); + goto Exit; + } + + ifpClient = f_new F_DefaultBackupClient( pszBackupPath); + + if (ifpClient == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + bReleaseClient = TRUE; + } + + // Allocate and initialize the backer stream object + + if( (pBackerStream = f_new F_BackerStream) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = pBackerStream->setup( FLM_BACKER_MTU_SIZE, ifpClient))) + { + goto Exit; + } + + // Allocate a temporary buffer + + uiBlkBufSize = FLM_BACKER_MTU_SIZE; + uiMaxCSBlocks = uiBlkBufSize / m_uiBlockSize; + if( RC_BAD( rc = f_alloc( uiBlkBufSize, &pucBlkBuf))) + { + goto Exit; + } + + // Setup the backup file header + + uiBlkBufOffset = 0; + f_memset( pucBlkBuf, 0, m_uiBlockSize); + f_memcpy( &pucBlkBuf[ FLM_BACKER_SIGNATURE_OFFSET], + FLM_BACKER_SIGNATURE, FLM_BACKER_SIGNATURE_SIZE); + + UD2FBA( FLM_BACKER_VERSION, + &pucBlkBuf[ FLM_BACKER_VERSION_OFFSET]); + UD2FBA( (FLMUINT32)m_uiBlockSize, + &pucBlkBuf[ FLM_BACKER_DB_BLOCK_SIZE_OFFSET]); + uiMaxFileSize = (FLMUINT)m_dbHdr.ui32MaxFileSize; + UD2FBA( (FLMUINT32)uiMaxFileSize, + &pucBlkBuf[ FLM_BACKER_BFMAX_OFFSET]); + UD2FBA( (FLMUINT32)FLM_BACKER_MTU_SIZE, + &pucBlkBuf[ FLM_BACKER_MTU_OFFSET]); + f_timeGetSeconds( &uiTime); + UD2FBA( (FLMUINT32)uiTime, + &pucBlkBuf[ FLM_BACKER_TIME_OFFSET]); + + uiCount = f_strlen( m_szDbPath); + + if( uiCount <= 3) + { + pucBlkBuf[ FLM_BACKER_DB_NAME_OFFSET] = + (FLMBYTE)m_szDbPath[ uiCount - 6]; + pucBlkBuf[ FLM_BACKER_DB_NAME_OFFSET + 1] = + (FLMBYTE)m_szDbPath[ uiCount - 5]; + pucBlkBuf[ FLM_BACKER_DB_NAME_OFFSET + 2] = + (FLMBYTE)m_szDbPath[ uiCount - 4]; + pucBlkBuf[ FLM_BACKER_DB_NAME_OFFSET + 3] = '\0'; + } + + UD2FBA( (FLMUINT32)m_eBackupType, + &pucBlkBuf[ FLM_BACKER_BACKUP_TYPE_OFFSET]); + + // Set the next incremental serial number in the backup's + // header so that it can be put into the database's log header + // after the backup has been restored. + + f_memcpy( &pucBlkBuf[ FLM_BACKER_NEXT_INC_SERIAL_NUM], + m_ucNextIncSerialNum, SFLM_SERIAL_NUM_SIZE); + + // Set the database version number + + UD2FBA( (FLMUINT32)m_uiDbVersion, + &pucBlkBuf[ FLM_BACKER_DB_VERSION]); + + uiBlkBufOffset += m_uiBlockSize; + + // Copy the database header into the backup's buffer + + f_memset( &pucBlkBuf[ uiBlkBufOffset], 0, m_uiBlockSize); + f_memcpy( &pucBlkBuf[ uiBlkBufOffset], &m_dbHdr, sizeof( SFLM_DB_HDR)); + pDbHdr = (SFLM_DB_HDR *)(&pucBlkBuf[ uiBlkBufOffset]); + uiBlkBufOffset += m_uiBlockSize; + + // Fix up the log header + + if( !pDbHdr->ui8RflKeepFiles) + { + // Put zero in as the last transaction offset so that the current + // RFL file will be created if it does not exist after the database + // is restored. This has basically the same effect as setting the + // offset to 512 if the RFL file has already been created. + + pDbHdr->ui32RflLastTransOffset = 0; + uiCPTransOffset = 512; + + // Create new serial numbers for the RFL. We don't want anyone + // to be able to branch into a "no-keep" RFL sequence. + + if (RC_BAD( rc = f_createSerialNumber( + pDbHdr->ucLastTransRflSerialNum))) + { + goto Exit; + } + + if (RC_BAD( rc = f_createSerialNumber( + pDbHdr->ucNextRflSerialNum))) + { + goto Exit; + } + } + else + { + uiCPTransOffset = (FLMUINT)pDbHdr->ui32RflLastTransOffset; + if( !uiCPTransOffset) + { + uiCPTransOffset = 512; + } + } + + // Set the CP offsets to the last trans offsets. This is done + // because the backup could actually read dirty (committed) blocks + // from the cache, resulting in a backup set that contains blocks + // that are more recent than the ones currently on disk. + + pDbHdr->ui32RflLastCPFileNum = pDbHdr->ui32RflCurrFileNum; + pDbHdr->ui64RflLastCPTransID = pDbHdr->ui64CurrTransID; + pDbHdr->ui32RflLastCPOffset = (FLMUINT32)uiCPTransOffset; + pDbHdr->ui32RblEOF = (FLMUINT32)m_uiBlockSize; + pDbHdr->ui32RblFirstCPBlkAddr = 0; + + // If a password was used, wrap the database key in that password + if (pszPassword && *pszPassword) + { + FLMBYTE * pucTmp = NULL; + + // Need to get a lock on the database - mostly to prevent the very + // unlikely possibility of another thread attempting to use the + // database key at the same time we are. + // (Carson found this in his random testing when one thread did + // a wrapKey while another did a backup.) + + if ((m_pDb->m_uiFlags & FDB_HAS_FILE_LOCK) == 0) + { + if (RC_BAD( rc = m_pDb->dbLock(SFLM_LOCK_EXCLUSIVE, 0, SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bMustUnlock = TRUE; + } + rc = m_pDb->getDatabase()->m_pWrappingKey->getKeyToStore( &pucTmp, + &pDbHdr->ui32DbKeyLen, + (FLMBYTE *)pszPassword, NULL); + if (bMustUnlock) + { + m_pDb->dbUnlock(); + bMustUnlock = FALSE; + } + if (RC_BAD( rc)) + { + if (pucTmp) + { + f_free( &pucTmp); + } + goto Exit; + } + + f_memcpy( pDbHdr->ucDbKey, pucTmp, pDbHdr->ui32DbKeyLen); + f_free( &pucTmp); + } + + // Header should already be in native format. + + flmAssert( !hdrIsNonNativeFormat( pDbHdr)); + + // Calculate and set the CRC + + pDbHdr->ui32HdrCRC = calcDbHdrCRC( pDbHdr); + + // Output the header + + if( RC_BAD( rc = pBackerStream->write( uiBlkBufOffset, pucBlkBuf))) + { + goto Exit; + } + + // There is no way to quickly compute the actual number of bytes + // that will be written to the backup. This is due, in part, to the + // fact that the backup compresses unused space out of blocks before + // storing them. + + backupInfo.ui64BytesToDo = FSGetSizeInBytes( uiMaxFileSize, m_uiLogicalEOF); + uiBlockFileOffset = 0; + uiBlockCount = 0; + iFileNum = 1; + + for( ;;) + { + if( uiBlockFileOffset >= uiMaxFileSize) + { + uiBlockFileOffset = 0; + iFileNum++; + } + + uiBlkAddr = FSBlkAddress( iFileNum, uiBlockFileOffset); + if( !FSAddrIsBelow( uiBlkAddr, m_uiLogicalEOF)) + { + break; + } + + // Get the block + + if( RC_BAD( rc = m_pDb->m_pDatabase->getBlock( m_pDb, NULL, + uiBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + + if( bFullBackup || + pSCache->getBlockPtr()->ui64TransID > m_ui64LastBackupTransId) + { + uiBlkBufOffset = 0; + if ((FLMUINT)pSCache->getBlockPtr()->ui16BlkBytesAvail > + m_uiBlockSize - blkHdrSize( pSCache->getBlockPtr())) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Output the backup header for the block + + UD2FBA( (FLMUINT32)uiBlkAddr, + &pucBlkBuf[ FLM_BACKER_BLK_ADDR_OFFSET]); + + uiBlkBufOffset += FLM_BACKER_BLK_HDR_SIZE; + + // Copy the block into the block buffer and prepare it + // for writing. + + f_memcpy( &pucBlkBuf[ uiBlkBufOffset], + pSCache->getBlockPtr(), m_uiBlockSize); + + // Encrypt the block if needed. + + if (RC_BAD( rc = m_pDb->m_pDatabase->encryptBlock( m_pDb->m_pDict, + &pucBlkBuf[ uiBlkBufOffset]))) + { + goto Exit; + } + + if (RC_BAD( rc = flmPrepareBlockToWrite( m_uiBlockSize, + (F_BLK_HDR *)(&pucBlkBuf [uiBlkBufOffset])))) + { + goto Exit; + } + + uiBlkBufOffset += m_uiBlockSize; + + // Write the block to the backup stream + + if( RC_BAD( rc = pBackerStream->write( + uiBlkBufOffset, pucBlkBuf))) + { + goto Exit; + } + + uiBlockCount++; + } + + ScaReleaseCache( pSCache, FALSE); + pSCache = NULL; + uiBlockFileOffset += m_uiBlockSize; + + // Call the status callback + + if ((uiBlockCount - uiBlockCountLastCB) > 100) + { + if( ifpStatus) + { + backupInfo.ui64BytesDone = FSGetSizeInBytes( uiMaxFileSize, + uiBlkAddr); + if( RC_BAD( rc = ifpStatus->backupStatus( + backupInfo.ui64BytesToDo, + backupInfo.ui64BytesDone))) + { + goto Exit; + } + } + + uiBlockCountLastCB = uiBlockCount; + } + } + + // Output the end-of-backup marker + + f_memset( pucBlkBuf, 0xFF, sizeof( FLM_BACKER_BLK_HDR_SIZE)); + if( RC_BAD( rc = pBackerStream->write( FLM_BACKER_BLK_HDR_SIZE, + pucBlkBuf))) + { + goto Exit; + } + + // Pad the backup so that FlmDbRestore will never read more + // data from the input stream than the backup wrote to it. + + uiBytesToPad = (FLMUINT32)(pBackerStream->getMTUSize() - + (pBackerStream->getByteCount() % pBackerStream->getMTUSize())); + + if( uiBytesToPad < pBackerStream->getMTUSize()) + { + f_memset( pucBlkBuf, 0, uiBytesToPad); + if( RC_BAD( rc = pBackerStream->write( uiBytesToPad, pucBlkBuf))) + { + goto Exit; + } + } + + // Because of the double buffering, we need to have one empty + // buffer at the end of the file. + + f_memset( pucBlkBuf, 0, pBackerStream->getMTUSize()); + if( RC_BAD( rc = pBackerStream->write( pBackerStream->getMTUSize(), + pucBlkBuf))) + { + goto Exit; + } + + if( RC_BAD( rc = pBackerStream->flush())) + { + goto Exit; + } + +Exit: + + if( pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + if( pBackerStream) + { + pBackerStream->Release(); + } + + // Call the status callback now that the background + // thread has terminated. + + if( RC_OK( rc) && ifpStatus) + { + backupInfo.ui64BytesDone = backupInfo.ui64BytesToDo; + (void)ifpStatus->backupStatus( backupInfo.ui64BytesToDo, + backupInfo.ui64BytesDone); + } + + if( pucBlkBuf) + { + f_free( &pucBlkBuf); + } + + if( RC_OK( rc)) + { + m_bCompletedBackup = TRUE; + } + + if ( bReleaseClient) + { + ifpClient->Release(); + } + + m_backupRc = rc; + return( rc); +} + +/**************************************************************************** +Area : MISC +Desc : Ends the backup, updating the log header if needed. +****************************************************************************/ +RCODE F_Backup::endBackup( void) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + + if( !m_bCompletedBackup) + { + goto Exit; + } + + // End the transaction + + flmAssert( m_eTransType != SFLM_NO_TRANS); + if( RC_BAD( rc = m_pDb->abortTrans())) + { + goto Exit; + } + m_eTransType = SFLM_NO_TRANS; + m_bTransStarted = FALSE; + + // Start an update transaction. + + if( RC_BAD( rc = m_pDb->beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Update log header fields + + m_pDb->m_pDatabase->m_uncommittedDbHdr.ui64LastBackupTransID = + m_ui64TransId; + + // Since there may have been transactions during the backup, + // we need to take into account the number of blocks that have + // changed during the backup when updating the + // ui32BlksChangedSinceBackup statistic. + + m_pDb->m_pDatabase->m_uncommittedDbHdr.ui32BlksChangedSinceBackup -= + (FLMUINT32)m_uiBlkChgSinceLastBackup; + + // Bump the incremental backup sequence number + + if( m_eBackupType == SFLM_INCREMENTAL_BACKUP) + { + m_pDb->m_pDatabase->m_uncommittedDbHdr.ui32IncBackupSeqNum++; + } + + // Always change the incremental backup serial number. This is + // needed so that if the user performs a full backup, runs some + // transactions against the database, performs another full backup, + // and then performs an incremental backup we will know that the + // incremental backup cannot be restored against the first full + // backup. + + f_memcpy( + m_pDb->m_pDatabase->m_uncommittedDbHdr.ucIncBackupSerialNum, + m_ucNextIncSerialNum, SFLM_SERIAL_NUM_SIZE); + + // Commit the transaction and perform a checkpoint so that the + // modified log header values will be written. + + bStartedTrans = FALSE; + if( RC_BAD( m_pDb->commitTrans( 0, TRUE))) + { + goto Exit; + } + +Exit: + + // Abort the active transaction (if any) + + if( bStartedTrans) + { + m_pDb->abortTrans(); + } + + // Unset the backup flag + + if( m_pDb) + { + m_pDb->m_pDatabase->lockMutex(); + m_pDb->m_pDatabase->m_bBackupActive = FALSE; + m_pDb->m_pDatabase->unlockMutex(); + } + + // Clear the object + + reset(); + + // Done. + + return( rc); +} + +/**************************************************************************** +Desc: Restores a database from backup +Notes: This routine does not restore referenced BLOBs. +****************************************************************************/ +RCODE F_DbSystem::dbRestore( + const char * pszDbPath, + // [IN] Path of database that is being restored. This is the + // same path format that FlmDbCreate expects + // (i.e., c:\flaim\flm.db). + const char * pszDataDir, + // [IN] Directory where data files are located. + const char * pszRflDir, + // [IN] RFL log file directory. NULL can be passed to indicate + // that the files are located in the same directory as the + // database (specified above). + const char * pszBackupPath, + // [IN] Directory and name of the backup file set. + // This parameter is required only if the default + // BACKER_READ_HOOK is used. Otherwise, NULL can be + // passed as the value of this parameter. + const char * pszPassword, + // [IN] Password that was used durning the backup + IF_RestoreClient * pRestoreObj, + // [IN] Object to be used to read data from the backup set. + IF_RestoreStatus * pRestoreStatus) + // [IN] Object for reporting the status of the restore + // operation +{ + IF_FileHdl * pFileHdl = NULL; + IF_FileHdl * pLockFileHdl = NULL; + F_SuperFileHdl * pSFile = NULL; + FLMBYTE szBasePath[ F_PATH_MAX_SIZE]; + char szTmpPath[ F_PATH_MAX_SIZE]; + FLMUINT uiDbVersion; + FLMUINT uiNextIncNum; + eRestoreAction eAction = SFLM_RESTORE_ACTION_CONTINUE; // default action... + FLMBOOL bRflPreserved; + FLMBOOL bMutexLocked = FALSE; + F_Db * pDb = NULL; + F_Database * pDatabase = NULL; + F_FSRestore * pFSRestoreObj = NULL; + FLMBOOL bOKToRetry; + RCODE rc = NE_SFLM_OK; + + // Set up the callback + + if( !pRestoreObj) + { + if( !pszBackupPath || *pszBackupPath == 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + if( (pFSRestoreObj = f_new F_FSRestore) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = pFSRestoreObj->setup( pszDbPath, + pszBackupPath, pszRflDir))) + { + goto Exit; + } + + // Note: If we wanted to be absolutely correct, we'd do an AddRef on + // pFSRestoreObj because there's going to be two pointers pointing at + // it. It really doesn't matter in this case, though because + // pFSRestoreObj is local to this function and will get deleted before + // the function exits. + + pRestoreObj = (IF_RestoreClient *)pFSRestoreObj; + } + + // Get the base path + + getDbBasePath( (char *)szBasePath, pszDbPath, NULL); + + // Lock the global mutex + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + + // Look up the file using findDatabase to see if the file is already open. + // May unlock and re-lock the global mutex.. + + if( RC_BAD( rc = findDatabase( pszDbPath, pszDataDir, &pDatabase))) + { + goto Exit; + } + + // If the database is open, we cannot perform a restore + + if( pDatabase) + { + rc = RC_SET( NE_SFLM_DATABASE_OPEN); + pDatabase = NULL; + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + goto Exit; + } + + // Allocate the F_Database object. This will prevent other threads from + // opening the database while the restore is being performed. + + if( RC_BAD( rc = allocDatabase( pszDbPath, pszDataDir, FALSE, &pDatabase))) + { + goto Exit; + } + + // Unlock the global mutex + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + // Create a lock file. If this fails, it could indicate + // that the destination database exists and is in use by another + // process. + + f_sprintf( szTmpPath, "%s.lck", szBasePath); + if( RC_BAD( rc = flmCreateLckFile( szTmpPath, &pLockFileHdl))) + { + goto Exit; + } + + // Create the control file and set up the super file object + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->createFile( + pszDbPath, FLM_IO_RDWR, &pFileHdl))) + { + goto Exit; + } + + // Allocate a super file object + // NOTE: Do not use extended cache for this super-file object. + + if( (pSFile = f_new F_SuperFileHdl) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = pSFile->setup( pszDbPath, pszDataDir))) + { + goto Exit; + } + + // Open the backup set + + if( RC_BAD( rc = pRestoreObj->openBackupSet())) + { + goto Exit; + } + + // Restore the data in the backup set + + if( RC_BAD( rc = flmRestoreFile( pRestoreObj, pRestoreStatus, + pSFile, FALSE, &uiDbVersion, &uiNextIncNum, &bRflPreserved, + &eAction, NULL))) + { + goto Exit; + } + + // See if we should continue + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + goto Exit; + } + + // Close the backup set + + if( RC_BAD( rc = pRestoreObj->close())) + { + goto Exit; + } + + // Apply any available incremental backups. uiNextIncNum will be 0 if + // the database version does not support incremental backups. + + if( uiNextIncNum) + { + FLMUINT uiCurrentIncNum; + + for( ;;) + { + uiCurrentIncNum = uiNextIncNum; + if( RC_BAD( rc = pRestoreObj->openIncFile( uiCurrentIncNum))) + { + if( rc == NE_FLM_IO_PATH_NOT_FOUND) + { + rc = NE_SFLM_OK; + break; + } + else + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = flmRestoreFile( pRestoreObj, pRestoreStatus, + pSFile, TRUE, &uiDbVersion, &uiNextIncNum, &bRflPreserved, + &eAction, &bOKToRetry))) + { + RCODE tmpRc; + + if( !bOKToRetry) + { + // Cannot retry the operation or continue ... the + // database is in an unknown state. + + goto Exit; + } + + if( pRestoreStatus) + { + if( RC_BAD( tmpRc = + pRestoreStatus->reportError( &eAction, rc))) + { + rc = tmpRc; + goto Exit; + } + } + + if( eAction == SFLM_RESTORE_ACTION_RETRY || + eAction == SFLM_RESTORE_ACTION_CONTINUE) + { + // Abort the current file (if any) + + if( RC_BAD( rc = pRestoreObj->abortFile())) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_CONTINUE) + { + // Break out and begin processing the RFL + + break; + } + + // Otherwise, retry opening the incremental file + + uiNextIncNum = uiCurrentIncNum; + continue; + } + goto Exit; + } + + // See if we should continue + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + goto Exit; + } + + // Close the current file + + if( RC_BAD( rc = pRestoreObj->close())) + { + goto Exit; + } + } + } + } + + // Force everything out to disk + + if( RC_BAD( rc = pSFile->flush())) + { + goto Exit; + } + + pSFile->Release(); + pSFile = NULL; + + // Don't do anything with the RFL if the preserve flag + // isn't set. + + if( !bRflPreserved) + { + pRestoreObj = NULL; + pRestoreStatus = NULL; + } + + // Open the file and apply any available RFL files. The + // lock file handle is passed to the openDatabase call so + // that we don't have to give up our lock until the + // restore is complete. Also, we don't want to resume + // any indexing at this point. By not resuming the indexes, + // we can perform a DB diff of two restored databases that + // should be identical without having differences in the + // tracker container due to background indexing. + + rc = openDatabase( pDatabase, + pszDbPath, pszDataDir, + pszRflDir, pszPassword, SFLM_DONT_RESUME_THREADS, + TRUE, pRestoreObj, pRestoreStatus, pLockFileHdl, &pDb); + pLockFileHdl = NULL; + + if( RC_BAD( rc)) + { + pDatabase = NULL; + goto Exit; + } + + // If a password was needed to open the database, we need to clear it so it + //can be opened without a password. + + if (pszPassword && pszPassword[0] != 0) + { + if (RC_BAD( rc = pDb->wrapKey())) + { + goto Exit; + } + } + + // Close the database + + pDb->Release(); + pDb = NULL; + +Exit: + + if( pSFile) + { + // Need to release the super file handle before cleaning up the + // FFILE because the super file still has a reference to the + // FFILE's file ID list. + + pSFile->Release(); + } + + if( pDatabase) + { + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + } + + if (RC_BAD( rc)) + { + pDatabase->newDatabaseFinish( rc); + } + + if( !pDatabase->m_uiOpenIFDbCount) + { + pDatabase->freeDatabase(); + } + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + } + + if( bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + if( pDb) + { + pDb->Release(); + } + + if( pFileHdl) + { + pFileHdl->Release(); + } + + if( pLockFileHdl) + { + pLockFileHdl->Release(); + } + + if( pFSRestoreObj) + { + pFSRestoreObj->Release(); + } + + // If restore failed, remove all database files (excluding RFL files) + + if( RC_BAD( rc)) + { + dbRemove( pszDbPath, pszDataDir, NULL, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc : Restores a full or incremental backup +*END************************************************************************/ +FSTATIC RCODE flmRestoreFile( + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus, + F_SuperFileHdl * pSFile, + FLMBOOL bIncremental, + FLMUINT * puiDbVersion, + FLMUINT * puiNextIncSeqNum, + FLMBOOL * pbRflPreserved, + eRestoreAction * peAction, + FLMBOOL * pbOKToRetry) +{ + FLMUINT uiBytesWritten; + FLMUINT uiLogicalEOF; + FLMUINT uiBlkAddr; + FLMUINT uiBlockCount = 0; + FLMUINT uiBlockSize; + FLMUINT uiDbVersion; + FLMUINT uiMaxFileSize; + FLMUINT uiBackupMaxFileSize; + FLMUINT uiPriorBlkFile = 0; + FLMUINT uiSectorSize; + SFLM_DB_HDR * pDbHdr; + FLMBYTE ucIncSerialNum[ SFLM_SERIAL_NUM_SIZE]; + FLMBYTE ucNextIncSerialNum[ SFLM_SERIAL_NUM_SIZE]; + FLMUINT uiIncSeqNum; + FLMBYTE * pucBlkBuf = NULL; + char szPath[ F_PATH_MAX_SIZE]; + FLMUINT uiBlkBufSize; + FLMUINT uiPriorBlkAddr = 0; + FLMUINT64 ui64BytesToDo = 0; + FLMUINT64 ui64BytesDone = 0; + eDbBackupType eBackupType; + F_BackerStream * pBackerStream = NULL; + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32CRC; + F_BLK_HDR * pBlkHdr; + + // Initialize the "ok-to-retry" flag + + if( pbOKToRetry) + { + *pbOKToRetry = TRUE; + } + +#ifdef FLM_WIN + + // Don't want to do extra file extensions or flush when file + // is extended. Setting the extend size to ~0 has the effect + // of not updating the directory entry (a time-consuming operation) + // on Windows platforms. On all other platforms, we will just + // go with the default behavior. + + pSFile->setExtendSize( (FLMUINT)(~0)); + +#endif + + + // Set up the backer stream object + + if( (pBackerStream = f_new F_BackerStream) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = pBackerStream->setup( FLM_BACKER_MTU_SIZE, pRestoreObj))) + { + goto Exit; + } + + // Get the path of the .DB file (file 0). + + if( RC_BAD( rc = pSFile->getFilePath( 0, szPath))) + { + goto Exit; + } + + // Get the sector size + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->getSectorSize( + szPath, &uiSectorSize))) + { + goto Exit; + } + + // Allocate a temporary buffer. Try to align the buffer on a sector + // boundary to avoid memcpy operatons in the file system. + + uiBlkBufSize = FLM_BACKER_MTU_SIZE; + if( uiSectorSize) + { + uiBlkBufSize = (((uiBlkBufSize / uiSectorSize) + 1) * uiSectorSize); + } + + if( RC_BAD( rc = f_allocAlignedBuffer( uiBlkBufSize, (void **)&pucBlkBuf))) + { + goto Exit; + } + + // Read and verify the backup header + + if( RC_BAD( rc = pBackerStream->read( FLM_BACKER_MIN_DB_BLOCK_SIZE, + pucBlkBuf))) + { + goto Exit; + } + + if( FB2UD( &pucBlkBuf[ FLM_BACKER_VERSION_OFFSET]) != FLM_BACKER_VERSION) + { + rc = RC_SET( NE_SFLM_UNSUPPORTED_VERSION); + goto Exit; + } + + if( f_strncmp( (const char *)&pucBlkBuf[ FLM_BACKER_SIGNATURE_OFFSET], + FLM_BACKER_SIGNATURE, FLM_BACKER_SIGNATURE_SIZE) != 0) + { + rc = RC_SET( NE_SFLM_UNSUPPORTED_VERSION); + goto Exit; + } + + uiBlockSize = (FLMUINT)FB2UW( &pucBlkBuf[ FLM_BACKER_DB_BLOCK_SIZE_OFFSET]); + if( uiBlockSize > FLM_BACKER_MAX_DB_BLOCK_SIZE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Get the maximum file size from the backup header. + + uiBackupMaxFileSize = (FLMUINT)FB2UD( &pucBlkBuf[ FLM_BACKER_BFMAX_OFFSET]); + + // Make sure the MTU is correct + + if( FB2UD( &pucBlkBuf[ FLM_BACKER_MTU_OFFSET]) != FLM_BACKER_MTU_SIZE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Make sure the backup type is correct + + eBackupType = (eDbBackupType)FB2UD( + &pucBlkBuf[ FLM_BACKER_BACKUP_TYPE_OFFSET]); + + if( (eBackupType == SFLM_INCREMENTAL_BACKUP && !bIncremental) || + (eBackupType == SFLM_FULL_BACKUP && bIncremental)) + { + // Do not allow an incremental backup to be restored directly. The + // only way to restore an incremental backup is to provide the + // incremental files when requested by FlmDbRestore. Also, we don't + // want to allow the user to mistakenly hand us a full backup when + // we are expecting an incremental backup. + + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Grab the "next" incremental backup serial number + + f_memcpy( ucNextIncSerialNum, + &pucBlkBuf[ FLM_BACKER_NEXT_INC_SERIAL_NUM], + SFLM_SERIAL_NUM_SIZE); + + // Get the database version from the backup header + + uiDbVersion = FB2UD( &pucBlkBuf[ FLM_BACKER_DB_VERSION]); + if( puiDbVersion) + { + *puiDbVersion = uiDbVersion; + } + + // Seek to the database header block + + if( uiBlockSize > FLM_BACKER_MIN_DB_BLOCK_SIZE) + { + if( RC_BAD( rc = pBackerStream->read( + uiBlockSize - FLM_BACKER_MIN_DB_BLOCK_SIZE, pucBlkBuf))) + { + goto Exit; + } + } + + // Read the database header block from the backup + + if( RC_BAD( rc = pBackerStream->read( uiBlockSize, pucBlkBuf))) + { + goto Exit; + } + + // Sanity check - make sure the block size in the backup header + // is the same as the size in the database header + + pDbHdr = (SFLM_DB_HDR *)pucBlkBuf; + + // Calculate the CRC before doing any conversions. + + ui32CRC = calcDbHdrCRC( pDbHdr); + + // Convert to native platform format, if necessary. + + if (hdrIsNonNativeFormat( pDbHdr)) + { + convertDbHdr( pDbHdr); + } + + // Validate the checksum + + if (ui32CRC != pDbHdr->ui32HdrCRC) + { + rc = RC_SET( NE_SFLM_HDR_CRC); + goto Exit; + } + + if( uiBlockSize != (FLMUINT)pDbHdr->ui16BlockSize) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Compare the database version in the DB header with + // the one extracted from the backup header + + if( (FLMUINT)pDbHdr->ui32DbVersion != uiDbVersion) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + uiMaxFileSize = (FLMUINT)pDbHdr->ui32MaxFileSize; + + // Set the database version number and block size into the + // super file handle. We only do this if the file being restored + // is the full backup. It will always be first in the restore sequence, + // and thus we only need to set these values into the super file handle + // at that time. + + if( !bIncremental) + { + pSFile->setBlockSize( uiBlockSize); + } + + // Make sure the maximum block file size matches what was read from the + // backup header. + + if( uiBackupMaxFileSize != uiMaxFileSize) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Get the logical EOF from the log header + + uiLogicalEOF = (FLMUINT)pDbHdr->ui32LogicalEOF; + + // Are RFL files being preserved? + + if( pbRflPreserved) + { + *pbRflPreserved = pDbHdr->ui8RflKeepFiles + ? TRUE + : FALSE; + } + + // Get the incremental backup sequence number + + uiIncSeqNum = (FLMUINT)pDbHdr->ui32IncBackupSeqNum; + *puiNextIncSeqNum = uiIncSeqNum; + + if( bIncremental) + { + (*puiNextIncSeqNum)++; + } + + // Get information about the incremental backup + + if( bIncremental) + { + FLMUINT uiTmp; + SFLM_DB_HDR dbHdr; + + f_memcpy( ucIncSerialNum, pDbHdr->ucIncBackupSerialNum, + SFLM_SERIAL_NUM_SIZE); + + // Compare the incremental backup sequence number to the value in the + // database's DB header. + + if( RC_BAD( rc = pSFile->readHeader( 0, sizeof( SFLM_DB_HDR), + &dbHdr, &uiTmp))) + { + goto Exit; + } + if (hdrIsNonNativeFormat( &dbHdr)) + { + convertDbHdr( &dbHdr); + } + + if( (FLMUINT)dbHdr.ui32IncBackupSeqNum != uiIncSeqNum) + { + rc = RC_SET( NE_SFLM_INVALID_FILE_SEQUENCE); + goto Exit; + } + + // Compare the incremental backup serial number to the value in the + // database's log header. + + if( f_memcmp( ucIncSerialNum, dbHdr.ucIncBackupSerialNum, + SFLM_SERIAL_NUM_SIZE) != 0) + { + rc = RC_SET( NE_SFLM_SERIAL_NUM_MISMATCH); + goto Exit; + } + + // Increment the incremental backup sequence number + + pDbHdr->ui32IncBackupSeqNum = (FLMUINT32)(uiIncSeqNum + 1); + } + + // At the start of a backup, either incremental or full, + // we generate a new incremental serial number. This is needed so + // that if the user performs a full backup, runs some transactions + // against the database, performs another full backup, and then + // performs an incremental backup we will know that the incremental + // backup cannot be restored against the first full backup. + + // Since the new serial number is not written to the database's log + // header until after the backup completes, we need to put the + // new serial number in the log header during the restore. In doing + // so, the log header will contain the correct serial number for a + // subsequent incremental backup that may have been made. + + f_memcpy( pDbHdr->ucIncBackupSerialNum, + ucNextIncSerialNum, SFLM_SERIAL_NUM_SIZE); + + // DB Header is in native format. Set the CRC + // before writing it out. + + pDbHdr->ui32HdrCRC = calcDbHdrCRC( pDbHdr); + pDbHdr = NULL; + + // Set the "ok-to-retry" flag + + if( pbOKToRetry) + { + *pbOKToRetry = FALSE; + } + + // Write the database header + + if( RC_BAD( rc = pSFile->writeHeader( 0, + uiBlockSize, pucBlkBuf, &uiBytesWritten))) + { + goto Exit; + } + + // The status callback will give a general idea of how much work + // is left to do. We don't have any way to get the total size + // of the stream to give a correct count, so a close estimate + // will have to suffice. + ui64BytesToDo = FSGetSizeInBytes( uiMaxFileSize, uiLogicalEOF); + + // Write the blocks in the backup file to the database + + for (;;) + { + if( RC_BAD( rc = pBackerStream->read( FLM_BACKER_BLK_HDR_SIZE, + pucBlkBuf))) + { + goto Exit; + } + + uiBlockCount++; + uiBlkAddr = FB2UD( &pucBlkBuf[ FLM_BACKER_BLK_ADDR_OFFSET]); + + // Are we done? + + if( uiBlkAddr == 0xFFFFFFFF) + { + break; + } + + if( !uiBlkAddr || + !FSAddrIsBelow( uiBlkAddr, uiLogicalEOF) || + (uiPriorBlkAddr && !FSAddrIsBelow( uiPriorBlkAddr, uiBlkAddr))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Read and process the block + + if( RC_BAD( rc = pBackerStream->read( uiBlockSize, pucBlkBuf))) + { + goto Exit; + } + + pBlkHdr = (F_BLK_HDR *)pucBlkBuf; + + // Convert the entire block to native format if necessary. + + if (RC_BAD( rc = flmPrepareBlockForUse( uiBlockSize, pBlkHdr))) + { + if (rc == NE_SFLM_BLOCK_CRC) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + } + goto Exit; + } + if( (FLMUINT)pBlkHdr->ui32BlkAddr != uiBlkAddr) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + // Prepare the block for writing. + + if (RC_BAD( rc = flmPrepareBlockToWrite( uiBlockSize, pBlkHdr))) + { + goto Exit; + } + + // Write the block to the database + +#ifdef FLM_UNIX + + // Unix systems can have sector sizes that are larger than our + // typical 4K database blocks. The Unix implementation of SectorWrite + // (called by WriteBlock) will write the passed-in block and clobber any + // additional data beyond the end of the block to the end of the sector if + // it has enough room in the block buffer to write a full sector. If the + // block buffer is less than a full sector, the Unix SectorWrite will only + // write out the amount requested, not a full sector. + + if( RC_BAD( rc = pSFile->writeBlock( uiBlkAddr, + uiBlockSize, pucBlkBuf, uiBlockSize, + NULL, &uiBytesWritten))) +#else + if( RC_BAD( rc = pSFile->writeBlock( uiBlkAddr, + uiBlockSize, pucBlkBuf, uiBlkBufSize, + NULL, &uiBytesWritten))) +#endif + { + if( rc == NE_FLM_IO_PATH_NOT_FOUND || + rc == NE_FLM_IO_INVALID_FILENAME) + { + // Create a new block file + + if( FSGetFileNumber( uiBlkAddr) != (uiPriorBlkFile + 1)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INCONSISTENT_BACKUP); + goto Exit; + } + + if( RC_BAD( rc = pSFile->createFile( FSGetFileNumber( uiBlkAddr)))) + { + goto Exit; + } + +#ifdef FLM_UNIX + if( RC_BAD( rc = pSFile->writeBlock( uiBlkAddr, + uiBlockSize, pucBlkBuf, uiBlockSize, + NULL, &uiBytesWritten))) +#else + if( RC_BAD( rc = pSFile->writeBlock( uiBlkAddr, + uiBlockSize, pucBlkBuf, uiBlkBufSize, + NULL, &uiBytesWritten))) +#endif + { + goto Exit; + } + } + else + { + goto Exit; + } + } + + uiPriorBlkAddr = uiBlkAddr; + uiPriorBlkFile = FSGetFileNumber( uiBlkAddr); + + if( pRestoreStatus && (uiBlockCount & 0x7F) == 0x7F) + { + ui64BytesDone = FSGetSizeInBytes( uiMaxFileSize, uiBlkAddr); + if( RC_BAD( rc = pRestoreStatus->reportProgress( peAction, + ui64BytesToDo, + ui64BytesDone))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + } + + if( pRestoreStatus) + { + // Call the status callback one last time. + + ui64BytesDone = ui64BytesToDo; + if( RC_BAD( rc = pRestoreStatus->reportProgress( peAction, ui64BytesToDo, + ui64BytesDone))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + // It is safe to jump to exit at this point + + goto Exit; + } + } + +Exit: + + if( pucBlkBuf) + { + f_freeAlignedBuffer( (void **)&pucBlkBuf); + } + + if( pBackerStream) + { + pBackerStream->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Constructor +****************************************************************************/ +F_DefaultBackupClient::F_DefaultBackupClient( + const char * pszBackupPath) +{ + m_pMultiFileHdl = NULL; + m_ui64Offset = 0; + m_rc = NE_SFLM_OK; + + f_strncpy( m_szPath, pszBackupPath, F_PATH_MAX_SIZE - 1); +} + +/**************************************************************************** +Desc: Destructor +****************************************************************************/ +F_DefaultBackupClient::~F_DefaultBackupClient() +{ + if (m_pMultiFileHdl) + { + m_pMultiFileHdl->close(); + m_pMultiFileHdl->Release(); + } +} + + +/**************************************************************************** +Desc: Default hook for creating a backup file set +****************************************************************************/ +RCODE F_DefaultBackupClient::WriteData( + const void * pvBuffer, + FLMUINT uiBytesToWrite) +{ + FLMUINT uiBytesWritten; + RCODE rc = m_rc; + + if( RC_BAD( rc)) + { + goto Exit; + } + + if( m_pMultiFileHdl == 0) + { + // Remove any existing backup files + + if( RC_BAD( rc = FlmAllocMultiFileHdl( &m_pMultiFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pMultiFileHdl->deleteMultiFile( m_szPath)) && + rc != NE_FLM_IO_PATH_NOT_FOUND && + rc != NE_FLM_IO_INVALID_FILENAME) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + goto Exit; + } + + if( RC_BAD( rc = m_pMultiFileHdl->create( m_szPath))) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + goto Exit; + } + } + + rc = m_pMultiFileHdl->write( m_ui64Offset, + uiBytesToWrite, (FLMBYTE *)pvBuffer, &uiBytesWritten); + m_ui64Offset += uiBytesWritten; + +Exit: + + if( RC_BAD( rc)) + { + m_rc = rc; + if( m_pMultiFileHdl) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + } + } + + return( rc); +} + +// F_BackerStream methods + +/**************************************************************************** +Desc: Constructor +****************************************************************************/ +F_BackerStream::F_BackerStream( void) +{ + m_bSetup = FALSE; + m_bFirstRead = TRUE; + m_bFinalRead = FALSE; + m_ui64ByteCount = 0; + m_uiBufOffset = 0; + m_pRestoreObj = NULL; + m_hDataSem = F_SEM_NULL; + m_hIdleSem = F_SEM_NULL; + m_pThread = NULL; + m_rc = NE_SFLM_OK; + m_pucInBuf = NULL; + m_puiInOffset = NULL; + m_pucOutBuf = NULL; + m_puiOutOffset = NULL; + m_pucBufs[ 0] = NULL; + m_pucBufs[ 1] = NULL; + m_uiOffsets[ 0] = 0; + m_uiOffsets[ 1] = 0; + m_uiMTUSize = 0; + m_pClient = NULL; +} + +/**************************************************************************** +Desc: Destructor +****************************************************************************/ +F_BackerStream::~F_BackerStream( void) +{ + shutdownThreads(); + + if( m_hDataSem != F_SEM_NULL) + { + f_semDestroy( &m_hDataSem); + } + + if( m_hIdleSem != F_SEM_NULL) + { + f_semDestroy( &m_hIdleSem); + } + + if( m_pucBufs[ 0]) + { + f_free( &m_pucBufs[ 0]); + } + + if( m_pucBufs[ 1]) + { + f_free( &m_pucBufs[ 1]); + } +} + + +/**************************************************************************** +Desc: Start any background threads +****************************************************************************/ +RCODE F_BackerStream::startThreads( void) +{ + RCODE rc = NE_SFLM_OK; + + if( m_pThread) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + // The semaphore handles better be null + + flmAssert( m_hDataSem == F_SEM_NULL); + flmAssert( m_hIdleSem == F_SEM_NULL); + + // Create a semaphore to signal the background thread + // that data is available + + if( RC_BAD( rc = f_semCreate( &m_hDataSem))) + { + goto Exit; + } + + // Create a semaphore to signal when the background thread + // is idle + + if( RC_BAD( rc = f_semCreate( &m_hIdleSem))) + { + goto Exit; + } + + // Start the thread + + if( m_pClient) + { + if( RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( &m_pThread, + F_BackerStream::writeThread, "backup", + 0, 0, (void *)this))) + { + goto Exit; + } + } + else if( m_pRestoreObj) + { + if( RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( &m_pThread, + F_BackerStream::readThread, "restore", + 0, 0, (void *)this))) + { + goto Exit; + } + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Shut down any background threads +****************************************************************************/ +void F_BackerStream::shutdownThreads( void) +{ + if( m_pThread) + { + // Shut down the background read or write thread. + + m_pThread->setShutdownFlag(); + f_semSignal( m_hDataSem); + m_pThread->stopThread(); + m_pThread->Release(); + m_pThread = NULL; + + // Now that the thread has terminated, it is safe + // to destroy the data and idle semaphores. + + f_semDestroy( &m_hDataSem); + f_semDestroy( &m_hIdleSem); + } +} + +/**************************************************************************** +Desc: Setup method to use the backer stream as an input stream +****************************************************************************/ +RCODE F_BackerStream::setup( + FLMUINT uiMTUSize, + IF_RestoreClient * pRestoreObj) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( pRestoreObj); + flmAssert( !m_bSetup); + + m_pRestoreObj = pRestoreObj; + m_uiMTUSize = uiMTUSize; + + if( RC_BAD( rc = _setup())) + { + goto Exit; + } + + // Fire up the background threads + + if( RC_BAD( rc = startThreads())) + { + goto Exit; + } + + m_bSetup = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup method to use the backer stream as an output stream +****************************************************************************/ +RCODE F_BackerStream::setup( + FLMUINT uiMTUSize, + IF_BackupClient * pClient) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( pClient); + flmAssert( !m_bSetup); + + m_pClient = pClient; + m_uiMTUSize = uiMTUSize; + + if( RC_BAD( rc = _setup())) + { + goto Exit; + } + + // Fire up the background threads + + if( RC_BAD( rc = startThreads())) + { + goto Exit; + } + + m_bSetup = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Performs setup operations common to read and write streams +****************************************************************************/ +RCODE F_BackerStream::_setup( void) +{ + RCODE rc = NE_SFLM_OK; + + // Allocate a buffer for reading or writing blocks + + if( (m_uiMTUSize < (2 * FLM_BACKER_MAX_DB_BLOCK_SIZE)) || + m_uiMTUSize % FLM_BACKER_MAX_DB_BLOCK_SIZE) + { + rc = RC_SET( NE_SFLM_INVALID_PARM); + goto Exit; + } + + // Allocate buffers for reading or writing + + if( RC_BAD( rc = f_alloc( m_uiMTUSize, &m_pucBufs[ 0]))) + { + goto Exit; + } + + if( RC_BAD( rc = f_alloc( m_uiMTUSize, &m_pucBufs[ 1]))) + { + goto Exit; + } + + m_pucInBuf = m_pucBufs[ 0]; + m_puiInOffset = &m_uiOffsets[ 0]; + + m_pucOutBuf = m_pucBufs[ 1]; + m_puiOutOffset = &m_uiOffsets[ 1]; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Reads data from the input stream +****************************************************************************/ +RCODE F_BackerStream::read( + FLMUINT uiLength, + FLMBYTE * pucData, + FLMUINT * puiBytesRead) +{ + FLMUINT uiBufSize; + FLMBYTE * pucBuf; + FLMUINT uiBytesRead = 0; + FLMUINT uiBytesAvail; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetup); + flmAssert( m_pRestoreObj); + flmAssert( uiLength); + + if( m_bFirstRead) + { + m_bFirstRead = FALSE; + + // Prime the pump. Call signalThread twice ... once to + // get the first chunk of data and a second time to have + // the background thread pre-fetch the next chunk. A backup + // will always have at least two MTU data chunks, so we should + // never get an IO_END_OF_FILE error. If we do, the restore + // operation needs to abort (which will happen because the + // error will be returned to the caller of this routine). + + if( RC_BAD( rc = signalThread()) || + RC_BAD( rc = signalThread())) + { + goto Exit; + } + } + + while( uiLength) + { + uiBufSize = *m_puiOutOffset; + pucBuf = m_pucOutBuf; + + uiBytesAvail = uiBufSize - m_uiBufOffset; + flmAssert( uiBytesAvail); + + if( uiBytesAvail < uiLength) + { + f_memcpy( &pucData[ uiBytesRead], + &pucBuf[ m_uiBufOffset], uiBytesAvail); + m_uiBufOffset += uiBytesAvail; + uiBytesRead += uiBytesAvail; + uiLength -= uiBytesAvail; + } + else + { + f_memcpy( &pucData[ uiBytesRead], + &pucBuf[ m_uiBufOffset], uiLength); + m_uiBufOffset += uiLength; + uiBytesRead += uiLength; + uiLength = 0; + } + + if( m_uiBufOffset == uiBufSize) + { + m_uiBufOffset = 0; + if( RC_BAD( rc = signalThread())) + { + // Since we are reading MTU-sized units and the restore + // code knows when to stop reading, we should never + // get an IO_END_OF_FILE error back from a call to + // signalThread(). If we do, we need to return the + // error to the caller (FlmDbRestore). + goto Exit; + } + } + } + +Exit: + + if( puiBytesRead) + { + *puiBytesRead = uiBytesRead; + } + m_ui64ByteCount += (FLMUINT64)uiBytesRead; + return( rc); +} + +/**************************************************************************** +Desc: Writes data to the output stream +****************************************************************************/ +RCODE F_BackerStream::write( + FLMUINT uiLength, + FLMBYTE * pucData, + FLMUINT * puiBytesWritten) +{ + FLMUINT uiMaxWriteSize; + FLMUINT uiBytesWritten = 0; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetup); + flmAssert( m_pClient); + flmAssert( uiLength); + + while( uiLength) + { + uiMaxWriteSize = m_uiMTUSize - *m_puiInOffset; + flmAssert( uiMaxWriteSize); + + if( uiMaxWriteSize < uiLength) + { + f_memcpy( &m_pucInBuf[ *m_puiInOffset], + &pucData[ uiBytesWritten], uiMaxWriteSize); + (*m_puiInOffset) += uiMaxWriteSize; + uiBytesWritten += uiMaxWriteSize; + uiLength -= uiMaxWriteSize; + } + else + { + f_memcpy( &m_pucInBuf[ *m_puiInOffset], + &pucData[ uiBytesWritten], uiLength); + (*m_puiInOffset) += uiLength; + uiBytesWritten += uiLength; + uiLength = 0; + } + + if( (*m_puiInOffset) == m_uiMTUSize) + { + if( RC_BAD( rc = signalThread())) + { + goto Exit; + } + } + } + +Exit: + + if( puiBytesWritten) + { + *puiBytesWritten = uiBytesWritten; + } + + m_ui64ByteCount += (FLMUINT64)uiBytesWritten; + return( rc); +} + +/**************************************************************************** +Desc: Flushes any pending writes to the output stream +****************************************************************************/ +RCODE F_BackerStream::flush( void) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetup); + + if( m_pClient && m_pThread) + { + if( *m_puiInOffset) + { + if( RC_BAD( rc = signalThread())) + { + goto Exit; + } + } + + // Wait for the background thread to become idle. When it + // does, we know that all writes have completed. + + if( RC_BAD( rc = f_semWait( m_hIdleSem, F_SEM_WAITFOREVER))) + { + goto Exit; + } + + // If the background thread set an error code, we need to return it. + + rc = m_rc; + + // At this point, we know the background thread is either waiting + // for the data semaphore to be signaled or it has exited due to + // an error. We need to re-signal the idle semaphore so that + // other F_BackerStream calls (i.e., additional calls to + // flush, etc.) will not block waiting for it to be signaled + // since it won't be signaled by the background thread until + // after the data semaphore has been signaled again. + + f_semSignal( m_hIdleSem); + + // Jump to exit if we have a bad rc + + if( RC_BAD( rc)) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Signals the read or write thread indicating that data is needed or + that data is available. +****************************************************************************/ +RCODE F_BackerStream::signalThread( void) +{ + FLMBYTE * pucTmp; + FLMUINT * puiTmp; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetup); + + // Return an error if we don't have a thread. + + if( !m_pThread) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + // Wait for the thread to become idle + + if( RC_BAD( rc = f_semWait( m_hIdleSem, F_SEM_WAITFOREVER))) + { + goto Exit; + } + + if( RC_BAD( rc = m_rc)) + { + // If m_rc is bad, we know that the background thread has + // exited and will not signal the idle semaphore again. + // Thus, we will re-signal the idle semaphore so that if the + // code using this class happens to call flush() or some + // other method that waits on the idle semaphore, we + // won't wait forever on something that will never happen. + + f_semSignal( m_hIdleSem); + + // Check the error code + + if( rc == NE_FLM_IO_END_OF_FILE && !m_bFinalRead) + { + m_bFinalRead = TRUE; + } + else + { + goto Exit; + } + } + + pucTmp = m_pucOutBuf; + puiTmp = m_puiOutOffset; + + m_pucOutBuf = m_pucInBuf; + m_puiOutOffset = m_puiInOffset; + + m_pucInBuf = pucTmp; + m_puiInOffset = puiTmp; + + *(m_puiInOffset) = 0; + + if( !m_bFinalRead) + { + // Signal the thread to read or write data + + f_semSignal( m_hDataSem); + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This thread reads data in the background +****************************************************************************/ +RCODE F_BackerStream::readThread( + IF_Thread * pThread) +{ + F_BackerStream * pBackerStream = (F_BackerStream *)pThread->getParm1(); + RCODE rc = NE_SFLM_OK; + + for( ;;) + { + f_semSignal( pBackerStream->m_hIdleSem); + + if( RC_BAD( rc = f_semWait( pBackerStream->m_hDataSem, + F_SEM_WAITFOREVER))) + { + goto Exit; + } + + if( pThread->getShutdownFlag()) + { + break; + } + + if( RC_BAD( rc = pBackerStream->m_pRestoreObj->read( + pBackerStream->m_uiMTUSize, pBackerStream->m_pucInBuf, + pBackerStream->m_puiInOffset))) + { + goto Exit; + } + } + +Exit: + + pBackerStream->m_rc = rc; + f_semSignal( pBackerStream->m_hIdleSem); + return( rc); +} + +/**************************************************************************** +Desc: This thread writes data in the background +****************************************************************************/ +RCODE F_BackerStream::writeThread( + IF_Thread * pThread) +{ + F_BackerStream * pBackerStream = (F_BackerStream *)pThread->getParm1(); + RCODE rc = NE_SFLM_OK; + + for( ;;) + { + f_semSignal( pBackerStream->m_hIdleSem); + + if( RC_BAD( rc = f_semWait( pBackerStream->m_hDataSem, + F_SEM_WAITFOREVER))) + { + goto Exit; + } + + if( *(pBackerStream->m_puiOutOffset)) + { + if( RC_BAD( rc = pBackerStream->m_pClient->WriteData( + pBackerStream->m_pucOutBuf, *(pBackerStream->m_puiOutOffset)))) + { + goto Exit; + } + + // Reset *puiOutOffset so that we won't re-write + // the same data again if we receive a shut-down + // signal. + + *(pBackerStream->m_puiOutOffset) = 0; + } + + // Need to put the thread shutdown check here + // so that if the thread is signaled twice, + // once to do final work and once to shut down, + // we will actually do the work before we exit. + + if( pThread->getShutdownFlag()) + { + break; + } + } + +Exit: + + pBackerStream->m_rc = rc; + f_semSignal( pBackerStream->m_hIdleSem); + return( rc); +} diff --git a/sql/src/flclose.cpp b/sql/src/flclose.cpp new file mode 100644 index 0000000..fbfa749 --- /dev/null +++ b/sql/src/flclose.cpp @@ -0,0 +1,224 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the destructor for the F_Db object. +// +// Tabs: 3 +// +// Copyright (c) 1990-1992, 1995-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flclose.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Destructor for F_Db object +****************************************************************************/ +F_Db::~F_Db() +{ + if( m_eTransType != SFLM_NO_TRANS) + { + // Someone forgot to close their transaction! + + RC_UNEXPECTED_ASSERT( NE_SFLM_TRANS_ACTIVE); + transAbort(); + } + + // Free the super file. + + if (m_pSFileHdl) + { + // Opened files will be released back to the + // file handle manager + + m_pSFileHdl->Release(); + } + + // Free up statistics. + + if (m_bStatsInitialized) + { + flmStatFree( &m_Stats); + } + + // Return the cached B-Tree (if any) to the + // global pool + + if( m_pCachedBTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &m_pCachedBTree); + } + + if( m_pKrefTbl) + { + f_free( &m_pKrefTbl); + m_uiKrefTblSize = 0; + } + + if( m_pucKrefKeyBuf) + { + f_free( &m_pucKrefKeyBuf); + } + + if (m_pKeyColl) + { + m_pKeyColl->Release(); + } + + if (m_pIxClient) + { + m_pIxClient->Release(); + } + + if (m_pIxStatus) + { + m_pIxStatus->Release(); + } + + if (m_pDeleteStatus) + { + m_pDeleteStatus->Release(); + } + + if (m_pCommitClient) + { + m_pCommitClient->Release(); + } + + if (m_hWaitSem != F_SEM_NULL) + { + f_semDestroy( &m_hWaitSem); + } + + m_tmpKrefPool.poolFree(); + m_tempPool.poolFree(); + + // Unlink the F_Db from the F_Database and F_Dict structures. + // IMPORTANT NOTE: The call to unlinkFromDatabase needs to + // be the last thing that is done, because it may unlock + // and relock the mutex. + + if (m_pDatabase) + { + m_pDatabase->lockMutex(); + unlinkFromDict(); + m_pDatabase->unlockMutex(); + + f_mutexLock( gv_SFlmSysData.hShareMutex); + unlinkFromDatabase(); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } +} + +/**************************************************************************** +Desc: Wait for a specific database to close +****************************************************************************/ +RCODE F_DbSystem::waitToClose( + const char * pszDbPath) +{ + RCODE rc = NE_SFLM_OK; + FBUCKET * pBucket; + FLMUINT uiBucket; + F_Database * pDatabase = NULL; + char szDbPathStr1[ F_PATH_MAX_SIZE]; + FLMBOOL bMutexLocked = FALSE; + F_SEM hWaitSem = F_SEM_NULL; + + if( RC_BAD( rc = f_semCreate( &hWaitSem))) + { + goto Exit; + } + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszDbPath, szDbPathStr1))) + { + goto Exit; + } + +Retry: + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + } + + pBucket = gv_SFlmSysData.pDatabaseHashTbl; + uiBucket = flmStrHashBucket( szDbPathStr1, pBucket, FILE_HASH_ENTRIES); + pDatabase = (F_Database *)pBucket [uiBucket].pFirstInBucket; + + while( pDatabase) + { + // Compare the strings. On non-Unix platforms we must use + // f_stricmp, because case does not matter for file names + // on those platforms. + +#ifdef FLM_UNIX + if( f_strcmp( szDbPathStr1, pDatabase->m_pszDbPath) == 0) +#else + if( f_stricmp( szDbPathStr1, pDatabase->m_pszDbPath) == 0) +#endif + { + break; + } + + pDatabase = pDatabase->m_pNext; + } + + if( !pDatabase) + { + // Didn't find a matching database. We are done. + + goto Exit; + } + + // If the file is in the process of being opened by another + // thread, wait for the open to complete. + + if( pDatabase->m_uiFlags & DBF_BEING_OPENED) + { + flmWaitNotifyReq( gv_SFlmSysData.hShareMutex, hWaitSem, + &pDatabase->m_pOpenNotifies, NULL); + goto Retry; + } + else + { + // The database is open. Put ourselves into the close notify list + // so that we will wake up when the database has been closed. + + if( RC_BAD( rc = flmWaitNotifyReq( + gv_SFlmSysData.hShareMutex, hWaitSem, + &pDatabase->m_pCloseNotifies, NULL))) + { + goto Exit; + } + } + +Exit: + + if( bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + if( hWaitSem != F_SEM_NULL) + { + f_semDestroy( &hWaitSem); + } + + return( rc); +} diff --git a/sql/src/flconvrt.cpp b/sql/src/flconvrt.cpp new file mode 100644 index 0000000..887320a --- /dev/null +++ b/sql/src/flconvrt.cpp @@ -0,0 +1,726 @@ +//------------------------------------------------------------------------------ +// Desc: Database upgrade routines. +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flconvrt.cpp 3112 2006-01-19 13:12:40 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc : Upgrades a database to the latest FLAIM version. +****************************************************************************/ +RCODE F_Db::upgrade( + IF_UpgradeClient * // pUpgradeClient + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + FLMBOOL bLockedDatabase = FALSE; + FLMUINT uiOldVersion = 0; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + SFLM_DB_HDR * pUncommittedDbHdr = &m_pDatabase->m_uncommittedDbHdr; + FLMUINT64 ui64SaveTransId; + FLMBOOL bUpgradeNeeded = FALSE; + FLMUINT uiDisableCount = 0; + + // Lock the database if not already locked. + // Cannot lose exclusive access between the checkpoint and + // the update transaction that does the conversion. + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if (RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, 15))) + { + goto Exit; + } + bLockedDatabase = TRUE; + } + + // Cannot have any transaction already going. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // NOTE: Don't get the current version number until AFTER obtaining + // the exclusive lock - to make sure nobody else can or will do + // an upgrade while we are in here. + + uiOldVersion = (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32DbVersion; + + // Verify that we can do the upgrade + + switch (uiOldVersion) + { + case SFLM_CURRENT_VERSION_NUM: + break; + + default: + rc = RC_SET( NE_SFLM_UNALLOWED_UPGRADE); + goto Exit; + } + + if( !bUpgradeNeeded) + { + goto Exit; + } + + // Change state of logging OFF to TRUE - don't want anything + // logged during conversion except for the upgrade packet. + + uiDisableCount = pRfl->disableLogging(); + m_uiFlags |= FDB_UPGRADING; + + ui64SaveTransId = m_pDatabase->m_lastCommittedDbHdr.ui64CurrTransID; + + // Flush everything do disk so that the roll forward log is empty. + // The upgrade doesn't put any special data in the roll forward log + // so if the roll forward log had stuff in it, it would roll forward + // on data that was a newer version - and never work! + // Start an update transaction and commit it, forcing it to be + // checkpointed. + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Don't want this transaction to change the transaction ID because + // we are only trying to force a checkpoint. We don't want to change + // the transaction ID until we have actually done the convert. + + m_pDatabase->m_uncommittedDbHdr.ui64CurrTransID = ui64SaveTransId; + m_ui64CurrTransID = ui64SaveTransId; + + // Set up things in the FDB to indicate where we should move the + // checkpoint file number and offset to. If we are in the middle + // of a recovery or restore operation, move the pointer forward + // to just BEFORE the upgrade packet. Down below when we do the + // checkpoint at the end of the upgrade, we will move the pointer + // forward to just AFTER the upgrade packet. + + if (m_uiFlags & FDB_REPLAYING_RFL) + { + m_uiUpgradeCPFileNum = pRfl->getCurrFileNum(); + m_uiUpgradeCPOffset = pRfl->getCurrPacketAddress(); + } + + // Commit the transaction, forcing it to be checkpointed. + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, TRUE))) + { + goto Exit; + } + + // Start an update transaction for the conversion. + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Make sure that commit does something. + + m_bHadUpdOper = TRUE; + + // NOTE: By this point, all conversions should be complete, except for + // committing and changing the version number. + + // Log the upgrade packet to the RFL + + pRfl->enableLogging(); + + // Log the upgrade packet. + + if( RC_BAD( rc = pRfl->logUpgrade( this, uiOldVersion))) + { + goto Exit; + } + + // Turn logging off again + + uiDisableCount = pRfl->disableLogging(); + + // Change the FLAIM version number to the new version number. + + pUncommittedDbHdr->ui32DbVersion = SFLM_CURRENT_VERSION_NUM; + + // Commit and force a checkpoint by passing TRUE. + // Set up things in the FDB to indicate where we should move the + // checkpoint file number and offset to. If we are in the middle + // of a recovery or restore operation, move the pointer forward + // to just AFTER the upgrade packet. + + if (m_uiFlags & FDB_REPLAYING_RFL) + { + m_uiUpgradeCPFileNum = pRfl->getCurrFileNum(); + m_uiUpgradeCPOffset = pRfl->getCurrReadOffset(); + } + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, TRUE))) + { + goto Exit; + } + +Exit: + + if( bStartedTrans) + { + // Failure condition, we jumped to exit + + pUncommittedDbHdr->ui32DbVersion = (FLMUINT32)uiOldVersion; + m_pDatabase->m_lastCommittedDbHdr.ui32DbVersion = + (FLMUINT32)uiOldVersion; + + (void)abortTrans(); + } + + if (uiDisableCount) + { + pRfl->enableLogging(); + } + + // Turn off the upgrade flag, in case it was turned on above. + + m_uiFlags &= (~(FDB_UPGRADING)); + + if (bLockedDatabase) + { + (void)dbUnlock(); + } + + return( rc ); +} + +/************************************************************************ +Desc : Enable encryption on the database. +*************************************************************************/ +RCODE F_Db::enableEncryption( void) +{ + RCODE rc = NE_SFLM_OK; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMBYTE * pucWrappingKey = NULL; + FLMUINT32 ui32KeyLen = 0; + SFLM_DB_HDR * pucUncommittedLogHdr = &m_pDatabase->m_uncommittedDbHdr; + FLMBOOL bLocked = FALSE; + FLMBOOL bStartedTrans = FALSE; + FLMBOOL bEnableLogging = FALSE; + + // We must will start our own transaction + + if( m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if ( RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, SFLM_NO_TIMEOUT))) + { + goto Exit; + } + + bLocked = TRUE; + } + + // Disable RFL logging + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Begin an update transaction. + + if (RC_BAD( rc = transBegin( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + + bStartedTrans = TRUE; + + // If we don't have a wrapping key, then create one. Normally + // this would be the case, since we are enabling encryption, + // but the test is "just to be sure" we don't + // overwrite an existing key. + + if (!m_pDatabase->m_pWrappingKey) + { + if ( RC_BAD( rc = createDbKey())) + { + goto Exit; + } + } + + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->getKeyToStore( + &pucWrappingKey, &ui32KeyLen, + m_pDatabase->m_pszDbPasswd, NULL))) + { + goto Exit; + } + + f_memcpy( pucUncommittedLogHdr->ucDbKey, pucWrappingKey, ui32KeyLen); + pucUncommittedLogHdr->ui32DbKeyLen = ui32KeyLen; + + m_pDatabase->m_rcLimitedCode = NE_SFLM_OK; + m_pDatabase->m_bInLimitedMode = FALSE; + m_pDatabase->m_bHaveEncKey = TRUE; + + // Log the upgrade packet + + pRfl->enableLogging(); + + if (RC_BAD( rc = pRfl->logEncryptionKey( this, + RFL_ENABLE_ENCRYPTION_PACKET, pucWrappingKey, ui32KeyLen))) + { + goto Exit; + } + + // Turn logging off again + + pRfl->disableLogging(); + + // Commit the transaction and force a checkpoint + + if( RC_BAD( rc = commitTrans( 0, TRUE, NULL))) + { + goto Exit; + } + + bStartedTrans = FALSE; + +Exit: + + if( bStartedTrans) + { + transAbort(); + } + + if (bEnableLogging) + { + pRfl->enableLogging(); + } + + if( bLocked) + { + dbUnlock(); + } + + if( pucWrappingKey) + { + f_free( &pucWrappingKey); + } + + return( rc); +} + +/**************************************************************************** +Desc : Change the database key from wrapped in the NICI server + key to shrouded in a password and vice-versa. + + If no password is specified, the key will be wrapped in the + NICI server key. If a password is specified, the key will be + shrouded in it. +****************************************************************************/ +RCODE F_Db::wrapKey( + const char * pszPassword) +{ + RCODE rc = NE_SFLM_OK; + SFLM_DB_HDR * pUncommittedDbHdr = &m_pDatabase->m_uncommittedDbHdr; + FLMBOOL bStartedTrans = FALSE; + FLMBYTE * pucTmp = NULL; + FLMUINT32 ui32KeyLen = SFLM_MAX_ENC_KEY_SIZE; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMBOOL bLocked = FALSE; + FLMBOOL bEnableLogging = FALSE; + + if( getTransType() != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + if( !(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if ( RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bLocked = TRUE; + } + + // Turn off logging. We only want to log the wrap key packet. + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Start the transaction + + if (RC_BAD( rc = transBegin( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Wrap or shroud the key + + if (RC_BAD( rc = m_pDatabase->m_pWrappingKey->getKeyToStore( + &pucTmp, &ui32KeyLen, + (FLMBYTE *)pszPassword, NULL))) + { + goto Exit; + } + + f_memcpy( pUncommittedDbHdr->ucDbKey, pucTmp, ui32KeyLen); + pUncommittedDbHdr->ui32DbKeyLen = ui32KeyLen; + + // Turn on logging. We only want to log the wrap key packet. + + pRfl->enableLogging(); + + // Log a wrapped key packet to record that the key + // has been wrapped/encrypted. + + if (RC_BAD( rc = pRfl->logEncryptionKey( this, + RFL_WRAP_KEY_PACKET, pucTmp, ui32KeyLen))) + { + goto Exit; + } + + // Turn logging off again + + pRfl->disableLogging(); + + // Make sure the log header gets written out... + + m_bHadUpdOper = TRUE; + + // Commit the transaction and force a checkpoint + + if (RC_BAD( rc = transCommit())) + { + goto Exit; + } + + bStartedTrans = FALSE; + + // Delete the old password + + if (m_pDatabase->m_pszDbPasswd) + { + f_free( &m_pDatabase->m_pszDbPasswd); + } + + // Store the new password + + if( pszPassword) + { + if (RC_BAD( rc = f_calloc( f_strlen( pszPassword) + 1, + &m_pDatabase->m_pszDbPasswd))) + { + goto Exit; + } + + f_memcpy( m_pDatabase->m_pszDbPasswd, pszPassword, + (FLMSIZET)f_strlen( pszPassword)); + } + +Exit: + + if( bStartedTrans) + { + transAbort(); + } + + if( bEnableLogging) + { + pRfl->enableLogging(); + } + + if( bLocked) + { + dbUnlock(); + } + + if( pucTmp) + { + f_free( &pucTmp); + } + + return( rc); +} + +/*************************************************************************** + * A private function that goes through all the potential trial-and-error + * involved in getting the strongest possible key we can use for the + * database key. + * NOTE: If an update transaction is needed, it is the responsibility of + * the caller to start the transaction! We can't check for this in the + * function because sometimes a transaction is required, and other times + * (such as during a db create) it's impossible. + ***************************************************************************/ +RCODE F_Db::createDbKey( void) +{ + RCODE rc = NE_SFLM_OK; + + if( m_pDatabase->m_pWrappingKey) + { + m_pDatabase->m_pWrappingKey->Release(); + m_pDatabase->m_pWrappingKey = NULL; + } + + if( (m_pDatabase->m_pWrappingKey = f_new F_CCS) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->init( + TRUE, SFLM_AES_ENCRYPTION))) + { + goto Exit; + } + + // Try to get the strongest encryption supported on this platform. + + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->generateWrappingKey( + SFLM_AES256_KEY_SIZE))) + { + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->generateWrappingKey( + SFLM_AES192_KEY_SIZE))) + { + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->generateWrappingKey( + SFLM_AES128_KEY_SIZE))) + { + // Try using DES3 + m_pDatabase->m_pWrappingKey->Release(); + + if ((m_pDatabase->m_pWrappingKey = f_new F_CCS) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = m_pDatabase->m_pWrappingKey->init( + TRUE, SFLM_DES3_ENCRYPTION))) + { + goto Exit; + } + + if (RC_BAD( rc = m_pDatabase->m_pWrappingKey->generateWrappingKey( + SFLM_DES3_168_KEY_SIZE))) + { + // No more choices... + goto Exit; + } + } + } + } + +Exit: + return rc; +} + +/**************************************************************************** +Desc : Generate a new database key and re-wrap all existing keys in it + NOTE: New database key will be wrapped in NICI server key, + even if the previous key was wrapped in a password. +****************************************************************************/ +RCODE F_Db::rollOverDbKey( void) +{ + RCODE rc = NE_SFLM_OK; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMBYTE * pucWrappingKey = NULL; + FLMUINT32 ui32KeyLen = 0; + FLMBOOL bIsNull; + FLMUINT uiEncDefNum; + F_ENCDEF * pEncDef; + FLMBYTE * pucBuf = NULL; + FLMBOOL bLocked = FALSE; + FLMBOOL bStartedTrans = FALSE; + FLMUINT32 ui32BufLen = 0; + F_Row * pRow = NULL; + FSTableCursor * pTableCursor = NULL; + FLMBOOL bEnableLogging = FALSE; + + if( getTransType() != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if ( RC_BAD( rc = dbLock( SFLM_LOCK_EXCLUSIVE, 0, SFLM_NO_TIMEOUT))) + { + goto Exit; + } + + bLocked = TRUE; + } + + // Turn off logging + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Start the transaction + + if (RC_BAD( rc = transBegin( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + + // Update the database header with the new key + + if( RC_BAD( rc = createDbKey())) + { + goto Exit; + } + + if( RC_BAD( rc = m_pDatabase->m_pWrappingKey->getKeyToStore( + &pucWrappingKey, &ui32KeyLen, + m_pDatabase->m_pszDbPasswd, NULL))) + { + goto Exit; + } + + f_memcpy( m_pDatabase->m_uncommittedDbHdr.ucDbKey, + pucWrappingKey, ui32KeyLen); + m_pDatabase->m_uncommittedDbHdr.ui32DbKeyLen = ui32KeyLen; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pTableCursor->setupRange( this, SFLM_TBLNUM_ENCDEFS, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Read through all rows in the encryption definition table. Each row + // defines an encryption definition whose key needs to be re-wrapped. + + for (;;) + { + if (RC_BAD( rc = pTableCursor->nextRow( this, &pRow, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // Get the encryption definition number - required. + + if (RC_BAD( rc = pRow->getUINT( this, SFLM_COLNUM_ENCDEFS_ENCDEF_NUM, + &uiEncDefNum, &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NULL_ENCDEF_NUM); + goto Exit; + } + + if ((pEncDef = m_pDict->getEncDef( uiEncDefNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + + if( RC_BAD( rc = pEncDef->pCcs->getKeyToStore( &pucBuf, &ui32BufLen, + NULL, m_pDatabase->m_pWrappingKey))) + { + goto Exit; + } + + if (RC_BAD( rc = pRow->setBinary( this, SFLM_COLNUM_ENCDEFS_ENC_KEY, + pucBuf, ui32BufLen))) + { + goto Exit; + } + } + + if( RC_BAD( rc = transCommit())) + { + goto Exit; + } + bStartedTrans = FALSE; + + pRfl->enableLogging(); + + if( RC_BAD( rc = m_pDatabase->m_pRfl->logRollOverDbKey( this))) + { + goto Exit; + } + +Exit: + + if( bStartedTrans) + { + transAbort(); + } + + if( bEnableLogging) + { + pRfl->enableLogging(); + } + + if( bLocked) + { + dbUnlock(); + } + + if( pucWrappingKey) + { + f_free( &pucWrappingKey); + } + + if( pucBuf) + { + f_free( &pucBuf); + } + + if (pRow) + { + pRow->ReleaseRow(); + } + + if (pTableCursor) + { + pTableCursor->Release(); + } + + return( rc); +} diff --git a/sql/src/flcreate.cpp b/sql/src/flcreate.cpp new file mode 100644 index 0000000..1818bc4 --- /dev/null +++ b/sql/src/flcreate.cpp @@ -0,0 +1,472 @@ +//------------------------------------------------------------------------------ +// Desc: Create a database. +// +// Tabs: 3 +// +// Copyright (c) 1990-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flcreate.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/*API~*********************************************************************** +Desc: Creates a new FLAIM database. +*END************************************************************************/ +RCODE F_DbSystem::dbCreate( + const char * pszFilePath, + const char * pszDataDir, + const char * pszRflDir, + SFLM_CREATE_OPTS * pCreateOpts, + FLMBOOL bTempDb, + F_Db ** ppDb) +{ + RCODE rc = NE_SFLM_OK; + F_Db * pDb = NULL; + F_Database * pDatabase = NULL; + FLMBOOL bDatabaseCreated = FALSE; + FLMBOOL bNewDatabase = FALSE; + FLMBOOL bMutexLocked = FALSE; + FLMBOOL bEnableLogging = FALSE; + + // Make sure the path looks valid + + if (!pszFilePath || !pszFilePath [0]) + { + rc = RC_SET( NE_FLM_IO_INVALID_FILENAME); + goto Exit; + } + + // Allocate and initialize an F_Db structure. + + if (RC_BAD( rc = allocDb( &pDb, FALSE))) + { + goto Exit; + } + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + + // Free any unused structures that have been unused for the maximum + // amount of time. May unlock and re-lock the global mutex. + + checkNotUsedObjects(); + + for( ;;) + { + + // See if we already have the file open. + // May unlock and re-lock the global mutex. + + if (RC_BAD( rc = findDatabase( pszFilePath, pszDataDir, &pDatabase))) + { + goto Exit; + } + + // Didn't find the database + + if (!pDatabase) + { + break; + } + + // See if file is open or being opened. + + if (pDatabase->m_uiOpenIFDbCount || (pDatabase->m_uiFlags & DBF_BEING_OPENED)) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } + + // Free the F_Database object. May temporarily unlock the global mutex. + // For this reason, we must call findDatabase again (see above) after + // freeing the object. + + pDatabase->freeDatabase(); + pDatabase = NULL; + } + + // Allocate a new F_Database object + + if (RC_BAD( rc = allocDatabase( pszFilePath, pszDataDir, bTempDb, &pDatabase))) + { + goto Exit; + } + bNewDatabase = TRUE; + + // Link the F_Db object to the F_Database object. + + rc = pDb->linkToDatabase( pDatabase); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + if (RC_BAD( rc)) + { + goto Exit; + } + + // If the database has not already been created, do so now. + + // Determine what to set file block size to. + + if (pCreateOpts != NULL) + { + pDb->m_pSFileHdl->setBlockSize( + flmAdjustBlkSize( pCreateOpts->uiBlockSize)); + } + else + { + pDb->m_pSFileHdl->setBlockSize( SFLM_DEFAULT_BLKSIZ); + } + + if (RC_OK( gv_SFlmSysData.pFileSystem->doesFileExist( pszFilePath))) + { + rc = RC_SET( NE_SFLM_DATABASE_EXISTS); + goto Exit; + } + + // Create the .db file. + + pDb->m_pSFileHdl->setMaxAutoExtendSize( gv_SFlmSysData.uiMaxFileSize); + pDb->m_pSFileHdl->setExtendSize( pDb->m_pDatabase->m_uiFileExtendSize); + if (RC_BAD( rc = pDb->m_pSFileHdl->createFile( 0))) + { + goto Exit; + } + bDatabaseCreated = TRUE; + + (void)flmStatGetDb( &pDb->m_Stats, pDatabase, + 0, &pDb->m_pDbStats, NULL, NULL); + + // We must have exclusive access. Create a lock file for that + // purpose, if there is not already a lock file. + // NOTE: No need for a lock file if this is a temporary database. + + if (!bTempDb) + { + if (RC_BAD( rc = pDatabase->getExclAccess( pszFilePath))) + { + goto Exit; + } + } + + if (RC_BAD( rc = pDb->initDbFiles( pszRflDir, pCreateOpts))) + { + goto Exit; + } + + // Disable RFL logging (m_pRfl was initialized in initDbFiles) + + if( pDatabase->m_pRfl) + { + bEnableLogging = pDatabase->m_pRfl->isLoggingEnabled(); + pDatabase->m_pRfl->disableLogging(); + } + + // Set FFILE stuff to same state as a completed checkpoint. + + pDatabase->m_uiFirstLogCPBlkAddress = 0; + pDatabase->m_uiLastCheckpointTime = (FLMUINT)FLM_GET_TIMER(); + + // Create a checkpoint thread - no need if this is a temporary + // database. + + if (!bTempDb) + { + if (RC_BAD( rc = pDatabase->startCPThread())) + { + goto Exit; + } + + if( RC_BAD( rc = pDatabase->startMaintThread())) + { + goto Exit; + } + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + if (pDb) + { + pDb->completeOpenOrCreate( rc, bNewDatabase); + + // completeOpenOrCreate will delete pDb if RC_BAD( rc) + + if (RC_BAD( rc)) + { + pDb = NULL; + } + else + { + *ppDb = pDb; + pDb = NULL; // This isn't strictly necessary, but it makes it + // obvious that we are no longer using the object. + } + } + + if (RC_BAD( rc)) + { + if (bDatabaseCreated) + { + F_DbSystem dbSystem; + + dbSystem.dbRemove( pszFilePath, pszDataDir, pszRflDir, TRUE); + } + } + else if (bEnableLogging) + { + pDatabase->m_pRfl->enableLogging(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Create a database - initialize all physical areas & data dictionary. +****************************************************************************/ +RCODE F_Db::initDbFiles( + const char * pszRflDir, + SFLM_CREATE_OPTS * pCreateOpts) // Create options for the database. +{ + RCODE rc = NE_SFLM_OK; + FLMUINT bTransStarted = FALSE; + FLMBYTE * pucBuf = NULL; + FLMUINT uiBlkSize; + FLMUINT uiWriteBytes; + SFLM_DB_HDR * pDbHdr; + F_BLK_HDR * pBlkHdr; + F_CachedBlock * pSCache = NULL; + FLMBYTE * pucWrappingKey = NULL; + FLMBOOL bEnableLogging = FALSE; +#ifdef FLM_USE_NICI + FLMUINT32 ui32KeyLen = 0; +#endif + + // Determine what size of buffer to allocate. + + uiBlkSize = (FLMUINT)(pCreateOpts + ? flmAdjustBlkSize( pCreateOpts->uiBlockSize) + : (FLMUINT)SFLM_DEFAULT_BLKSIZ); + + // Allocate a buffer for writing. + + if (RC_BAD( rc = f_alloc( (FLMUINT)uiBlkSize, &pucBuf))) + { + goto Exit; + } + + // Initialize the database header structure. + + pDbHdr = (SFLM_DB_HDR *)pucBuf; + flmInitDbHdr( pCreateOpts, TRUE, m_pDatabase->m_bTempDb, pDbHdr); + m_pDatabase->m_uiBlockSize = (FLMUINT)pDbHdr->ui16BlockSize; + m_pDatabase->m_uiDefaultLanguage = (FLMUINT)pDbHdr->ui8DefaultLanguage; + m_pDatabase->m_uiMaxFileSize = (FLMUINT)pDbHdr->ui32MaxFileSize; + m_pDatabase->m_uiSigBitsInBlkSize = calcSigBits( uiBlkSize); + + f_memcpy( &m_pDatabase->m_lastCommittedDbHdr, pDbHdr, sizeof( SFLM_DB_HDR)); + + // Create the first block file. + + if (!m_pDatabase->m_bTempDb) + { + if (RC_BAD( rc = m_pSFileHdl->createFile( 1))) + { + goto Exit; + } + } + +#ifdef FLM_USE_NICI + if (RC_BAD( rc = createDbKey())) + { + goto Exit; + } + + if (RC_BAD( rc = m_pDatabase->m_pWrappingKey->getKeyToStore( + &pucWrappingKey, + &ui32KeyLen, + m_pDatabase->m_pszDbPasswd, + NULL))) + { + goto Exit; + } + + f_memcpy( m_pDatabase->m_lastCommittedDbHdr.DbKey, + pucWrappingKey, + ui32KeyLen); + m_pDatabase->m_lastCommittedDbHdr.ui32DbKeyLen = ui32KeyLen; + + m_pDatabase->m_rcLimitedCode = NE_SFLM_OK; + m_pDatabase->m_bInLimitedMode = FALSE; + m_pDatabase->m_bHaveEncKey = TRUE; + +#else + m_pDatabase->m_rcLimitedCode = NE_SFLM_ENCRYPTION_UNAVAILABLE; + m_pDatabase->m_bInLimitedMode = TRUE; + m_pDatabase->m_bHaveEncKey = FALSE; +#endif + + // Write out the log header + + if (RC_BAD( rc = m_pDatabase->writeDbHdr( m_pDbStats, m_pSFileHdl, + &m_pDatabase->m_lastCommittedDbHdr, + NULL, TRUE))) + { + goto Exit; + } + + // Initialize and output the first LFH block + + if (m_pDatabase->m_bTempDb) + { + getDbHdrInfo( &m_pDatabase->m_lastCommittedDbHdr); + if (RC_BAD( rc = m_pDatabase->createBlock( this, &pSCache))) + { + goto Exit; + } + pBlkHdr = (F_BLK_HDR *)pSCache->m_pBlkHdr; + m_pDatabase->m_lastCommittedDbHdr.ui32FirstLFBlkAddr = pBlkHdr->ui32BlkAddr; + } + else + { + // Copy the Db header to the checkpointDbHdr buffer. + // This is now the first official checkpoint version of the log + // header. It must be copied to the checkpointDbHdr buffer so that + // it will not be lost in subsequent calls to flmWriteDbHdr. + + f_memcpy( &m_pDatabase->m_checkpointDbHdr, &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + + f_memset( pucBuf, 0, uiBlkSize); + pBlkHdr = (F_BLK_HDR *)pucBuf; + pBlkHdr->ui32BlkAddr = m_pDatabase->m_lastCommittedDbHdr.ui32FirstLFBlkAddr; + pBlkHdr->ui64TransID = 0; + } + pBlkHdr->ui8BlkType = BT_LFH_BLK; + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)(uiBlkSize - SIZEOF_STD_BLK_HDR); + blkSetNativeFormat( pBlkHdr); + + if (!m_pDatabase->m_bTempDb) + { + pBlkHdr->ui32BlkCRC = calcBlkCRC( pBlkHdr, SIZEOF_STD_BLK_HDR); + if (RC_BAD( rc = m_pSFileHdl->writeBlock( + (FLMUINT)pBlkHdr->ui32BlkAddr, + uiBlkSize, pucBuf, uiBlkSize, NULL, + &uiWriteBytes))) + { + goto Exit; + } + + // Force things to disk. + + if (RC_BAD( rc = m_pSFileHdl->flush())) + { + goto Exit; + } + } + + // Allocate the pRfl object. Could not do this until this point + // because we need to have the version number, block size, etc. + // setup in the database header + + flmAssert( !m_pDatabase->m_pRfl); + + if (!m_pDatabase->m_bTempDb) + { + if ((m_pDatabase->m_pRfl = f_new F_Rfl) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = m_pDatabase->m_pRfl->setup( m_pDatabase, pszRflDir))) + { + goto Exit; + } + + // Disable RFL logging + + bEnableLogging = m_pDatabase->m_pRfl->isLoggingEnabled(); + m_pDatabase->m_pRfl->disableLogging(); + + // Start an update transaction and populate the dictionary. + // This also creates the default collections and indexes. + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bTransStarted = TRUE; + + if (RC_BAD( rc = dictCreate())) + { + goto Exit; + } + + // Because the checkpoint thread has not yet been created, + // flmCommitDbTrans will force a checkpoint when it completes, + // ensuring a consistent database state. + + bTransStarted = FALSE; + if (RC_BAD( rc = commitTrans( 0, TRUE))) + { + goto Exit; + } + } + else + { + // The uncommitted header must have all of the stuff from the committed header + // in order for this to be able to work as if an update transaction was in + // progress. + + f_memcpy( &m_pDatabase->m_uncommittedDbHdr, &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + } + +Exit: + + // Free the temporary buffer, if it was allocated. + + f_free( &pucBuf); + + if (pucWrappingKey) + { + f_free( &pucWrappingKey); + } + + if (bTransStarted) + { + abortTrans(); + } + + if( bEnableLogging) + { + m_pDatabase->m_pRfl->enableLogging(); + } + + if (pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} diff --git a/sql/src/fldbglog.cpp b/sql/src/fldbglog.cpp new file mode 100644 index 0000000..4893fe7 --- /dev/null +++ b/sql/src/fldbglog.cpp @@ -0,0 +1,325 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the functions for debug logging. +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fldbglog.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#ifdef FLM_DBG_LOG + +// Local prototypes + +FSTATIC void _flmDbgLogFlush( void); + +FSTATIC void _flmDbgOutputMsg( + char * pszMsg); + +// Global data + +F_MUTEX g_hDbgLogMutex = F_MUTEX_NULL; +IF_FileHdl * g_pLogFile = NULL; +char * g_pszLogBuf = NULL; +FLMUINT g_uiLogBufOffset = 0; +FLMUINT g_uiLogFileOffset = 0; +FLMBOOL g_bDbgLogEnabled = TRUE; + +#define DBG_LOG_BUFFER_SIZE ((FLMUINT)512000) + + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogInit( void) +{ + char szLogPath[ 256]; + RCODE rc = NE_SFLM_OK; + + flmAssert( g_hDbgLogMutex == F_MUTEX_NULL); + + // Allocate a buffer for the log + + if (RC_BAD( rc = f_alloc( DBG_LOG_BUFFER_SIZE + 1024, &g_pszLogBuf))) + { + goto Exit; + } + + // Create the mutex + + if (RC_BAD( rc = f_mutexCreate( &g_hDbgLogMutex))) + { + goto Exit; + } + + // Build the file path + +#ifdef FLM_NLM + f_strcpy( szLogPath, "SYS:\\FLMDBG.LOG"); +#else + f_sprintf( szLogPath, "FLMDBG.LOG"); +#endif + + // Create the file - truncate if it exists already. + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->Create( szLogPath, + XFLM_IO_RDWR | XFLM_IO_SH_DENYNONE | XFLM_IO_DIRECT, &g_pLogFile))) + { + goto Exit; + } + +Exit: + + flmAssert( RC_OK( rc)); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogExit( void) +{ + if( g_bDbgLogEnabled) + { + // Output "Log End" message + f_mutexLock( g_hDbgLogMutex); + _flmDbgOutputMsg( "--- LOG END ---"); + f_mutexUnlock( g_hDbgLogMutex); + + // Flush the log + flmDbgLogFlush(); + } + + // Free all resources + + if( g_hDbgLogMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &g_hDbgLogMutex); + } + + if( g_pszLogBuf) + { + f_free( &g_pszLogBuf); + } + + if( g_pLogFile) + { + g_pLogFile->Truncate( g_uiLogFileOffset + g_uiLogBufOffset); + g_pLogFile->Close(); + g_pLogFile->Release(); + g_pLogFile = NULL; + } + + g_bDbgLogEnabled = FALSE; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogMsg( + char * pszMsg) +{ + if (!g_bDbgLogEnabled) + return; + f_mutexLock( g_hDbgLogMutex); + _flmDbgOutputMsg( pszMsg); + f_mutexUnlock( g_hDbgLogMutex); +} + + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogFlush( void) +{ + f_mutexLock( g_hDbgLogMutex); + _flmDbgLogFlush(); + f_mutexUnlock( g_hDbgLogMutex); +} + + +/**************************************************************************** +Desc: +****************************************************************************/ +FSTATIC void _flmDbgLogFlush( void) +{ + FLMUINT uiBytesToWrite; + FLMUINT uiBytesWritten; + char * pszBufPtr = g_pszLogBuf; + FLMUINT uiTotalToWrite = g_uiLogBufOffset; + RCODE rc = NE_SFLM_OK; + FLMUINT uiBufferSize = DBG_LOG_BUFFER_SIZE + 1024; + + while( uiTotalToWrite) + { + if( uiTotalToWrite > 0xFE00) + { + uiBytesToWrite = 0xFE00; + } + else + { + uiBytesToWrite = uiTotalToWrite; + } + + if( RC_BAD( rc = g_pLogFile->SectorWrite( + g_uiLogFileOffset, uiBytesToWrite, + pszBufPtr, uiBufferSize, NULL, &uiBytesWritten, FALSE))) + { + goto Exit; + } + + flmAssert( uiBytesToWrite == uiBytesWritten); + g_uiLogFileOffset += uiBytesWritten; + pszBufPtr += uiBytesWritten; + uiBufferSize -= uiBytesWritten; + uiTotalToWrite -= uiBytesWritten; + } + + if (g_uiLogBufOffset & 0x1FF) + { + if (g_uiLogBufOffset > 512) + { + f_memcpy( g_pszLogBuf, + &g_pszLogBuf [g_uiLogBufOffset & 0xFFFFFE00], + 512); + g_uiLogBufOffset &= 0x1FF; + } + g_uiLogFileOffset -= g_uiLogBufOffset; + } + else + { + g_uiLogBufOffset = 0; + } + +Exit: + + flmAssert( RC_OK( rc)); +} + + +/**************************************************************************** +Desc: +****************************************************************************/ +void _flmDbgOutputMsg( + char * pszMsg) +{ + char * pszBufPtr = (char *)(&(g_pszLogBuf[ g_uiLogBufOffset])); + + f_sprintf( (char *)pszBufPtr, "%s\n", pszMsg); + g_uiLogBufOffset += f_strlen( pszBufPtr); + + if( g_uiLogBufOffset >= DBG_LOG_BUFFER_SIZE) + { + _flmDbgLogFlush(); + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogWrite( + F_Database * pDatabase, + FLMUINT uiBlkAddress, + FLMUINT uiWriteAddress, + FLMUINT64 ui64TransId, + char * pszEvent) +{ + char pszTmpBuf[ 256]; + + if( !g_bDbgLogEnabled) + return; + + if( !uiWriteAddress) + { + f_sprintf( (char *)pszTmpBuf, "d%X b=%X t%I64u %s", + (unsigned)((FLMUINT)pDatabase), + (unsigned)uiBlkAddress, ui64TransId, pszEvent); + } + else + { + f_sprintf( (char *)pszTmpBuf, "d%X b=%X a=%X t%I64u %s", + (unsigned)((FLMUINT)pDatabase), + (unsigned)uiBlkAddress, (unsigned)uiWriteAddress, + ui64TransId, pszEvent); + } + f_mutexLock( g_hDbgLogMutex); + _flmDbgOutputMsg( pszTmpBuf); + f_mutexUnlock( g_hDbgLogMutex); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void flmDbgLogUpdate( + F_Database * pDatabase, + FLMUINT64 ui64TransId, + FLMUINT uiCollection, + FLMUINT64 ui64NodeId, + RCODE rc, + char * pszEvent) +{ + char pszTmpBuf[ 256]; + char szErr [12]; + + if (!g_bDbgLogEnabled) + { + return; + } + if (RC_BAD( rc)) + { + f_sprintf( szErr, " RC=%04X", (unsigned)rc); + } + else + { + szErr [0] = 0; + } + + if (uiCollection) + { + f_sprintf( (char *)pszTmpBuf, "d%X t%I64u c%u n%I64u %s%s", + (unsigned)((FLMUINT)pDatabase), + ui64TransId, (unsigned)uiCollection, + ui64NodeId, pszEvent, szErr); + } + else + { + f_sprintf( (char *)pszTmpBuf, "d%X t%I64u %s%s", + (unsigned)((FLMUINT)pDatabase), + ui64TransId, pszEvent, + szErr); + } + + f_mutexLock( g_hDbgLogMutex); + _flmDbgOutputMsg( pszTmpBuf); + f_mutexUnlock( g_hDbgLogMutex); +} + +#else + +// Must have something here for the Netware platform, or it won't build. + +#if defined( FLM_WATCOM_NLM) + void gv_fldbglog() + { + } +#endif + + +#endif // #ifdef FLM_DBG_LOG diff --git a/sql/src/flerror.cpp b/sql/src/flerror.cpp new file mode 100644 index 0000000..d20217c --- /dev/null +++ b/sql/src/flerror.cpp @@ -0,0 +1,153 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains error routines that are used throughout FLAIM. +// +// Tabs: 3 +// +// Copyright (c) 1997-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flerror.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: +****************************************************************************/ +const char * FlmCorruptStrings[ SFLM_NUM_CORRUPT_ERRORS] = +{ + + // WARNING: ANY CHANGES MADE TO THE FlmCorruptStrings TABLE MUST BE + // REFLECTED IN THE CHECK CODE DEFINES FOUND IN flaimsys.h + + "OK", // 0 + "BAD_CHAR", // 1 + "BAD_ASIAN_CHAR", // 2 + "BAD_CHAR_SET", // 3 + "BAD_TEXT_FIELD", // 4 + "BAD_NUMBER_FIELD", // 5 + "BAD_FIELD_TYPE", // 6 + "BAD_IX_DEF", // 7 + "BAD_NUMBER_KEY", // 8 + "BAD_BINARY_KEY", // 9 + "BAD_KEY_FIELD_TYPE", // 10 + "BAD_KEY_LEN", // 11 + "BAD_LFH_LIST_PTR", // 12 + "BAD_LFH_LIST_END", // 13 + "BAD_BLK_END", // 14 + "KEY_COUNT_MISMATCH", // 15 + "REF_COUNT_MISMATCH", // 16 + "BAD_BLK_HDR_ADDR", // 17 + "BAD_BLK_HDR_LEVEL", // 18 + "BAD_BLK_HDR_PREV", // 19 + + // WARNING: ANY CHANGES MADE TO THE FlmCorruptStrings TABLE MUST BE + // REFLECTED IN THE CHECK CODE DEFINES FOUND IN flaimsys.h + + "BAD_BLK_HDR_NEXT", // 20 + "BAD_BLK_HDR_TYPE", // 21 + "BAD_BLK_HDR_ROOT_BIT", // 22 + "BAD_BLK_HDR_BLK_END", // 23 + "BAD_BLK_HDR_LF_NUM", // 24 + "BAD_AVAIL_LIST_END", // 25 + "BAD_PREV_BLK_NEXT", // 26 + "BAD_FIRST_ELM_FLAG", // 27 + "BAD_LEM", // 28 + "BAD_ELM_LEN", // 29 + "BAD_ELM_KEY_SIZE", // 30 + "BAD_ELM_KEY", // 31 + "BAD_ELM_KEY_ORDER", // 32 + "BAD_CONT_ELM_KEY", // 34 + "NON_UNIQUE_FIRST_ELM_KEY", // 35 + "BAD_ELM_OFFSET", // 36 + "BAD_ELM_INVALID_LEVEL", // 37 + "BAD_ELM_FLD_NUM", // 38 + "BAD_ELM_FLD_LEN", // 39 + + // WARNING: ANY CHANGES MADE TO THE FlmCorruptStrings TABLE MUST BE + // REFLECTED IN THE CHECK CODE DEFINES FOUND IN flaimsys.h + + "BAD_ELM_FLD_TYPE", // 40 + "BAD_ELM_END", // 41 + "BAD_PARENT_KEY", // 42 + "BAD_ELM_IX_REF", // 43 + "BAD_LAST_BLK_NEXT", // 44 + "REBUILD_REC_EXISTS", // 45 + "REBUILD_KEY_NOT_UNIQUE", // 46 + "NON_UNIQUE_ELM_KEY_REF", // 47 + "OLD_VIEW", // 48 + "COULD_NOT_SYNC_BLK", // 49 + "IX_KEY_NOT_FOUND_IN_REC", // 50 + "KEY_NOT_IN_KEY_REFSET", // 51 + "BAD_BLK_CHECKSUM", // 52 + "BAD_FILE_SIZE", // 53 + "BAD_BLK_TYPE", // 54 + "BAD_ELEMENT_CHAIN", // 55 + "BAD_ELM_EXTRA_DATA", // 56 + "BAD_BLOCK_STRUCTURE", // 57 + "BAD_DATA_BLOCK_COUNT", // 58 + "BAD_AVAIL_SIZE", // 59 + "BAD_CHILD_ELM_COUNT", // 60 +}; + +/**************************************************************************** +Desc: The primary purpose of this function is to provide a way to easily + trap errors when they occur. Just put a breakpoint in this function + to catch them. +Note: Some of the most common errors will be coded so the use can set a + break point. +****************************************************************************/ +#ifdef FLM_DEBUG +RCODE flmMakeErr( + RCODE rc, + const char * pszFile, + int iLine, + FLMBOOL bAssert) +{ + if( bAssert) + { + flmAssert( 0); + } + + return( rc); +} +#endif + +#if defined( FLM_WATCOM_NLM) + int gv_iFlerrorDummy(void) + { + return( 0); + } +#endif + +/**************************************************************************** +Desc: Returns a pointer to the string representation of a corruption + error code. +****************************************************************************/ +const char * F_DbSystem::checkErrorToStr( + eCorruptionType eCorruption) +{ + if (eCorruption < SFLM_NUM_CORRUPT_ERRORS) + { + return( FlmCorruptStrings [(FLMINT)eCorruption]); + } + else + { + return( "Unknown Error"); + } +} + diff --git a/sql/src/flgethdr.cpp b/sql/src/flgethdr.cpp new file mode 100644 index 0000000..9a29b23 --- /dev/null +++ b/sql/src/flgethdr.cpp @@ -0,0 +1,53 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the flmGetHdrInfo routine -- a routine which +// reads the header information from a FLAIM database, verifies it +// and returns the header information in a structure. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flgethdr.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/*************************************************************************** +Desc: This routine reads the header information in a FLAIM database, + verifies the password, and returns the file header and log + header information. +*****************************************************************************/ +RCODE flmGetHdrInfo( + F_SuperFileHdl * pSFileHdl, + SFLM_DB_HDR * pDbHdr, + FLMUINT32 * pui32CalcCRC) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pCFileHdl; + + if( RC_BAD( rc = pSFileHdl->getFileHdl( 0, FALSE, &pCFileHdl))) + { + goto Exit; + } + + rc = flmReadAndVerifyHdrInfo( NULL, pCFileHdl, pDbHdr, pui32CalcCRC); + +Exit: + + return( rc); +} diff --git a/sql/src/flindex.cpp b/sql/src/flindex.cpp new file mode 100644 index 0000000..2abbc6f --- /dev/null +++ b/sql/src/flindex.cpp @@ -0,0 +1,1120 @@ +//------------------------------------------------------------------------------ +// Desc: Contains FLAIM API routines that aid the user in managing indexes. +// +// Tabs: 3 +// +// Copyright (c) 2000-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flindex.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC RCODE flmBackgroundIndexBuildThrd( + IF_Thread * pThread); + +/**************************************************************************** +Desc : Return the status of the index. +Notes: +****************************************************************************/ +RCODE F_Db::indexStatus( + FLMUINT uiIndexNum, + SFLM_INDEX_STATUS * pIndexStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + F_BKGND_IX * pBackgroundIx; + FLMBOOL bMutexLocked = FALSE; + + flmAssert( pIndexStatus); + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + if( m_eTransType != SFLM_NO_TRANS) + { + if( !okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + // Need to have at least a read transaction going. + + if( RC_BAD( rc = beginTrans( SFLM_READ_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + + pBackgroundIx = flmBackgroundIndexGet( m_pDatabase, uiIndexNum, TRUE); + if (pBackgroundIx) + { + f_memcpy( pIndexStatus, &pBackgroundIx->indexStatus, + sizeof( SFLM_INDEX_STATUS)); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + flmAssert( pIndexStatus->uiIndexNum == uiIndexNum); + } + else + { + F_INDEX * pIndex; + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + pIndex = m_pDict->getIndex( uiIndexNum); + + // Populate the index status structure. + + f_memset( pIndexStatus, 0, sizeof( SFLM_INDEX_STATUS)); + pIndexStatus->uiIndexNum = uiIndexNum; + + if( !(pIndex->uiFlags & (IXD_SUSPENDED | IXD_OFFLINE))) + { + pIndexStatus->ui64LastRowIndexed = ~((FLMUINT64)0); + pIndexStatus->eState = SFLM_INDEX_ONLINE; + } + else + { + // NOTE: If we are in a read transaction, the last node indexed + // value may not be read-consistent. It is only guaranteed to + // be correct for update transactions. + + pIndexStatus->ui64LastRowIndexed = pIndex->ui64LastRowIndexed; + pIndexStatus->eState = (pIndex->uiFlags & IXD_SUSPENDED) + ? SFLM_INDEX_SUSPENDED + : SFLM_INDEX_BRINGING_ONLINE; + } + } + +Exit: + + if( bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + if( bStartedTrans) + { + abortTrans(); + } + + return( rc); +} + +/**************************************************************************** +Desc : Suspend the selected index from doing any key updates on records + that are equal or higher than the next record ID value + in the container that the index references. If the index is offline + then the background process will be suspended. If the index is + online then it will be suspended. If the index is already + suspended NE_SFLM_OK will be returned. A suspended index is not + persistent if the database goes down. +Notes: An update transaction will be started if necessary. +****************************************************************************/ +RCODE F_Db::indexSuspend( + FLMUINT uiIndexNum) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + FLMUINT64 ui64HighestRowId; + FLMBOOL bStartedTrans = FALSE; + F_TABLE * pTable; + FLMBOOL bMustAbortOnError = FALSE; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMBOOL bEnableLogging = FALSE; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + else if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + } + else + { + // Need to have an update transaction going. + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + // See if the index is valid + + pIndex = m_pDict->getIndex( uiIndexNum); + + if (pIndex->uiFlags & IXD_SUSPENDED) + { + // Index is already suspended. + + goto Exit; + } + + // If the index is not currently offline, the highest row indexed + // is the tables's last row ID. Otherwise, it is + // simply the value that is stored in the F_INDEX. + + if (!(pIndex->uiFlags & IXD_OFFLINE)) + { + pTable = m_pDict->getTable( pIndex->uiTableNum); + ui64HighestRowId = pTable->lfInfo.ui64NextRowId - 1; + } + else + { + ui64HighestRowId = pIndex->ui64LastRowIndexed; + } + + // Disable RFL logging + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Must abort on error + + bMustAbortOnError = TRUE; + + // There may be a background thread still assigned to the + // index even though the index may be "on-line." This is because + // the background thread may have just commited a transaction that + // transitioned the index from off-line to on-line, but the thread + // has not yet exited (even though it will not do any more work + // to update the index). We want to wait for the thread to terminate + // before our transaction is allowed to commit. This is so that if + // we immediately call resume, it won't find the yet-to-terminate + // thread still running in the background. + + if (!(m_uiFlags & FDB_REPLAYING_RFL)) + { + if( RC_BAD( rc = addToStopList( uiIndexNum))) + { + goto Exit; + } + } + + if (RC_BAD( rc = setIxStateInfo( uiIndexNum, ui64HighestRowId, + IXD_SUSPENDED))) + { + goto Exit; + } + + // setIxStateInfo may have changed to a new dictionary, so pIndex is no + // good after this point + + pIndex = NULL; + + // Log the suspend packet to the RFL + + pRfl->enableLogging(); + + if (RC_BAD( rc = m_pDatabase->m_pRfl->logIndexSuspendOrResume( + this, uiIndexNum, RFL_INDEX_SUSPEND_PACKET))) + { + goto Exit; + } + +Exit: + + if( bEnableLogging) + { + pRfl->enableLogging(); + } + + if( RC_BAD( rc)) + { + if( bStartedTrans) + { + abortTrans(); + } + else if( bMustAbortOnError) + { + setMustAbortTrans( rc); + } + } + else if( bStartedTrans) + { + rc = commitTrans( 0, FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc : If the index was suspended, restart the background process that + will get the index up to date so that it will eventually be online. + Returns NE_SFLM_OK with no change if the index is already online. +Notes: An update transaction will be started if necessary. +****************************************************************************/ +RCODE F_Db::indexResume( + FLMUINT uiIndexNum) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + FLMBOOL bStartedTrans = FALSE; + FLMBOOL bMustAbortOnError = FALSE; + FLMBOOL bEnableLogging = FALSE; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + else if (m_eTransType == SFLM_READ_TRANS) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_TRANS_OP); + goto Exit; + } + } + else + { + // Need to have an update transaction going. + + if (RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + // See if the index is valid + + pIndex = m_pDict->getIndex( uiIndexNum); + + if (!(pIndex->uiFlags & (IXD_SUSPENDED | IXD_OFFLINE))) + { + // Index is already on-line + + goto Exit; + } + + if (!(pIndex->uiFlags & IXD_SUSPENDED)) + { + // Index is not suspended. It is offline (see test + // above), but a thread should already be building the + // index. + + flmAssert( flmBackgroundIndexGet( m_pDatabase, + uiIndexNum, FALSE) != NULL); + + goto Exit; + } + + // Disable RFL logging + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Point of no return -- must abort on error + + bMustAbortOnError = TRUE; + + // Better not have a background thread running + + flmAssert( flmBackgroundIndexGet( m_pDatabase, uiIndexNum, FALSE) == NULL); + + // Update the index state info to show "offline" + + if (RC_BAD( rc = setIxStateInfo( uiIndexNum, pIndex->ui64LastRowIndexed, + IXD_OFFLINE))) + { + goto Exit; + } + + // setIxStateInfo may have changed to a new dictionary, so pIndex is no + // good after this point + + pIndex = NULL; + + // Add an entry to the start list so that an indexing thread + // will be started when this transaction commits. + + if (!(m_uiFlags & FDB_REPLAYING_RFL)) + { + if (RC_BAD( rc = addToStartList( uiIndexNum))) + { + goto Exit; + } + } + + // Log the resume packet to the RFL + + pRfl->enableLogging(); + + if (RC_BAD( rc = pRfl->logIndexSuspendOrResume( + this, uiIndexNum, RFL_INDEX_RESUME_PACKET))) + { + goto Exit; + } + +Exit: + + if( bEnableLogging) + { + pRfl->enableLogging(); + } + + if( RC_BAD( rc)) + { + if( bStartedTrans) + { + abortTrans(); + } + else if( bMustAbortOnError) + { + setMustAbortTrans( rc); + } + } + else if( bStartedTrans) + { + rc = commitTrans( 0, FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc: Add the index to the stop list of background threads. +****************************************************************************/ +RCODE F_Db::addToStopList( + FLMUINT uiIndexNum) +{ + RCODE rc = NE_SFLM_OK; + F_BKGND_IX * pBackgroundIx; + F_BKGND_IX * pNextBackgroundIx; + + // We'd better not be replaying the RFL + + flmAssert( !(m_uiFlags & FDB_REPLAYING_RFL)); + + // First look in the start list and remove any index matches. + // This is need if you add an index and drop + // it within the same transaction. + + for( pBackgroundIx = m_pIxStartList; + pBackgroundIx; pBackgroundIx = pNextBackgroundIx) + { + pNextBackgroundIx = pBackgroundIx->pNext; + + if (pBackgroundIx->indexStatus.uiIndexNum == uiIndexNum) + { + if (pNextBackgroundIx) + { + pNextBackgroundIx->pPrev = pBackgroundIx->pPrev; + } + + if (pBackgroundIx->pPrev) + { + pBackgroundIx->pPrev->pNext = pNextBackgroundIx; + } + else + { + m_pIxStartList = pNextBackgroundIx; + } + + f_free( &pBackgroundIx); + } + } + + // See if we already have an entry in the stop list for the index. There + // is no reason to have the index in the list more than once. + + for (pBackgroundIx = m_pIxStopList; + pBackgroundIx; pBackgroundIx = pNextBackgroundIx) + { + pNextBackgroundIx = pBackgroundIx->pNext; + + if (pBackgroundIx->indexStatus.uiIndexNum == uiIndexNum) + { + goto Exit; // Should return NE_SFLM_OK + } + } + + // Allocate and add the thread structure to the thread list. + + if (RC_BAD( rc = f_calloc( (FLMUINT)( sizeof( F_BKGND_IX)), + &pBackgroundIx))) + { + goto Exit; + } + + pBackgroundIx->indexStatus.uiIndexNum = uiIndexNum; + pBackgroundIx->pPrev = NULL; + if ((pBackgroundIx->pNext = m_pIxStopList) != NULL) + { + m_pIxStopList->pPrev = pBackgroundIx; + } + m_pIxStopList = pBackgroundIx; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add the index to the start list of background threads. +****************************************************************************/ +RCODE F_Db::addToStartList( + FLMUINT uiIndexNum) +{ + RCODE rc = NE_SFLM_OK; + F_BKGND_IX * pBackgroundIx; + F_BKGND_IX * pNextBackgroundIx; + + // We'd better not be replaying the RFL + + flmAssert( !(m_uiFlags & FDB_REPLAYING_RFL)); + + // Look in the start list to make sure we don't already + // have an entry for this index. We don't want to + // start more than one thread per index. The background + // indexing code is not structured to handle multiple build + // threads on the same index. + + // NOTE: We don't want to remove any entries in the stop + // list corresponding to this index. The reason for this + // is the index may have been deleted, re-added, deleted, + // modified, etc. several times during the transaction. + // We want to make sure that an existing background indexing + // thread is terminated and a new one is started. The stop + // list is always processed first at transaction commit time. + // Then new indexing threads (in the start list) are started. + + for( pBackgroundIx = m_pIxStartList; + pBackgroundIx; pBackgroundIx = pNextBackgroundIx) + { + pNextBackgroundIx = pBackgroundIx->pNext; + + if (pBackgroundIx->indexStatus.uiIndexNum == uiIndexNum) + { + goto Exit; // Should return NE_SFLM_OK + } + } + + // Allocate and add the thread structure to the pDb thread list. + + if (RC_BAD( rc = f_calloc( (FLMUINT)( sizeof( F_BKGND_IX)), + &pBackgroundIx))) + { + goto Exit; + } + + pBackgroundIx->indexStatus.uiIndexNum = uiIndexNum; + pBackgroundIx->pPrev = NULL; + if ((pBackgroundIx->pNext = m_pIxStartList) != NULL) + { + m_pIxStartList->pPrev = pBackgroundIx; + } + m_pIxStartList = pBackgroundIx; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: After Abort and before we unlock, stop and start all indexing. +****************************************************************************/ +void F_Db::indexingAfterAbort( void) +{ + F_BKGND_IX * pStartIx; + F_BKGND_IX * pStopIx; + F_BKGND_IX * pNextIx; + + pStopIx = m_pIxStopList; + m_pIxStopList = NULL; + for( ; pStopIx; pStopIx = pNextIx) + { + pNextIx = pStopIx->pNext; + f_free( &pStopIx); + } + + pStartIx = m_pIxStartList; + m_pIxStartList = NULL; + for( ; pStartIx; pStartIx = pNextIx) + { + pNextIx = pStartIx->pNext; + f_free( &pStartIx); + } +} + +/**************************************************************************** +Desc: Stops a background indexing thread +Notes: This routine DOES NOT assume that the global mutex is locked. It + will lock and unlock the mutex as needed. +****************************************************************************/ +void F_Db::stopBackgroundIndexThread( + FLMUINT uiIndexNum, + FLMBOOL bWait, + FLMBOOL * pbStopped) +{ + F_BKGND_IX * pBackgroundIx; + FLMUINT uiThreadId; + FLMBOOL bMutexLocked = FALSE; + + if (pbStopped) + { + *pbStopped = FALSE; + } + + for (;;) + { + // Lock the global mutex + + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + } + + // Get the background index + + if ((pBackgroundIx = flmBackgroundIndexGet( m_pDatabase, + uiIndexNum, TRUE, &uiThreadId)) == NULL) + { + if (pbStopped) + { + *pbStopped = TRUE; + } + goto Exit; + } + + // Set the thread's shutdown flag first. + + gv_SFlmSysData.pThreadMgr->setThreadShutdownFlag( uiThreadId); + + // Unlock the global mutex + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + // The thread may be waiting to start a transaction. + + gv_SFlmSysData.pServerLockMgr->SignalLockWaiter( uiThreadId); + + if( !bWait) + { + break; + } + + // Wait for the thread to terminate + + f_sleep( 50); + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } +} + +/**************************************************************************** +Desc: After commit and before we unlock, stop and start all indexing. +****************************************************************************/ +void F_Db::indexingAfterCommit( void) +{ + F_BKGND_IX * pStartIx; + F_BKGND_IX * pStopIx; + F_BKGND_IX * pNextIx; + FLMBOOL bThreadsActive; + FLMBOOL bStopped; + + // Signal all background indexing threads in the stop list + // to shutdown. Poll until all have terminated. + + for( ;;) + { + bThreadsActive = FALSE; + for( pStopIx = m_pIxStopList; pStopIx; pStopIx = pStopIx->pNext) + { + stopBackgroundIndexThread( pStopIx->indexStatus.uiIndexNum, + FALSE, &bStopped); + if( !bStopped) + { + bThreadsActive = TRUE; + } + } + + if( !bThreadsActive) + { + break; + } + + f_sleep( 50); + } + + // Now that all of the threads have been stopped, discard the stop list + + pStopIx = m_pIxStopList; + m_pIxStopList = NULL; + for (; pStopIx; pStopIx = pNextIx) + { + pNextIx = pStopIx->pNext; + f_free( &pStopIx); + } + + // Start threads listed in the index start list. + + pStartIx = m_pIxStartList; + m_pIxStartList = NULL; + for (; pStartIx; pStartIx = pNextIx) + { + pNextIx = pStartIx->pNext; + (void)startIndexBuild( pStartIx->indexStatus.uiIndexNum); + f_free( &pStartIx); + } +} + +/**************************************************************************** +Desc: Thread that will build an index in the background. +****************************************************************************/ +RCODE F_Db::startIndexBuild( + FLMUINT uiIndexNum) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiGMT; + F_INDEX * pIndex; + F_BKGND_IX * pBackgroundIx = NULL; + char szThreadName[ F_PATH_MAX_SIZE]; + char szBaseName[ F_FILENAME_SIZE]; + + f_timeGetSeconds( &uiGMT ); + + if (flmBackgroundIndexGet( m_pDatabase, uiIndexNum, FALSE) != NULL) + { + // There is already a background thread running on this index. + + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + pIndex = m_pDict->getIndex( uiIndexNum); + + // Allocate the background thread and index status strucutures. + + if (RC_BAD( rc = f_calloc( (FLMUINT)sizeof( F_BKGND_IX), &pBackgroundIx))) + { + goto Exit; + } + + pBackgroundIx->pDatabase = m_pDatabase; + pBackgroundIx->indexStatus.eState = SFLM_INDEX_BRINGING_ONLINE; + pBackgroundIx->indexStatus.uiIndexNum = uiIndexNum; + pBackgroundIx->indexStatus.uiStartTime = uiGMT; + pBackgroundIx->indexStatus.ui64LastRowIndexed = + pIndex->ui64LastRowIndexed; + pBackgroundIx->indexStatus.ui64KeysProcessed = 0; + pBackgroundIx->indexStatus.ui64RowsProcessed = 0; + pBackgroundIx->indexStatus.ui64Transactions = 0; + + pBackgroundIx->uiIndexingAction = FTHREAD_ACTION_INDEX_OFFLINE; + pBackgroundIx->pPrev = NULL; + pBackgroundIx->pNext = NULL; + + // Generate the thread name + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( m_pDatabase->m_pszDbPath, + szThreadName, szBaseName))) + { + goto Exit; + } + + f_sprintf( (char *)szThreadName, "BldIX %u (%s)", + (unsigned)uiIndexNum, szBaseName); + + // Start the thread in the background indexing thread group. + // The new thread will cleanup pBackgroundIx on termination. + + if (RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( NULL, + flmBackgroundIndexBuildThrd, szThreadName, + gv_SFlmSysData.uiIndexingThreadGroup, uiIndexNum, + (void *)pBackgroundIx, NULL, 24000))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc) && pBackgroundIx) + { + f_free( &pBackgroundIx); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine is called by the thread that performs background + indexing. +****************************************************************************/ +RCODE F_Db::backgroundIndexBuild( + IF_Thread * pThread, + FLMBOOL * pbShutdown, + FLMINT * piErrorLine) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + F_BKGND_IX * pBackgroundIx = (F_BKGND_IX *)pThread->getParm1(); + FLMBOOL bStartedTrans = FALSE; + FLMUINT64 ui64FirstRowId; + FLMUINT uiIndexNum; + FLMBOOL bHitEnd; + SFLM_INDEX_STATUS savedIxStatus; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + *piErrorLine = (FLMINT)__LINE__; + goto Exit; + } + + flmAssert( m_eTransType == SFLM_NO_TRANS); + + m_uiFlags |= FDB_BACKGROUND_INDEXING; + uiIndexNum = pBackgroundIx->indexStatus.uiIndexNum; + m_pSFileHdl->enableFlushMinimize(); + + for (;;) + { + + // Set the thread's status + + pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); + + // See if we should shut down. + + if (pThread->getShutdownFlag()) + { + *pbShutdown = TRUE; + break; + } + + // Start a transaction + + if( RC_BAD( rc = beginBackgroundTrans( pThread))) + { + if (rc == NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT) + { + // This would only happen if we were signaled to shut down. + // So, it's ok to exit + + flmAssert( pThread->getShutdownFlag()); + *pbShutdown = TRUE; + rc = NE_SFLM_OK; + } + else + { + *piErrorLine = (FLMINT)__LINE__; + } + goto Exit; + } + bStartedTrans = TRUE; + + if ((pIndex = m_pDict->getIndex( uiIndexNum)) == NULL) + { + rc = RC_SET( NE_SFLM_INVALID_INDEX_NUM); + + // Index may have been deleted by another transaction, or + // there may have been some other error. + + *piErrorLine = (FLMINT)__LINE__; + goto Exit; + } + + // If the index is suspended, this thread should have been + // shut down. The suspending thread will keep the database + // locked until we exit. So, if we have the database locked, + // the index better not be suspended. + + flmAssert( !(pIndex->uiFlags & IXD_SUSPENDED)); + pBackgroundIx->indexStatus.eState = SFLM_INDEX_BRINGING_ONLINE; + + if ((ui64FirstRowId = pIndex->ui64LastRowIndexed) == ~((FLMUINT64)0)) + { + goto Exit; + } + + // Setup the NODE range we want to index. + + ui64FirstRowId++; + + // Set the thread's status + + pThread->setThreadStatus( "Indexing %u:%I64u", + (unsigned)pIndex->uiTableNum, ui64FirstRowId); + + // Read and index up to the highest row ID (or node higher than ui64EndNodeId) + // or until time runs out. The 500 is millisecs to take for the transaction. + + f_memcpy( &savedIxStatus, + &pBackgroundIx->indexStatus, sizeof( SFLM_INDEX_STATUS)); + + if (RC_BAD( rc = indexSetOfRows( uiIndexNum, + ui64FirstRowId, FLM_MAX_UINT64, NULL, NULL, + &pBackgroundIx->indexStatus, &bHitEnd, pThread))) + { + // Lock the mutex while copying the saved index status back to + // the main index status so that someone requesting the index status + // won't see the status while the memcpy is in progress. + + f_mutexLock( gv_SFlmSysData.hShareMutex); + f_memcpy( &pBackgroundIx->indexStatus, + &savedIxStatus, sizeof( SFLM_INDEX_STATUS)); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + *piErrorLine = (FLMINT)__LINE__; + goto Exit; + } + + pThread->setThreadStatus( FLM_THREAD_STATUS_COMMITTING_TRANS); + + // Commit the transaction (even if we didn't do any indexing work). + + bStartedTrans = FALSE; + if (RC_BAD( rc = commitTrans( 0, FALSE))) + { + *piErrorLine = (FLMINT)__LINE__; + goto Exit; + } + pBackgroundIx->indexStatus.ui64Transactions++; + + if (bHitEnd) + { + break; + } + } + +Exit: + + if (bStartedTrans) + { + (void)abortTrans(); + bStartedTrans = FALSE; + } + + return( rc); +} + +/**************************************************************************** +Desc: Thread that will build an index in the background. + Caller will create a pDb to use. This pDb must be + freed at the conclusion of the routine. +****************************************************************************/ +FSTATIC RCODE flmBackgroundIndexBuildThrd( + IF_Thread * pThread) +{ + RCODE rc = NE_SFLM_OK; + F_BKGND_IX * pBackgroundIx = (F_BKGND_IX *)pThread->getParm1(); + F_Db * pDb = NULL; + FLMBOOL bForcedShutdown = FALSE; + FLMUINT uiIndexNum; + char szMsg [128]; + FLMINT iErrorLine = 0; + F_DbSystem dbSystem; + + pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING); + +Loop_Again: + + rc = NE_SFLM_OK; + uiIndexNum = pBackgroundIx->indexStatus.uiIndexNum; + flmAssert( pThread->getThreadAppId() == uiIndexNum); + pDb = NULL; + + // We could loop forever on internalDbOpen errors, + // check if we should exit. + + if( pThread->getShutdownFlag()) + { + bForcedShutdown = TRUE; + goto Exit; + } + + if( RC_BAD( rc = dbSystem.internalDbOpen( + pBackgroundIx->pDatabase, &pDb))) + { + + // If the file is being closed, this is not an error. + + if (pBackgroundIx->pDatabase->getFlags() & DBF_BEING_CLOSED) + { + bForcedShutdown = TRUE; + rc = NE_SFLM_OK; + } + else + { + iErrorLine = (FLMINT)__LINE__; + } + goto Exit; + } + + if (RC_BAD( rc = pDb->backgroundIndexBuild( pThread, &bForcedShutdown, + &iErrorLine))) + { + goto Exit; + } + +Exit: + + pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); + + if (pDb) + { + pDb->Release(); + pDb = NULL; + } + + if (RC_BAD(rc) && !bForcedShutdown) + { + if (rc == NE_SFLM_MEM || rc == NE_FLM_IO_DISK_FULL || + rc == NE_SFLM_MUST_WAIT_CHECKPOINT) + { + // Log the error + + f_sprintf( szMsg, + "Background indexing thread %u (index %u)", + (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); + flmLogError( rc, szMsg, __FILE__, iErrorLine); + + // Sleep a half second and try again. + + f_sleep( 500); + goto Loop_Again; + } + else + { + f_sprintf( szMsg, + "Background indexing thread %u (index %u) -- unrecoverable error.", + (unsigned)pThread->getThreadId(), (unsigned)uiIndexNum); + flmLogError( rc, szMsg, __FILE__, iErrorLine); + } + } + + // Set the thread's app ID to 0, so that it will not + // be found now that the thread is terminating (we don't + // want flmBackgroundIndexGet to find the thread). + + pThread->setThreadAppId( 0); + + // Free the background index structure + + f_mutexLock( gv_SFlmSysData.hShareMutex); + pThread->setParm1( NULL); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + f_free( &pBackgroundIx); + + return( rc); +} + +/**************************************************************************** +Desc: Looks for a background indexing thread that is running with + a matching action and value. +Note: The shared semaphore must be locked on the outside while + calling this routine and accessing anything within the F_BKGND_IX + structure. +****************************************************************************/ +F_BKGND_IX * flmBackgroundIndexGet( + F_Database * pDatabase, + FLMUINT uiIndexNum, + FLMBOOL bMutexLocked, + FLMUINT * puiThreadId) +{ + RCODE rc = NE_SFLM_OK; + IF_Thread * pThread; + FLMUINT uiThreadId; + F_BKGND_IX * pBackgroundIx = NULL; + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + } + + uiThreadId = 0; + for( ;;) + { + if (RC_BAD( rc = gv_SFlmSysData.pThreadMgr->getNextGroupThread( + &pThread, gv_SFlmSysData.uiIndexingThreadGroup, &uiThreadId))) + { + if( rc == NE_SFLM_NOT_FOUND) + { + rc = NE_SFLM_OK; + break; + } + else + { + RC_UNEXPECTED_ASSERT( rc); + } + } + + if (pThread->getThreadAppId()) + { + F_BKGND_IX * pTmpIx = NULL; + + pTmpIx = (F_BKGND_IX *)pThread->getParm1(); + if (pTmpIx->indexStatus.uiIndexNum == uiIndexNum && + pTmpIx->pDatabase == pDatabase) + { + flmAssert( pThread->getThreadAppId() == uiIndexNum); + pBackgroundIx = pTmpIx; + pThread->Release(); + if( puiThreadId) + { + *puiThreadId = uiThreadId; + } + break; + } + } + pThread->Release(); + } + + if (!bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + return( pBackgroundIx); +} diff --git a/sql/src/flkeyret.cpp b/sql/src/flkeyret.cpp new file mode 100644 index 0000000..ba044d6 --- /dev/null +++ b/sql/src/flkeyret.cpp @@ -0,0 +1,293 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the F_Db::keyRetrieve method. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flkeyret.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/***************************************************************************** +Desc: Retrieves a key from an index. +******************************************************************************/ +RCODE F_Db::keyRetrieve( + FLMUINT uiIndex, + F_DataVector * pSearchKey, + FLMUINT uiFlags, + F_DataVector * pFoundKey) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex = NULL; + LFILE * pLFile; + FLMBYTE * pucSearchKey = NULL; + FLMBYTE * pucFoundKey = NULL; + void * pvMark = m_tempPool.poolMark(); + FLMUINT uiSearchKeyLen = 0; + FLMUINT uiFoundKeyLen; + FLMUINT uiOriginalFlags; + F_Btree * pbtree = NULL; + FLMUINT uiDataLen; + FLMBOOL bStartedTrans = FALSE; + FLMUINT uiIdMatchFlags = uiFlags & FLM_MATCH_ROW_ID; + IXKeyCompare compareObject; + FLMBOOL bCompareRowId = FALSE; + + // See if the database is being forced to close + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // If we are not in a transaction, we cannot read. + + if (m_eTransType == SFLM_NO_TRANS) + { + if( RC_BAD( rc = transBegin( SFLM_READ_TRANS))) + { + goto Exit; + } + + bStartedTrans = TRUE; + } + + // See if we have a transaction going which should be aborted. + + if( RC_BAD( m_AbortRc)) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + + // Allocate key buffers. + + if (pSearchKey) + { + if (RC_BAD( rc = m_tempPool.poolAlloc( SFLM_MAX_KEY_SIZE, + (void **)&pucSearchKey))) + { + goto Exit; + } + } + + if (RC_BAD( rc = m_tempPool.poolAlloc( SFLM_MAX_KEY_SIZE, (void **)&pucFoundKey))) + { + goto Exit; + } + + // Make sure it is a valid index definition + + pIndex = m_pDict->getIndex( uiIndex); + + if (RC_BAD( rc = flushKeys())) + { + goto Exit; + } + + if (uiFlags & FLM_FIRST || (!pSearchKey && + !(uiFlags & FLM_FIRST) && !(uiFlags & FLM_LAST))) + { + uiOriginalFlags = uiFlags = FLM_FIRST; + } + else if (uiFlags & FLM_LAST) + { + uiOriginalFlags = uiFlags = FLM_LAST; + } + else + { + uiOriginalFlags = uiFlags; + if (uiFlags & FLM_EXACT) + { + flmAssert( !(uiFlags & FLM_KEY_EXACT)); + uiFlags = FLM_EXACT; + } + else if (uiFlags & FLM_EXCL) + { + uiFlags = FLM_EXCL; + } + else + { + uiFlags = FLM_INCL; + } + + if (RC_BAD( rc = pSearchKey->outputKey( this, uiIndex, uiIdMatchFlags, + pucSearchKey, SFLM_MAX_KEY_SIZE, &uiSearchKeyLen, SEARCH_KEY_FLAG))) + { + goto Exit; + } + + // If we are not matching on the IDs and this is an FLM_EXCL + // search, tack on a 0xFF for the IDs, which should get us past + // all keys that match. We need to turn on the match IDs flags + // in this case so that the comparison routine will match on the + // 0xFF. + + if (!uiIdMatchFlags && (uiFlags & FLM_EXCL)) + { + pucSearchKey [uiSearchKeyLen++] = 0xFF; + bCompareRowId = TRUE; + } + else + { + if (uiIdMatchFlags & FLM_MATCH_ROW_ID) + { + bCompareRowId = TRUE; + } + } + } + + compareObject.setIxInfo( this, pIndex); + compareObject.setCompareRowId( bCompareRowId); + compareObject.setSearchKey( pSearchKey); + + // Get a btree + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbtree))) + { + goto Exit; + } + + if( RC_BAD( rc = pbtree->btOpen( this, pLFile, + (pIndex->uiFlags & IXD_ABS_POS) + ? TRUE + : FALSE, + (pIndex->pDataIcds) + ? TRUE + : FALSE, &compareObject))) + { + goto Exit; + } + + // Search the for the key + + if (uiSearchKeyLen) + { + f_memcpy( pucFoundKey, pucSearchKey, uiSearchKeyLen); + } + uiFoundKeyLen = uiSearchKeyLen; + + if( RC_BAD( rc = pbtree->btLocateEntry( + pucFoundKey, SFLM_MAX_KEY_SIZE, &uiFoundKeyLen, uiFlags, NULL, + &uiDataLen))) + { + if (rc == NE_SFLM_EOF_HIT && uiOriginalFlags & FLM_EXACT) + { + rc = RC_SET( NE_SFLM_NOT_FOUND); + } + goto Exit; + } + + // See if we are in the same key + + if (uiOriginalFlags & FLM_KEY_EXACT) + { + FLMINT iTmpCmp; + + if (RC_BAD( rc = ixKeyCompare( this, pIndex, + pSearchKey, NULL, NULL, + (uiIdMatchFlags == FLM_MATCH_ROW_ID) ? TRUE : FALSE, + pucFoundKey, uiFoundKeyLen, + pucSearchKey, uiSearchKeyLen, &iTmpCmp))) + { + goto Exit; + } + + if (iTmpCmp != 0) + { + rc = (uiOriginalFlags & (FLM_INCL | FLM_EXCL)) + ? RC_SET( NE_SFLM_EOF_HIT) + : RC_SET( NE_SFLM_NOT_FOUND); + goto Exit; + } + } + + // Parse the found key into its individual components + + if (pFoundKey) + { + pFoundKey->reset(); + if (RC_BAD( rc = pFoundKey->inputKey( this, uiIndex, + pucFoundKey, uiFoundKeyLen))) + { + goto Exit; + } + + // See if there is a data part + + if (pIndex->pDataIcds) + { + FLMUINT uiDataBufSize; + FLMBYTE * pucData; + + // If the data will fit in the search key buffer, just + // reuse it since we are not going to do anything with + // it after this. Otherwise, allocate a new buffer. + + if (uiDataLen <= SFLM_MAX_KEY_SIZE && pucSearchKey) + { + uiDataBufSize = SFLM_MAX_KEY_SIZE; + pucData = pucSearchKey; + } + else + { + uiDataBufSize = uiDataLen; + if (RC_BAD( rc = m_tempPool.poolAlloc( uiDataBufSize, + (void **)&pucData))) + { + goto Exit; + } + } + + // Retrieve the data + + if (RC_BAD( rc = pbtree->btGetEntry( + pucFoundKey, SFLM_MAX_KEY_SIZE, uiFoundKeyLen, + pucData, uiDataBufSize, &uiDataLen))) + { + goto Exit; + } + + // Parse the data + + if (RC_BAD( rc = pFoundKey->inputData( this, uiIndex, + pucData, uiDataLen))) + { + goto Exit; + } + } + } + +Exit: + + m_tempPool.poolReset( pvMark); + + if (pbtree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbtree); + } + + if( bStartedTrans) + { + transAbort(); + } + + return( rc); +} diff --git a/sql/src/flmstat.cpp b/sql/src/flmstat.cpp new file mode 100644 index 0000000..4dc1cbf --- /dev/null +++ b/sql/src/flmstat.cpp @@ -0,0 +1,1453 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains routines for updating FLAIM statistics. +// +// Tabs: 3 +// +// Copyright (c) 1997-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flmstat.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC RCODE flmStatGetDbByName( + SFLM_STATS * pFlmStats, + const char * pszDbName, + FLMUINT uiLowStart, + SFLM_DB_STATS ** ppDbStatsRV, + FLMUINT * puiDBAllocSeqRV, + FLMUINT * puiDbTblPosRV); + +FSTATIC void flmUpdateLFileStats( + SFLM_LFILE_STATS * pDest, + SFLM_LFILE_STATS * pSrc); + +FSTATIC RCODE flmUpdateDbStats( + SFLM_DB_STATS * pDest, + SFLM_DB_STATS * pSrc); + +FSTATIC FLMUINT flmDaysInMonth( + FLMUINT uiYear, + FLMUINT uiMonth); + +FSTATIC void flmAdjustTime( + F_TMSTAMP * pTime, + FLMINT iStartPoint); + +/**************************************************************************** +Desc: This routine returns a pointer to a particular database's + statistics block. +****************************************************************************/ +FSTATIC RCODE flmStatGetDbByName( + SFLM_STATS * pFlmStats, + const char * pszDbName, + FLMUINT uiLowStart, + SFLM_DB_STATS ** ppDbStatsRV, + FLMUINT * puiDBAllocSeqRV, + FLMUINT * puiDbTblPosRV) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTblSize; + SFLM_DB_STATS * pDbStatTbl; + FLMUINT uiLow; + FLMUINT uiMid = 0; + FLMUINT uiHigh; + FLMINT iCmp = 0; + FLMUINT uiNewSize; + FLMUINT uiElement; + char * pszTmpDbName = NULL; + + // If there is a database array, search it first. + + if (((pDbStatTbl = pFlmStats->pDbStats) != NULL) && + ((uiTblSize = pFlmStats->uiNumDbStats) != 0)) + { + for (uiHigh = --uiTblSize, uiLow = uiLowStart ; ; ) // Optimize: reduce uiTblSize + { + uiMid = (uiLow + uiHigh) >> 1; // (uiLow + uiHigh) / 2 + + // See if we found a match. + +#ifdef FLM_UNIX + if ((iCmp = f_strcmp( pszDbName, + pDbStatTbl [uiMid].pszDbName)) == 0) +#else + if ((iCmp = f_stricmp( pszDbName, + pDbStatTbl [uiMid].pszDbName)) == 0) +#endif + { + + // Found match. + + *ppDbStatsRV = &pDbStatTbl [uiMid]; + if (puiDBAllocSeqRV) + { + *puiDBAllocSeqRV = pFlmStats->uiDBAllocSeq; + } + if (puiDbTblPosRV) + { + *puiDbTblPosRV = uiMid; + } + goto Exit; + } + + // Check if we are done - where uiLow equals uiHigh. + + if (uiLow >= uiHigh) + { + + // Item not found. + + break; + } + + if (iCmp < 0) + { + if (uiMid == uiLowStart) + { + break; // Way too high? + } + uiHigh = uiMid - 1; // Too high + } + else + { + if (uiMid == uiTblSize) + { + break; // Done - Hit the top + } + uiLow = uiMid + 1; // Too low + } + } + } + + // If the array is full, or was never allocated, allocate a new one. + + if (pFlmStats->uiDbStatArraySize <= pFlmStats->uiNumDbStats) + { + if (!pFlmStats->pDbStats) + { + uiNewSize = INIT_DB_STAT_ARRAY_SIZE; + } + else + { + uiNewSize = pFlmStats->uiDbStatArraySize + + DB_STAT_ARRAY_INCR_SIZE; + } + if (RC_BAD( rc = f_calloc( (FLMUINT)(sizeof( SFLM_DB_STATS) * uiNewSize), + &pDbStatTbl))) + { + goto Exit; + } + + // Save whatever was in the old table, if any. + + if (pFlmStats->pDbStats && pFlmStats->uiNumDbStats) + { + f_memcpy( pDbStatTbl, pFlmStats->pDbStats, + (FLMINT)(sizeof( SFLM_DB_STATS) * pFlmStats->uiNumDbStats)); + } + if (pFlmStats->pDbStats) + { + f_free( &pFlmStats->pDbStats); + } + + pFlmStats->uiDBAllocSeq++; + pFlmStats->pDbStats = pDbStatTbl; + pFlmStats->uiDbStatArraySize = uiNewSize; + } + + // Allocate space for the database name + + if (RC_BAD( rc = f_alloc( f_strlen( pszDbName) + 1, &pszTmpDbName))) + { + goto Exit; + } + + // Insert the item into the array. + + if (iCmp != 0) + { + uiElement = pFlmStats->uiNumDbStats; + + // If our new database number is greater than database number of the + // element pointed to by uiMid, increment uiMid so that the new + // database number will be inserted after it instead of before it. + + if (iCmp > 0) + { + uiMid++; + } + + // Move everything up in the array, including the slot pointed to + // by uiMid. + + while (uiElement > uiMid) + { + f_memcpy( &pDbStatTbl [uiElement], &pDbStatTbl [uiElement - 1], + sizeof( SFLM_DB_STATS)); + uiElement--; + } + f_memset( &pDbStatTbl [uiMid], 0, sizeof( SFLM_DB_STATS)); + } + f_strcpy( pszTmpDbName, pszDbName); + pDbStatTbl [uiMid].pszDbName = pszTmpDbName; + pszTmpDbName = NULL; + pFlmStats->uiNumDbStats++; + *ppDbStatsRV = &pDbStatTbl [uiMid]; + if (puiDBAllocSeqRV) + { + *puiDBAllocSeqRV = pFlmStats->uiDBAllocSeq; + } + if (puiDbTblPosRV) + { + *puiDbTblPosRV = uiMid; + } +Exit: + if (pszTmpDbName) + { + f_free( &pszTmpDbName); + } + return( rc); +} + +/**************************************************************************** +Desc: This routine returns a pointer to a particular database's + statistics block. +****************************************************************************/ +RCODE flmStatGetDb( + SFLM_STATS * pFlmStats, + F_Database * pDatabase, + FLMUINT uiLowStart, + SFLM_DB_STATS ** ppDbStatsRV, + FLMUINT * puiDBAllocSeqRV, + FLMUINT * puiDbTblPosRV) +{ + if (!pFlmStats) + { + *ppDbStatsRV = NULL; + if (puiDBAllocSeqRV) + { + *puiDBAllocSeqRV = 0; + } + if (puiDbTblPosRV) + { + *puiDbTblPosRV = 0; + } + return( NE_SFLM_OK); + } + + return( flmStatGetDbByName( pFlmStats, pDatabase->getDbNamePtr(), + uiLowStart, ppDbStatsRV, puiDBAllocSeqRV, + puiDbTblPosRV)); +} + +/**************************************************************************** +Desc: This routine returns a pointer to a particular logical file in a + particular database's statistics block. +****************************************************************************/ +RCODE flmStatGetLFile( + SFLM_DB_STATS * pDbStats, + FLMUINT uiLFileNum, + eLFileType eLfType, + FLMUINT uiLowStart, + SFLM_LFILE_STATS ** ppLFileStatsRV, + FLMUINT * puiLFileAllocSeqRV, + FLMUINT * puiLFileTblPosRV) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTblSize; + SFLM_LFILE_STATS * pLFileStatTbl; + SFLM_LFILE_STATS * pLFileCurrStat; + FLMUINT uiLow; + FLMUINT uiMid = 0; + FLMUINT uiHigh; + FLMINT iCmp = 0; + FLMUINT uiNewSize; + FLMUINT uiElement; + + if (!pDbStats) + { + *ppLFileStatsRV = NULL; + if (puiLFileAllocSeqRV) + { + *puiLFileAllocSeqRV = 0; + } + if (puiLFileTblPosRV) + { + *puiLFileTblPosRV = 0; + } + goto Exit; + } + + // If there is a database array, search it first. + + if (((pLFileStatTbl = pDbStats->pLFileStats) != NULL) && + ((uiTblSize = pDbStats->uiNumLFileStats) != 0)) + { + for (uiHigh = --uiTblSize, uiLow = uiLowStart ; ; ) // Optimize: reduce uiTblSize + { + uiMid = (uiLow + uiHigh) >> 1; // (uiLow + uiHigh) / 2 + + // See if we found a match. + + pLFileCurrStat = &pLFileStatTbl [uiMid]; + if (uiLFileNum < pLFileCurrStat->uiLFileNum) + { + iCmp = -1; + } + else if (uiLFileNum > pLFileCurrStat->uiLFileNum) + { + iCmp = 1; + } + else if (eLfType < pLFileCurrStat->eLfType) + { + iCmp = -1; + } + else if (eLfType > pLFileCurrStat->eLfType) + { + iCmp = 1; + } + else + { + + // Found match. + + *ppLFileStatsRV = pLFileCurrStat; + if (puiLFileAllocSeqRV) + { + *puiLFileAllocSeqRV = pDbStats->uiLFileAllocSeq; + } + if (puiLFileTblPosRV) + { + *puiLFileTblPosRV = uiMid; + } + goto Exit; + } + + // Check if we are done - where uiLow equals uiHigh. + + if (uiLow >= uiHigh) + { + + // Item not found. + + break; + } + + if (iCmp < 0) + { + if (uiMid == uiLowStart) + { + break; // Way too high? + } + uiHigh = uiMid - 1; // Too high + } + else + { + if (uiMid == uiTblSize) + { + break; // Done - Hit the top + } + uiLow = uiMid + 1; // Too low + } + } + } + + // If the array is full, or was never allocated, allocate a new one. + + if (pDbStats->uiLFileStatArraySize <= pDbStats->uiNumLFileStats) + { + if (!pDbStats->pLFileStats) + { + uiNewSize = INIT_LFILE_STAT_ARRAY_SIZE; + } + else + { + uiNewSize = pDbStats->uiLFileStatArraySize + + LFILE_STAT_ARRAY_INCR_SIZE; + } + if (RC_BAD( rc = f_calloc( (FLMUINT)(sizeof( SFLM_LFILE_STATS) * uiNewSize), + &pLFileStatTbl))) + { + goto Exit; + } + + // Save whatever was in the old table, if any. + + if ((pDbStats->pLFileStats) && + (pDbStats->uiNumLFileStats)) + { + f_memcpy( pLFileStatTbl, pDbStats->pLFileStats, + (FLMUINT)(sizeof( SFLM_LFILE_STATS) * + pDbStats->uiNumLFileStats)); + } + if (pDbStats->pLFileStats) + { + f_free( &pDbStats->pLFileStats); + } + + pDbStats->uiLFileAllocSeq++; + pDbStats->pLFileStats = pLFileStatTbl; + pDbStats->uiLFileStatArraySize = uiNewSize; + } + + // Insert the item into the array. + + if (iCmp != 0) + { + uiElement = pDbStats->uiNumLFileStats; + + // If our new database number is greater than database number of the + // element pointed to by uiMid, increment uiMid so that the new + // database number will be inserted after it instead of before it. + + if (iCmp > 0) + { + uiMid++; + } + + // Move everything up in the array, including the slot pointed to + // by uiMid. + + while (uiElement > uiMid) + { + f_memcpy( &pLFileStatTbl [uiElement], &pLFileStatTbl [uiElement - 1], + sizeof( SFLM_LFILE_STATS)); + uiElement--; + } + f_memset( &pLFileStatTbl [uiMid], 0, sizeof( SFLM_LFILE_STATS)); + } + pLFileStatTbl [uiMid].uiLFileNum = uiLFileNum; + pLFileStatTbl [uiMid].eLfType = eLfType; + pDbStats->uiNumLFileStats++; + *ppLFileStatsRV = &pLFileStatTbl [uiMid]; + if (puiLFileAllocSeqRV) + { + *puiLFileAllocSeqRV = pDbStats->uiLFileAllocSeq; + } + if (puiLFileTblPosRV) + { + *puiLFileTblPosRV = uiMid; + } +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine returns a pointer to a particular LFILE's + statistics block. +****************************************************************************/ +SFLM_LFILE_STATS * F_Db::getLFileStatPtr( + LFILE * pLFile) +{ + if (!pLFile) + { + return( (SFLM_LFILE_STATS *)NULL); + } + + if ((!m_pLFileStats) || + (m_uiLFileAllocSeq != + m_pDbStats->uiLFileAllocSeq) || + (m_pLFileStats->uiLFileNum != pLFile->uiLfNum)) + { + if (RC_BAD( flmStatGetLFile( m_pDbStats, pLFile->uiLfNum, + pLFile->eLfType, 0, &m_pLFileStats, + &m_uiLFileAllocSeq, NULL))) + { + m_pLFileStats = NULL; + m_uiLFileAllocSeq = 0; + } + } + return( m_pLFileStats); +} + +/**************************************************************************** +Desc: This routine resets the statistics in a FLM_STAT structure. +****************************************************************************/ +void flmStatReset( + SFLM_STATS * pStats, + FLMBOOL bFree) +{ + FLMUINT uiDb; + SFLM_DB_STATS * pDbStats; + FLMUINT uiLFile; + SFLM_LFILE_STATS * pLFile; + + if ((pDbStats = pStats->pDbStats) != NULL) + { + for (uiDb = 0; uiDb < pStats->uiNumDbStats; uiDb++, pDbStats++) + { + if ((pLFile = pDbStats->pLFileStats) != NULL) + { + if (bFree) + { + f_free( &pDbStats->pLFileStats); + } + else + { + for (uiLFile = 0; + uiLFile < pDbStats->uiNumLFileStats; + uiLFile++, pLFile++) + { + FLMUINT uiSaveLFileNum = pLFile->uiLFileNum; + eLFileType eSaveLfType = pLFile->eLfType; + + f_memset( pLFile, 0, sizeof( SFLM_LFILE_STATS)); + pLFile->uiLFileNum = uiSaveLFileNum; + pLFile->eLfType = eSaveLfType; + } + } + } + if (!bFree) + { + const char * pszSaveDbName; + FLMUINT uiSaveLFileAllocSeq = pDbStats->uiLFileAllocSeq; + SFLM_LFILE_STATS * pSaveLFileStats = pDbStats->pLFileStats; + FLMUINT uiSaveLFileStatArraySize = + pDbStats->uiLFileStatArraySize; + FLMUINT uiSaveNumLFileStats = pDbStats->uiNumLFileStats; + + pszSaveDbName = pDbStats->pszDbName; + f_memset( pDbStats, 0, sizeof( SFLM_DB_STATS)); + pDbStats->pszDbName = pszSaveDbName; + pDbStats->uiLFileAllocSeq = uiSaveLFileAllocSeq; + pDbStats->pLFileStats = pSaveLFileStats; + pDbStats->uiLFileStatArraySize = uiSaveLFileStatArraySize; + pDbStats->uiNumLFileStats = uiSaveNumLFileStats; + } + else + { + f_free( &pDbStats->pszDbName); + } + } + if (bFree) + { + f_free( &pStats->pDbStats); + } + } + if ((bFree) || (!pDbStats)) + { + pStats->pDbStats = NULL; + pStats->uiDbStatArraySize = 0; + pStats->uiNumDbStats = 0; + } + pStats->uiStartTime = 0; + pStats->uiStopTime = 0; + + if (pStats->bCollectingStats) + { + f_timeGetSeconds( &pStats->uiStartTime); + } +} + +/**************************************************************************** +Desc: This routine updates statistics from one SFLM_RTRANS_STATS structure + into another. +****************************************************************************/ +FINLINE void flmUpdateRTransStats( + SFLM_RTRANS_STATS * pDest, + SFLM_RTRANS_STATS * pSrc) +{ + flmUpdateCountTimeStats( &pDest->CommittedTrans, &pSrc->CommittedTrans); + flmUpdateCountTimeStats( &pDest->AbortedTrans, &pSrc->AbortedTrans); +} + +/**************************************************************************** +Desc: This routine updates statistics from one SFLM_UTRANS_STATS structure + into another. +****************************************************************************/ +FINLINE void flmUpdateUTransStats( + SFLM_UTRANS_STATS * pDest, + SFLM_UTRANS_STATS * pSrc) +{ + flmUpdateCountTimeStats( &pDest->CommittedTrans, &pSrc->CommittedTrans); + flmUpdateCountTimeStats( &pDest->GroupCompletes, &pSrc->GroupCompletes); + pDest->ui64GroupFinished += pSrc->ui64GroupFinished; + flmUpdateCountTimeStats( &pDest->AbortedTrans, &pSrc->AbortedTrans); +} + +/**************************************************************************** +Desc: This routine updates statistics from one SFLM_BLOCKIO_STATS structure + into another. +****************************************************************************/ +void flmUpdateBlockIOStats( + SFLM_BLOCKIO_STATS * pDest, + SFLM_BLOCKIO_STATS * pSrc) +{ + flmUpdateDiskIOStats( &pDest->BlockReads, &pSrc->BlockReads); + flmUpdateDiskIOStats( &pDest->OldViewBlockReads, + &pSrc->OldViewBlockReads); + pDest->uiBlockChkErrs += pSrc->uiBlockChkErrs; + pDest->uiOldViewBlockChkErrs += pSrc->uiOldViewBlockChkErrs; + pDest->uiOldViewErrors += pSrc->uiOldViewErrors; + flmUpdateDiskIOStats( &pDest->BlockWrites, &pSrc->BlockWrites); +} + +/**************************************************************************** +Desc: This routine updates statistics from one SFLM_LFILE_STATS structure + into another. +****************************************************************************/ +FSTATIC void flmUpdateLFileStats( + SFLM_LFILE_STATS * pDest, + SFLM_LFILE_STATS * pSrc) +{ + pDest->bHaveStats = TRUE; + flmUpdateBlockIOStats( &pDest->RootBlockStats, &pSrc->RootBlockStats); + flmUpdateBlockIOStats( &pDest->MiddleBlockStats, &pSrc->MiddleBlockStats); + flmUpdateBlockIOStats( &pDest->LeafBlockStats, &pSrc->LeafBlockStats); + pDest->ui64BlockSplits += pSrc->ui64BlockSplits; + pDest->ui64BlockCombines += pSrc->ui64BlockCombines; +} + +/**************************************************************************** +Desc: This routine updates statistics from one SFLM_DB_STATS structure into + another. +****************************************************************************/ +FSTATIC RCODE flmUpdateDbStats( + SFLM_DB_STATS * pDestDb, + SFLM_DB_STATS * pSrcDb) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiSrcLFile; + FLMUINT uiDestLFile; + SFLM_LFILE_STATS * pDestLFile; + SFLM_LFILE_STATS * pSrcLFile; + FLMUINT uiLowLFileStart; + FLMUINT uiSaveNumLFiles; + + flmUpdateRTransStats( &pDestDb->ReadTransStats, + &pSrcDb->ReadTransStats); + flmUpdateUTransStats( &pDestDb->UpdateTransStats, + &pSrcDb->UpdateTransStats); + pDestDb->bHaveStats = TRUE; + flmUpdateBlockIOStats( &pDestDb->LFHBlockStats, + &pSrcDb->LFHBlockStats); + flmUpdateBlockIOStats( &pDestDb->AvailBlockStats, + &pSrcDb->AvailBlockStats); + flmUpdateDiskIOStats( &pDestDb->DbHdrWrites, + &pSrcDb->DbHdrWrites); + flmUpdateDiskIOStats( &pDestDb->LogBlockWrites, + &pSrcDb->LogBlockWrites); + flmUpdateDiskIOStats( &pDestDb->LogBlockRestores, + &pSrcDb->LogBlockRestores); + flmUpdateDiskIOStats( &pDestDb->LogBlockReads, + &pSrcDb->LogBlockReads); + pDestDb->uiLogBlockChkErrs += pSrcDb->uiLogBlockChkErrs; + pDestDb->uiReadErrors += pSrcDb->uiReadErrors; + pDestDb->uiWriteErrors += pSrcDb->uiWriteErrors; + flmUpdateCountTimeStats( &pDestDb->NoLocks, &pSrcDb->NoLocks); + flmUpdateCountTimeStats( &pDestDb->WaitingForLock, + &pSrcDb->WaitingForLock); + flmUpdateCountTimeStats( &pDestDb->HeldLock, &pSrcDb->HeldLock); + + // Go through the LFILE statistics. + + for (uiDestLFile = 0, uiSrcLFile = 0, uiLowLFileStart = 0, + pSrcLFile = pSrcDb->pLFileStats; + uiSrcLFile < pSrcDb->uiNumLFileStats; + uiSrcLFile++, pSrcLFile++) + { + if (!pSrcLFile->bHaveStats) + { + continue; + } + + // Find or add the store in the destination store array. + + uiSaveNumLFiles = pDestDb->uiNumLFileStats; + if (RC_BAD( rc = flmStatGetLFile( pDestDb, pSrcLFile->uiLFileNum, + pSrcLFile->eLfType, + uiLowLFileStart, &pDestLFile, NULL, + &uiLowLFileStart))) + { + goto Exit; + } + + if (uiLowLFileStart < pDestDb->uiNumLFileStats - 1) + { + uiLowLFileStart++; + } + + // If we created the LFILE, all we have to do is copy the + // LFILE statistics. It will be quicker. + + if (uiSaveNumLFiles != pDestDb->uiNumLFileStats) + { + f_memcpy( pDestLFile, pSrcLFile, sizeof( SFLM_LFILE_STATS)); + } + else + { + + // LFILE was already present, need to go through and + // update the statistics. + + flmUpdateLFileStats( pDestLFile, pSrcLFile); + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine updates statistics from one FLM_STAT structure into + the global statistics. +****************************************************************************/ +RCODE flmStatUpdate( + SFLM_STATS * pSrcStats) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiSrcDb; + FLMUINT uiDestDb; + SFLM_DB_STATS * pDestDb; + SFLM_DB_STATS * pSrcDb; + FLMUINT uiLowDbStart; + + // Do not update the statistics if the source statistics were started + // at an earlier time that the destination statistics start time. + + if (!gv_SFlmSysData.Stats.bCollectingStats || + pSrcStats->uiStartTime < gv_SFlmSysData.Stats.uiStartTime) + { + return( NE_SFLM_OK); + } + + f_mutexLock( gv_SFlmSysData.hStatsMutex); + + // Go through each of the source's databases + + for ( uiDestDb = 0, uiSrcDb = 0, uiLowDbStart = 0, + pSrcDb = pSrcStats->pDbStats; + uiSrcDb < pSrcStats->uiNumDbStats; + uiSrcDb++, pSrcDb++) + { + if (!pSrcDb->bHaveStats) + { + continue; + } + + // Find or add the store in the destination store array. + + if (RC_BAD( rc = flmStatGetDbByName( &gv_SFlmSysData.Stats, + pSrcDb->pszDbName, + uiLowDbStart, &pDestDb, NULL, + &uiLowDbStart))) + { + goto Exit; + } + + if (uiLowDbStart < gv_SFlmSysData.Stats.uiNumDbStats - 1) + { + uiLowDbStart++; + } + + if (RC_BAD( rc = flmUpdateDbStats( pDestDb, pSrcDb))) + { + goto Exit; + } + } + +Exit: + + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + + if (RC_OK( rc)) + { + flmStatReset( pSrcStats, FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine frees however many queries that are over the limit of + the number we can save. +Note: This routine will ALWAYS unlock the query mutex on leaving. It may + be locked when entering. +****************************************************************************/ +void flmFreeSavedQueries( + FLMBOOL bMutexAlreadyLocked) +{ + QUERY_HDR * pQueriesToFree = NULL; + + // Must determine the queries to free inside the mutex lock and then free + // them outside the mutex lock, because freeing a query may cause an + // embedded query to be put into the list again, which will cause the + // mutex to be locked again. + + if (!bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hQueryMutex); + } + while (gv_SFlmSysData.uiQueryCnt > gv_SFlmSysData.uiMaxQueries) + { + gv_SFlmSysData.pOldestQuery = gv_SFlmSysData.pOldestQuery->pPrev; + gv_SFlmSysData.uiQueryCnt--; + } + + // Whatever is found after pOldestQuery should be freed. Unlink + // those from the list and point to them with pQueriesToFree. + + if (!gv_SFlmSysData.pOldestQuery) + { + pQueriesToFree = gv_SFlmSysData.pNewestQuery; + gv_SFlmSysData.pNewestQuery = NULL; + } + else if (gv_SFlmSysData.pOldestQuery->pNext) + { + pQueriesToFree = gv_SFlmSysData.pOldestQuery->pNext; + pQueriesToFree->pPrev = NULL; + gv_SFlmSysData.pOldestQuery->pNext = NULL; + } + f_mutexUnlock( gv_SFlmSysData.hQueryMutex); + + // Now clean up each of the queries in the pQueriesToFree list. + // This can be done outside the mutex lock because we are now + // dealing with a completely local list that no other thread can + // see. Also, the mutex must NOT be locked at this point because + // FlmCursorCleanup may free an embedded query, which will want + // to lock the mutex again. + + while (pQueriesToFree) + { + QUERY_HDR * pQueryHdrToFree = pQueriesToFree; + + pQueriesToFree = pQueriesToFree->pNext; +// VISIT + flmAssert( 0); +// pQueryHdrToFree->pCursor->cleanup(); + f_free( &pQueryHdrToFree); + } +} + +/**************************************************************************** +Desc: This routine saves a query so it can be analyzed later. +****************************************************************************/ +void flmSaveQuery( + F_Query * pQuery) +{ + QUERY_HDR * pQueryHdr = NULL; + FLMBOOL bNeedToCleanup = TRUE; + FLMBOOL bMutexLocked = FALSE; + + // Allocate memory for the new query + + if (RC_BAD( f_calloc( sizeof( QUERY_HDR), &pQueryHdr))) + { + goto Exit; + } + + pQueryHdr->pQuery = pQuery; + + f_mutexLock( gv_SFlmSysData.hQueryMutex); + bMutexLocked = TRUE; + + // uiMaxQueries was originally checked outside of the mutex lock. + // Make sure it is still non-zero. + + if (gv_SFlmSysData.uiMaxQueries) + { + + // Link query to head of list. + + bNeedToCleanup = FALSE; + if ((pQueryHdr->pNext = gv_SFlmSysData.pNewestQuery) != NULL) + { + pQueryHdr->pNext->pPrev = pQueryHdr; + } + else + { + gv_SFlmSysData.pOldestQuery = pQueryHdr; + } + gv_SFlmSysData.pNewestQuery = pQueryHdr; + + if (++gv_SFlmSysData.uiQueryCnt > gv_SFlmSysData.uiMaxQueries) + { + flmFreeSavedQueries( TRUE); + + // flmFreeSavedQueries will always unlock the mutex. + + bMutexLocked = FALSE; + } + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hQueryMutex); + } + + // Must clean up the query if we didn't get it into the list for + // some reason. + + if (bNeedToCleanup) + { + if (pQueryHdr) + { + f_free( &pQueryHdr); + } +// VISIT + flmAssert( 0); +// pCursor->cleanup(); + } +} + +/**************************************************************************** +Desc: This routine copies statistics from one FLM_STAT structure into + another. This is used to retrieve statistics. +****************************************************************************/ +RCODE flmStatCopy( + SFLM_STATS * pDestStats, + SFLM_STATS * pSrcStats) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDb; + SFLM_DB_STATS * pDestDb; + SFLM_DB_STATS * pSrcDb; + FLMUINT uiCount; + FLMUINT uiLoop; + SFLM_DB_STATS * pDbStats; + SFLM_LFILE_STATS * pLFile; + + f_memcpy( pDestStats, pSrcStats, sizeof( SFLM_STATS)); + + // Zero out the database array. We need to do this + // so that if we get an error, we can correctly release memory for + // the destination structure. + + pDestStats->uiNumDbStats = 0; + pDestStats->uiDbStatArraySize = 0; + pDestStats->pDbStats = NULL; + uiCount = 0; + if (pSrcStats->uiNumDbStats) + { + for (uiLoop = 0, pDbStats = pSrcStats->pDbStats; + uiLoop < pSrcStats->uiNumDbStats; + uiLoop++, pDbStats++) + { + if (pDbStats->bHaveStats) + { + uiCount++; + } + } + } + if (uiCount) + { + if (RC_BAD( rc = f_calloc( (FLMUINT)sizeof( SFLM_DB_STATS) * uiCount, + &pDestStats->pDbStats))) + { + goto Exit; + } + for (uiLoop = 0, uiCount = 0, pDbStats = pSrcStats->pDbStats; + uiLoop < pSrcStats->uiNumDbStats; + uiLoop++, pDbStats++) + { + if (pDbStats->bHaveStats) + { + char * pszTmpDbName; + + pDestDb = &pDestStats->pDbStats [uiCount]; + f_memcpy( pDestDb, pDbStats, sizeof( SFLM_DB_STATS)); + + // Allocate space for the database name + + if (RC_BAD( rc = f_alloc( f_strlen( pDbStats->pszDbName) + 1, + &pszTmpDbName))) + { + goto Exit; + } + f_strcpy( pszTmpDbName, pDbStats->pszDbName); + pDestDb->pszDbName = pszTmpDbName; + + // Zero out each store's LFILE statistics. We need to do this + // so that if we get an error, we can correctly release memory for + // the destination structure. + + pDestDb->uiNumLFileStats = 0; + pDestDb->uiLFileStatArraySize = 0; + pDestDb->pLFileStats = NULL; + uiCount++; + } + } + pDestStats->uiDbStatArraySize = + pDestStats->uiNumDbStats = uiCount; + } + + for (uiDb = pSrcStats->uiNumDbStats, + pDestDb = pDestStats->pDbStats, + pSrcDb = pSrcStats->pDbStats; + uiDb; + uiDb--, pSrcDb++) + { + if (!pSrcDb->bHaveStats) + { + continue; + } + + pDestDb->uiNumLFileStats = 0; + pDestDb->uiLFileStatArraySize = 0; + pDestDb->pLFileStats = NULL; + uiCount = 0; + for (uiLoop = 0, pLFile = pSrcDb->pLFileStats; + uiLoop < pSrcDb->uiNumLFileStats; + uiLoop++, pLFile++) + { + if (pLFile->bHaveStats) + { + uiCount++; + } + } + if (uiCount) + { + if (RC_BAD( rc = f_calloc( + (FLMUINT)sizeof( SFLM_LFILE_STATS) * uiCount, + &pDestDb->pLFileStats))) + { + goto Exit; + } + uiCount = 0; + for (uiLoop = 0, pLFile = pSrcDb->pLFileStats; + uiLoop < pSrcDb->uiNumLFileStats; + uiLoop++, pLFile++) + { + if (pLFile->bHaveStats) + { + f_memcpy( &pDestDb->pLFileStats [uiCount], + pLFile, sizeof( SFLM_LFILE_STATS)); + uiCount++; + } + } + pDestDb->uiLFileStatArraySize = + pDestDb->uiNumLFileStats = uiCount; + } + pDestDb++; + } + +Exit: + + if (RC_BAD( rc)) + { + flmStatFree( pDestStats); + } + return( rc); +} + +/**************************************************************************** +Desc: This routine locates the appropriate SFLM_BLOCKIO_STATS structure for + a given block of data. NULL is returned if an appropriate one cannot + be found. +VISIT: uiBlkType passed in is a guess. Remove this parm and start using + pBlk. +****************************************************************************/ +SFLM_BLOCKIO_STATS * flmGetBlockIOStatPtr( + SFLM_DB_STATS * pDbStats, + SFLM_LFILE_STATS * pLFileStats, + FLMBYTE * pucBlk) +{ + F_BLK_HDR * pBlkHdr = (F_BLK_HDR *)pucBlk; + + if (pBlkHdr->ui8BlkType == BT_FREE) + { + pDbStats->bHaveStats = TRUE; + return( &pDbStats->AvailBlockStats); + } + else if (pBlkHdr->ui8BlkType == BT_LFH_BLK) + { + pDbStats->bHaveStats = TRUE; + return( &pDbStats->LFHBlockStats); + } + else if (pLFileStats) + { + pLFileStats->bHaveStats = + pDbStats->bHaveStats = TRUE; + + // VISIT: Consider a one level tree. + // Is it more important to count root stats over leaf stats? + // What about the Data Only Blocks? + + // Consider invalid type. + + if (pBlkHdr->ui8BlkType != BT_LEAF && + pBlkHdr->ui8BlkType != BT_NON_LEAF && + pBlkHdr->ui8BlkType != BT_NON_LEAF_COUNTS && + pBlkHdr->ui8BlkType != BT_LEAF_DATA) + { + return( &pLFileStats->LeafBlockStats); + } + if (pBlkHdr->ui32NextBlkInChain == 0 && + pBlkHdr->ui32PrevBlkInChain == 0) + { + return( &pLFileStats->RootBlockStats); + } + else if ((pBlkHdr->ui8BlkType != BT_LEAF) && + (pBlkHdr->ui8BlkType != BT_LEAF_DATA)) + { + return( &pLFileStats->MiddleBlockStats); + } + else + { + return( &pLFileStats->LeafBlockStats); + } + } + else + { + return( (SFLM_BLOCKIO_STATS *)NULL); + } +} + +/******************************************************************** +Desc: Determine if a given year is a leap year. +*********************************************************************/ +FINLINE FLMUINT flmLeapYear( + FLMUINT uiYear + ) +{ + if (uiYear % 4 != 0) + { + return( 0); + } + if (uiYear % 100 != 0) + { + return( 1); + } + if (uiYear % 400 != 0) + { + return( 0); + } + return( 1); +} + +/******************************************************************** +Desc: Calculate days in a given month of a given year. +*********************************************************************/ +FSTATIC FLMUINT flmDaysInMonth( + FLMUINT uiYear, + FLMUINT uiMonth + ) +{ + switch (uiMonth + 1) + { + case 4: + case 6: + case 9: + case 11: + return( 30); + case 2: + return( 28 + flmLeapYear( uiYear)); + default: + return( 31); + } +} + +/******************************************************************** +Desc: Adjust the time. +*********************************************************************/ +FSTATIC void flmAdjustTime( + F_TMSTAMP * pTime, + FLMINT iStartPoint + ) +{ + switch (iStartPoint) + { + case 1: + goto Adj_1; + case 2: + goto Adj_2; + case 3: + goto Adj_3; + case 4: + goto Adj_4; + case 5: + goto Adj_5; + case 6: + goto Adj_6; + } +Adj_1: + if (pTime->hundredth >= 100) + { + pTime->second++; + pTime->hundredth = 0; + } +Adj_2: + if (pTime->second == 60) + { + pTime->minute++; + pTime->second = 0; + } +Adj_3: + if (pTime->minute == 60) + { + pTime->hour++; + pTime->minute = 0; + } +Adj_4: + if (pTime->hour == 24) + { + pTime->day++; + pTime->hour = 0; + } +Adj_5: + if ((FLMUINT)pTime->day > flmDaysInMonth( pTime->year, pTime->month)) + { + pTime->month++; + pTime->day = 1; + } +Adj_6: + if (pTime->month > 11) + { + pTime->year++; + pTime->month = 1; + } +} + +/******************************************************************** +Desc: Calculate the elapsed time, including milliseconds. +*********************************************************************/ +void flmAddElapTime( + F_TMSTAMP * pStartTime, + FLMUINT64 * pui64ElapMilli + ) +{ + F_TMSTAMP StartTime; + F_TMSTAMP EndTime; + FLMUINT uiSec = 0; + FLMUINT uiHundredth = 0; + + f_timeGetTimeStamp( &EndTime); + f_memcpy( &StartTime, pStartTime, sizeof( F_TMSTAMP)); + + if (StartTime.year < EndTime.year) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + if (StartTime.second) + { + uiSec += (FLMUINT)(60 - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + if (StartTime.minute) + { + uiSec += (FLMUINT)((60 - StartTime.minute) * 60); + StartTime.minute = 0; + StartTime.hour++; + flmAdjustTime( &StartTime, 4); + } + if (StartTime.hour) + { + uiSec += (FLMUINT)((24 - StartTime.hour) * 3600); + StartTime.hour = 0; + StartTime.day++; + flmAdjustTime( &StartTime, 5); + } + if (StartTime.day > 1) + { + uiSec += (FLMUINT)(flmDaysInMonth( StartTime.year, StartTime.month) - + StartTime.day + 1) * (FLMUINT)86400; + StartTime.day = 1; + StartTime.month++; + flmAdjustTime( &StartTime, 6); + } + if (StartTime.month > 1) + { + while (StartTime.month <= 11) + { + uiSec += (FLMUINT)((FLMUINT)flmDaysInMonth( StartTime.year, + StartTime.month) * (FLMUINT)86400); + StartTime.month++; + } + StartTime.year++; + } + while (StartTime.year < EndTime.year) + { + uiSec += (FLMUINT)((FLMUINT)(365 + flmLeapYear( StartTime.year)) * + (FLMUINT)86400); + StartTime.year++; + } + } + + if (StartTime.month < EndTime.month) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + if (StartTime.second) + { + uiSec += (FLMUINT)(60 - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + if (StartTime.minute) + { + uiSec += (FLMUINT)((60 - StartTime.minute) * 60); + StartTime.minute = 0; + StartTime.hour++; + flmAdjustTime( &StartTime, 4); + } + if (StartTime.hour) + { + uiSec += (FLMUINT)((24 - StartTime.hour) * 3600); + StartTime.hour = 0; + StartTime.day++; + flmAdjustTime( &StartTime, 5); + } + if (StartTime.day > 1) + { + uiSec += (FLMUINT)(flmDaysInMonth( StartTime.year, StartTime.month) - + StartTime.day + 1) * (FLMUINT)86400; + StartTime.day = 1; + StartTime.month++; + flmAdjustTime( &StartTime, 6); + } + while (StartTime.month < EndTime.month) + { + uiSec += (FLMUINT)((FLMUINT)flmDaysInMonth( StartTime.year, + StartTime.month) * (FLMUINT)86400); + StartTime.month++; + } + } + + if (StartTime.day < EndTime.day) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + if (StartTime.second) + { + uiSec += (FLMUINT)(60 - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + if (StartTime.minute) + { + uiSec += (FLMUINT)((60 - StartTime.minute) * 60); + StartTime.minute = 0; + StartTime.hour++; + flmAdjustTime( &StartTime, 4); + } + if (StartTime.hour) + { + uiSec += (FLMUINT)((24 - StartTime.hour) * 3600); + StartTime.hour = 0; + StartTime.day++; + flmAdjustTime( &StartTime, 5); + } + uiSec += (FLMUINT)(EndTime.day - StartTime.day) * (FLMUINT)86400; + StartTime.day = 1; + StartTime.month++; + flmAdjustTime( &StartTime, 6); + } + + if (StartTime.hour < EndTime.hour) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + if (StartTime.second) + { + uiSec += (FLMUINT)(60 - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + if (StartTime.minute) + { + uiSec += (FLMUINT)((60 - StartTime.minute) * 60); + StartTime.minute = 0; + StartTime.hour++; + flmAdjustTime( &StartTime, 4); + } + uiSec += (FLMUINT)((EndTime.hour - StartTime.hour) * 3600); + StartTime.hour = 0; + StartTime.day++; + flmAdjustTime( &StartTime, 5); + } + + if (StartTime.minute < EndTime.minute) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + if (StartTime.second) + { + uiSec += (FLMUINT)(60 - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + uiSec += (FLMUINT)((EndTime.minute - StartTime.minute) * 60); + StartTime.minute = 0; + StartTime.hour++; + flmAdjustTime( &StartTime, 4); + } + + if (StartTime.second < EndTime.second) + { + if (StartTime.hundredth) + { + uiHundredth += (FLMUINT)(100 - StartTime.hundredth); + StartTime.hundredth = 0; + StartTime.second++; + flmAdjustTime( &StartTime, 2); + } + uiSec += (FLMUINT)(EndTime.second - StartTime.second); + StartTime.second = 0; + StartTime.minute++; + flmAdjustTime( &StartTime, 3); + } + + if (StartTime.hundredth < EndTime.hundredth) + { + uiHundredth += (FLMUINT)(EndTime.hundredth - StartTime.hundredth); + } + if (uiSec) + { + (*pui64ElapMilli) += (FLMUINT64)((uiHundredth * 10 + uiSec * 1000)); + } + else + { + (*pui64ElapMilli) += (FLMUINT64)(uiHundredth * 10); + } +} diff --git a/sql/src/flmstat.h b/sql/src/flmstat.h new file mode 100644 index 0000000..90255f8 --- /dev/null +++ b/sql/src/flmstat.h @@ -0,0 +1,135 @@ +//------------------------------------------------------------------------------ +// Desc: This include file contains the structure definitions and prototypes +// needed to capture statistics. +// +// Tabs: 3 +// +// Copyright (c) 1997-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flmstat.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FLMSTAT_H +#define FLMSTAT_H + +#define INIT_DB_STAT_ARRAY_SIZE 5 +#define DB_STAT_ARRAY_INCR_SIZE 5 +#define INIT_LFILE_STAT_ARRAY_SIZE 5 +#define LFILE_STAT_ARRAY_INCR_SIZE 5 + +/************************************************************************** + Various function prototypes. +**************************************************************************/ + +RCODE flmStatGetDb( // Source: flmstat.cpp + SFLM_STATS * pFlmStats, + F_Database * pDatabase, + FLMUINT uiLowStart, + SFLM_DB_STATS ** ppDbStatsRV, + FLMUINT * puiDbAllocSeqRV, + FLMUINT * puiDbTblPosRV); + +RCODE flmStatGetLFile( // Source: flmstat.cpp + SFLM_DB_STATS * pDbStats, + FLMUINT uiLFileNum, + eLFileType eLfType, + FLMUINT uiLowStart, + SFLM_LFILE_STATS ** ppLFileStatsRV, + FLMUINT * puiLFileAllocSeqRV, + FLMUINT * puiLFileTblPosRV); + +void flmStatReset( // Source: flmstat.cpp + SFLM_STATS * pStats, + FLMBOOL bFree); + +FINLINE void flmStatStart( + SFLM_STATS * pStats) +{ + pStats->bCollectingStats = TRUE; + flmStatReset( pStats, TRUE); +} + +FINLINE void flmStatStop( + SFLM_STATS * pStats) +{ + if (pStats->bCollectingStats) + { + pStats->bCollectingStats = FALSE; + f_timeGetSeconds( &pStats->uiStopTime); + } +} + +FINLINE void flmStatFree( + SFLM_STATS * pStats) +{ + pStats->bCollectingStats = FALSE; + flmStatReset( pStats, TRUE); +} + +void flmUpdateBlockIOStats( // Source: flmstat.cpp + SFLM_BLOCKIO_STATS * pDest, + SFLM_BLOCKIO_STATS * pSrc); + +RCODE flmStatUpdate( // Source: flmstat.cpp + SFLM_STATS * pSrcStats); + +void flmFreeSavedQueries( // Source: flmstat.cpp + FLMBOOL bMutexAlreadyLocked); + +void flmSaveQuery( // Source: flmstat.cpp + F_Query * pQuery); + +RCODE flmStatCopy( // Source: flmstat.cpp + SFLM_STATS * pDestStats, + SFLM_STATS * pSrcStats); + +SFLM_BLOCKIO_STATS * flmGetBlockIOStatPtr(// Source: flmstat.cpp + SFLM_DB_STATS * pDbStats, + SFLM_LFILE_STATS * pLFileStats, + FLMBYTE * pucBlk); + +void flmAddElapTime( // Source: flmstat.cpp + F_TMSTAMP * pStartTime, + FLMUINT64 * pui64ElapMilli); + +/**************************************************************************** + Inline Functions +****************************************************************************/ + + /* + Desc: This routine updates statistics from one DISKIO_STAT structure into + another. + */ + FINLINE void flmUpdateDiskIOStats( + SFLM_DISKIO_STAT * pDest, + SFLM_DISKIO_STAT * pSrc) + { + pDest->ui64Count += pSrc->ui64Count; + pDest->ui64TotalBytes += pSrc->ui64TotalBytes; + pDest->ui64ElapMilli += pSrc->ui64ElapMilli; + } + + FINLINE void flmUpdateCountTimeStats( + SFLM_COUNT_TIME_STAT * pDest, + SFLM_COUNT_TIME_STAT * pSrc) + { + pDest->ui64Count += pSrc->ui64Count; + pDest->ui64ElapMilli += pSrc->ui64ElapMilli; + } + +#endif // ifdef FLMSTAT_H diff --git a/sql/src/flog.cpp b/sql/src/flog.cpp new file mode 100644 index 0000000..60db788 --- /dev/null +++ b/sql/src/flog.cpp @@ -0,0 +1,112 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for logging messages from within FLAIM. +// +// Tabs: 3 +// +// Copyright (c) 2001-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flog.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Returns an IF_LogMessageClient object if logging is enabled for the + specified message type +****************************************************************************/ +IF_LogMessageClient * flmBeginLogMessage( + eLogMessageType eMsgType) +{ + IF_LogMessageClient * pNewMsg = NULL; + + f_mutexLock( gv_SFlmSysData.hLoggerMutex); + + if( !gv_SFlmSysData.pLogger) + { + goto Exit; + } + + if( (pNewMsg = gv_SFlmSysData.pLogger->beginMessage( eMsgType)) != NULL) + { + gv_SFlmSysData.uiPendingLogMessages++; + } + +Exit: + + f_mutexUnlock( gv_SFlmSysData.hLoggerMutex); + return( pNewMsg); +} + +/**************************************************************************** +Desc: Logs information about an error +****************************************************************************/ +void flmLogError( + RCODE rc, + const char * pszDoing, + const char * pszFileName, + FLMINT iLineNumber) +{ + FLMBYTE * pszMsgBuf = NULL; + IF_LogMessageClient * pLogMsg = NULL; + + if( (pLogMsg = flmBeginLogMessage( SFLM_GENERAL_MESSAGE)) != NULL) + { + if( RC_OK( f_alloc( 512, &pszMsgBuf))) + { + if( pszFileName) + { + f_sprintf( (char *)pszMsgBuf, + "Error %s: %e, File=%s, Line=%d.", + pszDoing, rc, pszFileName, (int)iLineNumber); + } + else + { + f_sprintf( (char *)pszMsgBuf, "Error %s: %e.", pszDoing, rc); + } + + pLogMsg->changeColor( FLM_YELLOW, FLM_BLACK); + pLogMsg->appendString( (char *)pszMsgBuf); + } + flmEndLogMessage( &pLogMsg); + } + + if( pszMsgBuf) + { + f_free( &pszMsgBuf); + } +} + +/**************************************************************************** +Desc: Ends a logging message +****************************************************************************/ +void flmEndLogMessage( + IF_LogMessageClient ** ppLogMessage) +{ + if( *ppLogMessage) + { + f_mutexLock( gv_SFlmSysData.hLoggerMutex); + flmAssert( gv_SFlmSysData.uiPendingLogMessages); + + (*ppLogMessage)->endMessage(); + (*ppLogMessage)->Release(); + *ppLogMessage = NULL; + + gv_SFlmSysData.uiPendingLogMessages--; + f_mutexUnlock( gv_SFlmSysData.hLoggerMutex); + } +} diff --git a/sql/src/flog.h b/sql/src/flog.h new file mode 100644 index 0000000..db9694e --- /dev/null +++ b/sql/src/flog.h @@ -0,0 +1,114 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the logging routines. They use the +// IF_Logger_Client and IF_LogMessage_Client classes. +// +// Tabs: 3 +// +// Copyright (c) 2001-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flog.h 3108 2006-01-19 13:05:19 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FLOG_H +#define FLOG_H + +// Special defines for use in the format string of flmLogPrintf + +#define F_BLACK "%0C" +#define F_BLUE "%1C" +#define F_GREEN "%2C" +#define F_CYAN "%3C" +#define F_RED "%4C" +#define F_PURPLE "%5C" +#define F_BROWN "%6C" +#define F_LIGHTGRAY "%7C" +#define F_DARKGRAY "%8C" +#define F_LIGHTBLUE "%9C" +#define F_LIGHTGREEN "%10C" +#define F_LIGHTCYAN "%11C" +#define F_LIGHTRED "%12C" +#define F_LIGHTPURPLE "%13C" +#define F_YELLOW "%14C" +#define F_WHITE "%15C" + +#define F_PUSHFORECOLOR "%+0C" +#define F_PUSHBACKCOLOR "%+1C" +#define F_POPFORECOLOR "%-0C" +#define F_POPBACKCOLOR "%-1C" + +#define F_PUSHCOLOR F_PUSHFORECOLOR F_PUSHBACKCOLOR +#define F_POPCOLOR F_POPFORECOLOR F_POPBACKCOLOR + +#define F_BLUE_ON_WHITE "%1.15C" + +// Logging functions for use within FLAIM + +IF_LogMessageClient * flmBeginLogMessage( + eLogMessageType eMsgType); + +void flmLogPrintf( + IF_LogMessageClient * pLogMessage, + const char * pszFormatStr, ...); + +void flmLogVPrintf( + IF_LogMessageClient * pLogMessage, + const char * szFormatStr, + f_va_list * args); + +void flmEndLogMessage( + IF_LogMessageClient ** ppLogMessage); + +/*============================================================================ + Debug Logging Functions +============================================================================*/ + +#ifdef FLM_DBG_LOG + + void scaLogWrite( + F_Database * pDatabase, + FLMUINT uiWriteAddress, + FLMBYTE * pucBlkBuf, + FLMUINT uiBufferLen, + FLMUINT uiBlockSize, + char * pszEvent); + + void flmDbgLogWrite( + F_Database * pDatabase, + FLMUINT uiBlkAddress, + FLMUINT uiWriteAddress, + FLMUINT64 ui64TransId, + char * pszEvent); + + void flmDbgLogUpdate( + F_Database * pDatabase, + FLMUINT64 ui64TransId, + FLMUINT uiCollection, + FLMUINT64 ui64NodeId, + RCODE rc, + char * pszEvent); + + void flmDbgLogMsg( + char * pszMsg); + + void flmDbgLogInit( void); + void flmDbgLogExit( void); + void flmDbgLogFlush( void); + +#endif // #ifdef FLM_DBG_LOG + +#endif // #ifndef FLOG_H diff --git a/sql/src/flopen.cpp b/sql/src/flopen.cpp new file mode 100644 index 0000000..e44a638 --- /dev/null +++ b/sql/src/flopen.cpp @@ -0,0 +1,1993 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the F_DbSystem::openDb method. +// +// Tabs: 3 +// +// Copyright (c) 1990-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flopen.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define MAX_DIRTY_PERCENT 70 + +FSTATIC void flmFreeCPInfo( + CP_INFO ** ppCPInfoRV); + +FSTATIC RCODE flmCPThread( + IF_Thread * pThread); + +/*************************************************************************** +Desc: Does most of the actual work of opening an existing database, but + doesn't provide COM interfaces... +****************************************************************************/ +RCODE F_DbSystem::openDb( + const char * pszDbFileName, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + F_Db ** ppDb) +{ + RCODE rc = NE_SFLM_OK; + + *ppDb = NULL; + if (!pszDbFileName || *pszDbFileName == 0) + { + rc = RC_SET( NE_FLM_IO_INVALID_FILENAME); + goto Exit; + } + + // Open the file + + if (RC_BAD( rc = openDatabase( NULL, pszDbFileName, pszDataDir, pszRflDir, + pszPassword, uiOpenFlags, FALSE, NULL, NULL, NULL, ppDb))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Constructor for F_Db object. +****************************************************************************/ +F_Db::F_Db( + FLMBOOL bInternalOpen) +{ + m_pDatabase = NULL; + m_pDict = NULL; + m_pNextForDatabase = NULL; + m_pPrevForDatabase = NULL; + m_pvAppData = NULL; + m_uiThreadId = 0; + m_bMustClose = FALSE; + m_pSFileHdl = NULL; + m_uiFlags = bInternalOpen ? FDB_INTERNAL_OPEN : 0; + m_uiTransCount = 0; + m_eTransType = SFLM_NO_TRANS; + m_AbortRc = NE_SFLM_OK; + m_ui64CurrTransID = 0; + m_uiFirstAvailBlkAddr = 0; + m_uiLogicalEOF = 0; + m_uiUpgradeCPFileNum = 0; + m_uiUpgradeCPOffset = 0; + m_uiTransEOF = 0; + f_memset( &m_TransStartTime, 0, sizeof( m_TransStartTime)); + m_bHadUpdOper = FALSE; + m_uiBlkChangeCnt = 0; + m_pIxdFixups = NULL; + m_pNextReadTrans = NULL; + m_pPrevReadTrans = NULL; + m_uiInactiveTime = 0; + m_uiKilledTime = 0; + m_bItemStateUpdOk = FALSE; + m_pDeleteStatus = NULL; + m_pIxClient = NULL; + m_pIxStatus = NULL; + m_pCommitClient = NULL; + m_pStats = NULL; + m_pDbStats = NULL; + m_pLFileStats = NULL; + m_uiLFileAllocSeq = 0; + f_memset( &m_Stats, 0, sizeof( m_Stats)); + m_bStatsInitialized = TRUE; + m_pIxStartList = NULL; + m_pIxStopList = NULL; + m_pCachedBTree = NULL; + m_pKeyColl = NULL; + m_uiDirtyRowCount = 0; + m_hWaitSem = F_SEM_NULL; + + m_bKrefSetup = FALSE; + m_pKrefTbl = NULL; + m_uiKrefTblSize = 0; + m_uiKrefCount = 0; + m_uiTotalKrefBytes = 0; + m_pucKrefKeyBuf = NULL; + m_pKrefPool = NULL; + m_bReuseKrefPool = FALSE; + m_bKrefCompoundKey = FALSE; + m_pKrefReset = NULL; + + m_tmpKrefPool.poolInit( 8192); + m_tempPool.poolInit( SFLM_MAX_KEY_SIZE * 4); +} + +/*************************************************************************** +Desc: Allocates and initializes an F_Db object for a database which + is to be opened or created. +****************************************************************************/ +RCODE F_DbSystem::allocDb( + F_Db ** ppDb, + FLMBOOL bInternalOpen) +{ + RCODE rc = NE_SFLM_OK; + F_Db * pDb = NULL; + + *ppDb = NULL; + + // Allocate the F_Db object. + + if ((pDb = f_new F_Db( bInternalOpen)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = f_semCreate( &pDb->m_hWaitSem))) + { + goto Exit; + } + + *ppDb = pDb; + pDb = NULL; + +Exit: + + if (pDb) + { + pDb->Release(); + } + return( rc); +} + +/**************************************************************************** +Desc: This routine performs all of the necessary steps to complete + a create or open of a database, including notifying other threads + waiting for the open or create to complete. +NOTE: If RC_BAD( rc), this routine will delete the F_Db object. +****************************************************************************/ +void F_Db::completeOpenOrCreate( + RCODE rc, + FLMBOOL bNewDatabase + ) +{ + if (RC_OK( rc)) + { + + // If this is a newly created F_Database, we need to notify any + // threads waiting for the database to be created or opened that + // the create or open is now complete. + + if (bNewDatabase) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + m_pDatabase->newDatabaseFinish( NE_SFLM_OK); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + } + else + { + F_Database * pDatabase = m_pDatabase; + + // Temporarily increment the open count on the F_Database structure + // so that it will NOT be freed when pDb is freed below. + + if (bNewDatabase) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + pDatabase->m_uiOpenIFDbCount++; + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + // NOTE: Cannot access this F_Db object after this! + // Must do this before potentially deleting the F_Database object + // below, so that the F_Db object will unlink itself from + // the F_Database object. + Release(); + + // If we allocated the F_Database object, notify any + // waiting threads. + + if (bNewDatabase) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + + // Decrement the use count to compensate for the increment + // that occurred above. + + pDatabase->m_uiOpenIFDbCount--; + + // If this is a newly created F_Database, we need to notify any + // threads waiting for the database to be created or opened that + // the create or open is now complete. + + pDatabase->newDatabaseFinish( rc); + pDatabase->freeDatabase(); + f_mutexUnlock ( gv_SFlmSysData.hShareMutex); + } + } +} + +/**************************************************************************** +Desc: Returns the length of the base part of a database name. If the + name ends with a '.' or ".db", this will not be included in the + returned length. +****************************************************************************/ +void F_DbSystem::getDbBasePath( + char * pszBaseDbName, + const char * pszDbName, + FLMUINT * puiBaseDbNameLen) +{ + FLMUINT uiBaseLen = f_strlen( pszDbName); + + if( uiBaseLen <= 3 || + f_stricmp( &pszDbName[ uiBaseLen - 3], ".db") != 0) + { + if( pszDbName[ uiBaseLen - 1] == '.') + { + uiBaseLen--; + } + } + else + { + uiBaseLen -= 3; + } + + f_memcpy( pszBaseDbName, pszDbName, uiBaseLen); + pszBaseDbName[ uiBaseLen] = 0; + + if( puiBaseDbNameLen) + { + *puiBaseDbNameLen = uiBaseLen; + } +} + +/**************************************************************************** +Desc: This routine will open a database, returning a pointer to an F_Db + object that can be used to access it. +****************************************************************************/ +RCODE F_DbSystem::openDatabase( + F_Database * pDatabase, + const char * pszDbPath, + const char * pszDataDir, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + FLMBOOL bInternalOpen, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus, + IF_FileHdl * pLockFileHdl, + F_Db ** ppDb) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bNewDatabase = FALSE; + FLMBOOL bMutexLocked = FALSE; + F_Db * pDb = NULL; + FLMBOOL bNeedToOpen = FALSE; + + // Allocate and initialize an F_Db object. + + if (RC_BAD( rc = allocDb( &pDb, bInternalOpen))) + { + goto Exit; + } + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + + // Free any unused structures that have been unused for the maximum + // amount of time. May unlock and re-lock the global mutex. + + checkNotUsedObjects(); + + // Look up the file using findDatabase to see if we already + // have the file open. + + if (!pDatabase) + { + bNeedToOpen = TRUE; + + // May unlock and re-lock the global mutex. + + if (RC_BAD( rc = findDatabase( pszDbPath, pszDataDir, &pDatabase))) + { + goto Exit; + } + } + + if (pDatabase) + { + if( RC_BAD( rc = pDatabase->checkState( __FILE__, __LINE__))) + { + goto Exit; + } + } + + if (!pDatabase) + { + if (RC_BAD( rc = allocDatabase( pszDbPath, pszDataDir, FALSE, &pDatabase))) + { + goto Exit; + } + flmAssert( !pLockFileHdl); + bNewDatabase = TRUE; + } + else if( pLockFileHdl) + { + flmAssert( pDatabase); + flmAssert( !pDatabase->m_uiOpenIFDbCount); + flmAssert( pDatabase->m_uiFlags & DBF_BEING_OPENED); + + pDatabase->m_pLockFileHdl = pLockFileHdl; + + // Set to NULL to prevent lock file from being released below + + pLockFileHdl = NULL; + + bNewDatabase = TRUE; + bNeedToOpen = TRUE; + } + else + { + FLMBOOL bWaited = FALSE; + flmAssert( !pLockFileHdl); + + if (RC_BAD( rc = pDatabase->verifyOkToUse( &bWaited))) + { + goto Exit; + } + + if (bWaited) + { + bNewDatabase = FALSE; + bNeedToOpen = FALSE; + } + } + + // Link the F_Db object to the F_Database object. + + rc = pDb->linkToDatabase( pDatabase); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + if (RC_BAD(rc)) + { + goto Exit; + } + + (void)flmStatGetDb( &pDb->m_Stats, pDatabase, + 0, &pDb->m_pDbStats, NULL, NULL); + + if (bNeedToOpen) + { + if (RC_BAD( rc = pDatabase->physOpen( + pDb, pszDbPath, pszRflDir, pszPassword, uiOpenFlags, + bNewDatabase, pRestoreObj, pRestoreStatus))) + { + goto Exit; + } + } + + // Start a checkpoint thread + + if( bNewDatabase && !(uiOpenFlags & SFLM_DONT_REDO_LOG)) + { + flmAssert( !pDatabase->m_pCPThrd); + flmAssert( !pDatabase->m_pMaintThrd); + + if( RC_BAD( rc = pDatabase->startCPThread())) + { + goto Exit; + } + + if( !(uiOpenFlags & SFLM_DONT_RESUME_THREADS)) + { + if( RC_BAD( rc = pDb->startBackgroundIndexing())) + { + goto Exit; + } + + if( RC_BAD( rc = pDatabase->startMaintThread())) + { + goto Exit; + } + } + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + if (pLockFileHdl) + { + pLockFileHdl->Release(); + } + + if (pDb) + { + // completeOpenOrCreate will delete pDb if RC_BAD( rc) + + pDb->completeOpenOrCreate( rc, bNewDatabase); + + if (RC_BAD( rc)) + { + pDb = NULL; + } + } + *ppDb = pDb; + return( rc); +} + +/**************************************************************************** +Desc: This routine checks to see if it is OK for another F_Db to use an + F_Database object. + If so, it increments the database's use counter. NOTE: This routine + assumes that the calling routine has locked the global mutex. +****************************************************************************/ +RCODE F_Database::verifyOkToUse( + FLMBOOL * pbWaited) +{ + RCODE rc = NE_SFLM_OK; + F_SEM hWaitSem = F_SEM_NULL; + + // Can't open the database if it is being closed by someone else. + + if (m_uiFlags & DBF_BEING_CLOSED) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } + + // If the file is in the process of being opened by another + // thread, wait for the open to complete. + + if (m_uiFlags & DBF_BEING_OPENED) + { + if( RC_BAD( rc = f_semCreate( &hWaitSem))) + { + goto Exit; + } + + *pbWaited = TRUE; + if (RC_BAD( rc = flmWaitNotifyReq( + gv_SFlmSysData.hShareMutex, hWaitSem, &m_pOpenNotifies, (void *)0))) + { + // If flmWaitNotifyReq returns a bad RC, assume that the other + // thread will unlock and free the F_Database object. This + // routine should only unlock the object if an error occurs at + // some other point. + + goto Exit; + } + } + else + { + *pbWaited = FALSE; + } + +Exit: + + if( hWaitSem != F_SEM_NULL) + { + f_semDestroy( &hWaitSem); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine obtains exclusive access to a database by creating + a .lck file. FLAIM holds the .lck file open as long as the database + is open. When the database is finally closed, it deletes the .lck + file. This is only used for 3.x databases. +****************************************************************************/ +RCODE flmCreateLckFile( + const char * pszFilePath, + IF_FileHdl ** ppLockFileHdlRV) +{ + RCODE rc = NE_SFLM_OK; + char szLockPath [F_PATH_MAX_SIZE]; + char szDbBaseName [F_FILENAME_SIZE]; + char * pszFileExt; + IF_FileHdl * pLockFileHdl = NULL; + char szFilePathStr[ F_PATH_MAX_SIZE]; + FLMUINT uiIoFlags; + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszFilePath, szFilePathStr))) + { + goto Exit; + } + + // Extract the 8.3 name and put a .lck extension on it to create + // the full path for the .lck file. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + szFilePathStr, szLockPath, szDbBaseName))) + { + goto Exit; + } + pszFileExt = &szDbBaseName [0]; + while ((*pszFileExt) && (*pszFileExt != '.')) + pszFileExt++; + f_strcpy( pszFileExt, ".lck"); + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + szLockPath, szDbBaseName))) + { + goto Exit; + } + + // Attempt to create the lock file. If that succeeds, we are + // OK to use the database. If it fails, the lock file may have + // been left because of a crash if FLAIM was not shut down properly. + // Hence, we first try to delete the file. If that succeeds, we + // then attempt to create the file again. If it, or the 2nd create + // fail, we simply return an access denied error. + +uiIoFlags = FLM_IO_RDWR | FLM_IO_EXCL | FLM_IO_SH_DENYRW; +#ifndef FLM_UNIX + uiIoFlags |= FLM_IO_DELETE_ON_RELEASE; +#endif + + if( RC_BAD( gv_SFlmSysData.pFileSystem->createFile( + szLockPath, uiIoFlags, &pLockFileHdl))) + { +#ifndef FLM_UNIX + if (RC_BAD( gv_SFlmSysData.pFileSystem->deleteFile( szLockPath))) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } + else if (RC_BAD( gv_SFlmSysData.pFileSystem->createFile( + szLockPath, uiIoFlags, &pLockFileHdl))) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } +#else + + if( RC_BAD( gv_SFlmSysData.pFileSystem->openFile( + szLockPath, FLM_IO_RDWR | FLM_IO_SH_DENYRW, &pLockFileHdl))) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } +#endif + } + +#ifdef FLM_UNIX + if( RC_BAD( pLockFileHdl->lock())) + { + rc = RC_SET( NE_FLM_IO_ACCESS_DENIED); + goto Exit; + } +#endif + + *ppLockFileHdlRV = (IF_FileHdl *)pLockFileHdl; + pLockFileHdl = NULL; +Exit: + if (pLockFileHdl) + { + (void)pLockFileHdl->close(); + pLockFileHdl->Release(); + pLockFileHdl = NULL; + } + return( rc); +} + +/**************************************************************************** +Desc: This routine obtains exclusive access to a database by creating + a .lck file. FLAIM holds the .lck file open as long as the database + is open. When the database is finally closed, it deletes the .lck + file. This is only used for 3.x databases. +****************************************************************************/ +RCODE F_Database::getExclAccess( + const char * pszFilePath) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bNotifyWaiters = FALSE; + FLMBOOL bMutexLocked = FALSE; + F_SEM hWaitSem = F_SEM_NULL; + + // If m_pLockFileHdl is non-NULL, it means that we currently + // have the database locked with a lock file. There is no need to make + // this test inside a mutex lock, because the lock file handle can only + // be set to NULL when the use count goes to zero, meaning that the thread + // that sets it to NULL will be the only thread accessing it. + + // However, it is possible that two or more threads will simultaneously + // test m_pLockFileHdl and discover that it is NULL. In that case, + // we allow one thread to proceed and attempt to get a lock on the database + // while the other threads wait to be notified of the results of the + // attempt to lock the database. + + if (m_pLockFileHdl) + { + goto Exit; + } + + lockMutex(); + bMutexLocked = TRUE; + + if (m_bBeingLocked) + { + // If the database is in the process of being locked by another + // thread, wait for the lock to complete. NOTE: flmWaitNotifyReq will + // re-lock the mutex before returning. + + if( RC_BAD( rc = f_semCreate( &hWaitSem))) + { + goto Exit; + } + + rc = flmWaitNotifyReq( m_hMutex, hWaitSem, &m_pLockNotifies, (void *)0); + goto Exit; + } + + // No other thread was attempting to lock the database, so + // set this thread up to make the attempt. Other threads + // coming in at this point will be required to wait and + // be notified of the results. + + m_bBeingLocked = TRUE; + bNotifyWaiters = TRUE; + unlockMutex(); + bMutexLocked = FALSE; + if (RC_BAD( rc = flmCreateLckFile( pszFilePath, &m_pLockFileHdl))) + { + goto Exit; + } + +Exit: + + if (bNotifyWaiters) + { + FNOTIFY * pNotify; + F_SEM hSem; + + // Notify any thread waiting on the lock what its status is. + + if( !bMutexLocked) + { + lockMutex(); + bMutexLocked = TRUE; + } + + pNotify = m_pLockNotifies; + while (pNotify) + { + *(pNotify->pRc) = rc; + hSem = pNotify->hSem; + pNotify = pNotify->pNext; + f_semSignal( hSem); + } + + m_bBeingLocked = FALSE; + m_pLockNotifies = NULL; + unlockMutex(); + bMutexLocked = FALSE; + } + + if( bMutexLocked) + { + unlockMutex(); + } + + if( hWaitSem != F_SEM_NULL) + { + f_semDestroy( &hWaitSem); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine checks to see if it is OK for another FDB to use a file. + If so, it increments the file's use counter. NOTE: This routine + assumes that the global mutex is NOT locked. +****************************************************************************/ +RCODE F_Database::physOpen( + F_Db * pDb, + const char * pszFilePath, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + FLMBOOL bNewDatabase, // Is this a new F_Database object? + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus) +{ + RCODE rc = NE_SFLM_OK; + + // See if we need to read in the database header. If the database was + // already open (bNewDatabase == FALSE), we don't need to again. + + if (bNewDatabase) + { + + // Read in the database header. + + if (RC_BAD( rc = readDbHdr( pDb->m_pDbStats, pDb->m_pSFileHdl, + (FLMBYTE *)pszPassword, + (uiOpenFlags & SFLM_ALLOW_LIMITED_MODE) ? TRUE : FALSE))) + { + goto Exit; + } + + // Allocate the pRfl object. Could not do this until this point + // because we need to have the version number, block size, etc. + // setup in the database header. + + flmAssert( !m_pRfl); + + if ((m_pRfl = f_new F_Rfl) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = m_pRfl->setup( this, pszRflDir))) + { + goto Exit; + } + } + + // We must have exclusive access. Create a lock file for that + // purpose, if there is not already a lock file. + + if (!m_pLockFileHdl) + { + if (RC_BAD( rc = getExclAccess( pszFilePath))) + { + goto Exit; + } + } + + // Do a recovery to ensure a consistent database + // state before going any further. The FO_DONT_REDO_LOG + // flag is used ONLY by the VIEW program. + + if (bNewDatabase && !(uiOpenFlags & SFLM_DONT_REDO_LOG)) + { + if (RC_BAD( rc = doRecover( pDb, pRestoreObj, pRestoreStatus))) + { + goto Exit; + } + } + +Exit: + + if (RC_BAD( rc)) + { + (void)pDb->m_pSFileHdl->releaseFiles( TRUE); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine finishes up after creating a new F_Database object. It + will notify any other threads waiting for the operation to complete + of the status of the operation. +****************************************************************************/ +void F_Database::newDatabaseFinish( + RCODE OpenRc) // Return code to send to other threads that are + // waiting for the open to complete. +{ + FNOTIFY * pNotify; + F_SEM hSem; + + // Notify anyone waiting on the operation what its status is. + + pNotify = m_pOpenNotifies; + while (pNotify) + { + *(pNotify->pRc) = OpenRc; + hSem = pNotify->hSem; + pNotify = pNotify->pNext; + f_semSignal( hSem); + } + + m_pOpenNotifies = NULL; + m_uiFlags &= (~(DBF_BEING_OPENED)); +} + +/**************************************************************************** +Desc: This routine is used to see if a file is already in use somewhere. + This is only called for files which are opened directly by the + application. +Notes: This routine assumes that the global mutex is locked, but it + may unlock and re-lock the mutex if needed. +****************************************************************************/ +RCODE F_DbSystem::findDatabase( + const char * pszDbPath, + const char * pszDataDir, + F_Database ** ppDatabase) +{ + RCODE rc = NE_SFLM_OK; + FBUCKET * pBucket; + FLMUINT uiBucket; + FLMBOOL bMutexLocked = TRUE; + F_Database * pDatabase; + char szDbPathStr1 [F_PATH_MAX_SIZE]; + char szDbPathStr2 [F_PATH_MAX_SIZE]; + F_SEM hWaitSem = F_SEM_NULL; + + *ppDatabase = NULL; + + // Normalize the path to a string before looking for it. + // NOTE: On non-UNIX, non-WIN platforms, this will basically convert + // the string to upper case. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszDbPath, szDbPathStr1))) + { + goto Exit; + } + +Retry: + + *ppDatabase = NULL; + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + } + + pBucket = gv_SFlmSysData.pDatabaseHashTbl; + uiBucket = flmStrHashBucket( szDbPathStr1, pBucket, FILE_HASH_ENTRIES); + pDatabase = (F_Database *)pBucket [uiBucket].pFirstInBucket; + while (pDatabase) + { + // Compare the strings. On non-Unix platforms we must use + // f_stricmp, because case does not matter for file names + // on those platforms. + +#ifdef FLM_UNIX + if( f_strcmp( szDbPathStr1, pDatabase->m_pszDbPath) == 0) +#else + if( f_stricmp( szDbPathStr1, pDatabase->m_pszDbPath) == 0) +#endif + { + + // Make sure data paths match. + + if (pszDataDir && *pszDataDir) + { + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszDataDir, szDbPathStr2))) + { + goto Exit; + } + + if (pDatabase->m_pszDataDir) + { + // f_stricmp must be used on non-unix platforms because file + // names are case insensitive on those platforms. +#ifdef FLM_UNIX + if (f_strcmp( pDatabase->m_pszDataDir, szDbPathStr2) != 0) +#else + if (f_stricmp( pDatabase->m_pszDataDir, szDbPathStr2) != 0) +#endif + { + rc = RC_SET( NE_SFLM_DATA_PATH_MISMATCH); + goto Exit; + } + } + else + { + rc = RC_SET( NE_SFLM_DATA_PATH_MISMATCH); + goto Exit; + } + } + else if (pDatabase->m_pszDataDir) + { + rc = RC_SET( NE_SFLM_DATA_PATH_MISMATCH); + goto Exit; + } + *ppDatabase = pDatabase; + break; + } + pDatabase = pDatabase->m_pNext; + } + + if (*ppDatabase && pDatabase->m_uiFlags & DBF_BEING_CLOSED) + { + if( RC_BAD( rc = f_semCreate( &hWaitSem))) + { + goto Exit; + } + + // Put ourselves into the notify list and then re-try + // the lookup when we wake up + + if (RC_BAD( rc = flmWaitNotifyReq( gv_SFlmSysData.hShareMutex, hWaitSem, + &pDatabase->m_pCloseNotifies, (void *)0))) + { + goto Exit; + } + + f_semDestroy( &hWaitSem); + + // The mutex will be locked at this point. Re-try the lookup. + // IMPORTANT NOTE: pDatabase will have been destroyed by this + // time. DO NOT use it for anything! + + goto Retry; + } + +Exit: + + if( hWaitSem != F_SEM_NULL) + { + f_semDestroy( &hWaitSem); + } + + // Make sure the global mutex is re-locked before exiting + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + } + + + return( rc); +} + +/**************************************************************************** +Desc: Make sure a database is NOT open. If it is, return an error. +****************************************************************************/ +RCODE F_DbSystem::checkDatabaseClosed( + const char * pszDbName, + const char * pszDataDir) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase; + + f_mutexLock( gv_SFlmSysData.hShareMutex); + rc = findDatabase( pszDbName, pszDataDir, &pDatabase); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + if (RC_BAD( rc)) + { + goto Exit; + } + if (pDatabase) + { + rc = RC_SET( NE_SFLM_DATABASE_OPEN); + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Constructor for F_Database object. +****************************************************************************/ +F_Database::F_Database( + FLMBOOL bTempDb) +{ + m_krefPool.poolInit( DEFAULT_KREF_POOL_BLOCK_SIZE * 8); + m_pNext = NULL; + m_pPrev = NULL; + m_pFirstSQLQuery = NULL; + m_pLastSQLQuery = NULL; + m_uiBlockSize = 0; + m_uiDefaultLanguage = 0; + m_uiMaxFileSize = 0; + m_uiOpenIFDbCount = 0; + m_bTempDb = bTempDb; + m_pFirstDb = NULL; + m_pszDbPath = NULL; + m_pszDataDir = NULL; + m_pSCacheList = NULL; + m_pFirstRow = NULL; + m_pLastRow = NULL; + m_pLastDirtyRow = NULL; + m_pPendingWriteList = NULL; + m_pLastDirtyBlk = NULL; + m_pFirstInLogList = NULL; + m_pLastInLogList = NULL; + m_uiLogListCount = 0; + m_pFirstInNewList = NULL; + m_pLastInNewList = NULL; + m_uiNewCount = 0; + m_uiDirtyCacheCount = 0; + m_uiLogCacheCount = 0; + m_ppBlocksDone = NULL; + m_uiBlocksDoneArraySize = 0; + m_uiBlocksDone = 0; + m_pTransLogList = NULL; + m_pOpenNotifies = NULL; + m_pCloseNotifies = NULL; + m_pDictList = NULL; + m_bMustClose = FALSE; + m_rcMustClose = NE_SFLM_OK; + m_uiSigBitsInBlkSize = 0; + if (!bTempDb) + { + m_uiFileExtendSize = SFLM_DEFAULT_FILE_EXTEND_SIZE; + } + else + { + m_uiFileExtendSize = 65536; + } + + m_pRfl = NULL; + + f_memset( &m_lastCommittedDbHdr, 0, sizeof( m_lastCommittedDbHdr)); + f_memset( &m_checkpointDbHdr, 0, sizeof( m_checkpointDbHdr)); + f_memset( &m_uncommittedDbHdr, 0, sizeof( m_uncommittedDbHdr)); + + m_pBufferMgr = NULL; + m_pCurrLogBuffer = NULL; + m_uiCurrLogWriteOffset = 0; + m_uiCurrLogBlkAddr = 0; + m_pDbHdrWriteBuf = NULL; + m_pucUpdBuffer = NULL; + m_uiUpdBufferSize = 0; + m_uiUpdByteCount = 0; + m_uiUpdCharCount = 0; + m_ePendingDataType = SFLM_UNKNOWN_TYPE; + m_pPendingBTree = NULL; + m_pucBTreeTmpBlk = NULL; + m_pucBTreeTmpDefragBlk = NULL; + m_pucEntryArray = NULL; + m_pucSortedArray = NULL; + m_pucBtreeBuffer = NULL; + m_pucReplaceStruct = NULL; + m_pDatabaseLockObj = NULL; + m_pWriteLockObj = NULL; + m_pLockFileHdl = NULL; + m_pLockNotifies = NULL; + m_bBeingLocked = FALSE; + m_pFirstReadTrans = NULL; + m_pLastReadTrans = NULL; + m_pFirstKilledTrans = NULL; + m_uiFirstLogBlkAddress = 0; + m_uiFirstLogCPBlkAddress = 0; + m_uiLastCheckpointTime = 0; + m_pCPThrd = NULL; + m_pCPInfo = NULL; + m_CheckpointRc = NE_SFLM_OK; + m_uiBucket = 0; + m_uiFlags = 0; + m_bBackupActive = FALSE; + m_pMaintThrd = NULL; + m_hMaintSem = F_SEM_NULL; + m_bAllowLimitedMode = FALSE; + m_bInLimitedMode = FALSE; + m_pszDbPasswd = NULL; + m_pWrappingKey = NULL; + m_bHaveEncKey = FALSE; + m_rcLimitedCode = NE_SFLM_OK; + m_hMutex = F_MUTEX_NULL; +} + +/**************************************************************************** +Desc: This destructor frees all of the structures associated with an + F_Database object. + Whoever called this routine has already determined that it is safe + to do so. +Notes: The global mutex is assumed to be locked when entering the + routine. It may be unlocked and re-locked before the routine + exits, however. +****************************************************************************/ +F_Database::~F_Database() +{ + FNOTIFY * pCloseNotifies; + F_Dict * pDict; + F_Dict * pTmpDict; + + // At this point, the use count better be zero + + flmAssert( !m_uiOpenIFDbCount); + + // Shut down all background threads before shutting down the CP thread. + + shutdownDatabaseThreads(); + + if (m_pRfl) + { + m_pRfl->closeFile(); + } + + // At this point, the use count better be zero + + flmAssert( !m_uiOpenIFDbCount); + + // Unlock the mutex + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + + // Shut down the checkpoint thread + + if( m_pCPThrd) + { + m_pCPThrd->stopThread(); + m_pCPThrd->Release(); + m_pCPThrd = NULL; + } + + // Unlink all of the F_Dict objects that are connected to the + // database. + + lockMutex(); + while (m_pDictList) + { + m_pDictList->unlinkFromDatabase(); + } + unlockMutex(); + + // Take the file out of its name hash bucket, if any. + + if (m_uiBucket != 0xFFFF) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + if (m_pPrev) + { + m_pPrev->m_pNext = m_pNext; + } + else + { + gv_SFlmSysData.pDatabaseHashTbl[ m_uiBucket].pFirstInBucket = m_pNext; + } + + if (m_pNext) + { + m_pNext->m_pPrev = m_pPrev; + } + m_uiBucket = 0xFFFF; + + // After this point, we should not need to keep the global mutex locked + // because the F_Database is no longer visible to any thread to find in + // the hash table. + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + + // Shouldn't have any queries at this point. But we will be nice in case + // we do and will unlink the queries from the list + + flmAssert( !m_pFirstSQLQuery); + while (m_pFirstSQLQuery) + { + SQLQuery * pQuery = m_pFirstSQLQuery; + + m_pFirstSQLQuery = m_pFirstSQLQuery->m_pNext; + pQuery->m_pPrev = NULL; + pQuery->m_pNext = NULL; + pQuery->m_pDatabase = NULL; + } + + // Free the RFL data, if any. + + if (m_pRfl) + { + m_pRfl->Release(); + m_pRfl = NULL; + } + + flmAssert( m_pOpenNotifies == NULL); + m_pOpenNotifies = NULL; + + // Save pCloseNotifies -- we will notify any waiters once the + // F_Database has been freed. + + pCloseNotifies = m_pCloseNotifies; + + // Free any dictionary usage structures associated with the database. + + pDict = m_pDictList; + while (pDict) + { + pTmpDict = pDict; + pDict = pDict->getNext(); + pTmpDict->Release(); + } + m_pDictList = NULL; + + // Free any shared cache associated with the database. + // IMPORTANT NOTE: + // Cannot have the global mutex locked when these are called because + // these routines lock the block cache mutex and the node cache mutex. + // If both the global mutex and the block or node cache mutexes are to be + // locked, the rule is that the block or node cache mutex must be locked + // before locking the global mutex. This is because neededByReadTrans + // will end up doing it in this order - when neededByReadTrans is called + // either the block or node cache mutex is already locked, and it will + // additionally lock the global mutex. Since that order is already + // required, we cannot have anyone else attempting to lock the mutexes + // in a different order. + + freeBlockCache(); + freeRowCache(); + + // Release the lock objects. + + if (m_pWriteLockObj) + { + m_pWriteLockObj->Release( FALSE); + m_pWriteLockObj = NULL; + } + + if (m_pDatabaseLockObj) + { + m_pDatabaseLockObj->Release(); + m_pDatabaseLockObj = NULL; + } + + // Close and delete the lock file. + + if (m_pLockFileHdl) + { + (void)m_pLockFileHdl->close(); + m_pLockFileHdl->Release(); + m_pLockFileHdl = NULL; + } + + // Free the write buffer managers. + + if (m_pBufferMgr) + { + m_pBufferMgr->Release(); + m_pBufferMgr = NULL; + } + + // Free the log header write buffer + + if (m_pDbHdrWriteBuf) + { + f_freeAlignedBuffer( (void **)&m_pDbHdrWriteBuf); + } + + // Free the update buffer + + if (m_pucUpdBuffer) + { + f_free( &m_pucUpdBuffer); + m_uiUpdBufferSize = 0; + } + + m_krefPool.poolFree(); + + if (m_ppBlocksDone) + { + f_free( &m_ppBlocksDone); + m_uiBlocksDoneArraySize = 0; + } + + // Notify waiters that the F_Database is gone + + while (pCloseNotifies) + { + F_SEM hSem; + + *(pCloseNotifies->pRc) = NE_SFLM_OK; + hSem = pCloseNotifies->hSem; + pCloseNotifies = pCloseNotifies->pNext; + f_semSignal( hSem); + } + + f_free( &m_pszDbPath); + + // Encryption stuff + if (m_pszDbPasswd) + { + f_free( &m_pszDbPasswd); + } + if (m_pWrappingKey) + { + delete m_pWrappingKey; + } + + flmAssert( !m_pFirstRow && !m_pLastRow && !m_pLastDirtyRow); + + if (m_hMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hMutex); + } + + // Global mutex is still expected to be locked at this point + + f_mutexLock( gv_SFlmSysData.hShareMutex); +} + +/**************************************************************************** +Desc: This frees an F_Database object. +Note: The global mutex is assumed to be locked when entering the + routine. It may be unlocked and re-locked during the destructor, + however. For this reason, this routine should be called instead of + directly deleting a database object - i.e., delete pDatabase. +****************************************************************************/ +void F_Database::freeDatabase( void) +{ + + // See if another thread is in the process of freeing + // this F_Database. It is possible for this to happen, since + // the monitor thread may have selected this F_Database to be + // freed because it has been unused for a period of time. + // At the same time, a foreground thread could have called + // FlmConfig to close all unused F_Databases. Since the + // destructor for the F_Database object + // may unlock and re-lock the mutex, there is a small window + // of opportunity for both threads to try to free the same + // F_Database. -- Therefore, we must do this check while the + // mutex is still locked. + + if (m_uiFlags & DBF_BEING_CLOSED) + { + return; + } + + // Set the DBF_BEING_CLOSED flag + + m_uiFlags |= DBF_BEING_CLOSED; + Release(); +} + +/**************************************************************************** +Desc: This routine sets up a new F_Database object, allocating member + variables, linking into lists, etc. + NOTE: This routine assumes that the global mutex has already + been locked. It may unlock it temporarily if there is an error, + but will always relock it before exiting. +****************************************************************************/ +RCODE F_Database::setupDatabase( + const char * pszDbPath, + const char * pszDataDir) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiAllocLen; + FLMUINT uiDbNameLen; + FLMUINT uiDirNameLen; + FFileItemId * pFileItemId1 = NULL; + FFileItemId * pFileItemId2 = NULL; + char szDbPathStr[ F_PATH_MAX_SIZE]; + char szDataDirStr[ F_PATH_MAX_SIZE]; + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszDbPath, szDbPathStr))) + { + goto Exit; + } + uiDbNameLen = f_strlen( szDbPathStr) + 1; + + if( pszDataDir && *pszDataDir) + { + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathToStorageString( + pszDataDir, szDataDirStr))) + { + goto Exit; + } + uiDirNameLen = f_strlen( szDataDirStr) + 1; + + } + else + { + szDataDirStr[0] = 0; + uiDirNameLen = 0; + } + + + if (RC_BAD( rc = f_mutexCreate( &m_hMutex))) + { + goto Exit; + } + + + uiAllocLen = (FLMUINT)(uiDbNameLen + uiDirNameLen); + if (RC_BAD( rc = f_alloc( uiAllocLen, &m_pszDbPath))) + { + goto Exit; + } + + // Allocate a buffer for writing the DB header + // If we are a temporary database, there is no need + // for this allocation. + + if (!m_bTempDb) + { + if( RC_BAD( rc = f_allocAlignedBuffer( + SFLM_MAX_BLOCK_SIZE, (void **)&m_pDbHdrWriteBuf))) + { + goto Exit; + } + } + + // Setup the write buffer managers. + + if( RC_BAD( rc = FlmAllocIOBufferMgr( &m_pBufferMgr))) + { + goto Exit; + } + + m_pBufferMgr->setMaxBuffers( MAX_PENDING_WRITES); + m_pBufferMgr->setMaxBytes( MAX_WRITE_BUFFER_BYTES); + + // Initialize members of F_Database object. + + m_uiBucket = 0xFFFF; + m_uiFlags = DBF_BEING_OPENED; + + // Copy the database name and directory. + // NOTE: uiDbNameLen includes the null terminating byte. + // and uiDirNameLen includes the null terminating byte. + + f_memcpy( m_pszDbPath, szDbPathStr, uiDbNameLen); + if (uiDirNameLen) + { + m_pszDataDir = m_pszDbPath + uiDbNameLen; + f_memcpy( m_pszDataDir, szDataDirStr, uiDirNameLen); + } + + // Link the file into the various lists it needs to be linked into. + + if (RC_BAD( rc = linkToBucket())) + { + goto Exit; + } + + // Allocate the lock objects - must be done AFTER setting up the + // file name stuff up above. + + if ((pFileItemId1 = f_new FFileItemId( this, TRUE)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if ((pFileItemId2 = f_new FFileItemId( this, FALSE)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Allocate a lock object for write locking. + + if ((m_pWriteLockObj = gv_SFlmSysData.pServerLockMgr->GetLockObject( + pFileItemId1)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + m_pWriteLockObj->AddRef(); + + // Allocate a lock object for file locking. + + if ((m_pDatabaseLockObj = gv_SFlmSysData.pServerLockMgr->GetLockObject( + pFileItemId2)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + m_pDatabaseLockObj->AddRef(); + +Exit: + + if (pFileItemId1) + { + pFileItemId1->Release(); + } + + if (pFileItemId2) + { + pFileItemId2->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine allocates a new F_Database object and links it + into its hash buckets. + NOTE: This routine assumes that the global mutex has already + been locked. It may unlock it temporarily if there is an error, + but will always relock it before exiting. +****************************************************************************/ +RCODE F_DbSystem::allocDatabase( + const char * pszDbPath, + const char * pszDataDir, + FLMBOOL bTempDb, + F_Database ** ppDatabase) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = NULL; + + if ((pDatabase = f_new F_Database( bTempDb)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = pDatabase->setupDatabase( pszDbPath, pszDataDir))) + { + goto Exit; + } + + *ppDatabase = pDatabase; + +Exit: + + if (RC_BAD( rc)) + { + if (pDatabase) + { + pDatabase->freeDatabase(); + } + } + return( rc); +} + +/*************************************************************************** +Desc: This routine reads the header information for an existing + flaim database and makes sure we have a valid database. +*****************************************************************************/ +RCODE F_Database::readDbHdr( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBYTE * pszPassword, + FLMBOOL bAllowLimited) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pCFileHdl = NULL; + + if (RC_BAD( rc = pSFileHdl->getFileHdl( 0, FALSE, &pCFileHdl))) + { + goto Exit; + } + + // Read and verify the database header. + + if (RC_BAD( rc = flmReadAndVerifyHdrInfo( pDbStats, pCFileHdl, + &m_lastCommittedDbHdr))) + { + goto Exit; + } + m_uiBlockSize = (FLMUINT)m_lastCommittedDbHdr.ui16BlockSize; + m_uiDefaultLanguage = (FLMUINT)m_lastCommittedDbHdr.ui8DefaultLanguage; + m_uiMaxFileSize = (FLMUINT)m_lastCommittedDbHdr.ui32MaxFileSize; + m_uiSigBitsInBlkSize = calcSigBits( m_uiBlockSize); + + // Initialize the master database key from the database header + m_bAllowLimitedMode = bAllowLimited; + + if (pszPassword && *pszPassword) + { + if (m_pszDbPasswd) + { + f_free( &m_pszDbPasswd); + } + if ( RC_BAD( rc = f_alloc( + (f_strlen( (const char *)pszPassword) + 1), &m_pszDbPasswd))) + { + goto Exit; + } + + f_strcpy( (char *)m_pszDbPasswd, (const char *)pszPassword); + } + + if ((m_pWrappingKey = f_new F_CCS()) == NULL) + { + RC_SET( rc = NE_SFLM_MEM); + goto Exit; + } + + if( RC_OK( rc = m_pWrappingKey->init( TRUE, SFLM_AES_ENCRYPTION))) + { + // If the key was encrypted in a password, then the pszPassword parameter better + // be the key used to encrypt it. If the key was not encrypted in a password, + // then pszPassword parameter should be NULL. + rc = m_pWrappingKey->setKeyFromStore( + m_lastCommittedDbHdr.ucDbKey, + pszPassword, NULL); + } + + if (RC_BAD( rc)) + { + // NE_SFLM_UNSUPPORTED_FEATURE is returned when we've been compiled + // without NICI support + if ((rc == NE_SFLM_UNSUPPORTED_FEATURE) || bAllowLimited) + { + m_bInLimitedMode = TRUE; + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + // Note that we might still end up in limited mode if we can't verify all the keys + // that are stored in the dictionary. + +Exit: + + // Need to close the .db file so that we can set the block size. + // This will allow direct I/O to be used when accessing the file later. + + if (pCFileHdl) + { + (void)pSFileHdl->releaseFile( (FLMUINT)0, TRUE); + pSFileHdl->setBlockSize( m_uiBlockSize); + } + + return( rc); +} + +/*************************************************************************** +Desc: This routine frees a CP_INFO structure and all associated data. +*****************************************************************************/ +FSTATIC void flmFreeCPInfo( + CP_INFO ** ppCPInfoRV) +{ + CP_INFO * pCPInfo; + + if ((pCPInfo = *ppCPInfoRV) != NULL) + { + if (pCPInfo->pSFileHdl) + { + pCPInfo->pSFileHdl->Release(); + } + + if (pCPInfo->bStatsInitialized) + { + flmStatFree( &pCPInfo->Stats); + } + + if( pCPInfo->hWaitSem != F_SEM_NULL) + { + f_semDestroy( &pCPInfo->hWaitSem); + } + + f_free( ppCPInfoRV); + } +} + +/*************************************************************************** +Desc: This routine begins a thread that will do checkpoints for the + passed in database. It gives the thread its own FLAIM session and its + own handle to the database. +*****************************************************************************/ +RCODE F_Database::startCPThread( void) +{ + RCODE rc = NE_SFLM_OK; + CP_INFO * pCPInfo = NULL; + char szThreadName[ F_PATH_MAX_SIZE]; + char szBaseName[ 32]; + + // Allocate a CP_INFO structure that will be passed into the + // thread when it is created. + + if (RC_BAD( rc = f_calloc( (FLMUINT)(sizeof( CP_INFO)), &pCPInfo))) + { + goto Exit; + } + pCPInfo->pDatabase = this; + + // Create a "wait" semaphore + + if( RC_BAD( rc = f_semCreate( &pCPInfo->hWaitSem))) + { + goto Exit; + } + + // Allocate a super file handle. + + if ((pCPInfo->pSFileHdl = f_new F_SuperFileHdl) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Set up the super file + + if (RC_BAD( rc = pCPInfo->pSFileHdl->setup( m_pszDbPath, m_pszDataDir))) + { + goto Exit; + } + + if (m_lastCommittedDbHdr.ui32DbVersion) + { + pCPInfo->pSFileHdl->setBlockSize( m_uiBlockSize); + } + + f_memset( &pCPInfo->Stats, 0, sizeof( SFLM_STATS)); + pCPInfo->bStatsInitialized = TRUE; + + // Generate the thread name + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( m_pszDbPath, + szThreadName, szBaseName))) + { + goto Exit; + } + + f_sprintf( (char *)szThreadName, "Checkpoint (%s)", (char *)szBaseName); + + // Start the checkpoint thread. + + if (RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( &m_pCPThrd, + flmCPThread, szThreadName, gv_SFlmSysData.uiCheckpointThreadGroup, + 0, pCPInfo, NULL, 32000))) + { + goto Exit; + } + + m_pCPInfo = pCPInfo; + pCPInfo = NULL; + +Exit: + + if( pCPInfo) + { + flmFreeCPInfo( &pCPInfo); + } + + return( rc); +} + +/**************************************************************************** +Desc: Try to perform a checkpoint on the database. Returns TRUE if we need + to terminate. +****************************************************************************/ +FLMBOOL F_Database::tryCheckpoint( + IF_Thread * pThread, + CP_INFO * pCPInfo) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bTerminate = FALSE; + FLMBOOL bForceCheckpoint; + eForceCPReason eForceReason; + FLMUINT uiCurrTime; + SFLM_DB_STATS * pDbStats; + + // See if we should terminate the thread. + + if (pThread->getShutdownFlag()) + { + // Set terminate flag to TRUE and then see if + // we have been set up to do one final checkpoint + // to flush dirty buffers to disk. + + bTerminate = TRUE; + } + + // Determine if we need to force a checkpoint. + + bForceCheckpoint = FALSE; + eForceReason = SFLM_CP_NO_REASON; + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + if (bTerminate) + { + bForceCheckpoint = TRUE; + eForceReason = SFLM_CP_SHUTTING_DOWN_REASON; + } + else if (!m_pRfl->seeIfRflVolumeOk() || RC_BAD( m_CheckpointRc)) + { + bForceCheckpoint = TRUE; + eForceReason = SFLM_CP_RFL_VOLUME_PROBLEM; + } + else if ((FLM_ELAPSED_TIME( uiCurrTime, m_uiLastCheckpointTime) >= + gv_SFlmSysData.uiMaxCPInterval) || + (!gv_SFlmSysData.uiMaxCPInterval)) + { + bForceCheckpoint = TRUE; + eForceReason = SFLM_CP_TIME_INTERVAL_REASON; + } + + if (gv_SFlmSysData.Stats.bCollectingStats) + { + + // Statistics are being collected for the system. Therefore, + // if we are not currently collecting statistics in the + // start. If we were collecting statistics, but the + // start time was earlier than the start time in the system + // statistics structure, reset the statistics. + + if (!pCPInfo->Stats.bCollectingStats) + { + flmStatStart( &pCPInfo->Stats); + } + else if (pCPInfo->Stats.uiStartTime < + gv_SFlmSysData.Stats.uiStartTime) + { + flmStatReset( &pCPInfo->Stats, FALSE); + } + (void)flmStatGetDb( &pCPInfo->Stats, this, + 0, &pDbStats, NULL, NULL); + } + else + { + pDbStats = NULL; + } + + // Lock write object - If we are forcing a checkpoint + // wait until we get the lock. Otherwise, if we can't get + // the lock without waiting, don't do anything. + + if (bForceCheckpoint || + (gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache && + (m_uiDirtyCacheCount + m_uiLogCacheCount) * m_uiBlockSize > + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache)) + { + if (RC_BAD( rc = dbWriteLock( pCPInfo->hWaitSem, pDbStats))) + { + + // THIS SHOULD NEVER HAPPEN BECAUSE dbWriteLock will + // wait forever for the lock! + + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + pThread->setThreadStatusStr( "Forcing checkpoint"); + + // Must wait for any RFL writes to complete. + + (void)m_pRfl->seeIfRflWritesDone( pCPInfo->hWaitSem, TRUE); + } + else + { + if (RC_BAD( dbWriteLock( pCPInfo->hWaitSem, pDbStats, 0))) + { + goto Exit; + } + + pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); + + // See if we actually need to do the checkpoint. If the + // current transaction ID and the last checkpoint transaction + // ID are the same, no updates have occurred that would require + // a checkpoint to take place. + + if (m_lastCommittedDbHdr.ui64RflLastCPTransID == + m_lastCommittedDbHdr.ui64CurrTransID || + !m_pRfl->seeIfRflWritesDone( pCPInfo->hWaitSem, FALSE)) + { + dbWriteUnlock( pDbStats); + goto Exit; + } + } + + // Do the checkpoint. + + (void)doCheckpoint( pCPInfo->hWaitSem, pDbStats, pCPInfo->pSFileHdl, FALSE, + bForceCheckpoint, eForceReason, 0, 0); + if (pDbStats) + { + (void)flmStatUpdate( &pCPInfo->Stats); + } + + dbWriteUnlock( pDbStats); + + // Set the thread's status + + pThread->setThreadStatus( FLM_THREAD_STATUS_SLEEPING); + +Exit: + + return( bTerminate); +} + +/**************************************************************************** +Desc: This routine functions as a thread. It monitors open files and + frees up files which have been closed longer than the maximum + close time. +****************************************************************************/ +FSTATIC RCODE flmCPThread( + IF_Thread * pThread) +{ + CP_INFO * pCPInfo = (CP_INFO *)pThread->getParm1(); + F_Database * pDatabase = pCPInfo->pDatabase; + + pThread->setThreadStatus( FLM_THREAD_STATUS_SLEEPING); + for (;;) + { + f_sleep( 1000); + if (pDatabase->tryCheckpoint( pThread, pCPInfo)) + { + break; + } + } + + pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); + + flmFreeCPInfo( &pCPInfo); + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: Recover a database on startup. +****************************************************************************/ +RCODE F_Database::doRecover( + F_Db * pDb, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus) +{ + RCODE rc = NE_SFLM_OK; + SFLM_DB_HDR * pLastCommittedDbHdr; + + // At this point, m_lastCommittedDbHdr contains the header + // that was read from disk, which will be the state of the + // header as of the last completed checkpoint. Therefore, + // we copy it into m_checkpointDbHdr. + + pLastCommittedDbHdr = &m_lastCommittedDbHdr; + f_memcpy( &m_checkpointDbHdr, pLastCommittedDbHdr, sizeof( SFLM_DB_HDR)); + + // Do a physical rollback on the database to restore the last + // checkpoint. + + if (RC_BAD( rc = pDb->physRollback( + (FLMUINT)pLastCommittedDbHdr->ui32RblEOF, + (FLMUINT)pLastCommittedDbHdr->ui32RblFirstCPBlkAddr, + TRUE, + pLastCommittedDbHdr->ui64RflLastCPTransID))) + { + goto Exit; + } + pLastCommittedDbHdr->ui32RblFirstCPBlkAddr = 0; + pLastCommittedDbHdr->ui32RblEOF = (FLMUINT32)m_uiBlockSize; + if (RC_BAD( rc = writeDbHdr( pDb->m_pDbStats, pDb->m_pSFileHdl, + pLastCommittedDbHdr, + &m_checkpointDbHdr, TRUE))) + { + goto Exit; + } + + // Set uiFirstLogCPBlkAddress to zero to indicate that no + // physical blocks have been logged for the current checkpoint. + // The above call to flmPhysRollback will have set the log header + // to the same thing. + + m_uiFirstLogCPBlkAddress = 0; + + // Set the checkpointDbHdr to be the same as the log header + + f_memcpy( &m_checkpointDbHdr, pLastCommittedDbHdr, sizeof( SFLM_DB_HDR)); + + // Open roll forward log and redo the transactions that + // occurred since the last checkpoint, if any. + + if( RC_BAD( rc = m_pRfl->recover( pDb, pRestoreObj, pRestoreStatus))) + { + goto Exit; + } + +Exit: + + return( rc); +} diff --git a/sql/src/flreduce.cpp b/sql/src/flreduce.cpp new file mode 100644 index 0000000..2a4498a --- /dev/null +++ b/sql/src/flreduce.cpp @@ -0,0 +1,849 @@ +//------------------------------------------------------------------------------ +// Desc: Reduce the database size by move 'N' free blocks the the end of +// the file and truncating the file. +// +// Tabs: 3 +// +// Copyright (c) 1992-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: flreduce.cpp 3113 2006-01-19 13:20:35 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc : Reduces the size of a FLAIM database file. +Notes: The size of the database file is reduced by freeing a specified + number of blocks from the available (unused) block list. The blocks + are moved to the end of the file and the file is truncated. If the + available block list is empty, FLAIM will attemp to add blocks to + the list by freeing log extent blocks. +****************************************************************************/ +RCODE F_Db::reduceSize( + FLMUINT uiCount, + FLMUINT * puiCount) +{ + RCODE rc = NE_SFLM_OK; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMUINT uiLogicalEOF; + FLMUINT uiBlkAddr; + FLMUINT uiNumBlksMoved; + FLMUINT uiBlkSize; + F_LARGEST_BLK_HDR blkHdr; + FLMINT iType; + FLMBOOL bFlagSet; + FLMBOOL bTransActive = FALSE; + FLMBOOL bLockedDb = FALSE; + FLMBOOL bEnableLogging = FALSE; + + uiNumBlksMoved = 0; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Make sure we are NOT in a database transaction. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // There must NOT be a shared lock on the file. + + if (m_uiFlags & FDB_FILE_LOCK_SHARED) + { + rc = RC_SET( NE_SFLM_SHARED_LOCK); + goto Exit; + } + + // Must acquire an exclusive file lock first, if it hasn't been + // acquired. + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if (RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Lock( this, m_hWaitSem, + TRUE, FALSE, TRUE, SFLM_NO_TIMEOUT, 0, m_pDbStats))) + { + goto Exit; + } + + bLockedDb = TRUE; + m_uiFlags |= FDB_HAS_FILE_LOCK; + } + + // Disable RFL logging - don't want anything logged during reduce + // except for the reduce packet. + + bEnableLogging = pRfl->isLoggingEnabled(); + pRfl->disableLogging(); + + // Keep looping to here until the count is satisfied or there + // are not any more log extent blocks to turn into avail blks. + // The loop does a begin transaction - move blocks - set logical + // EOF and commits the transaction. During the commit if there are + // not any avail blocks left then a log extent (if any) will be turned + // into more avail blocks and we can do this again with more avail + // blocks. + + // Start a database transaction + + if( RC_BAD(rc = beginTrans( SFLM_UPDATE_TRANS, + SFLM_NO_TIMEOUT, SFLM_DONT_POISON_CACHE))) + { + goto Exit; + } + bTransActive = TRUE; + + // Make sure that commit does something. + + m_bHadUpdOper = TRUE; + + uiBlkSize = m_pDatabase->m_uiBlockSize; + + // Get the logical end of file and use internally. + // Loop until there are not any more free blocks left or the + // input count is matched. Switch on each block type found + // while backing up through the file. + + uiLogicalEOF = m_uiLogicalEOF; + + while (m_uiFirstAvailBlkAddr && + (!uiCount || uiNumBlksMoved < uiCount)) + { + + // Read the last block and determine block type. + + if( FSGetFileOffset( uiLogicalEOF) == 0) + { + IF_FileHdl * pFileHdl; + FLMUINT uiFileNumber = FSGetFileNumber( uiLogicalEOF) - 1; + FLMUINT64 ui64FileSize; + FLMUINT64 ui64Temp; + + if (RC_BAD( rc = m_pSFileHdl->getFileHdl( + uiFileNumber, TRUE, &pFileHdl))) + { + goto Exit; + } + + if (RC_BAD( rc = pFileHdl->size( &ui64FileSize))) + { + goto Exit; + } + + // Adjust to a block bounds. + + ui64Temp = (ui64FileSize / uiBlkSize) * uiBlkSize; + if( ui64Temp < ui64FileSize) + { + ui64FileSize = ui64Temp + uiBlkSize; + } + + uiLogicalEOF = FSBlkAddress( uiFileNumber, (FLMUINT)ui64FileSize); + } + + uiBlkAddr = uiLogicalEOF - uiBlkSize; + if (RC_BAD( rc = readBlkHdr( uiBlkAddr, + (F_BLK_HDR *)&blkHdr, &iType))) + { + goto Exit; + } + + switch (iType) + { + case BT_FREE: + rc = m_pDatabase->freeAvailBlk( this, uiBlkAddr); + break; + + case BT_LEAF: + case BT_NON_LEAF: + case BT_NON_LEAF_COUNTS: + case BT_LEAF_DATA: + case BT_DATA_ONLY: + rc = m_pDatabase->moveBtreeBlk( this, uiBlkAddr, + (FLMUINT)blkHdr.all.BTreeBlkHdr.ui16LogicalFile, + getBlkLfType( &blkHdr.all.BTreeBlkHdr)); + break; + + case BT_LFH_BLK: + rc = m_pDatabase->moveLFHBlk( this, uiBlkAddr); + break; + + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + break; + } + + if (RC_BAD(rc)) + { + goto Exit; + } + + uiNumBlksMoved++; + + // Adjust the logical EOF to the new value. + // This is complex when dealing with block files. + + if (FSGetFileOffset( uiLogicalEOF) == 0) + { + FLMUINT uiFileNumber = FSGetFileNumber( uiLogicalEOF); + FLMUINT64 ui64FileOffset; + IF_FileHdl * pFileHdl; + + if (uiFileNumber <= 1) + { + break; + } + + // Leave the current file at zero bytes and move to the + // previous store file. + + uiFileNumber--; + + // Compute the end of the previous block file. + + if (RC_BAD( rc = m_pSFileHdl->getFileHdl( + uiFileNumber, TRUE, &pFileHdl))) + { + goto Exit; + } + + if (RC_BAD( rc = pFileHdl->size( &ui64FileOffset))) + { + goto Exit; + } + + uiLogicalEOF = FSBlkAddress( uiFileNumber, (FLMUINT)ui64FileOffset); + } + uiLogicalEOF -= uiBlkSize; + } + + // Log the reduce packet to the RFL if we are not in the middle of a + // restore or recovery. Will need to re-enable logging temporarily + // and then turn it back off after logging the packet. + + pRfl->enableLogging(); + + // Log the reduce. + + if( RC_BAD( rc = pRfl->logReduce( this, uiCount))) + { + goto Exit; + } + + // Turn logging back off. + + pRfl->disableLogging(); + + if (RC_BAD( rc)) + { + goto Exit; + } + + // Commit the transaction. + + if (m_uiFlags & FDB_DO_TRUNCATE) + { + bFlagSet = TRUE; + } + else + { + bFlagSet = FALSE; + m_uiFlags |= FDB_DO_TRUNCATE; + } + + bTransActive = FALSE; + rc = commitTrans( uiLogicalEOF, TRUE); + + if (!bFlagSet) + { + m_uiFlags &= (~(FDB_DO_TRUNCATE)); + } + + if (RC_BAD( rc)) + { + goto Exit; + } + +Exit: + + if( RC_BAD( rc) && bTransActive) + { + (void)abortTrans(); + uiNumBlksMoved = 0; + } + + if (puiCount) + { + // May be more than the count requested. + + *puiCount = uiNumBlksMoved; + } + + if (bEnableLogging) + { + pRfl->enableLogging(); + } + + if (bLockedDb) + { + (void)m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this); + m_uiFlags &= ~FDB_HAS_FILE_LOCK; + } + + return( rc); +} + +/**************************************************************************** +Desc: Read the block header and return the type of block it is +****************************************************************************/ +RCODE F_Db::readBlkHdr( + FLMUINT uiBlkAddress, + F_BLK_HDR * pBlkHdr, + FLMINT * piType + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesRead; + FLMUINT uiNumLooks; + IF_FileHdl * pTmpFileHdl = NULL; + F_CachedBlock * pBlkSCache; + SFLM_LFILE_STATS * pLFileStats; + F_TMSTAMP StartTime; + FLMUINT64 ui64ElapMilli = 0; + + // First see if the block is in cache. + // Previous writes may not have been forced out to cache. + + if (RC_BAD( rc = m_pDatabase->getBlock( this, NULL, uiBlkAddress, + &uiNumLooks, &pBlkSCache))) + { + goto Exit; + } + + if (pBlkSCache) // If found in cache ... + { + f_memcpy( pBlkHdr, pBlkSCache->getBlockPtr(), SIZEOF_LARGEST_BLK_HDR); + ScaReleaseCache( pBlkSCache, FALSE); + } + else + { + if (m_pDbStats) + { + ui64ElapMilli = 0; + f_timeGetTimeStamp( &StartTime); + } + + if (RC_OK( rc = m_pSFileHdl->getFileHdl( + FSGetFileNumber( uiBlkAddress), TRUE, &pTmpFileHdl))) + { + rc = pTmpFileHdl->read( FSGetFileOffset( uiBlkAddress), + SIZEOF_LARGEST_BLK_HDR, pBlkHdr, &uiBytesRead); + } + + if (m_pDbStats) + { + flmAddElapTime( &StartTime, &ui64ElapMilli); + if (RC_BAD( rc)) + { + m_pDbStats->bHaveStats = TRUE; + m_pDbStats->uiReadErrors++; + } + } + + // Convert the block header if necessary. + + if (blkIsNonNativeFormat( pBlkHdr)) + { + convertBlkHdr( pBlkHdr); + } + + if (m_pDbStats && RC_OK( rc)) + { + FLMUINT uiLFileNum; + SFLM_BLOCKIO_STATS * pBlockIOStats; + + uiLFileNum = (FLMUINT)((F_BTREE_BLK_HDR *)pBlkHdr)->ui16LogicalFile; + if (!uiLFileNum) + { + pLFileStats = NULL; + } + else + { + if( RC_BAD( flmStatGetLFile( m_pDbStats, + uiLFileNum, + getBlkLfType((F_BTREE_BLK_HDR *)pBlkHdr), + 0, + &pLFileStats, NULL, NULL))) + { + pLFileStats = NULL; + } + } + if ((pBlockIOStats = flmGetBlockIOStatPtr( m_pDbStats, + pLFileStats, (FLMBYTE *)pBlkHdr)) != NULL) + { + m_pDbStats->bHaveStats = TRUE; + if (pLFileStats) + { + pLFileStats->bHaveStats = TRUE; + } + pBlockIOStats->BlockReads.ui64ElapMilli += ui64ElapMilli; + pBlockIOStats->BlockReads.ui64Count++; + pBlockIOStats->BlockReads.ui64TotalBytes += SIZEOF_LARGEST_BLK_HDR; + } + } + + if (RC_BAD( rc)) + { + if (rc != NE_FLM_IO_END_OF_FILE && rc != NE_SFLM_MEM) + { + m_pSFileHdl->releaseFile( FSGetFileNumber( uiBlkAddress), + TRUE); + } + goto Exit; + } + } + + if (piType) + { + *piType = (FLMINT)pBlkHdr->ui8BlkType; + } + +Exit: + + return( rc ); +} + +/**************************************************************************** +Desc: Find where in the b-tree a matching block is located. Move to + a free block and change all pointers to the block. +Notes: Some of this code could be called in movePcodeLFHBlk but we have + to worry about if the block is a root or right most leaf block. +****************************************************************************/ +RCODE F_Database::moveBtreeBlk( + F_Db * pDb, + FLMUINT uiBlkAddr, + FLMUINT uiLfNumber, + eLFileType eLfType) +{ + RCODE rc; + F_INDEX * pIndex; + LFILE * pLFile; + F_TABLE * pTable = NULL; + F_CachedBlock * pFreeSCache = NULL; + FLMBOOL bReleaseCache = FALSE; + F_Btree * pbtree = NULL; + FLMBOOL bHaveCounts; + FLMBOOL bHaveData; + IXKeyCompare compareObject; + IF_ResultSetCompare * pCompareObject = NULL; + + // Get an F_Btree object + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbtree))) + { + goto Exit; + } + + if (eLfType == SFLM_LF_TABLE) + { + if ((pTable = pDb->m_pDict->getTable( uiLfNumber)) == NULL) + { + rc = RC_SET( NE_SFLM_INVALID_TABLE_NUM); + goto Exit; + } + pLFile = &pTable->lfInfo; + bHaveCounts = FALSE; + bHaveData = TRUE; + } + else + { + if ((pIndex = pDb->m_pDict->getIndex( uiLfNumber)) == NULL) + { + rc = RC_SET( NE_SFLM_INVALID_INDEX_NUM); + goto Exit; + } + pLFile = &pIndex->lfInfo; + bHaveCounts = (pIndex->uiFlags & IXD_ABS_POS) ? TRUE : FALSE; + bHaveData = (pIndex->uiNumDataComponents) ? TRUE : FALSE; + + pCompareObject = &compareObject; + compareObject.setIxInfo( pDb, pIndex); + } + + // Need to make sure that LFILE is up to date. + // Force reading it in. + + if (RC_BAD( rc = lFileRead( pDb, pLFile))) + { + goto Exit; + } + + // If we are moving the root block, create a new dictionary so + // that the LFILE can be modified safely to point to the new + // root block. + + if (pLFile->uiRootBlk == uiBlkAddr) + { + + // Create a new dictionary + + if (!(pDb->m_uiFlags & FDB_UPDATED_DICTIONARY)) + { + if (RC_BAD( rc = pDb->dictClone())) + { + goto Exit; + } + + // Re-get the LFile + + if (eLfType == SFLM_LF_TABLE) + { + pTable = pDb->m_pDict->getTable( uiLfNumber); + flmAssert( pTable); + pLFile = &pTable->lfInfo; + } + else + { + pIndex = pDb->m_pDict->getIndex( uiLfNumber); + flmAssert( pIndex); + pLFile = &pIndex->lfInfo; + pCompareObject = &compareObject; + compareObject.setIxInfo( pDb, pIndex); + } + } + } + + if (RC_BAD( rc = pbtree->btOpen( pDb, pLFile, bHaveCounts, bHaveData, + pCompareObject))) + { + goto Exit; + } + + // Get the next free block. + + if (RC_BAD( rc = blockUseNextAvail( pDb, &pFreeSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + // Move B-tree block to free block. + + if (RC_BAD( rc = pbtree->btMoveBlock( (FLMUINT32)uiBlkAddr, + (FLMUINT32)pFreeSCache->m_uiBlkAddress))) + { + goto Exit; + } + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pFreeSCache, FALSE); + } + + if (pbtree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbtree); + } + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/**************************************************************************** +Desc: Find where anb LFH input block is located. Move to + a free block and change all pointers to the block. +****************************************************************************/ +RCODE F_Database::moveLFHBlk( + F_Db * pDb, + FLMUINT uiBlkAddr) +{ + RCODE rc; + F_CachedBlock * pSCache; + FLMBOOL bReleaseCache = FALSE; + F_CachedBlock * pFreeSCache = NULL; + FLMBOOL bReleaseCache2 = FALSE; + F_BLK_HDR * pBlkHdr; + F_LF_HDR * pLfHdr; + F_BLK_HDR * pFreeBlkHdr; + FLMUINT uiLeftBlkAddr; + FLMUINT uiRightBlkAddr; + FLMUINT uiFreeBlkAddr; + FLMUINT uiSavePriorBlkImgAddr; + FLMUINT uiPos; + FLMUINT uiEndPos; + F_TABLE * pTmpTable; + F_INDEX * pTmpIndex; + LFILE * pTmpLFile; + + if (RC_BAD( rc = getBlock( pDb, NULL, uiBlkAddr, NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + pBlkHdr = pSCache->m_pBlkHdr; + + // Get left and rigth block addresses. + // Get next avail block and move data over + + uiLeftBlkAddr = (FLMUINT)pBlkHdr->ui32PrevBlkInChain; + uiRightBlkAddr = (FLMUINT)pBlkHdr->ui32NextBlkInChain; + + if (RC_BAD( rc = blockUseNextAvail( pDb, &pFreeSCache))) + { + goto Exit; + } + bReleaseCache2 = TRUE; + + pFreeBlkHdr = pFreeSCache->m_pBlkHdr; + uiFreeBlkAddr = (FLMUINT)pFreeBlkHdr->ui32BlkAddr; + + // The free block has been logged and set to dirty in + // blockUseNextAvail(). + // BUT, need to preserve prior image block address - it should + // NOT be copied over from the block we are switching with. + + uiSavePriorBlkImgAddr = (FLMUINT)pFreeBlkHdr->ui32PriorBlkImgAddr; + + f_memcpy( pFreeBlkHdr, pBlkHdr, m_uiBlockSize); + pFreeBlkHdr->ui32BlkAddr = (FLMUINT32)uiFreeBlkAddr; + + // Restore the saved previous transaction ID and block address. + + pFreeBlkHdr->ui32PriorBlkImgAddr = (FLMUINT32)uiSavePriorBlkImgAddr; + + // Fix up any LFile entries that were pointing to the + // original LFH block + + uiPos = SIZEOF_STD_BLK_HDR; + uiEndPos = blkGetEnd( m_uiBlockSize, SIZEOF_STD_BLK_HDR, pFreeBlkHdr); + + // Create a new dictionary + + if (!(pDb->m_uiFlags & FDB_UPDATED_DICTIONARY)) + { + if (RC_BAD( rc = pDb->dictClone())) + { + goto Exit; + } + } + + // Iterate over the set of LFiles in the block and + // update their LFH block addresses + + pLfHdr = (F_LF_HDR *)((FLMBYTE *)pFreeBlkHdr + SIZEOF_STD_BLK_HDR); + while (uiPos < uiEndPos) + { + if (pLfHdr->ui32LfType != (FLMUINT32)SFLM_LF_INVALID) + { + if (pLfHdr->ui32LfType == (FLMUINT32)SFLM_LF_TABLE) + { + if ((pTmpTable = pDb->m_pDict->getTable( + (FLMUINT)pLfHdr->ui32LfNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + pTmpLFile = &pTmpTable->lfInfo; + } + else + { + flmAssert( pLfHdr->ui32LfType == (FLMUINT32)SFLM_LF_INDEX); + if ((pTmpIndex = pDb->m_pDict->getIndex( + (FLMUINT)pLfHdr->ui32LfNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + pTmpLFile = &pTmpIndex->lfInfo; + } + + pTmpLFile->uiBlkAddress = uiFreeBlkAddr; + } + uiPos += sizeof( F_LF_HDR); + pLfHdr++; + } + + // Done with both blocks. + + ScaReleaseCache( pFreeSCache, FALSE); + ScaReleaseCache( pSCache, FALSE); + bReleaseCache2 = bReleaseCache = FALSE; + + // Read left and right blocks and adjust their + // pointers to point to the new block. + // This doesn't matter what level of the b-tree + // you are on. + + if (uiLeftBlkAddr) + { + if (RC_BAD( rc = getBlock( pDb, NULL, uiLeftBlkAddr, + NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + + pSCache->m_pBlkHdr->ui32NextBlkInChain = (FLMUINT32)uiFreeBlkAddr; + ScaReleaseCache( pSCache, FALSE); + bReleaseCache = FALSE; + } + + if (uiRightBlkAddr) + { + if (RC_BAD( rc = getBlock( pDb, NULL, uiRightBlkAddr, + NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + + pSCache->m_pBlkHdr->ui32PrevBlkInChain = (FLMUINT32)uiFreeBlkAddr; + ScaReleaseCache( pSCache, FALSE); + bReleaseCache = FALSE; + } + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + if (bReleaseCache2) + { + ScaReleaseCache( pFreeSCache, FALSE); + } + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + + +/**************************************************************************** +Desc: Free the input avail block. Link the block out of the free list +****************************************************************************/ +RCODE F_Database::freeAvailBlk( + F_Db * pDb, + FLMUINT uiBlkAddr) +{ + RCODE rc = NE_SFLM_OK; + F_LARGEST_BLK_HDR blkHdr; + FLMUINT uiPrevBlkAddr; + FLMUINT uiNextBlkAddr; + F_CachedBlock * pSCache; + + // Check for first avail block condition. + + if (uiBlkAddr == pDb->m_uiFirstAvailBlkAddr) + { + if (RC_OK( rc = blockUseNextAvail( pDb, &pSCache))) + { + ScaReleaseCache( pSCache, FALSE); + } + goto Exit; + } + + // Not first block, unlink from list. + + // Read the block header and get pointers + + if (RC_BAD( rc = pDb->readBlkHdr( uiBlkAddr, + (F_BLK_HDR *)&blkHdr, (FLMINT *)0))) + { + goto Exit; + } + uiPrevBlkAddr = (FLMUINT)blkHdr.all.stdBlkHdr.ui32PrevBlkInChain; + uiNextBlkAddr = (FLMUINT)blkHdr.all.stdBlkHdr.ui32NextBlkInChain; + + // Read the previous block, if any + + if (uiPrevBlkAddr) + { + if (RC_BAD( rc = getBlock( pDb, NULL, uiPrevBlkAddr, + NULL, &pSCache))) + { + goto Exit; + } + + // Log the block before modifying it. + + if (RC_OK( rc = logPhysBlk( pDb, &pSCache))) + { + pSCache->m_pBlkHdr->ui32NextBlkInChain = (FLMUINT32)uiNextBlkAddr; + } + ScaReleaseCache( pSCache, FALSE); + if (RC_BAD( rc)) + { + goto Exit; + } + } + + // Read the next block, if any. + + if (uiNextBlkAddr) + { + if (RC_BAD( rc = getBlock( pDb, NULL, uiNextBlkAddr, + NULL, &pSCache))) + { + goto Exit; + } + + // Log the block before modifying it. + + if (RC_OK( rc = logPhysBlk( pDb, &pSCache))) + { + pSCache->m_pBlkHdr->ui32PrevBlkInChain = (FLMUINT32)uiPrevBlkAddr; + } + ScaReleaseCache( pSCache, FALSE); + if (RC_BAD( rc)) + { + goto Exit; + } + } + +Exit: + + return( rc); +} diff --git a/sql/src/fltrabrt.cpp b/sql/src/fltrabrt.cpp new file mode 100644 index 0000000..b068189 --- /dev/null +++ b/sql/src/fltrabrt.cpp @@ -0,0 +1,449 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for aborting a transaction. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fltrabrt.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: This routine aborts an active transaction for a particular + database. If the database is open via a server, a message is + sent to the server to abort the transaction. Otherwise, the + transaction is rolled back locally. +****************************************************************************/ +RCODE F_Db::abortTrans( + FLMBOOL bOkToLogAbort) +{ + RCODE rc = NE_SFLM_OK; + eDbTransType eSaveTransType; + SFLM_DB_HDR * pLastCommittedDbHdr; + SFLM_DB_HDR * pUncommittedDbHdr; + FLMBOOL bDumpedCache = FALSE; + FLMBOOL bKeepAbortedTrans; + FLMUINT64 ui64TransId; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + RCODE tmpRc; + + // Should never be calling on a temporary database. + + flmAssert( !m_pDatabase->m_bTempDb); + + // Get transaction type + + if (m_eTransType == SFLM_NO_TRANS) + { + goto Exit; // Will return SUCCESS. + } + + // No recovery required if it is a read transaction. + + if (m_eTransType == SFLM_READ_TRANS) + { + if (m_bKrefSetup) + { + // krefCntrlFree could be called w/o checking bKrefSetup because + // it checks the flag, but it is more optimal to check the + // flag before making the call because most of the time it will + // be false. + + krefCntrlFree(); + } + + goto Unlink_From_Trans; + } + +#ifdef FLM_DBG_LOG + flmDbgLogUpdate( m_pDatabase, m_ui64CurrTransID, + 0, 0, NE_SFLM_OK, "TAbrt"); +#endif + + // Disable DB header writes + + pRfl->clearDbHdrs(); + + // If the transaction had no update operations, restore it + // to its pre-transaction state - make it appear that no + // transaction ever happened. + + pLastCommittedDbHdr = &m_pDatabase->m_lastCommittedDbHdr; + pUncommittedDbHdr = &m_pDatabase->m_uncommittedDbHdr; + ui64TransId = m_ui64CurrTransID; + + // Free up all keys associated with this database. This is done even + // if we didn't have any update operations because the KREF may + // have been initialized by key generation operations performed + // by cursors, etc. + + krefCntrlFree(); + + if (m_bHadUpdOper) + { + + // Dump any start and stop indexing stubs that should be aborted. + + indexingAfterAbort(); + + // Log the abort record to the rfl file, or throw away the logged + // records altogether, depending on the LOG_KEEP_ABORTED_TRANS_IN_RFL + // flag. If the RFL volume is bad, we will not attempt to keep this + // transaction in the RFL. + + if (!pRfl->seeIfRflVolumeOk()) + { + bKeepAbortedTrans = FALSE; + } + else + { + bKeepAbortedTrans = + (pUncommittedDbHdr->ui8RflKeepAbortedTrans) + ? TRUE + : FALSE; + } + } + else + { + bKeepAbortedTrans = FALSE; + } + + // Log an abort transaction record to the roll-forward log or + // throw away the entire transaction, depending on the + // bKeepAbortedTrans flag. + + // If the transaction is being "dumped" because of a failed commit, + // don't log anything to the RFL. + + if (bOkToLogAbort) + { +#ifdef FLM_DEBUG + if( pRfl->isLoggingEnabled()) + { + flmAssert( m_ui64CurrTransID == pRfl->getCurrTransID()); + } +#endif + + if (RC_BAD( rc = pRfl->logEndTransaction( + this, RFL_TRNS_ABORT_PACKET, !bKeepAbortedTrans))) + { + goto Exit1; + } + } +#ifdef FLM_DEBUG + else + { + // If bOkToLogAbort is FALSE, this always means that either a + // commit failed while trying to log an end transaction packet or a + // commit packet was logged and the transaction commit subsequently + // failed for some other reason. In either case, the RFL should be + // in a good state, with its current transaction ID reset to 0. If + // not, either bOkToLogAbort is being used incorrectly by the caller + // or there is a bug in the RFL logic. + + flmAssert( pRfl->getCurrTransID() == 0); + } +#endif + + // If there were no operations in the transaction, restore + // everything as if the transaction never happened. + + // Even empty transactions can have modified rows to clean up + // so we need to call this no matter what. + + m_pDatabase->freeModifiedRows( this, m_ui64CurrTransID - 1); + + if (!m_bHadUpdOper) + { + + // Pretend we dumped cache - shouldn't be any to worry about at + // this point. + + bDumpedCache = TRUE; + goto Exit1; + } + + // Dump ALL modified cache blocks associated with the DB. + // NOTE: This needs to be done BEFORE the call to flmGetDbHdrInfo + // below, because that call will change pDb->m_ui64CurrTransID, + // and that value is used by freeModifiedRows. + + m_pDatabase->freeModifiedBlocks( m_ui64CurrTransID); + bDumpedCache = TRUE; + + // Reset the Db header from the last committed DB header in pFile. + + getDbHdrInfo( pLastCommittedDbHdr); + if (RC_BAD( rc = physRollback( (FLMUINT)pUncommittedDbHdr->ui32RblEOF, + m_pDatabase->m_uiFirstLogBlkAddress, FALSE, 0))) + { + goto Exit1; + } + + m_pDatabase->lockMutex(); + + // Put the new transaction ID into the log header even though + // we are not committing. We want to keep the transaction IDs + // incrementing even though we aborted. + + pLastCommittedDbHdr->ui64CurrTransID = ui64TransId; + + // Preserve where we are at in the roll-forward log. Even though + // the transaction aborted, we may have kept it in the RFL instead of + // throw it away. + + pLastCommittedDbHdr->ui32RflCurrFileNum = + pUncommittedDbHdr->ui32RflCurrFileNum; + pLastCommittedDbHdr->ui32RflLastTransOffset = + pUncommittedDbHdr->ui32RflLastTransOffset; + f_memcpy( pLastCommittedDbHdr->ucLastTransRflSerialNum, + pUncommittedDbHdr->ucLastTransRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_memcpy( pLastCommittedDbHdr->ucNextRflSerialNum, + pUncommittedDbHdr->ucNextRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + + // The following items tell us where we are at in the roll-back log. + // During a transaction we may log blocks for the checkpoint or for + // read transactions. So, even though we are aborting this transaction, + // there may be other things in the roll-back log that we don't want + // to lose. These items should not be reset until we do a checkpoint, + // which is when we know it is safe to throw away the entire roll-back log. + + pLastCommittedDbHdr->ui32RblEOF = + pUncommittedDbHdr->ui32RblEOF; + pLastCommittedDbHdr->ui32RblFirstCPBlkAddr = + pUncommittedDbHdr->ui32RblFirstCPBlkAddr; + + m_pDatabase->unlockMutex(); + + pRfl->commitDbHdrs( pLastCommittedDbHdr, + &m_pDatabase->m_checkpointDbHdr); + +Exit1: + + // Dump cache, if not done above. + + if (!bDumpedCache) + { + m_pDatabase->freeModifiedBlocks( m_ui64CurrTransID); + m_pDatabase->freeModifiedRows( this, m_ui64CurrTransID - 1); + bDumpedCache = TRUE; + } + + // Throw away IXD_FIXUPs + + if (m_pIxdFixups) + { + IXD_FIXUP * pIxdFixup; + IXD_FIXUP * pDeleteIxdFixup; + + pIxdFixup = m_pIxdFixups; + while (pIxdFixup) + { + pDeleteIxdFixup = pIxdFixup; + pIxdFixup = pIxdFixup->pNext; + f_free( &pDeleteIxdFixup); + } + m_pIxdFixups = NULL; + } + + if (m_eTransType == SFLM_UPDATE_TRANS && + gv_SFlmSysData.EventHdrs[ SFLM_EVENT_UPDATES].pEventCBList) + { + flmTransEventCallback( SFLM_EVENT_ABORT_TRANS, this, rc, + ui64TransId); + } + +Unlink_From_Trans: + + eSaveTransType = m_eTransType; + + if (m_uiFlags & FDB_HAS_WRITE_LOCK) + { + if (RC_BAD( tmpRc = pRfl->completeTransWrites( this, FALSE, FALSE))) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + } + } + + if (eSaveTransType == SFLM_UPDATE_TRANS) + { + + // Before unlocking, restore collection information. + + if (m_uiFlags & FDB_UPDATED_DICTIONARY) + { + m_pDatabase->lockMutex(); + flmAssert( m_pDict); + unlinkFromDict(); + if (m_pDatabase->m_pDictList) + { + + // Link the F_Db to the right F_Dict object so it will + // fixup the correct F_COLLECTION structures. + + linkToDict( m_pDatabase->m_pDictList); + } + m_pDatabase->unlockMutex(); + } + + if (m_pDict) + { + F_TABLE * pTable; + FLMUINT uiLfNum; +#ifdef FLM_DEBUG + F_INDEX * pIndex; + FLMUINT uiRootBlk; +#endif + + // Only need to do tables. Nothing from the LFH of + // an index is stored in memory except for the root block + // address, and whenever that is changed, we get a new + // dictionary. Since the new dictionary will be discarded + // in that case, there is nothing to restore for an index. + + for (uiLfNum = 1, pTable = m_pDict->m_pTableTbl; + uiLfNum <= m_pDict->m_uiHighestTableNum; + uiLfNum++, pTable++) + { + if (pTable->uiTableNum) + { +#ifdef FLM_DEBUG + uiRootBlk = pTable->lfInfo.uiRootBlk; +#endif + if (RC_BAD( tmpRc = m_pDatabase->lFileRead( this, &pTable->lfInfo))) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + } +#ifdef FLM_DEBUG + else + { + // Make sure root block did not change - should not + // have because root block changes are done by creating + // a new dictionary, and we have already discarded + // any new dictionary. Hence, root block address should + // be the same in memory as it is no disk. + + flmAssert( uiRootBlk == pTable->lfInfo.uiRootBlk); + } +#endif + } + } + + // Do indexes in debug mode to make sure uiRootBlk is correct + +#ifdef FLM_DEBUG + for (uiLfNum = 1, pIndex = m_pDict->m_pIndexTbl; + uiLfNum <= m_pDict->m_uiHighestIndexNum; + uiLfNum++, pIndex++) + { + if (pIndex->uiIndexNum) + { + uiRootBlk = pIndex->lfInfo.uiRootBlk; + if (RC_BAD( tmpRc = m_pDatabase->lFileRead( this, &pIndex->lfInfo))) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + } + else + { + // Make sure root block did not change - should not + // have because root block changes are done by creating + // a new dictionary, and we have already discarded + // any new dictionary. Hence, root block address should + // be the same in memory as it is no disk. + + flmAssert( uiRootBlk == pIndex->lfInfo.uiRootBlk); + } + } + } +#endif + } + } + + // Unlink the database from the transaction list. + + unlinkFromTransList( FALSE); + + if (m_pDbStats) + { + FLMUINT64 ui64ElapMilli = 0; + + flmAddElapTime( &m_TransStartTime, &ui64ElapMilli); + m_pDbStats->bHaveStats = TRUE; + if (eSaveTransType == SFLM_READ_TRANS) + { + m_pDbStats->ReadTransStats.AbortedTrans.ui64Count++; + m_pDbStats->ReadTransStats.AbortedTrans.ui64ElapMilli += + ui64ElapMilli; + } + else + { + m_pDbStats->UpdateTransStats.AbortedTrans.ui64Count++; + m_pDbStats->UpdateTransStats.AbortedTrans.ui64ElapMilli += + ui64ElapMilli; + } + } + + if (m_pStats) + { + (void)flmStatUpdate( &m_Stats); + } + +Exit: + + m_AbortRc = NE_SFLM_OK; + return( rc); +} + +/*API~*********************************************************************** +Area : TRANSACTION +Desc : Aborts an active transaction. +*END************************************************************************/ +RCODE F_Db::transAbort( void) +{ + RCODE rc = NE_SFLM_OK; + + if (m_eTransType == SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + rc = abortTrans(); + +Exit: + + if (RC_OK( rc)) + { + rc = checkState( __FILE__, __LINE__); + } + + return( rc); +} diff --git a/sql/src/fltrbeg.cpp b/sql/src/fltrbeg.cpp new file mode 100644 index 0000000..3cec0ea --- /dev/null +++ b/sql/src/fltrbeg.cpp @@ -0,0 +1,987 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for starting a transaction. +// +// Tabs: 3 +// +// Copyright (c) 1991, 1994-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fltrbeg.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: This routine unlinks an F_Db from a transaction's list of F_Dbs. +****************************************************************************/ +void F_Db::unlinkFromTransList( + FLMBOOL bCommitting) +{ + flmAssert( m_pIxdFixups == NULL); + if( m_eTransType != SFLM_NO_TRANS) + { + if (m_uiFlags & FDB_HAS_WRITE_LOCK) + { + + // If this is a commit operation and we have a commit callback, + // call the callback function before unlocking the DIB. + + if (bCommitting && m_pCommitClient) + { + m_pCommitClient->commit( this); + } + unlockExclusive(); + } + + m_pDatabase->lockMutex(); + if (m_pDict) + { + unlinkFromDict(); + } + + // Unlink the transaction from the F_Database if it is a read transaction. + + if (m_eTransType == SFLM_READ_TRANS) + { + if (m_pNextReadTrans) + { + m_pNextReadTrans->m_pPrevReadTrans = m_pPrevReadTrans; + } + else if (!m_uiKilledTime) + { + m_pDatabase->m_pLastReadTrans = m_pPrevReadTrans; + } + if (m_pPrevReadTrans) + { + m_pPrevReadTrans->m_pNextReadTrans = m_pNextReadTrans; + } + else if (m_uiKilledTime) + { + m_pDatabase->m_pFirstKilledTrans = m_pNextReadTrans; + } + else + { + m_pDatabase->m_pFirstReadTrans = m_pNextReadTrans; + } + + // Zero out so it will be zero for next transaction begin. + + m_uiKilledTime = 0; + } + else + { + + // Reset to NULL or zero for next update transaction. + + m_pIxStartList = m_pIxStopList = NULL; + flmAssert( !m_pIxdFixups); + } + + m_pDatabase->unlockMutex(); + m_eTransType = SFLM_NO_TRANS; + m_uiFlags &= (~(FDB_UPDATED_DICTIONARY | + FDB_DONT_KILL_TRANS | + FDB_DONT_POISON_CACHE | + FDB_SWEEP_SCHEDULED)); + flmAssert( !m_uiDirtyRowCount); + } +} + +/**************************************************************************** +Desc: This routine reads a database's dictionary. This is called only + when we did not have a dictionary off of the F_Database object - + which will be the first transaction after a database is opened. +****************************************************************************/ +RCODE F_Db::readDictionary( void) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = dictOpen())) + { + goto Exit; + } + + m_pDatabase->lockMutex(); + + // At this point, we will not yet have opened the database for + // general use, so there is no way that any other thread can have + // created a dictionary yet. + + flmAssert( !m_pDatabase->m_pDictList); + + // Link the new local dictionary to its file structure. + + m_pDict->linkToDatabase( m_pDatabase); + m_pDatabase->unlockMutex(); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine starts a transaction for the specified database. The + transaction may be part of an overall larger transaction. +****************************************************************************/ +RCODE F_Db::beginTrans( + eDbTransType eTransType, + FLMUINT uiMaxLockWait, + FLMUINT uiFlags, + SFLM_DB_HDR * pDbHdr) +{ + RCODE rc = NE_SFLM_OK; + SFLM_DB_HDR * pLastCommittedDbHdr; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + FLMBOOL bMutexLocked = FALSE; + + // Should not be calling on a temporary database + + flmAssert( !m_pDatabase->m_bTempDb); + + // Check the state of the database engine + + if( RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Initialize a few things - as few as is necessary to avoid + // unnecessary overhead. + + m_AbortRc = NE_SFLM_OK; + pLastCommittedDbHdr = &m_pDatabase->m_lastCommittedDbHdr; + m_bKrefSetup = FALSE; + m_eTransType = eTransType; + m_uiThreadId = (FLMUINT)f_threadId(); + m_uiTransCount++; + + // Link the F_Db to the database's most current F_Dict structure, + // if there is one. Also, if it is a read transaction, link the F_Db + // into the list of read transactions off of the F_Database object. + + m_pDatabase->lockMutex(); + bMutexLocked = TRUE; + + if (m_pDatabase->m_pDictList) + { + // Link the F_Db to the right F_Dict object + + linkToDict( m_pDatabase->m_pDictList); + } + + // If it is a read transaction, link into the list of + // read transactions off of the F_Database object. Until we + // get the DB header transaction ID below, we set ui64CurrTransID + // to zero and link this transaction in at the beginning of the + // list. + + if (eTransType == SFLM_READ_TRANS) + { + getDbHdrInfo( pLastCommittedDbHdr); + + // Link in at the end of the transaction list. + + m_pNextReadTrans = NULL; + if ((m_pPrevReadTrans = m_pDatabase->m_pLastReadTrans) != NULL) + { + // Make sure transaction IDs are always in ascending order. They + // should be at this point. + + flmAssert( m_pDatabase->m_pLastReadTrans->m_ui64CurrTransID <= + m_ui64CurrTransID); + m_pDatabase->m_pLastReadTrans->m_pNextReadTrans = this; + } + else + { + m_pDatabase->m_pFirstReadTrans = this; + } + m_pDatabase->m_pLastReadTrans = this; + m_uiInactiveTime = 0; + + if (uiFlags & SFLM_DONT_KILL_TRANS) + { + m_uiFlags |= FDB_DONT_KILL_TRANS; + } + else + { + m_uiFlags &= ~FDB_DONT_KILL_TRANS; + } + + if (pDbHdr) + { + f_memcpy( pDbHdr, &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + } + } + + m_pDatabase->unlockMutex(); + bMutexLocked = FALSE; + + if (uiFlags & SFLM_DONT_POISON_CACHE) + { + m_uiFlags |= FDB_DONT_POISON_CACHE; + } + else + { + m_uiFlags &= ~FDB_DONT_POISON_CACHE; + } + + // Put an exclusive lock on the database if we are not in a read + // transaction. Read transactions require no lock. + + if (eTransType != SFLM_READ_TRANS) + { + // Set the m_bHadUpdOper to TRUE for all transactions to begin with. + // Many calls to beginTrans are internal, and we WANT the + // normal behavior at the end of the transaction when it is + // committed or aborted. The only time this flag will be set + // to FALSE is when the application starts the transaction as + // opposed to an internal starting of the transaction. + + m_bHadUpdOper = TRUE; + + // Initialize the count of blocks changed to be 0 + + m_uiBlkChangeCnt = 0; + + if (RC_BAD( rc = lockExclusive( uiMaxLockWait))) + { + goto Exit; + } + + // If there was a problem with the RFL volume, we must wait + // for a checkpoint to be completed before continuing. + // The checkpoint thread looks at this same flag and forces + // a checkpoint. If it completes one successfully, it will + // reset this flag. + // Also, if the last forced checkpoint had a problem + // (pFile->CheckpointRc != NE_SFLM_OK), we don't want to + // start up a new update transaction until it is resolved. + + if( !pRfl->seeIfRflVolumeOk() || RC_BAD( m_pDatabase->m_CheckpointRc)) + + { + rc = RC_SET( NE_SFLM_MUST_WAIT_CHECKPOINT); + goto Exit; + } + + // Set the first log block address to zero. + + m_pDatabase->m_uiFirstLogBlkAddress = 0; + + // Header must be read before opening roll forward log file to make + // sure we have the most current log file and log options. + + f_memcpy( &m_pDatabase->m_uncommittedDbHdr, pLastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + getDbHdrInfo( pLastCommittedDbHdr); + + // Need to increment the current checkpoint for update transactions + // so that it will be correct when we go to mark cache blocks. + + if (m_uiFlags & FDB_REPLAYING_RFL) + { + // During recovery we need to set the transaction ID to the + // transaction ID that was logged. + + m_ui64CurrTransID = pRfl->getCurrTransID(); + } + else + { + m_ui64CurrTransID++; + } + + // Link F_Db to the most current local dictionary, if there + // is one. + + m_pDatabase->lockMutex(); + if (m_pDatabase->m_pDictList != m_pDict && + m_pDatabase->m_pDictList) + { + linkToDict( m_pDatabase->m_pDictList); + } + m_pDatabase->unlockMutex(); + + // Set the transaction EOF to the current file EOF + + m_uiTransEOF = m_uiLogicalEOF; + + // Put the transaction ID into the uncommitted log header. + + m_pDatabase->m_uncommittedDbHdr.ui64CurrTransID = m_ui64CurrTransID; + + if (pDbHdr) + { + f_memcpy( pDbHdr, &m_pDatabase->m_uncommittedDbHdr, + sizeof( SFLM_DB_HDR)); + } + } + + // Set up to collect statistics. We only do this at transaction + // begin and not on any other type of operation. So this is the + // only time when an F_Db will sense that statistics have been + // turned on or off. + + if (!gv_SFlmSysData.Stats.bCollectingStats) + { + m_pStats = NULL; + m_pDbStats = NULL; + } + else + { + m_pStats = &m_Stats; + + // Statistics are being collected for the system. Therefore, + // if we are not currently collecting statistics in the + // session, start. If we were collecting statistics, but the + // start time was earlier than the start time in the system + // statistics structure, reset the statistics in the session. + + if (!m_Stats.bCollectingStats) + { + flmStatStart( &m_Stats); + } + else if (m_Stats.uiStartTime < gv_SFlmSysData.Stats.uiStartTime) + { + flmStatReset( &m_Stats, FALSE); + } + (void)flmStatGetDb( &m_Stats, m_pDatabase, + 0, &m_pDbStats, NULL, NULL); + m_pLFileStats = NULL; + } + + if (m_pDbStats) + { + f_timeGetTimeStamp( &m_TransStartTime); + } + + // If we do not have a dictionary, read it in from disk. + // NOTE: This should only happen when we are first opening + // the database. + + if (!m_pDict) + { + if (RC_BAD( rc = readDictionary())) + { + goto Exit; + } + } + +Exit: + + if( bMutexLocked) + { + m_pDatabase->unlockMutex(); + } + + if (eTransType != SFLM_READ_TRANS) + { + if (RC_OK( rc)) + { + rc = pRfl->logBeginTransaction( this); + } +#ifdef FLM_DBG_LOG + flmDbgLogUpdate( m_pDatabase, m_ui64CurrTransID, + 0, 0, rc, "TBeg"); +#endif + } + + if (eTransType == SFLM_UPDATE_TRANS && + gv_SFlmSysData.EventHdrs [SFLM_EVENT_UPDATES].pEventCBList) + { + flmTransEventCallback( SFLM_EVENT_BEGIN_TRANS, this, rc, + (FLMUINT)(RC_OK( rc) + ? m_ui64CurrTransID + : (FLMUINT64)0)); + } + + if (RC_BAD( rc)) + { + // If there was an error, unlink the database from the transaction + // structure as well as from the FDICT structure. Also dump any nodes + // that are already in the cache. + + unlinkFromTransList( FALSE); + + if (m_pStats) + { + (void)flmStatUpdate( &m_Stats); + } + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine starts a transaction for the specified database. It uses + the transaction information in the passed in pDb. +****************************************************************************/ +RCODE F_Db::beginTrans( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + + // Should not be calling on a temporary database + + flmAssert( !m_pDatabase->m_bTempDb); + + // pDb better be running a read transaction. + + flmAssert( pDb->m_eTransType == SFLM_READ_TRANS); + + if( RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Initialize a few things - as few as is necessary to avoid + // unnecessary overhead. + + m_AbortRc = NE_SFLM_OK; + m_bKrefSetup = FALSE; + m_eTransType = SFLM_READ_TRANS; + m_uiThreadId = (FLMUINT)f_threadId(); + m_uiTransCount++; + + // Link the F_Db to the database's most current F_Dict structure, + // if there is one. Also, if it is a read transaction, link the F_Db + // into the list of read transactions off of the F_Database object. + + m_pDatabase->lockMutex(); + bMutexLocked = TRUE; + + // Link to the same dictionary as pDb. + + linkToDict( pDb->m_pDict); + + // If it is a read transaction, link into the list of + // read transactions off of the F_Database object. Until we + // get the DB header transaction ID below, we set ui64CurrTransID + // to zero and link this transaction in at the beginning of the + // list. + + getDbHdrInfo( pDb); + + // Link into the transaction list right after the point where + // pDb is linked in. We need to keep transaction IDs in ascending + // order. + + m_pPrevReadTrans = pDb; + if ((m_pNextReadTrans = pDb->m_pNextReadTrans) != NULL) + { + m_pNextReadTrans->m_pPrevReadTrans = this; + } + else + { + m_pDatabase->m_pLastReadTrans = this; + } + pDb->m_pNextReadTrans = this; + + m_uiInactiveTime = 0; + + if (pDb->m_uiFlags & FDB_DONT_KILL_TRANS) + { + m_uiFlags |= FDB_DONT_KILL_TRANS; + } + else + { + m_uiFlags &= ~FDB_DONT_KILL_TRANS; + } + if (pDb->m_uiFlags & FDB_DONT_POISON_CACHE) + { + m_uiFlags |= FDB_DONT_POISON_CACHE; + } + else + { + m_uiFlags &= ~FDB_DONT_POISON_CACHE; + } + + m_pDatabase->unlockMutex(); + bMutexLocked = FALSE; + + // Set up to collect statistics. We only do this at transaction + // begin and not on any other type of operation. So this is the + // only time when an F_Db will sense that statistics have been + // turned on or off. + + if (!gv_SFlmSysData.Stats.bCollectingStats) + { + m_pStats = NULL; + m_pDbStats = NULL; + } + else + { + m_pStats = &m_Stats; + + // Statistics are being collected for the system. Therefore, + // if we are not currently collecting statistics in the + // session, start. If we were collecting statistics, but the + // start time was earlier than the start time in the system + // statistics structure, reset the statistics in the session. + + if (!m_Stats.bCollectingStats) + { + flmStatStart( &m_Stats); + } + else if (m_Stats.uiStartTime < gv_SFlmSysData.Stats.uiStartTime) + { + flmStatReset( &m_Stats, FALSE); + } + (void)flmStatGetDb( &m_Stats, m_pDatabase, + 0, &m_pDbStats, NULL, NULL); + m_pLFileStats = NULL; + } + + if (m_pDbStats) + { + f_timeGetTimeStamp( &m_TransStartTime); + } + +Exit: + + if( bMutexLocked) + { + m_pDatabase->unlockMutex(); + } + + if (RC_BAD( rc)) + { + // If there was an error, unlink the database from the transaction + // structure as well as from the FDICT structure. Also dump any nodes + // that are already in the cache. + + unlinkFromTransList( FALSE); + + if (m_pStats) + { + (void)flmStatUpdate( &m_Stats); + } + } + + return( rc); +} + +/**************************************************************************** +Desc : Starts a transaction. +****************************************************************************/ +RCODE F_Db::transBegin( + eDbTransType eTransType, + FLMUINT uiMaxLockWait, + FLMUINT uiFlags, + SFLM_DB_HDR * pDbHdr) +{ + RCODE rc = NE_SFLM_OK; + + // Verify the transaction type. + + if (eTransType != SFLM_UPDATE_TRANS && eTransType != SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_TYPE); + goto Exit; + } + + // Verify the transaction flags + + if ((uiFlags & SFLM_DONT_KILL_TRANS) && eTransType != SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_TYPE); + goto Exit; + } + + // Can't start an update transaction on a database that + // is locked in shared mode. + + if (eTransType == SFLM_UPDATE_TRANS && (m_uiFlags & FDB_FILE_LOCK_SHARED)) + { + rc = RC_SET( NE_SFLM_SHARED_LOCK); + goto Exit; + } + + // If the database is not running a transaction, start one. + + if (m_eTransType != SFLM_NO_TRANS) + { + + // Cannot nest transactions. + + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + if (RC_BAD( rc = beginTrans( eTransType, + uiMaxLockWait, uiFlags, pDbHdr))) + { + goto Exit; + } + + m_bHadUpdOper = FALSE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc : Starts a transaction. +****************************************************************************/ +RCODE F_Db::transBegin( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + + // Database cannot already be running a transaction. + + if (m_eTransType != SFLM_NO_TRANS) + { + + // Cannot nest transactions. + + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // Verify the transaction type. + + if (((F_Db *)pDb)->m_eTransType != SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_ILLEGAL_TRANS_TYPE); + goto Exit; + } + + if (RC_BAD( rc = beginTrans( (F_Db *)pDb))) + { + goto Exit; + } + m_bHadUpdOper = FALSE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc : Obtains a a lock on the database. +****************************************************************************/ +RCODE F_Db::dbLock( + eDbLockType eLockType, + FLMINT iPriority, + FLMUINT uiTimeout) +{ + RCODE rc = NE_SFLM_OK; + + // eLockType better be exclusive or shared + + if (eLockType != SFLM_LOCK_EXCLUSIVE && eLockType != SFLM_LOCK_SHARED) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Nesting of locks is not allowed - this test also keeps this call from + // being executed inside an update transaction that implicitly acquired + // the lock. + + if (m_uiFlags & + (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_SHARED | FDB_FILE_LOCK_IMPLICIT)) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Attempt to acquire the lock. + + if (RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Lock( this, m_hWaitSem, + TRUE, FALSE, (FLMBOOL)((eLockType == SFLM_LOCK_EXCLUSIVE) + ? (FLMBOOL)TRUE + : (FLMBOOL)FALSE), + uiTimeout, iPriority, m_pDbStats))) + { + goto Exit; + } + m_uiFlags |= FDB_HAS_FILE_LOCK; + if (eLockType == SFLM_LOCK_SHARED) + { + m_uiFlags |= FDB_FILE_LOCK_SHARED; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc : Releases a lock on the database +****************************************************************************/ +RCODE F_Db::dbUnlock( void) +{ + RCODE rc = NE_SFLM_OK; + + // If we don't have an explicit lock, can't do the unlock. It is + // also illegal to do the unlock during an update transaction. + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK) || + (m_uiFlags & FDB_FILE_LOCK_IMPLICIT) || + (m_eTransType == SFLM_UPDATE_TRANS)) + { + rc = RC_SET( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Unlock the file. + + if (RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this))) + { + goto Exit; + } + + // Unset the flags that indicated the file was explicitly locked. + + m_uiFlags &= (~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_SHARED)); + +Exit: + + if (RC_OK( rc)) + { + rc = checkState( __FILE__, __LINE__); + } + + return( rc); +} + +/**************************************************************************** +Desc : Returns information about current and pending locks on the + database. +****************************************************************************/ +RCODE F_Db::getLockInfo( + FLMINT iPriority, + eDbLockType * peCurrLockType, + FLMUINT * puiThreadId, + FLMUINT * puiNumExclQueued, + FLMUINT * puiNumSharedQueued, + FLMUINT * puiPriorityCount) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + m_pDatabase->m_pDatabaseLockObj->GetLockInfo( iPriority, + peCurrLockType, puiThreadId, + puiNumExclQueued, puiNumSharedQueued, + puiPriorityCount); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc : Returns information about the lock held by the specified database + handle. +****************************************************************************/ +RCODE F_Db::getLockType( + eDbLockType * peLockType, + FLMBOOL * pbImplicit) +{ + RCODE rc = NE_SFLM_OK; + + if (peLockType) + { + *peLockType = SFLM_LOCK_NONE; + } + + if (pbImplicit) + { + *pbImplicit = FALSE; + } + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + if (m_uiFlags & FDB_HAS_FILE_LOCK) + { + if (peLockType) + { + if (m_uiFlags & FDB_FILE_LOCK_SHARED) + { + *peLockType = SFLM_LOCK_SHARED; + } + else + { + *peLockType = SFLM_LOCK_EXCLUSIVE; + } + } + + if (pbImplicit) + { + *pbImplicit = (m_uiFlags & FDB_FILE_LOCK_IMPLICIT) + ? TRUE + : FALSE; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc : Forces a checkpoint on the database. +****************************************************************************/ +RCODE F_Db::doCheckpoint( + FLMUINT uiTimeout) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Start an update transaction. Must not already be one going. + + if (m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // If we get to this point, we need to start a transaction on the + // database. + + if( RC_BAD( rc = beginTrans( SFLM_UPDATE_TRANS, uiTimeout))) + { + goto Exit; + } + + // Commit the transaction, forcing it to be checkpointed. + + m_bHadUpdOper = FALSE; + if (RC_BAD( rc = commitTrans( 0, TRUE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine locks a database for exclusive access. +****************************************************************************/ +RCODE F_Db::lockExclusive( + FLMUINT uiMaxLockWait) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bGotFileLock = FALSE; + + flmAssert( !m_pDatabase->m_bTempDb); + + // There must NOT be a shared lock on the file. + + if (m_uiFlags & FDB_FILE_LOCK_SHARED) + { + rc = RC_SET( NE_SFLM_SHARED_LOCK); + goto Exit; + } + + // Must acquire an exclusive file lock first, if it hasn't been + // acquired. + + if (!(m_uiFlags & FDB_HAS_FILE_LOCK)) + { + if (RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Lock( this, + m_hWaitSem, TRUE, FALSE, TRUE, uiMaxLockWait, 0, m_pDbStats))) + { + goto Exit; + } + bGotFileLock = TRUE; + m_uiFlags |= (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); + } + + if (RC_OK( rc = m_pDatabase->dbWriteLock( m_hWaitSem, m_pDbStats))) + { + m_uiFlags |= FDB_HAS_WRITE_LOCK; + } + +Exit: + + if (rc == NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT) + { + if (bGotFileLock) + { + (void)m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this); + m_uiFlags &= (~(FDB_HAS_FILE_LOCK | + FDB_FILE_LOCK_IMPLICIT | FDB_HAS_WRITE_LOCK)); + } + + if (m_eTransType != SFLM_NO_TRANS) + { + + // Unlink the DB from the transaction. + + unlinkFromTransList( FALSE); + } + } + else if (RC_BAD( rc)) + { + if (bGotFileLock) + { + (void)m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this); + m_uiFlags &= (~(FDB_HAS_FILE_LOCK | + FDB_FILE_LOCK_IMPLICIT | FDB_HAS_WRITE_LOCK)); + } + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine unlocks a database that was previously locked + using the lockExclusive routine. +****************************************************************************/ +void F_Db::unlockExclusive( void) +{ + flmAssert( !m_pDatabase->m_bTempDb); + + // If we have the write lock, unlock it first. + + flmAssert( m_uiFlags & FDB_HAS_WRITE_LOCK); + + m_pDatabase->dbWriteUnlock( m_pDbStats); + m_uiFlags &= ~FDB_HAS_WRITE_LOCK; + + // Give up the file lock, if it was acquired implicitly. + + if (m_uiFlags & FDB_FILE_LOCK_IMPLICIT) + { + (void)m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this); + m_uiFlags &= (~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT)); + } + +} diff --git a/sql/src/fltrcmit.cpp b/sql/src/fltrcmit.cpp new file mode 100644 index 0000000..69b4e2a --- /dev/null +++ b/sql/src/fltrcmit.cpp @@ -0,0 +1,554 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for committing a transaction. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fltrcmit.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: This routine commits an active transaction for a particular + database. +****************************************************************************/ +RCODE F_Db::commitTrans( + FLMUINT uiNewLogicalEOF, // New logical end-of-file. This is only + // set by the reduceSize function when + // it is truncating the file. + FLMBOOL bForceCheckpoint, // Force a checkpoint? + FLMBOOL * pbEmpty + ) +{ + RCODE rc = NE_SFLM_OK; + SFLM_DB_HDR * pUncommittedDbHdr; + FLMUINT uiCPFileNum = 0; + FLMUINT uiCPOffset = 0; + FLMUINT64 ui64TransId = 0; + FLMBOOL bTransEndLogged; + FLMBOOL bForceCloseOnError = FALSE; + eDbTransType eSaveTransType; + FLMBOOL bOkToLogAbort = TRUE; + FLMBOOL bIndexAfterCommit = FALSE; + F_Rfl * pRfl = m_pDatabase->m_pRfl; + F_TABLE * pTable; + FLMUINT uiLfNum; + + // Not allowed to commit temporary databases. + + flmAssert( !m_pDatabase->m_bTempDb); + + // See if we even have a transaction going. + + if (m_eTransType == SFLM_NO_TRANS) + { + goto Exit; // Will return NE_SFLM_OK. + } + + // See if we have a transaction going which should be aborted. + + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit1; + } + + // If we are in a read transaction we can skip most of the stuff + // below because no updates would have occurred. This will help + // improve performance. + + if (m_eTransType == SFLM_READ_TRANS) + { + if (m_bKrefSetup) + { + + // krefCntrlFree could be called w/o checking bKrefSetup because + // it checks the flag, but it is more optimal to check the + // flag before making the call because most of the time it will + // be false. + + krefCntrlFree(); + } + + goto Exit1; + } + + // Disable DB header writes + + pRfl->clearDbHdrs(); + + // At this point, we know we have an update transaction. + + ui64TransId = m_ui64CurrTransID; +#ifdef FLM_DBG_LOG + flmDbgLogUpdate( m_pDatabase, ui64TransId, 0, 0, NE_SFLM_OK, "TCmit"); +#endif + + // Flush any dirty rows + + if( RC_BAD( rc = flushDirtyRows())) + { + goto Exit1; + } + + // Write out any LFILE changes for collections + + flmAssert( m_pDict); + + // Only need to do collections. Nothing from the LFH of + // an index is stored in memory except for the root block + // address, and whenever that is changed, we get a new + // dictionary. Since the new dictionary will be discarded + // in that case, there is nothing to restore for an index. + + for (uiLfNum = 1, pTable = m_pDict->m_pTableTbl; + uiLfNum <= m_pDict->m_uiHighestTableNum; + uiLfNum++, pTable++) + { + if (pTable->uiTableNum && pTable->lfInfo.bNeedToWriteOut) + { + if (RC_BAD( rc = m_pDatabase->lFileWrite( this, &pTable->lfInfo))) + { + goto Exit; + } + } + } + + // Commit any keys in the KREF buffers. + + if (RC_BAD( rc = keysCommit( TRUE))) + { + flmLogError( rc, "calling keysCommit from commitTrans"); + goto Exit1; + } + + // If the transaction had no update operations, restore it + // to its pre-transaction state - make it appear that no + // transaction ever happened. + + if (!m_bHadUpdOper) + { + bOkToLogAbort = FALSE; + + rc = pRfl->logEndTransaction( this, RFL_TRNS_COMMIT_PACKET, TRUE); + + // Even though we didn't have any update operations, there may have + // been operations during the transaction (i.e., query operations) + // that initialized the KREF in order to generate keys. + + krefCntrlFree(); + + // Restore everything as if the transaction never happened. + + if (pbEmpty) + { + *pbEmpty = TRUE; + } + + // Even if the transaction is empty, there could be "uncommitted" rows that + // were created, but never set to "dirty" - hence the m_bHadUpdOper flag + // would never have been set. So we need to call freeModifiedRows to get + // rid of them. + + m_pDatabase->freeModifiedRows( this, ui64TransId - 1); + goto Exit1; + } + + // Log commit record to roll-forward log + + bOkToLogAbort = FALSE; + if (RC_BAD( rc = pRfl->logEndTransaction( + this, RFL_TRNS_COMMIT_PACKET, FALSE, &bTransEndLogged))) + { + goto Exit1; + } + bForceCloseOnError = TRUE; + + // Reinitialize the log header. If the local dictionary was updated + // during the transaction, increment the local dictionary ID so that + // other concurrent users will know that it has been modified and + // that they need to re-read it into memory. + + // If we are in recovery mode, see if we need to force + // a checkpoint with what we have so far. We force a + // checkpoint on one of two conditions: + + // 1. If it appears that we have a buildup of dirty cache + // blocks. We force a checkpoint on this condition + // because it will be more efficient than replacing + // cache blocks one at a time. + // We check for this condition by looking to see if + // our LRU block is not used and it is dirty. That is + // a pretty good indicator that we have a buildup + // of dirty cache blocks. + // 2. We are at the end of the roll-forward log. We + // want to force a checkpoint here to complete the + // recovery phase. + + if (m_uiFlags & FDB_REPLAYING_RFL) + { + // If we are in the middle of upgrading, and are forcing + // a checkpoint, use the file number and offset that were + // set in the F_Db. + + if ((m_uiFlags & FDB_UPGRADING) && bForceCheckpoint) + { + uiCPFileNum = m_uiUpgradeCPFileNum; + uiCPOffset = m_uiUpgradeCPOffset; + } + else + { + FLMUINT uiCurrTime; + + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + f_mutexLock( gv_SFlmSysData.hShareMutex); + if (FLM_ELAPSED_TIME( uiCurrTime, m_pDatabase->m_uiLastCheckpointTime) >= + gv_SFlmSysData.uiMaxCPInterval || + !gv_SFlmSysData.uiMaxCPInterval || + (gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache && + (m_pDatabase->m_uiDirtyCacheCount + + m_pDatabase->m_uiLogCacheCount) * m_pDatabase->m_uiBlockSize > + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache) || + pRfl->atEndOfLog() || + bForceCheckpoint) + { + bForceCheckpoint = TRUE; + uiCPFileNum = pRfl->getCurrFileNum(); + uiCPOffset = pRfl->getCurrReadOffset(); + } + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } + } + + // Move information collected in the pDb into the + // uncommitted DB header. Other things that need to be + // set have already been set in the uncommitted log header + // at various places in the code. + + // Mutex does not have to be locked while we do this because + // the update transaction is the only one that ever accesses + // the uncommitted log header buffer. + + pUncommittedDbHdr = &m_pDatabase->m_uncommittedDbHdr; + + // Set the new logical EOF if passed in. + + if (uiNewLogicalEOF) + { + m_uiLogicalEOF = uiNewLogicalEOF; + } + pUncommittedDbHdr->ui32LogicalEOF = (FLMUINT32)m_uiLogicalEOF; + + // Increment the commit counter. + + pUncommittedDbHdr->ui64TransCommitCnt++; + + // Set the last committed transaction ID + + if (bTransEndLogged || (m_uiFlags & FDB_REPLAYING_COMMIT)) + { + pUncommittedDbHdr->ui64LastRflCommitID = ui64TransId; + } + + // Write the header + + pRfl->commitDbHdrs( pUncommittedDbHdr, + &m_pDatabase->m_checkpointDbHdr); + + // Commit row cache. + + m_pDatabase->commitRowCache(); + + // Push the IXD_FIXUP values back into the F_INDEX + + if (m_pIxdFixups) + { + IXD_FIXUP * pIxdFixup; + IXD_FIXUP * pDeleteIxdFixup; + F_INDEX * pIndex; + + pIxdFixup = m_pIxdFixups; + while (pIxdFixup) + { + if ((pIndex = m_pDict->getIndex( pIxdFixup->uiIndexNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_INDEX_NUM); + pIndex = NULL; + } + + if (pIndex) + { + pIndex->ui64LastRowIndexed = pIxdFixup->ui64LastRowIndexed; + } + pDeleteIxdFixup = pIxdFixup; + pIxdFixup = pIxdFixup->pNext; + f_free( &pDeleteIxdFixup); + } + m_pIxdFixups = NULL; + } + + // Set the update transaction ID back to zero only + // AFTER we know the transaction has safely committed. + + m_pDatabase->lockMutex(); + + f_memcpy( &m_pDatabase->m_lastCommittedDbHdr, pUncommittedDbHdr, + sizeof( SFLM_DB_HDR)); + + if (m_uiFlags & FDB_UPDATED_DICTIONARY) + { + + // Link the new local dictionary to its file. + // Since the new local dictionary will be linked at the head + // of the list of FDICT structures, see if the FDICT currently + // at the head of the list is unused and can be unlinked. + + if (m_pDatabase->m_pDictList && !m_pDatabase->m_pDictList->getUseCount()) + { + m_pDatabase->m_pDictList->unlinkFromDatabase(); + } + m_pDict->linkToDatabase( m_pDatabase); + } + m_pDatabase->unlockMutex(); + + // Log blocks must be released after the last committed database header + // has been updated. If done before, there is a slight window of + // opportunity for a caller to attempt to start a read transaction. Between + // the time releaseLogBlocks unlocks the block cache mutex and when + // the committed header is copied into m_pDatabase, the read transaction + // could see the prior committed transaction ID and use that as the basis + // for its transaction. In the mean time, releaseLogBlocks may release + // versions of the blocks that would be needed to satisfy the new reader's + // (incorrect) view of the database. + + m_pDatabase->releaseLogBlocks(); + +Exit1: + + if (RC_BAD( rc)) + { + // Since we failed to commit, do an abort. We are purposely not + // checking the return code from flmAbortDbTrans because we already + // have an error return code. If we attempted to log the transaction + // to the RFL and failed, we don't want to try to log an abort packet. + // The RFL code has already reset the log back to the starting point + // of the transaction, thereby discarding all operations. + + (void)abortTrans( bOkToLogAbort); + eSaveTransType = SFLM_NO_TRANS; + + // Do we need to force all handles to close? + + if (bForceCloseOnError) + { + // Since the commit packet has already been logged to the RFL, + // we must have failed when trying to write the log header. The + // database is in a bad state and must be closed. + + // Set the "must close" flag on all FDBs linked to the FFILE + // and set the FFILE's "must close" flag. This will cause any + // subsequent operations on the database to fail until all + // handles have been closed. + + m_pDatabase->setMustCloseFlags( rc, FALSE); + } + } + else + { + eSaveTransType = m_eTransType; + if (m_eTransType == SFLM_UPDATE_TRANS) + { + if (gv_SFlmSysData.EventHdrs [SFLM_EVENT_UPDATES].pEventCBList) + { + flmTransEventCallback( SFLM_EVENT_COMMIT_TRANS, this, rc, + ui64TransId); + } + + // Do the indexing work before we unlock the db. + + if (m_pIxStopList || m_pIxStartList) + { + // Must not call indexingAfterCommit until after + // completeTransWrites. Otherwise, there is a potential + // deadlock condition where indexingAfterCommit is + // waiting on an indexing thread to quit, but that + // thread is waiting to be signaled by this thread that + // writes are completed. However, indexingAfterCommit + // also must only be called while the database is still + // locked. If we were to leave the database locked for + // every call to completeTransWrites, however, we would + // lose the group commit capability. Hence, we opt to + // only lose it when there are actual indexing operations + // to start or stop - which should be very few transactions. + // That is what the bIndexAfterCommit flag is for. + + bIndexAfterCommit = TRUE; + } + } + } + + // Unlock the database, if the update transaction is still going. + // NOTE: We check m_eTransType because it may have been reset + // to SFLM_NO_TRANS up above if abortTrans was called. + + if (m_eTransType == SFLM_UPDATE_TRANS) + { + if (RC_BAD( rc)) + { + + // SHOULD NEVER HAPPEN - because it would have been taken + // care of above - abortTrans would have been called and + // m_eTransType would no longer be SFLM_UPDATE_TRANS. + + RC_UNEXPECTED_ASSERT( rc); + pRfl->completeTransWrites( this, FALSE, TRUE); + } + else if (!bForceCheckpoint) + { + if (bIndexAfterCommit) + { + rc = pRfl->completeTransWrites( this, TRUE, FALSE); + indexingAfterCommit(); + unlinkFromTransList( TRUE); + } + else + { + rc = pRfl->completeTransWrites( this, TRUE, TRUE); + } + } + else + { + // Do checkpoint, if forcing. Before doing the checkpoint + // we have to make sure the roll-forward log writes + // complete. We don't want to unlock the DB while the + // writes are happening in this case - thus, the FALSE + // parameter to completeTransWrites. + + if (RC_OK( rc = pRfl->completeTransWrites( this, TRUE, FALSE))) + { + bForceCloseOnError = FALSE; + rc = m_pDatabase->doCheckpoint( m_hWaitSem, m_pDbStats, m_pSFileHdl, + (m_uiFlags & FDB_DO_TRUNCATE) ? TRUE : FALSE, + TRUE, SFLM_CP_TIME_INTERVAL_REASON, + uiCPFileNum, uiCPOffset); + } + + if (bIndexAfterCommit) + { + indexingAfterCommit(); + } + + unlinkFromTransList( TRUE); + } + + if (RC_BAD( rc) && bForceCloseOnError) + { + + // Since the commit packet has already been logged to the RFL, + // we must have failed when trying to write the log header. The + // database is in a bad state and must be closed. + + // Set the "must close" flag on all F_Db objects linked to the + // F_Database object and set the F_Database object's "must close" + // flag. This will cause any subsequent operations on the + // database to fail until all handles have been closed. + + m_pDatabase->setMustCloseFlags( rc, FALSE); + } + } + else + { + + // Unlink the database from the transaction list + + unlinkFromTransList( FALSE); + } + + if (m_pDbStats && eSaveTransType != SFLM_NO_TRANS) + { + FLMUINT64 ui64ElapMilli = 0; + + flmAddElapTime( &m_TransStartTime, &ui64ElapMilli); + m_pDbStats->bHaveStats = TRUE; + if (eSaveTransType == SFLM_READ_TRANS) + { + m_pDbStats->ReadTransStats.CommittedTrans.ui64Count++; + m_pDbStats->ReadTransStats.CommittedTrans.ui64ElapMilli += + ui64ElapMilli; + } + else + { + m_pDbStats->UpdateTransStats.CommittedTrans.ui64Count++; + m_pDbStats->UpdateTransStats.CommittedTrans.ui64ElapMilli += + ui64ElapMilli; + } + } + + // Update stats + + if (m_pStats) + { + (void)flmStatUpdate( &m_Stats); + } + +Exit: + + return( rc); +} + +/*API~*********************************************************************** +Area : TRANSACTION +Desc : Commits an active transaction. +*END************************************************************************/ +RCODE F_Db::transCommit( + FLMBOOL * pbEmpty) // may be NULL +{ + RCODE rc = NE_SFLM_OK; + + if (m_eTransType == SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_NO_TRANS_ACTIVE); + goto Exit; + } + + // See if we have a transaction going which should be aborted. + + if (RC_BAD( m_AbortRc)) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + + if (pbEmpty) + { + *pbEmpty = FALSE; + } + + rc = commitTrans( 0, FALSE, pbEmpty); + +Exit: + + if( RC_OK( rc)) + { + rc = checkState( __FILE__, __LINE__); + } + + return( rc); +} diff --git a/sql/src/fnumber.cpp b/sql/src/fnumber.cpp new file mode 100644 index 0000000..2d66689 --- /dev/null +++ b/sql/src/fnumber.cpp @@ -0,0 +1,1001 @@ +//------------------------------------------------------------------------------ +// Desc: Routines that do conversions between internal number and numeric +// key format to platform number types. +// +// Tabs: 3 +// +// Copyright (c) 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fnumber.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#define DEFINE_NUMBER_MAXIMUMS +#include "flaimsys.h" + +/**************************************************************************** +Desc: Converts a UINT to its storage value +*****************************************************************************/ +RCODE FlmUINT2Storage( + FLMUINT uiNum, + FLMUINT * puiBufLen, // In (buffer size) / Out (bytes used) + FLMBYTE * pucBuf) +{ + RCODE rc = NE_SFLM_OK; + + if( RC_BAD( rc = flmNumber64ToStorage( uiNum, puiBufLen, + pucBuf, FALSE, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts an INT to its storage value +*****************************************************************************/ +RCODE FlmINT2Storage( + FLMINT iNum, + FLMUINT * puiBufLen, // In (buffer size) / Out (bytes used) + FLMBYTE * pucBuf) +{ + FLMBOOL bNeg = FALSE; + RCODE rc = NE_SFLM_OK; + + if( iNum < 0) + { + iNum = -iNum; + bNeg = TRUE; + } + + if( RC_BAD( rc = flmNumber64ToStorage( (FLMUINT64)iNum, + puiBufLen, pucBuf, bNeg, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a number to its storage or collation format +Notes: Changes to this code must also be made to flmUINT32ToStorage +*****************************************************************************/ +RCODE flmNumber64ToStorage( + FLMUINT64 ui64Num, + FLMUINT * puiBufLen, + FLMBYTE * pucBuf, + FLMBOOL bNegative, + FLMBOOL bCollation) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiByteCount = 0; + FLMUINT32 ui32Low = (FLMUINT32)ui64Num; + FLMUINT32 ui32High = (FLMUINT32)(ui64Num >> 32); + + if( *puiBufLen < FLM_MAX_NUM_BUF_SIZE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + // Negative flag should not be set if the number is 0 + + if( !ui64Num && bNegative) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // Build the storage format, either for an index key (if bCollation + // is TRUE) or for a data element. + + if( !bCollation) + { + // Build the data storage representation of the number + // Numbers are stored in a little-endian format + + do + { + *pucBuf++ = (FLMBYTE)ui32Low; + ui32Low >>= 8; + uiByteCount++; + } while (ui32Low); + + if( ui32High) + { + for( ; uiByteCount < 4; uiByteCount++) + { + *pucBuf++ = 0; + } + + for( ; ui32High; uiByteCount++) + { + *pucBuf++ = (FLMBYTE)ui32High; + ui32High >>= 8; + } + } + + // If a number is negative, the high bit of the right-most + // byte needs to be set. If the value of the number is such + // that its positive representation already has the high-bit + // set, we need to store an additional sign byte. + + if( !bNegative) + { + if( *(pucBuf - 1) & 0x80) + { + *pucBuf++ = 0; + uiByteCount++; + } + } + else + { + if( (*(pucBuf - 1) & 0x80) == 0) + { + *(pucBuf - 1) |= 0x80; + } + else + { + *pucBuf++ = 0x80; + uiByteCount++; + } + } + } + else + { + FLMBYTE * pucStart = pucBuf++; + + if( ui32High) + { + if( ui32High & 0xFF000000) + { + *pucBuf++ = (FLMBYTE)(ui32High >> 24); + *pucBuf++ = (FLMBYTE)(ui32High >> 16); + *pucBuf++ = (FLMBYTE)(ui32High >> 8); + *pucBuf++ = (FLMBYTE)ui32High; + uiByteCount += 4; + } + else if( ui32High & 0x00FF0000) + { + *pucBuf++ = (FLMBYTE)(ui32High >> 16); + *pucBuf++ = (FLMBYTE)(ui32High >> 8); + *pucBuf++ = (FLMBYTE)ui32High; + uiByteCount += 3; + } + else if( ui32High & 0x0000FF00) + { + *pucBuf++ = (FLMBYTE)(ui32High >> 8); + *pucBuf++ = (FLMBYTE)ui32High; + uiByteCount += 2; + } + else if( ui32High) + { + *pucBuf++ = (FLMBYTE)ui32High; + uiByteCount++; + } + } + + if( ui32Low) + { + if( ui32Low & 0xFF000000) + { + *pucBuf++ = (FLMBYTE)(ui32Low >> 24); + *pucBuf++ = (FLMBYTE)(ui32Low >> 16); + *pucBuf++ = (FLMBYTE)(ui32Low >> 8); + *pucBuf++ = (FLMBYTE)ui32Low; + uiByteCount += 4; + } + else if( ui32Low & 0x00FF0000) + { + *pucBuf++ = (FLMBYTE)(ui32Low >> 16); + *pucBuf++ = (FLMBYTE)(ui32Low >> 8); + *pucBuf++ = (FLMBYTE)ui32Low; + uiByteCount += 3; + } + else if( ui32Low & 0x0000FF00) + { + *pucBuf++ = (FLMBYTE)(ui32Low >> 8); + *pucBuf++ = (FLMBYTE)ui32Low; + uiByteCount += 2; + } + else if( ui32Low) + { + *pucBuf++ = (FLMBYTE)ui32Low; + uiByteCount++; + } + } + else if( !ui32High) + { + *pucBuf++ = 0; + uiByteCount++; + } + + if( !bNegative) + { + // Positive numbers must collate after negative numbers, + // so all positive numbers will start with a byte + // in the range of 0xC8 - 0xCF. + + *pucStart = (FLMBYTE)(0xC8 + (uiByteCount - 1)); + uiByteCount++; + } + else + { + FLMBYTE * pucTmp = pucStart + 1; + + while( pucTmp < pucBuf) + { + *pucTmp = ~(*pucTmp); + pucTmp++; + } + + // Negative numbers must collate before positive numbers, + // so all negative numbers will start with a byte + // in the range of 0xC0 - 0xC7. + + *pucStart = (FLMBYTE)(0xC8 - uiByteCount); + uiByteCount++; + } + } + + // Set the number of bytes in the buffer before returning. + + *puiBufLen = uiByteCount; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a storage value back into a number +Notes: Changes to this code must also be made to flmStorage2Number64 +*****************************************************************************/ +RCODE flmStorage2Number( + eDataType eDataTyp, + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT * puiNum, + FLMINT * piNum) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + FLMUINT uiNum = 0; + FLMBOOL bNeg = FALSE; + + if( !uiBufLen) + { + if( puiNum) + { + *puiNum = 0; + } + else + { + *piNum = 0; + } + goto Exit; + } + + if( !pucBuf) + { + rc = RC_SET( NE_SFLM_CONV_NULL_SRC); + goto Exit; + } + + switch( eDataTyp) + { + case SFLM_NUMBER_TYPE : + { + // Make sure the number buffer does not exceed the + // max length. If there is an extra byte for the + // sign (byte 9) make sure it has a value of either + // 0x80 or 0 (by masking of the high bit). + + if( uiBufLen > FLM_MAX_NUM_BUF_SIZE || + (uiBufLen == FLM_MAX_NUM_BUF_SIZE && + (pucBuf[ uiBufLen - 1] & 0x7F) != 0)) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + // Look at the high bit of the most-significant byte + // to determine if the number is signed + + if( pucBuf[ uiBufLen - 1] & 0x80) + { + if( puiNum) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + bNeg = TRUE; + } + + uiNum = pucBuf[ uiBufLen - 1] & 0x7F; + uiBufLen--; + + for( uiLoop = 1; uiLoop <= uiBufLen; uiLoop++) + { + if( gv_b32BitPlatform && (uiNum & 0xFF000000)) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + uiNum = (uiNum << 8) + pucBuf[ uiBufLen - uiLoop]; + } + + break; + } + + case SFLM_STRING_TYPE: + { + FLMBYTE ucNumBuf[ 64]; + FLMUINT uiNumBufLen = sizeof( ucNumBuf); + FLMBYTE * pucTmp; + + if( RC_BAD( rc = flmStorage2UTF8( SFLM_STRING_TYPE, uiBufLen, pucBuf, + &uiNumBufLen, ucNumBuf))) + { + goto Exit; + } + + pucTmp = &ucNumBuf[ 0]; + if( *pucTmp == ASCII_DASH) + { + if( puiNum) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + bNeg = TRUE; + pucTmp++; + } + + while( *pucTmp) + { + if( *pucTmp < ASCII_ZERO || *pucTmp > ASCII_NINE) + { + break; + } + + if( uiNum > (~(FLMUINT)0) / 10) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + uiNum *= (FLMUINT)10; + + if( uiNum > (~(FLMUINT)0) - (FLMUINT)(*pucTmp - ASCII_ZERO)) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + uiNum += (FLMUINT)(*pucTmp - ASCII_ZERO); + pucTmp++; + } + + break; + } + + default : + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + if( puiNum) + { + if( bNeg) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + + *puiNum = uiNum; + } + else + { + flmAssert( piNum); + + if( bNeg) + { + if( uiNum > gv_uiMaxSignedIntVal + 1) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + + *piNum = -(FLMINT)uiNum; + } + else + { + if( uiNum > gv_uiMaxSignedIntVal) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + *piNum = (FLMINT)uiNum; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a storage value back into a number +Notes: Changes to this code must also be made to flmStorage2Number +*****************************************************************************/ +RCODE flmStorage2Number64( + FLMUINT eDataTyp, + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT64 * pui64Num, + FLMINT64 * pi64Num) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + FLMUINT64 ui64Num = 0; + FLMBOOL bNeg = FALSE; + + if( !uiBufLen) + { + if( pui64Num) + { + *pui64Num = 0; + } + else + { + *pi64Num = 0; + } + goto Exit; + } + + if( !pucBuf) + { + rc = RC_SET( NE_SFLM_CONV_NULL_SRC); + goto Exit; + } + + switch( eDataTyp) + { + case SFLM_NUMBER_TYPE : + { + // Make sure the number buffer does not exceed the + // max length. If there is an extra byte for the + // sign (byte 9) make sure it has a value of either + // 0x80 or 0 (by masking of the high bit). + + if( uiBufLen > FLM_MAX_NUM_BUF_SIZE || + (uiBufLen == FLM_MAX_NUM_BUF_SIZE && + (pucBuf[ uiBufLen - 1] & 0x7F) != 0)) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + // Look at the high bit of the most-significant byte + // to determine if the number is signed + + if( pucBuf[ uiBufLen - 1] & 0x80) + { + if( pui64Num) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + bNeg = TRUE; + } + + ui64Num = pucBuf[ uiBufLen - 1] & 0x7F; + uiBufLen--; + + for( uiLoop = 1; uiLoop <= uiBufLen; uiLoop++) + { + ui64Num = (ui64Num << 8) + pucBuf[ uiBufLen - uiLoop]; + } + + break; + } + + case SFLM_STRING_TYPE : + { + FLMBYTE ucNumBuf[ 64]; + FLMUINT uiNumBufLen = sizeof( ucNumBuf); + FLMBYTE * pucTmp; + + if( RC_BAD( rc = flmStorage2UTF8( SFLM_NUMBER_TYPE, uiBufLen, pucBuf, + &uiNumBufLen, ucNumBuf))) + { + goto Exit; + } + + pucTmp = &ucNumBuf[ 0]; + + if( *pucTmp == ASCII_DASH) + { + if( pui64Num) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + bNeg = TRUE; + pucTmp++; + } + + while( *pucTmp) + { + if( *pucTmp < ASCII_ZERO || *pucTmp > ASCII_NINE) + { + break; + } + + if( ui64Num > (~(FLMUINT64)0) / 10) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + ui64Num *= (FLMUINT64)10; + + if( ui64Num > (~(FLMUINT64)0) - (FLMUINT64)(*pucTmp - ASCII_ZERO)) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + ui64Num += (FLMUINT64)(*pucTmp - ASCII_ZERO); + pucTmp++; + } + + break; + } + + default : + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + if( pui64Num) + { + *pui64Num = ui64Num; + } + else + { + flmAssert( pi64Num); + + if( bNeg) + { + if( ui64Num > gv_ui64MaxSignedIntVal + 1) + { + rc = RC_SET( NE_SFLM_CONV_NUM_UNDERFLOW); + goto Exit; + } + + *pi64Num = -(FLMINT64)ui64Num; + } + else + { + if( ui64Num > gv_ui64MaxSignedIntVal) + { + rc = RC_SET( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + *pi64Num = (FLMINT64)ui64Num; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a numeric storage value into a collation value +*****************************************************************************/ +RCODE flmStorageNum2CollationNum( + const FLMBYTE * pucStorageBuf, + FLMUINT uiStorageLen, + FLMBYTE * pucCollBuf, + FLMUINT * puiCollLen) +{ + FLMUINT uiLoop; + FLMUINT uiOffset; + FLMUINT uiMaxLen = *puiCollLen; + FLMBYTE ucVal; + FLMBOOL bNegative = FALSE; + RCODE rc = NE_SFLM_OK; + + if( !pucStorageBuf || !uiStorageLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + // Look at the high bit of the most-significant byte + // to determine if the number is signed + + if( pucStorageBuf[ uiStorageLen - 1] & 0x80) + { + bNegative = TRUE; + } + + uiOffset = 1; + if( (ucVal = pucStorageBuf[ uiStorageLen - 1] & 0x7F) != 0 || + uiStorageLen == 1) // Handle the special case of zero + { + if( bNegative) + { + ucVal = ~ucVal; + } + + if( uiOffset >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + pucCollBuf[ uiOffset++] = ucVal; + } + uiStorageLen--; + + // Check for overflow + + if( uiOffset + uiStorageLen >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + // Map the little-endian storage format to the big-endian + // collation format + + if( !bNegative) + { + for( uiLoop = 1; uiLoop <= uiStorageLen; uiLoop++) + { + pucCollBuf[ uiOffset++] = pucStorageBuf[ uiStorageLen - uiLoop]; + } + } + else + { + for( uiLoop = 1; uiLoop <= uiStorageLen; uiLoop++) + { + pucCollBuf[ uiOffset++] = ~pucStorageBuf[ uiStorageLen - uiLoop]; + } + } + + flmAssert( uiOffset >= 2); + + // Store the numeric collation marker and byte count + + if( !bNegative) + { + // Positive numbers must collate after negative numbers, + // so all positive numbers will start with a byte + // in the range of 0xC8 - 0xCF. + + pucCollBuf[ 0] = (FLMBYTE)(0xC8 + (uiOffset - 2)); + } + else + { + // Negative numbers must collate before positive numbers, + // so all negative numbers will start with a byte + // in the range of 0xC0 - 0xC7. + + pucCollBuf[ 0] = (FLMBYTE)(0xC8 - (uiOffset - 1)); + } + + // Set the key length + + *puiCollLen = uiOffset; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a numeric collation value into a storage value +*****************************************************************************/ +RCODE flmCollationNum2StorageNum( + const FLMBYTE * pucCollBuf, + FLMUINT uiCollLen, + FLMBYTE * pucStorageBuf, + FLMUINT * puiStorageLen) +{ + FLMUINT uiLoop; + FLMUINT uiOffset; + FLMUINT uiMaxOffset = *puiStorageLen; + FLMUINT uiNumKeyBytes; + FLMBOOL bNegative = FALSE; + RCODE rc = NE_SFLM_OK; + + if( !pucCollBuf || !uiCollLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + // Make sure this looks like a valid numeric key piece + + if( (pucCollBuf[ 0] & 0xC0) != 0xC0) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Get the byte count + + if( (uiNumKeyBytes = (FLMUINT)(pucCollBuf[ 0] & 0x0F)) >= 8) + { + uiNumKeyBytes -= 7; + } + else + { + uiNumKeyBytes = 8 - uiNumKeyBytes; + bNegative = TRUE; + } + pucCollBuf++; + uiCollLen--; + + // Make sure the buffer has at least the number of bytes + // we need + + if( uiCollLen != uiNumKeyBytes) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( uiNumKeyBytes >= uiMaxOffset) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + // Translate the collation value into a numeric value + + if( !bNegative) + { + for( uiLoop = 0; uiLoop < uiNumKeyBytes; uiLoop++) + { + pucStorageBuf[ uiNumKeyBytes - uiLoop - 1] = pucCollBuf[ uiLoop]; + } + } + else + { + for( uiLoop = 0; uiLoop < uiNumKeyBytes; uiLoop++) + { + pucStorageBuf[ uiNumKeyBytes - uiLoop - 1] = ~pucCollBuf[ uiLoop]; + } + } + + uiOffset = uiNumKeyBytes; + + // If a number is negative, the high bit of the right-most + // byte needs to be set. If the value of the number is such + // that its positive representation already has the high-bit + // set, we need to store an additional sign byte. + + if( !bNegative) + { + if( pucStorageBuf[ uiOffset - 1] & 0x80) + { + if( uiOffset >= uiMaxOffset) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + pucStorageBuf[ uiOffset++] = 0; + } + } + else + { + if( (pucStorageBuf[ uiOffset - 1] & 0x80) == 0) + { + pucStorageBuf[ uiOffset - 1] |= 0x80; + } + else + { + if( uiOffset >= uiMaxOffset) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + pucStorageBuf[ uiOffset++] = 0x80; + } + } + + // Set the storage length + + *puiStorageLen = uiOffset; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a collation value (numeric only) to a number +*****************************************************************************/ +RCODE flmCollation2Number( + FLMUINT uiBufLen, + const FLMBYTE * pucBuf, + FLMUINT64 * pui64Num, + FLMBOOL * pbNeg, + FLMUINT * puiBytesProcessed) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + FLMUINT uiNumBytes; + FLMUINT64 ui64Num; + FLMBOOL bNeg = FALSE; + + *pui64Num = 0; + + if( !uiBufLen) + { + goto Exit; + } + + if( !pucBuf) + { + rc = RC_SET( NE_SFLM_CONV_NULL_SRC); + goto Exit; + } + + // Make sure this looks like a valid numeric key piece + + if( (pucBuf[ 0] & 0xC0) != 0xC0) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Get the byte count + + if( (uiNumBytes = (FLMUINT)(pucBuf[ 0] & 0x0F)) >= 8) + { + uiNumBytes -= 7; + } + else + { + uiNumBytes = 8 - uiNumBytes; + bNeg = TRUE; + } + pucBuf++; + uiBufLen--; + + // Make sure the buffer has at least the number of bytes + // we need + + if( uiBufLen < uiNumBytes) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Reconstruct the number + + ui64Num = 0; + if( !bNeg) + { + for( uiLoop = 0; uiLoop < uiNumBytes; uiLoop++) + { + ui64Num += (((FLMUINT64)pucBuf[ uiLoop]) << + (8 * ((uiNumBytes - uiLoop) - 1))); + } + } + else + { + for( uiLoop = 0; uiLoop < uiNumBytes; uiLoop++) + { + ui64Num += (((FLMUINT64)((FLMBYTE)~pucBuf[ uiLoop])) << + (8 * ((uiNumBytes - uiLoop) - 1))); + } + } + + *pui64Num = ui64Num; + + if( puiBytesProcessed) + { + *puiBytesProcessed = uiNumBytes + 1; + } + + if( pbNeg) + { + *pbNeg = bNeg; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE flmStorageNumberToNumber( + const FLMBYTE * pucNumBuf, + FLMUINT uiNumBufLen, + FLMUINT64 * pui64Number, + FLMBOOL * pbNeg) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + FLMUINT64 ui64Num = 0; + FLMBOOL bNeg = FALSE; + + if( !uiNumBufLen) + { + goto Exit; + } + + // Make sure the number buffer does not exceed the + // max length. If there is an extra byte for the + // sign (byte 9) make sure it has a value of either + // 0x80 or 0 (by masking of the high bit). + + if( uiNumBufLen > FLM_MAX_NUM_BUF_SIZE || + (uiNumBufLen == FLM_MAX_NUM_BUF_SIZE && + (pucNumBuf[ uiNumBufLen - 1] & 0x7F) != 0)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + // Look at the high bit of the most-significant byte + // to determine if the number is signed + + if( pucNumBuf[ uiNumBufLen - 1] & 0x80) + { + bNeg = TRUE; + } + + ui64Num = pucNumBuf[ uiNumBufLen - 1] & 0x7F; + uiNumBufLen--; + + for( uiLoop = 1; uiLoop <= uiNumBufLen; uiLoop++) + { + ui64Num = (ui64Num << 8) + pucNumBuf[ uiNumBufLen - uiLoop]; + } + +Exit: + + *pui64Number = ui64Num; + *pbNeg = bNeg; + + return( rc); +} diff --git a/sql/src/frestore.cpp b/sql/src/frestore.cpp new file mode 100644 index 0000000..cc564dd --- /dev/null +++ b/sql/src/frestore.cpp @@ -0,0 +1,307 @@ +//------------------------------------------------------------------------------ +// Desc: Methods used during restore +// +// Tabs: 3 +// +// Copyright (c) 2001-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: frestore.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Constructor +****************************************************************************/ +F_FSRestore::F_FSRestore() +{ + m_pFileHdl = NULL; + m_pMultiFileHdl = NULL; + m_ui64Offset = 0; + m_bSetupCalled = FALSE; + m_szDbPath[ 0] = 0; + m_uiDbVersion = 0; + m_szBackupSetPath[ 0] = 0; + m_szRflDir[ 0] = 0; + m_bOpen = FALSE; +} + +/**************************************************************************** +Desc: Destructor +****************************************************************************/ +F_FSRestore::~F_FSRestore() +{ + if( m_bOpen) + { + (void)close(); + } +} + + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::setup( + const char * pszDbPath, + const char * pszBackupSetPath, + const char * pszRflDir) +{ + flmAssert( !m_bSetupCalled); + flmAssert( pszDbPath); + flmAssert( pszBackupSetPath); + + f_strcpy( m_szDbPath, pszDbPath); + f_strcpy( m_szBackupSetPath, pszBackupSetPath); + + if( pszRflDir) + { + f_strcpy( m_szRflDir, pszRflDir); + } + + + m_bSetupCalled = TRUE; + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::openBackupSet( void) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled); + flmAssert( !m_pMultiFileHdl); + + if( RC_BAD( rc = FlmAllocMultiFileHdl( &m_pMultiFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pMultiFileHdl->open( m_szBackupSetPath))) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + goto Exit; + } + + m_ui64Offset = 0; + m_bOpen = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::openRflFile( + FLMUINT uiFileNum) +{ + RCODE rc = NE_SFLM_OK; + char szRflPath[ F_PATH_MAX_SIZE]; + char szBaseName[ F_FILENAME_SIZE]; + FLMUINT uiBaseNameSize; + SFLM_DB_HDR dbHdr; + IF_FileHdl * pFileHdl = NULL; + + flmAssert( m_bSetupCalled); + flmAssert( uiFileNum); + flmAssert( !m_pFileHdl); + + // Read the database header to determine the version number + + if( !m_uiDbVersion) + { + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->openFile( m_szDbPath, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = flmReadAndVerifyHdrInfo( NULL, pFileHdl, &dbHdr))) + { + goto Exit; + } + + pFileHdl->close(); + pFileHdl->Release(); + pFileHdl = NULL; + + m_uiDbVersion = (FLMUINT)dbHdr.ui32DbVersion; + } + + // Generate the log file name. + + if( RC_BAD( rc = rflGetDirAndPrefix( m_szDbPath, m_szRflDir, szRflPath))) + { + goto Exit; + } + + uiBaseNameSize = sizeof( szBaseName); + rflGetBaseFileName( uiFileNum, szBaseName, &uiBaseNameSize, NULL); + gv_SFlmSysData.pFileSystem->pathAppend( szRflPath, szBaseName); + + // Open the file. + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->openBlockFile( szRflPath, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, 512, &m_pFileHdl))) + { + goto Exit; + } + + m_bOpen = TRUE; + m_ui64Offset = 0; + +Exit: + + if( pFileHdl) + { + pFileHdl->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::openIncFile( + FLMUINT uiFileNum) +{ + char szIncPath[ F_PATH_MAX_SIZE]; + char szIncFile[ F_FILENAME_SIZE]; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled); + flmAssert( !m_pMultiFileHdl); + + // Since this is a non-interactive restore, we will "guess" + // that incremental backups are located in the same parent + // directory as the main backup set. We will further assume + // that the incremental backup sets have been named XXXXXXXX.INC, + // where X is a hex digit. + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( m_szBackupSetPath, + szIncPath, NULL))) + { + goto Exit; + } + + f_sprintf( szIncFile, "%08X.INC", (unsigned)uiFileNum); + gv_SFlmSysData.pFileSystem->pathAppend( szIncPath, szIncFile); + + if( RC_BAD( rc = FlmAllocMultiFileHdl( &m_pMultiFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = m_pMultiFileHdl->open( szIncPath))) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + goto Exit; + } + + m_ui64Offset = 0; + m_bOpen = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::read( + FLMUINT uiLength, + void * pvBuffer, + FLMUINT * puiBytesRead) +{ + FLMUINT uiBytesRead = 0; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled); + flmAssert( m_pFileHdl || m_pMultiFileHdl); + + if( m_pMultiFileHdl) + { + if( RC_BAD( rc = m_pMultiFileHdl->read( m_ui64Offset, + uiLength, pvBuffer, &uiBytesRead))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = m_pFileHdl->read( (FLMUINT)m_ui64Offset, + uiLength, pvBuffer, &uiBytesRead))) + { + goto Exit; + } + } + +Exit: + + m_ui64Offset += uiBytesRead; + + if( puiBytesRead) + { + *puiBytesRead = uiBytesRead; + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::close( void) +{ + flmAssert( m_bSetupCalled); + + if( m_pMultiFileHdl) + { + m_pMultiFileHdl->Release(); + m_pMultiFileHdl = NULL; + } + + if( m_pFileHdl) + { + m_pFileHdl->Release(); + m_pFileHdl = NULL; + } + + m_bOpen = FALSE; + m_ui64Offset = 0; + + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FSRestore::abortFile( void) +{ + return( close()); +} + + diff --git a/sql/src/frow.cpp b/sql/src/frow.cpp new file mode 100644 index 0000000..7b6562c --- /dev/null +++ b/sql/src/frow.cpp @@ -0,0 +1,4574 @@ +//------------------------------------------------------------------------------ +// Desc: This is the row cache for FLAIM-SQL +// +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#if defined( FLM_NLM) && !defined( __MWERKS__) +// Disable "Warning! W549: col(XX) 'sizeof' operand contains +// compiler generated information" + #pragma warning 549 9 +#endif + +FSTATIC RCODE getStorageAsNumber( + const FLMBYTE * pucData, + FLMUINT uiDataLen, + eDataType eDataType, + FLMUINT64 * pui64Number, + FLMBOOL * pbNeg); + +/**************************************************************************** +Desc: Constructor +****************************************************************************/ +F_RowCacheMgr::F_RowCacheMgr() +{ + m_pPurgeList = NULL; + m_pHeapList = NULL; + m_pOldList = NULL; + f_memset( &m_Usage, 0, sizeof( m_Usage)); + m_ppHashBuckets = NULL; + m_uiNumBuckets = 0; + m_uiHashFailTime = 0; + m_uiHashMask = 0; + m_uiPendingReads = 0; + m_uiIoWaits = 0; + m_bReduceInProgress = FALSE; +#ifdef FLM_DEBUG + m_bDebug = FALSE; +#endif +} + +/**************************************************************************** +Desc: Constructor for F_Row +****************************************************************************/ +F_Row::F_Row() +{ + m_pPrevInBucket = NULL; + m_pNextInBucket = NULL; + m_pPrevInDatabase = NULL; + m_pNextInDatabase = NULL; + m_pOlderVersion = NULL; + m_pNewerVersion = NULL; + m_pPrevInHeapList = NULL; + m_pNextInHeapList = NULL; + m_pPrevInOldList = NULL; + m_pNextInOldList = NULL; + m_ui64LowTransId = 0; + + // Set the high transaction ID to FLM_MAX_UINT64 so that this will NOT + // be treated as one that had memory assigned to the old version rows. + + m_ui64HighTransId = FLM_MAX_UINT64; + m_pNotifyList = NULL; + m_uiCacheFlags = 0; + m_uiStreamUseCount = 0; + + // Items initialized in constructor + + m_uiColumnDataBufSize = 0; + m_pucColumnData = NULL; + m_pColumns = NULL; + m_uiFlags = 0; + + m_uiTableNum = 0; + m_ui64RowId = 0; + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif +} + +/**************************************************************************** +Desc: Destructor for F_Row object. + This routine assumes the global mutex is already locked. +****************************************************************************/ +F_Row::~F_Row() +{ + FLMUINT uiSize = memSize(); + FLMBYTE * pucActualAlloc; + + flmAssert( !m_uiStreamUseCount); + f_assertMutexLocked( gv_SFlmSysData.hRowCacheMutex); + + // If this is an old version, decrement the old version counters. + + if (m_ui64HighTransId != FLM_MAX_UINT64) + { + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes >= uiSize && + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes -= uiSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount--; + unlinkFromOldList(); + } + + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount >= uiSize && + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCount); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount -= uiSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCount--; + + if( m_uiFlags & FROW_HEAP_ALLOC) + { + unlinkFromHeapList(); + } + + // Free the column data, if any + + if (m_pucColumnData) + { + pucActualAlloc = getActualPointer( m_pucColumnData); + gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->freeBuf( + calcDataBufSize(m_uiColumnDataBufSize), + &pucActualAlloc); + m_pucColumnData = NULL; + } + + // Free the column item list + + if (m_pColumns) + { + pucActualAlloc = getActualPointer( m_pColumns); + gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->freeBuf( + calcColumnListBufSize( m_uiNumColumns), + &pucActualAlloc); + m_pColumns = NULL; + m_uiNumColumns = 0; + } + + if (shouldRehash( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCount, + gv_SFlmSysData.pRowCacheMgr->m_uiNumBuckets)) + { + if (checkHashFailTime( &gv_SFlmSysData.pRowCacheMgr->m_uiHashFailTime)) + { + (void)gv_SFlmSysData.pRowCacheMgr->rehash(); + } + } +} + +/**************************************************************************** +Desc: This routine frees a purged row from row cache. This routine assumes + that the row cache mutex has already been locked. +****************************************************************************/ +void F_Row::freePurged( void) +{ + + // Unlink the row from the purged list. + + unlinkFromPurged(); + + // Free the F_Row object. + + unsetPurged(); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + delete this; +} + +/**************************************************************************** +Desc: This routine frees a row in the row cache. This routine assumes + that the row cache mutex has already been locked. +****************************************************************************/ +void F_Row::freeCache( + FLMBOOL bPutInPurgeList) +{ + FLMBOOL bOldVersion; + + bOldVersion = (FLMBOOL)((m_ui64HighTransId != FLM_MAX_UINT64) + ? TRUE + : FALSE); + + // Unlink the row from its various lists. + + gv_SFlmSysData.pRowCacheMgr->m_MRUList.unlinkGlobal( + (F_CachedItem *)this); + unlinkFromDatabase(); + + if (!m_pNewerVersion) + { + F_Row * pOlderVersion = m_pOlderVersion; + + unlinkFromHashBucket(); + + // If there was an older version, it now needs to be + // put into the hash bucket. + + if (pOlderVersion) + { + unlinkFromVerList(); + pOlderVersion->linkToHashBucket(); + } + } + else + { + unlinkFromVerList(); + } + + if( m_uiFlags & FROW_HEAP_ALLOC) + { + unlinkFromHeapList(); + } + + // Free the F_Row structure if not putting in purge list. + + if (!bPutInPurgeList) + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + delete this; + } + else + { +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + if ((m_pNextInGlobal = gv_SFlmSysData.pRowCacheMgr->m_pPurgeList) != NULL) + { +#ifdef FLM_CACHE_PROTECT + m_pNextInGlobal->unprotectCachedItem(); +#endif + m_pNextInGlobal->m_pPrevInGlobal = this; +#ifdef FLM_CACHE_PROTECT + m_pNextInGlobal->protectCachedItem(); +#endif + } + gv_SFlmSysData.pRowCacheMgr->m_pPurgeList = this; + + // Unset the dirty flags - don't want anything in the purge list + // to be dirty. + + m_uiFlags &= ~(FROW_DIRTY | FROW_NEW); + setPurged(); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + flmAssert( !m_pPrevInGlobal); + } +} + +/**************************************************************************** +Desc: This routine initializes row cache manager. +****************************************************************************/ +RCODE F_RowCacheMgr::initCache( void) +{ + RCODE rc = NE_SFLM_OK; + + // Allocate the hash buckets. + + if (RC_BAD( rc = f_calloc( + (FLMUINT)sizeof( F_Row *) * + (FLMUINT)MIN_HASH_BUCKETS, + &m_ppHashBuckets))) + { + goto Exit; + } + m_uiNumBuckets = MIN_HASH_BUCKETS; + m_uiHashMask = m_uiNumBuckets - 1; + gv_SFlmSysData.pGlobalCacheMgr->incrTotalBytes( f_msize( m_ppHashBuckets)); + + if( RC_BAD( rc = FlmAllocFixedAllocator( &m_pRowAllocator))) + { + goto Exit; + } + + // Set up the F_Row object allocator + + if (RC_BAD( rc = m_pRowAllocator->setup( &m_rowRelocator, + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager, + sizeof( F_Row), &m_Usage.slabUsage))) + { + goto Exit; + } + + if (RC_BAD( rc = FlmAllocBufferAllocator( &m_pBufAllocator))) + { + goto Exit; + } + + // Set up the buffer allocator for F_Row objects + + if (RC_BAD( rc = m_pBufAllocator->setup( + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager, + &m_Usage.slabUsage))) + { + goto Exit; + } + +#ifdef FLM_DEBUG + m_bDebug = TRUE; +#endif + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Determine if a row can be moved. +Notes: This routine assumes the row cache mutex is locked + This is a static method, so there is no "this" pointer to the + F_RowCacheMgr object. +****************************************************************************/ +FLMBOOL F_RowRelocator::canRelocate( + void * pvAlloc) +{ + return( ((F_Row *)pvAlloc)->rowInUse() ? FALSE : TRUE); +} + +/**************************************************************************** +Desc: Fixes up all pointers needed to allow an F_Row object to be + moved to a different location in memory +Notes: This routine assumes the row cache mutex is locked. + This is a static method, so there is no "this" pointer to the + F_RowCacheMgr object. +****************************************************************************/ +void F_RowRelocator::relocate( + void * pvOldAlloc, + void * pvNewAlloc) +{ + F_Row * pOldRow = (F_Row *)pvOldAlloc; + F_Row * pNewRow = (F_Row *)pvNewAlloc; + F_Row ** ppBucket; + F_Database * pDatabase = pOldRow->m_pDatabase; + F_RowCacheMgr * pRowCacheMgr = gv_SFlmSysData.pRowCacheMgr; + FLMBYTE * pucActualAlloc; + + flmAssert( !pOldRow->rowInUse()); + flmAssert( pvNewAlloc < pvOldAlloc); + + // Update the F_Row pointer in the data buffer + + if (pNewRow->m_pucColumnData) + { + pucActualAlloc = getActualPointer( pNewRow->m_pucColumnData); + flmAssert( *((F_Row **)(pucActualAlloc)) == pOldRow); + pNewRow->setRowAndDataPtr( pucActualAlloc); + } + + if (pNewRow->m_pColumns) + { + pucActualAlloc = getActualPointer( pNewRow->m_pColumns); + flmAssert( *((F_Row **)(pucActualAlloc)) == pOldRow); + pNewRow->setColumnListPtr( pucActualAlloc); + } + + if (pNewRow->m_pPrevInDatabase) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInDatabase->unprotectCachedItem(); +#endif + pNewRow->m_pPrevInDatabase->m_pNextInDatabase = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInDatabase->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNextInDatabase) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInDatabase->unprotectCachedItem(); +#endif + pNewRow->m_pNextInDatabase->m_pPrevInDatabase = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInDatabase->protectCachedItem(); +#endif + } + + if (pNewRow->m_pPrevInGlobal) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInGlobal->unprotectCachedItem(); +#endif + pNewRow->m_pPrevInGlobal->m_pNextInGlobal = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInGlobal->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNextInGlobal) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInGlobal->unprotectCachedItem(); +#endif + pNewRow->m_pNextInGlobal->m_pPrevInGlobal = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInGlobal->protectCachedItem(); +#endif + } + + if (pNewRow->m_pPrevInBucket) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInBucket->unprotectCachedItem(); +#endif + pNewRow->m_pPrevInBucket->m_pNextInBucket = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInBucket->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNextInBucket) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInBucket->unprotectCachedItem(); +#endif + pNewRow->m_pNextInBucket->m_pPrevInBucket = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInBucket->protectCachedItem(); +#endif + } + + if (pNewRow->m_pOlderVersion) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pOlderVersion->unprotectCachedItem(); +#endif + pNewRow->m_pOlderVersion->m_pNewerVersion = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pOlderVersion->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNewerVersion) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNewerVersion->unprotectCachedItem(); +#endif + pNewRow->m_pNewerVersion->m_pOlderVersion = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNewerVersion->protectCachedItem(); +#endif + } + + if (pNewRow->m_pPrevInHeapList) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInHeapList->unprotectCachedItem(); +#endif + pNewRow->m_pPrevInHeapList->m_pNextInHeapList = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInHeapList->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNextInHeapList) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInHeapList->unprotectCachedItem(); +#endif + pNewRow->m_pNextInHeapList->m_pPrevInHeapList = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInHeapList->protectCachedItem(); +#endif + } + + if (pNewRow->m_pPrevInOldList) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInOldList->unprotectCachedItem(); +#endif + pNewRow->m_pPrevInOldList->m_pNextInOldList = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pPrevInOldList->protectCachedItem(); +#endif + } + + if (pNewRow->m_pNextInOldList) + { +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInOldList->unprotectCachedItem(); +#endif + pNewRow->m_pNextInOldList->m_pPrevInOldList = pNewRow; +#ifdef FLM_CACHE_PROTECT + pNewRow->m_pNextInOldList->protectCachedItem(); +#endif + } + + if( pDatabase) + { + if (pDatabase->m_pFirstRow == pOldRow) + { + pDatabase->m_pFirstRow = pNewRow; + } + + if( pDatabase->m_pLastRow == pOldRow) + { + pDatabase->m_pLastRow = pNewRow; + } + + if( pDatabase->m_pLastDirtyRow == pOldRow) + { + pDatabase->m_pLastDirtyRow = pNewRow; + } + } + + ppBucket = pRowCacheMgr->rowHash( pOldRow->m_ui64RowId); + if( *ppBucket == pOldRow) + { + *ppBucket = pNewRow; + } + + if (pRowCacheMgr->m_MRUList.m_pMRUItem == (F_CachedItem *)pOldRow) + { + pRowCacheMgr->m_MRUList.m_pMRUItem = pNewRow; + } + + if (pRowCacheMgr->m_MRUList.m_pLRUItem == (F_CachedItem *)pOldRow) + { + pRowCacheMgr->m_MRUList.m_pLRUItem = pNewRow; + } + + if (pRowCacheMgr->m_pHeapList == pOldRow) + { + pRowCacheMgr->m_pHeapList = pNewRow; + } + + if (pRowCacheMgr->m_pOldList == pOldRow) + { + pRowCacheMgr->m_pOldList = pNewRow; + } + + if (pRowCacheMgr->m_pPurgeList == pOldRow) + { + pRowCacheMgr->m_pPurgeList = pNewRow; + } +} + +/**************************************************************************** +Desc: Determine if a data buffer of an F_Row object can be moved. + This routine assumes that the row cache mutex is locked. +****************************************************************************/ +FLMBOOL F_ColumnDataRelocator::canRelocate( + void * pvAlloc) +{ + F_Row * pRow = *((F_Row **)pvAlloc); + + if( pRow->rowInUse()) + { + return( FALSE); + } + else + { + flmAssert( getActualPointer( pRow->m_pucColumnData) == (FLMBYTE *)pvAlloc); + return( TRUE); + } +} + +/**************************************************************************** +Desc: Relocate the data buffer of an F_Row object. This routine assumes + that the row cache mutex is locked. +****************************************************************************/ +void F_ColumnDataRelocator::relocate( + void * pvOldAlloc, + void * pvNewAlloc) +{ + F_Row * pRow = *((F_Row **)pvOldAlloc); + + flmAssert( !pRow->rowInUse()); + flmAssert( pvNewAlloc < pvOldAlloc); + flmAssert( getActualPointer( pRow->m_pucColumnData) == (FLMBYTE *)pvOldAlloc); + + pRow->setRowAndDataPtr( (FLMBYTE *)pvNewAlloc); +} + +/**************************************************************************** +Desc: Determine if an column list of an F_Row object can be moved. + This routine assumes that the row cache mutex is locked. +****************************************************************************/ +FLMBOOL F_ColumnListRelocator::canRelocate( + void * pvAlloc) +{ + F_Row * pRow = *((F_Row **)pvAlloc); + + if( pRow->rowInUse()) + { + return( FALSE); + } + else + { + flmAssert( getActualPointer( pRow->m_pColumns) == (FLMBYTE *)pvAlloc); + return( TRUE); + } +} + +/**************************************************************************** +Desc: Relocate the column list of an F_Row object. This routine assumes + that the row cache mutex is locked. +****************************************************************************/ +void F_ColumnListRelocator::relocate( + void * pvOldAlloc, + void * pvNewAlloc) +{ + F_Row * pRow = *((F_Row **)pvOldAlloc); + + flmAssert( !pRow->rowInUse()); + flmAssert( pvNewAlloc < pvOldAlloc); + flmAssert( getActualPointer( pRow->m_pColumns) == (FLMBYTE *)pvOldAlloc); + + pRow->setColumnListPtr( (FLMBYTE *)pvNewAlloc); +} + +/**************************************************************************** +Desc: This routine resizes the hash table for the cache manager. + NOTE: This routine assumes that the row cache mutex has been locked. +****************************************************************************/ +RCODE F_RowCacheMgr::rehash( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNewHashTblSize; + F_Row ** ppOldHashTbl; + FLMUINT uiOldHashTblSize; + F_Row ** ppBucket; + FLMUINT uiLoop; + F_Row * pTmpRow; + F_Row * pTmpNextRow; + FLMUINT uiOldMemSize; + + uiNewHashTblSize = caGetBestHashTblSize( m_Usage.uiCount); + + // At this point we better have a different hash table size + // or something is mucked up! + + flmAssert( uiNewHashTblSize != m_uiNumBuckets); + + // Save the old hash table and its size. + + if ((ppOldHashTbl = m_ppHashBuckets) != NULL) + { + uiOldMemSize = f_msize( ppOldHashTbl); + } + else + { + uiOldMemSize = 0; + } + uiOldHashTblSize = m_uiNumBuckets; + + // Allocate a new hash table. + + if (RC_BAD( rc = f_calloc( (FLMUINT)sizeof( F_Row *) * + (FLMUINT)uiNewHashTblSize, &m_ppHashBuckets))) + { + m_uiHashFailTime = FLM_GET_TIMER(); + m_ppHashBuckets = ppOldHashTbl; + goto Exit; + } + + // Subtract off old size and add in new size. + + gv_SFlmSysData.pGlobalCacheMgr->decrTotalBytes( uiOldMemSize); + gv_SFlmSysData.pGlobalCacheMgr->incrTotalBytes( f_msize( m_ppHashBuckets)); + + m_uiNumBuckets = uiNewHashTblSize; + m_uiHashMask = uiNewHashTblSize - 1; + + // Relink all of the rows into the new hash table. + + for (uiLoop = 0, ppBucket = ppOldHashTbl; + uiLoop < uiOldHashTblSize; + uiLoop++, ppBucket++) + { + pTmpRow = *ppBucket; + while (pTmpRow) + { + pTmpNextRow = pTmpRow->m_pNextInBucket; + pTmpRow->linkToHashBucket(); + pTmpRow = pTmpNextRow; + } + } + + // Throw away the old hash table. + + f_free( &ppOldHashTbl); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine shuts down the row cache manager and frees all + resources allocated by it. NOTE: Row cache mutex must be locked + already, or we must be shutting down so that only one thread is + calling this routine. +****************************************************************************/ +F_RowCacheMgr::~F_RowCacheMgr() +{ + F_CachedItem * pItem; + F_CachedItem * pNextItem; + + // Free all of the row cache objects. + + pItem = m_MRUList.m_pMRUItem; + while (pItem) + { + pNextItem = pItem->m_pNextInGlobal; + ((F_Row *)pItem)->freeCache( FALSE); + pItem = pNextItem; + } + flmAssert( !m_MRUList.m_pMRUItem && !m_MRUList.m_pLRUItem); + + // Must free those in the purge list too. + + while (m_pPurgeList) + { + m_pPurgeList->freePurged(); + } + + // The math better be consistent! + + flmAssert( m_Usage.uiCount == 0); + flmAssert( m_Usage.uiOldVerCount == 0); + flmAssert( m_Usage.uiOldVerBytes == 0); + + // Free the hash bucket array + + if (m_ppHashBuckets) + { + FLMUINT uiTotalMemory = f_msize( m_ppHashBuckets); + + f_free( &m_ppHashBuckets); + gv_SFlmSysData.pGlobalCacheMgr->decrTotalBytes( uiTotalMemory); + } + if (m_pRowAllocator) + { + m_pRowAllocator->Release(); + } + if (m_pBufAllocator) + { + m_pBufAllocator->Release(); + } +} + +/**************************************************************************** +Desc: This routine links a notify request into a row's notification list and + then waits to be notified that the event has occurred. + NOTE: This routine assumes that the row cache mutex is locked and that + it is supposed to unlock it. It will relock the mutex on its way out. +****************************************************************************/ +RCODE F_RowCacheMgr::waitNotify( + F_Db * pDb, + F_Row ** ppRow) +{ + return( flmWaitNotifyReq( gv_SFlmSysData.hRowCacheMutex, + pDb->m_hWaitSem, &((*ppRow)->m_pNotifyList), ppRow)); +} + +/**************************************************************************** +Desc: This routine notifies threads waiting for a pending read to complete. + NOTE: This routine assumes that the row cache mutex is already locked. +****************************************************************************/ +void F_RowCacheMgr::notifyWaiters( + FNOTIFY * pNotify, + F_Row * pUseRow, + RCODE NotifyRc) +{ + while (pNotify) + { + F_SEM hSem; + + *(pNotify->pRc) = NotifyRc; + if (RC_OK( NotifyRc)) + { + *((F_Row **)pNotify->pvUserData) = pUseRow; + pUseRow->incrRowUseCount(); + } + hSem = pNotify->hSem; + pNotify = pNotify->pNext; + f_semSignal( hSem); + } +} + +/**************************************************************************** +Desc: Allocate an F_Row object. +****************************************************************************/ +RCODE F_RowCacheMgr::allocRow( + F_Row ** ppRow, + FLMBOOL bMutexLocked) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bUnlockMutex = FALSE; + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bUnlockMutex = TRUE; + } + + if ((*ppRow = new F_Row) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Increment statistics. + + m_Usage.uiCount++; + m_Usage.uiByteCount += (*ppRow)->memSize(); + if (shouldRehash( m_Usage.uiCount, m_uiNumBuckets)) + { + if (checkHashFailTime( &m_uiHashFailTime)) + { + if (RC_BAD( rc = rehash())) + { + goto Exit; + } + } + } + +Exit: + + if( bUnlockMutex) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + return( rc); +} + +/**************************************************************************** +Desc: Cleanup old rows in cache that are no longer needed by any + transaction. This routine assumes that the row cache mutex + has been locked. +****************************************************************************/ +void F_RowCacheMgr::cleanupOldCache( void) +{ + F_Row * pCurRow; + F_Row * pNextRow; + + pCurRow = m_pOldList; + + // Stay in the loop until we have freed all old rows, or + // we have run through the entire list. + + while( pCurRow) + { + flmAssert( pCurRow->m_ui64HighTransId != FLM_MAX_UINT64); + + // Save the pointer to the next entry in the list because + // we may end up unlinking pCurRow below, in which case we would + // have lost the next row. + + pNextRow = pCurRow->m_pNextInOldList; + if (!pCurRow->rowInUse() && + !pCurRow->readingInRow() && + (!pCurRow->rowLinkedToDatabase() || + !pCurRow->m_pDatabase->neededByReadTrans( + pCurRow->m_ui64LowTransId, pCurRow->m_ui64HighTransId))) + { + pCurRow->freeRow(); + } + pCurRow = pNextRow; + } +} + +/**************************************************************************** +Desc: Cleanup rows that have been purged. This routine assumes that the + row cache mutex has been locked. +****************************************************************************/ +void F_RowCacheMgr::cleanupPurgedCache( void) +{ + F_Row * pCurRow; + F_Row * pNextRow; + + pCurRow = m_pPurgeList; + + // Stay in the loop until we have freed all purged rows, or + // we have run through the entire list. + + while( pCurRow) + { + // Save the pointer to the next entry in the list because + // we may end up unlinking pCurRow below, in which case we would + // have lost the next row. + + pNextRow = (F_Row *)pCurRow->m_pNextInGlobal; + flmAssert( pCurRow->rowPurged()); + + if (!pCurRow->rowInUse()) + { + pCurRow->freePurged(); + } + pCurRow = pNextRow; + } +} + +/**************************************************************************** +Desc: Reduce row cache to below the cache limit. NOTE: This routine assumes + that the row cache mutex is locked upon entering the routine, but + it may unlock and re-lock the mutex. +****************************************************************************/ +void F_RowCacheMgr::reduceCache( void) +{ + F_Row * pTmpRow; + F_Row * pPrevRow; + F_Row * pNextRow; + FLMUINT uiSlabSize; + FLMUINT uiByteThreshold; + FLMUINT uiSlabThreshold; + FLMBOOL bDoingReduce = FALSE; + + // Discard items that are allocated on the heap. These are large + // allocations that could not be satisfied by the buffer allocator and have + // the side effect of causing memory fragmentation. + + pTmpRow = m_pHeapList; + while( pTmpRow) + { + // Need to save the pointer to the next entry in the list because + // we may end up freeing pTmpRow below. + + pNextRow = pTmpRow->m_pNextInHeapList; + + // See if the item can be freed. + + if( pTmpRow->canBeFreed()) + { + // NOTE: This call will free the memory pointed to by + // pTmpRow. Hence, pTmpRow should NOT be used after + // this point. + + pTmpRow->freeRow(); + } + + pTmpRow = pNextRow; + } + + // If cache is not full, we are done. + + if( !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit() || m_bReduceInProgress) + { + goto Exit; + } + + m_bReduceInProgress = TRUE; + bDoingReduce = TRUE; + + // Cleanup cache that is no longer needed by anyone + + cleanupOldCache(); + cleanupPurgedCache(); + + // Determine the cache threshold + + uiSlabThreshold = gv_SFlmSysData.pGlobalCacheMgr->m_uiMaxSlabs >> 1; + uiSlabSize = gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->getSlabSize(); + + // Are we over the threshold? + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold) + { + goto Exit; + } + + // Remove items from cache starting from the LRU + + pTmpRow = (F_Row *)m_MRUList.m_pLRUItem; + uiByteThreshold = m_Usage.uiByteCount > uiSlabSize + ? m_Usage.uiByteCount - uiSlabSize + : 0; + + while( pTmpRow) + { + // Need to save the pointer to the next entry in the list because + // we may end up freeing pTmpRow below. + + pPrevRow = (F_Row *)pTmpRow->m_pPrevInGlobal; + + // See if the item can be freed. + + if( pTmpRow->canBeFreed()) + { + pTmpRow->freeRow(); + + if( m_Usage.uiByteCount <= uiByteThreshold) + { + if( pPrevRow) + { + pPrevRow->incrRowUseCount(); + } + + gv_SFlmSysData.pRowCacheMgr->defragmentMemory( TRUE); + + if( !pPrevRow) + { + break; + } + + pPrevRow->decrRowUseCount(); + + // We're going to quit when we get under 50 percent for row cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving row + // cache because block cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + + uiByteThreshold = uiByteThreshold > uiSlabSize + ? uiByteThreshold - uiSlabSize + : 0; + } + } + + pTmpRow = pPrevRow; + } + +Exit: + + if( bDoingReduce) + { + m_bReduceInProgress = FALSE; + } + + return; +} + +/**************************************************************************** +Desc: This routine finds a row in the row cache. If it cannot + find the row, it will return the position where the row should + be inserted. + NOTE: This routine assumes that the row cache mutex has been locked. +****************************************************************************/ +void F_RowCacheMgr::findRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT64 ui64VersionNeeded, + FLMBOOL bDontPoisonCache, + FLMUINT * puiNumLooks, + F_Row ** ppRow, + F_Row ** ppNewerRow, + F_Row ** ppOlderRow) +{ + F_Row * pRow; + FLMUINT uiNumLooks = 0; + FLMBOOL bFound; + F_Row * pNewerRow; + F_Row * pOlderRow; + F_Database * pDatabase = pDb->m_pDatabase; + + // Search down the hash bucket for the matching item. + +Start_Find: + + // NOTE: Need to always calculate hash bucket because + // the hash table may have been changed while we + // were waiting to be notified below - mutex can + // be unlocked, but it is guaranteed to be locked + // here. + + pRow = *(rowHash( ui64RowId)); + bFound = FALSE; + uiNumLooks = 1; + while (pRow && + (pRow->m_ui64RowId != ui64RowId || + pRow->m_uiTableNum != uiTableNum || + pRow->m_pDatabase != pDatabase)) + { + if ((pRow = pRow->m_pNextInBucket) != NULL) + { + uiNumLooks++; + } + } + + // If we found the row, see if we have the right version. + + if (!pRow) + { + pNewerRow = pOlderRow = NULL; + } + else + { + pNewerRow = NULL; + pOlderRow = pRow; + for (;;) + { + + // If this one is being read in, we need to wait on it. + + if (pRow->readingInRow()) + { + // We need to wait for this record to be read in + // in case it coalesces with other versions, resulting + // in a version that satisfies our request. + + m_uiIoWaits++; + if (RC_BAD( waitNotify( pDb, &pRow))) + { + + // Don't care what error the other thread had reading + // the thing in from disk - we'll bail out and start + // our find again. + + goto Start_Find; + } + + // The thread doing the notify "uses" the record cache + // on behalf of this thread to prevent the record + // from being replaced after it unlocks the mutex. + // At this point, since we have locked the mutex, + // we need to release the record - because we + // will put a "use" on it below. + + pRow->decrRowUseCount(); + + if (pRow->rowPurged()) + { + if (!pRow->rowInUse()) + { + pRow->freePurged(); + } + } + + // Start over with the find because the list + // structure has changed. + + goto Start_Find; + } + + // See if this record version is the one we need. + + if (ui64VersionNeeded < pRow->m_ui64LowTransId) + { + pNewerRow = pRow; + if ((pOlderRow = pRow = pRow->m_pOlderVersion) == NULL) + { + break; + } + uiNumLooks++; + } + else if (ui64VersionNeeded <= pRow->m_ui64HighTransId) + { + + // Make this the MRU record. + + if (puiNumLooks) + { + if (bDontPoisonCache) + { + m_MRUList.stepUpInGlobal( (F_CachedItem *)pRow); + } + else if (pRow->m_pPrevInGlobal) + { + m_MRUList.unlinkGlobal( (F_CachedItem *)pRow); + m_MRUList.linkGlobalAsMRU( (F_CachedItem *)pRow); + } + m_Usage.uiCacheHits++; + m_Usage.uiCacheHitLooks += uiNumLooks; + } + bFound = TRUE; + break; + } + else + { + pOlderRow = pRow; + pNewerRow = pRow->m_pNewerVersion; + + // Set pRow to NULL as an indicator that we did not + // find the version we needed. + + pRow = NULL; + break; + } + } + } + + *ppRow = pRow; + + if( ppOlderRow) + { + *ppOlderRow = pOlderRow; + } + + if( ppNewerRow) + { + *ppNewerRow = pNewerRow; + } + + if (puiNumLooks) + { + *puiNumLooks = uiNumLooks; + } +} + +/**************************************************************************** +Desc: This routine links a new row into the global list and + into the correct place in its hash bucket. This routine assumes that + the row cache mutex is already locked. +****************************************************************************/ +void F_RowCacheMgr::linkIntoRowCache( + F_Row * pNewerRow, + F_Row * pOlderRow, + F_Row * pRow, + FLMBOOL bLinkAsMRU + ) +{ + if( bLinkAsMRU) + { + m_MRUList.linkGlobalAsMRU( (F_CachedItem *)pRow); + } + else + { + m_MRUList.linkGlobalAsLRU( (F_CachedItem *)pRow); + } + + if (pNewerRow) + { + pRow->linkToVerList( pNewerRow, pOlderRow); + } + else + { + if (pOlderRow) + { + pOlderRow->unlinkFromHashBucket(); + } + pRow->linkToHashBucket(); + pRow->linkToVerList( NULL, pOlderRow); + } +} + +/**************************************************************************** +Desc: This routine links a new row to its F_Database according to whether + or not it is an update transaction or a read transaction. + It coalesces out any unnecessary versions. This routine assumes + that the row cache mutex is already locked. +****************************************************************************/ +void F_Row::linkToDatabase( + F_Database * pDatabase, + F_Db * pDb, + FLMUINT64 ui64LowTransId, + FLMBOOL bMostCurrent) +{ + F_Row * pTmpRow; + + m_ui64LowTransId = ui64LowTransId; + + // Before coalescing, link to F_Database. + // The following test determines if the row is an + // uncommitted version generated by the update transaction. + // If so, we mark it as such, and link it at the head of the + // F_Database list - so we can get rid of it quickly if we abort + // the transaction. + + if (pDb->getTransType() == SFLM_UPDATE_TRANS) + { + + // If we are in an update transaction, there better not + // be any newer versions in the list and the high + // transaction ID returned better be FLM_MAX_UINT64. + + flmAssert( m_pNewerVersion == NULL); + setTransID( FLM_MAX_UINT64); + + // If the low transaction ID is the same as the transaction, + // we may have modified this row during the transaction. + // Unfortunately, there is no sure way to tell, so we are + // forced to assume it may have been modified. If the + // transaction aborts, we will get rid if this version out + // of cache. + + if (ui64LowTransId == pDb->getTransID()) + { + setUncommitted(); + linkToDatabaseAtHead( pDatabase); + } + else + { + unsetUncommitted(); + linkToDatabaseAtEnd( pDatabase); + } + } + else + { + FLMUINT64 ui64HighTransId; + + // Adjust the high transaction ID to be the same as + // the transaction ID - we may have gotten a FLM_MAX_UINT64 + // back, but that is possible even if the row is + // not the most current version. Besides that, it is + // possible that in the mean time one or more update + // transactions have come along and created one or + // more newer versions of the row. + + if (bMostCurrent) + { + // This may be showing up as most current simply because we have + // a newer row that was dirty - meaning it would not have been + // written to block cache yet - so our read operation would have + // read the "most current" version of the block that contains + // this row - but it isn't really the most current version of + // the row. + + if (m_pNewerVersion && !m_pNewerVersion->readingInRow()) + { + ui64HighTransId = m_pNewerVersion->getLowTransId() - 1; + } + else + { + ui64HighTransId = FLM_MAX_UINT64; + } + } + else + { + ui64HighTransId = pDb->getTransID(); + } + + setTransID( ui64HighTransId); + + // For a read transaction, if there is a newer version, + // it better have a higher "low transaction ID" + +#ifdef FLM_DEBUG + if (m_pNewerVersion && !m_pNewerVersion->readingInRow()) + { + flmAssert( m_ui64HighTransId < m_pNewerVersion->m_ui64LowTransId); + if( m_ui64HighTransId >= m_pNewerVersion->m_ui64LowTransId) + { + checkReadFromDisk( pDb); + } + } +#endif + unsetUncommitted(); + linkToDatabaseAtEnd( pDatabase); + } + + // Coalesce any versions that overlap - can only + // coalesce older versions. For an updater, there + // should not be any newer versions. For a reader, it + // is impossible to know how high up it can coalesce. + // The read operation that read the row may have + // gotten back a FLM_MAX_UINT64 for its high transaction + // ID - but after that point in time, it is possible + // that one or more update transactions may have come + // along and created one or more newer versions that + // it would be incorrect to coalesce with. + // In reality, a read transaction has to ignore the + // FLM_MAX_UINT64 in the high transaction ID anyway + // because there is no way to know if it is correct. + + // Coalesce older versions. + + for (;;) + { + if ((pTmpRow = m_pOlderVersion) == NULL) + { + break; + } + + // Stop if we encounter one that is being read in. + + if (pTmpRow->readingInRow()) + { + break; + } + + // If there is no overlap between these two, there is + // nothing more to coalesce. + + if (m_ui64LowTransId > pTmpRow->m_ui64HighTransId) + { + break; + } + + if (m_ui64HighTransId <= pTmpRow->m_ui64HighTransId) + { + // This assert represents the following case, + // which should not be possible to hit: + + // pOlder->m_ui64HighTransId > m_ui64HighTransId. + // This cannot be, because if pOlder has a higher + // transaction ID, we would have found it up above and + // not tried to have read it in. + + flmAssert( 0); +#ifdef FLM_DEBUG + checkReadFromDisk( pDb); +#endif + } + else if (m_ui64LowTransId >= pTmpRow->m_ui64LowTransId) + { + m_ui64LowTransId = pTmpRow->m_ui64LowTransId; + pTmpRow->freeCache( + (FLMBOOL)((pTmpRow->rowInUse() || + pTmpRow->readingInRow()) + ? TRUE + : FALSE)); + } + else + { + // This assert represents the following case, + // which should not be possible to hit: + + // m_ui64LowTransId < pOlder->m_ui64LowTransId. + // This cannot be, because pOlder has to have been read + // in to memory by a transaction whose transaction ID is + // less than or equal to our own. That being the case, + // it would be impossible for our transaction to have + // found a version of the row that is older than pOlder. + + flmAssert( 0); +#ifdef FLM_DEBUG + checkReadFromDisk( pDb); +#endif + } + } +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::resizeDataBuffer( + FLMUINT uiSize, + FLMBOOL bMutexAlreadyLocked) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOldSize; + FLMUINT uiNewSize; + FLMUINT uiDataBufSize = calcDataBufSize( uiSize); + FLMBYTE * pucActualAlloc; + FLMBOOL bHeapAlloc = FALSE; + void * pvThis = this; + FLMBOOL bLockedMutex = FALSE; + + flmAssert( !m_uiColumnDataBufSize || m_pucColumnData); + + if( uiDataBufSize == calcDataBufSize(m_uiColumnDataBufSize)) + { + goto Exit; + } + + if( !bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bLockedMutex = TRUE; + } + + uiOldSize = memSize(); + + if (!m_pucColumnData) + { + pucActualAlloc = NULL; + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->allocBuf( + &gv_SFlmSysData.pRowCacheMgr->m_columnDataRelocator, + uiDataBufSize, &pvThis, sizeof( void *), + &pucActualAlloc, &bHeapAlloc))) + { + goto Exit; + } + } + else + { + pucActualAlloc = getActualPointer( m_pucColumnData); + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->reallocBuf( + &gv_SFlmSysData.pRowCacheMgr->m_columnDataRelocator, + calcDataBufSize(m_uiColumnDataBufSize), + uiDataBufSize, &pvThis, sizeof( void *), + &pucActualAlloc, &bHeapAlloc))) + { + goto Exit; + } + } + + flmAssert( *((F_Row **)pucActualAlloc) == this); + setRowAndDataPtr( pucActualAlloc); + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + m_uiColumnDataBufSize = uiSize; +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + + uiNewSize = memSize(); + + if (m_ui64HighTransId != FLM_MAX_UINT64) + { + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes >= uiOldSize); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes -= uiOldSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes += uiNewSize; + } + + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount >= uiOldSize); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount -= uiOldSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount += uiNewSize; + + if( bHeapAlloc) + { + linkToHeapList(); + } + else if( m_uiFlags & FROW_HEAP_ALLOC) + { + unlinkFromHeapList(); + } + +Exit: + + if( bLockedMutex) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + flmAssert( !m_uiColumnDataBufSize || m_pucColumnData); + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::resizeColumnList( + FLMUINT uiColumnCount, + FLMBOOL bMutexAlreadyLocked) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOldSize; + FLMBYTE * pucActualAlloc; + FLMBOOL bHeapAlloc = FALSE; + void * pvThis = this; +#ifdef FLM_CACHE_PROTECT + FLMBOOL bProtectRow = FALSE; +#endif + + if( uiColumnCount == m_uiNumColumns) + { + goto Exit; + } + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); + bProtectRow = TRUE; +#endif + + if( !bMutexAlreadyLocked) + { + flmAssert( rowInUse()); + } + + uiOldSize = memSize(); + + if( !uiColumnCount) + { + // The only thing we better be doing if we pass in a zero, is + // reducing the number of columns. Hence, the current + // column count better be non-zero. + + flmAssert( m_uiNumColumns); + pucActualAlloc = getActualPointer( m_pColumns); + gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->freeBuf( + calcColumnListBufSize( m_uiNumColumns), + &pucActualAlloc); + } + else + { + if( !m_uiNumColumns) + { + pucActualAlloc = NULL; + rc = gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->allocBuf( + &gv_SFlmSysData.pRowCacheMgr->m_columnListRelocator, + calcColumnListBufSize( uiColumnCount), + &pvThis, sizeof( void *), &pucActualAlloc, &bHeapAlloc); + } + else + { + pucActualAlloc = getActualPointer( m_pColumns); + rc = gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->reallocBuf( + &gv_SFlmSysData.pRowCacheMgr->m_columnListRelocator, + calcColumnListBufSize( m_uiNumColumns), + calcColumnListBufSize( uiColumnCount), + &pvThis, sizeof( void *), &pucActualAlloc, &bHeapAlloc); + } + + flmAssert( *((F_Row **)pucActualAlloc) == this); + } + + if (!bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + } + + if (RC_OK( rc)) + { + FLMUINT uiNewSize; + + m_uiNumColumns = uiColumnCount; + if (m_uiNumColumns) + { + setColumnListPtr( pucActualAlloc); + } + else + { + m_pColumns = NULL; + } + + uiNewSize = memSize(); + + if (m_ui64HighTransId != FLM_MAX_UINT64) + { + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes >= uiOldSize); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes -= uiOldSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes += uiNewSize; + } + + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount >= uiOldSize); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount -= uiOldSize; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount += uiNewSize; + + if( bHeapAlloc) + { + linkToHeapList(); + } + else if( m_uiFlags & FROW_HEAP_ALLOC) + { + unlinkFromHeapList(); + } + } + + if (!bMutexAlreadyLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + +Exit: + +#ifdef FLM_CACHE_PROTECT + if( bProtectRow) + { + protectCachedItem(); + } +#endif + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Db::getCachedBTree( + FLMUINT uiTableNum, + F_Btree ** ppBTree) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable = m_pDict->getTable( uiTableNum); + + if (m_pCachedBTree) + { + flmAssert( m_pCachedBTree->getRefCount() == 1); + m_pCachedBTree->btClose(); + } + else + { + // Reserve a B-Tree from the pool + + if( RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &m_pCachedBTree))) + { + goto Exit; + } + } + + // Set up the btree object + + if( RC_BAD( rc = m_pCachedBTree->btOpen( this, + &pTable->lfInfo, FALSE, TRUE))) + { + goto Exit; + } + + m_pCachedBTree->AddRef(); + *ppBTree = m_pCachedBTree; + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +F_BTreeIStreamPool::~F_BTreeIStreamPool() +{ + F_BTreeIStream * pTmpBTreeIStream; + + while ((pTmpBTreeIStream = m_pFirstBTreeIStream) != NULL) + { + m_pFirstBTreeIStream = m_pFirstBTreeIStream->m_pNextInPool; + pTmpBTreeIStream->m_refCnt = 0; + pTmpBTreeIStream->m_pNextInPool = NULL; + delete pTmpBTreeIStream; + } + + if (m_hMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hMutex); + } +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStreamPool::setup( void) +{ + RCODE rc = NE_SFLM_OK; + + if( RC_BAD( rc = f_mutexCreate( &m_hMutex))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStreamPool::allocBTreeIStream( + F_BTreeIStream ** ppBTreeIStream) +{ + RCODE rc = NE_SFLM_OK; + + if( m_pFirstBTreeIStream) + { + f_mutexLock( m_hMutex); + if( !m_pFirstBTreeIStream) + { + f_mutexUnlock( m_hMutex); + } + else + { + f_resetStackInfo( m_pFirstBTreeIStream); + *ppBTreeIStream = m_pFirstBTreeIStream; + m_pFirstBTreeIStream = m_pFirstBTreeIStream->m_pNextInPool; + (*ppBTreeIStream)->m_pNextInPool = NULL; + + f_mutexUnlock( m_hMutex); + goto Exit; + } + } + + if( (*ppBTreeIStream = f_new F_BTreeIStream) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +void F_BTreeIStreamPool::insertBTreeIStream( + F_BTreeIStream * pBTreeIStream) +{ + flmAssert( pBTreeIStream->m_refCnt == 1); + + pBTreeIStream->reset(); + f_mutexLock( m_hMutex); + pBTreeIStream->m_pNextInPool = m_pFirstBTreeIStream; + m_pFirstBTreeIStream = pBTreeIStream; + f_mutexUnlock( m_hMutex); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FLMINT F_BTreeIStream::Release( void) +{ + FLMATOMIC refCnt = --m_refCnt; + + if (m_refCnt == 0) + { + close(); + if( gv_SFlmSysData.pBTreeIStreamPool) + { + m_refCnt = 1; + gv_SFlmSysData.pBTreeIStreamPool->insertBTreeIStream( this); + return( 0); + } + else + { + delete this; + } + } + + return( refCnt); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStream::open( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT32 ui32BlkAddr, + FLMUINT uiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + F_Dict * pDict = pDb->m_pDict; + F_Btree * pBTree = NULL; + F_TABLE * pTable = pDict->getTable( uiTableNum); + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pBTree))) + { + goto Exit; + } + + // Set up the btree object + + if (RC_BAD( rc = pBTree->btOpen( pDb, &pTable->lfInfo, FALSE, TRUE))) + { + goto Exit; + } + + if (RC_BAD( rc = open( pDb, pBTree, FLM_EXACT, uiTableNum, ui64RowId, + ui32BlkAddr, uiOffsetIndex))) + { + goto Exit; + } + + pBTree = NULL; + m_bReleaseBTree = TRUE; + +Exit: + + if (pBTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pBTree); + } + + if( RC_BAD( rc)) + { + close(); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStream::open( + F_Db * pDb, + F_Btree * pBTree, + FLMUINT uiFlags, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + FLMUINT32 ui32BlkAddr, + FLMUINT uiOffsetIndex) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( !m_pBTree); + + m_pDb = pDb; + m_uiTableNum = uiTableNum; + m_pBTree = pBTree; + + // Save the key and key length + + m_uiKeyLength = sizeof( m_ucKey); + if( RC_BAD( rc = flmNumber64ToStorage( ui64RowId, &m_uiKeyLength, + m_ucKey, FALSE, TRUE))) + { + goto Exit; + } + + m_ui32BlkAddr = ui32BlkAddr; + m_uiOffsetIndex = uiOffsetIndex; + + if( RC_BAD( rc = m_pBTree->btLocateEntry( + m_ucKey, sizeof( m_ucKey), &m_uiKeyLength, uiFlags, + NULL, &m_uiStreamSize, &m_ui32BlkAddr, &m_uiOffsetIndex))) + { + if( rc == NE_SFLM_NOT_FOUND) + { + rc = RC_SET( NE_SFLM_ROW_NOT_FOUND); + } + goto Exit; + } + + if( uiFlags == FLM_EXACT) + { + m_ui64RowId = ui64RowId; + } + else + { + if( RC_BAD( rc = flmCollation2Number( m_uiKeyLength, m_ucKey, + &m_ui64RowId, NULL, NULL))) + { + goto Exit; + } + } + +Exit: + + if( RC_BAD( rc)) + { + close(); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStream::positionTo( + FLMUINT64 ui64Position) +{ + RCODE rc = NE_SFLM_OK; + + if( ui64Position >= m_uiBufferStartOffset && + ui64Position <= m_uiBufferStartOffset + m_uiBufferBytes) + { + m_uiBufferOffset = (FLMUINT)(ui64Position - m_uiBufferStartOffset); + } + else + { + if( RC_BAD( rc = m_pBTree->btSetReadPosition( m_ucKey, + m_uiKeyLength, (FLMUINT)ui64Position))) + { + goto Exit; + } + + m_uiBufferStartOffset = (FLMUINT)ui64Position; + m_uiBufferOffset = 0; + m_uiBufferBytes = 0; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_BTreeIStream::read( + void * pvBuffer, + FLMUINT uiBytesToRead, + FLMUINT * puiBytesRead) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucBuffer = (FLMBYTE *)pvBuffer; + FLMUINT uiBufBytesAvail; + FLMUINT uiTmp; + FLMUINT uiOffset = 0; + + flmAssert( m_pBTree); + + while( uiBytesToRead) + { + if( (uiBufBytesAvail = m_uiBufferBytes - m_uiBufferOffset) != 0) + { + uiTmp = f_min( uiBufBytesAvail, uiBytesToRead); + if( pucBuffer) + { + f_memcpy( &pucBuffer[ uiOffset], + &m_pucBuffer[ m_uiBufferOffset], uiTmp); + } + m_uiBufferOffset += uiTmp; + uiOffset += uiTmp; + if( (uiBytesToRead -= uiTmp) == 0) + { + break; + } + } + else + { + m_uiBufferStartOffset += m_uiBufferBytes; + m_uiBufferOffset = 0; + + if( RC_BAD( rc = m_pBTree->btGetEntry( + m_ucKey, m_uiKeyLength, m_uiKeyLength, + m_pucBuffer, m_uiBufferSize, &m_uiBufferBytes))) + { + if( rc == NE_SFLM_EOF_HIT) + { + if( !m_uiBufferBytes) + { + goto Exit; + } + rc = NE_SFLM_OK; + continue; + } + goto Exit; + } + } + } + +Exit: + + if( puiBytesRead) + { + *puiBytesRead = uiOffset; + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::flushRow( + F_Db * pDb, + F_Btree * pBTree) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + F_COLUMN * pColumn; + F_ENCDEF * pEncDef; + F_COLUMN_ITEM * pColItem; + FLMUINT uiIVLen; + FLMUINT uiEncLen; + FLMUINT uiEncOutputLen; + FLMUINT uiColumnDataLen; + FLMUINT uiColumnLenLen; + FLMBYTE * pucOutputColLengths; + FLMBYTE * pucOutputColData; + FLMBYTE * pucColumnData; + FLMBYTE * pucIV; + FLMBYTE ucKeyBuf[ FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiKeyLen; + F_DynaBuf dynaBuf( m_pDatabase->m_pucUpdBuffer, m_pDatabase->m_uiUpdBufferSize); + FLMUINT32 ui32BlkAddr; + FLMUINT uiOffsetIndex; + FLMBOOL bMustAbortOnError = FALSE; + FLMUINT uiLoop; + + // Row should be dirty + + flmAssert( rowIsDirty()); + + // Transaction IDs should match + + if (getLowTransId() != pDb->getTransID()) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + // First pass through columns is to calculate the total length that will + // be needed for column lengths and data. + + uiColumnDataLen = 0; + uiColumnLenLen = 0; + pTable = pDb->m_pDict->getTable( m_uiTableNum); + for (uiLoop = 0, pColItem = m_pColumns, pColumn = pTable->pColumns; + uiLoop < m_uiNumColumns; + uiLoop++, pColItem++, pColumn++) + { + uiColumnLenLen += f_getSENByteCount( pColItem->uiDataLen); + if (!pColumn->uiEncDefNum || !pColItem->uiDataLen) + { + uiColumnDataLen += pColItem->uiDataLen; + } + else + { + pEncDef = pDb->m_pDict->getEncDef( pColumn->uiEncDefNum); + uiIVLen = pEncDef->pCcs->getIVLen(); + flmAssert( uiIVLen == 8 || uiIVLen == 16); + + uiEncLen = getEncLen( pColItem->uiDataLen); + uiColumnDataLen += (f_getSENByteCount( uiEncLen) + uiIVLen + uiEncLen); + } + } + + // Allocate the space needed to output all column data. + + if (RC_BAD( rc = dynaBuf.allocSpace( uiColumnLenLen + uiColumnDataLen, + (void **)&pucOutputColLengths))) + { + goto Exit; + } + pucOutputColData = pucOutputColLengths + uiColumnLenLen; + + // Second pass is to output both column lengths and column data. + + for (uiLoop = 0, pColItem = m_pColumns, pColumn = pTable->pColumns; + uiLoop < m_uiNumColumns; + uiLoop++, pColItem++, pColumn++) + { + f_encodeSEN( pColItem->uiDataLen, &pucOutputColLengths); + if (!pColItem->uiDataLen) + { + continue; + } + pucColumnData = getColumnDataPtr( uiLoop + 1); + if (!pColumn->uiEncDefNum) + { + f_memcpy( pucOutputColData, pucColumnData, pColItem->uiDataLen); + } + else + { + pEncDef = pDb->m_pDict->getEncDef( pColumn->uiEncDefNum); + + uiIVLen = pEncDef->pCcs->getIVLen(); + flmAssert( uiIVLen == 8 || uiIVLen == 16); + + // Output the encryption data length + + uiEncLen = getEncLen( pColItem->uiDataLen); + f_encodeSEN( uiEncLen, &pucOutputColData); + + // Output the IV + + pucIV = pucOutputColData; + if( RC_BAD( rc = pEncDef->pCcs->generateIV( uiIVLen, pucOutputColData))) + { + goto Exit; + } + pucOutputColData += uiIVLen; + + // Output the data and encrypt it. + + f_memcpy( pucOutputColData, pucColumnData, pColItem->uiDataLen); + if (RC_BAD( rc = pDb->encryptData( pColumn->uiEncDefNum, pucIV, + pucOutputColData, uiEncLen, pColItem->uiDataLen, &uiEncOutputLen))) + { + goto Exit; + } + flmAssert( uiEncOutputLen == uiEncLen); + + pucOutputColData += uiEncLen; + } + } + + uiKeyLen = sizeof( ucKeyBuf); + if( RC_BAD( rc = flmNumber64ToStorage( m_ui64RowId, &uiKeyLen, ucKeyBuf, + FALSE, TRUE))) + { + goto Exit; + } + + ui32BlkAddr = m_ui32BlkAddr; + uiOffsetIndex = m_uiOffsetIndex; + + bMustAbortOnError = TRUE; + if (rowIsNew()) + { + if (RC_BAD( rc = pBTree->btInsertEntry( + ucKeyBuf, uiKeyLen, + dynaBuf.getBufferPtr(), dynaBuf.getDataLength(), + TRUE, TRUE, &ui32BlkAddr, &uiOffsetIndex))) + { + if( rc == NE_SFLM_NOT_UNIQUE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_EXISTS); + } + + goto Exit; + } + } + else + { + + // Replace the row on disk. + + if( RC_BAD( rc = pBTree->btReplaceEntry( + ucKeyBuf, uiKeyLen, + dynaBuf.getBufferPtr(), dynaBuf.getDataLength(), + TRUE, TRUE, TRUE, &ui32BlkAddr, + &uiOffsetIndex))) + { + goto Exit; + } + } + + m_ui32BlkAddr = ui32BlkAddr; + m_uiOffsetIndex = uiOffsetIndex; + + // Clear the dirty flag and the new flag. + + unsetRowDirtyAndNew( pDb); + +Exit: + + if (RC_BAD( rc) && bMustAbortOnError) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::flushRow( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + F_Btree * pBTree = NULL; + + if (!rowIsDirty()) + { + goto Exit; + } + + // Get a B-Tree object + + if( RC_BAD( rc = pDb->getCachedBTree( m_uiTableNum, &pBTree))) + { + goto Exit; + } + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + incrRowUseCount(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = FALSE; + + rc = flushRow( pDb, pBTree); + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + + decrRowUseCount(); +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif + + if( RC_BAD( rc)) + { + goto Exit; + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + if (pBTree) + { + pBTree->Release(); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Db::flushDirtyRows( void) +{ + RCODE rc = NE_SFLM_OK; + F_Row * pRow; + F_Btree * pBtree = NULL; + FLMUINT uiTableNum = 0; + + if( !m_uiDirtyRowCount) + { + goto Exit; + } + + // All of the dirty nodes should be at the front of the list. + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + while ((pRow = m_pDatabase->m_pFirstRow) != NULL) + { + if( !pRow->rowIsDirty()) + { + break; + } + + // Flushing the node should remove it from the front of the list. + // Need to increment the use count on the node to prevent it from + // being moved while we are flushing it to disk. + + pRow->incrRowUseCount(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + if (uiTableNum != pRow->m_uiTableNum) + { + if( pBtree) + { + pBtree->Release(); + } + + uiTableNum = pRow->m_uiTableNum; + if( RC_BAD( rc = getCachedBTree( uiTableNum, &pBtree))) + { + goto Exit; + } + } + + rc = pRow->flushRow( this, pBtree); + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + pRow->decrRowUseCount(); + + if( rc == NE_SFLM_NOT_FOUND) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + } + + if( RC_BAD( rc)) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + goto Exit; + } + } + + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + flmAssert( !m_uiDirtyRowCount); + +Exit: + + if (pBtree) + { + pBtree->Release(); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::readRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + IF_IStream * pIStream, + FLMUINT uiRowDataLength) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesRead; + F_TABLE * pTable = pDb->m_pDict->getTable( uiTableNum); + F_COLUMN * pColumn; + F_COLUMN_ITEM * pColItem; + FLMUINT uiLoop; + FLMBOOL bMutexLocked = FALSE; + FLMBYTE * pucColumnData; + FLMBYTE * pucColumnDataEnd; + FLMUINT uiEncDataLen = 0; + F_ENCDEF * pEncDef; + FLMUINT uiIVLen; + FLMUINT uiTotalColumnDataLen; + FLMBOOL bMustMoveOneColumnAtATime; + + // Allocate a buffer for the column data and a column list. + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + if (RC_BAD( rc = resizeDataBuffer( uiRowDataLength, TRUE))) + { + goto Exit; + } + if (RC_BAD( rc = resizeColumnList( pTable->uiNumColumns, TRUE))) + { + goto Exit; + } + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = FALSE; + + // Read the column data into memory. + + if (RC_BAD( rc = pIStream->read( m_pucColumnData, uiRowDataLength, + &uiBytesRead))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + } + goto Exit; + } + else if (uiBytesRead < uiRowDataLength) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Parse through all of the columns - some may be missing. + // All of the lengths precede the actual data. Sum them up to determine + // how big of a buffer we actually need in the end. + + uiTotalColumnDataLen = 0; + pucColumnData = m_pucColumnData; + pucColumnDataEnd = m_pucColumnData + uiRowDataLength; + bMustMoveOneColumnAtATime = FALSE; + for (uiLoop = 0, pColumn = pTable->pColumns, pColItem = m_pColumns; + uiLoop < pTable->uiNumColumns; + uiLoop++, pColumn++, pColItem++) + { + // Get the data length for the column + + if( RC_BAD( rc = f_decodeSEN( (const FLMBYTE **)&pucColumnData, + (const FLMBYTE *)pucColumnDataEnd, + &pColItem->uiDataLen))) + { + goto Exit; + } + if (pColumn->uiEncDefNum) + { + bMustMoveOneColumnAtATime = TRUE; + } + if (pColItem->uiDataLen > sizeof( FLMUINT)) + { + pColItem->uiDataOffset = uiTotalColumnDataLen; + uiTotalColumnDataLen += pColItem->uiDataLen; + } + else if (pColItem->uiDataLen) + { + bMustMoveOneColumnAtATime = TRUE; + } + } + + // Now move all of the data into place. If there were no encrypted + // columns and no columns whose size was <= sizeof( FLMUINT) we can do + // it all with a single memmove. + + if (!bMustMoveOneColumnAtATime) + { + flmAssert( pucColumnData + uiTotalColumnDataLen == pucColumnDataEnd); + f_memmove( m_pucColumnData, pucColumnData, uiTotalColumnDataLen); + } + else + { + + // Must move data a column at a time. + + for (uiLoop = 0, pColumn = pTable->pColumns, pColItem = m_pColumns; + uiLoop < pTable->uiNumColumns; + uiLoop++, pColumn++, pColItem++) + { + + // If there is no data for the column, skip it. + + if (!pColItem->uiDataLen) + { + continue; + + } + + // See if column is encrypted. If so, decrypt it in place before + // moving it to where it needs to go. + + if (pColumn->uiEncDefNum) + { + FLMBYTE * pucIV; + + pEncDef = pDb->m_pDict->getEncDef( pColumn->uiEncDefNum); + flmAssert( pEncDef); + + // Get the encrypted data length and the IV + + if (RC_BAD( rc = f_decodeSEN( (const FLMBYTE **)&pucColumnData, + (const FLMBYTE *)pucColumnDataEnd, &uiEncDataLen))) + { + goto Exit; + } + + uiIVLen = pEncDef->pCcs->getIVLen(); + flmAssert( uiIVLen == 8 || uiIVLen == 16); + + pucIV = pucColumnData; + pucColumnData += uiIVLen; + + // Decrypt the data in place, then move it to where it needs to go. + // We do NOT decrypt it directly to the destination, because the + // destination may overlap with the buffer we are decrypting. + // The decryptData routine does not deal with overlapping buffers + // except the case where the source and destination buffers are + // exactly the same (at least that is the assumption). + + if (RC_BAD( rc = pDb->decryptData( pColumn->uiEncDefNum, pucIV, + pucColumnData, uiEncDataLen, pucColumnData, uiEncDataLen))) + { + goto Exit; + } + } + if (pColItem->uiDataLen <= sizeof( FLMUINT)) + { + f_memcpy( &pColItem->uiDataOffset, pucColumnData, + pColItem->uiDataLen); + } + else + { + f_memmove( m_pucColumnData + pColItem->uiDataOffset, + pucColumnData, pColItem->uiDataLen); + } + + // Move pointer past the encrypted data + + if (pColumn->uiEncDefNum) + { + pucColumnData += uiEncDataLen; + } + else + { + pucColumnData += pColItem->uiDataLen; + } + } + } + + // We better have ended precisely on the end of the buffer. + + flmAssert( pucColumnData == pucColumnDataEnd); + + // Resize the buffer down. + + if (RC_BAD( rc = resizeDataBuffer( uiTotalColumnDataLen, bMutexLocked))) + { + goto Exit; + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine retrieves a row from disk. +****************************************************************************/ +RCODE F_RowCacheMgr::readRowFromDisk( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + F_Row * pRow, + FLMUINT64 * pui64LowTransId, + FLMBOOL * pbMostCurrent) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pBTree = NULL; + FLMBOOL bCloseIStream = FALSE; + F_BTreeIStream btreeIStream; + + if( RC_BAD( rc = pDb->getCachedBTree( uiTableNum, &pBTree))) + { + goto Exit; + } + if (RC_BAD( rc = btreeIStream.open( pDb, pBTree, FLM_EXACT, + uiTableNum, ui64RowId, 0, 0))) + { + goto Exit; + } + bCloseIStream = TRUE; + + // Read the row from the B-Tree + + if (RC_BAD( rc = pRow->readRow( pDb, uiTableNum, + ui64RowId, &btreeIStream, (FLMUINT)btreeIStream.remainingSize()))) + { + if( rc == NE_SFLM_EOF_HIT) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + } + + goto Exit; + } + + pRow->m_uiOffsetIndex = btreeIStream.getOffsetIndex(); + pRow->m_ui32BlkAddr = btreeIStream.getBlkAddr(); + + pBTree->btGetTransInfo( pui64LowTransId, pbMostCurrent); + +Exit: + + if (bCloseIStream) + { + btreeIStream.close(); + } + + if (pBTree) + { + pBTree->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine retrieves a row from the row cache. +****************************************************************************/ +RCODE F_RowCacheMgr::retrieveRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId, + F_Row ** ppRow) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + F_Database * pDatabase = pDb->m_pDatabase; + F_Row * pRow; + F_Row * pNewerRow; + F_Row * pOlderRow; + FLMUINT64 ui64LowTransId; + FLMBOOL bMostCurrent; + FLMUINT64 ui64CurrTransId; + FNOTIFY * pNotify; + FLMUINT uiNumLooks; + FLMBOOL bDontPoisonCache = pDb->m_uiFlags & FDB_DONT_POISON_CACHE + ? TRUE + : FALSE; + + if (RC_BAD( rc = pDb->checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + // Get the current transaction ID + + flmAssert( pDb->m_eTransType != SFLM_NO_TRANS); + ui64CurrTransId = pDb->getTransID(); + flmAssert( ui64RowId != 0); + + // Lock the row cache mutex + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + + // Reset the DB's inactive time + + pDb->m_uiInactiveTime = 0; + +Start_Find: + + findRow( pDb, uiTableNum, ui64RowId, + ui64CurrTransId, bDontPoisonCache, + &uiNumLooks, &pRow, + &pNewerRow, &pOlderRow); + + if (pRow) + { + // Have pRow point to the row we found + goto Exit1; + } + + // Did not find the row, fetch from disk + // Increment the number of faults only if we retrieve the record from disk. + + m_Usage.uiCacheFaults++; + m_Usage.uiCacheFaultLooks += uiNumLooks; + + // Create a place holder for the object. + + if (RC_BAD( rc = allocRow( &pRow, TRUE))) + { + goto Exit; + } + +#ifdef FLM_CACHE_PROTECT + pRow->unprotectCachedItem(); +#endif + + pRow->m_ui64RowId = ui64RowId; + pRow->m_uiTableNum = uiTableNum; + + // Set the F_Database so that other threads looking for this row in + // cache will find it and wait until the read has completed. If + // the F_Database is not set, other threads will attempt their own read, + // because they won't match a NULL F_Database. The result of not setting + // the F_Database is that multiple copies of the same version of a particular + // row could end up in cache. + + pRow->m_pDatabase = pDatabase; + + linkIntoRowCache( pNewerRow, pOlderRow, pRow, !bDontPoisonCache); + + pRow->setReadingIn(); + pRow->incrRowUseCount(); + pRow->m_pNotifyList = NULL; + + // Unlock mutex before reading in from disk. + + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = FALSE; + + // Read row from disk. + + rc = readRowFromDisk( pDb, uiTableNum, ui64RowId, pRow, + &ui64LowTransId, &bMostCurrent); + + // Relock mutex + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + + // If read was successful, link the row to its place in + // the F_Database list and coalesce any versions that overlap + // this one. + + if (RC_OK( rc)) + { + pRow->linkToDatabase( + pDb->m_pDatabase, pDb, ui64LowTransId, bMostCurrent); + } + + pRow->unsetReadingIn(); + + // Notify any threads waiting for the read to complete. + + pNotify = pRow->m_pNotifyList; + pRow->m_pNotifyList = NULL; + if (pNotify) + { + notifyWaiters( pNotify, + (F_Row *)((RC_BAD( rc)) + ? (F_Row *)NULL + : pRow), rc); + } + pRow->decrRowUseCount(); +#ifdef FLM_CACHE_PROTECT + pRow->protectCachedItem(); +#endif + + // If we did not succeed, free the F_Row structure. + + if (RC_BAD( rc)) + { + pRow->freeCache( FALSE); + goto Exit; + } + + // If this item was purged while we were reading it in, + // start over with the search. + + if (pRow->rowPurged()) + { + if (!pRow->rowInUse()) + { + pRow->freePurged(); + } + + // Start over with the find - this one has + // been marked for purging. + + goto Start_Find; + } + +Exit1: + + // Have *ppRow point to the row we read in from disk + + if (*ppRow) + { + (*ppRow)->decrRowUseCount(); + } + *ppRow = pRow; + pRow->incrRowUseCount(); + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine creates a row into the row cache. This is ONLY + called when a new row is being created. +****************************************************************************/ +RCODE F_RowCacheMgr::createRow( + F_Db * pDb, + FLMUINT uiTableNum, + F_Row ** ppRow) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = pDb->m_pDatabase; + F_Row * pRow = NULL; + F_Row * pNewerRow = NULL; + F_Row * pOlderRow = NULL; + FLMBOOL bMutexLocked = FALSE; + F_TABLE * pTable = pDb->m_pDict->getTable( uiTableNum); + + flmAssert( pTable); + + // Lock the row cache mutex + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + + // We are positioned to insert the new row. For an update, it + // must always be the newest version. + + flmAssert( !pNewerRow); + + // Create a new object. + + if (RC_BAD( rc = allocRow( &pRow, bMutexLocked))) + { + goto Exit; + } + +#ifdef FLM_CACHE_PROTECT + pRow->unprotectCachedItem(); +#endif + + pRow->m_ui64RowId = pTable->lfInfo.ui64NextRowId; + pTable->lfInfo.ui64NextRowId++; + pTable->lfInfo.bNeedToWriteOut = TRUE; + pRow->m_uiTableNum = uiTableNum; + pRow->m_uiOffsetIndex = 0; + pRow->m_ui32BlkAddr = 0; + + // NOTE: Not everything is initialized in pRow at this point, but + // no other thread should be accessing it anyway. The caller of this + // function must ensure that all of the necessary items get set before + // releasing the row. + + // Set the F_Database so that other threads looking for this row in + // cache will find it and wait until the read has completed. If + // the F_Database is not set, other threads will attempt their own read, + // because they won't match a NULL F_Database. The result of not setting + // the F_Database is that multiple copies of the same version of a particular + // row could end up in cache. + + pRow->m_pDatabase = pDatabase; + + linkIntoRowCache( pNewerRow, pOlderRow, pRow, TRUE); + + // Link the row to its place in the F_Database list + + pRow->linkToDatabase( pDatabase, pDb, pDb->m_ui64CurrTransID, TRUE); + + flmAssert( *ppRow == NULL); + *ppRow = pRow; + pRow->incrRowUseCount(); + +#ifdef FLM_CACHE_PROTECT + pRow->protectCachedItem(); +#endif + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine makes a writeable copy of the row pointed to by F_Row. +****************************************************************************/ +RCODE F_RowCacheMgr::_makeWriteCopy( + F_Db * pDb, + F_Row ** ppRow) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = pDb->m_pDatabase; + F_Row * pNewerRow = NULL; + F_Row * pOlderRow = *ppRow; + FLMBOOL bMutexLocked = FALSE; +#ifdef FLM_CACHE_PROTECT + FLMBOOL bProtectNewerRow = FALSE; +#endif + + flmAssert( pOlderRow->m_ui64HighTransId == FLM_MAX_UINT64); + flmAssert( !pOlderRow->m_pNewerVersion); + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + bMutexLocked = TRUE; + + // Create a new object. + + if (RC_BAD( rc = allocRow( &pNewerRow, TRUE))) + { + goto Exit; + } + + // If we found the last committed version, instead of replacing it, + // we want to change its high transaction ID, and go create a new + // row to put in cache. + + // Although this routine could be written to not do anything if we + // are already on the uncommitted version of the row, for performance + // reasons, we would prefer that they make the check on the outside + // before calling this routine. + + flmAssert( pOlderRow->m_ui64LowTransId < pDb->m_ui64CurrTransID); + pOlderRow->setTransID( pDb->m_ui64CurrTransID - 1); + flmAssert( pOlderRow->m_ui64HighTransId >= pOlderRow->m_ui64LowTransId); + + pOlderRow->setUncommitted(); + pOlderRow->setLatestVer(); + pOlderRow->unlinkFromDatabase(); + pOlderRow->linkToDatabaseAtHead( pDatabase); + +#ifdef FLM_CACHE_PROTECT + pNewerRow->unprotectCachedItem(); + bProtectNewerRow = TRUE; +#endif + + pNewerRow->m_pDatabase = pDatabase; + pNewerRow->m_uiFlags = pOlderRow->m_uiFlags; + pNewerRow->m_uiOffsetIndex = pOlderRow->m_uiOffsetIndex; + pNewerRow->m_ui32BlkAddr = pOlderRow->m_ui32BlkAddr; + + if( pNewerRow->m_uiFlags & FROW_HEAP_ALLOC) + { + pNewerRow->m_uiFlags &= ~FROW_HEAP_ALLOC; + } + + pNewerRow->m_uiTableNum = pOlderRow->m_uiTableNum; + pNewerRow->m_ui64RowId = pOlderRow->m_ui64RowId; + + if (pNewerRow->m_uiColumnDataBufSize) + { + if (RC_BAD( rc = pNewerRow->resizeDataBuffer( + pNewerRow->m_uiColumnDataBufSize, TRUE))) + { + goto Exit; + } + + f_memcpy( pNewerRow->m_pucColumnData, pOlderRow->m_pucColumnData, + pNewerRow->m_uiColumnDataBufSize); + } + + if (pOlderRow->m_uiNumColumns) + { + if (RC_BAD( rc = pNewerRow->copyColumnList( pDb, pOlderRow, TRUE))) + { + goto Exit; + } + } + + linkIntoRowCache( NULL, pOlderRow, pNewerRow, TRUE); + + // Link the row to its place in the F_Database list + + pNewerRow->linkToDatabase( pDatabase, pDb, pDb->m_ui64CurrTransID, TRUE); + + // Update the row pointer passed into the routine + + if( *ppRow) + { + (*ppRow)->decrRowUseCount(); + } + + *ppRow = pNewerRow; + pNewerRow->incrRowUseCount(); + +#ifdef FLM_CACHE_PROTECT + pNewerRow->protectCachedItem(); + bProtectNewerRow = FALSE; +#endif + + // Set pNewerRow to NULL so it won't get freed at Exit + + pNewerRow = NULL; + +Exit: + + // A non-NULL pNewerRow means there was an error of some kind where we will + // need to free up the cached item. + + if (pNewerRow) + { + flmAssert( RC_BAD( rc)); + delete pNewerRow; + } + + if( bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + + return( rc); +} + +/***************************************************************************** +Desc: +*****************************************************************************/ +RCODE F_Row::allocColumnDataSpace( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT uiSizeNeeded, + FLMBOOL bMutexAlreadyLocked) +{ + RCODE rc = NE_SFLM_OK; + F_COLUMN_ITEM * pColumnItem; + F_COLUMN_ITEM * pTmpColumnItem; + FLMUINT uiLoop; + FLMUINT uiOldLength; + FLMUINT uiOldOffset; + FLMUINT uiOldColumnDataBufSize; + FLMUINT uiChangeSize; + FLMBYTE * pucNextColumnDataStart; + FLMUINT uiSizeAfterCurrColumn; + + // If we have not yet allocated the column list array, allocate it now. + + if (!m_pColumns) + { + if (RC_BAD( rc = resizeColumnList( + pDb->m_pDict->getTable( m_uiTableNum)->uiNumColumns, + bMutexAlreadyLocked))) + { + goto Exit; + } + } + + pColumnItem = &m_pColumns [uiColumnNum - 1]; + uiOldLength = pColumnItem->uiDataLen; + uiOldOffset = pColumnItem->uiDataOffset; + uiOldColumnDataBufSize = m_uiColumnDataBufSize; + if (uiOldLength < uiSizeNeeded) + { + // Only need to increase the buffer size if the size needed is + // greater than sizeof( FLMUINT). Otherwise, both the old data + // and the new data would have fit in the data offset. Hence, + // there would be no need to change anything. + + if (uiSizeNeeded > sizeof( FLMUINT)) + { + if (uiOldLength <= sizeof( FLMUINT)) + { + uiChangeSize = uiSizeNeeded; + } + else + { + uiChangeSize = uiSizeNeeded - uiOldLength; + } + + // Increase the size of the buffer. + + if (RC_BAD( rc = resizeDataBuffer( m_uiColumnDataBufSize + uiChangeSize, + bMutexAlreadyLocked))) + { + goto Exit; + } + + // If the old data was not stored in the buffer, the column's offset + // can simply be set to the old end of the buffer. + + if (uiOldLength <= sizeof( FLMUINT)) + { + pColumnItem->uiDataOffset = uiOldColumnDataBufSize; + } + else + { + + // Move all of the data up for all columns that came after this + // column. The following check is to see if this is the last + // piece of data in the buffer. If it is the last one, no need + // to move anything down. + + if (uiOldOffset + uiOldLength < uiOldColumnDataBufSize) + { + pucNextColumnDataStart = m_pucColumnData + uiOldOffset + uiOldLength; + uiSizeAfterCurrColumn = uiOldColumnDataBufSize - uiOldOffset - uiOldLength; + + f_memmove( pucNextColumnDataStart + uiChangeSize, pucNextColumnDataStart, + uiSizeAfterCurrColumn); + + // Increment all of the data offsets after this current + // column - except for those that are not stored in the buffer. + + for (uiLoop = uiColumnNum, pTmpColumnItem = pColumnItem + 1; + uiLoop < m_uiNumColumns; + uiLoop++, pTmpColumnItem++) + { + + // Only increment the offset if the data is not stored + // in the offset. + + if (pTmpColumnItem->uiDataLen > sizeof( FLMUINT)) + { + pTmpColumnItem->uiDataOffset += uiChangeSize; + } + } + } + } + } + pColumnItem->uiDataLen = uiSizeNeeded; + } + else if (uiOldLength > uiSizeNeeded) + { + + // If the old length is less than or equal to sizeof( FLMUINT), then + // there is no need to change the size of the buffer, because the + // new data will fit where the old data did - inside the data offset. + + if (uiOldLength > sizeof( FLMUINT)) + { + if (uiSizeNeeded <= sizeof( FLMUINT)) + { + uiChangeSize = uiOldLength; + } + else + { + uiChangeSize = uiOldLength - uiSizeNeeded; + } + + // Scoot down all of the column data in the buffer that comes after + // the current column data, if any. + // BUFFER MUST NOT BE RESIZED UNTIL AFTER THIS IS DONE! + + if (uiOldOffset + uiOldLength < uiOldColumnDataBufSize) + { + pucNextColumnDataStart = m_pucColumnData + uiOldOffset + uiOldLength; + uiSizeAfterCurrColumn = uiOldColumnDataBufSize - uiOldOffset - uiOldLength; + + f_memmove( pucNextColumnDataStart - uiChangeSize, pucNextColumnDataStart, + uiSizeAfterCurrColumn); + + // Decrement all of the data offsets after this current + // column - except for those that are not stored in the buffer. + + for (uiLoop = uiColumnNum, pTmpColumnItem = pColumnItem + 1; + uiLoop < m_uiNumColumns; + uiLoop++, pTmpColumnItem++) + { + + // Only increment the offset if the data is not stored + // in the offset. + + if (pTmpColumnItem->uiDataLen > sizeof( FLMUINT)) + { + pTmpColumnItem->uiDataOffset -= uiChangeSize; + } + } + } + + // Need to scoot everything down in the column data buffer. + + if (RC_BAD( rc = resizeDataBuffer( m_uiColumnDataBufSize - uiChangeSize, + bMutexAlreadyLocked))) + { + goto Exit; + } + } + pColumnItem->uiDataLen = uiSizeNeeded; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::setNumber64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 ui64Value, + FLMBOOL bNeg) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable = pDb->m_pDict->getTable( m_uiTableNum); + F_COLUMN * pColumn = pDb->m_pDict->getColumn( pTable, uiColumnNum); + FLMUINT uiValLen; + FLMBYTE ucNumBuf [40]; + + switch (pColumn->eDataTyp) + { + case SFLM_NUMBER_TYPE: + { + if (ui64Value <= 0x7F) + { + if (RC_BAD( rc = allocColumnDataSpace( pDb, uiColumnNum, 1, FALSE))) + { + goto Exit; + } + *(getColumnDataPtr( uiColumnNum)) = (FLMBYTE)ui64Value; + goto Exit; + } + else + { + uiValLen = sizeof( ucNumBuf); + if( RC_BAD( rc = flmNumber64ToStorage( + ui64Value, &uiValLen, ucNumBuf, bNeg, FALSE))) + { + goto Exit; + } + if (RC_BAD( rc = allocColumnDataSpace( pDb, uiColumnNum, uiValLen, FALSE))) + { + goto Exit; + } + f_memcpy( getColumnDataPtr( uiColumnNum), ucNumBuf, uiValLen); + } + + break; + } + + case SFLM_STRING_TYPE: + { + FLMBYTE * pucSen; + FLMUINT uiSenLen; + + if( bNeg) + { + ucNumBuf[ 1] = '-'; + f_ui64toa( ui64Value, (char *)&ucNumBuf[ 2]); + } + else + { + f_ui64toa( ui64Value, (char *)&ucNumBuf[ 1]); + } + + uiValLen = f_strlen( (const char *)&ucNumBuf[ 1]); + pucSen = &ucNumBuf [0]; + uiSenLen = f_encodeSEN( uiValLen, &pucSen, (FLMUINT)0); + flmAssert( uiSenLen == 1); + uiValLen += uiSenLen + 1; + if (RC_BAD( rc = allocColumnDataSpace( pDb, uiColumnNum, uiValLen, FALSE))) + { + goto Exit; + } + f_memcpy( getColumnDataPtr( uiColumnNum), ucNumBuf, uiValLen); + break; + } + + default: + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_DATA_TYPE); + goto Exit; + } + } + setRowDirty( pDb, FALSE); + +Exit: + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +FSTATIC RCODE getStorageAsNumber( + const FLMBYTE * pucData, + FLMUINT uiDataLen, + eDataType eDataType, + FLMUINT64 * pui64Number, + FLMBOOL * pbNeg) +{ + RCODE rc = NE_SFLM_OK; + + *pbNeg = FALSE; + *pui64Number = 0; + + switch (eDataType) + { + case SFLM_NUMBER_TYPE: + if (RC_BAD( rc = flmStorageNumberToNumber( pucData, uiDataLen, + pui64Number, pbNeg))) + { + goto Exit; + } + break; + + case SFLM_STRING_TYPE : + { + const FLMBYTE * pucDataEnd = pucData + uiDataLen; + FLMBYTE ucChar; + FLMBOOL bHex = FALSE; + FLMUINT uiIncrAmount = 0; + FLMBOOL bFirstChar = TRUE; + + // Skip the character count + + if( RC_BAD( rc = f_decodeSEN64( &pucData, pucDataEnd, NULL))) + { + goto Exit; + } + + while (pucData < pucDataEnd) + { + ucChar = *pucData; + if (ucChar >= '0' && ucChar <= '9') + { + uiIncrAmount = (FLMUINT)(ucChar - '0'); + } + else if (ucChar >= 'A' && ucChar <= 'F') + { + if (bHex) + { + uiIncrAmount = (FLMUINT)(ucChar - 'A' + 10); + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_BAD_DIGIT); + goto Exit; + } + } + else if (ucChar >= 'a' && ucChar <= 'f') + { + if (bHex) + { + uiIncrAmount = (FLMUINT)(ucChar - 'a' + 10); + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_BAD_DIGIT); + goto Exit; + } + } + else if (ucChar == 'X' || ucChar == 'x') + { + if (*pui64Number == 0 && !bHex) + { + bHex = TRUE; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_BAD_DIGIT); + goto Exit; + } + } + else if (ucChar == '-' && bFirstChar) + { + *pbNeg = TRUE; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_BAD_DIGIT); + goto Exit; + } + + if (!bHex) + { + if (*pui64Number > (~(FLMUINT64)0) / 10) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + (*pui64Number) *= (FLMUINT64)10; + } + else + { + if (*pui64Number > (~(FLMUINT64)0) / 16) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + (*pui64Number) *= (FLMUINT64)16; + } + + if (*pui64Number > (~(FLMUINT64)0) - (FLMUINT64)uiIncrAmount) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_NUM_OVERFLOW); + goto Exit; + } + + (*pui64Number) += (FLMUINT64)uiIncrAmount; + pucData++; + bFirstChar = FALSE; + } + + break; + } + + default : + { + rc = RC_SET_AND_ASSERT( NE_SFLM_CONV_BAD_DIGIT); + goto Exit; + } + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::getNumber64( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT64 * pui64Value, + FLMBOOL * pbNeg, + FLMBOOL * pbIsNull) +{ + RCODE rc = NE_SFLM_OK; + F_COLUMN_ITEM * pColumnItem; + + *pbIsNull = FALSE; + if ((pColumnItem = getColumn( uiColumnNum)) == NULL) + { + *pbIsNull = TRUE; + goto Exit; + } + else + { + F_TABLE * pTable = pDb->m_pDict->getTable( m_uiTableNum); + F_COLUMN * pColumn = pDb->m_pDict->getColumn( pTable, uiColumnNum); + + if (RC_BAD( rc = getStorageAsNumber( getColumnDataPtr( uiColumnNum), + pColumnItem->uiDataLen, + pColumn->eDataTyp, pui64Value, pbNeg))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::setBinary( + F_Db * pDb, + FLMUINT uiColumnNum, + const void * pvValue, + FLMUINT uiValueLen) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = allocColumnDataSpace( pDb, uiColumnNum, uiValueLen, FALSE))) + { + goto Exit; + } + if( uiValueLen) + { + f_memcpy( getColumnDataPtr( uiColumnNum), pvValue, uiValueLen); + } + setRowDirty( pDb, FALSE); + +Exit: + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::getBinary( + F_Db * pDb, + FLMUINT uiColumnNum, + void * pvBuffer, + FLMUINT uiBufferLen, + FLMUINT * puiDataLen, + FLMBOOL * pbIsNull) +{ + RCODE rc = NE_SFLM_OK; + F_COLUMN_ITEM * pColumnItem; + FLMUINT uiCopySize; + + *pbIsNull = FALSE; + if ((pColumnItem = getColumn( uiColumnNum)) == NULL) + { + *pbIsNull = TRUE; + goto Exit; + } + + // If a NULL buffer is passed in, just return the + // data length + + if (!pvBuffer) + { + if (puiDataLen) + { + *puiDataLen = pColumnItem->uiDataLen; + } + goto Exit; + } + + uiCopySize = f_min( uiBufferLen, pColumnItem->uiDataLen); + if( uiCopySize) + { + f_memcpy( pvBuffer, getColumnDataPtr( uiColumnNum), uiCopySize); + } + + if (puiDataLen) + { + *puiDataLen = uiCopySize; + } + + // If we didn't return all of the data, return an overflow error. + + if (uiCopySize < pColumnItem->uiDataLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +void F_Row::setToNull( + F_Db * pDb, + FLMUINT uiColumnNum) +{ + F_COLUMN_ITEM * pColumnItem; + + // If column is already NULL, no need to do anything. + + if ((pColumnItem = getColumn( uiColumnNum)) != NULL) + { + pColumnItem->uiDataLen = 0; + setRowDirty( pDb, FALSE); + } +} + +/***************************************************************************** +Desc: +******************************************************************************/ +void F_Row::getDataLen( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMUINT * puiDataLen, + FLMBOOL * pbIsNull) +{ + F_COLUMN_ITEM * pColumnItem; + + *pbIsNull = FALSE; + if ((pColumnItem = getColumn( uiColumnNum)) == NULL) + { + *pbIsNull = TRUE; + if (puiDataLen) + { + *puiDataLen = 0; + } + } + else + { + if (puiDataLen) + { + *puiDataLen = pColumnItem->uiDataLen; + } + } +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::setUTF8( + F_Db * pDb, + FLMUINT uiColumnNum, + const char * pszValue, + FLMUINT uiNumBytesInValue, + FLMUINT uiNumCharsInValue) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable = pDb->m_pDict->getTable( m_uiTableNum); + F_COLUMN * pColumn = pDb->m_pDict->getColumn( pTable, uiColumnNum); + FLMUINT uiValLen = 0; + FLMUINT uiSenLen; + FLMBYTE * pucValue = (FLMBYTE *)pszValue; + FLMBOOL bNullTerminate = FALSE; + + if (pColumn->eDataTyp != SFLM_STRING_TYPE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + + if (pucValue && uiNumBytesInValue) + { + if (pucValue[ uiNumBytesInValue - 1] != 0) + { + bNullTerminate = TRUE; + } + uiSenLen = f_getSENByteCount( uiNumCharsInValue); + uiValLen = uiNumBytesInValue + uiSenLen + (bNullTerminate ? 1 : 0); + } + else + { + uiSenLen = 0; + uiValLen = 0; + } + + if (RC_BAD( rc = allocColumnDataSpace( pDb, uiColumnNum, uiValLen, FALSE))) + { + goto Exit; + } + if (uiValLen) + { + FLMBYTE * pucTmp = getColumnDataPtr( uiColumnNum); + + f_encodeSENKnownLength( uiNumCharsInValue, uiSenLen, &pucTmp); + f_memcpy( pucTmp, pucValue, uiNumBytesInValue); + + if (bNullTerminate) + { + pucTmp[ uiNumBytesInValue] = 0; + } + } + setRowDirty( pDb, FALSE); + +Exit: + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::getIStream( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMBOOL * pbIsNull, + F_BufferIStream * pBufferIStream, + eDataType * peDataType, + FLMUINT * puiDataLength) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + F_COLUMN * pColumn; + F_COLUMN_ITEM * pColumnItem; + + *pbIsNull = FALSE; + if ((pColumnItem = getColumn( uiColumnNum)) == NULL) + { + *pbIsNull = TRUE; + goto Exit; + } + if (puiDataLength) + { + *puiDataLength = pColumnItem->uiDataLen; + } + if (peDataType) + { + pTable = pDb->m_pDict->getTable( m_uiTableNum); + pColumn = pDb->m_pDict->getColumn( pTable, uiColumnNum); + *peDataType = pColumn->eDataTyp; + } + if (RC_BAD( rc = pBufferIStream->open( (const char *)getColumnDataPtr( uiColumnNum), + pColumnItem->uiDataLen, NULL))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::getTextIStream( + F_Db * pDb, + FLMUINT uiColumnNum, + FLMBOOL * pbIsNull, + F_BufferIStream * pBufferIStream, + FLMUINT * puiNumChars) +{ + RCODE rc = NE_SFLM_OK; + eDataType eDataTyp; + FLMUINT uiDataLength; + + *puiNumChars = 0; + + if( RC_BAD( rc = getIStream( pDb, uiColumnNum, pbIsNull, + pBufferIStream, &eDataTyp, &uiDataLength))) + { + goto Exit; + } + + if (eDataTyp != SFLM_STRING_TYPE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_DATA_TYPE); + goto Exit; + } + + if (*pbIsNull) + { + goto Exit; + } + + // Skip the leading SEN so that the stream is positioned to + // read raw utf8. + + if (pBufferIStream->remainingSize()) + { + if (RC_BAD( rc = f_readSEN( pBufferIStream, puiNumChars))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::getUTF8( + F_Db * pDb, + FLMUINT uiColumnNum, + char * pszValueBuffer, + FLMUINT uiBufferSize, + FLMBOOL * pbIsNull, + FLMUINT * puiCharsReturned, + FLMUINT * puiBufferBytesUsed) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + F_COLUMN * pColumn; + F_COLUMN_ITEM * pColumnItem; + const FLMBYTE * pucStart; + const FLMBYTE * pucEnd; + FLMUINT uiCharCount; + FLMUINT uiStrByteLen; + + *pbIsNull = FALSE; + if ((pColumnItem = getColumn( uiColumnNum)) == NULL) + { + *pbIsNull = TRUE; + goto Exit; + } + pTable = pDb->m_pDict->getTable( m_uiTableNum); + pColumn = pDb->m_pDict->getColumn( pTable, uiColumnNum); + + pucStart = getColumnDataPtr( uiColumnNum); + pucEnd = pucStart + pColumnItem->uiDataLen; + + if( RC_BAD( rc = f_decodeSEN( &pucStart, pucEnd, &uiCharCount))) + { + goto Exit; + } + + uiStrByteLen = (FLMUINT)(pucEnd - pucStart); + + // If buffer has room, we can simply memcpy the entire string. + + if (uiBufferSize >= uiStrByteLen || !pszValueBuffer) + { + if (pszValueBuffer) + { + f_memcpy( pszValueBuffer, pucStart, uiStrByteLen); + } + if (puiBufferBytesUsed) + { + *puiBufferBytesUsed = uiStrByteLen; + } + if (puiCharsReturned) + { + *puiCharsReturned = uiCharCount; + } + } + else + { + + // Get as many characters as will fit in the buffer. We will return + // an overflow error. + + uiStrByteLen = 0; + uiCharCount = 0; + while (pucStart < pucEnd) + { + if (!(*pucStart)) + { + pucEnd = pucStart; + break; + } + if (!uiBufferSize) + { + break; + } + + *pszValueBuffer++ = (char)(*pucStart); + if (*pucStart <= 0x7F) + { + *pszValueBuffer++ = (char)(*pucStart); + pucStart++; + uiBufferSize--; + uiStrByteLen++; + uiCharCount++; + } + else if ((pucStart [1] >> 6) != 0x02) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_UTF8); + goto Exit; + } + else if ((pucStart [0] >> 5) == 0x06) + { + if (uiBufferSize < 2) + { + break; + } + pszValueBuffer [0] = pucStart [0]; + pszValueBuffer [1] = pucStart [1]; + pszValueBuffer += 2; + pucStart += 2; + uiBufferSize -= 2; + uiStrByteLen += 2; + uiCharCount++; + } + else if ((pucStart [0] >> 4) != 0x0E || (pucStart [2] >> 6) != 0x02) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_UTF8); + goto Exit; + } + else + { + if (uiBufferSize < 3) + { + break; + } + pszValueBuffer [0] = pucStart [0]; + pszValueBuffer [1] = pucStart [1]; + pszValueBuffer [2] = pucStart [2]; + pszValueBuffer += 3; + pucStart += 3; + uiBufferSize -= 3; + uiStrByteLen += 3; + uiCharCount++; + } + } + if (puiBufferBytesUsed) + { + *puiBufferBytesUsed = uiStrByteLen; + } + if (puiCharsReturned) + { + *puiCharsReturned = uiCharCount; + } + + // If we didn't get all of the characters, return an overflow error. + + if (pucStart < pucEnd) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine is called to remove a row from cache. If this is + an uncommitted version of the row, it should remove that version + from cache. If the last committed version is in + cache, it should set the high transaction ID on that version to be + one less than the transaction ID of the update transaction. +****************************************************************************/ +void F_RowCacheMgr::removeRow( + F_Db * pDb, + F_Row * pRow, + FLMBOOL bDecrementUseCount, + FLMBOOL bMutexLocked) +{ + F_Database * pDatabase = pDb->m_pDatabase; + + flmAssert( pRow); + + // Lock the mutex + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + } + + // Decrement the row use count if told to by the caller. + + if (bDecrementUseCount) + { + pRow->decrRowUseCount(); + } + + // Unset the new and dirty flags + + pRow->unsetRowDirtyAndNew( pDb, TRUE); + + // Determine if pRow is the last committed version + // or a row that was added by this same transaction. + // If it is the last committed version, set its high transaction ID. + // Otherwise, remove the row from cache. + + if (pRow->m_ui64LowTransId < pDb->m_ui64CurrTransID) + { + + // The high transaction ID on pRow better be -1 - most current version. + + flmAssert( pRow->m_ui64HighTransId == FLM_MAX_UINT64); + + pRow->setTransID( (pDb->m_ui64CurrTransID - 1)); + flmAssert( pRow->m_ui64HighTransId >= pRow->m_ui64LowTransId); + pRow->setUncommitted(); + pRow->setLatestVer(); + pRow->unlinkFromDatabase(); + pRow->linkToDatabaseAtHead( pDatabase); + } + else + { + pRow->freeCache( pRow->rowInUse()); + } + + if( !bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } +} + +/**************************************************************************** +Desc: This routine is called to remove a row from cache. +****************************************************************************/ +void F_RowCacheMgr::removeRow( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64RowId) +{ + F_Row * pRow; + + // Lock the mutex + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + + // Find the row in cache + + findRow( pDb, uiTableNum, ui64RowId, + pDb->m_ui64CurrTransID, TRUE, NULL, &pRow, NULL, NULL); + + if( pRow) + { + removeRow( pDb, pRow, FALSE, TRUE); + } + + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); +} + +/**************************************************************************** +Desc: This routine is called when an F_Database object is going to be removed + from the shared memory area. At that point, we also need to get rid + of all rows that have been cached for that F_Database. +****************************************************************************/ +void F_Database::freeRowCache( void) +{ + FLMUINT uiNumFreed = 0; + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + while (m_pFirstRow) + { + m_pFirstRow->freeCache( m_pFirstRow->rowInUse()); + + // Release the CPU every 100 rows freed. + + if (++uiNumFreed == 100) + { + f_yieldCPU(); + uiNumFreed = 0; + } + } + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); +} + +/**************************************************************************** +Desc: This routine is called when an update transaction aborts. At that + point, we need to get rid of any uncommitted versions of rows in + the row cache. +****************************************************************************/ +void F_Database::freeModifiedRows( + F_Db * pDb, + FLMUINT64 ui64OlderTransId) +{ + F_Row * pRow; + F_Row * pOlderVersion; + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + pRow = m_pFirstRow; + while (pRow) + { + if (pRow->rowUncommitted()) + { + if (pRow->rowIsLatestVer()) + { + pRow->setTransID( FLM_MAX_UINT64); + pRow->unsetUncommitted(); + pRow->unsetLatestVer(); + pRow->unlinkFromDatabase(); + pRow->linkToDatabaseAtEnd( this); + } + else + { + // Save the older version - we may be changing its + // high transaction ID back to FLM_MAX_UINT64 + + pOlderVersion = pRow->m_pOlderVersion; + + // Clear the dirty and new flags + + pRow->unsetRowDirtyAndNew( pDb, TRUE); + + // Free the uncommitted version. + + pRow->freeCache( + (FLMBOOL)((pRow->rowInUse() || + pRow->readingInRow()) + ? TRUE + : FALSE)); + + // If the older version has a high transaction ID that + // is exactly one less than our current transaction, + // it is the most current version. Hence, we need to + // change its high transaction ID back to FLM_MAX_UINT64. + + if (pOlderVersion && + pOlderVersion->m_ui64HighTransId == ui64OlderTransId) + { + pOlderVersion->setTransID( FLM_MAX_UINT64); + } + } + pRow = m_pFirstRow; + } + else + { + // We can stop when we hit a committed version, because + // uncommitted versions are always linked in together at + // the head of the list. + + break; + } + } + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); +} + +/**************************************************************************** +Desc: This routine is called when an update transaction commits. At that + point, we need to unset the "uncommitted" flag on any rows + currently in row cache for the F_Database object. +****************************************************************************/ +void F_Database::commitRowCache( void) +{ + F_Row * pRow; + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + pRow = m_pFirstRow; + while (pRow) + { + if (pRow->rowUncommitted()) + { + pRow->unsetUncommitted(); + pRow->unsetLatestVer(); + pRow = pRow->m_pNextInDatabase; + } + else + { + + // We can stop when we hit a committed version, because + // uncommitted versions are always linked in together at + // the head of the list. + + break; + } + } + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); +} + +/**************************************************************************** +Desc: This routine is called when a table in the database is deleted. + All rows in row cache that are in that table must be removed from cache. +****************************************************************************/ +void F_Db::removeTableRows( + FLMUINT uiTableNum, + FLMUINT64 ui64TransId) +{ + F_Row * pRow; + F_Row * pNextRow; + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + pRow = m_pDatabase->m_pFirstRow; + + // Stay in the loop until we have freed all rows in the + // collection + + while (pRow) + { + + // Save the pointer to the previous entry in the list because + // we may end up unlinking pRow below, in which case we would + // have lost the previous entry. + + pNextRow = pRow->m_pNextInDatabase; + + // Only look at rows in this collection + + if (pRow->m_uiTableNum == uiTableNum) + { + flmAssert( pRow->m_pDatabase == m_pDatabase); + + // Only look at the most current versions. + + if (pRow->m_ui64HighTransId == FLM_MAX_UINT64) + { + + // Better not be a newer version. + + flmAssert( pRow->m_pNewerVersion == NULL); + + if (pRow->m_ui64LowTransId < ui64TransId) + { + + // This version was not added or modified by this + // transaction so it's high transaction ID should simply + // be set to one less than the current transaction ID. + + pRow->setTransID( ui64TransId - 1); + flmAssert( pRow->m_ui64HighTransId >= pRow->m_ui64LowTransId); + pRow->setUncommitted(); + pRow->setLatestVer(); + pRow->unlinkFromDatabase(); + pRow->linkToDatabaseAtHead( m_pDatabase); + } + else + { + + // The row was added or modified in this + // transaction. Simply remove it from cache. + + pRow->freeCache( pRow->rowInUse()); + } + } + else + { + + // If not most current version, the row's high transaction + // ID better already be less than transaction ID. + + flmAssert( pRow->m_ui64HighTransId < ui64TransId); + } + } + pRow = pNextRow; + + } + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +#ifdef FLM_DEBUG +void F_Row::checkReadFromDisk( + F_Db * pDb) +{ + FLMUINT64 ui64LowTransId; + FLMBOOL bMostCurrent; + RCODE rc; + + // Need to unlock the row cache mutex before doing the read. + + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + rc = gv_SFlmSysData.pRowCacheMgr->readRowFromDisk( pDb, + m_uiTableNum, m_ui64RowId, + this, &ui64LowTransId, &bMostCurrent); + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_Row::setRowDirty( + F_Db * pDb, + FLMBOOL bNew) +{ +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + if (!rowIsDirty()) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + + // Should already be the uncommitted version. + + flmAssert( rowUncommitted()); + + // Should NOT be the latest version - those cannot + // be set to dirty. - latest ver flag is only set + // for rows that should be returned to being the + // latest version of the row if the transaction + // aborts. + + flmAssert( !rowIsLatestVer()); + + // Unlink from its database, set the dirty flag, + // and relink at the head. + + unlinkFromDatabase(); + + if (bNew) + { + m_uiFlags |= (FROW_DIRTY | FROW_NEW); + } + else + { + m_uiFlags |= FROW_DIRTY; + } + + linkToDatabaseAtHead( pDb->m_pDatabase); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + pDb->m_uiDirtyRowCount++; + } + else if (bNew) + { + m_uiFlags |= FROW_NEW; + } + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_Row::unsetRowDirtyAndNew( + F_Db * pDb, + FLMBOOL bMutexAlreadyLocked) +{ +#ifdef FLM_CACHE_PROTECT + unprotectCachedItem(); +#endif + + // When outputting a binary or text stream, it is possible that the + // dirty flag was unset when the last buffer was output + + if (rowIsDirty()) + { + if( !bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + } + + // Unlink from its database, unset the dirty flag, + // and relink at the head. + + unlinkFromDatabase(); + if( m_uiFlags & FROW_DIRTY) + { + flmAssert( pDb->m_uiDirtyRowCount); + pDb->m_uiDirtyRowCount--; + } + + m_uiFlags &= ~(FROW_DIRTY | FROW_NEW); + linkToDatabaseAtHead( pDb->m_pDatabase); + + if( !bMutexAlreadyLocked) + { + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + } + } + +#ifdef FLM_CACHE_PROTECT + protectCachedItem(); +#endif +} + +/***************************************************************************** +Desc: +******************************************************************************/ +RCODE F_Row::copyColumnList( + F_Db * pDb, + F_Row * pSourceRow, + FLMBOOL bMutexAlreadyLocked) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( !m_uiNumColumns); + if( RC_BAD( rc = resizeColumnList( pSourceRow->m_uiNumColumns, + bMutexAlreadyLocked))) + { + goto Exit; + } + if (m_uiNumColumns) + { + f_memcpy( m_pColumns, pSourceRow->m_pColumns, + sizeof( F_COLUMN_ITEM) * m_uiNumColumns); + } + +Exit: + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_Row::resetRow( void) +{ + FLMBYTE * pucActualAlloc; + FLMUINT uiSize = memSize(); + + flmAssert( !m_pPrevInBucket); + flmAssert( !m_pNextInBucket); + flmAssert( !m_pOlderVersion); + flmAssert( !m_pNewerVersion); + flmAssert( !m_pPrevInOldList); + flmAssert( !m_pNextInOldList); + flmAssert( !m_pNotifyList); + flmAssert( !m_uiStreamUseCount); + flmAssert( !rowInUse()); + + f_assertMutexLocked( gv_SFlmSysData.hRowCacheMutex); + + // If this is an old version, decrement the old version counters. + + if (m_ui64HighTransId != FLM_MAX_UINT64) + { + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes >= uiSize && + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerCount); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes -= uiSize; + } + + flmAssert( gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount >= uiSize && + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCount); + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount -= uiSize; + + if( m_uiFlags & FROW_HEAP_ALLOC) + { + unlinkFromHeapList(); + } + + if (m_pucColumnData || m_pColumns) + { + f_assertMutexLocked( gv_SFlmSysData.hRowCacheMutex); + + if (m_pucColumnData) + { + pucActualAlloc = getActualPointer( m_pucColumnData); + gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->freeBuf( + calcDataBufSize(m_uiColumnDataBufSize), + &pucActualAlloc); + m_pucColumnData = NULL; + m_uiColumnDataBufSize = 0; + } + + if (m_pColumns) + { + pucActualAlloc = getActualPointer( m_pColumns); + gv_SFlmSysData.pRowCacheMgr->m_pBufAllocator->freeBuf( + calcColumnListBufSize( m_uiNumColumns), &pucActualAlloc); + m_pColumns = NULL; + m_uiNumColumns = 0; + } + } + + m_ui64LowTransId = 0; + m_ui64HighTransId = FLM_MAX_UINT64; + m_uiCacheFlags = 0; + m_pDatabase = NULL; + m_uiFlags = 0; + m_uiOffsetIndex = 0; + m_ui32BlkAddr = 0; + m_uiTableNum = 0; + m_ui64RowId = 0; + + uiSize = memSize(); + if (m_ui64HighTransId != FLM_MAX_UINT64) + { + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiOldVerBytes += uiSize; + } + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiByteCount += uiSize; +} + +#undef new +#undef delete + +/**************************************************************************** +Desc: +****************************************************************************/ +void * F_Row::operator new( + FLMSIZET uiSize) +#ifndef FLM_NLM + throw() +#endif +{ + void * pvCell; + +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( uiSize); +#endif + flmAssert( uiSize == sizeof( F_Row)); + f_assertMutexLocked( gv_SFlmSysData.hRowCacheMutex); + + if( (pvCell = + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->allocCell( + &gv_SFlmSysData.pRowCacheMgr->m_rowRelocator)) != NULL) + { +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->unprotectCell( pvCell); +#endif + } + + return( pvCell); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void * F_Row::operator new[]( FLMSIZET) +#ifndef FLM_NLM + throw() +#endif +{ + flmAssert( 0); + return( NULL); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +#ifdef FLM_DEBUG +void * F_Row::operator new( + FLMSIZET, // uiSize, + const char *, // pszFileName, + int) // iLineNum) +#ifndef FLM_NLM + throw() +#endif +{ + // This new should never be called + flmAssert( 0); + return( NULL); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +#ifdef FLM_DEBUG +void * F_Row::operator new[]( + FLMSIZET, // uiSize, + const char *, // pszFileName, + int) // iLine) +#ifndef FLM_NLM + throw() +#endif +{ + flmAssert( 0); + return( NULL); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_Row::operator delete( + void * ptr) +{ + if( !ptr) + { + return; + } + +#ifdef FLM_CACHE_PROTECT + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->protectCell( ptr); +#endif + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->freeCell( (FLMBYTE *)ptr, FALSE); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_Row::operator delete[]( + void * // ptr) + ) +{ + flmAssert( 0); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +#if defined( FLM_DEBUG) && !defined( __WATCOMC__) && !defined( FLM_SOLARIS) +void F_Row::operator delete( + void * ptr, + const char *, // pszFileName + int // iLineNum + ) +{ + if( !ptr) + { + return; + } + + gv_SFlmSysData.pRowCacheMgr->m_pRowAllocator->freeCell( (FLMBYTE *)ptr, FALSE); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +#if defined( FLM_DEBUG) && !defined( __WATCOMC__) && !defined( FLM_SOLARIS) +void F_Row::operator delete[]( + void *, // ptr, + const char *, // pszFileName + int // iLineNum + ) +{ + flmAssert( 0); +} +#endif + diff --git a/sql/src/fsblk_u.cpp b/sql/src/fsblk_u.cpp new file mode 100644 index 0000000..80dcd29 --- /dev/null +++ b/sql/src/fsblk_u.cpp @@ -0,0 +1,208 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for handling blocks in the avail list - putting +// blocks into the avail list and removing them from the avail list. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsblk_u.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/*************************************************************************** +Desc: Take current avail block off of free list. Have db header point + to next block in the chain. +*****************************************************************************/ +RCODE F_Database::blockUseNextAvail( + F_Db * pDb, + F_CachedBlock ** ppSCache // Returns pointer to cache structure. + ) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache = NULL; + F_CachedBlock * pNextSCache = NULL; + SFLM_DB_HDR * pDbHdr; + + pDbHdr = &m_uncommittedDbHdr; + + if (RC_BAD( rc = getBlock( pDb, NULL, pDb->m_uiFirstAvailBlkAddr, + NULL, &pSCache))) + { + goto Exit; + } + + // A corruption we have seen a couple of times is where a free + // block points to itself in the free list. This will hang the machine + // so this check has been added to verify that the block is a free block. + + if (pSCache->m_pBlkHdr->ui8BlkType != BT_FREE || + isEncryptedBlk( pSCache->m_pBlkHdr)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Log the block because we are changing it! + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + + *ppSCache = pSCache; + + pDb->m_uiFirstAvailBlkAddr = (FLMUINT)pSCache->m_pBlkHdr->ui32NextBlkInChain; + pDbHdr->ui32FirstAvailBlkAddr = (FLMUINT32)pDb->m_uiFirstAvailBlkAddr; + pSCache->m_pBlkHdr->ui32NextBlkInChain = 0; + + // Set the next block's previous to zero. + + if (pDb->m_uiFirstAvailBlkAddr) + { + if (RC_BAD( rc = getBlock( pDb, NULL, pDb->m_uiFirstAvailBlkAddr, + NULL, &pNextSCache))) + { + goto Exit; + } + + if (pNextSCache->m_pBlkHdr->ui8BlkType != BT_FREE) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Log the block because we are changing it + + if (RC_BAD( rc = logPhysBlk( pDb, &pNextSCache))) + { + goto Exit; + } + pNextSCache->m_pBlkHdr->ui32PrevBlkInChain = 0; + ScaReleaseCache( pNextSCache, FALSE); + pNextSCache = NULL; + } + +Exit: + + if (RC_BAD( rc)) + { + if (pSCache) + { + ScaReleaseCache( pSCache, FALSE); + } + if (pNextSCache) + { + ScaReleaseCache( pNextSCache, FALSE); + } + } + + return( rc); +} + +/*************************************************************************** +Desc: Put a block into the avail list. +Notes: This routine assumes that the block pointed to by pSCache has + been locked into memory. Regardless of whether or not the block + is actually freed on disk, its cache will be released. The + cached block should NOT be accessed after a call to FSBlockFree. +*****************************************************************************/ +RCODE F_Database::blockFree( + F_Db * pDb, + F_CachedBlock * pSCache // Pointer to pointer of cache block + // that is to be freed. NOTE: Regardless of whether + // or not the block is actually freed, it will be + // released. + ) +{ + RCODE rc = NE_SFLM_OK; + F_BLK_HDR * pBlkHdr; + F_CachedBlock * pFirstAvailSCache; + SFLM_DB_HDR * pDbHdr; + + pDbHdr = &m_uncommittedDbHdr; + + // Log the block before modifying it. + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + pBlkHdr = pSCache->m_pBlkHdr; + + // Modify header to be an avail block. + + if (isEncryptedBlk( pBlkHdr)) + { + // If block was previously encrypted, need to zero it + // out so that clear data will not be written to disk + // when this avail block is written out. + + unsetBlockEncrypted( pBlkHdr); + f_memset( ((FLMBYTE *)pBlkHdr) + SIZEOF_STD_BLK_HDR, 0, + m_uiBlockSize - SIZEOF_STD_BLK_HDR); + } + pBlkHdr->ui8BlkType = BT_FREE; + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)(m_uiBlockSize - + SIZEOF_STD_BLK_HDR); + + // Modify back chain of first block in avail list, if any, + // to point back to this block. + + if (pDbHdr->ui32FirstAvailBlkAddr) + { + + // Read the block at head of avail list. + + if (RC_BAD( rc = getBlock( pDb, NULL, + (FLMUINT)pDbHdr->ui32FirstAvailBlkAddr, + NULL, &pFirstAvailSCache))) + { + goto Exit; + } + + // Log the block + + if (RC_OK( rc = logPhysBlk( pDb, &pFirstAvailSCache))) + { + + // Previous block pointer better be zero at this point. + + flmAssert( pFirstAvailSCache->m_pBlkHdr->ui32PrevBlkInChain == 0); + pFirstAvailSCache->m_pBlkHdr->ui32PrevBlkInChain = pBlkHdr->ui32BlkAddr; + } + ScaReleaseCache( pFirstAvailSCache, FALSE); + if (RC_BAD( rc)) + { + goto Exit; + } + } + + // Link block at head of avail list. + + pBlkHdr->ui32PrevBlkInChain = 0; + pBlkHdr->ui32NextBlkInChain = pDbHdr->ui32FirstAvailBlkAddr; + pDbHdr->ui32FirstAvailBlkAddr = pBlkHdr->ui32BlkAddr; + pDb->m_uiFirstAvailBlkAddr = (FLMUINT)pBlkHdr->ui32BlkAddr; + +Exit: + + ScaReleaseCache( pSCache, FALSE); + return( rc); +} diff --git a/sql/src/fscursor.cpp b/sql/src/fscursor.cpp new file mode 100644 index 0000000..b02f918 --- /dev/null +++ b/sql/src/fscursor.cpp @@ -0,0 +1,1037 @@ +//------------------------------------------------------------------------------ +// Desc: Cursor routines to get the complexity of the file system out +// of the search code. +// +// Tabs: 3 +// +// Copyright (c) 2000-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fscursor.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" +#include "fscursor.h" + +/**************************************************************************** +Desc: +****************************************************************************/ +FSIndexCursor::FSIndexCursor() +{ + m_pbTree = NULL; + m_bTreeOpen = FALSE; + m_pucCurKeyDataBuf = NULL; + m_uiCurKeyDataBufSize = 0; + m_uiCurKeyDataLen = 0; + m_pIndex = NULL; + m_pLFile = NULL; + m_pDb = NULL; + m_eTransType = SFLM_NO_TRANS; + resetCursor(); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FSIndexCursor::~FSIndexCursor() +{ + closeBTree(); + if (m_pucCurKeyDataBuf) + { + f_free( &m_pucCurKeyDataBuf); + } + if (m_pbTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &m_pbTree); + } +} + +/**************************************************************************** +Desc: Resets any allocations, keys, state, etc. +****************************************************************************/ +void FSIndexCursor::resetCursor( void) +{ + closeBTree(); + + m_uiIndexNum = 0; + m_uiBlkChangeCnt = 0; + m_ui64CurrTransId = 0; + m_curKey.uiKeyLen = 0; + m_uiCurKeyDataLen = 0; + m_bAtBOF = TRUE; + m_bAtEOF = FALSE; + m_bSetup = FALSE; +} + +/**************************************************************************** +Desc: Resets to a new transaction that may change the read consistency of + the query. +****************************************************************************/ +RCODE FSIndexCursor::resetTransaction( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + LFILE * pLFile; + F_INDEX * pIndex; + + if ((pIndex = pDb->m_pDict->getIndex( m_uiIndexNum)) == NULL) + { + rc = RC_SET( NE_SFLM_INVALID_INDEX_NUM); + goto Exit; + } + if (pIndex->uiFlags & (IXD_OFFLINE | IXD_SUSPENDED)) + { + rc = RC_SET( NE_SFLM_INDEX_OFFLINE); + goto Exit; + } + pLFile = &pIndex->lfInfo; + if (m_pDb != pDb || pLFile != m_pLFile || pIndex != m_pIndex) + { + m_pLFile = pLFile; + m_pIndex = pIndex; + if (m_bTreeOpen) + { + closeBTree(); + } + m_pDb = pDb; + m_eTransType = pDb->m_eTransType; + } + m_ixCompare.setIxInfo( pDb, m_pIndex); + m_ui64CurrTransId = pDb->m_ui64CurrTransID; + m_uiBlkChangeCnt = pDb->m_uiBlkChangeCnt; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Setup the from and until keys in the cursor. Return counts + after positioning to the from and until key in the index. +****************************************************************************/ +RCODE FSIndexCursor::setupKeys( + F_Db * pDb, + F_INDEX * pIndex, + F_TABLE * pTable, + SQL_PRED * pPred, + FLMBOOL * pbDoRowMatch, + FLMBOOL * pbCanCompareOnKey, + FLMUINT64 * pui64LeafBlocksBetween, // [out] blocks between the stacks + FLMUINT64 * pui64TotalRefs, // [out] total references + FLMBOOL * pbTotalsEstimated) // [out] set to TRUE when estimating. +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pUntilBTree = NULL; + FLMINT iCompare; + + m_uiIndexNum = pIndex->uiIndexNum; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + +// VISIT: NEED TO REDO THIS + + if (RC_BAD( rc = flmBuildFromAndUntilKeys( pDb->getDict(), pIndex, pTable, + pPred, &m_fromExtKey, m_fromKey.ucKey, &m_fromKey.uiKeyLen, + &m_untilExtKey, m_untilKey.ucKey, &m_untilKey.uiKeyLen, + pbDoRowMatch, pbCanCompareOnKey))) + { + goto Exit; + } + + m_curKey.uiKeyLen = 0; + m_bSetup = TRUE; + + // Want any of the counts back? + + if (pui64LeafBlocksBetween || pui64TotalRefs) + { + + // Get a btree object + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pUntilBTree))) + { + goto Exit; + } + + if (RC_OK( rc = setKeyPosition( pDb, TRUE, FALSE, + &m_fromExtKey, &m_fromKey, &m_curKey, TRUE, NULL, NULL, NULL))) + { + + // All keys between FROM and UNTIL may be gone. + + if (RC_BAD( rc = ixKeyCompare( m_pDb, m_pIndex, &m_untilExtKey, + NULL, NULL, FALSE, + m_curKey.ucKey, m_curKey.uiKeyLen, + m_untilKey.ucKey, m_untilKey.uiKeyLen, + &iCompare))) + { + goto Exit; + } + if (iCompare <= 0) + { + KEYPOS tmpUntilKey; + + if (RC_OK( rc = pUntilBTree->btOpen( pDb, m_pLFile, + isAbsolutePositionable(), FALSE, + &m_ixCompare))) + { + + // Going forward so position is exclusive + + rc = setKeyPosition( pDb, FALSE, FALSE, + &m_untilExtKey, &m_untilKey, + &tmpUntilKey, FALSE, NULL, pUntilBTree, NULL); + } + } + else + { + rc = RC_SET( NE_SFLM_BOF_HIT); + m_bAtBOF = TRUE; + m_bAtEOF = FALSE; + } + } + else + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + m_bAtBOF = FALSE; + } + else if (rc == NE_SFLM_BOF_HIT) + { + m_bAtEOF = FALSE; + m_bAtBOF = TRUE; + } + } + + if (RC_BAD( rc)) + { + + // Empty tree or empty set case. + + if (rc == NE_SFLM_EOF_HIT || rc == NE_SFLM_BOF_HIT) + { + if (pui64LeafBlocksBetween) + { + *pui64LeafBlocksBetween = 0; + } + if (pui64TotalRefs) + { + *pui64TotalRefs = 0; + } + if (pbTotalsEstimated) + { + *pbTotalsEstimated = FALSE; + } + rc = NE_SFLM_OK; + } + goto Exit; + } + else + { + if (RC_BAD( rc = m_pbTree->btComputeCounts( pUntilBTree, + pui64LeafBlocksBetween, pui64TotalRefs, + pbTotalsEstimated, + (pDb->m_pDatabase->m_uiBlockSize * 3) / 4))) + { + goto Exit; + } + } + } + m_bAtBOF = TRUE; + +Exit: + + if (pUntilBTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pUntilBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: Open the F_Btree object if not already open. +****************************************************************************/ +RCODE FSIndexCursor::openBTree( + F_Db * pDb + ) +{ + RCODE rc = NE_SFLM_OK; + + if (!m_bTreeOpen) + { + if ( !m_pbTree) + { + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &m_pbTree))) + { + goto Exit; + } + } +Open_Btree: + if (RC_BAD( rc = m_pbTree->btOpen( pDb, m_pLFile, + isAbsolutePositionable(), FALSE, + &m_ixCompare))) + { + goto Exit; + } + m_bTreeOpen = TRUE; + m_pDb = pDb; + m_eTransType = pDb->m_eTransType; + m_ixCompare.setIxInfo( m_pDb, m_pIndex); + } + else + { + if (pDb != m_pDb || pDb->m_eTransType != m_eTransType) + { + closeBTree(); + goto Open_Btree; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Get a key's data part. +****************************************************************************/ +RCODE FSIndexCursor::getKeyData( + F_Btree * pBTree, + FLMUINT uiDataLen) +{ + RCODE rc = NE_SFLM_OK; + + m_uiCurKeyDataLen = 0; + + // See if there is a data part + + if (m_pIndex->pDataIcds && uiDataLen) + { + + // If the data will fit in the search key buffer, just + // reuse it since we are not going to do anything with + // it after this. Otherwise, allocate a new buffer. + + if (uiDataLen > m_uiCurKeyDataBufSize) + { + FLMBYTE * pucNewBuf; + FLMUINT uiNewLen = uiDataLen; + + if (uiNewLen < 256) + { + uiNewLen = 256; + } + + if (RC_BAD( rc = f_alloc( uiNewLen, &pucNewBuf))) + { + goto Exit; + } + if (m_pucCurKeyDataBuf) + { + f_free( &m_pucCurKeyDataBuf); + } + m_pucCurKeyDataBuf = pucNewBuf; + m_uiCurKeyDataBufSize = uiNewLen; + } + + // Retrieve the data + + if (RC_BAD( rc = pBTree->btGetEntry( + m_curKey.ucKey, SFLM_MAX_KEY_SIZE, m_curKey.uiKeyLen, + m_pucCurKeyDataBuf, m_uiCurKeyDataBufSize, + &m_uiCurKeyDataLen))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set the key position given some KEYPOS structure. + Please note that the blocks in the stack may or may not be used. +****************************************************************************/ +RCODE FSIndexCursor::setKeyPosition( + F_Db * pDb, + FLMBOOL bGoingForward, + FLMBOOL bExcludeKey, + F_DataVector * pExtSrchKey, + KEYPOS * pSearchKey, // Search key + KEYPOS * pFoundKey, + FLMBOOL bGetKeyData, + FLMUINT * puiDataLen, + F_Btree * pBTree, // BTree to use. NULL means use our + // internal one. + FLMUINT * puiAbsolutePos) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDataLen; + FLMINT iCompare = 0; + + // if pBTree is NULL, we are to use m_pbTree. Otherwise, we + // need to open the pBTree and use it. + + if (!pBTree) + { + if (RC_BAD( rc = openBTree( pDb))) + { + goto Exit; + } + pBTree = m_pbTree; + } + + if (pFoundKey != pSearchKey) + { + f_memcpy( pFoundKey->ucKey, pSearchKey->ucKey, + pSearchKey->uiKeyLen); + pFoundKey->uiKeyLen = pSearchKey->uiKeyLen; + } + + m_ixCompare.setSearchKey( pExtSrchKey); + m_ixCompare.setCompareRowId( pExtSrchKey ? FALSE : TRUE); + if (RC_BAD( rc = pBTree->btLocateEntry( pFoundKey->ucKey, SFLM_MAX_KEY_SIZE, + &pFoundKey->uiKeyLen, + (bGoingForward && bExcludeKey) + ? FLM_EXCL + : FLM_INCL, + puiAbsolutePos, &uiDataLen, NULL, NULL))) + { + if (rc != NE_SFLM_EOF_HIT) + { + goto Exit; + } + } + + if (bGoingForward) + { + if (rc == NE_SFLM_EOF_HIT) + { + goto Exit; + } + } + else + { + + // Going backwards or to last. See if we positioned too far. + + if (rc == NE_SFLM_EOF_HIT) + { + + // Position to last key in tree. + + if (RC_BAD( rc = pBTree->btLastEntry( pFoundKey->ucKey, SFLM_MAX_KEY_SIZE, + &pFoundKey->uiKeyLen, + &uiDataLen, NULL, NULL))) + { + goto Exit; + } + } + else + { + + // We want to go to the previous key if we went past the key + // we were aiming for, or if we landed on that key, but we + // are doing an exclusive lookup. + + if (!bExcludeKey) + { + if (RC_BAD( rc = ixKeyCompare( m_pDb, m_pIndex, pExtSrchKey, + NULL, NULL, + pExtSrchKey ? FALSE : TRUE, + pFoundKey->ucKey, pFoundKey->uiKeyLen, + pSearchKey->ucKey, pSearchKey->uiKeyLen, + &iCompare))) + { + goto Exit; + } + } + + if (bExcludeKey || iCompare > 0) + { + + // Position to the previous key. + + if (RC_BAD( rc = pBTree->btPrevEntry( pFoundKey->ucKey, SFLM_MAX_KEY_SIZE, + &pFoundKey->uiKeyLen, + &uiDataLen, NULL, NULL))) + { + goto Exit; + } + } + } + } + + // See if there is any data to get + + if (bGetKeyData) + { + flmAssert( pFoundKey == &m_curKey); + + if (RC_BAD( rc = getKeyData( pBTree, uiDataLen))) + { + goto Exit; + } + } + if (puiDataLen) + { + *puiDataLen = uiDataLen; + } + +Exit: + + if (RC_BAD( rc)) + { + pFoundKey->uiKeyLen = 0; + if (pBTree == m_pbTree) + { + closeBTree(); + } + } + return( rc); +} + +/**************************************************************************** +Desc: Populate a key from the current key. +****************************************************************************/ +RCODE FSIndexCursor::populateKey( + F_DataVector * pKey) +{ + RCODE rc = NE_SFLM_OK; + + pKey->reset(); + if (RC_BAD( rc = pKey->inputKey( m_pDb, m_uiIndexNum, + m_curKey.ucKey, m_curKey.uiKeyLen))) + { + goto Exit; + } + + // See if there is a data part + + if (m_pIndex->pDataIcds && m_uiCurKeyDataLen) + { + + // Parse the data + + if (RC_BAD( rc = pKey->inputData( m_pDb, m_uiIndexNum, m_pucCurKeyDataBuf, + m_uiCurKeyDataLen))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Return the current record and record id. +****************************************************************************/ +RCODE FSIndexCursor::currentKey( + F_Db * pDb, + F_DataVector * pKey) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + if (m_bAtBOF) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + if (m_bAtEOF) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + // If we get this far, we are positioned on some key. + + flmAssert( m_curKey.uiKeyLen); + + if (RC_BAD( rc = populateKey( pKey))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Make sure the current key is positioned in a key set. If not, + move to the next or previous key set until it is. +****************************************************************************/ +RCODE FSIndexCursor::checkIfKeyInRange( + FLMBOOL bPositionForward) +{ + RCODE rc = NE_SFLM_OK; + FLMINT iCompare; + + if (bPositionForward) + { + + // See if the key is within the bounds of the current key set. + // It is already guaranteed to be >= the fromKey, we just need to + // make sure it is <= the until key. + + if (RC_BAD( rc = ixKeyCompare( m_pDb, m_pIndex, &m_untilExtKey, + NULL, NULL, FALSE, + m_curKey.ucKey, m_curKey.uiKeyLen, + m_untilKey.ucKey, m_untilKey.uiKeyLen, + &iCompare))) + { + goto Exit; + } + + if (iCompare > 0) + { + m_bAtEOF = TRUE; + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + } + else + { + + // See if the key is within the bounds of the current key set. + // It is already guaranteed to be <= the untilKey, we just need to + // make sure it is >= the from key. + + if (RC_BAD( rc = ixKeyCompare( m_pDb, m_pIndex, &m_fromExtKey, + NULL, NULL, FALSE, + m_curKey.ucKey, m_curKey.uiKeyLen, + m_fromKey.ucKey, m_fromKey.uiKeyLen, + &iCompare))) + { + goto Exit; + } + + if (iCompare < 0) + { + m_bAtBOF = TRUE; + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Position to and return the first key. +****************************************************************************/ +RCODE FSIndexCursor::firstKey( + F_Db * pDb, + F_DataVector * pKey) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + + // If at BOF and we have a key, then we are positioned on the first + // key already - this would have happened if we had positioned to + // calculate a cost. Rather than do the positioning again, we simply + // set m_bAtBOF to FALSE. + + if (m_bAtBOF && m_curKey.uiKeyLen) + { + m_bAtBOF = FALSE; + } + else + { + m_bAtBOF = m_bAtEOF = FALSE; + if (RC_BAD( rc = setKeyPosition( pDb, TRUE, FALSE, + &m_fromExtKey, &m_fromKey, + &m_curKey, TRUE, NULL, NULL, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + } + + // Make sure the current key is within the FROM/UNTIL range. + + if (RC_BAD( rc = checkIfKeyInRange( TRUE))) + { + goto Exit; + } + + if (RC_BAD( rc = populateKey( pKey))) + { + goto Exit; + } + + +Exit: + + if (RC_BAD( rc)) + { + m_curKey.uiKeyLen = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to the next key. +****************************************************************************/ +RCODE FSIndexCursor::nextKey( + F_Db * pDb, + F_DataVector * pKey, + FLMBOOL bSkipCurrKey) +{ + RCODE rc = NE_SFLM_OK; + KEYPOS saveCurrentKey; + FLMUINT uiDataLen; + FLMINT iCompare; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + + flmAssert( m_bSetup); + if (m_bAtEOF) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + if (m_bAtBOF || !m_curKey.uiKeyLen) + { + rc = firstKey( pDb, pKey); + goto Exit; + } + + // Save the current key, so we can tell when we have gone beyond it. + + if (bSkipCurrKey) + { + getCurrKey( &saveCurrentKey); + } + + // See if we need to reset the b-tree object we are using + + if (m_bTreeOpen && + (pDb != m_pDb || pDb->m_eTransType != m_eTransType)) + { + closeBTree(); + } + for (;;) + { + + // checkTransaction may have closed the B-tree, in which case we + // need to reopen the b-tree and position exclusive of the current key. + + if (!m_bTreeOpen) + { + if (RC_BAD( rc = setKeyPosition( pDb, TRUE, TRUE, + NULL, &m_curKey, &m_curKey, FALSE, &uiDataLen, + NULL, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + } + else + { + + // set the compare object's search key to NULL because m_curKey + // should always be something we obtained from the index, and + // we should not need a search key for doing extended + // comparisons on truncated keys. + + m_ixCompare.setSearchKey( NULL); + m_ixCompare.setCompareRowId( TRUE); + + // Get the next key, if any + + if (RC_BAD( rc = m_pbTree->btNextEntry( m_curKey.ucKey, SFLM_MAX_KEY_SIZE, + &m_curKey.uiKeyLen, &uiDataLen, NULL, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + } + if (!bSkipCurrKey) + { +Check_Key: + if (RC_BAD( rc = getKeyData( m_pbTree, uiDataLen))) + { + goto Exit; + } + + // We got to the next key, make sure it is in the key range. + + if (RC_BAD( rc = checkIfKeyInRange( TRUE))) + { + goto Exit; + } + + if (RC_BAD( rc = populateKey( pKey))) + { + goto Exit; + } + + break; + } + + // If the bSkipCurrKey flag is TRUE, we want to skip keys until + // we come to one where the key part is different. + + if (RC_BAD( rc = ixKeyCompare( pDb, m_pIndex, NULL, NULL, NULL, FALSE, + m_curKey.ucKey, m_curKey.uiKeyLen, + saveCurrentKey.ucKey, saveCurrentKey.uiKeyLen, + &iCompare))) + { + goto Exit; + } + + // See if the key part is the same. + + if (iCompare != 0) + { + goto Check_Key; + } + } + +Exit: + + if (RC_BAD( rc)) + { + m_curKey.uiKeyLen = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to and return the last key in the range. +****************************************************************************/ +RCODE FSIndexCursor::lastKey( + F_Db * pDb, + F_DataVector * pKey) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + + // Position to the until key. + + m_bAtBOF = m_bAtEOF = FALSE; + if (RC_BAD( rc = setKeyPosition( pDb, FALSE, FALSE, + &m_untilExtKey, &m_untilKey, + &m_curKey, TRUE, NULL, NULL, NULL))) + { + if (rc == NE_SFLM_BOF_HIT) + { + m_bAtBOF = TRUE; + } + goto Exit; + } + + // Make sure the current key is within one of the FROM/UNTIL sets. + + if (RC_BAD( rc = checkIfKeyInRange( FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = populateKey( pKey))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc)) + { + m_curKey.uiKeyLen = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to the PREVIOUS key in the range. +****************************************************************************/ +RCODE FSIndexCursor::prevKey( + F_Db * pDb, + F_DataVector * pKey, + FLMBOOL bSkipCurrKey) +{ + RCODE rc = NE_SFLM_OK; + KEYPOS saveCurrentKey; + FLMUINT uiDataLen; + FLMINT iCompare; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + + flmAssert( m_bSetup); + + if (m_bAtBOF) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + + if (m_bAtEOF || !m_curKey.uiKeyLen) + { + rc = lastKey( pDb, pKey); + goto Exit; + } + + // Save the current key, so we can tell when we have gone beyond it. + + if (bSkipCurrKey) + { + getCurrKey( &saveCurrentKey); + } + + // See if we need to reset the b-tree object we are using + + if (m_bTreeOpen && + (pDb != m_pDb || pDb->m_eTransType != m_eTransType)) + { + closeBTree(); + } + for (;;) + { + + // checkTransaction may have closed the B-tree, in which case we + // need to reopen the b-tree and position exclusive of the current key. + + if (!m_bTreeOpen) + { + if (RC_BAD( rc = setKeyPosition( pDb, FALSE, TRUE, + NULL, &m_curKey, &m_curKey, + FALSE, &uiDataLen, NULL, NULL))) + { + if (rc == NE_SFLM_BOF_HIT) + { + m_bAtBOF = TRUE; + } + goto Exit; + } + } + else + { + + // set the compare object's search key to NULL because m_curKey + // should always be something we obtained from the index, and + // we should not need a search key for doing extended + // comparisons on truncated keys. + + m_ixCompare.setSearchKey( NULL); + m_ixCompare.setCompareRowId( TRUE); + + // Get the previous key, if any + + if (RC_BAD( rc = m_pbTree->btPrevEntry( m_curKey.ucKey, SFLM_MAX_KEY_SIZE, + &m_curKey.uiKeyLen, &uiDataLen, NULL, NULL))) + { + if (rc == NE_SFLM_BOF_HIT) + { + m_bAtBOF = TRUE; + } + goto Exit; + } + } + if (!bSkipCurrKey) + { +Check_Key: + if (RC_BAD( rc = getKeyData( m_pbTree, uiDataLen))) + { + goto Exit; + } + + // We got to the previous key, make sure it is in the key range. + + if (RC_BAD( rc = checkIfKeyInRange( FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = populateKey( pKey))) + { + goto Exit; + } + + break; + } + + // If the bSkipCurrKey flag is TRUE, we want to skip keys until + // we come to one where the key part is different. + + + if (RC_BAD( rc = ixKeyCompare( pDb, m_pIndex, NULL, NULL, NULL, FALSE, + m_curKey.ucKey, m_curKey.uiKeyLen, + saveCurrentKey.ucKey, saveCurrentKey.uiKeyLen, + &iCompare))) + { + goto Exit; + } + + // See if the key part is the same. + + if (iCompare != 0) + { + goto Check_Key; + } + } + +Exit: + + if (RC_BAD( rc)) + { + m_curKey.uiKeyLen = 0; + } + + return( rc); +} + diff --git a/sql/src/fscursor.h b/sql/src/fscursor.h new file mode 100644 index 0000000..d24289c --- /dev/null +++ b/sql/src/fscursor.h @@ -0,0 +1,334 @@ +//------------------------------------------------------------------------------ +// Desc: This is the header file that contains the FSIndexCursor class. +// +// Tabs: 3 +// +// Copyright (c) 2000-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fscursor.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FSCURSOR_H +#define FSCURSOR_H + +#include "fdynsset.h" + +typedef struct KeyPosition +{ + FLMBYTE ucKey [SFLM_MAX_KEY_SIZE]; + FLMUINT uiKeyLen; +} KEYPOS; + +/*============================================================================ +Desc: File system implementation of a cursor for an index. +============================================================================*/ +class FSIndexCursor : public F_Object +{ +public: + + // Constructors & Destructor + + FSIndexCursor(); + virtual ~FSIndexCursor(); + + void resetCursor( void); + + RCODE resetTransaction( + F_Db * pDb); + + RCODE setupKeys( + F_Db * pDb, + F_INDEX * pIndex, + F_TABLE * pTable, + SQL_PRED * pPred, + FLMBOOL * pbDoRowMatch, + FLMBOOL * pbCanCompareOnKey, + FLMUINT64 * pui64LeafBlocksBetween, + FLMUINT64 * pui64TotalRefs, + FLMBOOL * pbTotalsEstimated); + + RCODE currentKey( + F_Db * pDb, + F_DataVector * pKey); + + RCODE firstKey( + F_Db * pDb, + F_DataVector * pKey); + + RCODE lastKey( + F_Db * pDb, + F_DataVector * pKey); + + RCODE nextKey( + F_Db * pDb, + F_DataVector * pKey, + FLMBOOL bSkipCurrKey); + + RCODE prevKey( + F_Db * pDb, + F_DataVector * pKey, + FLMBOOL bSkipCurrKey); + +private: + + RCODE useNewDb( + F_Db * pDb); + + RCODE openBTree( + F_Db * pDb); + + // Does this index support native absolute positioning? + + FLMBOOL isAbsolutePositionable() + { + return (m_pIndex->uiFlags & IXD_ABS_POS) ? TRUE : FALSE; + } + + RCODE getKeyData( + F_Btree * pBTree, + FLMUINT uiDataLen); + + RCODE setKeyPosition( + F_Db * pDb, + FLMBOOL bGoingForward, + FLMBOOL bExcludeKey, + F_DataVector * pExtSrchKey, + KEYPOS * pSearchKey, + KEYPOS * pFoundKey, + FLMBOOL bGetKeyData, + FLMUINT * puiDataLen, + F_Btree * pBTree, + FLMUINT * puiAbsolutePos); + + FINLINE void closeBTree( void) + { + if (m_bTreeOpen) + { + m_pbTree->btClose(); + m_bTreeOpen = FALSE; + m_pDb = NULL; + m_eTransType = SFLM_NO_TRANS; + } + } + + FINLINE RCODE checkTransaction( + F_Db * pDb) + { + RCODE rc = NE_SFLM_OK; + if (RC_OK( rc = pDb->flushKeys())) + { + rc = (RCODE)((m_ui64CurrTransId != pDb->m_ui64CurrTransID || + m_uiBlkChangeCnt != pDb->m_uiBlkChangeCnt) + ? resetTransaction( pDb) + : NE_SFLM_OK); + } + return( rc); + } + + RCODE populateKey( + F_DataVector * pKey); + + RCODE checkIfKeyInRange( + FLMBOOL bPositionForward); + + FINLINE void getCurrKey( + KEYPOS * pKey + ) + { + f_memcpy( pKey->ucKey, m_curKey.ucKey, m_curKey.uiKeyLen); + pKey->uiKeyLen = m_curKey.uiKeyLen; + } + + // Database information + + FLMUINT64 m_ui64CurrTransId; + FLMUINT m_uiBlkChangeCnt; + FLMUINT m_uiIndexNum; + LFILE * m_pLFile; + F_INDEX * m_pIndex; + F_Db * m_pDb; + eDbTransType m_eTransType; + + // Key range information + + FLMBOOL m_bSetup; + KEYPOS m_fromKey; + KEYPOS m_untilKey; + + // State information. + + FLMBOOL m_bAtBOF; // Before the first key. + FLMBOOL m_bAtEOF; // After the last key. + KEYPOS m_curKey; // Current key + FLMBYTE * m_pucCurKeyDataBuf; + FLMUINT m_uiCurKeyDataBufSize; + FLMUINT m_uiCurKeyDataLen; + F_Btree * m_pbTree; + FLMBOOL m_bTreeOpen; + IXKeyCompare m_ixCompare; + F_DataVector m_fromExtKey; + F_DataVector m_untilExtKey; + + friend class F_Query; +}; + +/*============================================================================ +Desc: File system implementation of a cursor for a collection. +============================================================================*/ +class FSTableCursor : public F_Object +{ +public: + + // Constructors & Destructor + + FSTableCursor(); + virtual ~FSTableCursor(); + + void resetCursor(); + + RCODE resetTransaction( + F_Db * pDb); + + RCODE setupRange( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64LowRowId, + FLMUINT64 ui64HighRowId, + FLMUINT64 * pui64LeafBlocksBetween, + FLMUINT64 * pui64TotalRows, + FLMBOOL * pbTotalsEstimated); + + RCODE currentRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + + RCODE firstRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + + RCODE lastRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + + RCODE nextRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + + RCODE prevRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + +private: + + RCODE setRowPosition( + F_Db * pDb, + FLMBOOL bGoingForward, + FLMUINT64 ui64RowId, + FLMBOOL bPopulateCurRowId, + F_Btree * pBTree); + + RCODE openBTree( + F_Db * pDb); + + FINLINE void closeBTree( void) + { + if (m_bTreeOpen) + { + m_pbTree->btClose(); + m_bTreeOpen = FALSE; + m_pDb = NULL; + m_eTransType = SFLM_NO_TRANS; + } + } + + FINLINE RCODE checkTransaction( + F_Db * pDb) + { + RCODE rc = NE_SFLM_OK; + + if (pDb->m_uiDirtyRowCount) + { + if (RC_BAD( rc = pDb->flushDirtyRows())) + { + goto Exit; + } + } + rc = (RCODE)((m_pDb != pDb || + m_ui64CurrTransId != pDb->m_ui64CurrTransID || + m_uiBlkChangeCnt != pDb->m_uiBlkChangeCnt) + ? resetTransaction( pDb) + : NE_SFLM_OK); + Exit: + return( rc); + } + + RCODE populateRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId); + + RCODE checkIfRowInRange( + FLMBOOL bPositionForward); + + // Database Information + + FLMUINT64 m_ui64CurrTransId; + FLMUINT m_uiBlkChangeCnt; + FLMUINT m_uiTableNum; + F_TABLE * m_pTable; + LFILE * m_pLFile; + F_Db * m_pDb; + eDbTransType m_eTransType; + + // Key range information + + FLMBOOL m_bSetup; + FLMUINT64 m_ui64FromRowId; + FLMUINT64 m_ui64UntilRowId; + + // State information. + + FLMBOOL m_bAtBOF; // Before the first row. + FLMBOOL m_bAtEOF; // After the last row. + FLMUINT64 m_ui64CurRowId; // Current row id + FLMBYTE m_ucCurRowKey [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT m_uiCurRowKeyLen; + F_Btree * m_pbTree; + FLMBOOL m_bTreeOpen; +}; + +RCODE flmBuildFromAndUntilKeys( + F_Dict * pDict, + F_INDEX * pIndex, + F_TABLE * pTable, + SQL_PRED * pPred, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbDoRowMatch, + FLMBOOL * pbCanCompareOnKey); + +#endif diff --git a/sql/src/fsdatacu.cpp b/sql/src/fsdatacu.cpp new file mode 100644 index 0000000..845efc7 --- /dev/null +++ b/sql/src/fsdatacu.cpp @@ -0,0 +1,783 @@ +//------------------------------------------------------------------------------ +// Desc: Cursor routines to get the complexity of the file system out +// of the search code. +// +// Tabs: 3 +// +// Copyright (c) 2000-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" +#include "fscursor.h" + +/**************************************************************************** +Desc: +****************************************************************************/ +FSTableCursor::FSTableCursor() +{ + m_pbTree = NULL; + m_bTreeOpen = FALSE; + m_pTable = NULL; + m_pLFile = NULL; + m_pDb = NULL; + m_eTransType = SFLM_NO_TRANS; + resetCursor(); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FSTableCursor::~FSTableCursor() +{ + closeBTree(); + if (m_pbTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &m_pbTree); + } +} + +/**************************************************************************** +Desc: Resets any allocations, keys, state, etc. +****************************************************************************/ +void FSTableCursor::resetCursor( void) +{ + closeBTree(); + m_uiTableNum = 0; + m_uiBlkChangeCnt = 0; + m_ui64CurrTransId = 0; + m_ui64CurRowId = 0; + m_bAtBOF = TRUE; + m_bAtEOF = FALSE; + m_bSetup = FALSE; +} + +/**************************************************************************** +Desc: Resets to a new transaction that may change the read consistency of + the query. +****************************************************************************/ +RCODE FSTableCursor::resetTransaction( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable; + + if ((pTable = pDb->m_pDict->getTable( m_uiTableNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_TABLE_NUM); + goto Exit; + } + if (pTable != m_pTable) + { + m_pTable = pTable; + m_pLFile = &pTable->lfInfo; + if (m_bTreeOpen) + { + closeBTree(); + } + m_pDb = pDb; + m_eTransType = pDb->m_eTransType; + } + m_ui64CurrTransId = pDb->m_ui64CurrTransID; + m_uiBlkChangeCnt = pDb->m_uiBlkChangeCnt; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Open the F_Btree object if not already open. +****************************************************************************/ +RCODE FSTableCursor::openBTree( + F_Db * pDb + ) +{ + RCODE rc = NE_SFLM_OK; + + if (!m_bTreeOpen) + { + if (!m_pbTree) + { + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &m_pbTree))) + { + goto Exit; + } + } +Open_Btree: + if (RC_BAD( rc = m_pbTree->btOpen( pDb, m_pLFile, FALSE, FALSE))) + { + goto Exit; + } + m_bTreeOpen = TRUE; + m_pDb = pDb; + m_eTransType = pDb->m_eTransType; + } + else + { + if (pDb != m_pDb || pDb->m_eTransType != m_eTransType) + { + closeBTree(); + goto Open_Btree; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set the row position. +****************************************************************************/ +RCODE FSTableCursor::setRowPosition( + F_Db * pDb, + FLMBOOL bGoingForward, + FLMUINT64 ui64RowId, + FLMBOOL bPopulateCurRowId, + F_Btree * pBTree // BTree to use. NULL means use our + // internal one. + ) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bNeg; + FLMBYTE ucRowKey [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiRowKeyLen; + FLMBYTE * pucRowKey; + FLMUINT * puiRowKeyLen; + FLMUINT uiKeyBufSize; + FLMUINT uiBytesProcessed; + FLMUINT64 ui64TmpRowId; + + // if pBTree is NULL, we are to use m_pbTree. Otherwise, we + // need to open the pBTree and use it. + + if (!pBTree) + { + if (RC_BAD( rc = openBTree( pDb))) + { + goto Exit; + } + pBTree = m_pbTree; + } + + if (bPopulateCurRowId) + { + pucRowKey = &m_ucCurRowKey [0]; + puiRowKeyLen = &m_uiCurRowKeyLen; + uiKeyBufSize = sizeof( m_ucCurRowKey); + } + else + { + pucRowKey = &ucRowKey [0]; + puiRowKeyLen = &uiRowKeyLen; + uiKeyBufSize = sizeof( ucRowKey); + } + + *puiRowKeyLen = uiKeyBufSize; + if (RC_BAD( rc = flmNumber64ToStorage( ui64RowId, puiRowKeyLen, + pucRowKey, FALSE, TRUE))) + { + goto Exit; + } + + if (RC_BAD( rc = pBTree->btLocateEntry( pucRowKey, uiKeyBufSize, + puiRowKeyLen, FLM_INCL))) + { + if (rc != NE_SFLM_EOF_HIT) + { + goto Exit; + } + } + + if (bGoingForward) + { + if (rc == NE_SFLM_EOF_HIT) + { + goto Exit; + } + } + else + { + + // Going backwards or to last. See if we positioned too far. + + if (rc == NE_SFLM_BOF_HIT || rc == NE_SFLM_EOF_HIT) + { + + // Position to last key in tree. + + if (RC_BAD( rc = pBTree->btLastEntry( pucRowKey, uiKeyBufSize, + puiRowKeyLen, + NULL, NULL, NULL))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = flmCollation2Number( *puiRowKeyLen, pucRowKey, + &ui64TmpRowId, &bNeg, &uiBytesProcessed))) + { + goto Exit; + } + if (ui64TmpRowId > ui64RowId) + { + + // Position to the previous key. + + if (RC_BAD( rc = pBTree->btPrevEntry( pucRowKey, uiKeyBufSize, + puiRowKeyLen, NULL, NULL, NULL))) + { + goto Exit; + } + } + } + } + + if (bPopulateCurRowId) + { + if (RC_BAD( rc = flmCollation2Number( *puiRowKeyLen, pucRowKey, + &m_ui64CurRowId, &bNeg, &uiBytesProcessed))) + { + goto Exit; + } + } + +Exit: + + if (RC_BAD( rc)) + { + if (pBTree == m_pbTree) + { + closeBTree(); + } + } + return( rc); +} + +/**************************************************************************** +Desc: Setup the from and until keys in the cursor. Return counts + after positioning to the from and until key in the table. +****************************************************************************/ +RCODE FSTableCursor::setupRange( + F_Db * pDb, + FLMUINT uiTableNum, + FLMUINT64 ui64LowRowId, + FLMUINT64 ui64HighRowId, + FLMUINT64 * pui64LeafBlocksBetween, // [out] blocks between the stacks + FLMUINT64 * pui64TotalRows, // [out] + FLMBOOL * pbTotalsEstimated) // [out] set to TRUE when estimating. +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pUntilBTree = NULL; + + m_bAtBOF = TRUE; + m_bAtEOF = FALSE; + m_uiTableNum = uiTableNum; + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + + m_ui64FromRowId = ui64LowRowId; + m_ui64UntilRowId = ui64HighRowId; + m_bSetup = TRUE; + m_ui64CurRowId = 0; + + // Want any of the counts back? + + if (pui64LeafBlocksBetween || pui64TotalRows) + { + if (pui64LeafBlocksBetween) + { + *pui64LeafBlocksBetween = 0; + } + if (pui64TotalRows) + { + *pui64TotalRows = 0; + } + if (pbTotalsEstimated) + { + *pbTotalsEstimated = FALSE; + } + + // Position to the FROM and UNTIL key so we can get the stats. + + if (RC_BAD( rc = setRowPosition( pDb, TRUE, + m_ui64FromRowId, TRUE, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + // All nodes between FROM and UNTIL may be gone. + + if (m_ui64CurRowId < m_ui64UntilRowId) + { + + // Get a btree object + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( + &pUntilBTree))) + { + goto Exit; + } + if (RC_BAD( rc = pUntilBTree->btOpen( pDb, m_pLFile, + FALSE, FALSE))) + { + goto Exit; + } + + // We better be able to at least find m_ui64CurRowId going + // backward from m_ui64UntilRowId. + + if (RC_BAD( rc = setRowPosition( pDb, FALSE, + m_ui64UntilRowId, FALSE, pUntilBTree))) + { + goto Exit; + } + if (RC_BAD( rc = m_pbTree->btComputeCounts( pUntilBTree, + pui64LeafBlocksBetween, pui64TotalRows, + pbTotalsEstimated, + (pDb->m_pDatabase->m_uiBlockSize * 3) / 4))) + { + goto Exit; + } + } + } + +Exit: + + if (pUntilBTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pUntilBTree); + } + + return( rc); +} + +/**************************************************************************** +Desc: Return the current row. +****************************************************************************/ +RCODE FSTableCursor::currentRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId + ) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + if (m_bAtBOF) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + if (m_bAtEOF) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + + flmAssert( m_ui64CurRowId); + + if (RC_BAD( rc = populateRow( pDb, ppRow, pui64RowId))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Fetch the current row. B-tree object is assumed to be positioned on + the row. +****************************************************************************/ +RCODE FSTableCursor::populateRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId) +{ + if (pui64RowId) + { + *pui64RowId = m_ui64CurRowId; + } + if (ppRow) + { + return( gv_SFlmSysData.pRowCacheMgr->retrieveRow( pDb, m_uiTableNum, + m_ui64CurRowId, ppRow)); + } + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: Make sure the current node is positioned in the range for the cursor. +****************************************************************************/ +RCODE FSTableCursor::checkIfRowInRange( + FLMBOOL bPositionForward) +{ + RCODE rc = NE_SFLM_OK; + + if (bPositionForward) + { + if (m_ui64CurRowId > m_ui64UntilRowId) + { + m_bAtEOF = TRUE; + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + } + else + { + if (m_ui64CurRowId < m_ui64FromRowId) + { + m_bAtBOF = TRUE; + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Position to and return the first row. +****************************************************************************/ +RCODE FSTableCursor::firstRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId) +{ + RCODE rc = NE_SFLM_OK; + + if( RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + + // If at BOF and we have a node, then we are positioned on the first + // node already - this would have happened if we had positioned to + // calculate a cost. Rather than do the positioning again, we simply + // set m_bAtBOF to FALSE. + + if (m_bAtBOF && m_ui64CurRowId) + { + m_bAtBOF = FALSE; + } + else + { + m_bAtBOF = m_bAtEOF = FALSE; + if (RC_BAD( rc = setRowPosition( pDb, TRUE, m_ui64FromRowId, + TRUE, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + } + + // Make sure the current row ID is within the FROM/UNTIL range. + + if (RC_BAD( rc = checkIfRowInRange( TRUE))) + { + goto Exit; + } + if (RC_BAD( rc = populateRow( pDb, ppRow, pui64RowId))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc)) + { + m_ui64CurRowId = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to the next row. +****************************************************************************/ +RCODE FSTableCursor::nextRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + if (m_bAtEOF) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + goto Exit; + } + if (m_bAtBOF || !m_ui64CurRowId) + { + rc = firstRow( pDb, ppRow, pui64RowId); + goto Exit; + } + + // Get the next row, if any + + if (m_ui64CurRowId == FLM_MAX_UINT64) + { + rc = RC_SET( NE_SFLM_EOF_HIT); + m_bAtEOF = TRUE; + goto Exit; + } + + // See if we need to reset the b-tree object we are using + + if (m_bTreeOpen && + (pDb != m_pDb || pDb->m_eTransType != m_eTransType)) + { + closeBTree(); + } + + // checkTransaction may have closed the B-tree, in which case we + // need to reopen the b-tree and position to the next row. + + if (!m_bTreeOpen) + { + if (RC_BAD( rc = setRowPosition( pDb, TRUE, + m_ui64CurRowId + 1, TRUE, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + } + else + { + FLMBOOL bNeg; + FLMUINT uiBytesProcessed; + + // If we have a b-tree open, it is more efficient to call btNextEntry + // directly. This may allow us to avoid repositioning down the b-tree. + + if (RC_BAD( rc = m_pbTree->btNextEntry( m_ucCurRowKey, + sizeof( m_ucCurRowKey), + &m_uiCurRowKeyLen, NULL, NULL, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + if (RC_BAD( rc = flmCollation2Number( m_uiCurRowKeyLen, m_ucCurRowKey, + &m_ui64CurRowId, &bNeg, &uiBytesProcessed))) + { + goto Exit; + } + } + + // Make sure the current row ID is within the FROM/UNTIL range. + + if (RC_BAD( rc = checkIfRowInRange( TRUE))) + { + goto Exit; + } + if (RC_BAD( rc = populateRow( pDb, ppRow, pui64RowId))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc)) + { + m_ui64CurRowId = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to and return the last row. +****************************************************************************/ +RCODE FSTableCursor::lastRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId) +{ + RCODE rc = NE_SFLM_OK; + + if( RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + + // Position to the until node + + m_bAtBOF = m_bAtEOF = FALSE; + if (RC_BAD( rc = setRowPosition( pDb, FALSE, m_ui64UntilRowId, + TRUE, NULL))) + { + if (rc == NE_SFLM_BOF_HIT) + { + m_bAtBOF = TRUE; + } + goto Exit; + } + + // Make sure the current row ID is within the FROM/UNTIL range. + + if (RC_BAD( rc = checkIfRowInRange( FALSE))) + { + goto Exit; + } + if (RC_BAD( rc = populateRow( pDb, ppRow, pui64RowId))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc)) + { + m_ui64CurRowId = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: Position to the previous row. +****************************************************************************/ +RCODE FSTableCursor::prevRow( + F_Db * pDb, + F_Row ** ppRow, + FLMUINT64 * pui64RowId) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = checkTransaction( pDb))) + { + goto Exit; + } + flmAssert( m_bSetup); + if (m_bAtBOF) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + goto Exit; + } + if (m_bAtEOF || !m_ui64CurRowId) + { + rc = lastRow( pDb, ppRow, pui64RowId); + goto Exit; + } + + // Get the previous row, if any + + if (m_ui64CurRowId == 1) + { + rc = RC_SET( NE_SFLM_BOF_HIT); + m_bAtBOF = TRUE; + goto Exit; + } + + // See if we need to reset the b-tree object we are using + + if (m_bTreeOpen && + (pDb != m_pDb || pDb->m_eTransType != m_eTransType)) + { + closeBTree(); + } + + // checkTransaction may have closed the B-tree, in which case we + // need to reopen the b-tree and position to the previous row. + + if (!m_bTreeOpen) + { + if (RC_BAD( rc = setRowPosition( pDb, TRUE, + m_ui64CurRowId - 1, TRUE, NULL))) + { + if (rc == NE_SFLM_BOF_HIT) + { + m_bAtBOF = TRUE; + } + goto Exit; + } + } + else + { + FLMBOOL bNeg; + FLMUINT uiBytesProcessed; + + // If we have a b-tree open, it is more efficient to call btPrevEntry + // directly. This may allow us to avoid repositioning down the b-tree. + + if (RC_BAD( rc = m_pbTree->btPrevEntry( m_ucCurRowKey, + sizeof( m_ucCurRowKey), + &m_uiCurRowKeyLen, NULL, NULL, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + m_bAtEOF = TRUE; + } + goto Exit; + } + if (RC_BAD( rc = flmCollation2Number( m_uiCurRowKeyLen, m_ucCurRowKey, + &m_ui64CurRowId, &bNeg, &uiBytesProcessed))) + { + goto Exit; + } + } + + // Make sure the current row ID is within the FROM/UNTIL range. + + if (RC_BAD( rc = checkIfRowInRange( FALSE))) + { + goto Exit; + } + if (RC_BAD( rc = populateRow( pDb, ppRow, pui64RowId))) + { + goto Exit; + } + +Exit: + + if (RC_BAD( rc)) + { + m_ui64CurRowId = 0; + } + return( rc); +} + diff --git a/sql/src/fslfile.cpp b/sql/src/fslfile.cpp new file mode 100644 index 0000000..e6a6977 --- /dev/null +++ b/sql/src/fslfile.cpp @@ -0,0 +1,503 @@ +//------------------------------------------------------------------------------ +// Desc: Routines for reading and writing logical file headers. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fslfile.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC FLMUINT FSLFileFindEmpty( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr); + +/*************************************************************************** +Desc: Searches a block for an empty LFH slot. This is called whenever + a new logical file is create so we re-use the slots. +Ret: 0-Empty slot not found + non-zero - offset in the block of the empty slot +***************************************************************************/ +FSTATIC FLMUINT FSLFileFindEmpty( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr) +{ + FLMUINT uiPos = SIZEOF_STD_BLK_HDR; + FLMUINT uiEndPos = blkGetEnd( uiBlockSize, SIZEOF_STD_BLK_HDR, + pBlkHdr); + F_LF_HDR * pLfHdr = (F_LF_HDR *)((FLMBYTE *)pBlkHdr + SIZEOF_STD_BLK_HDR); + + while (uiPos < uiEndPos) + { + if (pLfHdr->ui32LfType == (FLMUINT32)SFLM_LF_INVALID) + { + break; + } + + uiPos += sizeof( F_LF_HDR); + pLfHdr++; + } + return( (uiPos < uiEndPos) ? uiPos : 0); +} + +/*************************************************************************** +Desc: Retrieves the logical file header record from disk & updates LFILE. + Called when it is discovered that the LFH for a + particular logical file is out of date. +*****************************************************************************/ +RCODE F_Database::lFileRead( + F_Db * pDb, + LFILE * pLFile) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache; + FLMBOOL bReleaseCache = FALSE; + + // Read in the block containing the logical file header + + if (RC_BAD( rc = getBlock( pDb, NULL, + pLFile->uiBlkAddress, NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + // Copy the LFH from the block to the LFILE + + FSLFileIn( (FLMBYTE *)(pSCache->m_pBlkHdr) + pLFile->uiOffsetInBlk, + pLFile, pLFile->uiBlkAddress, pLFile->uiOffsetInBlk); + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + } + + return( rc); +} + +/*************************************************************************** +Desc: Update the LFH data on disk. +*****************************************************************************/ +RCODE F_Database::lFileWrite( + F_Db * pDb, + LFILE * pLFile) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache; + FLMBOOL bReleaseCache = FALSE; + F_LF_HDR * pLfHdr; +#ifdef DEBUG + F_CachedBlock * pTmpSCache = NULL; +#endif + + flmAssert( !pDb->m_pDatabase->m_pRfl || + !pDb->m_pDatabase->m_pRfl->isLoggingEnabled()); + + // Read in the block containing the logical file header + + if (RC_BAD( rc = getBlock( pDb, NULL, + pLFile->uiBlkAddress, NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + // Log the block before modifying it + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + + // Now modify the block and set its status to dirty + + pLfHdr = (F_LF_HDR *)((FLMBYTE *)(pSCache->m_pBlkHdr) + + pLFile->uiOffsetInBlk); + + // If deleted, fill with 0, except for type - it is set below + + if (pLFile->eLfType == SFLM_LF_INVALID) + { + f_memset( pLfHdr, 0, sizeof( F_LF_HDR)); + pLfHdr->ui32LfType = (FLMUINT32)SFLM_LF_INVALID; + } + else + { + +#ifdef DEBUG + if (RC_BAD( rc = getBlock( pDb, NULL, + pLFile->uiRootBlk, NULL, &pTmpSCache))) + { + goto Exit; + } + + if (!isRootBlk( (F_BTREE_BLK_HDR *)pTmpSCache->m_pBlkHdr)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + } +#endif + + pLfHdr->ui32LfType = (FLMUINT32)pLFile->eLfType; + pLfHdr->ui32RootBlkAddr = (FLMUINT32)pLFile->uiRootBlk; + pLfHdr->ui32LfNum = (FLMUINT32)pLFile->uiLfNum; + pLfHdr->ui32EncDefNum = (FLMUINT32)pLFile->uiEncDefNum; + + if (pLFile->eLfType == SFLM_LF_TABLE) + { + pLfHdr->ui64NextRowId = pLFile->ui64NextRowId; + } + else + { + flmAssert( pLFile->eLfType == SFLM_LF_INDEX); + pLfHdr->ui64NextRowId = 0; + } + } + pLFile->bNeedToWriteOut = FALSE; + + // If the LFILE was deleted, we need to set pLFile->uiLfNum to zero. + // This should happen only AFTER the LFILE update is logged, because + // logLFileUpdate relies on pLFile->uiLfNum being non-zero. + + if (pLFile->eLfType == SFLM_LF_INVALID) + { + pLFile->uiLfNum = 0; + } + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + } + +#ifdef DEBUG + if (pTmpSCache) + { + ScaReleaseCache( pTmpSCache, FALSE); + } +#endif + + return( rc); +} + +/*************************************************************************** +Desc: Creates and initializes a LFILE structure on disk and in memory. +*****************************************************************************/ +RCODE F_Database::lFileCreate( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiLfNum, + eLFileType eLfType, + FLMBOOL bCounts, + FLMBOOL bHaveData, + FLMUINT uiEncDefNum) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pNewSCache = NULL; + F_CachedBlock * pSCache = NULL; + F_BLK_HDR * pBlkHdr = NULL; + FLMUINT uiBlkAddress = 0; + FLMUINT uiNextBlkAddress; + FLMUINT uiEndPos = 0; + FLMUINT uiPos = 0; + FLMBOOL bReleaseCache2 = FALSE; + FLMBOOL bReleaseCache = FALSE; + F_Btree * pbTree = NULL; + + flmAssert( !pDb->m_pDatabase->m_pRfl || + !pDb->m_pDatabase->m_pRfl->isLoggingEnabled()); + + if (eLfType == SFLM_LF_TABLE) + { + if( bCounts) + { + // Force bCounts to be FALSE in this case + flmAssert( 0); + bCounts = FALSE; + } + + if( !bHaveData) + { + // Force bHaveData to be TRUE in this case + flmAssert( 0); + bHaveData = TRUE; + } + } + + // Find an available slot to create the LFH -- follow the linked list + // of LFH blocks to find one. + + uiNextBlkAddress = (FLMUINT)m_uncommittedDbHdr.ui32FirstLFBlkAddr; + + // Better be at least one LFH block. + + if (uiNextBlkAddress == 0) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + while (uiNextBlkAddress != 0) + { + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + bReleaseCache = FALSE; + } + uiBlkAddress = uiNextBlkAddress; + + if (RC_BAD( rc = getBlock( pDb, NULL, + uiBlkAddress, NULL, &pSCache))) + { + goto Exit; + } + bReleaseCache = TRUE; + + pBlkHdr = pSCache->m_pBlkHdr; + uiNextBlkAddress = (FLMUINT)pBlkHdr->ui32NextBlkInChain; + uiEndPos = blkGetEnd( m_uiBlockSize, SIZEOF_STD_BLK_HDR, pBlkHdr); + if ((uiPos = FSLFileFindEmpty( m_uiBlockSize, pBlkHdr)) != 0) + { + break; + } + } + + // If we did not find a deleted slot we can use, see if there + // is room for a new logical file in the last block + // in the chain. If not, allocate a new block. + + if (uiPos == 0) + { + uiEndPos = blkGetEnd( m_uiBlockSize, SIZEOF_STD_BLK_HDR, pBlkHdr); + + // Allocate new block? + + if (uiEndPos + sizeof( F_LF_HDR) >= m_uiBlockSize) + { + if (RC_BAD( rc = createBlock( pDb, &pNewSCache))) + { + goto Exit; + } + bReleaseCache2 = TRUE; + + pBlkHdr = pNewSCache->m_pBlkHdr; + uiNextBlkAddress = (FLMUINT)pBlkHdr->ui32BlkAddr; + + // Modify the new block's next pointer and other fields. + + pBlkHdr->ui32NextBlkInChain = 0; + pBlkHdr->ui32PrevBlkInChain = (FLMUINT32)uiBlkAddress; + pBlkHdr->ui8BlkType = BT_LFH_BLK; + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)(m_uiBlockSize - + SIZEOF_STD_BLK_HDR); + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + pSCache->m_pBlkHdr->ui32NextBlkInChain = (FLMUINT32)uiNextBlkAddress; + + // Set everything up so we are pointing to the new block. + + ScaReleaseCache( pSCache, FALSE); + pSCache = pNewSCache; + bReleaseCache2 = FALSE; + uiEndPos = blkGetEnd( m_uiBlockSize, SIZEOF_STD_BLK_HDR, + pSCache->m_pBlkHdr); + uiBlkAddress = uiNextBlkAddress; + } + + // Modify the end of block pointer -- log block before modifying. + + uiPos = uiEndPos; + uiEndPos += sizeof( F_LF_HDR); + } + + // Call memset to ensure unused bytes are zero. + // pBlkHdr, uiPos and uiEndPos should ALL be set. + + if (RC_BAD( rc = logPhysBlk( pDb, &pSCache))) + { + goto Exit; + } + pBlkHdr = pSCache->m_pBlkHdr; + f_memset( (FLMBYTE *)(pBlkHdr) + uiPos, 0, sizeof( F_LF_HDR)); + flmAssert( uiEndPos >= SIZEOF_STD_BLK_HDR && + uiEndPos <= m_uiBlockSize); + pBlkHdr->ui16BlkBytesAvail = (FLMUINT16)(m_uiBlockSize - uiEndPos); + + // Done with block in this routine + + ScaReleaseCache( pSCache, FALSE); + bReleaseCache = FALSE; + + // Set the variables in the LFILE structure to later save to disk + + pLFile->uiLfNum = uiLfNum; + pLFile->eLfType = eLfType; + + // NOTE: call to btCreate below sets pLFile->uiRootBlk + + // pLFile->uiRootBlk = 0; + pLFile->uiBlkAddress = uiBlkAddress; + pLFile->uiOffsetInBlk = uiPos; + pLFile->uiEncDefNum = uiEncDefNum; + if (eLfType == SFLM_LF_TABLE) + { + pLFile->ui64NextRowId = 1; + pLFile->bNeedToWriteOut = TRUE; + } + else + { + pLFile->ui64NextRowId = 0; + } + + // Get the btree... + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbTree))) + { + goto Exit; + } + + if (RC_BAD( rc = pbTree->btCreate( pDb, pLFile, bCounts, bHaveData))) + { + goto Exit; + } + + if (RC_BAD( rc = lFileWrite( pDb, pLFile))) + { + goto Exit; + } + +Exit: + + if (bReleaseCache) + { + ScaReleaseCache( pSCache, FALSE); + } + if (bReleaseCache2) + { + ScaReleaseCache( pNewSCache, FALSE); + } + if (pbTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbTree); + } + return( rc); +} + +/*************************************************************************** +Desc: Delete a logical file. +*****************************************************************************/ +RCODE F_Database::lFileDelete( + F_Db * pDb, + LFILE * pLFile, + FLMBOOL bCounts, + FLMBOOL bHaveData) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pbTree = NULL; + FLMUINT uiLoop; + FLMUINT uiBlkChains[ BH_MAX_LEVELS]; + FLMUINT uiChainCount; + F_Row * pRow = NULL; + + flmAssert( pDb->m_uiFlags & FDB_UPDATED_DICTIONARY); + + // Get a btree + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbTree))) + { + goto Exit; + } + + // Delete the logical file's B-Tree blocks. + // If there is no root block, no need to do anything. + + flmAssert( pLFile->uiRootBlk); + + if (RC_BAD( rc = pbTree->btOpen( pDb, pLFile, bCounts, bHaveData))) + { + goto Exit; + } + + if (RC_BAD( rc = pbTree->btGetBlockChains( uiBlkChains, &uiChainCount))) + { + goto Exit; + } + + // Indexes and tables are always deleted in the background. + // Add a row to the block chain table for each chain found. + + for (uiLoop = 0; uiLoop < uiChainCount; uiLoop++) + { + if (pRow) + { + pRow->ReleaseRow(); + pRow = NULL; + } + + // Create the row + + if (RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->createRow( pDb, + SFLM_TBLNUM_BLOCK_CHAINS, &pRow))) + { + goto Exit; + } + + // Set the block address column in the row. + + if (RC_BAD( rc = pRow->setUINT( pDb, + SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS, + uiBlkChains [uiLoop]))) + { + goto Exit; + } + } + + // Signal the maintenance thread that it has work to do + + f_semSignal( m_hMaintSem); + + // Delete the LFILE entry. + + pLFile->uiRootBlk = 0; + pLFile->eLfType = SFLM_LF_INVALID; + + if( RC_BAD( rc = lFileWrite( pDb, pLFile))) + { + goto Exit; + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + if( pbTree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbTree); + } + + return( rc); +} + diff --git a/sql/src/fslfileu.cpp b/sql/src/fslfileu.cpp new file mode 100644 index 0000000..579dbcc --- /dev/null +++ b/sql/src/fslfileu.cpp @@ -0,0 +1,1118 @@ +//------------------------------------------------------------------------------ +// Desc: Routines to perform dictionary updates. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fslfileu.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Copies an existing dictionary to a new dictionary. +****************************************************************************/ +RCODE F_Db::dictClone( void) +{ + RCODE rc = NE_SFLM_OK; + F_Dict * pNewDict = NULL; + + // Allocate a new FDICT structure + + if ((pNewDict = f_new F_Dict) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Nothing to do is not a legal state. + + if (!m_pDict) + { + flmAssert( 0); + m_pDict = pNewDict; + goto Exit; + } + + // Copy the dictionary. + + if (RC_BAD( rc = pNewDict->cloneDict( m_pDict))) + { + goto Exit; + } + + m_pDatabase->lockMutex(); + unlinkFromDict(); + m_pDatabase->unlockMutex(); + m_pDict = pNewDict; + pNewDict = NULL; + m_uiFlags |= FDB_UPDATED_DICTIONARY; + +Exit: + + if (RC_BAD( rc) && pNewDict) + { + pNewDict->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Build an index. +****************************************************************************/ +RCODE F_Db::buildIndex( + FLMUINT uiIndexNum, + FLMUINT uiState) +{ + RCODE rc = NE_SFLM_OK; + LFILE * pIxLFile; + F_INDEX * pIndex; + + // Flush any KY keys and free the tables because they may grow! + + if (RC_BAD( rc = keysCommit( TRUE))) + { + goto Exit; + } + + if (RC_BAD( rc = krefCntrlCheck())) + { + goto Exit; + } + + pIndex = m_pDict->getIndex( uiIndexNum); + flmAssert( pIndex); + pIxLFile = &pIndex->lfInfo; + + // NON-BLOCKING INDEX BUILD - NOTE: The IXD_SUSPENDED flag may + // also be set, which indicates that we should NOT start the + // background maintenance thread right now. + + if (uiState & IXD_OFFLINE) + { + if (RC_BAD( rc = setIxStateInfo( uiIndexNum, 0, uiState))) + { + goto Exit; + } + + // setIxStateInfo may have changed to a new dictionary, so pIndex is no + // good after this point + + pIndex = NULL; + + // Don't schedule a maintenance thread if index is to start + // out life in a suspended state, or if we are replaying + // the roll-forward log. + + if (!(uiState & IXD_SUSPENDED) && !(m_uiFlags & FDB_REPLAYING_RFL)) + { + if (RC_BAD( rc = addToStartList( uiIndexNum))) + { + goto Exit; + } + } + + // Done + + goto Exit; + } + + // There may be "new" rows in the row cache. + // Need to flush them to the database so that + // the B-Tree lookups done by the indexing code will + // work correctly + + if( RC_BAD( rc = flushDirtyRows())) + { + goto Exit; + } + + // NORMAL INDEX BUILD - BLOCKING. uiIndexToBeUpdated better be + // zero at this point since we are not working in the background. + + if (RC_BAD( rc = indexSetOfRows( uiIndexNum, 1, + FLM_MAX_UINT64, m_pIxStatus, m_pIxClient, NULL, NULL, NULL))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Logs information about an index being built +****************************************************************************/ +void flmLogIndexingProgress( + FLMUINT uiIndexNum, + FLMUINT64 ui64LastRowId) +{ + IF_LogMessageClient * pLogMsg = NULL; + char szMsg[ 128]; + + if( (pLogMsg = flmBeginLogMessage( SFLM_GENERAL_MESSAGE)) != NULL) + { + pLogMsg->changeColor( FLM_YELLOW, FLM_BLACK); + if (ui64LastRowId) + { + f_sprintf( (char *)szMsg, + "Indexing progress: Index %u is offline. Last row processed = %I64u.", + (unsigned)uiIndexNum, ui64LastRowId); + } + else + { + f_sprintf( (char *)szMsg, + "Indexing progress: Index %u is online.", + (unsigned)uiIndexNum); + } + pLogMsg->appendString( szMsg); + } + flmEndLogMessage( &pLogMsg); +} + +/**************************************************************************** +Desc: Index a set of documents or until time runs out. +****************************************************************************/ +RCODE F_Db::indexSetOfRows( + FLMUINT uiIndexNum, + FLMUINT64 ui64StartRowId, + FLMUINT64 ui64EndRowId, + IF_IxStatus * pIxStatus, + IF_IxClient * pIxClient, + SFLM_INDEX_STATUS * pIndexStatus, + FLMBOOL * pbHitEnd, + IF_Thread * pThread) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64RowId; + FLMUINT64 ui64LastRowId = 0; + F_INDEX * pIndex = NULL; + F_TABLE * pTable; + ServerLockObject * + pDatabaseLockObj = m_pDatabase->m_pDatabaseLockObj; + FLMBOOL bHitEnd = FALSE; + FLMUINT uiCurrTime; + FLMUINT uiLastStatusTime = 0; + FLMUINT uiStartTime; + FLMUINT uiMinTU; + FLMUINT uiStatusIntervalTU; + FLMUINT64 ui64RowsProcessed = 0; + FLMBOOL bRelinquish = FALSE; + FLMBYTE ucKey[ FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiKeyLen; + void * pvTmpPoolMark = m_tempPool.poolMark(); + F_Btree * pbtree = NULL; + FLMBOOL bNeg; + FLMUINT uiBytesProcessed; + F_Row * pRow = NULL; + + uiMinTU = FLM_MILLI_TO_TIMER_UNITS( 500); + uiStatusIntervalTU = FLM_SECS_TO_TIMER_UNITS( 10); + uiStartTime = FLM_GET_TIMER(); + + if (RC_BAD( rc = krefCntrlCheck())) + { + goto Exit; + } + + pIndex = m_pDict->getIndex( uiIndexNum); + flmAssert( pIndex); + + flmAssert( !(pIndex->uiFlags & IXD_SUSPENDED)); + + // Get a btree + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbtree))) + { + goto Exit; + } + + pTable = m_pDict->getTable( pIndex->uiTableNum); + flmAssert( pTable); + + if (RC_BAD( rc = pbtree->btOpen( this, &pTable->lfInfo, + FALSE, TRUE))) + { + goto Exit; + } + + uiKeyLen = sizeof( ucKey); + if (RC_BAD( rc = flmNumber64ToStorage( ui64StartRowId, &uiKeyLen, + ucKey, FALSE, TRUE))) + { + goto Exit; + } + if( RC_BAD( rc = pbtree->btLocateEntry( + ucKey, sizeof( ucKey), &uiKeyLen, FLM_INCL))) + { + if (rc == NE_SFLM_EOF_HIT || rc == NE_SFLM_NOT_FOUND) + { + rc = NE_SFLM_OK; + bHitEnd = TRUE; + goto Commit_Keys; + } + + goto Exit; + } + + for (;;) + { + + // See what row we're on + + if (RC_BAD( rc = flmCollation2Number( uiKeyLen, ucKey, + &ui64RowId, &bNeg, &uiBytesProcessed))) + { + goto Exit; + } + + if (ui64RowId > ui64EndRowId) + { + break; + } + + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( this, + pIndex->uiTableNum, ui64RowId, &pRow))) + { + goto Exit; + } + + if (RC_BAD( rc = buildKeys( pIndex, pTable, NULL, pRow))) + { + goto Exit; + } + + // See if there is an indexing callback + + if (pIxClient) + { + if (RC_BAD( rc = pIxClient->doIndexing( this, uiIndexNum, + pIndex->uiTableNum, pRow))) + { + goto Exit; + } + } + + ui64LastRowId = ui64RowId; + ui64RowsProcessed++; + + if (pIndexStatus) + { + pIndexStatus->ui64RowsProcessed++; + pIndexStatus->ui64LastRowIndexed = ui64LastRowId; + } + + // Get the current time + + uiCurrTime = FLM_GET_TIMER(); + + // Break out if someone is waiting for an update transaction. + + if (pThread) + { + if (pThread->getShutdownFlag()) + { + bRelinquish = TRUE; + break; + } + + if (pDatabaseLockObj->ThreadWaitingLock()) + { + // See if our minimum run time has elapsed + + if (FLM_ELAPSED_TIME( uiCurrTime, uiStartTime) >= uiMinTU) + { + if (ui64RowsProcessed < 50) + { + // If there are higher priority waiters in the lock queue, + // we want to relinquish. + + if (pDatabaseLockObj->haveHigherPriorityWaiter( + FLM_BACKGROUND_LOCK_PRIORITY)) + { + bRelinquish = TRUE; + break; + } + } + else + { + bRelinquish = TRUE; + break; + } + } + } + else + { + + // Even if no one has requested a lock for a long time, we + // still want to periodically commit our transaction so + // we won't lose more than uiMaxCPInterval timer units worth + // of work if we crash. We will run until we exceed the checkpoint + // interval and we see that someone (the checkpoint thread) is + // waiting for the write lock. + + if (FLM_ELAPSED_TIME( uiCurrTime, uiStartTime) > + gv_SFlmSysData.uiMaxCPInterval && + m_pDatabase->m_pWriteLockObj->ThreadWaitingLock()) + { + bRelinquish = TRUE; + break; + } + } + } + + if (FLM_ELAPSED_TIME( uiCurrTime, uiLastStatusTime) >= + uiStatusIntervalTU) + { + uiLastStatusTime = uiCurrTime; + if( pIxStatus) + { + if( RC_BAD( rc = pIxStatus->reportIndex( ui64LastRowId))) + { + goto Exit; + } + } + + // Send indexing completed event notification + + if( gv_SFlmSysData.EventHdrs[ SFLM_EVENT_UPDATES].pEventCBList) + { + flmDoEventCallback( SFLM_EVENT_UPDATES, + SFLM_EVENT_INDEXING_PROGRESS, this, f_threadId(), + 0, uiIndexNum, ui64LastRowId, + NE_SFLM_OK); + } + + // Log a progress message + + flmLogIndexingProgress( uiIndexNum, ui64LastRowId); + } + + // Need to go to the next row. + + if( RC_BAD( rc = pbtree->btNextEntry( + ucKey, sizeof( ucKey), &uiKeyLen))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + bHitEnd = TRUE; + break; + } + goto Exit; + } + } + +Commit_Keys: + + if (RC_BAD( rc = keysCommit( TRUE))) + { + goto Exit; + } + + // If at the end, change index state. + + if (bHitEnd) + { + if (RC_BAD( rc = setIxStateInfo( uiIndexNum, 0, 0))) + { + goto Exit; + } + + // setIxStateInfo may have changed to a new dictionary, so pIxd is no + // good after this point + + pIndex = NULL; + } + else if (ui64RowsProcessed) + { + if (RC_BAD( rc = setIxStateInfo( uiIndexNum, ui64LastRowId, + IXD_OFFLINE))) + { + goto Exit; + } + + // setIxStateInfo may have changed to a new dictionary, so pIndex is no + // good after this point + + pIndex = NULL; + } + +Exit: + + // We want to make one last call if we are in the foreground or if + // we actually did some indexing. + + if (gv_SFlmSysData.EventHdrs[ SFLM_EVENT_UPDATES].pEventCBList) + { + flmDoEventCallback( SFLM_EVENT_UPDATES, + SFLM_EVENT_INDEXING_PROGRESS, this, f_threadId(), + 0, uiIndexNum, + (FLMUINT64)(bHitEnd ? (FLMUINT64)0 : ui64LastRowId), + NE_SFLM_OK); + } + + flmLogIndexingProgress( uiIndexNum, + (FLMUINT64)(bHitEnd ? (FLMUINT64)0 : ui64LastRowId)); + + if (pIxStatus) + { + (void) pIxStatus->reportIndex( ui64LastRowId); + } + + if (pbHitEnd) + { + *pbHitEnd = bHitEnd; + } + + krefCntrlFree(); + m_tempPool.poolReset( pvTmpPoolMark); + + if (pbtree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbtree); + } + + if (pRow) + { + pRow->ReleaseRow(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Set information in the index definition row. +****************************************************************************/ +RCODE F_Db::setIxStateInfo( + FLMUINT uiIndexNum, + FLMUINT64 ui64LastRowIndexed, + FLMUINT uiState) +{ + RCODE rc = NE_SFLM_OK; + IXD_FIXUP * pIxdFixup; + F_INDEX * pIndex; + FLMBOOL bMustAbortOnError = FALSE; + F_Row * pRow = NULL; + + pIndex = m_pDict->getIndex( uiIndexNum); + flmAssert( pIndex); + + // See if this index is in our fixup list. + + pIxdFixup = m_pIxdFixups; + while (pIxdFixup && pIxdFixup->uiIndexNum != uiIndexNum) + { + pIxdFixup = pIxdFixup->pNext; + } + + if (!pIxdFixup) + { + if (RC_BAD( rc = f_calloc( (FLMUINT)sizeof( IXD_FIXUP), &pIxdFixup))) + { + goto Exit; + } + pIxdFixup->pNext = m_pIxdFixups; + m_pIxdFixups = pIxdFixup; + pIxdFixup->uiIndexNum = uiIndexNum; + pIxdFixup->ui64LastRowIndexed = pIndex->ui64LastRowIndexed; + } + + bMustAbortOnError = TRUE; + + // Update the last row indexed, if it changed. + + if (pIxdFixup->ui64LastRowIndexed != ui64LastRowIndexed) + { + pIxdFixup->ui64LastRowIndexed = ui64LastRowIndexed; + + // First, retrieve the index definition row. + + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( this, + SFLM_TBLNUM_INDEXES, pIndex->ui64DefRowId, &pRow))) + { + goto Exit; + } + + if (ui64LastRowIndexed) + { + if (RC_BAD( rc = pRow->setUINT64( this, SFLM_COLNUM_INDEXES_LAST_ROW_INDEXED, + ui64LastRowIndexed))) + { + goto Exit; + } + } + else + { + pRow->setToNull( this, SFLM_COLNUM_INDEXES_LAST_ROW_INDEXED); + } + } + + // If IXD_SUSPENDED is set, then IXD_OFFLINE must also be set. + // There are places in the code that only check for IXD_OFFLINE + // that don't care if the index is also suspended. + + if (uiState & IXD_SUSPENDED) + { + uiState = IXD_SUSPENDED | IXD_OFFLINE; + } + else if (uiState & IXD_OFFLINE) + { + uiState = IXD_OFFLINE; + } + else + { + uiState = 0; + } + + // See if we need to change state. + + if ((pIndex->uiFlags & (IXD_SUSPENDED | IXD_OFFLINE)) != uiState) + { + const char * pszStateStr; + FLMUINT uiStateStrLen; + + if (uiState & IXD_SUSPENDED) + { + pszStateStr = SFLM_INDEX_SUSPENDED_STR; + } + else if (uiState & IXD_OFFLINE) + { + pszStateStr = SFLM_INDEX_OFFLINE_STR; + } + else + { + pszStateStr = SFLM_INDEX_ONLINE_STR; + } + + // At this point we know we need to change the state. That means we need + // to create a new dictionary, if we have not already done so. + + if (!(m_uiFlags & FDB_UPDATED_DICTIONARY)) + { + if (RC_BAD( rc = dictClone())) + { + goto Exit; + } + + // Get a pointer to the new F_INDEX + + pIndex = m_pDict->getIndex( uiIndexNum); + } + + // Retrieve the index definition row if it was not fetched above. + + if (!pRow) + { + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( this, + SFLM_TBLNUM_INDEXES, pIndex->ui64DefRowId, &pRow))) + { + goto Exit; + } + } + uiStateStrLen = f_strlen( pszStateStr); + if (RC_BAD( rc = pRow->setUTF8( this, SFLM_COLNUM_INDEXES_INDEX_STATE, + pszStateStr, uiStateStrLen, uiStateStrLen))) + { + goto Exit; + } + + // Put the state into the F_INDEX. + + pIndex->uiFlags = (pIndex->uiFlags & (~(IXD_SUSPENDED | IXD_OFFLINE))) | + uiState; + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + if( RC_BAD( rc) && bMustAbortOnError) + { + setMustAbortTrans( rc); + } + + return( rc); +} + +/**************************************************************************** +Desc: See if any F_INDEX structures need indexing in the background. +****************************************************************************/ +RCODE F_Db::startBackgroundIndexing( void) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bStartedTrans = FALSE; + FLMUINT uiIndexNum; + F_INDEX * pIndex; + + if (RC_BAD( rc = checkState( __FILE__, __LINE__))) + { + goto Exit; + } + + if (m_eTransType != SFLM_NO_TRANS) + { + if (!okToCommitTrans()) + { + rc = RC_SET( NE_SFLM_ABORT_TRANS); + goto Exit; + } + } + else + { + + // Need to have at least a read transaction going. + + if (RC_BAD( rc = beginTrans( SFLM_READ_TRANS))) + { + goto Exit; + } + bStartedTrans = TRUE; + } + + for (uiIndexNum = 1, pIndex = m_pDict->m_pIndexTbl; + uiIndexNum <= m_pDict->m_uiHighestIndexNum; + uiIndexNum++, pIndex++) + { + + // Restart any indexes that are off-line but not suspended + + if ((pIndex->uiFlags & (IXD_OFFLINE | IXD_SUSPENDED)) == IXD_OFFLINE) + { + flmAssert( flmBackgroundIndexGet( m_pDatabase, + uiIndexNum, FALSE) == NULL); + + if (RC_BAD( rc = startIndexBuild( uiIndexNum))) + { + goto Exit; + } + } + } + +Exit: + + if (bStartedTrans) + { + (void)abortTrans(); + } + + return( rc); +} + +/*************************************************************************** +Desc: +*****************************************************************************/ +RCODE F_Database::startMaintThread( void) +{ + RCODE rc = NE_SFLM_OK; + char szThreadName[ F_PATH_MAX_SIZE]; + char szBaseName[ 32]; + + flmAssert( !m_pMaintThrd); + flmAssert( m_hMaintSem == F_SEM_NULL); + + // Generate the thread name + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + m_pszDbPath, szThreadName, szBaseName))) + { + goto Exit; + } + + f_sprintf( (char *)szThreadName, "Maintenance (%s)", (char *)szBaseName); + + // Create the maintenance semaphore + + if( RC_BAD( rc = f_semCreate( &m_hMaintSem))) + { + goto Exit; + } + + // Start the thread. + + if( RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( &m_pMaintThrd, + F_Database::maintenanceThread, szThreadName, + 0, 0, this, NULL, 32000))) + { + goto Exit; + } + + // Signal the thread to check for any queued work + + f_semSignal( m_hMaintSem); + +Exit: + + if( RC_BAD( rc)) + { + if( m_hMaintSem != F_SEM_NULL) + { + f_semDestroy( &m_hMaintSem); + } + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_Db::beginBackgroundTrans( + IF_Thread * pThread) +{ + RCODE rc = NE_SFLM_OK; + +RetryLock: + + // Obtain the file lock + + flmAssert( !(m_uiFlags & FDB_HAS_FILE_LOCK)); + + if( RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Lock( this, m_hWaitSem, + TRUE, FALSE, TRUE, SFLM_NO_TIMEOUT, FLM_BACKGROUND_LOCK_PRIORITY, + m_pDbStats))) + { + if( rc == NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT) + { + // This would only happen if we were signaled to shut down. + // So, it's ok to exit + + flmAssert( pThread->getShutdownFlag()); + } + goto Exit; + } + + // The lock needs to be marked as implicit so that commitTrans + // will unlock the database and allow the next update transaction to + // begin before all writes are complete. + + m_uiFlags |= (FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); + + // If there are higher priority waiters in the lock queue, + // we want to relinquish. + + if( m_pDatabase->m_pDatabaseLockObj->haveHigherPriorityWaiter( + FLM_BACKGROUND_LOCK_PRIORITY)) + { + if( pThread->getShutdownFlag()) + { + goto Exit; + } + if( RC_BAD( rc = m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this))) + { + goto Exit; + } + + m_uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); + goto RetryLock; + } + + // If we are shutting down, relinquish and exit. + + if( pThread->getShutdownFlag()) + { + rc = RC_SET( NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT); + goto Exit; + } + + // Start an update transaction + + if( RC_BAD( rc = beginTrans( + SFLM_UPDATE_TRANS, SFLM_NO_TIMEOUT, SFLM_DONT_POISON_CACHE))) + { + if( rc == NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT) + { + // This would only happen if we were signaled to shut down. + // So, it's ok to exit + + flmAssert( pThread->getShutdownFlag()); + } + goto Exit; + } + +Exit: + + if( RC_BAD( rc)) + { + if( m_uiFlags & FDB_HAS_FILE_LOCK) + { + (void)m_pDatabase->m_pDatabaseLockObj->Unlock( TRUE, this); + m_uiFlags &= ~(FDB_HAS_FILE_LOCK | FDB_FILE_LOCK_IMPLICIT); + } + } + + return( rc); +} + +/**************************************************************************** +Desc: Thread that will delete block chains from deleted indexes and + tables in the background. +****************************************************************************/ +RCODE F_Database::maintenanceThread( + IF_Thread * pThread) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = (F_Database *)pThread->getParm1(); + F_Db * pDb; + F_Row * pRow; + FLMUINT64 ui64MaintRowId; + FLMBOOL bStartedTrans; + FLMBOOL bShutdown; + F_DbSystem * pDbSystem; + FSTableCursor * pTableCursor; + FLMUINT uiBlkAddress; + FLMBOOL bIsNull; + FLMUINT uiBlocksToFree; + FLMUINT uiBlocksFreed; + +Retry: + + rc = NE_SFLM_OK; + pDb = NULL; + pRow = NULL; + bStartedTrans = FALSE; + bShutdown = FALSE; + pDbSystem = NULL; + pTableCursor = NULL; + + if( (pDbSystem = f_new F_DbSystem) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + pThread->setThreadStatus( FLM_THREAD_STATUS_INITIALIZING); + + if( RC_BAD( rc = pDbSystem->internalDbOpen( pDatabase, &pDb))) + { + // If the file is being closed, this is not an error. + + if( pDatabase->getFlags() & DBF_BEING_CLOSED) + { + rc = NE_SFLM_OK; + bShutdown = TRUE; + } + goto Exit; + } + pDbSystem->Release(); + pDbSystem = NULL; + + if ((pTableCursor = f_new FSTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + for( ;;) + { + pThread->setThreadStatus( FLM_THREAD_STATUS_RUNNING); + + for( ;;) + { + if( RC_BAD( rc = pDb->beginBackgroundTrans( pThread))) + { + goto Exit; + } + bStartedTrans = TRUE; + + pTableCursor->resetCursor(); + if (RC_BAD( rc = pTableCursor->setupRange( pDb, SFLM_TBLNUM_BLOCK_CHAINS, + 1, FLM_MAX_UINT64, NULL, NULL, NULL))) + { + goto Exit; + } + + // Free up to 25 blocks per transaction. + + uiBlocksToFree = 25; + while (uiBlocksToFree) + { + + if (RC_BAD( rc = pTableCursor->nextRow( pDb, &pRow, &ui64MaintRowId))) + { + if (rc != NE_SFLM_EOF_HIT) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + rc = NE_SFLM_OK; + break; + } + if (RC_BAD( rc = pRow->getUINT( pDb, + SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS, &uiBlkAddress, + &bIsNull))) + { + goto Exit; + } + if (bIsNull) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = pDb->maintBlockChainFree( + ui64MaintRowId, uiBlkAddress, uiBlocksToFree, 0, &uiBlocksFreed))) + { + goto Exit; + } + uiBlocksToFree -= uiBlocksFreed; + } + + bStartedTrans = FALSE; + if( RC_BAD( rc = pDb->commitTrans( 0, FALSE))) + { + goto Exit; + } + } + + pThread->setThreadStatus( FLM_THREAD_STATUS_SLEEPING); + f_semWait( pDatabase->m_hMaintSem, F_SEM_WAITFOREVER); + + if (pThread->getShutdownFlag()) + { + bShutdown = TRUE; + goto Exit; + } + } + +Exit: + + pThread->setThreadStatus( FLM_THREAD_STATUS_TERMINATING); + + if (pDbSystem) + { + pDbSystem->Release(); + } + + if (pRow) + { + pRow->ReleaseRow(); + } + + if( bStartedTrans) + { + pDb->abortTrans(); + } + + if (pDb) + { + pDb->Release(); + pDb = NULL; + } + + if (!bShutdown) + { + flmAssert( RC_BAD( rc)); + f_sleep( 250); + f_semSignal( pDatabase->m_hMaintSem); + goto Retry; + } + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_Db::maintBlockChainFree( + FLMUINT64 ui64MaintRowId, + FLMUINT uiStartBlkAddr, + FLMUINT uiBlocksToFree, + FLMUINT uiExpectedEndBlkAddr, + FLMUINT * puiBlocksFreed) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlocksFreed = 0; + FLMUINT uiEndBlkAddr = 0; + F_Row * pRow = NULL; + + // Make sure an update transaction is going and that a + // non-zero number of blocks was specified + + if( getTransType() != SFLM_UPDATE_TRANS || !uiBlocksToFree) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_ILLEGAL_OP); + goto Exit; + } + + if( RC_BAD( rc = btFreeBlockChain( + this, NULL, uiStartBlkAddr, uiBlocksToFree, + &uiBlocksFreed, &uiEndBlkAddr, NULL))) + { + goto Exit; + } + + flmAssert( uiBlocksFreed <= uiBlocksToFree); + + if (!uiEndBlkAddr) + { + gv_SFlmSysData.pRowCacheMgr->removeRow( this, SFLM_TBLNUM_BLOCK_CHAINS, + ui64MaintRowId); + } + else + { + if (RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( this, + SFLM_TBLNUM_BLOCK_CHAINS, ui64MaintRowId, + &pRow))) + { + goto Exit; + } + if (RC_BAD( rc = pRow->setUINT( this, + SFLM_COLNUM_BLOCK_CHAINS_BLOCK_ADDRESS, uiEndBlkAddr))) + { + goto Exit; + } + } + + if (uiExpectedEndBlkAddr) + { + if (uiBlocksToFree != uiBlocksFreed || + uiEndBlkAddr != uiExpectedEndBlkAddr) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + } + + if( RC_BAD( rc = m_pDatabase->m_pRfl->logBlockChainFree( + this, ui64MaintRowId, uiStartBlkAddr, uiEndBlkAddr, uiBlocksFreed))) + { + goto Exit; + } + + if (puiBlocksFreed) + { + *puiBlocksFreed = uiBlocksFreed; + } + +Exit: + + if (pRow) + { + pRow->ReleaseRow(); + } + + return( rc); +} + diff --git a/sql/src/fsrefupd.cpp b/sql/src/fsrefupd.cpp new file mode 100644 index 0000000..5a32abf --- /dev/null +++ b/sql/src/fsrefupd.cpp @@ -0,0 +1,100 @@ +//------------------------------------------------------------------------------ +// Desc: Insert and delete keys in an index B-Tree. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsrefupd.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/*************************************************************************** +Desc: Update (add or delete) a single reference +*****************************************************************************/ +RCODE F_Db::refUpdate( + F_INDEX * pIndex, + KREF_ENTRY * pKrefEntry, + FLMBOOL bNormalUpdate) +{ + RCODE rc = NE_SFLM_OK; + F_Btree * pbtree = NULL; + IXKeyCompare compareObject; + + // Get a btree + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpReserveBtree( &pbtree))) + { + goto Exit; + } + + flmAssert( pIndex->lfInfo.uiRootBlk); + + compareObject.setIxInfo( this, pIndex); + if (bNormalUpdate && pKrefEntry->bDelete) + { + compareObject.setOldRow( pKrefEntry->pRow); + } + if( RC_BAD( rc = pbtree->btOpen( this, &pIndex->lfInfo, + (pIndex->uiFlags & IXD_ABS_POS) ? TRUE : FALSE, + (pIndex->pDataIcds) ? TRUE : FALSE, &compareObject))) + { + goto Exit; + } + + if (!pKrefEntry->bDelete) + { + pbtree->btResetBtree(); + if( RC_BAD( rc = pbtree->btInsertEntry( + (FLMBYTE *)&pKrefEntry [1], pKrefEntry->ui16KeyLen, + pKrefEntry->uiDataLen + ? ((FLMBYTE *)(&pKrefEntry [1])) + 1 + pKrefEntry->ui16KeyLen + : NULL, + pKrefEntry->uiDataLen, TRUE, TRUE))) + { + goto Exit; + } + } + else + { + pbtree->btResetBtree(); + if (RC_BAD( rc = pbtree->btRemoveEntry( + (FLMBYTE *)&pKrefEntry [1], pKrefEntry->ui16KeyLen))) + { + if (rc == NE_SFLM_NOT_FOUND) + { + // Already been deleted, ignore the error condition and go on. + + RC_UNEXPECTED_ASSERT( rc); + rc = NE_SFLM_OK; + } + + goto Exit; + } + } + +Exit: + + if (pbtree) + { + gv_SFlmSysData.pBtPool->btpReturnBtree( &pbtree); + } + + return( rc); +} diff --git a/sql/src/fsrvlock.cpp b/sql/src/fsrvlock.cpp new file mode 100644 index 0000000..edef7d1 --- /dev/null +++ b/sql/src/fsrvlock.cpp @@ -0,0 +1,1204 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the methods for the ServerLockManager and +// ServerLockObject classes. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsrvlock.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define LOCK_HASH_ENTRIES 256 + +/**************************************************************************** +Desc: +****************************************************************************/ +ServerLockManager::~ServerLockManager() +{ + // Free everything in the avail lock list. This is where all + // of the lock objects should be at this point. + + if (m_hMutex != F_MUTEX_NULL) + { + lockMutex( FALSE); + + // Signal all pending lock waiters. + + CheckLockTimeouts( TRUE, TRUE); + + while (m_pAvailLockList) + { + ServerLockObject_p pLockObject = m_pAvailLockList; + + UnlinkLockObject( pLockObject, FALSE); + + pLockObject->Release(); + } + unlockMutex( FALSE); + f_mutexDestroy( &m_hMutex); + } + + // Free the hash table. + + f_free( &m_pHashTbl); +} + +/**************************************************************************** +Desc: Initializes the lock manager's hash table. +****************************************************************************/ +RCODE ServerLockManager::SetupHashTbl() +{ + return( flmAllocHashTbl( LOCK_HASH_ENTRIES, &m_pHashTbl)); +} + +/**************************************************************************** +Desc: Finds the lock object for the passed in item identifier. + If one is not found, one will be created. +****************************************************************************/ +ServerLockObject_p ServerLockManager::GetLockObject( + F_ItemId_p pItemId) +{ + FLMUINT uiBucket; + ServerLockObject * pLockObject; + ServerLockObject * pTmpLockObject; + + // Get the hash bucket. + + uiBucket = pItemId->GetHashBucket( m_pHashTbl, LOCK_HASH_ENTRIES); + + // See if the desired file is already in the hash bucket. + + lockMutex( FALSE); + + pLockObject = (ServerLockObject_p)m_pHashTbl [uiBucket].pFirstInBucket; + + // See if any of the objects match. + + while (pLockObject) + { + if (pItemId->IsEqual( pLockObject->m_pItemId)) + { + goto Exit; + } + pLockObject = pLockObject->m_pNext; + } + + // If we didn't find a matching object, allocate an object and link it into + // the hash bucket. Check to see if we have any in the avail list + // first so that we don't have to allocate memory if we can avoid it. + + if ((pLockObject = m_pAvailLockList) != NULL) + { + UnlinkLockObject( pLockObject, FALSE); + } + else + { + if ((pLockObject = f_new ServerLockObject) == NULL) + { + goto Exit; + } + } + + // Setup the new object and put it into the hash bucket. + + pLockObject->Setup( this, pItemId, uiBucket); + pTmpLockObject = + (ServerLockObject_p)m_pHashTbl [uiBucket].pFirstInBucket; + pLockObject->m_pPrev = NULL; + pLockObject->m_pNext = pTmpLockObject; + if (pTmpLockObject) + { + pTmpLockObject->m_pPrev = pLockObject; + } + m_pHashTbl [uiBucket].pFirstInBucket = pLockObject; + +Exit: + + unlockMutex( FALSE); + return( pLockObject); +} + +/**************************************************************************** +Desc: Unlinks a lock object from whatever list it is in. This routine + assumes that the server lock manager's mutex is already locked. +****************************************************************************/ +void ServerLockManager::UnlinkLockObject( + ServerLockObject * pLockObject, + FLMBOOL bPutInAvailList) +{ + ServerLockObject * pTmpLockObject; + FLMUINT uiBucket; + + // If hash bucket 0xFFFF, unlink from the avail list. Otherwise, + // unlink from the hash bucket it is in. + + if ((uiBucket = pLockObject->m_uiBucket) == 0xFFFF) + { + if ((pTmpLockObject = pLockObject->m_pPrev) == NULL) + { + m_pAvailLockList = pLockObject->m_pNext; + } + else + { + pTmpLockObject->m_pNext = pLockObject->m_pNext; + } + m_uiNumAvail--; + } + else + { + if ((pTmpLockObject = pLockObject->m_pPrev) == NULL) + { + m_pHashTbl [uiBucket].pFirstInBucket = pLockObject->m_pNext; + } + else + { + pTmpLockObject->m_pNext = pLockObject->m_pNext; + } + } + if ((pTmpLockObject = pLockObject->m_pNext) != NULL) + { + pTmpLockObject->m_pPrev = pLockObject->m_pPrev; + } + + if (bPutInAvailList) + { + if (m_uiNumAvail >= 50) + { + flmAssert( getRefCount() == 1); + pLockObject->Release(); + } + else + { + pLockObject->Setup( this, NULL, 0xFFFF); + if (m_pAvailLockList) + { + m_pAvailLockList->m_pPrev = pLockObject; + } + pLockObject->m_pPrev = NULL; + pLockObject->m_pNext = m_pAvailLockList; + m_pAvailLockList = pLockObject; + m_uiNumAvail++; + } + } +} + +/**************************************************************************** +Desc: Checks for any pending lock requests that have timed out. +****************************************************************************/ +void ServerLockManager::CheckLockTimeouts( + FLMBOOL bMutexAlreadyLocked, + FLMBOOL bTimeoutAll) +{ + FLMUINT uiCurrTime; + LOCK_WAITER_p pLockWaiter; + + lockMutex( bMutexAlreadyLocked); + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + while ((m_pFirstLockWaiter) && + ((bTimeoutAll) || + ((m_pFirstLockWaiter->uiWaitTime) && + (FLM_ELAPSED_TIME( uiCurrTime, m_pFirstLockWaiter->uiWaitStartTime) >= + m_pFirstLockWaiter->uiWaitTime)))) + { + // Sanity check + + flmAssert( m_pFirstLockWaiter->pPrevGlobal == NULL); + + // Lock waiter has timed out. + + pLockWaiter = m_pFirstLockWaiter; + + // Remove from global list and lock object's list + + RemoveWaiter( pLockWaiter); + pLockWaiter->pLockObject->RemoveWaiter( pLockWaiter); + + // Tell the waiter that the lock request timed out. + + *(pLockWaiter->pRc) = RC_SET( NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT); + f_semSignal( pLockWaiter->hESem); + } + + unlockMutex( bMutexAlreadyLocked); +} + + +/**************************************************************************** +Desc: Signal a lock waiter that has a matching thread id. +****************************************************************************/ +void ServerLockManager::SignalLockWaiter( + FLMUINT uiThreadId) +{ + FLMUINT uiCurrTime; + LOCK_WAITER_p pLockWaiter; + LOCK_WAITER_p pNextWaiter; + + lockMutex( FALSE); + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + for( pLockWaiter = m_pFirstLockWaiter; + pLockWaiter; + pLockWaiter = pNextWaiter) + { + pNextWaiter = pLockWaiter->pNextGlobal; + + if( pLockWaiter->uiThreadId == uiThreadId) + { + // Remove from global list and lock object's list + + RemoveWaiter( pLockWaiter); + pLockWaiter->pLockObject->RemoveWaiter( pLockWaiter); + + // Tell the waiter that the lock request timed out. + + *(pLockWaiter->pRc) = RC_SET( NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT); + f_semSignal( pLockWaiter->hESem); + break; + } + } + + unlockMutex( FALSE); + return; +} + +/**************************************************************************** +Desc: Inserts a waiter into the global list of waiters, sorted by + its end wait time. + + NOTE: This routine assumes that the lock manager's semaphore + is already locked. +****************************************************************************/ +void ServerLockManager::InsertWaiter( + LOCK_WAITER_p pLockWaiter) +{ + LOCK_WAITER_p pPrevLockWaiter; + + // Determine where in the list this lock waiter should go. + + if ((pPrevLockWaiter = m_pFirstLockWaiter) != NULL) + { + FLMUINT uiCurrTime = FLM_GET_TIMER(); + FLMUINT uiElapTime; + FLMUINT uiTimeLeft; + + while (pPrevLockWaiter) + { + // Waiters with zero wait time go to end of list. + // They never time out. + + if (!pPrevLockWaiter->uiWaitTime) + { + + // Should go BEFORE the first zero waiter. + + pPrevLockWaiter = pPrevLockWaiter->pPrevGlobal; + break; + } + else if (!pLockWaiter->uiWaitTime) + { + if (!pPrevLockWaiter->pNextGlobal) + { + break; + } + pPrevLockWaiter = pPrevLockWaiter->pNextGlobal; + } + else + { + // Determine how much time is left on the previous + // lock waiter's timer. If it is less than the + // new lock waiter's wait time, the new lock waiter + // should be inserted AFTER it. Otherwise, the + // new lock waiter should be inserted BEFORE it. + + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, + pPrevLockWaiter->uiWaitStartTime); + if (uiElapTime >= pPrevLockWaiter->uiWaitTime) + { + uiTimeLeft = 0; + } + else + { + uiTimeLeft = pPrevLockWaiter->uiWaitTime - uiElapTime; + } + + // New lock waiter will time out before previous lock + // waiter - insert it BEFORE the previous lock waiter. + + if (pLockWaiter->uiWaitTime < uiTimeLeft) + { + pPrevLockWaiter = pPrevLockWaiter->pPrevGlobal; + break; + } + else + { + if (!pPrevLockWaiter->pNextGlobal) + break; + pPrevLockWaiter = pPrevLockWaiter->pNextGlobal; + } + } + } + } + + // Insert into list AFTER pPrevLockWaiter. + + if ((pLockWaiter->pPrevGlobal = pPrevLockWaiter) != NULL) + { + if ((pLockWaiter->pNextGlobal = pPrevLockWaiter->pNextGlobal) != NULL) + { + pLockWaiter->pNextGlobal->pPrevGlobal = pLockWaiter; + } + pPrevLockWaiter->pNextGlobal = pLockWaiter; + } + else + { + if( (pLockWaiter->pNextGlobal = m_pFirstLockWaiter) != NULL) + { + m_pFirstLockWaiter->pPrevGlobal = pLockWaiter; + } + m_pFirstLockWaiter = pLockWaiter; + } +} + +/**************************************************************************** +Desc: See if this item ID is equal to another F_ItemId. +****************************************************************************/ +FLMBOOL FFileItemId::IsEqual( + F_ItemId_p pItemId) +{ + FFileItemId_p pFFileItemId; + RFileItemId_p pRFileItemId; + char szName1 [F_FILENAME_SIZE]; + char szName2 [F_FILENAME_SIZE]; + FLMUINT uiItemType = pItemId->GetItemType(); + + switch (uiItemType) + { + case FFILE_ITEM: + case FFILE_TRANS_ITEM: + if (m_uiItemType != uiItemType) + return( FALSE); + pFFileItemId = (FFileItemId_p)pItemId; + + /* First see if the FFILE pointers are the same. */ + + if (pFFileItemId->getDatabase() == + this->getDatabase()) + { + return( TRUE); + } + + /* Next see if the file names are the same. */ + + this->GetFileName( szName1); + pFFileItemId->GetFileName( szName2); +#if !defined( FLM_UNIX) + if (f_stricmp( szName1, szName2) == 0) + return( TRUE); +#else + if (f_strcmp( szName1, szName2) == 0) + return( TRUE); +#endif + break; + case RFILE_ITEM: + case RFILE_TRANS_ITEM: + if ((uiItemType == RFILE_ITEM && + m_uiItemType != FFILE_ITEM) || + (uiItemType == RFILE_TRANS_ITEM && + m_uiItemType != FFILE_TRANS_ITEM)) + { + return( FALSE); + } + pRFileItemId = (RFileItemId_p)pItemId; + + /* See if the file names are the same. */ + + this->GetFileName( szName1); + pRFileItemId->GetFileName( szName2); +#if !defined( FLM_UNIX) + if (f_stricmp( szName1, szName2) == 0) + return( TRUE); +#else + if (f_strcmp( szName1, szName2) == 0) + return( TRUE); +#endif + break; + default: + break; + } + return( FALSE); +} + +/**************************************************************************** +Desc: Get file name for this file item. +****************************************************************************/ +void FFileItemId::GetFileName( + char * pszFileNameRV) +{ + // _ExtractFileName( m_pFile, pszFileNameRV); + char szTmpPath[ F_PATH_MAX_SIZE]; + + (void)gv_SFlmSysData.pFileSystem->pathReduce( + m_pDatabase->m_pszDbPath, szTmpPath, pszFileNameRV); + + // Convert to upper case for consistency when hashing. + +#if !defined( FLM_UNIX) + while (*pszFileNameRV) + { + *pszFileNameRV = (char)f_toupper( *pszFileNameRV); + pszFileNameRV++; + } +#endif +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RFileItemId::RFileItemId( + char * pszFileName, + FLMBOOL bTrans) +{ + char * pszTmp = &m_szFileName [0]; + + while (*pszFileName) + { + // Convert to uppercase for consistency when hashing. + +#if !defined( FLM_UNIX) + *pszFileName = (char)f_toupper( *pszFileName); +#else + *pszTmp++ = *pszFileName; +#endif + pszFileName++; + } + *pszTmp = 0; + m_uiItemType = (FLMUINT)((bTrans) + ? (FLMUINT)RFILE_TRANS_ITEM + : (FLMUINT)RFILE_ITEM); +} + +/**************************************************************************** +Desc: See if this item ID is equal to another F_ItemId. +****************************************************************************/ +FLMBOOL RFileItemId::IsEqual( + F_ItemId * pItemId) +{ + FFileItemId * pFFileItemId; + RFileItemId * pRFileItemId; + char szName1 [F_FILENAME_SIZE]; + char szName2 [F_FILENAME_SIZE]; + FLMUINT uiItemType = pItemId->GetItemType(); + + switch (uiItemType) + { + case FFILE_ITEM: + case FFILE_TRANS_ITEM: + if ((uiItemType == FFILE_ITEM && + m_uiItemType != RFILE_ITEM) || + (uiItemType == FFILE_TRANS_ITEM && + m_uiItemType != RFILE_TRANS_ITEM)) + { + return( FALSE); + } + pFFileItemId = (FFileItemId_p)pItemId; + + /* See if the file names are the same. */ + + this->GetFileName( szName1); + pFFileItemId->GetFileName( szName2); +#if !defined( FLM_UNIX) + if (f_stricmp( szName1, szName2) == 0) + return( TRUE); +#else + if (f_strcmp( szName1, szName2) == 0) + return( TRUE); +#endif + break; + case RFILE_ITEM: + case RFILE_TRANS_ITEM: + if (m_uiItemType != uiItemType) + return( FALSE); + pRFileItemId = (RFileItemId_p)pItemId; + + /* See if the file names are the same. */ + + this->GetFileName( szName1); + pRFileItemId->GetFileName( szName2); +#if !defined( FLM_UNIX) + if (f_stricmp( szName1, szName2) == 0) + return( TRUE); +#else + if (f_strcmp( szName1, szName2) == 0) + return( TRUE); +#endif + break; + default: + break; + } + return( FALSE); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +ServerLockObject::ServerLockObject() +{ + m_pServerLockMgr = NULL; + m_pItemId = NULL; + m_uiLockThreadId = 0; + m_uiLockTime = 0; + m_uiLockCnt = 0; + m_pFirstLockWaiter = + m_pLastLockWaiter = NULL; + m_uiNumWaiters = 0; + m_pNext = m_pPrev = NULL; + m_uiSharedLockCnt = 0; + m_bExclLock = FALSE; + m_uiBucket = 0xFFFF; + m_bStartTimeSet = FALSE; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FLMINT ServerLockObject::Release( + FLMBOOL bMutexAlreadyLocked) +{ + FLMINT iRefCnt = --m_refCnt; + + if( !iRefCnt) + { + delete this; + goto Exit; + } + + // When it is no longer pointed to from anything but the server lock + // manager, put it into the avail list. + + if (iRefCnt == 1) + { + LOCK_WAITER * pLockWaiter; + + // Signal all waiters that they cannot get the lock. + + m_pServerLockMgr->lockMutex( bMutexAlreadyLocked); + while (m_pFirstLockWaiter) + { + pLockWaiter = m_pFirstLockWaiter; + + // Remove from global list and lock object's list + + RemoveWaiter( pLockWaiter); + m_pServerLockMgr->RemoveWaiter( pLockWaiter); + + // Tell the waiter that the lock request timed out and signal + // the thread to wake it up. + + *(pLockWaiter->pRc) = RC_SET( NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT); + f_semSignal( pLockWaiter->hESem); + } + + m_pServerLockMgr->UnlinkLockObject( this, TRUE); + m_pServerLockMgr->unlockMutex( bMutexAlreadyLocked); + } + +Exit: + + return( iRefCnt); +} + +/**************************************************************************** +Desc: Initialize some data for the lock object. +****************************************************************************/ +void ServerLockObject::Setup( + ServerLockManager_p pServerLockMgr, + F_ItemId_p pItemId, + FLMUINT uiBucket) +{ + m_pServerLockMgr = pServerLockMgr; + if (m_pItemId) + { + m_pItemId->Release(); + m_pItemId = NULL; + } + if ((m_pItemId = pItemId) != NULL) + { + m_pItemId->AddRef(); + } + m_uiBucket = uiBucket; +} + +/**************************************************************************** +Desc: Removes a waiter from the list of waiters on this object. + NOTE: This routine assumes that the lock manager's semaphore + is already locked. +****************************************************************************/ +void ServerLockObject::RemoveWaiter( + LOCK_WAITER_p pLockWaiter) +{ + if (pLockWaiter->pNext) + pLockWaiter->pNext->pPrev = pLockWaiter->pPrev; + else + m_pLastLockWaiter = pLockWaiter->pPrev; + + if (pLockWaiter->pPrev) + pLockWaiter->pPrev->pNext = pLockWaiter->pNext; + else + m_pFirstLockWaiter = pLockWaiter->pNext; + flmAssert( m_uiNumWaiters > 0); + m_uiNumWaiters--; +} + +/**************************************************************************** +Desc: Lock this object. If object is locked, wait the specified + number of seconds. +****************************************************************************/ +RCODE ServerLockObject::Lock( + F_Db * pDb, + F_SEM hWaitSem, + FLMBOOL bLogEvent, + FLMBOOL bSendSuspendEvent, // Send suspend event, as opposed to + // waiting event, when waiting. + FLMBOOL bExclReq, // Exclusive or shared lock? + FLMUINT uiMaxWaitSecs, // Maximum wait time in seconds. + FLMINT iPriority, // Lock priority + SFLM_DB_STATS * pDbStats) // Place to collect stats. +{ + RCODE rc = NE_SFLM_OK; + RCODE TempRc; + LOCK_WAITER LockWait; + FLMBOOL bMutexLocked = FALSE; + + flmAssert( hWaitSem != F_SEM_NULL); + + m_pServerLockMgr->lockMutex( FALSE); + bMutexLocked = TRUE; + + if ((m_pFirstLockWaiter) || + (m_bExclLock) || + ((bExclReq) && (m_uiSharedLockCnt))) + { + + // Object is locked, wait to get lock. + + if (!uiMaxWaitSecs) + { + rc = RC_SET( NE_SFLM_DATABASE_LOCK_REQ_TIMEOUT); + goto Exit; + } + + // Set up to wait for the lock. + + f_memset( &LockWait, 0, sizeof( LockWait)); + LockWait.pLockObject = this; + LockWait.hESem = hWaitSem; + + // Link into list of waiters on this object. + + if ((LockWait.pPrev = m_pLastLockWaiter) != NULL) + { + LockWait.pPrev->pNext = &LockWait; + } + else + { + m_pFirstLockWaiter = &LockWait; + } + m_pLastLockWaiter = &LockWait; + m_uiNumWaiters++; + + LockWait.uiThreadId = f_threadId(); + LockWait.pRc = &rc; + + rc = RC_SET( NE_SFLM_FAILURE); + + LockWait.bExclReq = bExclReq; + LockWait.iPriority = iPriority; + LockWait.uiWaitStartTime = (FLMUINT)FLM_GET_TIMER(); + + if (bExclReq && pDbStats) + { + f_timeGetTimeStamp( &LockWait.StartTime); + LockWait.pDbStats = pDbStats; + } + + if (uiMaxWaitSecs == SFLM_NO_TIMEOUT) + { + LockWait.uiWaitTime = 0; + } + else + { + LockWait.uiWaitTime = FLM_SECS_TO_TIMER_UNITS( uiMaxWaitSecs); + } + + // Link to list of global waiters - ordered by end time. + + m_pServerLockMgr->InsertWaiter( &LockWait); + + m_pServerLockMgr->unlockMutex( FALSE); + bMutexLocked = FALSE; + + // Do the event callback, if any registered. + + if (bLogEvent && + gv_SFlmSysData.EventHdrs [SFLM_EVENT_LOCKS].pEventCBList) + { + flmDoEventCallback( SFLM_EVENT_LOCKS, + (eEventType)((bSendSuspendEvent) + ? SFLM_EVENT_LOCK_SUSPENDED + : SFLM_EVENT_LOCK_WAITING), + pDb, LockWait.uiThreadId, + 0, 0, 0, NE_SFLM_OK); + } + + // Now just wait to be signaled. + + if (RC_BAD( TempRc = f_semWait( LockWait.hESem, F_SEM_WAITFOREVER))) + { + RC_UNEXPECTED_ASSERT( TempRc); + rc = TempRc; + } + else + { + // Process that signaled us better set the rc to something + // besides NE_SFLM_FAILURE. + + if (rc == NE_SFLM_FAILURE) + { + RC_UNEXPECTED_ASSERT( rc); + } + } + + // Do the event callback, if any registered. + + if (bLogEvent && + gv_SFlmSysData.EventHdrs [SFLM_EVENT_LOCKS].pEventCBList) + { + if (RC_BAD( rc)) + { + flmDoEventCallback( SFLM_EVENT_LOCKS, + SFLM_EVENT_LOCK_TIMEOUT, + pDb, LockWait.uiThreadId, + 0, 0, 0, NE_SFLM_OK); + } + else + { + flmDoEventCallback( SFLM_EVENT_LOCKS, + (eEventType)((bSendSuspendEvent) + ? SFLM_EVENT_LOCK_RESUMED + : SFLM_EVENT_LOCK_GRANTED), + pDb, LockWait.uiThreadId, + 0, 0, 0, NE_SFLM_OK); + } + } + } + else + { + + // Object is NOT locked in a conflicting mode. Grant the + // lock immediately. + + m_uiLockThreadId = f_threadId(); + m_bExclLock = bExclReq; + if (!bExclReq) + { + m_uiSharedLockCnt++; + } + else + { + m_uiLockTime = (FLMUINT)FLM_GET_TIMER(); + flmAssert( m_uiSharedLockCnt == 0); + + // Take care of statistics gathering. + + if (pDbStats) + { + + // If m_bStartTimeSet is TRUE, we started the + // clock the last time nobody had the exclusive + // lock, so we need to sum up idle time now. + + if (m_bStartTimeSet) + { + flmAddElapTime( &m_StartTime, &pDbStats->NoLocks.ui64ElapMilli); + pDbStats->NoLocks.ui64Count++; + } + + // Restart the clock for this locker. + + f_timeGetTimeStamp( &m_StartTime); + m_bStartTimeSet = TRUE; + } + else + { + m_bStartTimeSet = FALSE; + } + } + + // Do the event callback, if any registered. + + if (bLogEvent && + !bSendSuspendEvent && + gv_SFlmSysData.EventHdrs [SFLM_EVENT_LOCKS].pEventCBList) + { + m_pServerLockMgr->unlockMutex( FALSE); + bMutexLocked = FALSE; + flmDoEventCallback( SFLM_EVENT_LOCKS, + SFLM_EVENT_LOCK_GRANTED, + pDb, m_uiLockThreadId, + 0, 0, 0, NE_SFLM_OK); + } + } + +Exit: + + if (RC_OK( rc)) + { + m_uiLockCnt++; + } + + if (bMutexLocked) + { + m_pServerLockMgr->unlockMutex( FALSE); + } + + return( rc); +} + +/**************************************************************************** +Desc: Unlock this object. If there is a pending lock request, give + the lock to the next waiter. +****************************************************************************/ +RCODE ServerLockObject::Unlock( + FLMBOOL bLogEvent, + F_Db * pDb, + FLMBOOL bRelease, + SFLM_DB_STATS * pDbStats) +{ + RCODE rc = NE_SFLM_OK; + F_SEM hESem; + LOCK_WAITER_p pLockWaiter; + + m_pServerLockMgr->lockMutex( FALSE); + + if (m_bExclLock) + { + flmAssert( m_uiSharedLockCnt == 0); + m_bExclLock = FALSE; + + // Record how long the lock was held, if we were tracking + // it. + + if (pDbStats && m_bStartTimeSet) + { + flmAddElapTime( &m_StartTime, &pDbStats->HeldLock.ui64ElapMilli); + pDbStats->HeldLock.ui64Count++; + } + m_bStartTimeSet = FALSE; + } + else + { + flmAssert( m_uiSharedLockCnt > 0); + m_uiSharedLockCnt--; + } + + // Do the event callback, if any registered. + // NOTE: flmDoEventCallback locks the event mutex. + // Since we are inside the lock manager's mutex lock, + // the callback should not do ANYTHING that would cause + // us to end up in here again! + + if (bLogEvent && + gv_SFlmSysData.EventHdrs [SFLM_EVENT_LOCKS].pEventCBList) + { + flmDoEventCallback( SFLM_EVENT_LOCKS, + SFLM_EVENT_LOCK_RELEASED, + pDb, m_uiLockThreadId, + 0, 0, 0, NE_SFLM_OK); + } + + m_uiLockThreadId = 0; + + /* See if we need to signal the next set of waiters. */ + + if (m_pFirstLockWaiter && !m_uiSharedLockCnt) + { + m_bExclLock = m_pFirstLockWaiter->bExclReq; + while (m_pFirstLockWaiter) + { + if (!m_bExclLock) + { + m_uiSharedLockCnt++; + } + + + pLockWaiter = m_pFirstLockWaiter; + hESem = pLockWaiter->hESem; + + // Unlink the waiter from the list of waiters on this lock + // object and then from the global list of waiters. + // IMPORTANT NOTE: Do NOT signal the semaphore until AFTER + // doing this unlinking. This is because LOCK_WAITER + // structures exist only on the stack of the thread + // being signaled. If we tried to assign m_pFirstLockWaiter after + // signaling the semaphore, the LOCK_WAITER structure could + // disappear and m_pFirstLockWaiter would get garbage. + + RemoveWaiter( pLockWaiter); + m_pServerLockMgr->RemoveWaiter( pLockWaiter); + + // Update statistics for the waiter. + + if (pLockWaiter->pDbStats) + { + flmAddElapTime( &pLockWaiter->StartTime, + &pLockWaiter->pDbStats->WaitingForLock.ui64ElapMilli); + pLockWaiter->pDbStats->WaitingForLock.ui64Count++; + } + + // Grant the lock to this waiter and signal the thread + // to wake it up. + + m_uiLockThreadId = pLockWaiter->uiThreadId; + if (m_bExclLock) + { + m_uiLockTime = (FLMUINT)FLM_GET_TIMER(); + + // Restart the stats timer + + if (pDbStats) + { + m_bStartTimeSet = TRUE; + f_timeGetTimeStamp( &m_StartTime); + } + } + + *(pLockWaiter->pRc) = NE_SFLM_OK; + + f_semSignal( hESem); + + // If the next waiter is not a shared lock request or + // the lock that was granted was exclusive, we stop + // here. + + if (m_bExclLock || + (m_pFirstLockWaiter && m_pFirstLockWaiter->bExclReq)) + { + break; + } + } + } + else if (bRelease && + !m_pFirstLockWaiter && // No one is wating for the object + !m_uiSharedLockCnt) // No one has the object locked + { + // Release the object. If the reference count drops to 1, + // the object will be put in the avail list. The caller + // should have performed an AddRef() on the object at + // some point prior to calling this method. Once this routine + // returns the caller should not attempt further access of the object. + Release( TRUE); + bRelease = FALSE; + } + + // Start timer, if not already running. If the timer is not set at + // this point, it will be because nobody has been granted the exclusive + // lock. If someone was granted the exclusive lock, the timer would + // have been started above. We start it here so we can track idle + // time. + + if (pDbStats && !m_bStartTimeSet && !bRelease) + { + flmAssert( !m_bExclLock); + m_bStartTimeSet = TRUE; + f_timeGetTimeStamp( &m_StartTime); + } + + // If we get to this point and bRelease is still TRUE, someone was + // waiting to acquire the lock or there is still a shared lock + // count. + + if( bRelease) + { + // All lock waiters should have done an AddRef on the lock object. + // At this point we should still have our reference to the object, + // the lock manager's reference, and a reference from at least one + // waiter (that may have been granted above). + flmAssert( m_refCnt >= 3); + m_refCnt--; + } + + m_pServerLockMgr->unlockMutex( FALSE); + return( rc); +} + +/**************************************************************************** +Desc: Returns information about the pending lock requests. +****************************************************************************/ +void ServerLockObject::GetLockInfo( + FLMINT iPriority, + eDbLockType * peCurrLockType, + FLMUINT * puiThreadId, + FLMUINT * puiNumExclQueued, + FLMUINT * puiNumSharedQueued, + FLMUINT * puiPriorityCount) +{ + LOCK_WAITER_p pLockWaiter; + + *puiNumExclQueued = 0; + *puiNumSharedQueued = 0; + *puiPriorityCount = 0; + + m_pServerLockMgr->lockMutex( FALSE); + + // Get the type of lock, if any. + + if (m_bExclLock) + { + *peCurrLockType = SFLM_LOCK_EXCLUSIVE; + *puiThreadId = m_uiLockThreadId; + } + else if (m_uiSharedLockCnt) + { + *peCurrLockType = SFLM_LOCK_SHARED; + *puiThreadId = 0; + } + else + { + *peCurrLockType = SFLM_LOCK_NONE; + *puiThreadId = 0; + } + + // Get information on pending lock requests. + + pLockWaiter = m_pFirstLockWaiter; + for ( ; pLockWaiter; pLockWaiter = pLockWaiter->pNext ) + { + + // Count the number of exclusive and shared waiters. + + if (pLockWaiter->bExclReq) + { + (*puiNumExclQueued)++; + } + else + { + (*puiNumSharedQueued)++; + } + + // Count the number of waiters at or above input priority. + + if (pLockWaiter->iPriority >= iPriority) + { + (*puiPriorityCount)++; + } + } + + m_pServerLockMgr->unlockMutex( FALSE); +} + +/**************************************************************************** +Desc: Return the lock waiters for this object. +****************************************************************************/ +RCODE ServerLockObject::GetLockInfo( + IF_LockInfoClient * pLockInfo) +{ + RCODE rc = NE_SFLM_OK; + LOCK_WAITER * pLockWaiter; + FLMUINT uiCnt; + FLMUINT uiElapTime; + FLMUINT uiCurrTime; + FLMUINT uiMilli; + + m_pServerLockMgr->lockMutex( FALSE); + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + if (!m_uiNumWaiters && !m_uiLockThreadId) + { + pLockInfo->setLockCount( 0); + goto Exit; + } + uiCnt = m_uiNumWaiters + 1; // Add one for lock holder. + if( pLockInfo->setLockCount( uiCnt) == FALSE) + { + goto Exit; + } + + // Output the lock holder first. + + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, m_uiLockTime); + uiMilli = FLM_TIMER_UNITS_TO_MILLI( uiElapTime); + if( pLockInfo->addLockInfo( 0, m_uiLockThreadId, uiMilli) == FALSE) + { + goto Exit; + } + uiCnt--; + + // Output the lock waiters. + + pLockWaiter = m_pFirstLockWaiter; + while( pLockWaiter && uiCnt) + { + uiElapTime = FLM_ELAPSED_TIME( uiCurrTime, pLockWaiter->uiWaitStartTime); + uiMilli = FLM_TIMER_UNITS_TO_MILLI( uiElapTime); + if( pLockInfo->addLockInfo( (m_uiNumWaiters - uiCnt) + 1, + pLockWaiter->uiThreadId, uiMilli) == FALSE) + { + goto Exit; + } + pLockWaiter = pLockWaiter->pNext; + uiCnt--; + } + flmAssert( pLockWaiter == NULL && uiCnt == 0); + +Exit: + + m_pServerLockMgr->unlockMutex( FALSE); + return( rc); +} + +/**************************************************************************** +Desc: Returns TRUE if there are lock waiters with a priority > iPriority +****************************************************************************/ +FLMBOOL ServerLockObject::haveHigherPriorityWaiter( + FLMINT iPriority) +{ + LOCK_WAITER_p pLockWaiter; + FLMBOOL bWaiters = FALSE; + + m_pServerLockMgr->lockMutex( FALSE); + + pLockWaiter = m_pFirstLockWaiter; + for ( ; pLockWaiter; pLockWaiter = pLockWaiter->pNext ) + { + // If we find a waiter with a priority > the specified + // priority, we're done. + + if (pLockWaiter->iPriority > iPriority) + { + bWaiters = TRUE; + break; + } + } + + m_pServerLockMgr->unlockMutex( FALSE); + return( bWaiters); +} diff --git a/sql/src/fsrvlock.h b/sql/src/fsrvlock.h new file mode 100644 index 0000000..05e4f8e --- /dev/null +++ b/sql/src/fsrvlock.h @@ -0,0 +1,424 @@ +//------------------------------------------------------------------------------ +// Desc: This include file contains the class definitions for FLAIM's +// ServerLockManager and ServerLockObject classes. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsrvlock.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FSRVLOCK_H +#define FSRVLOCK_H + +FLMUINT flmStrHashBucket( + char * pszStr, + FBUCKET * pHashTbl, + FLMUINT uiNumBuckets); + + /* + *** Define the 'C++' classes for FLAIM's File Handle cache code. + */ + +class ServerLockManager; // Forward Reference +class ServerLockObject; // Forward Reference +class F_ItemId; // Forward Reference +class FFileItemId; // Forward Reference +class RFileItemId; // Forward Reference + +typedef ServerLockManager * ServerLockManager_p; +typedef ServerLockObject * ServerLockObject_p; +typedef F_ItemId * F_ItemId_p; +typedef FFileItemId * FFileItemId_p; +typedef RFileItemId * RFileItemId_p; + +/************************************************************************** +Struct: LOCK_WAITER (Lock Waiter) +Desc: This structure is used to keep track of threads waiting for a + lock. +**************************************************************************/ +typedef struct Lock_Waiter * LOCK_WAITER_p; + +typedef struct Lock_Waiter +{ + ServerLockObject_p pLockObject; // Pointer to lock object. + FLMUINT uiThreadId; // Thread of waiter + F_SEM hESem; // Semaphore to signal when lock is + // granted (or denied). + RCODE * pRc; // Pointer to return code that is to + // be set when lock is granted or + // denied. + FLMUINT uiWaitStartTime; + // Time we started waiting. + FLMUINT uiWaitTime; // Time pending lock request should + // wait before being timed out. + // Zero means should not be timed out. + FLMBOOL bExclReq; // TRUE if exclusive lock request. + FLMINT iPriority; // Priority of waiter. + F_TMSTAMP StartTime; // Time we started waiting (for stats) + SFLM_DB_STATS * pDbStats; // Statistics to update. + LOCK_WAITER_p pNext; // Next lock waiter in list. + LOCK_WAITER_p pPrev; // Previous lock waiter in list. + LOCK_WAITER_p pNextGlobal; // Next lock waiter in global list + // that is ordered according to + // udWaitEndTime. + LOCK_WAITER_p pPrevGlobal; // Previous lock waiter in global list +} LOCK_WAITER; + +/*=========================================================================== +Class: ServerLockManager +Desc: The ServerLockManager class manages ServerLockObject objects. +===========================================================================*/ + +class ServerLockManager : public F_Object +{ +public: + + ServerLockManager() + { + m_hMutex = F_MUTEX_NULL; + m_pFirstLockWaiter = NULL; + m_pHashTbl = NULL; + m_uiNumAvail = 0; + m_pAvailLockList = NULL; + } + + virtual ~ServerLockManager(); // ServerLockManager Destructor - free + // ServerLockObjects owned by this + // ServerLockManager. + + FINLINE RCODE setupLockMgr( void) + { + return( f_mutexCreate( &m_hMutex)); + } + + RCODE SetupHashTbl(); // Setup hash table for lock manager. + + void CheckLockTimeouts( // See if any pending lock requests have + FLMBOOL bMutexAlreadyLocked, + FLMBOOL bTimeoutAll); // timed out. + + void InsertWaiter( // Insert waiter into global list. + LOCK_WAITER * pLockWaiter); + + FINLINE void RemoveWaiter( + LOCK_WAITER * pLockWaiter) + { + if (pLockWaiter->pNextGlobal) + pLockWaiter->pNextGlobal->pPrevGlobal = pLockWaiter->pPrevGlobal; + + if (pLockWaiter->pPrevGlobal) + { + pLockWaiter->pPrevGlobal->pNextGlobal = pLockWaiter->pNextGlobal; + } + else + { + m_pFirstLockWaiter = pLockWaiter->pNextGlobal; + } + } + + ServerLockObject_p GetLockObject(// Return a lock object for the file. + F_ItemId * pItemId); + + void SignalLockWaiter( // Unlink a lock object from lists + FLMUINT uiThreadId); + + void UnlinkLockObject( // Unlink a lock object from lists + ServerLockObject * pLockObject, + FLMBOOL bPutInAvailList); + + FINLINE void lockMutex( + FLMBOOL bMutexAlreadyLocked) + { + if (m_hMutex != F_MUTEX_NULL && !bMutexAlreadyLocked) + { + f_mutexLock( m_hMutex); + } + } + + FINLINE void unlockMutex( + FLMBOOL bMutexAlreadyLocked) + { + if (m_hMutex != F_MUTEX_NULL && !bMutexAlreadyLocked) + { + f_mutexUnlock( m_hMutex); + } + } + +private: + + // Private variables + + F_MUTEX m_hMutex; + FBUCKET * m_pHashTbl; // Hash table. + LOCK_WAITER * m_pFirstLockWaiter; // Pointer to first in list of global + // lock waiters. + FLMUINT m_uiNumAvail; // Number of lock objects in avail + // list. + ServerLockObject * + m_pAvailLockList; // List of available lock objects. + +friend class F_ServerLockMgrPage; +friend class ServerLockObject; + +}; + +/*=========================================================================== +Class: F_ItemId +Desc: The item id that identifies a particular object. +===========================================================================*/ +class F_ItemId : public F_Object +{ +public: + F_ItemId(); // F_ItemId Constructor + + virtual ~F_ItemId(); // F_ItemId Destructor + + virtual FLMBOOL IsEqual( // Compare to another F_ItemId. + F_ItemId * pItemId) = 0; + + virtual FLMUINT GetHashBucket( // Get hash bucket for lock item id + FBUCKET * pHashTbl, + FLMUINT uiHashTblSize) = 0; + + FLMUINT GetItemType() // Returns the type of item. + { return m_uiItemType; } + +protected: + FLMUINT m_uiItemType; // Item type. +#define FFILE_ITEM 1 +#define RFILE_ITEM 2 +#define FFILE_TRANS_ITEM 3 +#define RFILE_TRANS_ITEM 4 +}; + + + /* + Public: constructor, destructor + */ + FINLINE F_ItemId::F_ItemId() + { + m_uiItemType = 0; + } + + FINLINE F_ItemId::~F_ItemId() + { + } + +/*=========================================================================== +Class: FFileItemId +Desc: The item id that identifies an FFILE object. +===========================================================================*/ +class FFileItemId : public F_ItemId +{ +public: + + // Constructor + + FFileItemId( + F_Database * pDatabase, + FLMBOOL bTrans) + { + m_pDatabase = pDatabase; + m_uiItemType = (FLMUINT)(bTrans + ? (FLMUINT)FFILE_TRANS_ITEM + : (FLMUINT)FFILE_ITEM); + } + + virtual ~FFileItemId() + { + } + + FLMBOOL IsEqual( // Compare to another F_ItemId. + F_ItemId * pItemId); + + FINLINE FLMUINT GetHashBucket( + FBUCKET * pHashTbl, + FLMUINT uiHashTblSize) + { + char szFileName[ F_PATH_MAX_SIZE]; + + // Extract the file name + + this->GetFileName( szFileName); + + // Determine what hash bucket the file should be in - based on file name. + + return( flmStrHashBucket( szFileName, pHashTbl, uiHashTblSize)); + } + + FINLINE F_Database * getDatabase( void) // was GetFilePtr + { + return m_pDatabase; + } + + void GetFileName( // Returns file name + char * pszFileNameRV); + +private: + F_Database * m_pDatabase; +}; + +/*=========================================================================== +Class: RFileItemId +Desc: The item id that identifies a file being used by rebuild. +===========================================================================*/ +class RFileItemId : public F_ItemId +{ +public: + RFileItemId( // RFileItemId Constructor + char * pszFileName, + FLMBOOL bTrans = FALSE); + + virtual ~RFileItemId() // RFileItemId Destructor + { + } + + FLMBOOL IsEqual( // Compare to another F_ItemId. + F_ItemId * pItemId); + + FINLINE FLMUINT GetHashBucket( // Get hash bucket for lock item id + FBUCKET * pHashTbl, + FLMUINT uiHashTblSize) + { + // Determine what hash bucket the file should be in - based on file name. + + return( flmStrHashBucket( m_szFileName, pHashTbl, uiHashTblSize)); + } + + FINLINE void GetFileName( // Returns file name + char * pszFileNameRV) + { + f_strcpy( pszFileNameRV, m_szFileName); + } + +private: + char m_szFileName [F_PATH_MAX_SIZE]; // File's name. +}; + +/*=========================================================================== +Desc: The ServerLockObject is used to lock and unlock a particular + object. +===========================================================================*/ +class ServerLockObject : public F_Object +{ +public: + ServerLockObject(); // ServerLockObject Constructor + + virtual ~ServerLockObject() + { + if( m_pItemId) + { + m_pItemId->Release(); + } + } + + void Setup( + ServerLockManager * pServerLockMgr, + F_ItemId * pItemId, + FLMUINT uiBucket); + + RCODE Lock( + F_Db * pDb, + F_SEM hWaitSem, + FLMBOOL bLogEvent, + FLMBOOL bSendSuspendEvent, // Send suspend event when waiting + // for lock. + FLMBOOL bExclLock, // Exclusive or shared lock? + FLMUINT uiMaxWaitSecs, // Maximum wait time in seconds. + FLMINT iPriority, // Lock priority + SFLM_DB_STATS * pDbStats = NULL); // Place to gather DB stats. + + RCODE Unlock( + FLMBOOL bLogEvent, + F_Db * pDb, // used for event callbacks. + FLMBOOL bRelease = FALSE, // Release object if no one is waiting + SFLM_DB_STATS * pDbStats = NULL); // Place to gather DB stats. + + FLMINT Release( + FLMBOOL bMutexAlreadyLocked); + // Decrement ref count, when it gets + // down to 1, put it in the avail list. + + FLMINT Release( void) + { + return( Release( FALSE)); + } + + void RemoveWaiter( + LOCK_WAITER * pLockWaiter); + + FLMBOOL ThreadWaitingLock( void) + { + return( ((m_pFirstLockWaiter) ? (FLMBOOL)TRUE : (FLMBOOL)FALSE) ); + } + + FLMUINT LockCnt( void) + { + return( m_uiLockCnt); + } + + void GetLockInfo( // Returns lock information + FLMINT iPriority, // A count of all lock requests with + // a priority >= to this will be returned + // in pLockInfo. + eDbLockType * peCurrLockType, + FLMUINT * puiThreadId, + FLMUINT * puiNumExclQueued, + FLMUINT * puiNumSharedQueued, + FLMUINT * puiPriorityCount); + + RCODE GetLockInfo( + IF_LockInfoClient * pLockInfo); + + FLMBOOL haveHigherPriorityWaiter( + FLMINT iPriority); + +private: + + ServerLockManager * m_pServerLockMgr; // Server Lock Manager pointer. + F_ItemId * m_pItemId; // ID for object this lock + // object represents + FLMUINT m_uiLockThreadId; // Thread of thread that has this + // object locked. Zero if none. + FLMUINT m_uiLockTime; // Time lock was granted, if + // exclusive lock. + FLMUINT m_uiLockCnt; // Number of locks that have been + // granted thus far. + LOCK_WAITER * m_pFirstLockWaiter; + // Pointer to first in list of + // lock waiters. + LOCK_WAITER * m_pLastLockWaiter;// Pointer to last in list of + // lock waiters. + FLMUINT m_uiNumWaiters; // Number of threads waiting. + ServerLockObject * m_pNext; // Next in hash bucket or avail list + ServerLockObject * m_pPrev; // Prev in hash bucket or avail list + FLMUINT m_uiSharedLockCnt;// Number of shared locks that have + // been granted. + FLMBOOL m_bExclLock; // Is the granted lock exclusive? + FLMUINT m_uiBucket; // Hash bucket this object is in. + // 0xFFFF means it is in avail list. + F_TMSTAMP m_StartTime; // Time exclusive lock was grabbed. + FLMBOOL m_bStartTimeSet; // Was m_StartTime set? + +friend class ServerLockManager; + +}; + +#endif // FSRVLOCK_H diff --git a/sql/src/fstructs.h b/sql/src/fstructs.h new file mode 100644 index 0000000..7539669 --- /dev/null +++ b/sql/src/fstructs.h @@ -0,0 +1,1377 @@ +//------------------------------------------------------------------------------ +// Desc: Various structures and classes used internally by FLAIM. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#ifndef FSTRUCTS_H +#define FSTRUCTS_H + +#if defined( FLM_WIN) || defined( FLM_NLM) || defined( FLM_LINUX) + #pragma pack( push, 1) +#else + #pragma pack( 1) +#endif + +class HRequest; +class F_BtPool; +class F_FileIdList; +class F_Rfl; +class F_SuperFileHdl; +class F_Btree; +class F_DbRebuild; +class F_DbCheck; +class ServerLockManager; +class ServerLockObject; +class F_FileHdlMgr; +class F_Cursor; +class F_MultiAlloc; +class F_Row; +class F_CachedBlock; +class F_BlockCacheMgr; +class F_RowCacheMgr; +class F_GlobalCacheMgr; +class F_BTreeInfo; +class F_RowRelocator; +class F_ColumnDataRelocator; +class F_ColumnListRelocator; +class F_BTreeIStreamPool; +class SQLQuery; + +/**************************************************************************** +Desc: Tests to see if database is NOT in native platform format. +****************************************************************************/ +FINLINE FLMBOOL hdrIsNonNativeFormat( + SFLM_DB_HDR * pDbHdr) +{ + return( (FLMBOOL)(pDbHdr->ui8IsLittleEndian == + SFLM_NATIVE_IS_LITTLE_ENDIAN + ? (FLMBOOL)FALSE + : (FLMBOOL)TRUE)); +} + +/**************************************************************************** +Desc: Block header - on-disk format. +****************************************************************************/ +typedef struct FlmBlockHdrTag +{ + FLMUINT32 ui32BlkAddr; // BH_ADDR + FLMUINT32 ui32PrevBlkInChain; // BH_PREV_BLK + FLMUINT32 ui32NextBlkInChain; // BH_NEXT_BLK + FLMUINT32 ui32PriorBlkImgAddr; // BH_PREV_BLK_ADDR + FLMUINT64 ui64TransID; // BH_TRANS_ID + FLMUINT32 ui32BlkCRC; // Block CRC + FLMUINT16 ui16BlkBytesAvail; // BH_BLK_END, BH_ELM_END + FLMUINT8 ui8BlkFlags; // Flags for the block + #define BLK_FORMAT_IS_LITTLE_ENDIAN 0x01 + #define BLK_IS_BEFORE_IMAGE 0x02 + // This bit gets ORed into type if the + // block is a Before Image block that + // should be restored on transaction + // abort. This is only set when a block + // is written to the log, so it only + // needs to be unset when the block is + // read back from the log. + #define BLK_IS_ENCRYPTED 0x04 + + + FLMUINT8 ui8BlkType; // BH_TYPE + #define BT_FREE 0 // Free block - avail list + #define BT_LFH_BLK 1 // LFH Header block + #define BT_LEAF 2 // New B-Tree Leaf block + #define BT_NON_LEAF 3 // New B-Tree Non-leaf block block - fixed key size + #define BT_NON_LEAF_COUNTS 4 // New B-Tree Non-leaf index with counts + #define BT_LEAF_DATA 5 // New B-Tree Leaf block with Data + #define BT_DATA_ONLY 6 // Data-only block + // NOTE: IF adding more types, may need to modify the blkIsNewBTree function + // below. + + // IMPORTANT NOTE: If anything is changed in here, need to make + // corresponding changes to convertBlkHdr routine and + // flmVerifyDiskStructOffsets routine. + +#define F_BLK_HDR_ui32BlkAddr_OFFSET 0 +#define F_BLK_HDR_ui32PrevBlkInChain_OFFSET 4 +#define F_BLK_HDR_ui32NextBlkInChain_OFFSET 8 +#define F_BLK_HDR_ui32PriorBlkImgAddr_OFFSET 12 +#define F_BLK_HDR_ui64TransID_OFFSET 16 +#define F_BLK_HDR_ui32BlkCRC_OFFSET 24 +#define F_BLK_HDR_ui16BlkBytesAvail_OFFSET 28 +#define F_BLK_HDR_ui8BlkFlags_OFFSET 30 +#define F_BLK_HDR_ui8BlkType_OFFSET 31 +} F_BLK_HDR; + +#define SIZEOF_STD_BLK_HDR sizeof( F_BLK_HDR) + +/**************************************************************************** +Desc: Test to see if block is in non-native format. +****************************************************************************/ +FINLINE FLMBOOL blkIsNonNativeFormat( + F_BLK_HDR * pBlkHdr) +{ +#if SFLM_NATIVE_IS_LITTLE_ENDIAN + return( (pBlkHdr->ui8BlkFlags & BLK_FORMAT_IS_LITTLE_ENDIAN) + ? FALSE + : TRUE); +#else + return( (pBlkHdr->ui8BlkFlags & BLK_FORMAT_IS_LITTLE_ENDIAN) + ? TRUE + : FALSE); +#endif +} + +/**************************************************************************** +Desc: Set block flags to indicate it is in native format. +****************************************************************************/ +FINLINE void blkSetNativeFormat( + F_BLK_HDR * pBlkHdr) +{ +#if SFLM_NATIVE_IS_LITTLE_ENDIAN + pBlkHdr->ui8BlkFlags |= BLK_FORMAT_IS_LITTLE_ENDIAN; +#else + pBlkHdr->ui8BlkFlags &= ~(BLK_FORMAT_IS_LITTLE_ENDIAN); +#endif +} + +/**************************************************************************** +Desc: Test to see if a block type is a B-Tree block type. +****************************************************************************/ +FINLINE FLMBOOL blkIsBTree( + F_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BlkType != BT_FREE && + pBlkHdr->ui8BlkType != BT_LFH_BLK) + ? TRUE + : FALSE); +} + +/**************************************************************************** +Desc: Test to see if a block type is a NEW B-Tree block type. +****************************************************************************/ +FINLINE FLMBOOL blkIsNewBTree( + F_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BlkType >= BT_LEAF) + ? TRUE + : FALSE); +} + +/**************************************************************************** +Desc: Determine where the block ends. +****************************************************************************/ +FINLINE FLMUINT blkGetEnd( + FLMUINT uiBlockSize, + FLMUINT uiBlkHdrSize, + F_BLK_HDR * pBlkHdr + ) +{ + return( (FLMUINT)(blkIsNewBTree( pBlkHdr) + ? uiBlockSize + : (FLMUINT)(pBlkHdr->ui16BlkBytesAvail > + uiBlockSize - uiBlkHdrSize + ? uiBlkHdrSize + : uiBlockSize - + (FLMUINT)pBlkHdr->ui16BlkBytesAvail))); +} + +FINLINE void setBlockEncrypted( + F_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BlkFlags |= BLK_IS_ENCRYPTED; +} + +FINLINE void unsetBlockEncrypted( + F_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BlkFlags &= (~(BLK_IS_ENCRYPTED)); +} + +FINLINE FLMBOOL isEncryptedBlk( + F_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BlkFlags & BLK_IS_ENCRYPTED) ? TRUE : FALSE); +} + +/**************************************************************************** +Desc: B-Tree block header - on-disk format. +****************************************************************************/ +typedef struct FlmBTreeBlkHdr +{ + F_BLK_HDR stdBlkHdr; // Standard block header + FLMUINT16 ui16LogicalFile; // BH_LOG_FILE_NUM + FLMUINT16 ui16NumKeys; // Number of keys + FLMUINT8 ui8BlkLevel; // BH_LEVEL + #define BH_MAX_LEVELS 8 // Max allowable b-tree levels + #define MAX_LEVELS BH_MAX_LEVELS + FLMUINT8 ui8BTreeFlags; // Flags for BTree + #define BLK_IS_ROOT 0x01 + #define BLK_IS_INDEX 0x02 + FLMUINT16 ui16HeapSize; // Contiguous available space +#define F_BTREE_BLK_HDR_stdBlkHdr_OFFSET 0 +#define F_BTREE_BLK_HDR_ui16LogicalFile_OFFSET 32 +#define F_BTREE_BLK_HDR_ui16NumKeys_OFFSET 34 +#define F_BTREE_BLK_HDR_ui8BlkLevel_OFFSET 36 +#define F_BTREE_BLK_HDR_ui8BTreeFlags_OFFSET 37 +#define F_BTREE_BLK_HDR_ui16HeapSize_OFFSET 38 +} F_BTREE_BLK_HDR; + +/**************************************************************************** +Desc: Encrypted B-Tree block header - on-disk format. +****************************************************************************/ +typedef struct +{ + F_BTREE_BLK_HDR btree; + FLMUINT64 ui64Reserved; // Reserving 8 bytes to ensure the data segment + // of the block is sized to an even 16 byte boundary + // as needed by encryption. +} F_ENC_BTREE_BLK_HDR; + +FINLINE FLMUINT sizeofBTreeBlkHdr( + F_BTREE_BLK_HDR * pBlkHdr) +{ + if (!isEncryptedBlk( (F_BLK_HDR *)pBlkHdr)) + { + return sizeof( F_BTREE_BLK_HDR); + } + else + { + return sizeof( F_ENC_BTREE_BLK_HDR); + } +} + +/**************************************************************************** +Desc: Encrypted Data-only block header - on-disk format. +****************************************************************************/ +typedef struct +{ + F_BLK_HDR blk; + FLMUINT32 ui32EncDefNum; + FLMBYTE ucReserved[12]; // Reserving 12 bytes to ensure the data segment + // of the block is sized to an even 16 byte boundary + // as needed by encryption. +} F_ENC_DO_BLK_HDR; + +FINLINE FLMUINT sizeofDOBlkHdr( + F_BLK_HDR * pBlkHdr + ) +{ + if (!isEncryptedBlk( pBlkHdr)) + { + return sizeof( F_BLK_HDR); + } + else + { + return sizeof( F_ENC_DO_BLK_HDR); + } +} + +FINLINE FLMBOOL isRootBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BTreeFlags & BLK_IS_ROOT) ? TRUE : FALSE); +} + +FINLINE void setRootBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BTreeFlags |= BLK_IS_ROOT; +} + +FINLINE void unsetRootBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BTreeFlags &= (~(BLK_IS_ROOT)); +} + +FINLINE FLMBOOL isIndexBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BTreeFlags & BLK_IS_INDEX) ? TRUE : FALSE); +} + +FINLINE eLFileType getBlkLfType( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + return( (eLFileType)((pBlkHdr->ui8BTreeFlags & BLK_IS_INDEX) + ? (eLFileType)SFLM_LF_INDEX + : (eLFileType)SFLM_LF_TABLE)); +} + +FINLINE void setBlkLfType( + F_BTREE_BLK_HDR * pBlkHdr, + FLMUINT uiLfType + ) +{ + if (uiLfType == SFLM_LF_TABLE) + { + pBlkHdr->ui8BTreeFlags &= (~(BLK_IS_INDEX)); + } + else + { + pBlkHdr->ui8BTreeFlags |= BLK_IS_INDEX; + } +} + +FINLINE FLMBOOL isTableBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BTreeFlags & BLK_IS_INDEX) ? FALSE : TRUE); +} + +FINLINE void setIndexBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BTreeFlags |= BLK_IS_INDEX; +} + +FINLINE void setContainerBlk( + F_BTREE_BLK_HDR * pBlkHdr + ) +{ + pBlkHdr->ui8BTreeFlags &= (~(BLK_IS_INDEX)); +} + +/**************************************************************************** +Desc: Get block header size. +****************************************************************************/ +FINLINE FLMUINT blkHdrSize( + F_BLK_HDR * pBlkHdr + ) +{ + return( (pBlkHdr->ui8BlkType != BT_FREE && + pBlkHdr->ui8BlkType != BT_LFH_BLK) + ? (pBlkHdr->ui8BlkType == BT_DATA_ONLY + ? sizeofDOBlkHdr( pBlkHdr) + : sizeofBTreeBlkHdr((F_BTREE_BLK_HDR *)pBlkHdr)) + : SIZEOF_STD_BLK_HDR); +} + +/**************************************************************************** +Desc: This is a union of all block header types - so that we can have + something that gives us the largest block header type in one + structure. Reduce uses this. +****************************************************************************/ +typedef struct FlmLargestBlkHdr +{ + union + { + F_BLK_HDR stdBlkHdr; + F_BTREE_BLK_HDR BTreeBlkHdr; + } all; +} F_LARGEST_BLK_HDR; + +#define SIZEOF_LARGEST_BLK_HDR sizeof( F_LARGEST_BLK_HDR) + +/**************************************************************************** +Desc: Logical File (b-tree) header - on-disk format. +****************************************************************************/ +typedef struct F_LF_HDR +{ + FLMUINT64 ui64NextRowId; // Only used for tables. + FLMUINT32 ui32LfType; + FLMUINT32 ui32RootBlkAddr; + FLMUINT32 ui32LfNum; + FLMUINT32 ui32EncDefNum; + FLMBYTE ucZeroes[ 40]; // Reserve 40 bytes of zero. + + // IMPORTANT NOTE: If anything is changed in here, need to make + // corresponding changes to convertLfHdr routine and + // flmVerifyDiskStructOffsets routine. + +#define F_LF_HDR_ui64NextRowId_OFFSET 0 +#define F_LF_HDR_ui32LfType_OFFSET 8 +#define F_LF_HDR_ui32RootBlkAddr_OFFSET 12 +#define F_LF_HDR_ui32LfNum_OFFSET 16 +#define F_LF_HDR_ui32EncDefNum_OFFSET 20 +#define F_LF_HDR_ucZeroes_OFFSET 24 +} F_LF_HDR; + +/*************************************************************************** +Desc: This is the notify request structure. Notify requests are linked + off of open requests for files or read requests for files so that + when an operation is complete that multiple threads are waiting + on, all of them will be notified. +***************************************************************************/ +typedef struct FNotify +{ + FNotify * pNext; // Pointer to next FNOTIFY structure in list. + FLMUINT uiThreadId; // ID of thread requesting the notify + RCODE * pRc; // Pointer to a return code variable that is to + // be filled in when the operation is completed. + // The thread requesting notification supplies + // the return code variable to be filled in. + void * pvUserData; // Other user data that the notifier might use + // to transfer other information to the waiter. + F_SEM hSem; // Semaphore that will be signaled when the + // operation is complete. +} FNOTIFY; + +// Flags for the uiKrAction parameter - used in sorting/indexing. + +#define KREF_DEL_KEYS 0x01 +#define KREF_ADD_KEYS 0x02 +#define KREF_INDEXING_ONLY 0x04 +#define KREF_IN_MODIFY 0x10 +#define KREF_MISSING_KEYS_OK 0x20 + +// Maximum length of a key. + +#define MAX_ID_SIZE 256 // Cannot be more than 256 because + // we can only use one byte to + // represent the total ID size. + +#define DEFAULT_KREF_TBL_SIZE 4096 +#define DEFAULT_KREF_POOL_BLOCK_SIZE 8192 + +/**************************************************************************** +Desc: This structure is used to sort keys before the keys are actually + added to an index. +****************************************************************************/ +typedef struct Kref_Entry +{ + FLMBOOL bDelete; // Delete the key if TRUE + FLMUINT uiSequence; // Sequence of updates within trans. + FLMUINT uiDataLen; // Data length for this entry. The + // data, if any, comes after the key. + // Note that there will be a null + // terminating byte between the key and + // the data. + + // Note: used uint16 below to reduce memory allocations. + + FLMUINT16 ui16IxNum; // Index number + FLMUINT16 ui16KeyLen; // Key Length for this entry. The key + // comes immediately after this structure. + F_Row * pRow; // Row data came from. +} KREF_ENTRY; + +typedef struct IXD_FIXUP +{ + FLMUINT uiIndexNum; + FLMUINT64 ui64LastRowIndexed; + IXD_FIXUP * pNext; +} IXD_FIXUP; + +#define FTHREAD_ACTION_IDLE 0 +#define FTHREAD_ACTION_INDEX_OFFLINE 1 + +/*************************************************************************** +Desc: Contains elements for passing parms into the background thread. +***************************************************************************/ +typedef struct F_BkgndIx +{ + F_Database * pDatabase; + FLMUINT uiIndexingAction; + SFLM_INDEX_STATUS indexStatus; + F_BkgndIx * pPrev; + F_BkgndIx * pNext; +} F_BKGND_IX; + +/**************************************************************************** +Desc: Structure used to pass information to the checkpoint thread for 3.x + databases. +****************************************************************************/ +typedef struct +{ + F_Database * pDatabase; + F_SuperFileHdl * pSFileHdl; + F_SEM hWaitSem; + SFLM_STATS Stats; + FLMBOOL bStatsInitialized; + FLMBOOL bShuttingDown; + FLMBOOL bDoingCheckpoint; + FLMUINT uiStartTime; + FLMBOOL bForcingCheckpoint; + FLMUINT uiForceCheckpointStartTime; + eForceCPReason eForceCheckpointReason; + FLMUINT uiLogBlocksWritten; + FLMBOOL bWritingDataBlocks; + FLMUINT uiDataBlocksWritten; + FLMUINT uiStartWaitTruncateTime; +} CP_INFO; + +#define MAX_WRITE_BUFFER_BYTES (4 * 1024 * 1024) +#define MAX_PENDING_WRITES (MAX_WRITE_BUFFER_BYTES / 4096) +#define MAX_LOG_BUFFER_SIZE (256 * 1024) + +typedef struct TMP_READ_STATS +{ + SFLM_DISKIO_STAT BlockReads; // Statistics on block reads. + SFLM_DISKIO_STAT OldViewBlockReads; // Statistics on old view block + // reads. + FLMUINT uiBlockChkErrs; // Number of times we had + // check errors reading blocks. + FLMUINT uiOldViewBlockChkErrs; // Number of times we had + // check errors reading an + // old view of a block. +} TMP_READ_STATS; + +// Flags for F_Database->m_uiFlags + +#define DBF_BEING_OPENED 0x01 // Flag indicating whether this database is + // in the process of being opened. +#define DBF_BEING_CLOSED 0x02 // Database is being closed - cannot open. + +/***************************************************************************** +Desc: Shared database object - only to be used internally +*****************************************************************************/ +class F_Database : F_Object +{ +public: + + F_Database( + FLMBOOL bTempDb); + + ~F_Database(); + + void freeDatabase( void); + + RCODE setupDatabase( + const char * pszDbPath, + const char * pszDataDir); + + RCODE dbWriteLock( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats = NULL, + FLMUINT uiTimeout = SFLM_NO_TIMEOUT); + + void dbWriteUnlock( + SFLM_DB_STATS * pDbStats = NULL); + + void shutdownDatabaseThreads( void); + + RCODE linkToBucket( void); // was flmLinkFileToBucket + + void setMustCloseFlags( // was flmSetMustCloseFlags + RCODE rcMustClose, + FLMBOOL bMutexLocked); + + FINLINE RCODE checkState( + const char * pszFileName, + FLMINT iLineNumber) + + { + RCODE rc = NE_SFLM_OK; + + if (m_bMustClose) + { + logMustCloseReason( pszFileName, iLineNumber); + rc = RC_SET( NE_SFLM_MUST_CLOSE_DATABASE); + } + return( rc); + } + + RCODE startCPThread( void); + + FLMBOOL tryCheckpoint( + IF_Thread * pThread, + CP_INFO * pCPInfo); + + void newDatabaseFinish( + RCODE OpenRc); + + RCODE getExclAccess( + const char * pszFilePath); + + RCODE verifyOkToUse( + FLMBOOL * pbWaited); + + RCODE writeDbHdr( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + SFLM_DB_HDR * pDbHdr, + SFLM_DB_HDR * pCPDbHdr, + FLMBOOL bIsCheckpoint); + + FINLINE char * getDbNamePtr( void) + { + return m_pszDbPath; + } + + FINLINE SFLM_DB_HDR * getUncommittedDbHdr( void) + { + return &m_uncommittedDbHdr; + } + + FINLINE FLMUINT getBlockSize( void) + { + return( m_uiBlockSize); + } + + FINLINE FLMUINT getMaxFileSize( void) + { + return( m_uiMaxFileSize); + } + + FINLINE FLMUINT getSigBitsInBlkSize( void) + { + return m_uiSigBitsInBlkSize; + } + + FINLINE F_CachedBlock * getTransLogList( void) + { + return m_pTransLogList; + } + + void releaseLogBlocks( void); + + FINLINE FLMUINT getDirtyCacheCount( void) + { + return m_uiDirtyCacheCount; + } + + FINLINE void incrementDirtyCacheCount( void) + { + m_uiDirtyCacheCount++; + } + + FINLINE void decrementDirtyCacheCount( void) + { + m_uiDirtyCacheCount--; + } + + FLMBOOL neededByReadTrans( + FLMUINT64 ui64LowTransId, + FLMUINT64 ui64HighTransId); + + RCODE getBlock( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiBlkAddress, + FLMUINT * puiNumLooks, + F_CachedBlock ** ppSCache); + + RCODE logPhysBlk( + F_Db * pDb, + F_CachedBlock ** ppSCache, + F_CachedBlock ** ppOldCache = NULL); + + RCODE createBlock( + F_Db * pDb, + F_CachedBlock ** ppSCache); + + RCODE blockUseNextAvail( + F_Db * pDb, + F_CachedBlock ** ppSCache); + + RCODE blockFree( + F_Db * pDb, + F_CachedBlock * pSCache); + + RCODE moveBtreeBlk( + F_Db * pDb, + FLMUINT uiBlkAddr, + FLMUINT uiLfNumber, + eLFileType eLfType); + + RCODE freeAvailBlk( + F_Db * pDb, + FLMUINT uiBlkAddr); + + RCODE moveLFHBlk( + F_Db * pDb, + FLMUINT uiBlkAddr); + + void unlinkTransLogBlocks( void); + + void freeBlockCache( void); + + void freeRowCache( void); + + void freeModifiedBlocks( + FLMUINT64 ui64CurrTransId); + + void freeModifiedRows( + F_Db * pDb, + FLMUINT64 ui64OlderTransId); + + void commitRowCache( void); + + void getCPInfo( + SFLM_CHECKPOINT_INFO * pCheckpointInfo); + + FINLINE FLMUINT getFlags( void) + { + return( m_uiFlags); + } + + // NOTE: This routine expects that the global mutex is locked when + // it is called. + + FINLINE void incrOpenCount( void) + { + m_uiOpenIFDbCount++; + } + + // NOTE: This routine expects that the global mutex is locked when + // it is called. It may temporarily unlock the mutex (when it + // calls freeDatabase), but the mutex will be locked when it returns. + + FINLINE void decrOpenCount( void) + { + flmAssert( m_uiOpenIFDbCount); + m_uiOpenIFDbCount--; + if (!m_uiOpenIFDbCount) + { + freeDatabase(); + } + } + + RCODE startMaintThread( void); + + FINLINE FLMBOOL inLimitedMode() + { + return m_bInLimitedMode; + } + + RCODE encryptBlock( + F_Dict * pDict, + FLMBYTE * pucBuffer); + + RCODE decryptBlock( + F_Dict * pDict, + FLMBYTE * pucBuffer); + + FINLINE void lockMutex( void) + { + f_mutexLock( m_hMutex); + } + + FINLINE void unlockMutex( void) + { + f_mutexUnlock( m_hMutex); + } + +private: + + void logMustCloseReason( + const char * pszFileName, + FLMINT iLineNumber); + + RCODE readDbHdr( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBYTE * pszPassword, + FLMBOOL bAllowLimited); + + RCODE physOpen( + F_Db * pDb, + const char * pszFilePath, + const char * pszRflDir, + const char * pszPassword, + FLMUINT uiOpenFlags, + FLMBOOL bNewDatabase, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus); + + RCODE doRecover( + F_Db * pDb, + IF_RestoreClient * pRestoreObj, + IF_RestoreStatus * pRestoreStatus); + + RCODE outputRow( + FLMBYTE * pucKeyBuf, + F_Row * pRow, + FLMBOOL bAdd, + F_Btree * pBTree); + + RCODE readTheBlock( + F_Db * pDb, + TMP_READ_STATS * pTmpReadStats, + F_BLK_HDR * pBlkHdr, + FLMUINT uiFilePos, + FLMUINT uiBlkAddress); + + RCODE readBlock( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiFilePos, + FLMUINT uiBlkAddress, + FLMUINT64 ui64NewerBlkLowTransID, + F_CachedBlock * pSCache, + FLMBOOL * pbFoundVerRV, + FLMBOOL * pbDiscardRV); + + RCODE readIntoCache( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiBlkAddress, + F_CachedBlock * pPrevInVerList, + F_CachedBlock * pNextInVerList, + F_CachedBlock ** ppSCacheRV, + FLMBOOL * pbGotFromDisk); + + void setBlkDirty( + F_CachedBlock * pSCache); + + RCODE allocBlocksArray( + FLMUINT uiNewSize, + FLMBOOL bOneArray); + + RCODE flushLogBlocks( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bIsCPThread, + FLMUINT uiMaxDirtyCache, + FLMBOOL * pbForceCheckpoint, + FLMBOOL * pbWroteAll); + + RCODE reduceNewBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT * puiBlocksFlushed); + + RCODE writeContiguousBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + IF_IOBuffer * pIOBuffer, + FLMUINT uiBlkAddress, + FLMBOOL bDoAsync); + + RCODE writeSortedBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT uiMaxDirtyCache, + FLMUINT * puiDirtyCacheLeft, + FLMBOOL * pbForceCheckpoint, + FLMBOOL bIsCPThread, + FLMBOOL bDoAsync, + FLMUINT uiNumSortedBlocks, + FLMBOOL * pbWroteAll); + + RCODE flushDirtyBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT uiMaxDirtyCache, + FLMBOOL bForceCheckpoint, + FLMBOOL bIsCPThread, + FLMBOOL * pbWroteAll); + + RCODE reduceDirtyCache( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl); + + RCODE finishCheckpoint( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoTruncate, + FLMUINT uiCPFileNum, + FLMUINT uiCPOffset, + FLMUINT uiCPStartTime, + FLMUINT uiTotalToWrite); + + RCODE doCheckpoint( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoTruncate, + FLMBOOL bForceCheckpoint, + eForceCPReason eForceReason, + FLMUINT uiCPFileNum, + FLMUINT uiCPOffset); + + RCODE lgFlushLogBuffer( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoAsync); + + RCODE lgOutputBlock( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + F_CachedBlock * pLogBlock, + F_BLK_HDR * pBlkHdr, + FLMBOOL bDoAsync, + FLMUINT * puiLogEofRV); + + FLMUINT lFileFindEmpty( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr); + + RCODE lFileRead( + F_Db * pDb, + LFILE * pLFile); + + RCODE lFileWrite( + F_Db * pDb, + LFILE * pLFile); + + RCODE lFileCreate( + F_Db * pDb, + LFILE * pLFile, + FLMUINT uiLfNum, + eLFileType eLfType, + FLMBOOL bCounts, + FLMBOOL bHaveData, + FLMUINT uiEncDefNum); + + RCODE lFileDelete( + F_Db * pDb, + LFILE * pLFile, + FLMBOOL bCounts, + FLMBOOL bHaveData); + + static RCODE maintenanceThread( + IF_Thread * pThread); + + F_Database * m_pNext; // Next F_Database structure in in name hash + // bucket, dependent store hash + // bucket, or avail list. + F_Database * m_pPrev; // Previous F_Database structure in name hash + // bucket or dependent store hash + // bucket. + SQLQuery * m_pFirstSQLQuery; // First query currently running on this database + SQLQuery * m_pLastSQLQuery; // Last query currently running on this database + FLMUINT m_uiBlockSize; // Block size for database + FLMUINT m_uiDefaultLanguage; // Default language for database. + FLMUINT m_uiMaxFileSize; // Maximum file size for the database. + FLMUINT m_uiOpenIFDbCount; // Number of F_Dbs currently using this + // database. Does NOT count internal uses + FLMBOOL m_bTempDb; // Is this a temporary database? If so, + // minimize writing out to disk. + F_Db * m_pFirstDb; // List of ALL F_Db's associated with + // this database. + char * m_pszDbPath; // Database file name. + char * m_pszDataDir; // Path for data files. + F_Database * m_pNextNUDatabase; // Next F_Database structure in list of + // unused databases. When use count goes + // to zero, the structure is linked + // into a list of unused databases off of + // the FSYSDATA structure. + F_Database * m_pPrevNUDatabase; // Previous F_Database structure in list of + // unused databases. + F_CachedBlock * m_pSCacheList; // This is a pointer to a linked list + // of all shared cache blocks + // belonging to this database. + F_Row * m_pFirstRow; // First cached row that belongs + // to this database. Also points to + // the first dirty row, if any. + F_Row * m_pLastRow; // Last cached row that belongs + // to this database. + F_Row * m_pLastDirtyRow; // Last dirty row that belongs + // to this database. + F_CachedBlock * m_pPendingWriteList; // This is a pointer to a linked list + // of all shared cache blocks + // that are in the pending-write state. + F_CachedBlock * m_pLastDirtyBlk; // Pointer to last dirty block in the + // list. + F_CachedBlock * m_pFirstInLogList; // First block that needs to be logged + F_CachedBlock * m_pLastInLogList; // Last block that needs to be logged + FLMUINT m_uiLogListCount; // Number of items in the log list + F_CachedBlock * m_pFirstInNewList; // First new block that is dirty + F_CachedBlock * m_pLastInNewList; // Last new block that is dirty + FLMUINT m_uiNewCount; // Number of items in new list + FLMUINT m_uiDirtyCacheCount; // Number of dirty blocks + FLMUINT m_uiLogCacheCount; // Log blocks needing to be written. + F_CachedBlock ** m_ppBlocksDone; // List of blocks to be written to rollback + // log or database. + FLMUINT m_uiBlocksDoneArraySize;// Size of ppBlocksDone array. + FLMUINT m_uiBlocksDone; // Number of blocks currently in the + // ppBlocksDone array. + F_CachedBlock * m_pTransLogList; // This is a pointer to a linked list + // of all shared cache blocks + // belonging to this database that need + // to be logged to the rollback log + // for the current transaction. + FNOTIFY * m_pOpenNotifies; // Pointer to a list of notifies to + // perform when this database is finally + // opened (points to a linked list of + // FNOTIFY structures). + FNOTIFY * m_pCloseNotifies; // Pointer to a list of notifies to + // perform when this database is finally + // closed (points to a linked list of + // FNOTIFY structures). + F_Dict * m_pDictList; // Pointer to linked list of + // dictionaries currently being used + // for this database. The linked list + // is a list of versions of the + // dictionary. When a version is no + // longer used, it is removed from the + // list. Hence, the list is usually + // has only one member. + FLMBOOL m_bMustClose; // The database is being forced to close + // because of a critical error. + RCODE m_rcMustClose; // Return code that caused bMustClose to + // be set. + F_Pool m_krefPool; // Kref pool to be used during update + // transactions. + FLMUINT m_uiSigBitsInBlkSize;// Significant bits in the database's + // block size. + FLMUINT m_uiFileExtendSize; // Bytes to extend files by. + F_Rfl * m_pRfl; // Pointer RFL object. + + + SFLM_DB_HDR m_lastCommittedDbHdr;// This is the last committed DBheader. + SFLM_DB_HDR m_checkpointDbHdr; // This is the DB header as of the start + // of the last checkpoint. + SFLM_DB_HDR m_uncommittedDbHdr; // This is the uncommitted DB header. + // It is used by the current update + // transaction. + F_FileIdList * m_pFileIdList; // List of unique IDs that have been + // assigned to the physical files that + // are mananaged by the FFILE. + IF_IOBufferMgr * m_pBufferMgr; + + IF_IOBuffer * m_pCurrLogBuffer; + FLMUINT m_uiCurrLogWriteOffset; // Offset in current write buffer + FLMUINT m_uiCurrLogBlkAddr; + // Address of first block in the current + // buffer. + SFLM_DB_HDR * m_pDbHdrWriteBuf; // Aligned buffer (on win32) for writing + // the DB header. + FLMBYTE * m_pucUpdBuffer; // Buffer for writing out records. + FLMUINT m_uiUpdBufferSize; // Size of update buffer. + FLMBYTE m_ucIV [16]; // Used when outputting encrypted data. + F_Btree * m_pPendingBTree; // B-Tree used by row that is having + // its value set across multiple calls + FLMBOOL m_bUpdFirstBuf; + FLMUINT m_uiUpdByteCount; + FLMUINT m_uiUpdCharCount; + eDataType m_ePendingDataType; + FLMBYTE * m_pucBTreeTmpBlk; // Temporary buffer for F_Btree object + // to use during a call that updates + // a B-Tree - btInsertEntry, + // btReplaceEntry, etc. SHOULD NOT + // be used in read/find operations! + FLMBYTE * m_pucBTreeTmpDefragBlk; + FLMBYTE * m_pucEntryArray; // Temporary buffer for F_Btree object + // to use during update operations. + FLMBYTE * m_pucSortedArray; // Temporary buffer for F_Btree object + // to use during update operations. + FLMBYTE * m_pucBtreeBuffer; // Buffer used by the Btree during moves + // between blocks. + FLMBYTE * m_pucReplaceStruct; // Buffer used by the Btree to hold additional + // replace information during updates *only*. + ServerLockObject * m_pDatabaseLockObj; // Object for locking the database. + ServerLockObject * m_pWriteLockObj; // Object for locking to do writing. + IF_FileHdl * m_pLockFileHdl; // Lock file handle. + FNOTIFY * m_pLockNotifies; // Pointer to a list of notifies to + // perform when this database is finally + // locked (points to a linked list of + // FNOTIFY structures). + FLMBOOL m_bBeingLocked; // Flag indicating whether or not this + // database is in the process of being + // locked for exclusive access. + F_Db * m_pFirstReadTrans; // Pointer to first read transaction for + // this database. + F_Db * m_pLastReadTrans; // Pointer to last read transaction for + // this database. + F_Db * m_pFirstKilledTrans; // List of read transactions that have + // been killed. + FLMUINT m_uiFirstLogBlkAddress; + // Address of first block logged for the + // current update transaction. + + FLMUINT m_uiFirstLogCPBlkAddress; + // Address of first block logged for the + // current checkpoint. + FLMUINT m_uiLastCheckpointTime; + // Last time we successfully completed a + // checkpoint. + IF_Thread * m_pCPThrd; // Checkpoint thread. + CP_INFO * m_pCPInfo; // Pointer to checkpoint thread's + // information buffer - used for + // communicating information to the + // checkpoint thread. + RCODE m_CheckpointRc; // Return code from last checkpoint + // that was attempted. + FLMUINT m_uiBucket; // Hash bucket this database is in. + // 0xFFFF means it is not currently + // in a bucket. + FLMUINT m_uiFlags; // Flags for this database. + FLMBOOL m_bBackupActive; // Backup is currently being run against the + // database. + IF_Thread * m_pMaintThrd; // Background maintenance thread + F_SEM m_hMaintSem; // Maintenance thread "work-to-do" semaphore + FLMBYTE * m_pszDbPasswd; // The database encryption password + F_CCS * m_pWrappingKey; // The database wrapping key + FLMBOOL m_bHaveEncKey; // + FLMBOOL m_bAllowLimitedMode; // Is this database allowed to be opened in limited mode? + FLMBOOL m_bInLimitedMode; // Has this database been opened in limited mode? + RCODE m_rcLimitedCode; + F_MUTEX m_hMutex; // Database mutex. + +friend class F_Db; +friend class F_Rfl; +friend class F_Btree; +friend class F_Dict; +friend class F_DbSystem; +friend class F_Backup; +friend class FFileItemId; +friend class F_Row; +friend class F_BTreeIStream; +friend class F_DbRebuild; +friend class F_DbCheck; +friend class F_Query; +friend class F_BtResultSet; +friend class F_BtRSFactory; +friend class FSIndexCursor; +friend class FSTableCursor; +friend class F_CachedBlock; +friend class F_BlockCacheMgr; +friend class F_RowCacheMgr; +friend class F_GlobalCacheMgr; +friend class F_QueryResultSet; +friend class F_BTreeInfo; +friend class F_RowRelocator; +friend class F_ColumnDataRelocator; +friend class F_ColumnListRelocator; +friend class F_BlockRelocator; +friend class SQLQuery; +}; + +/*************************************************************************** +Desc: This is the hash bucket header structure. Each bucket header + points to a list of items that belong to the bucket. +***************************************************************************/ +typedef struct FBucket +{ + void * pFirstInBucket; // Pointer to first item in the bucket. + // The type of structure being pointed to + // depends on the usage of the hash bucket. + FLMUINT uiHashValue; // Hash value for this bucket. +} FBUCKET; + +typedef struct QUERY_HDR +{ + F_Query * pQuery; + QUERY_HDR * pNext; + QUERY_HDR * pPrev; +} QUERY_HDR; + +/*************************************************************************** +Desc: This is the FLAIM Event Structure. It keeps track of a registered + event callback function that has been registered for a particular + event category. +***************************************************************************/ +typedef struct F_Event +{ + IF_EventClient * pEventClient; + F_Event * pNext; + F_Event * pPrev; +} FEVENT; + +/*************************************************************************** +Desc: This is the FLAIM Event Header Structure. It is the header for + the list of events that have been registered for a particular + event category. +***************************************************************************/ +typedef struct FEVENT_HDR +{ + FEVENT * pEventCBList; // List of registered event callbacks. + F_MUTEX hMutex; // Mutex to control access to the + // the event list. +} FEVENT_HDR; + +/*************************************************************************** +Desc: This is the FLAIM Shared System Data Structure. It is the anchor + for all of the other shared structures. +***************************************************************************/ +typedef struct FLMSYSDATA +{ + FBUCKET * pDatabaseHashTbl; // Database name hash table (array of FBUCKET). +#define FILE_HASH_ENTRIES 256 + + F_MUTEX hShareMutex; // Mutex for controlling access to + // various global items. + F_MUTEX hRowCacheMutex; + // Mutex for controlling access to + // row cache. + F_MUTEX hBlockCacheMutex; + // Mutex for controlling access to + // block cache. + F_FileHdlMgr * pFileHdlMgr; // Used to Manage all FileHdl objects + + FLMBOOL bTempDirSet; // TRUE if temporary directory has been set + + FLMBOOL bOkToDoAsyncWrites; + // OK To do async writes, if available. + FLMBOOL bOkToUseESM; // OK to use Extended Server Memory, + // if available + ServerLockManager * pServerLockMgr; + // Pointer to server lock manager. + FLMUINT uiMaxCPInterval; + // Maximum number of seconds to allow between + // checkpoints + F_GlobalCacheMgr * pGlobalCacheMgr; + F_BlockCacheMgr * pBlockCacheMgr; + F_RowCacheMgr * pRowCacheMgr; // row cache manager + FLMUINT uiRehashAfterFailureBackoffTime; + // Amount of time to wait after trying + // to reallocate a cache manager's hash + // table before trying again. + F_BTreeIStreamPool * pBTreeIStreamPool; // Pool of b-tree i-streams that can be re-used + IF_Thread * pMonitorThrd; // Monitor thread + IF_Thread * pCacheCleanupThrd; + SFLM_STATS Stats; // Statistics structure + F_MUTEX hStatsMutex; // Mutex for statistics structure + + F_MUTEX hQueryMutex; // Mutex for managing query list + QUERY_HDR * pNewestQuery; // Head of query list (newest) + QUERY_HDR * pOldestQuery; // Tail of query list (oldest) + FLMUINT uiQueryCnt; // Number of queries in the list + FLMUINT uiMaxQueries; // Maximum number of queries to keep around + FLMBOOL bNeedToUnsetMaxQueries; + // When TRUE, indicates that a call to stop + // statistics should also stop saving + // queries. + FLMBOOL bStatsInitialized; + // Has statistics structure been + // initialized? + + char szTempDir[ F_PATH_MAX_SIZE]; + // Temporary working directory for + // ResultSets, RecordCache + // and other sub-systems that need + // temporary files. This is aligned + // on a 4-byte boundary + + FLMUINT uiMaxUnusedTime; + // Maximum number of timer units to keep + // unused structures in memory before + // freeing them. + FEVENT_HDR EventHdrs [SFLM_MAX_EVENT_CATEGORIES]; + F_Pool * pKRefPool; // Memory Pool that is only used by + // record updaters for key building + + FLMUINT uiMaxFileSize; + IF_LoggerClient * pLogger; + FLMUINT uiPendingLogMessages; + F_MUTEX hLoggerMutex; + +#ifdef FLM_LINUX + FLMUINT uiLinuxMajorVer; + FLMUINT uiLinuxMinorVer; + FLMUINT uiLinuxRevision; +#endif + +#ifdef FLM_DEBUG + // Variables for memory allocation tracking. + + FLMBOOL bTrackLeaks; + FLMBOOL bLogLeaks; + FLMBOOL bStackWalk; + FLMBOOL bMemTrackingInitialized; + FLMUINT uiInitThreadId; + F_MUTEX hMemTrackingMutex; + void ** ppvMemTrackingPtrs; + FLMUINT uiMemTrackingPtrArraySize; + FLMUINT uiMemNumPtrs; + FLMUINT uiMemNextPtrSlotToUse; + FLMUINT uiAllocCnt; + #if defined( FLM_WIN) + HANDLE hMemProcess; + #endif + + #ifdef DEBUG_SIM_OUT_OF_MEM + FLMUINT uiOutOfMemSimEnabledFlag; + // We pick a random number for the flag so that it is hard to accidentally + // turn this flag on by writing memory out-of-bounds. + + #define OUT_OF_MEM_SIM_ENABLED_FLAG 2149614134UL + + F_RandomGenerator memSimRandomGen; + FLMUINT uiSimOutOfMemFailTotal; + FLMUINT uiSimOutOfMemFailSequence; + #endif +#endif + + IF_FileSystem * pFileSystem; + F_MUTEX hIniMutex; + IF_ThreadMgr * pThreadMgr; + FLMUINT uiIndexingThreadGroup; + FLMUINT uiCheckpointThreadGroup; + F_MUTEX hHttpSessionMutex; + F_BtPool * pBtPool; +#ifdef FLM_NLM + FLMBOOL bUseNSSFileHdls; +#endif +} FLMSYSDATA; + +#ifndef ALLOCATE_SYS_DATA + extern FLMSYSDATA gv_SFlmSysData; +#else + FLMSYSDATA gv_SFlmSysData; +#endif + +FINLINE FLMBOOL f_isWhiteSpace( + FLMBYTE ucChar) +{ + return( ucChar == ASCII_SPACE || + ucChar == ASCII_TAB || + ucChar == ASCII_NEWLINE ? TRUE : FALSE); +} + +FINLINE FLMBOOL f_UniIsWhiteSpace( + FLMUNICODE uzChar) +{ + return( uzChar == (FLMUNICODE)ASCII_SPACE || + uzChar == (FLMUNICODE)ASCII_TAB || + uzChar == (FLMUNICODE)ASCII_NEWLINE ? TRUE : FALSE); +} + +FINLINE FLMUNICODE flmConvertChar( + FLMUNICODE uzChar, + FLMUINT uiCompareRules) +{ + if (uzChar == ASCII_SPACE || + (uzChar == ASCII_UNDERSCORE && + (uiCompareRules & FLM_COMP_NO_UNDERSCORES)) || + (f_UniIsWhiteSpace( uzChar) && + (uiCompareRules & FLM_COMP_WHITESPACE_AS_SPACE))) + { + return( (FLMUNICODE)((uiCompareRules & + (FLM_COMP_NO_WHITESPACE | + FLM_COMP_IGNORE_LEADING_SPACE)) + ? (FLMUNICODE)0 + : (FLMUNICODE)ASCII_SPACE)); + } + else if (uzChar == ASCII_DASH && (uiCompareRules & FLM_COMP_NO_DASHES)) + { + return( (FLMUNICODE)0); + } + else + { + return( uzChar); + } +} + +#if defined( FLM_WIN) || defined( FLM_NLM) || defined( FLM_LINUX) + #pragma pack(pop) +#else + #pragma pack() +#endif + +#endif // FSTRUCTS_H diff --git a/sql/src/fsuperfl.cpp b/sql/src/fsuperfl.cpp new file mode 100644 index 0000000..8ce3375 --- /dev/null +++ b/sql/src/fsuperfl.cpp @@ -0,0 +1,1003 @@ +//------------------------------------------------------------------------------ +// Desc: This include file contains the methods for FLAIM's +// super file class. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsuperfl.cpp 3114 2006-01-19 13:22:45 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC char base24ToDigit( + FLMUINT uiBaseValue); + +/**************************************************************************** +Desc: +****************************************************************************/ +F_SuperFileHdl::F_SuperFileHdl( void) +{ + m_pszDbFileName = NULL; + m_pszDataFileNameBase = NULL; + f_memset( &m_CheckedOutFileHdls[ 0], 0, sizeof( m_CheckedOutFileHdls)); + m_pCheckedOutFileHdls = &m_CheckedOutFileHdls [0]; + m_uiCkoArraySize = MAX_CHECKED_OUT_FILE_HDLS + 1; + m_uiBlockSize = 0; + m_uiExtendSize = SFLM_DEFAULT_FILE_EXTEND_SIZE; + m_uiMaxAutoExtendSize = gv_SFlmSysData.uiMaxFileSize; + m_uiLowestDirtySlot = 1; + m_uiHighestDirtySlot = 0; + m_uiHighestUsedSlot = 0; + m_uiHighestFileNumber = 0; + m_bMinimizeFlushes = FALSE; + m_bSetupCalled = FALSE; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +F_SuperFileHdl::~F_SuperFileHdl() +{ + if( m_bSetupCalled) + { + (void)releaseFiles( TRUE); + } + + if (m_pszDbFileName) + { + f_free( &m_pszDbFileName); + } +} + +/**************************************************************************** +Desc: Configures the super file object +****************************************************************************/ +RCODE F_SuperFileHdl::setup( + const char * pszDbFileName, + const char * pszDataDir) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNameLen; + FLMUINT uiDataNameLen; + char szDir [F_PATH_MAX_SIZE]; + char szBaseName [F_FILENAME_SIZE]; + + flmAssert( !m_bSetupCalled); + + if( !pszDbFileName && *pszDbFileName == 0) + { + rc = RC_SET( NE_FLM_IO_INVALID_FILENAME); + goto Exit; + } + + uiNameLen = f_strlen( pszDbFileName); + if (pszDataDir && *pszDataDir) + { + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( + pszDbFileName, szDir, szBaseName))) + { + goto Exit; + } + f_strcpy( szDir, pszDataDir); + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + szDir, szBaseName))) + { + goto Exit; + } + uiDataNameLen = f_strlen( szDir); + + if (RC_BAD( rc = f_alloc( (uiNameLen + 1) + (uiDataNameLen + 1), + &m_pszDbFileName))) + { + goto Exit; + } + + f_memcpy( m_pszDbFileName, pszDbFileName, uiNameLen + 1); + m_pszDataFileNameBase = m_pszDbFileName + uiNameLen + 1; + F_DbSystem::getDbBasePath( m_pszDataFileNameBase, szDir, &m_uiDataExtOffset); + m_uiExtOffset = uiNameLen - (uiDataNameLen - m_uiDataExtOffset); + } + else + { + if (RC_BAD( rc = f_alloc( (uiNameLen + 1) * 2, &m_pszDbFileName))) + { + goto Exit; + } + + f_memcpy( m_pszDbFileName, pszDbFileName, uiNameLen + 1); + m_pszDataFileNameBase = m_pszDbFileName + uiNameLen + 1; + F_DbSystem::getDbBasePath( m_pszDataFileNameBase, + m_pszDbFileName, &m_uiDataExtOffset); + m_uiExtOffset = m_uiDataExtOffset; + } + + m_bSetupCalled = TRUE; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Creates a file +****************************************************************************/ +RCODE F_SuperFileHdl::createFile( + FLMUINT uiFileNumber) +{ + RCODE rc = NE_SFLM_OK; + char szFilePath[ F_PATH_MAX_SIZE]; + IF_FileHdl * pFileHdl = NULL; +// FLMUINT uiFileId; + + // Sanity checks + + flmAssert( m_bSetupCalled && m_uiBlockSize); + flmAssert( uiFileNumber <= MAX_LOG_BLOCK_FILE_NUMBER); + + // See if we already have an open file handle (or if we can open the file). + // If so, truncate the file and use it. + + if( RC_OK( rc = getFileHdl( uiFileNumber, TRUE, &pFileHdl))) + { + rc = pFileHdl->truncate( 0); + pFileHdl = NULL; + goto Exit; + } + else if( rc != NE_FLM_IO_PATH_NOT_FOUND) + { + goto Exit; + } + + // Build the file path + + if( RC_BAD( rc = getFilePath( uiFileNumber, szFilePath))) + { + goto Exit; + } + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->createFile( szFilePath, + FLM_IO_RDWR | FLM_IO_EXCL | FLM_IO_DIRECT | FLM_IO_SH_DENYNONE, + &pFileHdl))) + { + goto Exit; + } + + pFileHdl->setBlockSize( m_uiBlockSize); + +Exit: + + if( pFileHdl) + { + pFileHdl->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Reads a database block into a buffer +****************************************************************************/ +RCODE F_SuperFileHdl::readBlock( + FLMUINT uiBlkAddress, + FLMUINT uiBytesToRead, + void * pvBuffer, + FLMUINT * puiBytesRead) +{ + IF_FileHdl * pFileHdl = NULL; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled && m_uiBlockSize); + + if( RC_BAD( rc = getFileHdl( + FSGetFileNumber( uiBlkAddress), FALSE, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pFileHdl->sectorRead( + FSGetFileOffset( uiBlkAddress), uiBytesToRead, + pvBuffer, puiBytesRead))) + { + if (rc != NE_FLM_IO_END_OF_FILE && rc != NE_SFLM_MEM) + { + releaseFile( FSGetFileNumber( uiBlkAddress), TRUE); + } + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Writes a block to the database +****************************************************************************/ +RCODE F_SuperFileHdl::writeBlock( + FLMUINT uiBlkAddress, + FLMUINT uiBytesToWrite, + const void * pvBuffer, + FLMUINT uiBufferSize, + IF_IOBuffer * pIOBuffer, + FLMUINT * puiBytesWritten) +{ + IF_FileHdl * pFileHdl = NULL; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled && m_uiBlockSize); + +Get_Handle: + if( RC_BAD( rc = getFileHdl( + FSGetFileNumber( uiBlkAddress), TRUE, &pFileHdl))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND) + { + if (RC_BAD( rc = createFile( FSGetFileNumber( uiBlkAddress)))) + { + goto Exit; + } + else + { + goto Get_Handle; + } + } + goto Exit; + } + + pFileHdl->setExtendSize( m_uiExtendSize); + pFileHdl->setMaxAutoExtendSize( m_uiMaxAutoExtendSize); + if( RC_BAD( rc = pFileHdl->sectorWrite( + FSGetFileOffset( uiBlkAddress), uiBytesToWrite, + pvBuffer, uiBufferSize, pIOBuffer, puiBytesWritten))) + { + if (rc != NE_FLM_IO_DISK_FULL && rc != NE_SFLM_MEM) + { + releaseFile( FSGetFileNumber( uiBlkAddress), TRUE); + } + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Reads data from the database header +****************************************************************************/ +RCODE F_SuperFileHdl::readHeader( + FLMUINT uiOffset, + FLMUINT uiBytesToRead, + void * pvBuffer, + FLMUINT * puiBytesRead) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pFileHdl; + +#ifdef FLM_DEBUG + if( m_uiBlockSize) + { + // Note: Block size may not be set because we are in the process of + // opening the file for the first time and we don't know the block + // size until after the header has been read. + + flmAssert( (FLMUINT)(uiOffset + uiBytesToRead) <= m_uiBlockSize); + } +#endif + + if( RC_BAD( rc = getFileHdl( 0, TRUE, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pFileHdl->read( uiOffset, + uiBytesToRead, pvBuffer, puiBytesRead))) + { + if (rc != NE_FLM_IO_END_OF_FILE && rc != NE_SFLM_MEM) + { + releaseFile( (FLMUINT)0, TRUE); + } + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Writes data to the database header +****************************************************************************/ +RCODE F_SuperFileHdl::writeHeader( + FLMUINT uiOffset, + FLMUINT uiBytesToWrite, + const void * pvBuffer, + FLMUINT * puiBytesWritten) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pFileHdl; + +#ifdef FLM_DEBUG + if( m_uiBlockSize) + { + flmAssert( (FLMUINT)(uiOffset + uiBytesToWrite) <= m_uiBlockSize); + } +#endif + + if( RC_BAD( rc = getFileHdl( 0, TRUE, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pFileHdl->write( uiOffset, + uiBytesToWrite, pvBuffer, puiBytesWritten))) + { + if (rc != NE_FLM_IO_DISK_FULL && rc != NE_SFLM_MEM) + { + releaseFile( (FLMUINT)0, TRUE); + } + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Releases all file handle objects and optionally closes the files +****************************************************************************/ +RCODE F_SuperFileHdl::releaseFile( + FLMUINT uiFileNum, + FLMBOOL bCloseFile) +{ + RCODE rc = NE_SFLM_OK; + CHECKED_OUT_FILE_HDL * pCkoFileHdl; + FLMUINT uiSlot; + + pCkoFileHdl = getCkoFileHdlPtr( uiFileNum, &uiSlot); + if( pCkoFileHdl->uiFileNumber == uiFileNum) + { + if( RC_BAD( rc = releaseFile( pCkoFileHdl, bCloseFile))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Releases all file handle objects and optionally closes the files +****************************************************************************/ +RCODE F_SuperFileHdl::releaseFiles( + FLMBOOL bCloseFiles) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + + flmAssert( m_bSetupCalled); + + for( uiLoop = 0; uiLoop <= m_uiHighestUsedSlot; uiLoop++) + { + if( RC_BAD( rc = releaseFile( + &m_CheckedOutFileHdls[ uiLoop], bCloseFiles))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Releases a file handle object +****************************************************************************/ +RCODE F_SuperFileHdl::releaseFile( + CHECKED_OUT_FILE_HDL * pCkoFileHdl, + FLMBOOL bCloseFile) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pFileHdl = pCkoFileHdl->pFileHdl; + + if( pFileHdl) + { +// flmAssert( pFileHdl->getFileId()); + + if( pCkoFileHdl->bDirty) + { + (void)pFileHdl->flush(); + } + + if( bCloseFile) + { + FLMUINT uiRefCnt; + + uiRefCnt = pFileHdl->Release(); + flmAssert( uiRefCnt == 0); + } + + clearCkoFileHdl( pCkoFileHdl); + } + + return( rc); +} + +/**************************************************************************** +Desc: Copy one CKO array into another. +****************************************************************************/ +void F_SuperFileHdl::copyCkoFileHdls( + CHECKED_OUT_FILE_HDL * pSrcCkoArray, + FLMUINT uiSrcHighestUsedSlot + ) +{ + FLMUINT uiNewSlot; + FLMUINT uiSrcSlot; + + // Zeroeth element is always copied. + + f_memcpy( m_pCheckedOutFileHdls, pSrcCkoArray, + sizeof( CHECKED_OUT_FILE_HDL)); + + // Memset the rest of the destination array to zero. + + f_memset( &m_pCheckedOutFileHdls[1], 0, sizeof( CHECKED_OUT_FILE_HDL) * + (m_uiCkoArraySize - 1)); + + m_uiHighestUsedSlot = 0; + m_uiLowestDirtySlot = 1; + m_uiHighestDirtySlot = 0; + for (uiSrcSlot = 1, pSrcCkoArray++; + uiSrcSlot <= uiSrcHighestUsedSlot; + uiSrcSlot++, pSrcCkoArray++) + { + if (pSrcCkoArray->pFileHdl && pSrcCkoArray->uiFileNumber) + { + uiNewSlot = pSrcCkoArray->uiFileNumber % (m_uiCkoArraySize - 1) + 1; + + // Only overwrite the destination one if the file number is + // lower than the one already there + + if (pSrcCkoArray->uiFileNumber < + m_pCheckedOutFileHdls [uiNewSlot].uiFileNumber || + !m_pCheckedOutFileHdls [uiNewSlot].uiFileNumber) + { + if (m_pCheckedOutFileHdls [uiNewSlot].uiFileNumber) + { + releaseFile( &m_pCheckedOutFileHdls [uiNewSlot], FALSE); + } + f_memcpy( &m_pCheckedOutFileHdls [uiNewSlot], pSrcCkoArray, + sizeof( CHECKED_OUT_FILE_HDL)); + if (uiNewSlot > m_uiHighestUsedSlot) + { + m_uiHighestUsedSlot = uiNewSlot; + } + if (m_uiHighestFileNumber < pSrcCkoArray->uiFileNumber) + { + m_uiHighestFileNumber = pSrcCkoArray->uiFileNumber; + } + if (pSrcCkoArray->bDirty) + { + if (m_uiLowestDirtySlot > m_uiHighestDirtySlot) + + { + m_uiLowestDirtySlot = + m_uiHighestDirtySlot = uiNewSlot; + } + else if( m_uiHighestDirtySlot < uiNewSlot) + { + m_uiHighestDirtySlot = uiNewSlot; + } + else if (m_uiLowestDirtySlot < uiNewSlot) + { + m_uiLowestDirtySlot = uiNewSlot; + } + } + } + else + { + releaseFile( pSrcCkoArray, FALSE); + } + } + } +} + +/**************************************************************************** +Desc: Disable flush minimizing. +****************************************************************************/ +void F_SuperFileHdl::disableFlushMinimize( void) +{ + + // Copy the allocated array back into the fixed array. + // This doesn't necessarily copy all of the file handles. + + if (m_pCheckedOutFileHdls != &m_CheckedOutFileHdls [0]) + { + CHECKED_OUT_FILE_HDL * pOldCkoArray = m_pCheckedOutFileHdls; + FLMUINT uiOldHighestUsedSlot = m_uiHighestUsedSlot; + + m_pCheckedOutFileHdls = &m_CheckedOutFileHdls [0]; + m_uiCkoArraySize = MAX_CHECKED_OUT_FILE_HDLS + 1; + copyCkoFileHdls( pOldCkoArray, uiOldHighestUsedSlot); + + f_free( &pOldCkoArray); + } + m_bMinimizeFlushes = FALSE; +} + +/**************************************************************************** +Desc: Flush dirty files to disk. +****************************************************************************/ +RCODE F_SuperFileHdl::flush( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLoop; + + // Flush all dirty files + + for (uiLoop = m_uiLowestDirtySlot; + uiLoop <= m_uiHighestDirtySlot; + uiLoop++) + { + if( m_pCheckedOutFileHdls[ uiLoop].bDirty) + { + RCODE tmpRc; + + if (RC_BAD( tmpRc = + m_pCheckedOutFileHdls[ uiLoop].pFileHdl->flush())) + { + rc = tmpRc; + releaseFile( &m_pCheckedOutFileHdls [uiLoop], TRUE); + } + m_pCheckedOutFileHdls[ uiLoop].bDirty = FALSE; + } + } + m_uiLowestDirtySlot = 1; + m_uiHighestDirtySlot = 0; + return( rc); +} + +/**************************************************************************** +Desc: Truncates back to an end of file block address. + This may only be called from reduce() because there cannot + be any other cases to reduce a 3x block file. +****************************************************************************/ +RCODE F_SuperFileHdl::truncateFile( + FLMUINT uiEOFBlkAddress) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNumber = (FLMUINT)FSGetFileNumber( uiEOFBlkAddress); + FLMUINT uiBlockOffset = (FLMUINT)FSGetFileOffset( uiEOFBlkAddress); + IF_FileHdl * pFileHdl; + + // Truncate the current block file. + + if( RC_BAD( rc = getFileHdl( uiFileNumber, TRUE, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pFileHdl->truncate( uiBlockOffset))) + { + releaseFile( uiFileNumber, TRUE); + goto Exit; + } + + // Visit the rest of the high block files until a NULL file hdl is hit. + + for( ;;) + { + if( RC_BAD( getFileHdl( ++uiFileNumber, TRUE, &pFileHdl))) + { + break; + } + + if( RC_BAD( rc = pFileHdl->truncate( (FLMUINT)0 ))) + { + releaseFile( uiFileNumber, TRUE); + goto Exit; + } + + if( RC_BAD( rc = releaseFile( uiFileNumber, TRUE))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Truncate to zero length any files between the specified start + and end files. +****************************************************************************/ +void F_SuperFileHdl::truncateFiles( + FLMUINT uiStartFileNum, + FLMUINT uiEndFileNum) +{ + FLMUINT uiFileNumber; + IF_FileHdl * pFileHdl; + + for( uiFileNumber = uiStartFileNum; + uiFileNumber <= uiEndFileNum; + uiFileNumber++ ) + { + if( RC_OK( getFileHdl( uiFileNumber, TRUE, &pFileHdl ))) + { + (void)pFileHdl->truncate( (FLMUINT)0 ); + (void)releaseFile( uiFileNumber, TRUE); + } + } +} + +/**************************************************************************** +Desc: Returns the physical size of a file +****************************************************************************/ +RCODE F_SuperFileHdl::getFileSize( + FLMUINT uiFileNumber, + FLMUINT64 * pui64FileSize) +{ + IF_FileHdl * pFileHdl = NULL; + RCODE rc = NE_SFLM_OK; + + flmAssert( m_bSetupCalled); + flmAssert( pui64FileSize); + + *pui64FileSize = 0; + + // Get the file handle. + + if( RC_BAD( rc = getFileHdl( uiFileNumber, FALSE, &pFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pFileHdl->size( pui64FileSize))) + { + releaseFile( uiFileNumber, TRUE); + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Returns the path of a file given its file number +****************************************************************************/ +RCODE F_SuperFileHdl::getFilePath( + FLMUINT uiFileNumber, + char * pszIoPath) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiExtOffset; + + // Sanity checks + + flmAssert( m_bSetupCalled); + + if (!uiFileNumber) + { + f_strcpy( pszIoPath, m_pszDbFileName); + goto Exit; + } + + if( uiFileNumber <= MAX_DATA_BLOCK_FILE_NUMBER) + { + f_memcpy( pszIoPath, m_pszDataFileNameBase, m_uiDataExtOffset); + uiExtOffset = m_uiDataExtOffset; + } + else + { + f_memcpy( pszIoPath, m_pszDbFileName, m_uiExtOffset); + uiExtOffset = m_uiExtOffset; + } + + // Modify the file's extension. + + bldSuperFileExtension( uiFileNumber, &pszIoPath[ uiExtOffset]); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Reallocates the checked out file handle array. +****************************************************************************/ +RCODE F_SuperFileHdl::reallocCkoArray( + FLMUINT uiFileNum + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNewSize; + CHECKED_OUT_FILE_HDL * pNewCkoArray; + CHECKED_OUT_FILE_HDL * pOldCkoArray; + FLMUINT uiOldHighestUsedSlot; + + if (uiFileNum < m_uiHighestFileNumber) + { + uiFileNum = m_uiHighestFileNumber; + } + uiNewSize = uiFileNum + 128; + + // Reallocate so we can guarantee that all of the current file + // numbers will copy and there is room for this new one as well. + + if (uiNewSize > MAX_LOG_BLOCK_FILE_NUMBER + 1) + { + flmAssert( uiFileNum <= MAX_LOG_BLOCK_FILE_NUMBER); + uiNewSize = MAX_LOG_BLOCK_FILE_NUMBER + 1; + } + + // No need to call f_calloc, because copyCkoFileHdls will initialize + // it below. + + if (RC_BAD( rc = f_alloc( sizeof( CHECKED_OUT_FILE_HDL) * uiNewSize, + &pNewCkoArray))) + { + goto Exit; + } + + pOldCkoArray = m_pCheckedOutFileHdls; + uiOldHighestUsedSlot = m_uiHighestUsedSlot; + + m_pCheckedOutFileHdls = pNewCkoArray; + m_uiCkoArraySize = uiNewSize; + + copyCkoFileHdls( pOldCkoArray, uiOldHighestUsedSlot); + + // Can't free the old one until after the copy! + + if (pOldCkoArray != &m_CheckedOutFileHdls [0]) + { + f_free( &pOldCkoArray); + } + +Exit: + + return( rc); + +} + +/**************************************************************************** +Desc: Returns a file handle given the file's number +****************************************************************************/ +RCODE F_SuperFileHdl::getFileHdl( + FLMUINT uiFileNum, + FLMBOOL bGetForUpdate, + IF_FileHdl ** ppFileHdl) +{ + RCODE rc = NE_SFLM_OK; + IF_FileHdl * pFileHdl = NULL; + CHECKED_OUT_FILE_HDL * pCkoFileHdl; + char szFilePath[ F_PATH_MAX_SIZE]; + FLMUINT uiSlot; + + pCkoFileHdl = getCkoFileHdlPtr( uiFileNum, &uiSlot); + if( pCkoFileHdl->uiFileNumber != uiFileNum && + pCkoFileHdl->pFileHdl) + { + if (pCkoFileHdl->bDirty && m_bMinimizeFlushes) + { + flmAssert( pCkoFileHdl->uiFileNumber); + if (RC_BAD( reallocCkoArray( uiFileNum))) + { + goto Exit; + } + pCkoFileHdl = getCkoFileHdlPtr( uiFileNum, &uiSlot); + + // Better have reallocated so that the new slot for + // the file number has nothing in it. + + flmAssert( !pCkoFileHdl->uiFileNumber && + !pCkoFileHdl->pFileHdl); + } + else + { + if( RC_BAD( rc = releaseFile( pCkoFileHdl, FALSE))) + { + goto Exit; + } + } + } + + if( !pCkoFileHdl->pFileHdl) + { + if (!pFileHdl) + { + // Build the file path + + if( RC_BAD( rc = getFilePath( uiFileNum, szFilePath))) + { + goto Exit; + } + + // Open the file + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->openFile( szFilePath, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + &pFileHdl))) + { + goto Exit; + } + + pFileHdl->setBlockSize( m_uiBlockSize); + } + + pCkoFileHdl->pFileHdl = pFileHdl; + pFileHdl = NULL; + pCkoFileHdl->uiFileNumber = uiFileNum; + pCkoFileHdl->bDirty = FALSE; + + if( m_uiHighestUsedSlot < uiSlot) + { + m_uiHighestUsedSlot = uiSlot; + } + if (m_uiHighestFileNumber < uiFileNum) + { + m_uiHighestFileNumber = uiFileNum; + } + } + + *ppFileHdl = pCkoFileHdl->pFileHdl; + if( bGetForUpdate) + { + pCkoFileHdl->bDirty = TRUE; + if (m_uiLowestDirtySlot > m_uiHighestDirtySlot) + + { + m_uiLowestDirtySlot = + m_uiHighestDirtySlot = uiSlot; + } + else if( m_uiHighestDirtySlot < uiSlot) + { + m_uiHighestDirtySlot = uiSlot; + } + else if (m_uiLowestDirtySlot < uiSlot) + { + m_uiLowestDirtySlot = uiSlot; + } + } + +Exit: + + if( pFileHdl) + { + pFileHdl->Release(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Generates a file name given a super file number. + Adds ".xx" to pFileExtension. Use lower case characters. +Notes: This is a base 24 alphanumeric value where + { a, b, c, d, e, f, i, l, o, r, u, v } values are removed. +****************************************************************************/ +void bldSuperFileExtension( + FLMUINT uiFileNum, + char * pszFileExtension) +{ + char ucLetter; + + if (uiFileNum <= MAX_DATA_BLOCK_FILE_NUMBER - 1536) + { + // No additional letter - File numbers 1 to 511 + // This is just like pre-4.3 numbering. + ucLetter = 0; + } + else if (uiFileNum <= MAX_DATA_BLOCK_FILE_NUMBER - 1024) + { + // File numbers 512 to 1023 + ucLetter = 'r'; + } + else if (uiFileNum <= MAX_DATA_BLOCK_FILE_NUMBER - 512) + { + // File numbers 1024 to 1535 + ucLetter = 's'; + } + else if (uiFileNum <= MAX_DATA_BLOCK_FILE_NUMBER) + { + // File numbers 1536 to 2047 + ucLetter = 't'; + } + else if (uiFileNum <= MAX_LOG_BLOCK_FILE_NUMBER - 1536) + { + // File numbers 2048 to 2559 + ucLetter = 'v'; + } + else if (uiFileNum <= MAX_LOG_BLOCK_FILE_NUMBER - 1024) + { + // File numbers 2560 to 3071 + ucLetter = 'w'; + } + else if (uiFileNum <= MAX_LOG_BLOCK_FILE_NUMBER - 512) + { + // File numbers 3072 to 3583 + ucLetter = 'x'; + } + else + { + flmAssert( uiFileNum <= MAX_LOG_BLOCK_FILE_NUMBER); + + // File numbers 3584 to 4095 + ucLetter = 'z'; + } + + *pszFileExtension++ = '.'; + *pszFileExtension++ = (char)(base24ToDigit( (uiFileNum & 511) / 24)); + *pszFileExtension++ = (char)(base24ToDigit( (uiFileNum & 511) % 24)); + *pszFileExtension++ = ucLetter; + *pszFileExtension = 0; +} + +/**************************************************************************** +Desc: Turn a base 24 value into a native alphanumeric value. +Notes: This is a base 24 alphanumeric value where + {a, b, c, d, e, f, i, l, o, r, u, v } values are removed. +****************************************************************************/ +FSTATIC char base24ToDigit( + FLMUINT uiValue) +{ + flmAssert( uiValue <= 23); + + if( uiValue <= 9) + { + uiValue += (FLMUINT) NATIVE_ZERO; + } + else + { + uiValue = f_toascii(uiValue) - 10 + (FLMUINT)f_toascii('g'); + if( uiValue >= (FLMUINT)'i') + { + uiValue++; + if( uiValue >= (FLMUINT)'l') + { + uiValue++; + if( uiValue >= (FLMUINT)'o') + { + uiValue++; + if( uiValue >= (FLMUINT)'r') + { + uiValue++; + if( uiValue >= (FLMUINT)'u') + { + uiValue++; + if( uiValue >= (FLMUINT)'v') + { + uiValue++; + } + } + } + } + } + } + } + return (char)uiValue; +} diff --git a/sql/src/fsuperfl.h b/sql/src/fsuperfl.h new file mode 100644 index 0000000..afac08a --- /dev/null +++ b/sql/src/fsuperfl.h @@ -0,0 +1,215 @@ +//------------------------------------------------------------------------------ +// Desc: This include file contains the class definitions for FLAIM's +// super file class. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsuperfl.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FSUPERFL_H +#define FSUPERFL_H + +#include "fsrvlock.h" + +#define MAX_CHECKED_OUT_FILE_HDLS 8 + +void bldSuperFileExtension( + FLMUINT uiFileNum, + char * pszFileExtension); + +typedef struct +{ + IF_FileHdl * pFileHdl; + FLMUINT uiFileNumber; + FLMBOOL bDirty; +} CHECKED_OUT_FILE_HDL; + +/**************************************************************************** +Desc: The F_SuperFileHdl object manages the control and block files + associated with a FLAIM Super File. This class also provides + backward compatibility with prior file formats. +Note: +****************************************************************************/ +class F_SuperFileHdl : public F_Object +{ +public: + F_SuperFileHdl(); + + ~F_SuperFileHdl(); + + RCODE setup( + const char * pszDbFileName, + const char * pszDataDir); + + RCODE createFile( + FLMUINT uiFileNumber); + + RCODE readBlock( + FLMUINT uiBlkAddress, + FLMUINT uiBytesToRead, + void * pvBuffer, + FLMUINT * puiBytesRead); + + RCODE writeBlock( + FLMUINT uiBlkAddress, + FLMUINT uiBytesToWrite, + const void * pvBuffer, + FLMUINT uiBufferSize, + IF_IOBuffer * pIOBuffer, + FLMUINT * puiBytesWritten); + + RCODE readHeader( + FLMUINT uiOffset, + FLMUINT uiBytesToRead, + void * pvBuffer, + FLMUINT * puiBytesRead); + + RCODE writeHeader( + FLMUINT uiOffset, + FLMUINT uiBytesToWrite, + const void * pvBuffer, + FLMUINT * puiBytesWritten); + + RCODE getFilePath( + FLMUINT uiFileNumber, + char * pszPath); + + RCODE getFileHdl( + FLMUINT uiFileNumber, + FLMBOOL bGetForUpdate, + IF_FileHdl ** ppFileHdlRV); + + RCODE getFileSize( + FLMUINT uiFileNumber, + FLMUINT64 * pui64FileSize); + + RCODE releaseFile( + FLMUINT uiFileNum, + FLMBOOL bCloseFile); + + RCODE releaseFiles( + FLMBOOL bCloseFiles); + + RCODE truncateFile( + FLMUINT uiEOFBlkAddress); + + void truncateFiles( + FLMUINT uiStartFileNum, + FLMUINT uiEndFileNum); + + RCODE releaseFile( + CHECKED_OUT_FILE_HDL * pChkFileHdl, + FLMBOOL bCloseFile); + + FINLINE void enableFlushMinimize( void) + { + m_bMinimizeFlushes = TRUE; + } + + void disableFlushMinimize( void); + + RCODE flush( void); + + FINLINE void setBlockSize( + FLMUINT uiBlockSize) + { + m_uiBlockSize = uiBlockSize; + } + + FINLINE void setExtendSize( + FLMUINT uiExtendSize) + { + m_uiExtendSize = uiExtendSize; + } + + FINLINE void setMaxAutoExtendSize( + FLMUINT uiMaxAutoExtendSize) + { + m_uiMaxAutoExtendSize = uiMaxAutoExtendSize; + } + + FINLINE FLMBOOL canDoAsync( void) + { + if (m_pCheckedOutFileHdls[ 0].pFileHdl) + { + return( m_pCheckedOutFileHdls[ 0].pFileHdl->canDoAsync()); + } + else + { + IF_FileHdl * pFileHdl; + + if( RC_OK( getFileHdl( 0, FALSE, &pFileHdl))) + { + return( pFileHdl->canDoAsync()); + } + } + + return( FALSE); + } + +private: + + FINLINE CHECKED_OUT_FILE_HDL * getCkoFileHdlPtr( + FLMUINT uiFileNum, + FLMUINT * puiSlot) + { + *puiSlot = (uiFileNum + ? (uiFileNum % (m_uiCkoArraySize - 1)) + 1 + : 0); + + return( &m_pCheckedOutFileHdls[ *puiSlot]); + } + + FINLINE void clearCkoFileHdl( + CHECKED_OUT_FILE_HDL * pCkoFileHdl) + { + pCkoFileHdl->pFileHdl = NULL; + pCkoFileHdl->uiFileNumber = 0; + pCkoFileHdl->bDirty = FALSE; + } + + void copyCkoFileHdls( + CHECKED_OUT_FILE_HDL * pSrcCkoArray, + FLMUINT uiSrcHighestUsedSlot); + + RCODE reallocCkoArray( + FLMUINT uiFileNum); + + char * m_pszDbFileName; + char * m_pszDataFileNameBase; + FLMUINT m_uiExtOffset; + FLMUINT m_uiDataExtOffset; + CHECKED_OUT_FILE_HDL m_CheckedOutFileHdls[ + MAX_CHECKED_OUT_FILE_HDLS + 1]; + CHECKED_OUT_FILE_HDL * m_pCheckedOutFileHdls; + FLMUINT m_uiCkoArraySize; + FLMUINT m_uiBlockSize; + FLMUINT m_uiExtendSize; + FLMUINT m_uiMaxAutoExtendSize; + FLMUINT m_uiLowestDirtySlot; + FLMUINT m_uiHighestDirtySlot; + FLMUINT m_uiHighestUsedSlot; + FLMUINT m_uiHighestFileNumber; + FLMBOOL m_bMinimizeFlushes; + FLMBOOL m_bSetupCalled; +}; + +#endif // FSUPERFL_H diff --git a/sql/src/fsysdata.cpp b/sql/src/fsysdata.cpp new file mode 100644 index 0000000..897190c --- /dev/null +++ b/sql/src/fsysdata.cpp @@ -0,0 +1,3308 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the routines that initialize and shut down FLAIM, +// as well as routines for configuring FLAIM. +// +// Tabs: 3 +// +// Copyright (c) 1995-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fsysdata.cpp 3123 2006-01-24 17:19:50 -0700 (Tue, 24 Jan 2006) ahodgkinson $ +//------------------------------------------------------------------------------ + +#define ALLOCATE_SYS_DATA +#define ALLOC_ERROR_TABLES + +#include "flaimsys.h" + +#define HIGH_FLMUINT (~((FLMUINT)0)) +#define FLM_MIN_FREE_BYTES (2 * 1024 * 1024) + +#ifdef FLM_32BIT + #if defined( FLM_LINUX) + + // With mmap'd memory on Linux, you're effectively limited to about ~2 GB. + // Userspace only gets ~3GB of usable address space anyway, and then you + // have all of the thread stacks too, which you can't have + // overlapping the heap. + + #define FLM_MAX_CACHE_SIZE (1500 * 1024 * 1024) + #else + #define FLM_MAX_CACHE_SIZE (2000 * 1024 * 1024) + #endif +#else + #define FLM_MAX_CACHE_SIZE (~((FLMUINT)0)) +#endif + +FLMATOMIC F_DbSystem::m_flmSysSpinLock = 0; +FLMUINT F_DbSystem::m_uiFlmSysStartupCount = 0; + +FSTATIC RCODE flmGetCacheBytes( + FLMUINT uiPercent, + FLMUINT uiMin, + FLMUINT uiMax, + FLMUINT uiMinToLeave, + FLMBOOL bCalcOnAvailMem, + FLMUINT uiBytesCurrentlyInUse, + FLMUINT * puiCacheBytes); + +FSTATIC RCODE flmVerifyDiskStructOffsets( void); + +FSTATIC void flmFreeEvent( + FEVENT * pEvent, + F_MUTEX hMutex, + FEVENT ** ppEventListRV); + +FSTATIC void flmGetStringParam( + const char * pszParamName, + char ** ppszValue, + IF_IniFile * pIniFile); + +FSTATIC void flmGetNumParam( + char ** ppszParam, + FLMUINT * puiNum); + +FSTATIC void flmGetUintParam( + const char * pszParamName, + FLMUINT uiDefaultValue, + FLMUINT * puiUint, + IF_IniFile * pIniFile); + +void flmGetBoolParam( + const char * pszParamName, + FLMBOOL uiDefaultValue, + FLMBOOL * pbBool, + IF_IniFile * pIniFile); + +FSTATIC RCODE flmGetIniFileName( + FLMBYTE * pszIniFileName, + FLMUINT uiBufferSz); + +FLMBOOL gv_bToolkitStarted = FALSE; + +/**************************************************************************** +Desc: This routine allocates and initializes a hash table. +****************************************************************************/ +RCODE flmAllocHashTbl( + FLMUINT uiHashTblSize, + FBUCKET ** ppHashTblRV) +{ + RCODE rc = NE_SFLM_OK; + FBUCKET * pHashTbl = NULL; + IF_RandomGenerator * pRandGen = NULL; + FLMUINT uiCnt; + FLMUINT uiRandVal; + FLMUINT uiTempVal; + + // Allocate memory for the hash table + + if (RC_BAD( rc = f_calloc( + (FLMUINT)(sizeof( FBUCKET)) * uiHashTblSize, &pHashTbl))) + { + goto Exit; + } + + // Set up the random number generator + + if( RC_BAD( rc = FlmAllocRandomGenerator( &pRandGen))) + { + goto Exit; + } + + pRandGen->setSeed( 1); + + for (uiCnt = 0; uiCnt < uiHashTblSize; uiCnt++) + { + pHashTbl [uiCnt].uiHashValue = (FLMBYTE)uiCnt; + pHashTbl [uiCnt].pFirstInBucket = NULL; + } + + if( uiHashTblSize <= 256) + { + for( uiCnt = 0; uiCnt < uiHashTblSize - 1; uiCnt++) + { + uiRandVal = (FLMBYTE) pRandGen->getUINT32( (FLMUINT32)uiCnt, + (FLMUINT32)(uiHashTblSize - 1)); + if( uiRandVal != uiCnt) + { + uiTempVal = (FLMBYTE)pHashTbl [uiCnt].uiHashValue; + pHashTbl [uiCnt].uiHashValue = pHashTbl [uiRandVal].uiHashValue; + pHashTbl [uiRandVal].uiHashValue = uiTempVal; + } + } + } + +Exit: + + if( pRandGen) + { + pRandGen->Release(); + } + + *ppHashTblRV = pHashTbl; + return( rc); +} + +/**************************************************************************** +Desc: This routine determines the number of cache bytes to use for caching + based on a percentage of available physical memory or a percentage + of physical memory (depending on bCalcOnAvailMem flag). + uiBytesCurrentlyInUse indicates how many bytes are currently allocated + by FLAIM - so it can factor that in if the calculation is to be based + on the available memory. + Lower limit is 1 megabyte. +****************************************************************************/ +FSTATIC RCODE flmGetCacheBytes( + FLMUINT uiPercent, + FLMUINT uiMin, + FLMUINT uiMax, + FLMUINT uiMinToLeave, + FLMBOOL bCalcOnAvailMem, + FLMUINT uiBytesCurrentlyInUse, + FLMUINT * puiCacheBytes) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiMem = 0; + FLMUINT64 ui64TotalPhysMem; + FLMUINT64 ui64AvailPhysMem; + + if( RC_BAD( rc = f_getMemoryInfo( &ui64TotalPhysMem, &ui64AvailPhysMem))) + { + goto Exit; + } + + if( ui64TotalPhysMem > HIGH_FLMUINT) + { + ui64TotalPhysMem = HIGH_FLMUINT; + } + + if( ui64AvailPhysMem > ui64TotalPhysMem) + { + ui64AvailPhysMem = ui64TotalPhysMem; + } + + uiMem = (FLMUINT)((bCalcOnAvailMem) + ? (FLMUINT)ui64TotalPhysMem + : (FLMUINT)ui64AvailPhysMem); + + // If we are basing the calculation on available physical memory, + // take in to account what has already been allocated. + + if (bCalcOnAvailMem) + { + if (uiMem > HIGH_FLMUINT - uiBytesCurrentlyInUse) + { + uiMem = HIGH_FLMUINT; + } + else + { + uiMem += uiBytesCurrentlyInUse; + } + } + + // If uiMax is zero, use uiMinToLeave to calculate the maximum. + + if (!uiMax) + { + if (!uiMinToLeave) + { + uiMax = uiMem; + } + else if (uiMinToLeave < uiMem) + { + uiMax = uiMem - uiMinToLeave; + } + else + { + uiMax = 0; + } + } + + // Calculate memory as a percentage of memory. + + uiMem = (FLMUINT)((uiMem > HIGH_FLMUINT / 100) + ? (FLMUINT)(uiMem / 100) * uiPercent + : (FLMUINT)(uiMem * uiPercent) / 100); + + // Don't go above the maximum. + + if (uiMem > uiMax) + { + uiMem = uiMax; + } + + // Don't go below the minimum. + + if (uiMem < uiMin) + { + uiMem = uiMin; + } + +Exit: + + *puiCacheBytes = uiMem; + return( rc); +} + +/*************************************************************************** +Desc: Verify that the distance (in bytes) between pvStart and pvEnd is + what was specified in uiOffset. +****************************************************************************/ +FINLINE void flmVerifyOffset( + FLMUINT uiCompilerOffset, + FLMUINT uiOffset, + RCODE * pRc) +{ + if (RC_OK( *pRc)) + { + if ( uiCompilerOffset != uiOffset) + { + *pRc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + } +} + +/*************************************************************************** +Desc: Verify the offsets of each member of every on-disk structure. This + is a safety check to ensure that things work correctly on every + platform. +****************************************************************************/ +FSTATIC RCODE flmVerifyDiskStructOffsets( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiSizeOf; + + // Verify the SFLM_DB_HDR offsets. + + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, szSignature[0]), + SFLM_DB_HDR_szSignature_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8IsLittleEndian), + SFLM_DB_HDR_ui8IsLittleEndian_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8DefaultLanguage), + SFLM_DB_HDR_ui8DefaultLanguage_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui16BlockSize), + SFLM_DB_HDR_ui16BlockSize_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32DbVersion), + SFLM_DB_HDR_ui32DbVersion_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8BlkChkSummingEnabled), + SFLM_DB_HDR_ui8BlkChkSummingEnabled_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8RflKeepFiles), + SFLM_DB_HDR_ui8RflKeepFiles_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8RflAutoTurnOffKeep), + SFLM_DB_HDR_ui8RflAutoTurnOffKeep_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui8RflKeepAbortedTrans), + SFLM_DB_HDR_ui8RflKeepAbortedTrans_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflCurrFileNum), + SFLM_DB_HDR_ui32RflCurrFileNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui64LastRflCommitID), + SFLM_DB_HDR_ui64LastRflCommitID_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflLastFileNumDeleted), + SFLM_DB_HDR_ui32RflLastFileNumDeleted_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflLastTransOffset), + SFLM_DB_HDR_ui32RflLastTransOffset_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflLastCPFileNum), + SFLM_DB_HDR_ui32RflLastCPFileNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflLastCPOffset), + SFLM_DB_HDR_ui32RflLastCPOffset_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui64RflLastCPTransID), + SFLM_DB_HDR_ui64RflLastCPTransID_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflMinFileSize), + SFLM_DB_HDR_ui32RflMinFileSize_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RflMaxFileSize), + SFLM_DB_HDR_ui32RflMaxFileSize_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui64CurrTransID), + SFLM_DB_HDR_ui64CurrTransID_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui64TransCommitCnt), + SFLM_DB_HDR_ui64TransCommitCnt_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RblEOF), + SFLM_DB_HDR_ui32RblEOF_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32RblFirstCPBlkAddr), + SFLM_DB_HDR_ui32RblFirstCPBlkAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32FirstAvailBlkAddr), + SFLM_DB_HDR_ui32FirstAvailBlkAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32FirstLFBlkAddr), + SFLM_DB_HDR_ui32FirstLFBlkAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32LogicalEOF), + SFLM_DB_HDR_ui32LogicalEOF_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32MaxFileSize), + SFLM_DB_HDR_ui32MaxFileSize_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui64LastBackupTransID), + SFLM_DB_HDR_ui64LastBackupTransID_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32IncBackupSeqNum), + SFLM_DB_HDR_ui32IncBackupSeqNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32BlksChangedSinceBackup), + SFLM_DB_HDR_ui32BlksChangedSinceBackup_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucDbSerialNum[0]), + SFLM_DB_HDR_ucDbSerialNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucLastTransRflSerialNum[0]), + SFLM_DB_HDR_ucLastTransRflSerialNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucNextRflSerialNum[0]), + SFLM_DB_HDR_ucNextRflSerialNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucIncBackupSerialNum[0]), + SFLM_DB_HDR_ucIncBackupSerialNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32DbKeyLen), + SFLM_DB_HDR_ui32DbKeyLen, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucReserved[0]), + SFLM_DB_HDR_ucReserved_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ui32HdrCRC), + SFLM_DB_HDR_ui32HdrCRC_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(SFLM_DB_HDR, ucDbKey[0]), + SFLM_DB_HDR_ucDbKey, &rc); + + // Have to use a variable for sizeof. If we don't, compiler barfs + // because we are comparing two constants. + + uiSizeOf = SFLM_DB_HDR_ucDbKey + SFLM_MAX_ENC_KEY_SIZE; + if( sizeof( SFLM_DB_HDR) != uiSizeOf) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + + // Verify the offsets in the F_BLK_HDR structure. + + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui32BlkAddr), + F_BLK_HDR_ui32BlkAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui32PrevBlkInChain), + F_BLK_HDR_ui32PrevBlkInChain_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui32NextBlkInChain), + F_BLK_HDR_ui32NextBlkInChain_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui32PriorBlkImgAddr), + F_BLK_HDR_ui32PriorBlkImgAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui64TransID), + F_BLK_HDR_ui64TransID_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui32BlkCRC), + F_BLK_HDR_ui32BlkCRC_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui16BlkBytesAvail), + F_BLK_HDR_ui16BlkBytesAvail_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui8BlkFlags), + F_BLK_HDR_ui8BlkFlags_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.stdBlkHdr.ui8BlkType), + F_BLK_HDR_ui8BlkType_OFFSET, &rc); + + // Have to use a variable for sizeof. If we don't, compiler barfs + // because we are comparing two constants. + + uiSizeOf = SIZEOF_STD_BLK_HDR; + if (sizeof( F_BLK_HDR) != uiSizeOf) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + + // Verify the offsets in the F_BTREE_BLK_HDR structure + + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.stdBlkHdr), + F_BTREE_BLK_HDR_stdBlkHdr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.ui16LogicalFile), + F_BTREE_BLK_HDR_ui16LogicalFile_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.ui16NumKeys), + F_BTREE_BLK_HDR_ui16NumKeys_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.ui8BlkLevel), + F_BTREE_BLK_HDR_ui8BlkLevel_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.ui8BTreeFlags), + F_BTREE_BLK_HDR_ui8BTreeFlags_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LARGEST_BLK_HDR, all.BTreeBlkHdr.ui16HeapSize), + F_BTREE_BLK_HDR_ui16HeapSize_OFFSET, &rc); + + // Have to use a variable for sizeof. If we don't, compiler barfs + // because we are comparing two constants. + + uiSizeOf = 40; + if (sizeof( F_BTREE_BLK_HDR) != uiSizeOf) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + + // Have to use a variable for sizeof. If we don't, compiler barfs + // because we are comparing two constants. + + uiSizeOf = 40; + if (sizeof( F_LARGEST_BLK_HDR) != uiSizeOf) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + + // Verify the offsets in the F_LF_HDR structure + + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ui64NextRowId), + F_LF_HDR_ui64NextRowId_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ui32LfType), + F_LF_HDR_ui32LfType_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ui32RootBlkAddr), + F_LF_HDR_ui32RootBlkAddr_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ui32LfNum), + F_LF_HDR_ui32LfNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ui32EncDefNum), + F_LF_HDR_ui32EncDefNum_OFFSET, &rc); + flmVerifyOffset( (FLMUINT)f_offsetof(F_LF_HDR, ucZeroes[0]), + F_LF_HDR_ucZeroes_OFFSET, &rc); + + // Have to use a variable for sizeof. If we don't, compiler barfs + // because we are comparing two constants. + + uiSizeOf = 64; + if (sizeof( F_LF_HDR) != uiSizeOf) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_PLATFORM_FORMAT); + } + + return( rc); +} + +/**************************************************************************** +Desc: Logs the reason for the "must close" flag being set +****************************************************************************/ +void F_Database::logMustCloseReason( + const char * pszFileName, + FLMINT iLineNumber) +{ + char * pszMsgBuf = NULL; + IF_LogMessageClient * pLogMsg = NULL; + + // Log a message indicating why the "must close" flag was set + + if( (pLogMsg = flmBeginLogMessage( SFLM_GENERAL_MESSAGE)) != NULL) + { + if( RC_OK( f_alloc( F_PATH_MAX_SIZE + 512, &pszMsgBuf))) + { + f_sprintf( (char *)pszMsgBuf, + "Database (%s) must be closed because of a 0x%04X error, " + "File=%s, Line=%d.", + (char *)(m_pszDbPath ? (char *)m_pszDbPath : (char *)""), + (unsigned)m_rcMustClose, + pszFileName, (int)iLineNumber); + + pLogMsg->changeColor( FLM_YELLOW, FLM_BLACK); + pLogMsg->appendString( pszMsgBuf); + } + flmEndLogMessage( &pLogMsg); + } + + if( pszMsgBuf) + { + f_free( &pszMsgBuf); + } +} + +/**************************************************************************** +Desc: Acquires a write lock on the database +****************************************************************************/ +RCODE F_Database::dbWriteLock( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + FLMUINT uiTimeout) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = m_pWriteLockObj->Lock( NULL, hWaitSem, FALSE, + (FLMBOOL)(uiTimeout ? TRUE : FALSE), + TRUE, uiTimeout, 0, pDbStats))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine unlocks the write lock on a database. +****************************************************************************/ +void F_Database::dbWriteUnlock( + SFLM_DB_STATS * pDbStats) +{ + (void)m_pWriteLockObj->Unlock( FALSE, NULL, FALSE, pDbStats); +} + +/**************************************************************************** +Desc: This shuts down the background threads +Note: This routine assumes that the global mutex is locked. The mutex will + be unlocked internally, but will always be locked on exit. +****************************************************************************/ +void F_Database::shutdownDatabaseThreads( void) +{ + RCODE rc = NE_SFLM_OK; + F_BKGND_IX * pBackgroundIx; + IF_Thread * pThread; + FLMUINT uiThreadId; + FLMUINT uiThreadCount; + FLMBOOL bMutexLocked = TRUE; + + // Signal all background indexing threads to shutdown that are associated + // with this F_Database. + + for( ;;) + { + uiThreadCount = 0; + + // Shut down all background threads. + + uiThreadId = 0; + for( ;;) + { + if( RC_BAD( rc = gv_SFlmSysData.pThreadMgr->getNextGroupThread( + &pThread, gv_SFlmSysData.uiIndexingThreadGroup, &uiThreadId))) + { + if( rc == NE_SFLM_NOT_FOUND) + { + rc = NE_SFLM_OK; + break; + } + else + { + RC_UNEXPECTED_ASSERT( rc); + } + } + else + { + pBackgroundIx = (F_BKGND_IX *)pThread->getParm1(); + if( pBackgroundIx && pBackgroundIx->pDatabase == this) + { + // Set the thread's terminate flag. + + uiThreadCount++; + pThread->setShutdownFlag(); + } + + pThread->Release(); + pThread = NULL; + } + } + + if( !uiThreadCount) + { + break; + } + + // Unlock the global mutex + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + bMutexLocked = FALSE; + + // Give the threads a chance to terminate + + f_sleep( 50); + + // Re-lock the mutex and see if any threads are still active + + f_mutexLock( gv_SFlmSysData.hShareMutex); + bMutexLocked = TRUE; + + } + + // Shut down the maintenance thread + + if( m_pMaintThrd) + { + flmAssert( bMutexLocked); + + m_pMaintThrd->setShutdownFlag(); + f_semSignal( m_hMaintSem); + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + m_pMaintThrd->stopThread(); + f_mutexLock( gv_SFlmSysData.hShareMutex); + + m_pMaintThrd->Release(); + m_pMaintThrd = NULL; + f_semDestroy( &m_hMaintSem); + } + + // Re-lock the mutex + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + } +} + +/**************************************************************************** +Desc: This routine frees a registered event. +****************************************************************************/ +FSTATIC void flmFreeEvent( + FEVENT * pEvent, + F_MUTEX hMutex, + FEVENT ** ppEventListRV) +{ + pEvent->pEventClient->Release(); + f_mutexLock( hMutex); + if (pEvent->pPrev) + { + pEvent->pPrev->pNext = pEvent->pNext; + } + else + { + *ppEventListRV = pEvent->pNext; + } + if (pEvent->pNext) + { + pEvent->pNext->pPrev = pEvent->pPrev; + } + f_mutexUnlock( hMutex); + f_free( &pEvent); +} + +/**************************************************************************** +Desc: This routine determines the hash bucket for a string. +****************************************************************************/ +FLMUINT flmStrHashBucket( + char * pszStr, // Pointer to string to be hashed + FBUCKET * pHashTbl, // Hash table to use + FLMUINT uiNumBuckets) // Number of hash buckets +{ + FLMUINT uiHashIndex; + + if ((uiHashIndex = (FLMUINT)*pszStr) >= uiNumBuckets) + { + uiHashIndex -= uiNumBuckets; + } + + while (*pszStr) + { + if ((uiHashIndex = + (FLMUINT)((pHashTbl [uiHashIndex].uiHashValue) ^ (FLMUINT)(f_toupper( *pszStr)))) >= + uiNumBuckets) + { + uiHashIndex -= uiNumBuckets; + } + pszStr++; + } + + return( uiHashIndex); +} + +/**************************************************************************** +Desc: This routine links a notify request into a notification list and + then waits to be notified that the event has occurred. +Notes: This routine assumes that the shared mutex is locked and that + it is supposed to unlock it. +****************************************************************************/ +RCODE flmWaitNotifyReq( + F_MUTEX hMutex, + F_SEM hSem, + FNOTIFY ** ppNotifyListRV, + void * pvUserData) +{ + RCODE rc = NE_SFLM_OK; + RCODE TempRc; + FNOTIFY notifyItem; + + notifyItem.hSem = hSem; + notifyItem.uiThreadId = f_threadId(); + notifyItem.pRc = &rc; + notifyItem.pvUserData = pvUserData; + notifyItem.pNext = *ppNotifyListRV; + *ppNotifyListRV = ¬ifyItem; + + f_mutexUnlock( hMutex); + + if( RC_BAD( TempRc = f_semWait( notifyItem.hSem, F_SEM_WAITFOREVER))) + { + rc = TempRc; + } + + f_mutexLock( hMutex); + return( rc); +} + +/**************************************************************************** +Desc: This routine links an F_Database structure to its name hash bucket. + NOTE: This function assumes that the global mutex has been + locked. +****************************************************************************/ +RCODE F_Database::linkToBucket( void) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pTmpDatabase; + FBUCKET * pBucket; + FLMUINT uiBucket; + + pBucket = gv_SFlmSysData.pDatabaseHashTbl; + uiBucket = flmStrHashBucket( m_pszDbPath, pBucket, FILE_HASH_ENTRIES); + pBucket = &pBucket [uiBucket]; + if (pBucket->pFirstInBucket) + { + pTmpDatabase = (F_Database *)pBucket->pFirstInBucket; + pTmpDatabase->m_pPrev = this; + } + + m_uiBucket = uiBucket; + m_pPrev = NULL; + m_pNext = (F_Database *)pBucket->pFirstInBucket; + pBucket->pFirstInBucket = this; + + return( rc); +} + +/**************************************************************************** +Desc: This routine checks unused structures to see if any have been unused + longer than the maximum unused time. If so, it frees them up. +Note: This routine assumes that the calling routine has locked the global + mutex prior to calling this routine. The mutex may be unlocked and + re-locked by one of the called routines. +****************************************************************************/ +void F_DbSystem::checkNotUsedObjects( void) +{ +} + +/**************************************************************************** +Desc: This routine links an FDB structure to an F_Database structure. + NOTE: This routine assumes that the global mutex has been + locked. +****************************************************************************/ +RCODE F_Db::linkToDatabase( + F_Database * pDatabase + ) +{ + RCODE rc = NE_SFLM_OK; + + // If the use count on the file used to be zero, unlink it from the + // unused list. + + flmAssert( !m_pDatabase); + m_pPrevForDatabase = NULL; + if ((m_pNextForDatabase = pDatabase->m_pFirstDb) != NULL) + { + pDatabase->m_pFirstDb->m_pPrevForDatabase = this; + } + + pDatabase->m_pFirstDb = this; + m_pDatabase = pDatabase; + + if (!(m_uiFlags & FDB_INTERNAL_OPEN)) + { + pDatabase->incrOpenCount(); + } + + // Allocate the super file object + + if (!m_pSFileHdl) + { + if ((m_pSFileHdl = f_new F_SuperFileHdl) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + // Set up the super file + + if( RC_BAD( rc = m_pSFileHdl->setup( + pDatabase->m_pszDbPath, pDatabase->m_pszDataDir))) + { + goto Exit; + } + + if( pDatabase->m_lastCommittedDbHdr.ui32DbVersion) + { + m_pSFileHdl->setBlockSize( pDatabase->m_uiBlockSize); + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine unlinks F_Db object from its F_Database structure. + NOTE: This routine assumes that the global mutex has been + locked. +****************************************************************************/ +void F_Db::unlinkFromDatabase( void) +{ + if (!m_pDatabase) + { + return; + } + + // Unlink the F_Db from the F_Database. + + if (m_pNextForDatabase) + { + m_pNextForDatabase->m_pPrevForDatabase = m_pPrevForDatabase; + } + + if (m_pPrevForDatabase) + { + m_pPrevForDatabase->m_pNextForDatabase = m_pNextForDatabase; + } + else + { + m_pDatabase->m_pFirstDb = m_pNextForDatabase; + } + m_pNextForDatabase = m_pPrevForDatabase = NULL; + + // Decrement use counts in the F_Database, unless this was + // an internal open. + + if (!(m_uiFlags & FDB_INTERNAL_OPEN)) + { + m_pDatabase->decrOpenCount(); + } + m_pDatabase = NULL; +} + +/**************************************************************************** +Desc: This routine dynamically adjusts the cache limit if that is the mode + we are in. +****************************************************************************/ +RCODE F_GlobalCacheMgr::adjustCache( + FLMUINT * puiCurrTime, + FLMUINT * puiLastCacheAdjustTime) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCurrTime = *puiCurrTime; + FLMUINT uiLastCacheAdjustTime = *puiLastCacheAdjustTime; + FLMBOOL bMutexLocked = FALSE; + + if (m_bDynamicCacheAdjust && + FLM_ELAPSED_TIME( uiCurrTime, uiLastCacheAdjustTime) >= + m_uiCacheAdjustInterval) + { + FLMUINT uiCacheBytes; + + lockMutex(); + bMutexLocked = TRUE; + + // Make sure the dynamic adjust flag is still set. + + if (m_bDynamicCacheAdjust && + FLM_ELAPSED_TIME( uiCurrTime, uiLastCacheAdjustTime) >= + m_uiCacheAdjustInterval) + { + if( RC_BAD( rc = flmGetCacheBytes( m_uiCacheAdjustPercent, + m_uiCacheAdjustMin, m_uiCacheAdjustMax, + m_uiCacheAdjustMinToLeave, TRUE, totalBytes(), &uiCacheBytes))) + { + goto Exit; + } + + if (RC_BAD( rc = setCacheLimit( uiCacheBytes, FALSE))) + { + unlockMutex(); + goto Exit; + } + } + + unlockMutex(); + bMutexLocked = FALSE; + + *puiCurrTime = *puiLastCacheAdjustTime = FLM_GET_TIMER(); + } + +Exit: + + if( bMutexLocked) + { + unlockMutex(); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine functions as a thread. It monitors open files and + frees up files which have been closed longer than the maximum + close time. +****************************************************************************/ +RCODE F_DbSystem::monitorThrd( + IF_Thread * pThread) +{ + FLMUINT uiLastUnusedCleanupTime = 0; + FLMUINT uiCurrTime; + FLMUINT uiLastCacheAdjustTime = 0; + + for (;;) + { + // See if we should shut down + + if( pThread->getShutdownFlag()) + { + break; + } + + uiCurrTime = FLM_GET_TIMER(); + + // Check the not used stuff and lock timeouts. + + if ( FLM_ELAPSED_TIME( uiCurrTime, uiLastUnusedCleanupTime) >= + gv_SFlmSysData.pGlobalCacheMgr->m_uiUnusedCleanupInterval) + { + // See if any unused structures have bee unused longer than the + // maximum unused time. Free them if they have. + // May unlock and re-lock the global mutex. + + f_mutexLock( gv_SFlmSysData.hShareMutex); + F_DbSystem::checkNotUsedObjects(); + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + + // Reset the timer + + uiCurrTime = uiLastUnusedCleanupTime = FLM_GET_TIMER(); + } + + // Call the lock manager to check timeouts. It is critial + // that this routine be called on a regular interval to + // timeout lock waiters that have expired. + + gv_SFlmSysData.pServerLockMgr->CheckLockTimeouts( FALSE, FALSE); + + // Check the adjusting cache limit + + (void)gv_SFlmSysData.pGlobalCacheMgr->adjustCache( &uiCurrTime, + &uiLastCacheAdjustTime); + + f_sleep( 1000); + } + + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_DbSystem::cacheCleanupThrd( + IF_Thread * pThread) +{ + FLMUINT uiCurrTime; + FLMUINT uiLastDefragTime = 0; + FLMUINT uiLastCleanupTime = 0; + FLMUINT uiDefragInterval; + FLMUINT uiCleanupInterval = gv_SFlmSysData.pGlobalCacheMgr->m_uiCacheCleanupInterval; + FLMBOOL bDoRowCacheFirst = TRUE; + + uiDefragInterval = FLM_SECS_TO_TIMER_UNITS( 120); + + for (;;) + { + if( pThread->getShutdownFlag()) + { + break; + } + + uiCurrTime = FLM_GET_TIMER(); + + // Alternate between reducing row cache and block cache first. + + if (gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit() || + FLM_ELAPSED_TIME( uiCurrTime, uiLastCleanupTime) >= uiCleanupInterval) + { + if (bDoRowCacheFirst) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->reduceCache(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + (void)gv_SFlmSysData.pBlockCacheMgr->reduceCache( NULL); + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + bDoRowCacheFirst = FALSE; + } + else + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + (void)gv_SFlmSysData.pBlockCacheMgr->reduceCache( NULL); + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->reduceCache(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + bDoRowCacheFirst = TRUE; + } + + uiLastCleanupTime = FLM_GET_TIMER(); + } + + if( FLM_ELAPSED_TIME( uiCurrTime, uiLastDefragTime) >= uiDefragInterval) + { + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory(); + gv_SFlmSysData.pRowCacheMgr->defragmentMemory(); + uiLastDefragTime = FLM_GET_TIMER(); + } + + f_sleep( 500); + } + + return( NE_SFLM_OK); +} + +/**************************************************************************** +Desc: This routine does an event callback. Note that the mutex is + locked during the callback. +****************************************************************************/ +void flmDoEventCallback( + eEventCategory eCategory, + eEventType eEvent, + F_Db * pDb, + FLMUINT uiThreadId, + FLMUINT64 ui64TransID, + FLMUINT uiIndexOrTable, + FLMUINT64 ui64RowId, + RCODE rc) +{ + FEVENT * pEvent; + + f_mutexLock( gv_SFlmSysData.EventHdrs [eCategory].hMutex); + pEvent = gv_SFlmSysData.EventHdrs [eCategory].pEventCBList; + while (pEvent) + { + pEvent->pEventClient->catchEvent( eEvent, pDb, uiThreadId, ui64TransID, + uiIndexOrTable, ui64RowId, rc); + pEvent = pEvent->pNext; + } + f_mutexUnlock( gv_SFlmSysData.EventHdrs [eCategory].hMutex); +} + +/**************************************************************************** +Desc: This routine sets the "must close" flags on the + F_Database and its FDBs +****************************************************************************/ +void F_Database::setMustCloseFlags( + RCODE rcMustClose, + FLMBOOL bMutexLocked) +{ + F_Db * pTmpDb; + + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hShareMutex); + } + + if( !m_bMustClose) + { + m_bMustClose = TRUE; + m_rcMustClose = rcMustClose; + pTmpDb = m_pFirstDb; + while( pTmpDb) + { + pTmpDb->m_bMustClose = TRUE; + pTmpDb = pTmpDb->m_pNextForDatabase; + } + + // Log a message indicating why the "must close" flag has been + // set. Calling checkState with the bMustClose flag + // already set to TRUE will cause a message to be logged. + + (void)checkState( __FILE__, __LINE__); + } + + if( !bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + } +} + +/*************************************************************************** +Desc: Lock the system data structure for access - called only by startup + and shutdown. NOTE: On platforms that do not support atomic exchange + this is less than perfect - won't handle tight race conditions. +***************************************************************************/ +void F_DbSystem::lockSysData( void) +{ + // Obtain the spin lock + + while( f_atomicExchange( &m_flmSysSpinLock, 1) == 1) + { + f_sleep( 10); + } +} + +/*************************************************************************** +Desc: Unlock the system data structure for access - called only by startup + and shutdown. +***************************************************************************/ +void F_DbSystem::unlockSysData( void) +{ + (void)f_atomicExchange( &m_flmSysSpinLock, 0); +} + +/**************************************************************************** +Desc : Constructor for global cache manager. +****************************************************************************/ +F_GlobalCacheMgr::F_GlobalCacheMgr() +{ + m_pSlabManager = NULL; + m_bCachePreallocated = FALSE; + + m_bDynamicCacheAdjust = f_canGetMemoryInfo(); + m_uiCacheAdjustPercent = SFLM_DEFAULT_CACHE_ADJUST_PERCENT; + m_uiCacheAdjustMin = SFLM_DEFAULT_CACHE_ADJUST_MIN; + m_uiCacheAdjustMax = SFLM_DEFAULT_CACHE_ADJUST_MAX; + m_uiCacheAdjustMinToLeave = SFLM_DEFAULT_CACHE_ADJUST_MIN_TO_LEAVE; + m_uiCacheAdjustInterval = FLM_SECS_TO_TIMER_UNITS( + SFLM_DEFAULT_CACHE_ADJUST_INTERVAL); + flmGetCacheBytes( m_uiCacheAdjustPercent, m_uiCacheAdjustMin, + m_uiCacheAdjustMax, m_uiCacheAdjustMinToLeave, TRUE, 0, &m_uiMaxBytes); + m_uiCacheCleanupInterval = FLM_SECS_TO_TIMER_UNITS( + SFLM_DEFAULT_CACHE_CLEANUP_INTERVAL); + m_uiUnusedCleanupInterval = FLM_SECS_TO_TIMER_UNITS( + SFLM_DEFAULT_UNUSED_CLEANUP_INTERVAL); + m_hMutex = F_MUTEX_NULL; + m_uiMaxSlabs = 0; +} + +/**************************************************************************** +Desc: +****************************************************************************/ +F_GlobalCacheMgr::~F_GlobalCacheMgr() +{ + if (m_pSlabManager) + { + m_pSlabManager->Release(); + } + + if (m_hMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hMutex); + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_GlobalCacheMgr::setup( void) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE szTmpBuffer[ 64]; + + flmAssert( !m_pSlabManager); + + if (RC_BAD( rc = f_mutexCreate( &m_hMutex))) + { + goto Exit; + } + + if( RC_BAD( rc = FlmAllocSlabManager( &m_pSlabManager))) + { + goto Exit; + } + + f_getenv( "SFLM_PREALLOC_CACHE_SIZE", szTmpBuffer, sizeof( szTmpBuffer)); + + if( RC_BAD( rc = m_pSlabManager->setup( f_atoi( (char *)szTmpBuffer)))) + { + m_pSlabManager->Release(); + m_pSlabManager = NULL; + goto Exit; + } + + // Need to set max slabs here because we didn't know the slab size + // in the constructor. + + m_uiMaxSlabs = m_uiMaxBytes / m_pSlabManager->getSlabSize(); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine sets the limits for record cache and block cache - dividing + the total cache between the two caches. It uses the same ratio + currently in force. +****************************************************************************/ +RCODE F_GlobalCacheMgr::setCacheLimit( + FLMUINT uiNewTotalCacheSize, + FLMBOOL bPreallocateCache) +{ + RCODE rc = NE_SFLM_OK; + + if( uiNewTotalCacheSize > FLM_MAX_CACHE_SIZE) + { + uiNewTotalCacheSize = FLM_MAX_CACHE_SIZE; + } + + if( m_bDynamicCacheAdjust || !bPreallocateCache) + { +DONT_PREALLOCATE: + + if( uiNewTotalCacheSize < m_uiMaxBytes) + { + m_uiMaxBytes = uiNewTotalCacheSize; + m_uiMaxSlabs = m_uiMaxBytes / m_pSlabManager->getSlabSize(); + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->reduceCache(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + rc = gv_SFlmSysData.pBlockCacheMgr->reduceCache( NULL); + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + if( RC_BAD( rc)) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = m_pSlabManager->resize( 0))) + { + goto Exit; + } + } + + m_bCachePreallocated = FALSE; + } + else + { + if( RC_BAD( m_pSlabManager->resize( + uiNewTotalCacheSize, &uiNewTotalCacheSize))) + { + goto DONT_PREALLOCATE; + } + + m_bCachePreallocated = TRUE; + } + + m_uiMaxBytes = uiNewTotalCacheSize; + m_uiMaxSlabs = m_uiMaxBytes / m_pSlabManager->getSlabSize(); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_GlobalCacheMgr::clearCache( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiSavedMaxBytes; + FLMUINT uiSavedMaxSlabs; + + lockMutex(); + + uiSavedMaxBytes = m_uiMaxBytes; + uiSavedMaxSlabs = m_uiMaxSlabs; + + m_uiMaxBytes = 0; + m_uiMaxSlabs = 0; + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->reduceCache(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + rc = gv_SFlmSysData.pBlockCacheMgr->reduceCache( (F_Db *)pDb); + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + if( RC_BAD( rc)) + { + goto Exit; + } + +Exit: + + m_uiMaxBytes = uiSavedMaxBytes; + m_uiMaxSlabs = uiSavedMaxSlabs; + unlockMutex(); + return( rc); +} + +/**************************************************************************** +Desc : Startup the database engine. +Notes: This routine may be called multiple times. However, if that is done + exit() should be called for each time this is called successfully. + This routine does not handle race conditions on platforms that do + not support atomic increment. +****************************************************************************/ +RCODE F_DbSystem::init( void) +{ + RCODE rc = NE_SFLM_OK; + FLMINT iEventCategory; +#ifdef FLM_USE_NICI + int iHandle; +#endif + + lockSysData(); + + // See if FLAIM has already been started. If so, + // we are done. + + if (++m_uiFlmSysStartupCount > 1) + { + goto Exit; + } + + if( RC_BAD( rc = ftkStartup())) + { + goto Exit; + } + gv_bToolkitStarted = TRUE; + + // The memset needs to be first. + + f_memset( &gv_SFlmSysData, 0, sizeof( FLMSYSDATA)); + gv_SFlmSysData.uiMaxFileSize = f_getMaxFileSize(); + + // Get the thread manager + + if( RC_BAD( rc = FlmGetThreadMgr( &gv_SFlmSysData.pThreadMgr))) + { + goto Exit; + } + + // Get the file system manager + + if( RC_BAD( rc = FlmGetFileSystem( &gv_SFlmSysData.pFileSystem))) + { + goto Exit; + } + + gv_SFlmSysData.uiIndexingThreadGroup = + gv_SFlmSysData.pThreadMgr->allocGroupId(); + + gv_SFlmSysData.uiCheckpointThreadGroup = + gv_SFlmSysData.pThreadMgr->allocGroupId(); + + // Sanity check -- make sure we are using the correct + // byte-swap macros for this platform + + flmAssert( FB2UD( (FLMBYTE *)"\x0A\x0B\x0C\x0D") == 0x0D0C0B0A); + flmAssert( FB2UW( (FLMBYTE *)"\x0A\x0B") == 0x0B0A); + +#ifdef FLM_DEBUG + + // Variables for memory allocation tracking. + + gv_SFlmSysData.bTrackLeaks = TRUE; + gv_SFlmSysData.hMemTrackingMutex = F_MUTEX_NULL; +#endif + + gv_SFlmSysData.hRowCacheMutex = F_MUTEX_NULL; + gv_SFlmSysData.hBlockCacheMutex = F_MUTEX_NULL; + gv_SFlmSysData.hShareMutex = F_MUTEX_NULL; + gv_SFlmSysData.hStatsMutex = F_MUTEX_NULL; + gv_SFlmSysData.hLoggerMutex = F_MUTEX_NULL; + gv_SFlmSysData.hIniMutex = F_MUTEX_NULL; + + // Initialize the event categories to have no mutex. + + for (iEventCategory = 0; + iEventCategory < SFLM_MAX_EVENT_CATEGORIES; + iEventCategory++) + { + gv_SFlmSysData.EventHdrs [iEventCategory].hMutex = F_MUTEX_NULL; + } + + initFastBlockCheckSum(); + + if (RC_BAD( rc = flmVerifyDiskStructOffsets())) + { + goto Exit; + } + + // Initialize all of the fields + + gv_SFlmSysData.uiMaxUnusedTime = FLM_SECS_TO_TIMER_UNITS( + SFLM_DEFAULT_MAX_UNUSED_TIME); + gv_SFlmSysData.uiMaxCPInterval = FLM_SECS_TO_TIMER_UNITS( + SFLM_DEFAULT_MAX_CP_INTERVAL); + + gv_SFlmSysData.uiRehashAfterFailureBackoffTime = + FLM_SECS_TO_TIMER_UNITS( SFLM_DEFAULT_REHASH_BACKOFF_INTERVAL); + + if ((gv_SFlmSysData.pGlobalCacheMgr = f_new F_GlobalCacheMgr) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if( RC_BAD( rc = gv_SFlmSysData.pGlobalCacheMgr->setup())) + { + goto Exit; + } + + // Create the mutexes for controlling access to cache and global structures. + + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hRowCacheMutex))) + { + goto Exit; + } + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hBlockCacheMutex))) + { + goto Exit; + } + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hShareMutex))) + { + goto Exit; + } + + if ((gv_SFlmSysData.pBlockCacheMgr = f_new F_BlockCacheMgr) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = gv_SFlmSysData.pBlockCacheMgr->initCache())) + { + goto Exit; + } + if ((gv_SFlmSysData.pRowCacheMgr = f_new F_RowCacheMgr) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->initCache())) + { + goto Exit; + } + + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hQueryMutex))) + { + goto Exit; + } + + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hIniMutex))) + { + goto Exit; + } + + // Initialize a statistics structure. + + if (RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hStatsMutex))) + { + goto Exit; + } + + f_memset( &gv_SFlmSysData.Stats, 0, sizeof( SFLM_STATS)); + gv_SFlmSysData.bStatsInitialized = TRUE; + + // Initialize the logging mutex + + if( RC_BAD( rc = f_mutexCreate( &gv_SFlmSysData.hLoggerMutex))) + { + goto Exit; + } + + // Allocate memory for the file name hash table. + + if (RC_BAD(rc = flmAllocHashTbl( FILE_HASH_ENTRIES, + &gv_SFlmSysData.pDatabaseHashTbl))) + { + goto Exit; + } + +#ifdef FLM_DBG_LOG + flmDbgLogInit(); +#endif + + // Allocate and Initialize FLAIM Server Lock Manager. + + if ((gv_SFlmSysData.pServerLockMgr = f_new ServerLockManager) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = gv_SFlmSysData.pServerLockMgr->setupLockMgr())) + { + goto Exit; + } + + // Set up hash table for lock manager. + + if (RC_BAD( rc = gv_SFlmSysData.pServerLockMgr->SetupHashTbl())) + { + goto Exit; + } + + // Set up mutexes for the event table. + + for (iEventCategory = 0; + iEventCategory < SFLM_MAX_EVENT_CATEGORIES; + iEventCategory++) + { + if (RC_BAD( rc = f_mutexCreate( + &gv_SFlmSysData.EventHdrs [iEventCategory].hMutex))) + { + goto Exit; + } + } + + // Start the monitor thread + + if (RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( + &gv_SFlmSysData.pMonitorThrd, + F_DbSystem::monitorThrd, "DB Monitor"))) + { + goto Exit; + } + + // Start the cache cleanup thread + + if (RC_BAD( rc = gv_SFlmSysData.pThreadMgr->createThread( + &gv_SFlmSysData.pCacheCleanupThrd, + F_DbSystem::cacheCleanupThrd, "Cache Cleanup Thread"))) + { + goto Exit; + } + + if ((gv_SFlmSysData.pBtPool = f_new F_BtPool()) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = gv_SFlmSysData.pBtPool->btpInit())) + { + goto Exit; + } + + if ((gv_SFlmSysData.pBTreeIStreamPool = f_new F_BTreeIStreamPool) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + if (RC_BAD( rc = gv_SFlmSysData.pBTreeIStreamPool->setup())) + { + goto Exit; + } + +#ifdef FLM_USE_NICI + iHandle = f_getpid(); + + // Initialize NICI + + if( RC_BAD( rc = CCS_Init( &iHandle))) + { + rc = RC_SET( NE_SFLM_NICI_INIT_FAILED); + goto Exit; + } +#endif + + // Initialize the XFlaim cache settings from the .ini file (if present) + + readIniFile(); + +Exit: + + // If not successful, free up any resources that were allocated. + + if (RC_BAD( rc)) + { + cleanup(); + } + + unlockSysData(); + return( rc); +} + +/**************************************************************************** +Desc: Shuts the database engine down. +Notes: This routine allows itself to be called multiple times, even before + init() is called or if init() fails. May not handle race + conditions very well on platforms that do not support atomic + exchange. +****************************************************************************/ +void F_DbSystem::exit( void) +{ + lockSysData(); + cleanup(); + unlockSysData(); +} + +/************************************************************************ +Desc : Cleans up - assumes that the spin lock has already been + obtained. This allows it to be called directly from + init() on error conditions. +************************************************************************/ +void F_DbSystem::cleanup( void) +{ + FLMUINT uiCnt; + FLMINT iEventCategory; + + // NOTE: We are checking and decrementing a global variable here. + // However, on platforms that properly support atomic exchange, + // we are OK, because the caller has obtained a spin lock before + // calling this routine, so we are guaranteed to be the only thread + // executing this code at this point. On platforms that don't + // support atomic exchange, our spin lock will be less reliable for + // really tight race conditions. But in reality, nobody should be + // calling FlmStartup and FlmShutdown in race conditions like that + // anyway. We are only doing the spin lock stuff to try and be + // nice about it if they are. + + // This check allows FlmShutdown to be called before calling + // FlmStartup, or even if FlmStartup fails. + + if (!m_uiFlmSysStartupCount) + { + return; + } + + // If we decrement the count and it doesn't go to zero, we are not + // ready to do cleanup yet. + + if (--m_uiFlmSysStartupCount > 0) + { + return; + } + + // Free any queries that have been saved in the query list. + + if (gv_SFlmSysData.hQueryMutex != F_MUTEX_NULL) + { + + // Setting uiMaxQueries to zero will cause flmFreeSavedQueries + // to free the entire list. Also, embedded queries will not be + // added back into the list when uiMaxQueries is zero. + + gv_SFlmSysData.uiMaxQueries = 0; + flmFreeSavedQueries( FALSE); + } + + // Shut down the monitor thread, if there is one. + + if( gv_SFlmSysData.pMonitorThrd) + { + gv_SFlmSysData.pMonitorThrd->stopThread(); + gv_SFlmSysData.pMonitorThrd->Release(); + gv_SFlmSysData.pMonitorThrd = NULL; + } + + // Shut down the cache reduce thread + + if( gv_SFlmSysData.pCacheCleanupThrd) + { + gv_SFlmSysData.pCacheCleanupThrd->stopThread(); + gv_SFlmSysData.pCacheCleanupThrd->Release(); + gv_SFlmSysData.pCacheCleanupThrd = NULL; + } + + // Release the thread manager + + if( gv_SFlmSysData.pThreadMgr) + { + gv_SFlmSysData.pThreadMgr->Release(); + gv_SFlmSysData.pThreadMgr = NULL; + } + + // Free all of the files and associated structures + + if (gv_SFlmSysData.pDatabaseHashTbl) + { + FBUCKET * pDatabaseHashTbl; + + // F_Database destructor expects the global mutex to be locked + // IMPORTANT NOTE: pDatabaseHashTbl is ALWAYS allocated + // AFTER the mutex is allocated, so we are guaranteed + // to have a mutex if pDatabaseHashTbl is non-NULL. + + f_mutexLock( gv_SFlmSysData.hShareMutex); + for (uiCnt = 0, pDatabaseHashTbl = gv_SFlmSysData.pDatabaseHashTbl; + uiCnt < FILE_HASH_ENTRIES; + uiCnt++, pDatabaseHashTbl++) + { + F_Database * pDatabase = (F_Database *)pDatabaseHashTbl->pFirstInBucket; + F_Database * pTmpDatabase; + + while (pDatabase) + { + pTmpDatabase = pDatabase; + pDatabase = pDatabase->m_pNext; + pTmpDatabase->freeDatabase(); + } + pDatabaseHashTbl->pFirstInBucket = NULL; + } + + // Unlock the global mutex + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + + // Free the hash table + f_free( &gv_SFlmSysData.pDatabaseHashTbl); + } + + // Free the statistics. + + if (gv_SFlmSysData.bStatsInitialized) + { + f_mutexLock( gv_SFlmSysData.hStatsMutex); + flmStatFree( &gv_SFlmSysData.Stats); + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + gv_SFlmSysData.bStatsInitialized = FALSE; + } + + if (gv_SFlmSysData.hStatsMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hStatsMutex); + } + + // Free (release) FLAIM's Server Lock Manager. + + if (gv_SFlmSysData.pServerLockMgr) + { + FLMUINT uiRefCnt; + + // Release all locks. + + gv_SFlmSysData.pServerLockMgr->CheckLockTimeouts( FALSE, TRUE); + + // Release the lock manager. + + uiRefCnt = gv_SFlmSysData.pServerLockMgr->Release(); + + // No one else should have a reference to the server lock manager + // at this point, so lets trip a flmAssert if the object was really + // not deleted. + +#ifdef FLM_DEBUG + flmAssert( 0 == uiRefCnt); +#else + // Quiet the compiler about the unused variable + (void)uiRefCnt; +#endif + gv_SFlmSysData.pServerLockMgr = NULL; + } + + // Make sure the purge list is empty + + if( gv_SFlmSysData.pRowCacheMgr->m_pPurgeList) + { + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->cleanupPurgedCache(); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + flmAssert( !gv_SFlmSysData.pRowCacheMgr->m_pPurgeList); + } + + // Free the row cache manager + + if( gv_SFlmSysData.pRowCacheMgr) + { + gv_SFlmSysData.pRowCacheMgr->Release(); + gv_SFlmSysData.pRowCacheMgr = NULL; + } + + // Free the block cache manager + + if( gv_SFlmSysData.pBlockCacheMgr) + { + gv_SFlmSysData.pBlockCacheMgr->Release(); + gv_SFlmSysData.pBlockCacheMgr = NULL; + } + + // Free up callbacks that have been registered for events. + + for (iEventCategory = 0; + iEventCategory < SFLM_MAX_EVENT_CATEGORIES; + iEventCategory++) + { + if (gv_SFlmSysData.EventHdrs [iEventCategory].hMutex != F_MUTEX_NULL) + { + while (gv_SFlmSysData.EventHdrs [iEventCategory].pEventCBList) + { + flmFreeEvent( + gv_SFlmSysData.EventHdrs [iEventCategory].pEventCBList, + gv_SFlmSysData.EventHdrs [iEventCategory].hMutex, + &gv_SFlmSysData.EventHdrs [iEventCategory].pEventCBList); + } + f_mutexDestroy( &gv_SFlmSysData.EventHdrs [iEventCategory].hMutex); + } + } + + // Release the file system object + + if (gv_SFlmSysData.pFileSystem) + { + gv_SFlmSysData.pFileSystem->Release(); + gv_SFlmSysData.pFileSystem = NULL; + } + +#ifdef FLM_DBG_LOG + flmDbgLogExit(); +#endif + + // Release the logger + + if( gv_SFlmSysData.pLogger) + { + flmAssert( !gv_SFlmSysData.uiPendingLogMessages); + gv_SFlmSysData.pLogger->Release(); + gv_SFlmSysData.pLogger = NULL; + } + + if( gv_SFlmSysData.hLoggerMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hLoggerMutex); + } + + // Free the btree pool + + if( gv_SFlmSysData.pBtPool) + { + gv_SFlmSysData.pBtPool->Release(); + gv_SFlmSysData.pBtPool = NULL; + } + + // Free the b-tree i-stream pool + + if( gv_SFlmSysData.pBTreeIStreamPool) + { + gv_SFlmSysData.pBTreeIStreamPool->Release(); + gv_SFlmSysData.pBTreeIStreamPool = NULL; + } + + // Release the global cache manager. NOTE: This must happen + // only AFTER releasing the row cache manager and block cache + // manager. + + if( gv_SFlmSysData.pGlobalCacheMgr) + { + gv_SFlmSysData.pGlobalCacheMgr->Release(); + gv_SFlmSysData.pGlobalCacheMgr = NULL; + } + +#ifdef FLM_USE_NICI + CCS_Shutdown(); +#endif + + // Free the mutexes last of all. + + if (gv_SFlmSysData.hQueryMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hQueryMutex); + } + + if (gv_SFlmSysData.hRowCacheMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hRowCacheMutex); + } + if (gv_SFlmSysData.hBlockCacheMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hBlockCacheMutex); + } + if (gv_SFlmSysData.hShareMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hShareMutex); + } + + if (gv_SFlmSysData.hIniMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &gv_SFlmSysData.hIniMutex); + } + + if( gv_bToolkitStarted) + { + ftkShutdown(); + gv_bToolkitStarted = FALSE; + } +} + +/**************************************************************************** +Desc: Configures how memory will be dynamically regulated. +****************************************************************************/ +RCODE F_GlobalCacheMgr::setDynamicMemoryLimit( + FLMUINT uiCacheAdjustPercent, + FLMUINT uiCacheAdjustMin, + FLMUINT uiCacheAdjustMax, + FLMUINT uiCacheAdjustMinToLeave) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCacheBytes; + FLMBOOL bMutexLocked = FALSE; + + if( !f_canGetMemoryInfo()) + { + rc = RC_SET( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + + lockMutex(); + bMutexLocked = TRUE; + + m_bDynamicCacheAdjust = TRUE; + flmAssert( uiCacheAdjustPercent > 0 && uiCacheAdjustPercent <= 100); + m_uiCacheAdjustPercent = uiCacheAdjustPercent; + m_uiCacheAdjustMin = uiCacheAdjustMin; + m_uiCacheAdjustMax = uiCacheAdjustMax; + m_uiCacheAdjustMinToLeave = uiCacheAdjustMinToLeave; + + if( RC_BAD( rc = flmGetCacheBytes( m_uiCacheAdjustPercent, + m_uiCacheAdjustMin, m_uiCacheAdjustMax, + m_uiCacheAdjustMinToLeave, TRUE, totalBytes(), + &uiCacheBytes))) + { + goto Exit; + } + + if( RC_BAD( rc = setCacheLimit( uiCacheBytes, FALSE))) + { + goto Exit; + } + +Exit: + + if( bMutexLocked) + { + unlockMutex(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Sets a hard memory limit for cache. +****************************************************************************/ +RCODE F_GlobalCacheMgr::setHardMemoryLimit( + FLMUINT uiPercent, + FLMBOOL bPercentOfAvail, + FLMUINT uiMin, + FLMUINT uiMax, + FLMUINT uiMinToLeave, + FLMBOOL bPreallocate) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + + lockMutex(); + bMutexLocked = TRUE; + + m_bDynamicCacheAdjust = FALSE; + if (uiPercent) + { + FLMUINT uiCacheBytes; + + if( RC_BAD( rc = flmGetCacheBytes( uiPercent, uiMin, + uiMax, uiMinToLeave, bPercentOfAvail, + totalBytes(), &uiCacheBytes))) + { + goto Exit; + } + + if( RC_BAD( rc = setCacheLimit( uiCacheBytes, bPreallocate))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = setCacheLimit( uiMax, bPreallocate))) + { + goto Exit; + } + } + +Exit: + + if( bMutexLocked) + { + unlockMutex(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Returns information about memory usage +****************************************************************************/ +void F_GlobalCacheMgr::getCacheInfo( + SFLM_CACHE_INFO * pCacheInfo) +{ + f_memset( pCacheInfo, 0, sizeof( SFLM_CACHE_INFO)); + + lockMutex(); + pCacheInfo->uiMaxBytes = m_uiMaxBytes; + pCacheInfo->uiTotalBytesAllocated = totalBytes(); + pCacheInfo->bDynamicCacheAdjust = m_bDynamicCacheAdjust; + pCacheInfo->uiCacheAdjustPercent = m_uiCacheAdjustPercent; + pCacheInfo->uiCacheAdjustMin = m_uiCacheAdjustMin; + pCacheInfo->uiCacheAdjustMax = m_uiCacheAdjustMax; + pCacheInfo->uiCacheAdjustMinToLeave = m_uiCacheAdjustMinToLeave; + pCacheInfo->bPreallocatedCache = m_bCachePreallocated; + unlockMutex(); + + // Return block cache information. + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + f_memcpy( &pCacheInfo->BlockCache, &gv_SFlmSysData.pBlockCacheMgr->m_Usage, + sizeof( SFLM_CACHE_USAGE)); + + pCacheInfo->uiFreeBytes = gv_SFlmSysData.pBlockCacheMgr->m_uiFreeBytes; + pCacheInfo->uiFreeCount = gv_SFlmSysData.pBlockCacheMgr->m_uiFreeCount; + pCacheInfo->uiReplaceableCount = gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableCount; + pCacheInfo->uiReplaceableBytes = gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableBytes; + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + // Return row cache information. + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + f_memcpy( &pCacheInfo->RowCache, &gv_SFlmSysData.pRowCacheMgr->m_Usage, + sizeof( SFLM_CACHE_USAGE)); + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + // Gather per-database statistics + + f_mutexLock( gv_SFlmSysData.hShareMutex); + if( gv_SFlmSysData.pDatabaseHashTbl) + { + FLMUINT uiLoop; + F_Database * pDatabase; + + for( uiLoop = 0; uiLoop < FILE_HASH_ENTRIES; uiLoop++) + { + if( (pDatabase = (F_Database *)gv_SFlmSysData.pDatabaseHashTbl[ + uiLoop].pFirstInBucket) != NULL) + { + while( pDatabase) + { + if( pDatabase->m_uiDirtyCacheCount) + { + pCacheInfo->uiDirtyBytes += + pDatabase->m_uiDirtyCacheCount * pDatabase->m_uiBlockSize; + pCacheInfo->uiDirtyCount += pDatabase->m_uiDirtyCacheCount; + } + + if( pDatabase->m_uiNewCount) + { + pCacheInfo->uiNewBytes += + pDatabase->m_uiNewCount * pDatabase->m_uiBlockSize; + pCacheInfo->uiNewCount += pDatabase->m_uiNewCount; + } + + if( pDatabase->m_uiLogCacheCount) + { + pCacheInfo->uiLogBytes += + pDatabase->m_uiLogCacheCount * pDatabase->m_uiBlockSize; + pCacheInfo->uiLogCount += pDatabase->m_uiLogCacheCount; + } + + pDatabase = pDatabase->m_pNext; + } + } + } + } + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); +} + +/**************************************************************************** +Desc: Close all files in the file handle manager that have not been + used for the specified number of seconds. +****************************************************************************/ +RCODE F_DbSystem::closeUnusedFiles( + FLMUINT uiSeconds) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiValue; + FLMUINT uiCurrTime; + FLMUINT uiSave; + + // Convert seconds to timer units + + uiValue = FLM_SECS_TO_TIMER_UNITS( uiSeconds); + + // Free any other unused structures that have not been used for the + // specified amount of time. + + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + f_mutexLock( gv_SFlmSysData.hShareMutex); + + // Temporarily set the maximum unused seconds in the FLMSYSDATA structure + // to the value that was passed in to Value1. Restore it after + // calling checkNotUsedObject. + + uiSave = gv_SFlmSysData.uiMaxUnusedTime; + gv_SFlmSysData.uiMaxUnusedTime = uiValue; + + // May unlock and re-lock the global mutex. + + checkNotUsedObjects(); + gv_SFlmSysData.uiMaxUnusedTime = uiSave; + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + + return( rc); +} + +/**************************************************************************** +Desc: Enable/disable cache debugging mode +****************************************************************************/ +void F_DbSystem::enableCacheDebug( + FLMBOOL bDebug) +{ +#ifdef FLM_DEBUG + gv_SFlmSysData.pBlockCacheMgr->m_bDebug = bDebug; + gv_SFlmSysData.pRowCacheMgr->m_bDebug = bDebug; +#else + F_UNREFERENCED_PARM( bDebug); +#endif +} + +/**************************************************************************** +Desc: Returns cache debugging mode +****************************************************************************/ +FLMBOOL F_DbSystem::cacheDebugEnabled( void) +{ +#ifdef FLM_DEBUG + return( gv_SFlmSysData.pBlockCacheMgr->m_bDebug || + gv_SFlmSysData.pRowCacheMgr->m_bDebug); +#else + return( FALSE); +#endif +} + +/**************************************************************************** +Desc: Start gathering statistics. +****************************************************************************/ +void F_DbSystem::startStats( void) +{ + f_mutexLock( gv_SFlmSysData.hStatsMutex); + flmStatStart( &gv_SFlmSysData.Stats); + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + + // Start query statistics, if they have not + // already been started. + + f_mutexLock( gv_SFlmSysData.hQueryMutex); + if (!gv_SFlmSysData.uiMaxQueries) + { + gv_SFlmSysData.uiMaxQueries = 20; + gv_SFlmSysData.bNeedToUnsetMaxQueries = TRUE; + } + f_mutexUnlock( gv_SFlmSysData.hQueryMutex); +} + +/**************************************************************************** +Desc: Stop gathering statistics. +****************************************************************************/ +void F_DbSystem::stopStats( void) +{ + f_mutexLock( gv_SFlmSysData.hStatsMutex); + flmStatStop( &gv_SFlmSysData.Stats); + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + + // Stop query statistics, if they were + // started by a call to FLM_START_STATS. + + f_mutexLock( gv_SFlmSysData.hQueryMutex); + if (gv_SFlmSysData.bNeedToUnsetMaxQueries) + { + gv_SFlmSysData.uiMaxQueries = 0; + flmFreeSavedQueries( TRUE); + // NOTE: flmFreeSavedQueries unlocks the mutex. + } + else + { + f_mutexUnlock( gv_SFlmSysData.hQueryMutex); + } +} + +/**************************************************************************** +Desc: Reset statistics. +****************************************************************************/ +void F_DbSystem::resetStats( void) +{ + FLMUINT uiSaveMax; + + // Reset the block cache statistics. + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + gv_SFlmSysData.pBlockCacheMgr->m_uiIoWaits = 0; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheHits = 0; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheHitLooks = 0; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaults = 0; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaultLooks = 0; + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + // Reset the row cache statistics. + + f_mutexLock( gv_SFlmSysData.hRowCacheMutex); + gv_SFlmSysData.pRowCacheMgr->m_uiIoWaits = 0; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCacheHits = 0; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCacheHitLooks = 0; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCacheFaults = 0; + gv_SFlmSysData.pRowCacheMgr->m_Usage.uiCacheFaultLooks = 0; + f_mutexUnlock( gv_SFlmSysData.hRowCacheMutex); + + f_mutexLock( gv_SFlmSysData.hStatsMutex); + flmStatReset( &gv_SFlmSysData.Stats, TRUE); + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + + f_mutexLock( gv_SFlmSysData.hQueryMutex); + uiSaveMax = gv_SFlmSysData.uiMaxQueries; + gv_SFlmSysData.uiMaxQueries = 0; + flmFreeSavedQueries( TRUE); + // NOTE: flmFreeSavedQueries unlocks the mutex. + + // Restore the old maximum + + if( uiSaveMax) + { + // flmFreeSavedQueries unlocks the mutex, so we + // must relock it to restore the old maximum. + + f_mutexLock( gv_SFlmSysData.hQueryMutex); + gv_SFlmSysData.uiMaxQueries = uiSaveMax; + f_mutexUnlock( gv_SFlmSysData.hQueryMutex); + } +} + +/**************************************************************************** +Desc: Returns statistics that have been collected for a share. +Notes: The statistics returned will be the statistics for ALL databases +****************************************************************************/ +RCODE F_DbSystem::getStats( + SFLM_STATS * pFlmStats) +{ + RCODE rc = NE_SFLM_OK; + + // Get the statistics + + f_mutexLock( gv_SFlmSysData.hStatsMutex); + rc = flmStatCopy( pFlmStats, &gv_SFlmSysData.Stats); + f_mutexUnlock( gv_SFlmSysData.hStatsMutex); + if (RC_BAD( rc)) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Frees memory allocated to a FLM_STATS structure +****************************************************************************/ +void F_DbSystem::freeStats( + SFLM_STATS * pFlmStats) +{ + flmStatFree( pFlmStats); +} + +/**************************************************************************** +Desc: Sets the path for all temporary files that come into use within a + FLAIM share structure. The share mutex should be locked when + settting when called from FlmConfig(). +****************************************************************************/ +RCODE F_DbSystem::setTempDir( + const char * pszPath) +{ + RCODE rc = NE_SFLM_OK; + + f_mutexLock( gv_SFlmSysData.hShareMutex); + + // First, test the path + + if( RC_BAD( rc = gv_SFlmSysData.pFileSystem->doesFileExist( pszPath))) + { + goto Exit; + } + + f_strcpy( gv_SFlmSysData.szTempDir, pszPath); + gv_SFlmSysData.bTempDirSet = TRUE; + +Exit: + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + return( rc); +} + +/**************************************************************************** +Desc: Get the temporary directory. +****************************************************************************/ +RCODE F_DbSystem::getTempDir( + char * pszPath) +{ + RCODE rc = NE_SFLM_OK; + + f_mutexLock( gv_SFlmSysData.hShareMutex); + + if( !gv_SFlmSysData.bTempDirSet ) + { + *pszPath = 0; + rc = RC_SET( NE_FLM_IO_PATH_NOT_FOUND); + goto Exit; + } + else + { + f_strcpy( pszPath, gv_SFlmSysData.szTempDir); + } + +Exit: + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); + return( rc); +} + +/**************************************************************************** +Desc: Sets the maximum seconds between checkpoints +****************************************************************************/ +void F_DbSystem::setCheckpointInterval( + FLMUINT uiSeconds) +{ + gv_SFlmSysData.uiMaxCPInterval = FLM_SECS_TO_TIMER_UNITS( uiSeconds); +} + +/**************************************************************************** +Desc: Gets the maximum seconds between checkpoints +****************************************************************************/ +FLMUINT F_DbSystem::getCheckpointInterval( void) +{ + return( FLM_TIMER_UNITS_TO_SECS( gv_SFlmSysData.uiMaxCPInterval)); +} + +/**************************************************************************** +Desc: Sets the interval for dynamically adjusting the cache limit. +****************************************************************************/ +void F_DbSystem::setCacheAdjustInterval( + FLMUINT uiSeconds) +{ + gv_SFlmSysData.pGlobalCacheMgr->m_uiCacheAdjustInterval = + FLM_SECS_TO_TIMER_UNITS( uiSeconds); +} + +/**************************************************************************** +Desc: Sets the interval for dynamically adjusting the cache limit. +****************************************************************************/ +FLMUINT F_DbSystem::getCacheAdjustInterval( void) +{ + return( FLM_TIMER_UNITS_TO_SECS( + gv_SFlmSysData.pGlobalCacheMgr->m_uiCacheAdjustInterval)); +} + +/**************************************************************************** +Desc: Sets the interval for dynamically cleaning out old + cache blocks and records +****************************************************************************/ +void F_DbSystem::setCacheCleanupInterval( + FLMUINT uiSeconds) +{ + gv_SFlmSysData.pGlobalCacheMgr->m_uiCacheCleanupInterval = + FLM_SECS_TO_TIMER_UNITS( uiSeconds); +} + +/**************************************************************************** +Desc: Gets the interval for dynamically cleaning out old + cache blocks and records +****************************************************************************/ +FLMUINT F_DbSystem::getCacheCleanupInterval( void) +{ + return( FLM_TIMER_UNITS_TO_SECS( + gv_SFlmSysData.pGlobalCacheMgr->m_uiCacheCleanupInterval)); +} + +/**************************************************************************** +Desc: Set interval for cleaning up unused structures +****************************************************************************/ +void F_DbSystem::setUnusedCleanupInterval( + FLMUINT uiSeconds) +{ + gv_SFlmSysData.pGlobalCacheMgr->m_uiUnusedCleanupInterval = + FLM_SECS_TO_TIMER_UNITS( uiSeconds); +} + +/**************************************************************************** +Desc: Gets the interval for cleaning up unused structures +****************************************************************************/ +FLMUINT F_DbSystem::getUnusedCleanupInterval( void) +{ + return( FLM_TIMER_UNITS_TO_SECS( + gv_SFlmSysData.pGlobalCacheMgr->m_uiUnusedCleanupInterval)); +} + +/**************************************************************************** +Desc: Set the maximum time for an item to be unused before it is + cleaned up +****************************************************************************/ +void F_DbSystem::setMaxUnusedTime( + FLMUINT uiSeconds) +{ + gv_SFlmSysData.uiMaxUnusedTime = FLM_SECS_TO_TIMER_UNITS( uiSeconds); +} + +/**************************************************************************** +Desc: Gets the maximum time for an item to be unused +****************************************************************************/ +FLMUINT F_DbSystem::getMaxUnusedTime( void) +{ + return( FLM_TIMER_UNITS_TO_SECS( gv_SFlmSysData.uiMaxUnusedTime)); +} + +/**************************************************************************** +Desc: Sets the logging object to be used for internal status messages +****************************************************************************/ +void F_DbSystem::setLogger( + IF_LoggerClient * pLogger) +{ + IF_LoggerClient * pOldLogger = NULL; + + f_mutexLock( gv_SFlmSysData.hLoggerMutex); + + for( ;;) + { + // While waiting for the pending message count to go to zero, other + // threads may have come in to set different logger objects. To handle + // this case, we need to check gv_SFlmSysData.pLogger and release it + // if non-NULL. + + if( gv_SFlmSysData.pLogger) + { + if( pOldLogger) + { + pOldLogger->Release(); + } + + pOldLogger = gv_SFlmSysData.pLogger; + gv_SFlmSysData.pLogger = NULL; + } + + if( !gv_SFlmSysData.uiPendingLogMessages) + { + break; + } + + f_mutexUnlock( gv_SFlmSysData.hLoggerMutex); + f_sleep( 100); + f_mutexLock( gv_SFlmSysData.hLoggerMutex); + } + + if( pOldLogger) + { + pOldLogger->Release(); + } + + if( (gv_SFlmSysData.pLogger = pLogger) != NULL) + { + gv_SFlmSysData.pLogger->AddRef(); + } + + f_mutexUnlock( gv_SFlmSysData.hLoggerMutex); +} + +/**************************************************************************** +Desc: Enables or disables the use of ESM (if available) +****************************************************************************/ +void F_DbSystem::enableExtendedServerMemory( + FLMBOOL bEnable) +{ + gv_SFlmSysData.bOkToUseESM = bEnable; +} + +/**************************************************************************** +Desc: Returns the state of ESM +****************************************************************************/ +FLMBOOL F_DbSystem::extendedServerMemoryEnabled( void) +{ + return( gv_SFlmSysData.bOkToUseESM); +} + +/**************************************************************************** +Desc: Deactivates open database handles, forcing the database to be + (eventually) closed +Notes: Passing NULL for the path values will cause all active database + handles to be deactivated +****************************************************************************/ +void F_DbSystem::deactivateOpenDb( + const char * pszDbFileName, + const char * pszDataDir) +{ + F_Database * pTmpDatabase; + + f_mutexLock( gv_SFlmSysData.hShareMutex); + if( pszDbFileName) + { + // Look up the file using findDatabase to see if we have the + // file open. May unlock and re-lock the global mutex. + + if( RC_OK( findDatabase( pszDbFileName, + pszDataDir, &pTmpDatabase)) && pTmpDatabase) + { + pTmpDatabase->setMustCloseFlags( NE_SFLM_OK, TRUE); + } + } + else + { + if( gv_SFlmSysData.pDatabaseHashTbl) + { + FLMUINT uiLoop; + + for( uiLoop = 0; uiLoop < FILE_HASH_ENTRIES; uiLoop++) + { + pTmpDatabase = + (F_Database *)gv_SFlmSysData.pDatabaseHashTbl[ uiLoop].pFirstInBucket; + + while( pTmpDatabase) + { + pTmpDatabase->setMustCloseFlags( NE_SFLM_OK, TRUE); + pTmpDatabase = pTmpDatabase->m_pNext; + } + } + } + } + + f_mutexUnlock( gv_SFlmSysData.hShareMutex); +} + +/**************************************************************************** +Desc: Sets the maximum number of queries to save +****************************************************************************/ +void F_DbSystem::setQuerySaveMax( + FLMUINT uiMaxToSave) +{ + f_mutexLock( gv_SFlmSysData.hQueryMutex); + gv_SFlmSysData.uiMaxQueries = uiMaxToSave; + gv_SFlmSysData.bNeedToUnsetMaxQueries = FALSE; + flmFreeSavedQueries( TRUE); +} + +/**************************************************************************** +Desc: Gets the maximum number of queries to save +****************************************************************************/ +FLMUINT F_DbSystem::getQuerySaveMax( void) +{ + return( gv_SFlmSysData.uiMaxQueries); +} + +/**************************************************************************** +Desc: Sets the maximum amount of dirty cache allowed +****************************************************************************/ +void F_DbSystem::setDirtyCacheLimits( + FLMUINT uiMaxDirty, + FLMUINT uiLowDirty) +{ + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + if( !uiMaxDirty) + { + gv_SFlmSysData.pBlockCacheMgr->m_bAutoCalcMaxDirty = TRUE; + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache = 0; + gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache = 0; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_bAutoCalcMaxDirty = FALSE; + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache = uiMaxDirty; + + // Low threshhold must be no higher than maximum! + + if ((gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache = uiLowDirty) > + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache = + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache; + } + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: Gets the maximum amount of dirty cache allowed +****************************************************************************/ +void F_DbSystem::getDirtyCacheLimits( + FLMUINT * puiMaxDirty, + FLMUINT * puiLowDirty) +{ + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + if( puiMaxDirty) + { + *puiMaxDirty = gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache; + } + + if( puiLowDirty) + { + *puiLowDirty = gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache; + } + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: Returns information about threads owned by the database engine +****************************************************************************/ +RCODE F_DbSystem::getThreadInfo( + IF_ThreadInfo ** ppThreadInfo) +{ + return( FlmGetThreadInfo( ppThreadInfo)); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_DbSystem::getFileSystem( + IF_FileSystem ** ppFileSystem) +{ + FlmGetFileSystem( ppFileSystem); +} + +/**************************************************************************** +Desc: Registers for an event +****************************************************************************/ +RCODE F_DbSystem::registerForEvent( + eEventCategory eCategory, + IF_EventClient * pEventClient) +{ + RCODE rc = NE_SFLM_OK; + FEVENT * pEvent; + + // Make sure it is a legal event category to register for. + + if (eCategory >= SFLM_MAX_EVENT_CATEGORIES) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + + // Allocate an event structure + + if (RC_BAD( rc = f_calloc( + (FLMUINT)(sizeof( FEVENT)), &pEvent))) + { + goto Exit; + } + + // Initialize the structure members and linkt to the + // list of events off of the event category. + + pEvent->pEventClient = pEventClient; + pEvent->pEventClient->AddRef(); + // pEvent->pPrev = NULL; // done by f_calloc above. + + // Mutex should be locked to link into list. + + f_mutexLock( gv_SFlmSysData.EventHdrs [eCategory].hMutex); + if ((pEvent->pNext = + gv_SFlmSysData.EventHdrs [eCategory].pEventCBList) != NULL) + { + pEvent->pNext->pPrev = pEvent; + } + gv_SFlmSysData.EventHdrs [eCategory].pEventCBList = pEvent; + f_mutexUnlock( gv_SFlmSysData.EventHdrs [eCategory].hMutex); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: De-registers for an event +****************************************************************************/ +void F_DbSystem::deregisterForEvent( + eEventCategory eCategory, + IF_EventClient * pEventClient) +{ + FEVENT * pEvent = gv_SFlmSysData.EventHdrs [eCategory].pEventCBList; + + // Find the event and remove from the list. If it is not + // there, don't worry about it. + + while (pEvent) + { + if (pEventClient == pEvent->pEventClient) + { + flmFreeEvent( pEvent, + gv_SFlmSysData.EventHdrs[ eCategory].hMutex, + &gv_SFlmSysData.EventHdrs[ eCategory].pEventCBList); + break; + } + pEvent = pEvent->pNext; + } +} + +/**************************************************************************** +Desc: Returns TRUE if the specified error indicates that the database + is corrupt +****************************************************************************/ +RCODE F_DbSystem::getNextMetaphone( + IF_IStream * pIStream, + FLMUINT * puiMetaphone, + FLMUINT * puiAltMetaphone) +{ + return( f_getNextMetaphone( + pIStream, puiMetaphone, puiAltMetaphone)); +} + +/**************************************************************************** +Desc: Returns TRUE if the specified error indicates that the database + is corrupt +****************************************************************************/ +FLMBOOL F_DbSystem::errorIsFileCorrupt( + RCODE rc) +{ + FLMBOOL bIsCorrupt = FALSE; + + switch( rc) + { + case NE_SFLM_BTREE_ERROR : + case NE_SFLM_DATA_ERROR : + case NE_SFLM_NOT_FLAIM : + case NE_SFLM_BLOCK_CRC : + case NE_SFLM_HDR_CRC : + case NE_SFLM_INCOMPLETE_LOG : + bIsCorrupt = TRUE; + break; + default : + break; + } + + return( bIsCorrupt); +} + +/**************************************************************************** +Desc: Increment the database system use count +****************************************************************************/ +FLMINT F_DbSystem::AddRef(void) +{ + FLMINT iRefCnt = f_atomicInc( &m_refCnt); + + // Note: We don't bother with a call to LockModule() here because + // it's done in the constructor. In fact, it's unlikely that + // this AddRef() will ever be called, because the constructor + // already sets the ref count to 1 and it's unlikely that there + // will be to references to the same DbSystem object... + + return( iRefCnt); +} + +/**************************************************************************** +Desc: Decrement the database system use count +****************************************************************************/ +FLMINT F_DbSystem::Release(void) +{ + FLMINT iRefCnt = f_atomicDec( &m_refCnt); + + if (iRefCnt == 0) + { + delete this; + } + + return( iRefCnt); +} + +/**************************************************************************** +Desc: Check for the existence of FLAIM performance tuning file. If + found this routine will parse the contents and set the global + performance tuning variables. + + Looks for the following strings within the .ini file + cache= # Set a hard memory limit + cache= # Set a hard memory limit or dynamically + # adjusting limit. + # Multiple options may be specified, in + # any order, separated by commas. All + # are optional. + %: # percentage of available or physical memory. + AVAIL or TOTAL # percentage is for available physical + # memory or total physical memory. NOTE: + # these are ignored for a dynamic limit. + MIN: # minimum number of bytes + MAX: # maximum number of bytes + LEAVE: # minimum nmber of bytes to leave + DYN or HARD # dynamic or hard limit + PRE # preallocate cache - valid only with HARD setting. + + Examples: + + cache=DYN,%:75,MIN:16000000 # Dynamic limit of 75% available memory, + # minimum of 16 megabytes. + cache=HARD,%:75,MIN:16000000 # Hard limit of 75% total physical memory, + # minimum of 16 megabytes. + cache=HARD,%:75,MIN:16000000,PRE + # Hard limit of 75% total physical memory, + # minimum of 16 megabytes, preallocated. + cache=8000000 # Old style - hard limit of 8 megabytes. + + cacheadjustinterval= + cachecleanupinterval= +****************************************************************************/ + +#define SFLM_INI_FILE_NAME "_sflm.ini" + +RCODE F_DbSystem::readIniFile() +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + IF_IniFile * pIniFile = NULL; + char szIniFileName [F_PATH_MAX_SIZE]; + FLMUINT uiParamValue; + FLMUINT uiMaxDirtyCache; + FLMUINT uiLowDirtyCache; + + // Initialize the ini file object... + + if( RC_BAD( rc = FlmAllocIniFile( &pIniFile))) + { + goto Exit; + } + + // Lock the ini file mutex + + f_mutexLock( gv_SFlmSysData.hIniMutex); + bMutexLocked = TRUE; + + if (RC_BAD( rc = flmGetIniFileName( (FLMBYTE *)szIniFileName, + F_PATH_MAX_SIZE))) + { + goto Exit; + } + + if (RC_BAD( rc = pIniFile->read( szIniFileName))) + { + goto Exit; + } + + // Set FLAIM parameters from the buffer read from the .ini file. + + setCacheParams( pIniFile); + + flmGetUintParam( SFLM_INI_CACHE_ADJUST_INTERVAL, + SFLM_DEFAULT_CACHE_CLEANUP_INTERVAL, + &uiParamValue, pIniFile); + setCacheAdjustInterval( uiParamValue); + + flmGetUintParam( SFLM_INI_CACHE_CLEANUP_INTERVAL, + SFLM_DEFAULT_CACHE_CLEANUP_INTERVAL, + &uiParamValue, pIniFile); + setCacheCleanupInterval( uiParamValue); + + flmGetUintParam( SFLM_INI_MAX_DIRTY_CACHE, 0, &uiMaxDirtyCache, pIniFile); + + flmGetUintParam( SFLM_INI_LOW_DIRTY_CACHE, 0, &uiLowDirtyCache, pIniFile); + + // Use FLAIM's default behavior if uiMaxDirtyCache is zero. FLAIM's + // default behavior is to use a maximum value for dirty cache. + + if (uiMaxDirtyCache) + { + setDirtyCacheLimits( uiMaxDirtyCache, uiLowDirtyCache); + } + +Exit: + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hIniMutex); + } + + pIniFile->Release(); + return( rc); +} + +/**************************************************************************** +Desc: Given a tag and a value this function will open/create the + ini file and replace or insert the tag with it's value. + NOTE: This function expects gv_SFlmSysData.hIniMutex to already be + locked! +****************************************************************************/ +RCODE F_DbSystem::updateIniFile( + const char * pszParamName, + const char * pszValue) +{ + RCODE rc = NE_SFLM_OK; + IF_IniFile * pIniFile = NULL; + char szIniFileName [F_PATH_MAX_SIZE]; + + if( RC_BAD( rc = FlmAllocIniFile( &pIniFile))) + { + goto Exit; + } + + if (RC_BAD( rc = flmGetIniFileName( (FLMBYTE *)szIniFileName, + F_PATH_MAX_SIZE))) + { + goto Exit; + } + + if (RC_BAD( rc = pIniFile->read( szIniFileName))) + { + goto Exit; + } + + if (RC_BAD( rc = pIniFile->setParam( pszParamName, pszValue))) + { + goto Exit; + } + + if (RC_BAD( rc = pIniFile->write())) + { + goto Exit; + } + +Exit: + + pIniFile->Release(); + return( rc); +} + +/**************************************************************************** +Desc: Sets the parameters for controlling cache. +****************************************************************************/ +RCODE F_DbSystem::setCacheParams( + IF_IniFile * pIniFile) +{ + RCODE rc = NE_SFLM_OK; + char * pszCacheParam = NULL; + char cChar; + char * pszSave; + char * pszValueName; + FLMBOOL bDynamicCacheAdjust; + FLMBOOL bCalcLimitOnAvail = FALSE; + FLMUINT uiCachePercent; + FLMUINT uiCacheMin; + FLMUINT uiCacheMax; + FLMUINT uiCacheMinToLeave; + FLMBOOL bPreallocated = FALSE; + + // Set up appropriate defaults. + + bDynamicCacheAdjust = getDynamicCacheSupported(); + + if( bDynamicCacheAdjust) + { + uiCachePercent = SFLM_DEFAULT_CACHE_ADJUST_PERCENT; + bCalcLimitOnAvail = TRUE; + uiCacheMin = SFLM_DEFAULT_CACHE_ADJUST_MIN; + uiCacheMax = SFLM_DEFAULT_CACHE_ADJUST_MAX; + uiCacheMinToLeave = 0; + } + else + { + uiCachePercent = 0; + uiCacheMin = + uiCacheMax = SFLM_DEFAULT_CACHE_ADJUST_MIN; + uiCacheMinToLeave = 0; + } + + // Extract values from the cache parameter + flmGetStringParam( SFLM_INI_CACHE, &pszCacheParam, pIniFile); + + if ((pszCacheParam != NULL) && (pszCacheParam[0] != '\0')) + { + // Parse until we hit white space. + + while (*pszCacheParam && + *pszCacheParam != ' ' && + *pszCacheParam != '\n' && + *pszCacheParam != '\r' && + *pszCacheParam != '\t') + { + + // Get the value name. + + pszValueName = pszCacheParam; + while (*pszCacheParam && + *pszCacheParam != ' ' && + *pszCacheParam != '\n' && + *pszCacheParam != '\r' && + *pszCacheParam != '\t' && + *pszCacheParam != ':' && + *pszCacheParam != ',' && + *pszCacheParam != ';') + { + pszCacheParam++; + } + pszSave = pszCacheParam; + cChar = *pszSave; + *pszSave = 0; + + // New options only supported on platforms that have ways + // of getting physical memory size and available physical + // memory size. + + if ((f_stricmp( pszValueName, "MAX") == 0) || + (f_stricmp( pszValueName, "MAXIMUM") == 0)) + { + if (cChar == ':') + { + pszCacheParam++; + flmGetNumParam( &pszCacheParam, + &uiCacheMax); + } + } + else if ((f_stricmp( pszValueName, "MIN") == 0) || + (f_stricmp( pszValueName, "MINIMUM") == 0)) + { + if (cChar == ':') + { + pszCacheParam++; + flmGetNumParam( &pszCacheParam, + &uiCacheMin); + } + } + else if ((f_stricmp( pszValueName, "%") == 0) || + (f_stricmp( pszValueName, "PERCENT") == 0)) + { + if (cChar == ':') + { + pszCacheParam++; + flmGetNumParam( &pszCacheParam, + &uiCachePercent); + } + } + else if ((f_stricmp( pszValueName, "DYNAMIC") == 0) || + (f_stricmp( pszValueName, "DYN") == 0)) + { + if (cChar == ':' || cChar == ',' || cChar == ';') + { + pszCacheParam++; + } + bDynamicCacheAdjust = TRUE; + } + else if (f_stricmp( pszValueName, "AVAIL") == 0) + { + if (cChar == ':' || cChar == ',' || cChar == ';') + { + pszCacheParam++; + } + bCalcLimitOnAvail = TRUE; + } + else if (f_stricmp( pszValueName, "TOTAL") == 0) + { + if (cChar == ':' || cChar == ',' || cChar == ';') + { + pszCacheParam++; + } + bCalcLimitOnAvail = FALSE; + } + else if (f_stricmp( pszValueName, "LEAVE") == 0) + { + if (cChar == ':') + { + pszCacheParam++; + flmGetNumParam( &pszCacheParam, + &uiCacheMinToLeave); + } + } + else if (f_stricmp( pszValueName, "HARD") == 0) + { + if (cChar == ':' || cChar == ',' || cChar == ';') + { + pszCacheParam++; + } + bDynamicCacheAdjust = FALSE; + } + else if (f_stricmp( pszValueName, "PRE") == 0) + { + if (cChar == ':' || cChar == ',' || cChar == ';') + { + pszCacheParam++; + } + bPreallocated = TRUE; + } + else + { + *pszSave = cChar; + + // This will handle the old way where they just put in a + // number for the byte maximum. + + bDynamicCacheAdjust = FALSE; + uiCachePercent = 0; + uiCacheMax = f_atol( pszValueName); + + // Don't really have to set these, but do it just to + // be clean. + + uiCacheMin = 0; + uiCacheMinToLeave = 0; + bCalcLimitOnAvail = FALSE; + break; + } + *pszSave = cChar; + } + } + + // Set up the stuff for controlling cache - hard limit or dynamically + // adjusting limit. + + if( bDynamicCacheAdjust) + { + if (RC_BAD( rc = setDynamicMemoryLimit( + uiCachePercent, + uiCacheMin, uiCacheMax, + uiCacheMinToLeave))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = setHardMemoryLimit( uiCachePercent, + bCalcLimitOnAvail, uiCacheMin, + uiCacheMax, uiCacheMinToLeave, + bPreallocated))) + { + goto Exit; + } + } + +Exit: + + return rc; +} + +/**************************************************************************** +Desc: Gets a string parameter from the .ini file. +****************************************************************************/ +FSTATIC void flmGetStringParam( + const char * pszParamName, + char ** ppszValue, + IF_IniFile * pIniFile) +{ + flmAssert( pIniFile); + (void)pIniFile->getParam( pszParamName, ppszValue); +} + +/**************************************************************************** +Desc: Gets one of the number parameters for setting cache stuff. +****************************************************************************/ +FSTATIC void flmGetNumParam( + char ** ppszParam, + FLMUINT * puiNum) +{ + char * pszTmp = *ppszParam; + FLMUINT uiNum = 0; + + while ((*pszTmp >= '0') && (*pszTmp <= '9')) + { + uiNum *= 10; + uiNum += (FLMUINT)(*pszTmp - '0'); + pszTmp++; + } + + // Run past any other junk up to allowed terminators. + + while (*pszTmp && + *pszTmp != ' ' && + *pszTmp != '\n' && + *pszTmp != '\r' && + *pszTmp != '\t' && + *pszTmp != ':' && + *pszTmp != ',' && + *pszTmp != ';') + { + pszTmp++; + } + + // Skip past trailing colon, comma, or semicolon. + + if (*pszTmp == ':' || *pszTmp == ',' || *pszTmp == ';') + { + pszTmp++; + } + + *puiNum = uiNum; + *ppszParam = pszTmp; +} + +/**************************************************************************** +Desc: Gets a FLMUINT parameter from the .ini file. +****************************************************************************/ +FSTATIC void flmGetUintParam( + const char * pszParamName, + FLMUINT uiDefaultValue, + FLMUINT * puiUint, + IF_IniFile * pIniFile) +{ + flmAssert( pIniFile); + if (!pIniFile->getParam( pszParamName, puiUint)) + { + *puiUint = uiDefaultValue; + } +} + +/**************************************************************************** +Desc: Gets a FLMBOOL parameter from the .ini file. +****************************************************************************/ +void flmGetBoolParam( + const char * pszParamName, + FLMBOOL bDefaultValue, + FLMBOOL * pbBool, + IF_IniFile * pIniFile) +{ + flmAssert( pIniFile); + if (!pIniFile->getParam( pszParamName, pbBool)) + { + *pbBool = bDefaultValue; + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +FSTATIC RCODE flmGetIniFileName( + FLMBYTE * pszIniFileName, + FLMUINT uiBufferSz) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNameLen = 0; + IF_DirHdl * pDirHdl = NULL; + + if (!uiBufferSz) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_FAILURE); + goto Exit; + } + + // See if we have an environment variable to tell us wher the ini file should be. + + f_getenv( "SFLM_INI_PATH", + pszIniFileName, + uiBufferSz, + &uiFileNameLen); + + if( !uiFileNameLen) + { + // Perhaps we can find a data directory. If there is one, we will + // look in there. + + if( RC_OK( rc = gv_SFlmSysData.pFileSystem->openDir( + (char *)".", (char *)"data", &pDirHdl))) + { + if (RC_OK( rc = pDirHdl->next())) + { + if (pDirHdl->currentItemIsDir()) + { + // Directory exists. We will look there. + f_strcpy( (char *)pszIniFileName, "data"); + } + else + { + goto NoDataDir; + } + } + else + { + rc = NE_SFLM_OK; + goto NoDataDir; + } + } + else + { +NoDataDir: + // Data directory does not exist. We will look in the + // current (default) dir. + + f_strcpy( (char *)pszIniFileName, "."); + } + } + + gv_SFlmSysData.pFileSystem->pathAppend( + (char *)pszIniFileName, SFLM_INI_FILE_NAME); + +Exit: + + if (pDirHdl) + { + pDirHdl->Release(); + } + + return( rc); +} + diff --git a/sql/src/funicode.cpp b/sql/src/funicode.cpp new file mode 100644 index 0000000..8b91805 --- /dev/null +++ b/sql/src/funicode.cpp @@ -0,0 +1,1075 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the Unicode conversion routines +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: funicode.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +//#define DEF_FLM_UNI_GLOBALS +#include "flaimsys.h" + +/**************************************************************************** +Desc: Encode a string into FLAIM's internal text format + (SEN-prefixed, null-terminated UTF8). The string is prefixed with + a SEN that indicates the number of characters, not including the + terminating NULL. + + This routine can be called with a NULL input buffer. If it is + called in this way, the length of the encoded string will be + returned in *puiBufLength, but no encoding will actually be + performed. +****************************************************************************/ +RCODE flmUnicode2Storage( + const FLMUNICODE * puzStr, // UNICODE string to encode + FLMUINT uiStrLen, // 0 = Unknown + FLMBYTE * pucBuf, // Destination buffer + FLMUINT * puiBufLength, // [IN ] Size of pucBuf, + // [OUT] Amount of pucBuf used + FLMUINT * puiCharCount) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEncPtr; + const FLMUNICODE * puzPtr = NULL; + FLMUINT uiMaxLen; + FLMUINT uiCharsEncoded = 0; + FLMUINT uiEncodedLen = 0; + FLMUINT uiTmp; + FLMUNICODE uChar; + FLMBYTE ucTmpSen[ 5]; + FLMBYTE * pucTmpSen = &ucTmpSen[ 0]; + + if( !pucBuf) + { + uiMaxLen = (~(FLMUINT)0); + } + else + { + uiMaxLen = *puiBufLength; + } + + // If uiStrLen is 0, determine the number of characters. + + if( !uiStrLen) + { + uiStrLen = f_unilen( puzStr); + } + else if( puzStr[ uiStrLen] != 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + if( puiCharCount) + { + *puiCharCount = uiStrLen; + } + + if( !uiStrLen) + { + // Nothing to encode + + *puiBufLength = 0; + goto Exit; + } + + pucEncPtr = pucBuf; + + // Encode the number of characters as a SEN. If pucEncPtr is + // NULL, the caller is only interested in the length of the encoded + // string, so a temporary buffer is used to call f_encodeSEN. + + uiTmp = f_encodeSEN( uiStrLen, &pucTmpSen); + + if( pucEncPtr) + { + if( (uiEncodedLen + uiTmp) >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( uiTmp == 1) + { + *pucEncPtr++ = ucTmpSen[ 0]; + } + else + { + f_memcpy( pucEncPtr, &ucTmpSen[ 0], uiTmp); + pucEncPtr += uiTmp; + } + } + uiEncodedLen += uiTmp; + + // Encode the string using UTF-8 + + puzPtr = puzStr; + if( uiStrLen) + { + while( (uChar = *puzPtr) != 0) + { + if( (uiTmp = uiMaxLen - uiEncodedLen) == 0) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( uChar <= 127) + { + if( pucEncPtr) + { + *pucEncPtr++ = (FLMBYTE)uChar; + } + + uiEncodedLen++; + } + else + { + if( RC_BAD( rc = f_uni2UTF8( uChar, pucEncPtr, &uiTmp))) + { + goto Exit; + } + + if( pucEncPtr) + { + pucEncPtr += uiTmp; + } + + uiEncodedLen += uiTmp; + } + + puzPtr++; + uiCharsEncoded++; + } + + // Make sure the string length (which may have been provided by + // the caller) was correct. + + if( uiCharsEncoded != uiStrLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + } + + // Terminate the string with a 0 byte + + if( (uiMaxLen - uiEncodedLen) < 1) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( pucEncPtr) + { + *pucEncPtr++ = 0; + } + uiEncodedLen++; + + // Return the length of the encoded string + + *puiBufLength = uiEncodedLen; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Encode a string into FLAIM's internal text format + (SEN-prefixed, null-terminated UTF8). The string is prefixed with + a SEN that indicates the number of characters, not including the + terminating NULL. + + This routine can be called with a NULL input buffer. If it is + called in this way, the length of the encoded string will be + returned in *puiBufLength, but no encoding will actually be + performed. +****************************************************************************/ +RCODE flmNative2Storage( + char * pszStr, // Native string to encode + FLMUINT uiStrLen, // 0 = Unknown + FLMBYTE * pucBuf, // Destination buffer + FLMUINT * puiBufLength, // [IN ] Size of pucBuf, + // [OUT] Amount of pucBuf used + FLMUINT * puiCharCount) +{ + FLMBYTE * pucEncPtr; + char * pszPtr = NULL; + FLMUINT uiMaxLen; + FLMUINT uiCharsEncoded = 0; + FLMUINT uiEncodedLen = 0; + FLMUINT uiTmp; + FLMBYTE ucTmpSen[ 5]; + FLMBYTE * pucTmpSen = &ucTmpSen[ 0]; + FLMBOOL bDirectCopy = FALSE; + RCODE rc = NE_SFLM_OK; + + if( !pucBuf) + { + uiMaxLen = (~(FLMUINT)0); + } + else + { + uiMaxLen = *puiBufLength; + } + + // If uiStrLen is 0, determine the number of characters. + + if( !uiStrLen) + { +#ifdef FLM_ASCII_PLATFORM + bDirectCopy = TRUE; + for( pszPtr = pszStr; *pszPtr; pszPtr++, uiStrLen++) + { + if( (FLMBYTE)*pszPtr > 0x7F) + { + bDirectCopy = FALSE; + } + } +#else + uiStrLen = f_strlen( pszStr); +#endif + } + else if( pszStr[ uiStrLen] != 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + + if( puiCharCount) + { + *puiCharCount = uiStrLen; + } + + if( !uiStrLen) + { + // Nothing to encode + + *puiBufLength = 0; + goto Exit; + } + + pucEncPtr = pucBuf; + + // Encode the number of characters as a SEN. If pucEncPtr is + // NULL, the caller is only interested in the length of the encoded + // string, so a temporary buffer is used to call f_encodeSEN. + + uiTmp = f_encodeSEN( uiStrLen, &pucTmpSen); + if( pucEncPtr) + { + if( (uiEncodedLen + uiTmp) >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucEncPtr, &ucTmpSen[ 0], uiTmp); + pucEncPtr += uiTmp; + } + uiEncodedLen += uiTmp; + + // Encode the string using UTF-8 + + if( uiStrLen) + { + if( bDirectCopy) + { + // Since all of the characters are ASCII and have + // values <= 0x7F, the string can be copied directly + // into the destination buffer buffer. + + if( uiEncodedLen + uiStrLen >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( pucEncPtr) + { + f_memcpy( pucEncPtr, pszStr, uiStrLen); + pucEncPtr += uiStrLen; + } + uiEncodedLen += uiStrLen; + } + else + { + pszPtr = pszStr; + while( *pszPtr) + { + if( (uiTmp = uiMaxLen - uiEncodedLen) == 0) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + // Convert the native character to ASCII before + // mapping it to Unicode. + + if( RC_BAD( rc = f_uni2UTF8( (FLMUNICODE)f_toascii( *pszPtr), + pucEncPtr, &uiTmp))) + { + goto Exit; + } + + uiEncodedLen += uiTmp; + + if( pucEncPtr) + { + pucEncPtr += uiTmp; + } + + pszPtr++; + uiCharsEncoded++; + } + + // Make sure the string length (which may have been provided by + // the caller) was correct. + + if( uiCharsEncoded != uiStrLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_PARM); + goto Exit; + } + } + } + + // Terminate the string with a 0 byte + + if( (uiMaxLen - uiEncodedLen) == 0) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( pucEncPtr) + { + *pucEncPtr++ = 0; + } + uiEncodedLen++; + + // Return the length of the encoded string + + *puiBufLength = uiEncodedLen; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Encode a string into FLAIM's internal text format + (SEN-prefixed, null-terminated UTF8). The string is prefixed with + a SEN that indicates the number of characters, not including the + terminating NULL. + + This routine can be called with a NULL input buffer. If it is + called in this way, the length of the encoded string will be + returned in *puiBufLength, but no encoding will actually be + performed. +****************************************************************************/ +RCODE flmUTF8ToStorage( + const FLMBYTE * pucUTF8, // UTF8 string to encode + FLMUINT uiBytesInBuffer, // Maximum bytes to process from source + FLMBYTE * pucBuf, // Destination buffer + FLMUINT * puiBufLength) // [IN ] Size of pucBuf, + // [OUT] Amount of pucBuf used +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucEncPtr; + const FLMBYTE * pucPtr = NULL; + FLMUINT uiMaxLen; + FLMUINT uiEncodedLen = 0; + FLMUINT uiCharCount; + FLMUINT uiByteCount; + FLMUINT uiTmp; + FLMUNICODE uChar; + FLMBYTE ucTmpSen[ 5]; + FLMBYTE * pucTmpSen = &ucTmpSen[ 0]; + const FLMBYTE * pucEnd = NULL; + + if( !pucBuf) + { + uiMaxLen = (~(FLMUINT)0); + } + else + { + uiMaxLen = *puiBufLength; + } + + if( uiBytesInBuffer) + { + pucEnd = &pucUTF8[ uiBytesInBuffer]; + } + + // Determine the number of bytes and characters in the + // string + + uiCharCount = 0; + pucPtr = pucUTF8; + for( ;;) + { + if( RC_BAD( rc = f_getCharFromUTF8Buf( &pucPtr, pucEnd, &uChar))) + { + goto Exit; + } + + if( !uChar) + { + break; + } + + uiCharCount++; + } + + if( !uiCharCount) + { + *puiBufLength = 0; + goto Exit; + } + + pucEncPtr = pucBuf; + + // Encode the number of characters as a SEN. If pucEncPtr is + // NULL, the caller is only interested in the length of the encoded + // string, so a temporary buffer is used to call f_encodeSEN. + + uiTmp = f_encodeSEN( uiCharCount, &pucTmpSen); + if( pucEncPtr) + { + if( (uiEncodedLen + uiTmp) >= uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucEncPtr, &ucTmpSen[ 0], uiTmp); + pucEncPtr += uiTmp; + } + uiEncodedLen += uiTmp; + + // Copy the UTF8 characters into the destination buffer + + uiByteCount = (FLMUINT)(pucPtr - pucUTF8); + if( pucEncPtr) + { + if( (uiMaxLen - uiEncodedLen) < uiByteCount) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucEncPtr, pucUTF8, uiByteCount); + pucEncPtr += uiByteCount; + } + uiEncodedLen += uiByteCount; + + // Terminate the string with a 0 byte + + if( uiEncodedLen == uiMaxLen) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + if( pucEncPtr) + { + *pucEncPtr++ = 0; + } + uiEncodedLen++; + + // Return the length of the encoded string + + *puiBufLength = uiEncodedLen; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Convert text storage string to Unicode. +Notes: If puzOutBuf is NULL, only a count is returned + in puiOutBufLen to indicate the number of bytes needed to + contain the data. Two (unicode) bytes must be + added to this value to account for null termination. +****************************************************************************/ +RCODE flmStorage2Unicode( + FLMUINT uiType, + FLMUINT uiBufLength, + const FLMBYTE * pucBuffer, + FLMUINT * puiOutBufLen, + // [IN] Number of bytes available in buffer + // [OUT] Returns the number of bytes that are needed to + // represent the data. The null termination byte(s) are + // not included in this value. + void * pOutBuf) + // [IN/OUT] Buffer to hold the data. +{ + RCODE rc = NE_SFLM_OK; + FLMUNICODE uChar; + FLMUINT uiOffset = 0; + FLMUINT uiSenLen; + FLMUINT uiNumChars; + FLMUINT uiMaxOutChars; + FLMBYTE ucTempBuf[ 64]; + FLMUNICODE * puzOutBuf = NULL; + FLMBYTE * pszOutBuf = NULL; + const FLMBYTE * pucEnd; + FLMUINT uiDecodeCount; + + if( !pucBuffer || !uiBufLength) + { + ucTempBuf[ 0] = 0; // SEN encoding of 0 + ucTempBuf[ 1] = 0; // String terminator + pucBuffer = &ucTempBuf[ 0]; + uiBufLength = 2; + } + else if( uiType != SFLM_STRING_TYPE) + { + // If the value is a number, convert to text + + if( uiType == SFLM_NUMBER_TYPE) + { + FLMUINT uiTmp; + + uiTmp = sizeof( ucTempBuf); + if( RC_BAD( rc = flmStorageNum2StorageText( pucBuffer, uiBufLength, + ucTempBuf, &uiTmp))) + { + goto Exit; + } + pucBuffer = &ucTempBuf[ 0]; + uiBufLength = uiTmp; + } + else + { + rc = RC_SET( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + if( !uiBufLength) + { + if( puiOutBufLen) + { + if( *puiOutBufLen >= 2) + { + *((FLMUNICODE *)pOutBuf) = 0; + } + + *puiOutBufLen = 0; + } + goto Exit; + } + + pucEnd = &pucBuffer[ uiBufLength]; + + uiSenLen = f_getSENLength( *pucBuffer); + if( pucBuffer + uiSenLen >= pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucBuffer, pucEnd, &uiNumChars))) + { + goto Exit; + } + + // If only a length is needed (number of bytes), we can + // return that without parsing the string + + if( !pOutBuf) + { + uiOffset = uiNumChars; + goto Exit; + } + + flmAssert( puiOutBufLen); + uiMaxOutChars = (*puiOutBufLen) / sizeof( FLMUNICODE); + puzOutBuf = (FLMUNICODE *)pOutBuf; + + // If we have a zero-length string, jump to exit. + + if( !uiNumChars) + { + if( (pucBuffer + 1) != pucEnd || *pucBuffer != 0) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( *pucBuffer != 0) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + } + else if (!uiMaxOutChars) + { + goto Overflow_Error; + } + + if( puzOutBuf) + { + *puzOutBuf = 0; + } + else + { + *pszOutBuf = 0; + } + + goto Exit; + } + + // Parse through the string, outputting data to the buffer as we go. + + uChar = 0; + uiDecodeCount = 0; + for( ;;) + { + // Decode the bytes. + + if( RC_BAD( rc = f_getCharFromUTF8Buf( &pucBuffer, pucEnd, &uChar))) + { + goto Exit; + } + + if( !uChar) + { + break; + } + + if( uiOffset == uiMaxOutChars) + { + goto Overflow_Error; + } + + if( puzOutBuf) + { + puzOutBuf[ uiOffset++] = uChar; + } + else + { + if ( uChar <= 0xFF) + { + uChar = (FLMUNICODE)f_tonative( (FLMBYTE)uChar); + pszOutBuf[ uiOffset++] = (FLMBYTE)uChar; + } + else + { + rc = RC_SET( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + uiDecodeCount++; + } + + if( uChar || uiDecodeCount != uiNumChars) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // There is room for the 0 terminating character, but we + // will not increment return length. + + if( uiOffset < uiMaxOutChars) + { + if( puzOutBuf) + { + puzOutBuf[ uiOffset] = 0; + } + else + { + pszOutBuf[ uiOffset] = 0; + } + } + else + { +Overflow_Error: + flmAssert( uiOffset == uiMaxOutChars); + + // If uiOffset is zero, so is uiMaxOutChars, which means + // that we can't even put out the zero terminator. + + if (uiOffset) + { + uiOffset--; + if( puzOutBuf) + { + puzOutBuf[ uiOffset] = 0; + } + else + { + pszOutBuf[ uiOffset] = 0; + } + } + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + +Exit: + + if( puiOutBufLen) + { + *puiOutBufLen = uiOffset + uiOffset; + } + + return( rc); +} + +/**************************************************************************** +Desc: Converts storage formats to UNICODE +****************************************************************************/ +RCODE flmStorage2Unicode( + FLMUINT uiType, + FLMUINT uiStorageLength, + const FLMBYTE * pucStorageBuffer, + F_DynaBuf * pBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucTempBuf[ 80]; + const FLMBYTE * pucEnd; + FLMUINT uiSenLen; + FLMUINT uiNumChars; + FLMUNICODE * puzDestBuffer; + + pBuffer->truncateData( 0); + + if( uiType != SFLM_STRING_TYPE) + { + // If the value is a number, convert to text + + if( uiType == SFLM_NUMBER_TYPE) + { + FLMUINT uiTmp; + + uiStorageLength = sizeof( ucTempBuf); + if( RC_BAD( rc = flmStorageNum2StorageText( pucStorageBuffer, + uiStorageLength, ucTempBuf, &uiTmp))) + { + goto Exit; + } + pucStorageBuffer = &ucTempBuf[ 0]; + uiStorageLength = uiTmp; + } + else + { + rc = RC_SET( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + pucEnd = &pucStorageBuffer[ uiStorageLength]; + uiSenLen = f_getSENLength( *pucStorageBuffer); + + if( pucStorageBuffer + uiSenLen >= pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucStorageBuffer, pucEnd, &uiNumChars))) + { + goto Exit; + } + + if( RC_BAD( rc = pBuffer->allocSpace( + (uiNumChars + 1) * sizeof( FLMUNICODE), (void **)&puzDestBuffer))) + { + goto Exit; + } + + // Parse through the string outputting data to the buffer as we go + + for( ;;) + { + if( RC_BAD( rc = f_getCharFromUTF8Buf( + &pucStorageBuffer, pucEnd, puzDestBuffer))) + { + goto Exit; + } + + if( !(*puzDestBuffer)) + { + break; + } + + puzDestBuffer++; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts a storage buffer to UTF-8 (null-terminated) text +****************************************************************************/ +RCODE flmStorage2UTF8( + FLMUINT uiType, + FLMUINT uiBufLength, + const FLMBYTE * pucBuffer, + FLMUINT * puiOutBufLen, + FLMBYTE * pucOutBuf) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucEnd; + FLMBYTE ucTempBuf[ 64]; + FLMUINT uiSenLen; + + if( !pucBuffer) + { + ucTempBuf[ 0] = 0; // SEN encoding of 0 + ucTempBuf[ 1] = 0; // String terminator + pucBuffer = &ucTempBuf[ 0]; + uiBufLength = 2; + } + else if( uiType != SFLM_STRING_TYPE) + { + // If the value is a number, convert to text + + if( uiType == SFLM_NUMBER_TYPE) + { + FLMUINT uiTmp; + + uiTmp = sizeof( ucTempBuf); + if( RC_BAD( rc = flmStorageNum2StorageText( pucBuffer, uiBufLength, + ucTempBuf, &uiTmp))) + { + goto Exit; + } + pucBuffer = &ucTempBuf[ 0]; + uiBufLength = uiTmp; + } + else + { + rc = RC_SET( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + if( !uiBufLength) + { + if( *puiOutBufLen && pucOutBuf) + { + *pucOutBuf = 0; + } + *puiOutBufLen = 0; + goto Exit; + } + + pucEnd = &pucBuffer[ uiBufLength]; + + uiSenLen = f_getSENLength( *pucBuffer); + if( pucBuffer + uiSenLen >= pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucBuffer, pucEnd, NULL))) + { + goto Exit; + } + + if( pucOutBuf) + { + if( *puiOutBufLen >= uiBufLength - uiSenLen) + { + f_memcpy( pucOutBuf, pucBuffer, uiBufLength - uiSenLen); + } + } + + *puiOutBufLen = (uiBufLength - uiSenLen) - 1; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Converts storage formats to UTF8 +****************************************************************************/ +RCODE flmStorage2UTF8( + FLMUINT uiType, + FLMUINT uiStorageLength, + const FLMBYTE * pucStorageBuffer, + F_DynaBuf * pBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucTempBuf[ 80]; + FLMUINT uiSenLen; + FLMBYTE * pucDestBuffer; + + pBuffer->truncateData( 0); + + if( uiType != SFLM_STRING_TYPE) + { + // If the value is a number, convert to text + + if( uiType == SFLM_NUMBER_TYPE) + { + FLMUINT uiTmp; + + uiStorageLength = sizeof( ucTempBuf); + if( RC_BAD( rc = flmStorageNum2StorageText( pucStorageBuffer, + uiStorageLength, ucTempBuf, &uiTmp))) + { + goto Exit; + } + pucStorageBuffer = &ucTempBuf[ 0]; + uiStorageLength = uiTmp; + } + else + { + rc = RC_SET( NE_SFLM_CONV_ILLEGAL); + goto Exit; + } + } + + uiSenLen = f_getSENLength( *pucStorageBuffer); + if (uiSenLen >= uiStorageLength) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = pBuffer->allocSpace( uiStorageLength - uiSenLen, + (void **)&pucDestBuffer))) + { + goto Exit; + } + f_memcpy( pucDestBuffer, pucStorageBuffer + uiSenLen, + uiStorageLength - uiSenLen); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Reads and returns the value of the SEN indicating the number of + characters encoded into the storage (UTF-8) string +****************************************************************************/ +RCODE flmGetCharCountFromStorageBuf( + const FLMBYTE ** ppucBuf, + FLMUINT uiBufSize, + FLMUINT * puiNumChars, + FLMUINT * puiSenLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiSenLen; + FLMUINT uiNumChars; + + if( !uiBufSize) + { + if( puiNumChars) + { + *puiNumChars = 0; + } + + if( puiSenLen) + { + *puiSenLen = 0; + } + goto Exit; + } + + if( (uiSenLen = f_getSENLength( (*ppucBuf)[ 0])) >= uiBufSize) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( ppucBuf, *ppucBuf + uiSenLen, &uiNumChars))) + { + goto Exit; + } + + if( puiNumChars) + { + *puiNumChars = uiNumChars; + } + + if( puiSenLen) + { + *puiSenLen = uiSenLen; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine converts an internal number to internal (ASCII) text. +Notes: If the buffer pointer is NULL, the routine just determines how + much buffer space is needed to store the number in a text string. +****************************************************************************/ +RCODE flmStorageNum2StorageText( + const FLMBYTE * pucNum, + FLMUINT uiNumLen, + FLMBYTE * pucBuffer, + FLMUINT * puiBufLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64Num; + FLMINT64 i64Num; + FLMUINT uiOffset = 0; + FLMBOOL bNeg = FALSE; + char szTmpBuf[ 64]; + + if( RC_BAD( rc = flmStorage2Number64( SFLM_NUMBER_TYPE, uiNumLen, + pucNum, &ui64Num, NULL))) + { + if( rc == NE_SFLM_CONV_NUM_UNDERFLOW) + { + if( RC_BAD( rc = flmStorage2Number64( SFLM_NUMBER_TYPE, uiNumLen, + pucNum, NULL, &i64Num))) + { + goto Exit; + } + + ui64Num = (FLMUINT64)-i64Num; + bNeg = TRUE; + } + else + { + goto Exit; + } + } + + if( bNeg) + { + szTmpBuf[ uiOffset++] = '-'; + } + + uiOffset += f_sprintf( &szTmpBuf[ uiOffset], "%I64u", ui64Num); + + if( RC_BAD( rc = flmNative2Storage( szTmpBuf, uiOffset, + pucBuffer, puiBufLen, NULL))) + { + goto Exit; + } + +Exit: + + return( rc); +} diff --git a/sql/src/fvector.cpp b/sql/src/fvector.cpp new file mode 100644 index 0000000..fdaa50e --- /dev/null +++ b/sql/src/fvector.cpp @@ -0,0 +1,1355 @@ +//------------------------------------------------------------------------------ +// Desc: Contains the code for the F_DataVector class. +// +// Tabs: 3 +// +// Copyright (c) 2003-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fvector.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#if defined( FLM_WATCOM_NLM) + // Disable "Warning! W549: col(XX) 'sizeof' operand contains + // compiler generated information" + #pragma warning 549 9 +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +F_DataVector::F_DataVector() +{ + m_pVectorElements = &m_VectorArray [0]; + m_uiVectorArraySize = MIN_VECTOR_ELEMENTS; + m_pucDataBuf = m_ucIntDataBuf; + m_uiDataBufLength = sizeof( m_ucIntDataBuf); + reset(); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +F_DataVector::~F_DataVector() +{ + if (m_pVectorElements != &m_VectorArray [0]) + { + f_free( &m_pVectorElements); + } + if (m_pucDataBuf && m_pucDataBuf != m_ucIntDataBuf) + { + f_free( &m_pucDataBuf); + } + reset(); +} + + +/**************************************************************************** +Desc: Clear the data vector, but don't free any buffers that have been + allocated. That will only happen in the destructor. The reset() + method is so that we can get efficient re-use of the vector. So, if + it has allocated buffers, etc. we don't want to free them. +****************************************************************************/ +void F_DataVector::reset( void) +{ + m_ui64RowId = 0; + m_uiNumElements = 0; + m_uiDataBufOffset = 0; +} + +/**************************************************************************** +Desc: Make sure the vector array is allocated at least up to the element + number that is passed in. +****************************************************************************/ +RCODE F_DataVector::allocVectorArray( + FLMUINT uiElementNumber) +{ + RCODE rc = NE_SFLM_OK; + + if (uiElementNumber >= m_uiNumElements) + { + + // May need to allocate a new vector array + + if (uiElementNumber >= m_uiVectorArraySize) + { + FLMUINT uiNewArraySize = uiElementNumber + 32; + F_VECTOR_ELEMENT * pNewVector; + + if (m_pVectorElements == &m_VectorArray [0]) + { + if (RC_BAD( rc = f_alloc( uiNewArraySize * sizeof( F_VECTOR_ELEMENT), + &pNewVector))) + { + goto Exit; + } + if (m_uiNumElements) + { + f_memcpy( pNewVector, m_pVectorElements, + m_uiNumElements * sizeof( F_VECTOR_ELEMENT)); + } + } + else + { + pNewVector = m_pVectorElements; + + if (RC_BAD( rc = f_realloc( uiNewArraySize * sizeof( F_VECTOR_ELEMENT), + &pNewVector))) + { + goto Exit; + } + + } + m_pVectorElements = pNewVector; + m_uiVectorArraySize = uiNewArraySize; + } + + // Initialized everything between the old last element and + // the new element, including the new element, to zeroes. + + f_memset( &m_pVectorElements [m_uiNumElements], 0, + sizeof( F_VECTOR_ELEMENT) * + (uiElementNumber - m_uiNumElements + 1)); + + m_uiNumElements = uiElementNumber + 1; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Store a data value into its vector element. +****************************************************************************/ +RCODE F_DataVector::storeValue( + FLMINT uiElementNumber, + eDataType eDataTyp, + const FLMBYTE * pucData, + FLMUINT uiDataLen, + FLMBYTE ** ppucDataPtr) +{ + RCODE rc = NE_SFLM_OK; + F_VECTOR_ELEMENT * pVector; + FLMBYTE * pucDataPtr; + FLMUINT uiTemp; + + // Find or allocate space for the vector + + if (RC_BAD( rc = allocVectorArray( uiElementNumber))) + { + goto Exit; + } + + pVector = &m_pVectorElements [uiElementNumber]; + + // Will the data fit inside uiDataOffset? + + if (uiDataLen <= sizeof( FLMUINT)) + { + pucDataPtr = (FLMBYTE *)&pVector->uiDataOffset; + } + else if (uiDataLen <= pVector->uiDataLength) + { + + // New data will fit in original space. Simply reuse it. + + pucDataPtr = m_pucDataBuf + pVector->uiDataOffset; + } + else + { + + // New data will not fit in originally allocated space. + // Must allocate new space. + + // Always align the new allocation so that if it gets + // reused later for binary data it will be properly aligned. + + if ((m_uiDataBufOffset & FLM_ALLOC_ALIGN) != 0) + { + uiTemp = (FLM_ALLOC_ALIGN + 1) - (m_uiDataBufOffset & FLM_ALLOC_ALIGN); + m_uiDataBufOffset += uiTemp; + } + + if (uiDataLen + m_uiDataBufOffset > m_uiDataBufLength) + { + // Re-allocate the data buffer. + + if( m_pucDataBuf == m_ucIntDataBuf) + { + if (RC_BAD( rc = f_alloc( + m_uiDataBufOffset + uiDataLen + 512, + &m_pucDataBuf))) + { + goto Exit; + } + + f_memcpy( m_pucDataBuf, m_ucIntDataBuf, m_uiDataBufOffset); + } + else + { + if (RC_BAD( rc = f_realloc( + m_uiDataBufOffset + uiDataLen + 512, + &m_pucDataBuf))) + { + goto Exit; + } + } + + m_uiDataBufLength = m_uiDataBufOffset + uiDataLen + 512; + } + pucDataPtr = m_pucDataBuf + m_uiDataBufOffset; + pVector->uiDataOffset = m_uiDataBufOffset; + m_uiDataBufOffset += uiDataLen; + } + + // Store the data - may be zero length. + + if( pucData) + { + if( uiDataLen > 1) + { + f_memcpy( pucDataPtr, pucData, uiDataLen); + } + else if( uiDataLen) + { + *pucDataPtr = *pucData; + } + } + + pVector->uiFlags |= VECT_SLOT_HAS_DATA; + pVector->uiDataLength = uiDataLen; + pVector->eDataTyp = eDataTyp; + + if( ppucDataPtr) + { + *ppucDataPtr = pucDataPtr; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set the column number for a vector element. +****************************************************************************/ +RCODE F_DataVector::setColumnNum( + FLMUINT uiElementNumber, + FLMUINT uiColumnNum, + FLMBOOL bIsData) +{ + RCODE rc = NE_SFLM_OK; + F_VECTOR_ELEMENT * pVector; + + // Find or allocate space for the vector element + + if (RC_BAD( rc = allocVectorArray( uiElementNumber))) + { + goto Exit; + } + pVector = &m_pVectorElements [uiElementNumber]; + + pVector->uiFlags |= VECT_SLOT_HAS_COLUMN_NUM; + if (bIsData) + { + pVector->uiFlags |= VECT_SLOT_IS_DATA; + } + else + { + pVector->uiFlags &= (~(VECT_SLOT_IS_DATA)); + } + pVector->uiColumnNum = uiColumnNum; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a FLMINT value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setINT( + FLMUINT uiElementNumber, + FLMINT iNum) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucStorageBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + FLMBOOL bNeg = FALSE; + + if (iNum < 0) + { + bNeg = TRUE; + iNum = -iNum; + } + + uiStorageLen = sizeof( ucStorageBuf); + if( ((FLMUINT)iNum) <= gv_uiMaxUInt32Val) + { + if( RC_BAD( rc = flmNumber64ToStorage( (FLMUINT64)iNum, + &uiStorageLen, ucStorageBuf, bNeg, FALSE))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = flmNumber64ToStorage( (FLMUINT64)iNum, + &uiStorageLen, ucStorageBuf, bNeg, FALSE))) + { + goto Exit; + } + } + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_NUMBER_TYPE, ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a FLMINT64 value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setINT64( + FLMUINT uiElementNumber, + FLMINT64 i64Num) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucStorageBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + FLMBOOL bNeg = FALSE; + + if (i64Num < 0) + { + bNeg = TRUE; + i64Num = -i64Num; + } + + uiStorageLen = sizeof( ucStorageBuf); + if (RC_BAD( rc = flmNumber64ToStorage( i64Num, &uiStorageLen, + ucStorageBuf, bNeg, FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_NUMBER_TYPE, ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a FLMUINT value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setUINT( + FLMUINT uiElementNumber, + FLMUINT uiNum) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucStorageBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + + uiStorageLen = sizeof( ucStorageBuf); + if (uiNum <= gv_uiMaxUInt32Val) + { + if (RC_BAD( rc = flmNumber64ToStorage( uiNum, + &uiStorageLen, ucStorageBuf, FALSE, FALSE))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = flmNumber64ToStorage( uiNum, + &uiStorageLen, ucStorageBuf, FALSE, FALSE))) + { + goto Exit; + } + } + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_NUMBER_TYPE, ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a FLMUINT64 value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setUINT64( + FLMUINT uiElementNumber, + FLMUINT64 ui64Num) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucStorageBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + + uiStorageLen = sizeof( ucStorageBuf); + if (RC_BAD( rc = flmNumber64ToStorage( ui64Num, + &uiStorageLen, ucStorageBuf, FALSE, FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_NUMBER_TYPE, ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a FLMUNICODE value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setUnicode( + FLMUINT uiElementNumber, + const FLMUNICODE * puzUnicode) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucDataPtr; + FLMUINT uiLen; + FLMUINT uiCharCount; + FLMBYTE ucTmpBuf [64]; + + // A NULL or empty puzUnicode string is allowed - on those cases + // just set the data type. + + if (puzUnicode == NULL || *puzUnicode == 0) + { + rc = storeValue( uiElementNumber, SFLM_STRING_TYPE, NULL, 0); + goto Exit; + } + + // See if it will fit in our temporary buffer on the stack. + + uiLen = sizeof( ucTmpBuf); + if (RC_OK( rc = flmUnicode2Storage( puzUnicode, 0, ucTmpBuf, + &uiLen, &uiCharCount))) + { + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_STRING_TYPE, ucTmpBuf, uiLen))) + { + goto Exit; + } + } + else if (rc != NE_SFLM_CONV_DEST_OVERFLOW) + { + goto Exit; + } + else + { + + // Determine the length needed. + + if (RC_BAD( rc = flmUnicode2Storage( puzUnicode, 0, NULL, + &uiLen, &uiCharCount))) + { + goto Exit; + } + + // Allocate space for it in the vector and get a pointer + // back so we can then store it. + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_STRING_TYPE, NULL, uiLen, &pucDataPtr))) + { + goto Exit; + } + + // Store it out to the space we just allocated. + + if (RC_BAD( rc = flmUnicode2Storage( puzUnicode, uiCharCount, + pucDataPtr, &uiLen, NULL))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Set a UTF8 value for a vector element. +****************************************************************************/ +RCODE F_DataVector::setUTF8( + FLMUINT uiElementNumber, + const FLMBYTE * pszUTF8, + FLMUINT uiBytesInBuffer) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucDataPtr; + FLMUINT uiLen; + FLMBYTE ucTmpBuf [64]; + + // A NULL or empty pszNative string is allowed - on those cases + // just set the data type. + + if (pszUTF8 == NULL || *pszUTF8 == 0) + { + rc = storeValue( uiElementNumber, SFLM_STRING_TYPE, NULL, 0); + goto Exit; + } + + // See if it will fit in our temporary buffer on the stack. + + uiLen = sizeof( ucTmpBuf); + if (RC_OK( rc = flmUTF8ToStorage( + pszUTF8, uiBytesInBuffer, ucTmpBuf, &uiLen))) + { + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_STRING_TYPE, ucTmpBuf, uiLen))) + { + goto Exit; + } + } + else if (rc != NE_SFLM_CONV_DEST_OVERFLOW) + { + goto Exit; + } + else + { + // Determine the length needed. + + if (RC_BAD( rc = flmUTF8ToStorage( + pszUTF8, uiBytesInBuffer, NULL, &uiLen))) + { + goto Exit; + } + + // Allocate space for it in the vector and get a pointer + // back so we can then store it. + + if (RC_BAD( rc = storeValue( uiElementNumber, + SFLM_STRING_TYPE, NULL, uiLen, &pucDataPtr))) + { + goto Exit; + } + + // Store it out to the space we just allocated. + + if (RC_BAD( rc = flmUTF8ToStorage( + pszUTF8, uiBytesInBuffer, pucDataPtr, &uiLen))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Get a pointer to the UTF8 - no conversions are done. +****************************************************************************/ +RCODE F_DataVector::getUTF8Ptr( + FLMUINT uiElementNumber, + const FLMBYTE ** ppszUTF8, + FLMUINT * puiBufLen) +{ + RCODE rc = NE_SFLM_OK; + F_VECTOR_ELEMENT * pVector = getVector( uiElementNumber, + VECT_SLOT_HAS_DATA); + void * pvValue; + FLMUINT uiStorageLen; + FLMUINT uiSenLen; + + if (!pVector) + { + *ppszUTF8 = NULL; + if (puiBufLen) + { + *puiBufLen = 0; + } + goto Exit; + } + if (pVector->eDataTyp != SFLM_STRING_TYPE) + { + rc = RC_SET( NE_SFLM_BAD_DATA_TYPE); + goto Exit; + } + + if ((pvValue = getDataPtr( pVector)) != NULL) + { + *ppszUTF8 = (FLMBYTE *)pvValue; + uiStorageLen = pVector->uiDataLength; + if( RC_BAD( rc = flmGetCharCountFromStorageBuf( ppszUTF8, + uiStorageLen, NULL, &uiSenLen))) + { + goto Exit; + } + + flmAssert( uiStorageLen > uiSenLen); + uiStorageLen -= uiSenLen; + } + else + { + *ppszUTF8 = NULL; + uiStorageLen = 0; + } + + if (puiBufLen) + { + *puiBufLen = uiStorageLen; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Allocate data for a unicode element and retrieve it. +****************************************************************************/ +RCODE F_DataVector::getUnicode( + FLMUINT uiElementNumber, + FLMUNICODE ** ppuzUnicode) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLen; + + // Get the unicode length (does not include NULL terminator) + + if (RC_BAD( rc = getUnicode( uiElementNumber, NULL, &uiLen))) + { + goto Exit; + } + + if (uiLen) + { + + // Account for NULL character. + + uiLen += sizeof( FLMUNICODE); + + if( RC_BAD( rc = f_alloc( uiLen, ppuzUnicode))) + { + goto Exit; + } + + if (RC_BAD( rc = getUnicode( uiElementNumber, *ppuzUnicode, + &uiLen))) + { + goto Exit; + } + } + else + { + *ppuzUnicode = NULL; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Compose a key buffer from the vector's components. +****************************************************************************/ +RCODE F_DataVector::outputKey( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMUINT uiMatchFlags, + FLMBYTE * pucKeyBuf, + FLMUINT uiKeyBufSize, + FLMUINT * puiKeyLen, + FLMUINT uiSearchKeyFlag) +{ + RCODE rc = NE_SFLM_OK; + ICD * pIcd; + F_Dict * pDict = pDb->getDict(); + F_INDEX * pIndex = pDict->getIndex( uiIndexNum); + F_TABLE * pTable = pDict->getTable( pIndex->uiTableNum); + F_COLUMN * pColumn; + FLMBYTE * pucToKey; + FLMBYTE * pucKeyLenPos; + FLMUINT uiToKeyLen; + FLMUINT uiKeyLen; + FLMUINT uiKeyComponent; + FLMBOOL bDataTruncated; + eDataType eDataTyp; + FLMUINT uiLanguage; + F_VECTOR_ELEMENT * pVector = NULL; + FLMBYTE ucIDBuf [10]; + FLMUINT uiIDLen = 0; + FLMUINT uiMaxKeySize; + FLMUINT uiDataLen; + const FLMBYTE * pucDataPtr; + FLMUINT uiIDMatchFlags = uiMatchFlags & FLM_MATCH_ROW_ID; + F_BufferIStream bufferIStream; + + if (uiIDMatchFlags) + { + pucToKey = &ucIDBuf [0]; + uiIDLen = f_encodeSEN( m_ui64RowId, &pucToKey); + if (uiIDLen >= uiKeyBufSize) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + } + + // Output the key components + + uiMaxKeySize = uiKeyBufSize - uiIDLen; + uiLanguage = pIndex->uiLanguage; + uiKeyLen = 0; + pucToKey = pucKeyBuf; + for (uiKeyComponent = 0, pIcd = pIndex->pKeyIcds; + uiKeyComponent < pIndex->uiNumKeyComponents; + uiKeyComponent++, pIcd++) + { + pucKeyLenPos = pucToKey; + pucToKey += 2; + uiKeyLen += 2; + pColumn = pDict->getColumn( pTable, pIcd->uiColumnNum); + + eDataTyp = pColumn->eDataTyp; + + // Find matching node in the tree - if not found skip and continue. + + if ((pVector = getVector( uiKeyComponent, VECT_SLOT_HAS_DATA)) == NULL) + { + UW2FBA( 0, pucKeyLenPos); + } + else + { + uiToKeyLen = 0; + bDataTruncated = FALSE; + + // Take the dictionary number and make it the key + + if (pIcd->uiFlags & ICD_PRESENCE) + { + // Output the column number + + if (uiKeyLen + 4 > uiMaxKeySize) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + + f_UINT32ToBigEndian( (FLMUINT32)pIcd->uiColumnNum, pucToKey); + uiToKeyLen = 4; + } + else if (pIcd->uiFlags & ICD_METAPHONE) + { + FLMUINT uiMeta; + FLMBYTE ucStorageBuf[ FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + + if (eDataTyp != SFLM_STRING_TYPE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_DATA_TYPE); + goto Exit; + } + + if (pVector->eDataTyp == SFLM_STRING_TYPE) + { + if (RC_BAD( rc = getUTF8Ptr( uiKeyComponent, + &pucDataPtr, &uiDataLen))) + { + goto Exit; + } + + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucDataPtr, uiDataLen))) + { + goto Exit; + } + + if (RC_BAD( rc = f_getNextMetaphone( &bufferIStream, &uiMeta))) + { + if( rc == NE_SFLM_EOF_HIT) + { + rc = RC_SET( NE_SFLM_BAD_DATA_TYPE); + } + goto Exit; + } + bufferIStream.close(); + } + else if (pVector->eDataTyp == SFLM_NUMBER_TYPE) + { + if( RC_BAD( rc = getUINT( uiKeyComponent, &uiMeta))) + { + goto Exit; + } + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_DATA_TYPE); + goto Exit; + } + + if (uiMeta) + { + uiStorageLen = FLM_MAX_NUM_BUF_SIZE; + if( RC_BAD( rc = flmNumber64ToStorage( uiMeta, + &uiStorageLen, ucStorageBuf, FALSE, FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = bufferIStream.open( + (const char *)ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + + // Output the metaphone key piece + + uiToKeyLen = uiMaxKeySize - uiKeyLen; + if( RC_BAD( rc = KYCollateValue( pucToKey, &uiToKeyLen, + &bufferIStream, SFLM_NUMBER_TYPE, + pIcd->uiFlags, pIcd->uiCompareRules, pIcd->uiLimit, + NULL, NULL, uiLanguage, + FALSE, FALSE, &bDataTruncated, NULL))) + { + goto Exit; + } + bufferIStream.close(); + } + } + else + { + if (eDataTyp == SFLM_STRING_TYPE) + { + if (RC_BAD( rc = getUTF8Ptr( uiKeyComponent, + &pucDataPtr, &uiDataLen))) + { + goto Exit; + } + } + else + { + pucDataPtr = (FLMBYTE *)getDataPtr( pVector); + uiDataLen = pVector->uiDataLength; + } + + if (uiDataLen) + { + + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucDataPtr, uiDataLen))) + { + goto Exit; + } + + uiToKeyLen = uiMaxKeySize - uiKeyLen; + if( RC_BAD( rc = KYCollateValue( pucToKey, &uiToKeyLen, + &bufferIStream, eDataTyp, + pIcd->uiFlags, pIcd->uiCompareRules, pIcd->uiLimit, + NULL, NULL, uiLanguage, + (FLMBOOL) ((pIcd->uiFlags & ICD_SUBSTRING) + ? (isLeftTruncated( pVector) + ? FALSE : TRUE) + : FALSE), + isRightTruncated( pVector), + &bDataTruncated, NULL))) + { + goto Exit; + } + + bufferIStream.close(); + } + } + + if (uiToKeyLen) + { + + // Increment total key length + + pucToKey += uiToKeyLen; + uiKeyLen += uiToKeyLen; + } + if (!bDataTruncated) + { + UW2FBA( (FLMUINT16)(uiToKeyLen | uiSearchKeyFlag), + pucKeyLenPos); + } + else + { + UW2FBA( (FLMUINT16)(uiToKeyLen | TRUNCATED_FLAG | + uiSearchKeyFlag), pucKeyLenPos); + } + } + } + + // Output the row ID, if requested. + + if (uiIDMatchFlags) + { + + // There will always be room at this point for the + // row ID - because it was subtracted out above. + + f_memcpy( pucToKey, ucIDBuf, uiIDLen); + } + *puiKeyLen = uiKeyLen + uiIDLen; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Populate a vector's components from the key part of an index key. +****************************************************************************/ +RCODE F_DataVector::inputKey( + F_Db * pDb, + FLMUINT uiIndexNum, + const FLMBYTE * pucKey, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + F_Dict * pDict = pDb->getDict(); + F_INDEX * pIndex = pDict->getIndex( uiIndexNum); + F_TABLE * pTable = pDict->getTable( pIndex->uiTableNum); + F_COLUMN * pColumn; + const FLMBYTE * pucKeyEnd = pucKey + uiKeyLen; + FLMBYTE ucDataBuf [SFLM_MAX_KEY_SIZE]; + FLMUINT uiDataLen; + ICD * pIcd; + FLMUINT uiLanguage = pIndex->uiLanguage; + FLMUINT uiComponentLen; + eDataType eDataTyp; + FLMBOOL bDataRightTruncated; + FLMBOOL bFirstSubstring; + FLMBOOL bIsText; + FLMUINT uiComponent; + FLMUINT uiColumnNum; + + flmAssert( uiKeyLen); + + // Loop for each compound piece of key + + for (uiComponent = 0, pIcd = pIndex->pKeyIcds; + uiComponent < pIndex->uiNumKeyComponents; + pIcd++, uiComponent++) + { + if (uiKeyLen < 2) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + uiColumnNum = pIcd->uiColumnNum; + pColumn = pDict->getColumn( pTable, uiColumnNum); + + uiComponentLen = getKeyComponentLength( pucKey); + bDataRightTruncated = isKeyComponentTruncated( pucKey); + uiKeyLen -= 2; + pucKey += 2; + + if (uiComponentLen > uiKeyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + bFirstSubstring = FALSE; + eDataTyp = pColumn->eDataTyp; + bIsText = (eDataTyp == SFLM_STRING_TYPE && + !(pIcd->uiFlags & (ICD_PRESENCE | ICD_METAPHONE))) + ? TRUE + : FALSE; + + if (uiComponentLen) + { + if (pIcd->uiFlags & ICD_PRESENCE) + { + FLMUINT uiNum; + + if (uiComponentLen != 4 || bDataRightTruncated) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + uiNum = (FLMUINT)f_bigEndianToUINT32( pucKey); + + // What is stored in the key better match the column + // number of the ICD. + + if (uiNum != uiColumnNum) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + if (RC_BAD( rc = setUINT( uiComponent, uiNum))) + { + goto Exit; + } + } + else if (pIcd->uiFlags & ICD_METAPHONE) + { + uiDataLen = sizeof( ucDataBuf); + + if (uiComponentLen) + { + if( RC_BAD( rc = flmCollationNum2StorageNum( pucKey, + uiComponentLen, ucDataBuf, &uiDataLen))) + { + goto Exit; + } + } + else + { + uiDataLen = 0; + } + + if (bDataRightTruncated) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + + // Allocate and copy value into the component. NOTE: + // storeValue handles zero length data. + + if (RC_BAD( rc = storeValue( uiComponent, SFLM_NUMBER_TYPE, + ucDataBuf, uiDataLen))) + { + goto Exit; + } + } + else + { + + // Grab only the Nth section of key if compound key + + switch (eDataTyp) + { + case SFLM_STRING_TYPE: + { + FLMBOOL bTmpTruncated = FALSE; + + if (uiComponentLen) + { + uiDataLen = sizeof( ucDataBuf); + if (RC_BAD( rc = flmColText2StorageText( pucKey, + uiComponentLen, + ucDataBuf, &uiDataLen, uiLanguage, + &bTmpTruncated, &bFirstSubstring))) + { + goto Exit; + } + + if (bTmpTruncated != bDataRightTruncated) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + } + else + { + uiDataLen = 0; + } + break; + } + + case SFLM_NUMBER_TYPE: + { + if (uiComponentLen) + { + uiDataLen = sizeof( ucDataBuf); + if( RC_BAD( rc = flmCollationNum2StorageNum( pucKey, + uiComponentLen, ucDataBuf, &uiDataLen))) + { + goto Exit; + } + } + else + { + uiDataLen = 0; + } + + if (bDataRightTruncated) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BTREE_ERROR); + goto Exit; + } + break; + } + + case SFLM_BINARY_TYPE: + { + uiDataLen = uiComponentLen; + if (uiComponentLen > sizeof( ucDataBuf)) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + if (uiComponentLen) + { + f_memcpy( ucDataBuf, pucKey, uiComponentLen); + } + break; + } + + default: + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Allocate and copy value into the component. NOTE: + // storeValue handles zero length data. + + if (RC_BAD( rc = storeValue( uiComponent, eDataTyp, ucDataBuf, + uiDataLen))) + { + goto Exit; + } + + // Set first sub-string and truncated flags. + + if ((pIcd->uiFlags & ICD_SUBSTRING) && !bFirstSubstring) + { + setLeftTruncated( uiComponent); + } + if (bDataRightTruncated) + { + setRightTruncated( uiComponent); + } + } + } + + // Store the column number + + if (RC_BAD( rc = setColumnNum( uiComponent, uiColumnNum, FALSE))) + { + goto Exit; + } + + // Position to the end of this component + + flmAssert( uiKeyLen >= uiComponentLen); + pucKey += uiComponentLen; + uiKeyLen -= uiComponentLen; + } + + // See if we have a row ID. + + if (RC_BAD( rc = f_decodeSEN64( &pucKey, pucKeyEnd, &m_ui64RowId))) + { + goto Exit; + } + + // Store the column numbers for the data components, if any. + + for (uiComponent = 0, pIcd = pIndex->pDataIcds; + uiComponent < pIndex->uiNumDataComponents; + pIcd++, uiComponent++) + { + + // Store the column number + + if (RC_BAD( rc = setColumnNum( uiComponent + pIndex->uiNumKeyComponents, + pIcd->uiColumnNum, TRUE))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Compose a data buffer from the vector's components. +****************************************************************************/ +RCODE F_DataVector::outputData( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMBYTE * pucDataBuf, + FLMUINT uiDataBufSize, + FLMUINT * puiDataLen) +{ + RCODE rc = NE_SFLM_OK; + F_Dict * pDict = pDb->getDict(); + F_INDEX * pIndex = pDict->getIndex( uiIndexNum); + F_TABLE * pTable = pDict->getTable( pIndex->uiTableNum); + F_COLUMN * pColumn; + ICD * pIcd; + FLMUINT uiDataComponent; + F_VECTOR_ELEMENT * pVector; + FLMBYTE * pucData; + FLMUINT uiDataLength; + FLMUINT uiTotalLength = 0; + FLMBYTE ucTmpSen [32]; + FLMBYTE * pucTmpSen = &ucTmpSen [0]; + FLMUINT uiSENLen; + FLMUINT uiLastDataLen = 0; + + // Loop for each data component of key + + for (uiDataComponent = 0, pIcd = pIndex->pDataIcds; + uiDataComponent < pIndex->uiNumDataComponents; + pIcd++, uiDataComponent++) + { + pColumn = pDict->getColumn( pTable, pIcd->uiColumnNum); + if ((pVector = getVector( uiDataComponent + pIndex->uiNumKeyComponents, + VECT_SLOT_HAS_DATA)) != NULL) + { + + // Cannot do data conversions right now. + + flmAssert( pVector->eDataTyp == pColumn->eDataTyp); + uiDataLength = pVector->uiDataLength; + pucData = (FLMBYTE *)getDataPtr( pVector); + } + else + { + uiDataLength = 0; + pucData = NULL; + } + + // Output the length of the data as a SEN value + + uiSENLen = f_encodeSEN( uiDataLength, &pucTmpSen); + if (uiTotalLength + uiSENLen > uiDataBufSize) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucDataBuf, ucTmpSen, uiSENLen); + pucDataBuf += uiSENLen; + uiTotalLength += uiSENLen; + + // Output the data + + if (uiDataLength) + { + if (uiTotalLength + uiDataLength > uiDataBufSize) + { + rc = RC_SET( NE_SFLM_CONV_DEST_OVERFLOW); + goto Exit; + } + f_memcpy( pucDataBuf, pucData, uiDataLength); + pucDataBuf += uiDataLength; + uiTotalLength += uiDataLength; + uiLastDataLen = uiTotalLength; + } + } + +Exit: + + // Even if rc == NE_SFLM_CONV_DEST_OVERFLOW, return a length + + *puiDataLen = uiLastDataLen; + + return( rc); +} + +/**************************************************************************** +Desc: Populate a vector's data components from the data part of a key. +****************************************************************************/ +RCODE F_DataVector::inputData( + F_Db * pDb, + FLMUINT uiIndexNum, + const FLMBYTE * pucData, + FLMUINT uiInputLen) +{ + RCODE rc = NE_SFLM_OK; + F_Dict * pDict = pDb->getDict(); + F_INDEX * pIndex = pDict->getIndex( uiIndexNum); + F_TABLE * pTable = pDict->getTable( pIndex->uiTableNum); + F_COLUMN * pColumn; + ICD * pIcd; + FLMUINT uiDataComponent = 0; + FLMUINT uiDataLength; + FLMUINT uiSENLen; + + // Loop for each data component of key + + for (uiDataComponent = 0, pIcd = pIndex->pDataIcds; + uiDataComponent < pIndex->uiNumDataComponents; + pIcd++, uiDataComponent++) + { + pColumn = pDict->getColumn( pTable, pIcd->uiColumnNum); + if (RC_BAD( rc = setColumnNum( uiDataComponent + pIndex->uiNumKeyComponents, + pIcd->uiColumnNum, TRUE))) + { + goto Exit; + } + if (!uiInputLen) + { + continue; + } + + // Get the data length - it is stored as a SEN + + uiSENLen = f_getSENLength( *pucData); + if (uiSENLen > uiInputLen) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucData, + &pucData[ uiSENLen], &uiDataLength))) + { + goto Exit; + } + + uiInputLen -= uiSENLen; + if (uiDataLength > uiInputLen) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Store the data into the vector. + + if (RC_BAD( rc = storeValue( uiDataComponent + pIndex->uiNumKeyComponents, + pColumn->eDataTyp, + pucData, uiDataLength, NULL))) + { + goto Exit; + } + pucData += uiDataLength; + uiInputLen -= uiDataLength; + } + +Exit: + + return( rc); +} + diff --git a/sql/src/fxml.h b/sql/src/fxml.h new file mode 100644 index 0000000..c238fc1 --- /dev/null +++ b/sql/src/fxml.h @@ -0,0 +1,582 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the FLAIM XML wrapper class +// +// Tabs: 3 +// +// Copyright (c) 1999-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fxml.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FXML_H +#define FXML_H + +/*============================================================================ +Desc: FLAIM's XML namespace class +============================================================================*/ +class F_XMLNamespace : public F_Object +{ +public: + + FINLINE F_XMLNamespace() + { + m_puzPrefix = NULL; + m_puzURI = NULL; + m_pNext = NULL; + } + + FINLINE ~F_XMLNamespace() + { + flmAssert( !m_pNext); + + if( m_puzPrefix) + { + f_free( &m_puzPrefix); + } + + if( m_puzURI) + { + f_free( &m_puzURI); + } + } + + RCODE setPrefix( + FLMUNICODE * puzPrefix); + + RCODE setURI( + FLMUNICODE * puzURI); + + RCODE setup( + FLMUNICODE * puzPrefix, + FLMUNICODE * puzURI, + F_XMLNamespace * pNext); + + FINLINE FLMUNICODE * getPrefixPtr( void) + { + return( m_puzPrefix); + } + + FINLINE FLMUNICODE * getURIPtr( void) + { + return( m_puzURI); + } + +private: + + FLMUNICODE * m_puzPrefix; + FLMUNICODE * m_puzURI; + F_XMLNamespace * m_pNext; + +friend class F_XMLNamespaceMgr; +}; + +/*============================================================================ +Desc: Namespace manager class +============================================================================*/ +class F_XMLNamespaceMgr : public F_Object +{ +public: + + F_XMLNamespaceMgr(); + + ~F_XMLNamespaceMgr(); + + RCODE findNamespace( + FLMUNICODE * puzPrefix, + F_XMLNamespace ** ppNamespace, + FLMUINT uiMaxSearchSize = ~((FLMUINT)0)); + + RCODE pushNamespace( + FLMUNICODE * puzPrefix, + FLMUNICODE * puzNamespaceURI); + + RCODE pushNamespace( + F_XMLNamespace * pNamespace); + + void popNamespaces( + FLMUINT uiCount); + + FLMUINT getNamespaceCount( void) + { + return( m_uiNamespaceCount); + } + +private: + + F_XMLNamespace * m_pFirstNamespace; + FLMUINT m_uiNamespaceCount; +}; + +// Typedefs + +typedef enum +{ + XML_STATS +} eXMLStatus; + +// This callback is currently only used by the non-com utilities, +// which is why we haven't bothered to make it an interface +typedef RCODE (* XML_STATUS_HOOK)( + eXMLStatus eStatusType, + void * pvArg1, + void * pvArg2, + void * pvArg3, + void * pvUserData); + +/*============================================================================ +Desc: FLAIM's XML import class +============================================================================*/ +class F_XMLImport : public F_XMLNamespaceMgr +{ +public: + + F_XMLImport(); + + ~F_XMLImport(); + + RCODE setup( void); + + void reset( void); + + RCODE import( + IF_IStream * pStream, + F_Db * pDb, + FLMUINT uiCollection, + FLMUINT uiFlags, + F_DOMNode * pNodeToLinkTo, + eNodeInsertLoc eInsertLoc, + F_DOMNode ** ppNewNode, + XFLM_IMPORT_STATS * pImportStats); + + FINLINE void setStatusCallback( + XML_STATUS_HOOK fnStatus, + void * pvUserData) + { + m_fnStatus = fnStatus; + m_pvCallbackData = pvUserData; + } + +private: + + #define F_DEFAULT_NS_DECL 0x01 + #define F_PREFIXED_NS_DECL 0x02 + + typedef struct xmlattr + { + FLMUINT uiLineNum; + FLMUINT uiLineOffset; + FLMUINT uiLineFilePos; + FLMUINT uiLineBytes; + FLMUINT uiValueLineNum; + FLMUINT uiValueLineOffset; + FLMUNICODE * puzPrefix; + FLMUNICODE * puzLocalName; + FLMUNICODE * puzVal; + FLMUINT uiFlags; + xmlattr * pPrev; + xmlattr * pNext; + } XML_ATTR; + + // Methods + + RCODE getFieldTagAndType( + FLMUNICODE * puzName, + FLMBOOL bOkToAdd, + FLMUINT * puiTagNum, + FLMUINT * puiDataType); + + RCODE getByte( + FLMBYTE * pucByte); + + FINLINE void ungetByte( + FLMBYTE ucByte) + { + // Can only unget a single byte. + + flmAssert( !m_ucUngetByte); + m_ucUngetByte = ucByte; + m_importStats.uiChars--; + } + + RCODE getLine( void); + + FINLINE FLMUNICODE getChar( void) + { + if (m_uiCurrLineOffset == m_uiCurrLineNumChars) + { + return( (FLMUNICODE)0); + } + else + { + FLMUNICODE uzChar = m_puzCurrLineBuf [m_uiCurrLineOffset++]; + return( uzChar); + } + } + + FINLINE FLMUNICODE peekChar( void) + { + if (m_uiCurrLineOffset == m_uiCurrLineNumChars) + { + return( (FLMUNICODE)0); + } + else + { + return( m_puzCurrLineBuf [m_uiCurrLineOffset]); + } + } + + FINLINE void ungetChar( void) + { + + // There should never be a reason to unget past the beginning of the current + // line. + + flmAssert( m_uiCurrLineOffset); + m_uiCurrLineOffset--; + } + + RCODE getName( + FLMUINT * puiChars); + + RCODE getQualifiedName( + FLMUINT * puiChars, + FLMUNICODE ** ppuzPrefix, + FLMUNICODE ** ppuzLocal, + FLMBOOL * pbNamespaceDecl, + FLMBOOL * pbDefaultNamespaceDecl); + + void getNmtoken( + FLMUINT * puiChars); + + RCODE getPubidLiteral( void); + + RCODE getSystemLiteral( void); + + RCODE getElementValue( + FLMUNICODE * puBuf, + FLMUINT * puiMaxChars, + FLMBOOL * pbEntity); + + RCODE processEntityValue( void); + + RCODE getEntity( + FLMUNICODE * puBuf, + FLMUINT * puiChars, + FLMBOOL * pbTranslated, + FLMUNICODE * puTransChar); + + RCODE processReference( + FLMUNICODE * puChar = NULL); + + RCODE processCDATA( + F_DOMNode * pParent, + FLMUINT uiSavedLineNum, + FLMUINT uiSavedOffset, + FLMUINT uiSavedFilePos, + FLMUINT uiSavedLineBytes); + + RCODE processAttributeList( void); + + RCODE processComment( + F_DOMNode * pParent, + FLMUINT uiSavedLineNum, + FLMUINT uiSavedOffset, + FLMUINT uiSavedFilePos, + FLMUINT uiSavedLineBytes); + + RCODE processProlog( void); + + RCODE processXMLDecl( void); + + RCODE processVersion( void); + + RCODE processEncodingDecl( void); + + RCODE processSDDecl( void); + + RCODE processMisc( void); + + RCODE processDocTypeDecl( void); + + RCODE processPI( + F_DOMNode * pParent, + FLMUINT uiSavedLineNum, + FLMUINT uiSavedOffset, + FLMUINT uiSavedFilePos, + FLMUINT uiSavedLineBytes); + + RCODE processElement( + F_DOMNode * pNodeToLinkTo, + eNodeInsertLoc eInsertLoc, + F_DOMNode ** ppNewNode); + + RCODE unicodeToNumber64( + FLMUNICODE * puzVal, + FLMUINT64 * pui64Val, + FLMBOOL * pbNeg); + + RCODE flushElementValue( + F_DOMNode * pParent, + FLMBYTE * pucValue, + FLMUINT uiValueLen); + + RCODE getBinaryVal( + FLMUINT * puiLength); + + RCODE fixNamingTag( + F_DOMNode * pNode); + + FLMBOOL lineHasToken( + const char * pszToken); + + RCODE processMarkupDecl( void); + + RCODE processPERef( void); + + RCODE processElementDecl( void); + + RCODE processEntityDecl( void); + + RCODE processNotationDecl( void); + + RCODE processAttListDecl( void); + + RCODE processContentSpec( void); + + RCODE processMixedContent( void); + + RCODE processChildContent( void); + + RCODE processAttDef( void); + + RCODE processAttType( void); + + RCODE processAttValue( + XML_ATTR * pAttr); + + RCODE processDefaultDecl( void); + + RCODE processID( + FLMBOOL bPublicId); + + RCODE processSTag( + F_DOMNode * pNodeToLinkTo, + eNodeInsertLoc eInsertLoc, + FLMBOOL * pbHasContent, + F_DOMNode ** ppElement); + + RCODE skipWhitespace( + FLMBOOL bRequired); + + RCODE resizeValBuffer( + FLMUINT uiSize); + + // Attribute management + + void resetAttrList( void) + { + m_pFirstAttr = NULL; + m_pLastAttr = NULL; + m_attrPool.poolReset( NULL); + } + + RCODE allocAttribute( + XML_ATTR ** ppAttr) + { + XML_ATTR * pAttr = NULL; + RCODE rc = NE_SFLM_OK; + + if( RC_BAD( rc = m_attrPool.poolCalloc( + sizeof( XML_ATTR), (void **)&pAttr))) + { + goto Exit; + } + + if( (pAttr->pPrev = m_pLastAttr) == NULL) + { + m_pFirstAttr = pAttr; + } + else + { + m_pLastAttr->pNext = pAttr; + } + + m_pLastAttr = pAttr; + + Exit: + + *ppAttr = pAttr; + return( rc); + } + + RCODE setPrefix( + XML_ATTR * pAttr, + FLMUNICODE * puzPrefix) + { + RCODE rc = NE_SFLM_OK; + FLMUINT uiStrLen; + + if( !puzPrefix) + { + pAttr->puzPrefix = NULL; + goto Exit; + } + + uiStrLen = f_unilen( puzPrefix); + + if( RC_BAD( rc = m_attrPool.poolAlloc( + sizeof( FLMUNICODE) * (uiStrLen + 1), (void **)&pAttr->puzPrefix))) + { + goto Exit; + } + + f_memcpy( pAttr->puzPrefix, puzPrefix, + sizeof( FLMUNICODE) * (uiStrLen + 1)); + + Exit: + + return( rc); + } + + RCODE setLocalName( + XML_ATTR * pAttr, + FLMUNICODE * puzLocalName) + { + RCODE rc = NE_SFLM_OK; + FLMUINT uiStrLen; + + if( !puzLocalName) + { + pAttr->puzLocalName = NULL; + goto Exit; + } + + uiStrLen = f_unilen( puzLocalName); + + if( RC_BAD( rc = m_attrPool.poolAlloc( + sizeof( FLMUNICODE) * (uiStrLen + 1), + (void **)&pAttr->puzLocalName))) + { + goto Exit; + } + + f_memcpy( pAttr->puzLocalName, puzLocalName, + sizeof( FLMUNICODE) * (uiStrLen + 1)); + + Exit: + + return( rc); + } + + RCODE setUnicode( + XML_ATTR * pAttr, + FLMUNICODE * puzUnicode) + { + RCODE rc = NE_SFLM_OK; + FLMUINT uiStrLen; + + if( !puzUnicode) + { + pAttr->puzVal = NULL; + goto Exit; + } + + uiStrLen = f_unilen( puzUnicode); + + if( RC_BAD( rc = m_attrPool.poolAlloc( + sizeof( FLMUNICODE) * (uiStrLen + 1), + (void **)&pAttr->puzVal))) + { + goto Exit; + } + + f_memcpy( pAttr->puzVal, puzUnicode, + sizeof( FLMUNICODE) * (uiStrLen + 1)); + + Exit: + + return( rc); + } + + RCODE addAttributesToElement( + F_DOMNode * pElement); + + FINLINE void setErrInfo( + FLMUINT uiErrLineNum, + FLMUINT uiErrLineOffset, + XMLParseError eErrorType, + FLMUINT uiErrLineFilePos, + FLMUINT uiErrLineBytes) + { + m_importStats.uiErrLineNum = uiErrLineNum; + m_importStats.uiErrLineOffset = uiErrLineOffset; + m_importStats.eErrorType = eErrorType; + m_importStats.uiErrLineFilePos = uiErrLineFilePos; + m_importStats.uiErrLineBytes = uiErrLineBytes; + } + + // Data + + F_Db * m_pDb; + FLMUINT m_uiCollection; + FLMBYTE m_ucUngetByte; + FLMUNICODE * m_puzCurrLineBuf; + FLMUINT m_uiCurrLineBufMaxChars; + FLMUINT m_uiCurrLineNumChars; + FLMUINT m_uiCurrLineOffset; + FLMUINT m_uiCurrLineNum; + FLMUINT m_uiCurrLineFilePos; + FLMUINT m_uiCurrLineBytes; +#define FLM_XML_MAX_CHARS 128 + FLMUNICODE m_uChars[ FLM_XML_MAX_CHARS]; + FLMBOOL m_bSetup; + IF_IStream * m_pStream; + FLMBYTE * m_pucValBuf; + FLMUINT m_uiValBufSize; // Number of Unicode characters + FLMUINT m_uiFlags; + FLMBOOL m_bExtendDictionary; + XMLEncoding m_eXMLEncoding; + XML_STATUS_HOOK m_fnStatus; + void * m_pvCallbackData; + XFLM_IMPORT_STATS m_importStats; + F_Pool m_tmpPool; + + // Attribute management + + XML_ATTR * m_pFirstAttr; + XML_ATTR * m_pLastAttr; + F_Pool m_attrPool; +}; + +#define FLM_XML_EXTEND_DICT_FLAG 0x00000001 +#define FLM_XML_COMPRESS_WHITESPACE_FLAG 0x00000002 +#define FLM_XML_TRANSLATE_ESC_FLAG 0x00000004 + +FINLINE FLMBOOL isXMLNS( + FLMUNICODE * puzName) +{ + return( (puzName [0] == FLM_UNICODE_x || puzName [0] == FLM_UNICODE_X) && + (puzName [1] == FLM_UNICODE_m || puzName [1] == FLM_UNICODE_M) && + (puzName [2] == FLM_UNICODE_l || puzName [2] == FLM_UNICODE_L) && + (puzName [3] == FLM_UNICODE_n || puzName [3] == FLM_UNICODE_N) && + (puzName [4] == FLM_UNICODE_s || puzName [4] == FLM_UNICODE_S) + ? TRUE + : FALSE); +} + +#endif // FXML_H diff --git a/sql/src/fxpath.h b/sql/src/fxpath.h new file mode 100644 index 0000000..246cb16 --- /dev/null +++ b/sql/src/fxpath.h @@ -0,0 +1,454 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the FLAIM XML import and export utility classes +// +// Tabs: 3 +// +// Copyright (c) 1999-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: fxpath.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef FXPATH_H +#define FXPATH_H + +typedef enum +{ + UNKNOWN_TOKEN = 0, // 0 + OP_AND_TOKEN = XFLM_AND_OP, // 1 + OP_OR_TOKEN = XFLM_OR_OP, // 2 + OP_NOT_TOKEN = XFLM_NOT_OP, // 3 + OP_EQ_TOKEN = XFLM_EQ_OP, // 4 + OP_NE_TOKEN = XFLM_NE_OP, // 5 + OP_APPROX_EQ_TOKEN = XFLM_APPROX_EQ_OP, // 6 + OP_LT_TOKEN = XFLM_LT_OP, // 7 + OP_LE_TOKEN = XFLM_LE_OP, // 8 + OP_GT_TOKEN = XFLM_GT_OP, // 9 + OP_GE_TOKEN = XFLM_GE_OP, // 10 + OP_BITAND_TOKEN = XFLM_BITAND_OP, // 11 + OP_BITOR_TOKEN = XFLM_BITOR_OP, // 12 + OP_BITXOR_TOKEN = XFLM_BITXOR_OP, // 13 + OP_MULT_TOKEN = XFLM_MULT_OP, // 14 + OP_DIV_TOKEN = XFLM_DIV_OP, // 15 + OP_MOD_TOKEN = XFLM_MOD_OP, // 16 + OP_PLUS_TOKEN = XFLM_PLUS_OP, // 17 + OP_MINUS_TOKEN = XFLM_MINUS_OP, // 18 + OP_NEG_TOKEN = XFLM_NEG_OP, // 19 + OP_LPAREN_TOKEN = XFLM_LPAREN_OP, // 20 + OP_RPAREN_TOKEN = XFLM_RPAREN_OP, // 21 + OP_COMMA_TOKEN = XFLM_COMMA_OP, // 22 + OP_LBRACKET_TOKEN = XFLM_LBRACKET_OP, // 23 + OP_RBRACKET_TOKEN = XFLM_RBRACKET_OP, // 24 + OP_FSLASH_TOKEN, // 25 + OP_DOUBLE_FSLASH_TOKEN, // 26 + OP_UNION_TOKEN, // 27 + PERIOD_TOKEN, // 28 + DOUBLE_PERIOD_TOKEN, // 29 + COMMA_TOKEN, // 30 + DOUBLE_COLON_TOKEN, // 31 + NAME_TEST_WILD_TOKEN, // 32 + NAME_TEST_NCWILD_TOKEN, // 33 + NAME_TEST_QNAME_TOKEN, // 34 + NODE_TYPE_COMMENT_TOKEN, // 35 + NODE_TYPE_TEXT_TOKEN, // 36 + NODE_TYPE_PI_TOKEN, // 37 + NODE_TYPE_NODE_TOKEN, // 38 + AXIS_ANCESTOR_TOKEN, // 39 + AXIS_ANCESTOR_OR_SELF_TOKEN, // 40 + AXIS_ATTRIB_TOKEN, // 41 + AXIS_CHILD_TOKEN, // 42 + AXIS_DESCENDANT_TOKEN, // 43 + AXIS_DESCENDANT_OR_SELF_TOKEN, // 44 + AXIS_FOLLOWING_TOKEN, // 45 + AXIS_FOLLOWING_SIB_TOKEN, // 46 + AXIS_NAMESPACE_TOKEN, // 47 + AXIS_PARENT_TOKEN, // 48 + AXIS_PRECEDING_TOKEN, // 49 + AXIS_PRECEDING_SIB_TOKEN, // 50 + AXIS_SELF_TOKEN, // 51 + AXIS_ATSIGN_TOKEN, // 52 + AXIS_META_TOKEN, // 53 + LITERAL_TOKEN, // 54 + NUMBER_TOKEN, // 55 + VAR_REF_TOKEN, // 56 + LBRACE_TOKEN, // 57 + RBRACE_TOKEN, // 58 + FUNC_LAST_TOKEN, // 59 + FUNC_POSITION_TOKEN, // 60 + FUNC_COUNT_TOKEN, // 61 + FUNC_ID_TOKEN, // 62 + FUNC_LOCAL_NAME_TOKEN, // 63 + FUNC_NAMESPACE_URI_TOKEN, // 64 + FUNC_NAME_TOKEN, // 65 + FUNC_STRING_TOKEN, // 66 + FUNC_CONCAT_TOKEN, // 67 + FUNC_STARTS_WITH_TOKEN, // 68 + FUNC_CONTAINS_TOKEN, // 69 + FUNC_SUBSTR_BEFORE_TOKEN, // 70 + FUNC_SUBSTR_AFTER_TOKEN, // 71 + FUNC_SUBSTR_TOKEN, // 72 + FUNC_STR_LEN_TOKEN, // 73 + FUNC_NORM_SPACE_TOKEN, // 74 + FUNC_TRANSLATE_TOKEN, // 75 + FUNC_NOT_TOKEN, // 76 + FUNC_TRUE_TOKEN, // 77 + FUNC_FALSE_TOKEN, // 78 + FUNC_UNKNOWN_TOKEN, // 79 + FUNC_LANG_TOKEN, // 80 + FUNC_NUMBER_TOKEN, // 81 + FUNC_SUM_TOKEN, // 82 + FUNC_FLOOR_TOKEN, // 83 + FUNC_CEILING_TOKEN, // 84 + FUNC_ROUND_TOKEN, // 85 + BINARY_TOKEN, // 86 + FUNC_CB_TOKEN, // 87 + END_TOKEN // 88 +} eXPathTokenType; + +class F_XPathBase : public F_Object +{ +public: + + FINLINE FLMBOOL isOperator( + eXPathTokenType eType) + { + switch( eType) + { + case OP_AND_TOKEN: + case OP_OR_TOKEN: + case OP_MOD_TOKEN: + case OP_DIV_TOKEN: + case OP_MULT_TOKEN: + case OP_FSLASH_TOKEN: + case OP_DOUBLE_FSLASH_TOKEN: + case OP_UNION_TOKEN: + case OP_PLUS_TOKEN: + case OP_MINUS_TOKEN: + case OP_EQ_TOKEN: + case OP_NE_TOKEN: + case OP_LT_TOKEN: + case OP_LE_TOKEN: + case OP_GT_TOKEN: + case OP_GE_TOKEN: + return( TRUE); + default: + break; + } + + return( FALSE); + } + + FINLINE FLMBOOL tokenCanHaveFlags( + eXPathTokenType eType) + { + switch( eType) + { + case OP_EQ_TOKEN: + case OP_NE_TOKEN: + case OP_LT_TOKEN: + case OP_LE_TOKEN: + case OP_GT_TOKEN: + case OP_GE_TOKEN: + return( TRUE); + default: + break; + } + + return( FALSE); + } + + FINLINE FLMBOOL isAxisSpecifier( + eXPathTokenType eType) + { + switch( eType) + { + case AXIS_ANCESTOR_TOKEN: + case AXIS_ANCESTOR_OR_SELF_TOKEN: + case AXIS_ATTRIB_TOKEN: + case AXIS_CHILD_TOKEN: + case AXIS_DESCENDANT_TOKEN: + case AXIS_DESCENDANT_OR_SELF_TOKEN: + case AXIS_FOLLOWING_TOKEN: + case AXIS_FOLLOWING_SIB_TOKEN: + case AXIS_NAMESPACE_TOKEN: + case AXIS_PARENT_TOKEN: + case AXIS_PRECEDING_TOKEN: + case AXIS_PRECEDING_SIB_TOKEN: + case AXIS_SELF_TOKEN: + case AXIS_ATSIGN_TOKEN: + return( TRUE); + default: + break; + } + + return( FALSE); + } +}; + +/***************************************************************************** +Desc: +******************************************************************************/ +class F_XPathToken : public F_XPathBase +{ +public: + + F_XPathToken() + { + m_pValBuf = NULL; + m_uiValBufSize = 0; + reset(); + } + + ~F_XPathToken() + { + if( m_pValBuf) + { + f_free( &m_pValBuf); + } + } + + FINLINE void reset( void) + { + m_eTokenType = UNKNOWN_TOKEN; + m_uiTokenFlags = 0; + m_ui64Val = 0; + m_puzPrefix = NULL; + m_puzLocal = NULL;; + } + + FINLINE RCODE resizeBuffer( + FLMUINT uiNewSize) + { + RCODE rc = NE_SFLM_OK; + void * pOrigBuf = m_pValBuf; + + if( !m_pValBuf) + { + if( RC_BAD( rc = f_alloc( uiNewSize, &m_pValBuf))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = f_realloc( uiNewSize, &m_pValBuf))) + { + goto Exit; + } + + if( m_puzPrefix) + { + m_puzPrefix = (FLMUNICODE *)(((FLMBYTE *)m_puzPrefix - + (FLMBYTE *)pOrigBuf) + (FLMBYTE *)m_pValBuf); + } + + if( m_puzLocal) + { + m_puzLocal = (FLMUNICODE *)(((FLMBYTE *)m_puzLocal - + (FLMBYTE *)pOrigBuf) + (FLMBYTE *)m_pValBuf); + } + } + + m_uiValBufSize = uiNewSize; + + Exit: + + return( rc); + } + + FINLINE eXPathTokenType getType( void) + { + return( m_eTokenType); + } + + FINLINE FLMUNICODE * getPrefixPtr( void) + { + return( m_puzPrefix); + } + + FINLINE FLMUNICODE * getLocalPtr( void) + { + return( m_puzLocal); + } + + FINLINE FLMUINT64 getNumber( void) + { + return( m_ui64Val); + } + + FINLINE FLMUINT getTokenFlags( void) + { + return( m_uiTokenFlags); + } + +private: + + eXPathTokenType m_eTokenType; + FLMUINT m_uiTokenFlags; + void * m_pValBuf; + FLMUINT m_uiValBufSize; + FLMUINT m_uiValBufLen; + FLMUINT64 m_ui64Val; + FLMUNICODE * m_puzPrefix; + FLMUNICODE * m_puzLocal; + +friend class F_XPathTokenizer; +friend class F_XPath; +}; + +/***************************************************************************** +Desc: +******************************************************************************/ +class F_XPathTokenizer : public F_XPathBase, public F_XMLNamespaceMgr +{ +public: + + F_XPathTokenizer() + { + m_pIStream = NULL; + m_uiUngetCount = 0; + m_eLastTokenType = UNKNOWN_TOKEN; + } + + ~F_XPathTokenizer() + { + if( m_pIStream) + { + m_pIStream->Release(); + } + } + + RCODE setup( + IF_IStream * pIStream); + + RCODE getNextToken( + F_XPathToken * pToken); + +private: + + RCODE skipWhitespace( void); + + RCODE getChar( + FLMUNICODE * puChar); + + RCODE peekChar( + FLMUNICODE * puChar); + + RCODE ungetChar( + FLMUNICODE uChar); + + RCODE getNumber( + F_XPathToken * pToken); + + RCODE getName( + F_XPathToken * pToken); + + RCODE getBinary( + F_XPathToken * pToken); + + RCODE getLiteral( + F_XPathToken * pToken); + + IF_IStream * m_pIStream; + eXPathTokenType m_eLastTokenType; + FLMUINT m_uiUngetCount; +#define XPATH_MAX_UNGET_CHARS 4 + FLMUNICODE m_uUngetBuf[ XPATH_MAX_UNGET_CHARS]; +}; + +class F_XPathExpr; +class F_XPathPredicate; +class F_XPathAxisProducer; +class F_XPathStep; + +/***************************************************************************** +Desc: +******************************************************************************/ +class F_XPath : public F_XPathBase, public F_XMLNamespaceMgr +{ +public: + + F_XPath() + { + } + + ~F_XPath() + { + } + + RCODE parseQuery( + F_Db * pDb, + IF_IStream * pIStream, + IF_Query * pQuery); + + RCODE parseQuery( + F_Db * pDb, + char * pszQuery, + IF_Query * pQuery); + +private: + + RCODE processFilterExpr( + F_XPathExpr ** ppExpr); + + RCODE processPathExpr( + F_XPathExpr ** ppExpr); + + RCODE processUnionExpr( + F_XPathExpr ** ppExpr); + + RCODE processNodeTest( + FLMBOOL bAttr, + F_XPathExpr ** ppExpr); + + RCODE processStep( + F_XPathExpr ** ppExpr); + + RCODE processRelativeLocationPath( + F_XPathExpr ** ppExpr); + + RCODE processUnaryExpr( + F_XPathExpr ** ppExpr); + + RCODE processOrExpr( + F_XPathExpr ** ppExpr); + + RCODE processAndExpr( + F_XPathExpr ** ppExpr); + + RCODE processEqualityExpr( + F_XPathExpr ** ppExpr); + + RCODE processRelationalExpr( + F_XPathExpr ** ppExpr); + + RCODE processAdditiveExpr( + F_XPathExpr ** ppExpr); + + RCODE processMultiplicativeExpr( + F_XPathExpr ** ppExpr); + + RCODE processPrimaryExpr( + F_XPathExpr ** ppExpr); + + RCODE getNextToken( void); + + F_XPathTokenizer m_tokenizer; + F_XPathToken m_curToken; +}; + +#endif // FXPATH_H diff --git a/sql/src/kybldkey.cpp b/sql/src/kybldkey.cpp new file mode 100644 index 0000000..59ec3b4 --- /dev/null +++ b/sql/src/kybldkey.cpp @@ -0,0 +1,2001 @@ +//------------------------------------------------------------------------------ +// Desc: Build from and until keys from a predicate +// +// Tabs: 3 +// +// Copyright (c) 1996-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC FLMUINT kyAddInclComponent( + FLMUINT uiNumKeyComponents, + FLMUINT uiKeyComponent, + FLMBYTE * pucKeyEnd, + FLMBOOL bFromKey, + FLMUINT uiMaxSpaceLeft); + +FINLINE void flmSetupFirstToLastKey( + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen); + +FSTATIC RCODE flmAddNonTextKeyPiece( + SQL_PRED * pPred, + F_INDEX * pIndex, + FLMUINT uiKeyComponent, + F_COLUMN * pColumn, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbCanCompareOnKey); + +FSTATIC RCODE flmUTF8FindWildcard( + const FLMBYTE * pucValue, + FLMUINT * puiCharPos, + FLMUINT * puiCompareRules); + +FSTATIC RCODE flmCountCharacters( + const FLMBYTE * pucValue, + FLMUINT uiValueLen, + FLMUINT uiMaxToCount, + FLMUINT * puiCompareRules, + FLMUINT * puiCount); + +FSTATIC RCODE flmSelectBestSubstr( + const FLMBYTE ** ppucValue, + FLMUINT * puiValueLen, + FLMUINT * puiCompareRules, + FLMBOOL * pbTrailingWildcard, + FLMBOOL * pbNotUsingFirstOfString); + +FSTATIC void setFromCaseByte( + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + FLMUINT uiCaseLen, + FLMBOOL bIsDBCS, + FLMBOOL bAscending, + FLMBOOL bExcl); + +FSTATIC void setUntilCaseByte( + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMUINT uiCaseLen, + FLMBOOL bIsDBCS, + FLMBOOL bAscending, + FLMBOOL bExcl); + +FSTATIC RCODE flmAddTextKeyPiece( + SQL_PRED * pPred, + F_INDEX * pIndex, + FLMUINT uiKeyComponent, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbCanCompareOnKey); + +/**************************************************************************** +Desc: Add what is needed to an until key so that it is greater than or equal + for all possible components that come after the primary component. + In other words, this key should be less than any keys whose primary + component is greater than it, but greater than all keys whose primary + component is less than or equal to it. +****************************************************************************/ +FSTATIC FLMUINT kyAddInclComponent( + FLMUINT uiNumKeyComponents, + FLMUINT uiKeyComponent, + FLMBYTE * pucKeyEnd, + FLMBOOL bFromKey, + FLMUINT uiMaxSpaceLeft) +{ + FLMUINT uiBytesAdded = 0; + + // If there is a next key component that would be expected, set it to + // the highest possible value. + + if (uiKeyComponent < uiNumKeyComponents) + { + + // Must at least be room for a 2 byte length. + + if (uiMaxSpaceLeft >= 2) + { + // Need 2nd key component to sort lower if it is the from key + // higher if it is the until key. Note that KEY_LOW_VALUE and + // KEY_HIGH_VALUE always sort lower/higher no matter whether the + // component is ascending or descending. + + if (bFromKey) + { + UW2FBA( (FLMUINT16)KEY_LOW_VALUE, pucKeyEnd); + } + else + { + UW2FBA( (FLMUINT16)KEY_HIGH_VALUE, pucKeyEnd); + } + uiBytesAdded = 2; + } + } + else + { + + // There are no more key components. + // Output one byte of 0xFF - which should be higher than any + // possible SEN that could be output for document ID and node IDs. + // Only do this for until keys. For from keys, no need to add + // anything, because an empty list of IDs will be be inclusive + // on the from side - it will sort lower, and therefore be inclusive. + + if (uiMaxSpaceLeft && !bFromKey) + { + *pucKeyEnd = 0xFF; + uiBytesAdded = 1; + } + } + + return( uiBytesAdded); +} + +/**************************************************************************** +Desc: Setup a first-to-last key for the index. +****************************************************************************/ +FINLINE void flmSetupFirstToLastKey( + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen + ) +{ + UW2FBA( KEY_LOW_VALUE, pucFromKey); + UW2FBA( KEY_HIGH_VALUE, pucUntilKey); + *puiFromKeyLen = 2; + *puiUntilKeyLen = 2; +} + +/**************************************************************************** +Desc: Add a key piece to the from and until key. Text fields are not + handled in this routine because of their complexity. +Notes: The goal of this code is to build a the collated compound piece + for the 'from' and 'until' key only once instead of twice. +****************************************************************************/ +FSTATIC RCODE flmAddNonTextKeyPiece( + SQL_PRED * pPred, + F_INDEX * pIndex, + FLMUINT uiKeyComponent, + F_COLUMN * pColumn, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbCanCompareOnKey) +{ + RCODE rc = NE_SFLM_OK; + ICD * pIcd = pIndex->pKeyIcds + uiKeyComponent - 1; + FLMUINT uiFromKeyLen = 0; + FLMUINT uiUntilKeyLen = 0; + FLMBYTE * pucFromKeyLenPos = pucFromKey; + FLMBYTE * pucUntilKeyLenPos = pucUntilKey; + FLMBOOL bDataTruncated; + FLMBYTE * pucFromBuf; + FLMUINT uiFromBufLen; + FLMBYTE * pucUntilBuf; + FLMUINT uiUntilBufLen; + FLMBYTE ucFromNumberBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMBYTE ucUntilNumberBuf [FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiValue; + FLMINT iValue; + FLMBOOL bNeg; + FLMUINT64 ui64Value; + FLMINT64 i64Value; + FLMUINT uiFromFlags = 0; + FLMUINT uiUntilFlags = 0; + SQL_VALUE * pFromValue; + SQL_VALUE * pUntilValue; + FLMBOOL bInclFrom; + FLMBOOL bInclUntil; + FLMBOOL bAscending = (pIcd->uiFlags & ICD_DESCENDING) ? FALSE: TRUE; + F_BufferIStream bufferIStream; + + // Leave room for the component length + + pucFromKey += 2; + pucUntilKey += 2; + + // Handle the presence case here - this is not done in kyCollate. + + if (pIcd->uiFlags & ICD_PRESENCE) + { + f_UINT32ToBigEndian( (FLMUINT32)pIcd->uiColumnNum, pucFromKey); + uiFromKeyLen = uiUntilKeyLen = 4; + f_memcpy( pucUntilKey, pucFromKey, uiUntilKeyLen); + } + else if (pIcd->uiFlags & ICD_METAPHONE) + { + if (pPred->eOperator != SQL_APPROX_EQ_OP || + pPred->pFromValue->eValType != SQL_UTF8_VAL) + { + flmSetupFirstToLastKey( pucFromKeyLenPos, puiFromKeyLen, + pucUntilKeyLenPos, puiUntilKeyLen); + if (pPred->eOperator != SQL_EXISTS_OP) + { + *pbCanCompareOnKey = FALSE; + } + goto Exit; + } + *pbCanCompareOnKey = FALSE; + + // The value type in pPred->pFromValue is SQL_UTF8_VAL, but the + // calling routine should have put the metaphone value into + // pPred->pFromValue->val.uiVal. Sort of weird, but was the + // only way we could evaluate the cost of multiple words in + // the string. + + uiFromBufLen = sizeof( ucFromNumberBuf); + if( RC_BAD( rc = FlmUINT2Storage( pPred->pFromValue->val.uiVal, + &uiFromBufLen, ucFromNumberBuf))) + { + goto Exit; + } + pucFromBuf = &ucFromNumberBuf [0]; + + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucFromBuf, uiFromBufLen))) + { + goto Exit; + } + + uiFromKeyLen = SFLM_MAX_KEY_SIZE - 2; + bDataTruncated = FALSE; + + // Pass 0 for compare rules because it is non-text + + if (RC_BAD( rc = KYCollateValue( pucFromKey, &uiFromKeyLen, + &bufferIStream, SFLM_NUMBER_TYPE, + pIcd->uiFlags, 0, + pIcd->uiLimit, NULL, NULL, + pIndex->uiLanguage, FALSE, FALSE, + &bDataTruncated, NULL))) + { + goto Exit; + } + + bufferIStream.close(); + + if (bDataTruncated) + { + // This should never happen on numeric data. + + flmAssert( 0); + *pbCanCompareOnKey = FALSE; + } + + if (uiFromKeyLen) + { + f_memcpy( pucUntilKey, pucFromKey, uiFromKeyLen); + } + + uiUntilKeyLen = uiFromKeyLen; + } + else + { + if (pPred->eOperator == SQL_EXISTS_OP || + pPred->eOperator == SQL_NE_OP || + pPred->eOperator == SQL_APPROX_EQ_OP) + { + + // Setup a first-to-last key + + flmSetupFirstToLastKey( pucFromKeyLenPos, puiFromKeyLen, + pucUntilKeyLenPos, puiUntilKeyLen); + goto Exit; + } + + // Only other operator possible is the range operator + + flmAssert( pPred->eOperator == SQL_RANGE_OP); + + if (bAscending) + { + pFromValue = pPred->pFromValue; + bInclFrom = pPred->bInclFrom; + pUntilValue = pPred->pUntilValue; + bInclUntil = pPred->bInclUntil; + } + else + { + pFromValue = pPred->pUntilValue; + bInclFrom = pPred->bInclUntil; + pUntilValue = pPred->pFromValue; + bInclUntil = pPred->bInclFrom; + } + + // Set up from buffer + + if (!pFromValue) + { + pucFromBuf = NULL; + uiFromBufLen = 0; + } + else + { + switch (pFromValue->eValType) + { + case SQL_UINT_VAL: + uiValue = pFromValue->val.uiVal; + if (!bInclFrom) + { + if (bAscending) + { + uiValue++; + } + else + { + uiValue--; + } + } + uiFromBufLen = sizeof( ucFromNumberBuf); + if( RC_BAD( rc = FlmUINT2Storage( uiValue, &uiFromBufLen, + ucFromNumberBuf))) + { + goto Exit; + } + pucFromBuf = &ucFromNumberBuf [0]; + break; + + case SQL_INT_VAL: + iValue = pFromValue->val.iVal; + if (!bInclFrom) + { + if (bAscending) + { + iValue++; + } + else + { + iValue--; + } + } + uiFromBufLen = sizeof( ucFromNumberBuf); + if (RC_BAD( rc = FlmINT2Storage( iValue, &uiFromBufLen, + ucFromNumberBuf))) + { + goto Exit; + } + pucFromBuf = &ucFromNumberBuf [0]; + break; + + case SQL_UINT64_VAL: + ui64Value = pFromValue->val.ui64Val; + if (!bInclFrom) + { + if (bAscending) + { + ui64Value++; + } + else + { + ui64Value--; + } + } + uiFromBufLen = sizeof( ucFromNumberBuf); + if (RC_BAD( rc = flmNumber64ToStorage( ui64Value, &uiFromBufLen, + ucFromNumberBuf, FALSE, FALSE))) + { + goto Exit; + } + pucFromBuf = &ucFromNumberBuf [0]; + break; + + case SQL_INT64_VAL: + i64Value = pFromValue->val.i64Val; + if (!bInclFrom) + { + if (bAscending) + { + i64Value++; + } + else + { + i64Value--; + } + } + if (i64Value < 0) + { + bNeg = TRUE; + ui64Value = (FLMUINT64)-i64Value; + } + else + { + bNeg = FALSE; + ui64Value = (FLMUINT64)i64Value; + } + + uiFromBufLen = sizeof( ucFromNumberBuf); + if (RC_BAD( rc = flmNumber64ToStorage( ui64Value, &uiFromBufLen, + ucFromNumberBuf, bNeg, FALSE))) + { + goto Exit; + } + pucFromBuf = &ucFromNumberBuf [0]; + break; + + case SQL_BINARY_VAL: + pucFromBuf = pFromValue->val.pucBuf; + uiFromBufLen = pFromValue->uiDataLen; + if (!bInclFrom) + { + + // Should use EXCLUSIVE_GT_FLAG even if in descending + // order, because the comparison routines will take + // that into account. + + uiFromFlags |= EXCLUSIVE_GT_FLAG; + } + break; + + default: + + // Text type should have been taken care of elsewhere. + + rc = RC_SET_AND_ASSERT( NE_SFLM_QUERY_SYNTAX); + goto Exit; + } + } + + // Set up until buffer. + + if (!pUntilValue) + { + pucUntilBuf = NULL; + uiUntilBufLen = 0; + } + else if (pUntilValue == pFromValue) + { + pucUntilBuf = pucFromBuf; + uiUntilBufLen = uiFromBufLen; + } + else + { + switch (pUntilValue->eValType) + { + case SQL_UINT_VAL: + uiValue = pUntilValue->val.uiVal; + if (!bInclUntil) + { + if (bAscending) + { + uiValue--; + } + else + { + uiValue++; + } + } + uiUntilBufLen = sizeof( ucUntilNumberBuf); + if( RC_BAD( rc = FlmUINT2Storage( uiValue, &uiUntilBufLen, + ucUntilNumberBuf))) + { + goto Exit; + } + pucUntilBuf = &ucUntilNumberBuf [0]; + break; + + case SQL_INT_VAL: + iValue = pUntilValue->val.iVal; + if (!bInclUntil) + { + if (bAscending) + { + iValue--; + } + else + { + iValue++; + } + } + uiUntilBufLen = sizeof( ucUntilNumberBuf); + if (RC_BAD( rc = FlmINT2Storage( iValue, &uiUntilBufLen, + ucUntilNumberBuf))) + { + goto Exit; + } + pucUntilBuf = &ucUntilNumberBuf [0]; + break; + + case SQL_UINT64_VAL: + ui64Value = pUntilValue->val.ui64Val; + if (!bInclUntil) + { + if (bAscending) + { + ui64Value--; + } + else + { + ui64Value++; + } + } + uiUntilBufLen = sizeof( ucUntilNumberBuf); + if (RC_BAD( rc = flmNumber64ToStorage( ui64Value, &uiUntilBufLen, + ucUntilNumberBuf, FALSE, FALSE))) + { + goto Exit; + } + pucUntilBuf = &ucUntilNumberBuf [0]; + break; + + case SQL_INT64_VAL: + i64Value = pUntilValue->val.i64Val; + if (!bInclUntil) + { + if (bAscending) + { + i64Value--; + } + else + { + i64Value++; + } + } + if (i64Value < 0) + { + bNeg = TRUE; + ui64Value = (FLMUINT64)-i64Value; + } + else + { + bNeg = FALSE; + ui64Value = (FLMUINT64)i64Value; + } + + uiUntilBufLen = sizeof( ucUntilNumberBuf); + if (RC_BAD( rc = flmNumber64ToStorage( ui64Value, &uiUntilBufLen, + ucUntilNumberBuf, bNeg, FALSE))) + { + goto Exit; + } + pucUntilBuf = &ucUntilNumberBuf [0]; + break; + + case SQL_BINARY_VAL: + pucUntilBuf = pUntilValue->val.pucBuf; + uiUntilBufLen = pUntilValue->uiDataLen; + if (!bInclUntil) + { + + // Should use EXCLUSIVE_LT_FLAG even if in descending + // order, because the comparison routines will take + // that into account. + + uiUntilFlags |= EXCLUSIVE_LT_FLAG; + } + break; + + default: + + // Text type should have been taken care of elsewhere. + + rc = RC_SET_AND_ASSERT( NE_SFLM_QUERY_SYNTAX); + goto Exit; + } + } + + // Generate the keys using the from and until buffers that + // have been set up. + + if (!pucFromBuf && !pucUntilBuf) + { + + // setup a first-to-last key + + flmSetupFirstToLastKey( pucFromKeyLenPos, puiFromKeyLen, + pucUntilKeyLenPos, puiUntilKeyLen); + goto Exit; + } + + // Set up the from key + + if (!pucFromBuf) + { + uiFromKeyLen = KEY_LOW_VALUE; + } + else + { + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucFromBuf, uiFromBufLen))) + { + goto Exit; + } + + uiFromKeyLen = SFLM_MAX_KEY_SIZE - 2; + bDataTruncated = FALSE; + + // Pass 0 for compare rules on non-text component. + + if (RC_BAD( rc = KYCollateValue( pucFromKey, &uiFromKeyLen, + &bufferIStream, pColumn->eDataTyp, + pIcd->uiFlags, 0, + pIcd->uiLimit, NULL, NULL, + pIndex->uiLanguage, FALSE, FALSE, + &bDataTruncated, NULL))) + { + goto Exit; + } + + bufferIStream.close(); + + if (bDataTruncated) + { + *pbCanCompareOnKey = FALSE; + + // Save the original data into pFromSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + // Better only be a binary data type at this point. + + flmAssert( pFromValue->eValType == SQL_BINARY_VAL); + if (RC_BAD( rc = pFromSearchKey->setBinary( uiKeyComponent - 1, + pucFromBuf, uiFromBufLen))) + { + goto Exit; + } + uiFromFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + } + + // Set up the until key + + if (!pucUntilBuf) + { + uiUntilKeyLen = KEY_HIGH_VALUE; + } + else if (pucUntilBuf == pucFromBuf) + { + if (uiFromKeyLen) + { + f_memcpy( pucUntilKey, pucFromKey, uiFromKeyLen); + } + uiUntilKeyLen = uiFromKeyLen; + + // The "exclusive" flags better not have been set in this + // case - because this should only be possible if the operator + // was an EQ. + + flmAssert( !(uiFromFlags & EXCLUSIVE_GT_FLAG) && + !(uiUntilFlags & EXCLUSIVE_LT_FLAG)); + + if (uiFromFlags & SEARCH_KEY_FLAG) + { + + // Save the original data into pUntilSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + // Better only be a binary data type at this point. + + flmAssert( pUntilValue->eValType == SQL_BINARY_VAL); + if (RC_BAD( rc = pUntilSearchKey->setBinary( uiKeyComponent - 1, + pucUntilBuf, uiUntilBufLen))) + { + goto Exit; + } + uiUntilFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + } + else + { + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucUntilBuf, uiUntilBufLen))) + { + goto Exit; + } + uiUntilKeyLen = SFLM_MAX_KEY_SIZE - 2; + bDataTruncated = FALSE; + + // Pass 0 for compare rule because it is a non-text piece. + + if (RC_BAD( rc = KYCollateValue( pucUntilKey, &uiUntilKeyLen, + &bufferIStream, pColumn->eDataTyp, + pIcd->uiFlags, 0, + pIcd->uiLimit, NULL, NULL, + pIndex->uiLanguage, FALSE, FALSE, + &bDataTruncated, NULL))) + { + goto Exit; + } + + bufferIStream.close(); + + if (bDataTruncated) + { + *pbCanCompareOnKey = FALSE; + + // Save the original data into pUntilSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + // Better only be a binary data type at this point. + + flmAssert( pUntilValue->eValType == SQL_BINARY_VAL); + if (RC_BAD( rc = pUntilSearchKey->setBinary( uiKeyComponent - 1, + pucUntilBuf, uiUntilBufLen))) + { + goto Exit; + } + uiUntilFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + } + } + + UW2FBA( (FLMUINT16)(uiFromKeyLen | uiFromFlags), pucFromKeyLenPos); + UW2FBA( (FLMUINT16)(uiUntilKeyLen | uiUntilFlags), pucUntilKeyLenPos); + + if (!(uiFromFlags & EXCLUSIVE_GT_FLAG) && uiFromKeyLen < SFLM_MAX_KEY_SIZE - 2) + { + uiFromKeyLen += kyAddInclComponent( pIndex->uiNumKeyComponents, uiKeyComponent, + &pucFromKey [uiFromKeyLen], + TRUE, SFLM_MAX_KEY_SIZE - 2 - uiFromKeyLen); + } + if (!(uiUntilFlags & EXCLUSIVE_LT_FLAG) && uiUntilKeyLen < SFLM_MAX_KEY_SIZE - 2) + { + uiUntilKeyLen += kyAddInclComponent( pIndex->uiNumKeyComponents, uiKeyComponent, + &pucUntilKey [uiUntilKeyLen], + FALSE, SFLM_MAX_KEY_SIZE - 2 - uiUntilKeyLen); + } + + // Set the FROM and UNTIL key length return values. + + if (uiFromKeyLen != KEY_HIGH_VALUE && uiFromKeyLen != KEY_LOW_VALUE) + { + *puiFromKeyLen = uiFromKeyLen + 2; + } + else + { + *puiFromKeyLen = 2; + } + if (uiUntilKeyLen != KEY_HIGH_VALUE && uiUntilKeyLen != KEY_LOW_VALUE) + { + *puiUntilKeyLen = uiUntilKeyLen + 2; + } + else + { + *puiUntilKeyLen = 2; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Finds the location of a wildcard in the internal text string, if any. +****************************************************************************/ +FSTATIC RCODE flmUTF8FindWildcard( + const FLMBYTE * pucValue, + FLMUINT * puiCharPos, + FLMUINT * puiCompareRules) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucSaveVal; + const FLMBYTE * pucStart = pucValue; + FLMUNICODE uzChar; + FLMUINT uiCompareRules = *puiCompareRules; + + flmAssert( pucValue); + *puiCharPos = FLM_MAX_UINT; + + for( ;;) + { + pucSaveVal = pucValue; + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucValue, NULL, &uzChar))) + { + goto Exit; + } + + if (!uzChar) + { + break; + } + + if ((uzChar = f_convertChar( uzChar, uiCompareRules)) == 0) + { + continue; + } + if (uzChar == ASCII_WILDCARD) + { + *puiCharPos = (FLMUINT)(pucSaveVal - pucStart); + goto Exit; + } + if (uzChar != ASCII_SPACE) + { + + // Once we hit a non-space character - except for the wildcard, + // we can remove the ignore leading space rule. + + uiCompareRules &= (~(FLM_COMP_IGNORE_LEADING_SPACE)); + if (uzChar == ASCII_BACKSLASH) + { + + // Skip the escaped character + + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucValue, NULL, &uzChar))) + { + goto Exit; + } + + if (!uzChar) + { + rc = RC_SET( NE_SFLM_Q_BAD_SEARCH_STRING); + goto Exit; + } + } + } + } + +Exit: + + *puiCompareRules = uiCompareRules; + + return( rc); +} + +/**************************************************************************** +Desc: Count the number of characters that would be returned. +****************************************************************************/ +FSTATIC RCODE flmCountCharacters( + const FLMBYTE * pucValue, + FLMUINT uiValueLen, + FLMUINT uiMaxToCount, + FLMUINT * puiCompareRules, + FLMUINT * puiCharCount) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumChars = 0; + FLMUINT uiCompareRules = *puiCompareRules; + const FLMBYTE * pucEnd = &pucValue [uiValueLen]; + FLMUNICODE uzChar; + FLMBOOL bLastCharWasSpace = FALSE; + FLMUINT uiNumSpaces = 0; + + while (uiNumChars < uiMaxToCount) + { + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucValue, pucEnd, &uzChar))) + { + goto Exit; + } + + if (!uzChar) + { + if (bLastCharWasSpace) + { + // The spaces are trailing spaces, so if the ignore trailing + // space flag is set, we do nothing. If the compress space + // flag is set, we will increment by one. Otherwise, we will + // add in a count for all of the spaces. + + if (!(uiCompareRules & FLM_COMP_IGNORE_TRAILING_SPACE)) + { + if (uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE) + { + uiNumChars++; + } + else + { + uiNumChars += uiNumSpaces; + } + } + } + break; + } + + if ((uzChar = f_convertChar( uzChar, uiCompareRules)) == 0) + { + continue; + } + + if (uzChar == ASCII_SPACE) + { + if (!bLastCharWasSpace) + { + bLastCharWasSpace = TRUE; + uiNumSpaces = 0; + } + uiNumSpaces++; + } + else + { + + // Once we hit a non-space character, disable the ignore + // leading space flag. + + uiCompareRules &= (~(FLM_COMP_IGNORE_LEADING_SPACE)); + if (bLastCharWasSpace) + { + bLastCharWasSpace = FALSE; + if (uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE) + { + + // Consecutive spaces are compressed to a single space. + + uiNumChars++; + } + else + { + + // The spaces were not trailing spaces and were not compressed + // so we need to count all of them. + + uiNumChars += uiNumSpaces; + } + } + if (uzChar == ASCII_BACKSLASH) + { + + // Skip the next character, no matter what it is - only want + // to count one character here. A backslash followed by any + // character is only a single character. + + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucValue, pucEnd, &uzChar))) + { + goto Exit; + } + } + uiNumChars++; + } + } + +Exit: + + *puiCharCount = uiNumChars; + *puiCompareRules = uiCompareRules; + return( rc); +} + +/**************************************************************************** +Desc: Select the best substring for a CONTAINS or MATCH_END search. +****************************************************************************/ +FSTATIC RCODE flmSelectBestSubstr( + const FLMBYTE ** ppucValue, // [in/out] + FLMUINT * puiValueLen, // [in/out] + FLMUINT * puiCompareRules, + FLMBOOL * pbTrailingWildcard, // [in] change if found a wildcard + FLMBOOL * pbNotUsingFirstOfString) +{ + RCODE rc = NE_SFLM_OK; + const FLMBYTE * pucValue = *ppucValue; + const FLMBYTE * pucCurValue; + const FLMBYTE * pucBest; + const FLMBYTE * pucEnd; + const FLMBYTE * pucTmp; + FLMBOOL bBestTerminatesWithWildCard = *pbTrailingWildcard; + FLMUINT uiCurLen; + FLMUINT uiBestNumChars; + FLMUINT uiBestValueLen; + FLMUINT uiWildcardPos; + FLMUINT uiTargetNumChars; + FLMUINT uiNumChars; + FLMBOOL bNotUsingFirstOfString = FALSE; + FLMUNICODE uzChar; + FLMUNICODE uzDummy; + +#define GOOD_ENOUGH_CHARS 16 + + // There may not be any wildcards at all. Find the first one. + + if (RC_BAD( rc = flmUTF8FindWildcard( pucValue, + &uiWildcardPos, puiCompareRules))) + { + goto Exit; + } + + // FLM_MAX_UINT is returned if no wildcard was found. + + if (uiWildcardPos == FLM_MAX_UINT) + { + goto Exit; + } + + pucEnd = &pucValue [*puiValueLen]; + bBestTerminatesWithWildCard = TRUE; + pucBest = pucValue; + + // Skip past the wildcard + + pucTmp = &pucValue [uiWildcardPos]; + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucTmp, pucEnd, &uzDummy))) + { + goto Exit; + } + + uiCurLen = *puiValueLen - (FLMUINT)(pucTmp - pucValue); + pucCurValue = pucTmp; + + uiBestValueLen = uiWildcardPos; + if (RC_BAD( rc = flmCountCharacters( pucValue, uiWildcardPos, + GOOD_ENOUGH_CHARS, puiCompareRules, &uiBestNumChars))) + { + goto Exit; + } + uiTargetNumChars = uiBestNumChars + uiBestNumChars; + + // Here is the great FindADoubleLengthThatIsBetter algorithm. + // Below are the values to pick a next better contains key. + // First Key Size Next Key Size that will be used + // 1 * 2 2 // Single char searches are REALLY BAD + // 2 * 2 4 + // 3 * 2 6 + // 4 * 2 8 + // ... ... + // At each new key piece, increment the target length by 2 so that it + // will be even harder to find a better key. + + while (uiBestNumChars < GOOD_ENOUGH_CHARS) + { + pucTmp = pucCurValue; + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucTmp, pucEnd, &uzChar))) + { + goto Exit; + } + + if (!uzChar) + { + break; + } + + if (RC_BAD( rc = flmUTF8FindWildcard( pucCurValue, &uiWildcardPos, + puiCompareRules))) + { + goto Exit; + } + + // FLM_MAX_UINT is returned when no wildcard is found. + + if (uiWildcardPos == FLM_MAX_UINT) + { + + // No wildcard found + // Check the last section that may or may not have trailing *. + + if (RC_BAD( rc = flmCountCharacters( pucCurValue, uiCurLen, + GOOD_ENOUGH_CHARS, puiCompareRules, &uiNumChars))) + { + goto Exit; + } + + if (uiNumChars >= uiTargetNumChars) + { + pucBest = pucCurValue; + uiBestValueLen = uiCurLen; + bBestTerminatesWithWildCard = *pbTrailingWildcard; + } + break; + } + else + { + if (RC_BAD( rc = flmCountCharacters( pucCurValue, uiWildcardPos, + GOOD_ENOUGH_CHARS, puiCompareRules, &uiNumChars))) + { + goto Exit; + } + + if (uiNumChars >= uiTargetNumChars) + { + pucBest = pucCurValue; + uiBestValueLen = uiWildcardPos; + uiBestNumChars = uiNumChars; + uiTargetNumChars = uiNumChars + uiNumChars; + } + else + { + uiTargetNumChars += 2; + } + + // Skip past the wildcard + + pucTmp = &pucCurValue[ uiWildcardPos]; + if (RC_BAD( rc = f_getCharFromUTF8Buf( &pucTmp, pucEnd, &uzDummy))) + { + goto Exit; + } + + uiCurLen -= (FLMUINT)(pucTmp - pucCurValue); + pucCurValue = pucTmp; + } + } + + if (pucBest != *ppucValue) + { + bNotUsingFirstOfString = TRUE; + } + + *ppucValue = pucBest; + *puiValueLen = uiBestValueLen; + *pbTrailingWildcard = bBestTerminatesWithWildCard; + +Exit: + + *pbNotUsingFirstOfString = bNotUsingFirstOfString; + return( rc); +} + +/**************************************************************************** +Desc: Set the case byte on the from key for a case-insensitive search + that is using a case sensitive index. +****************************************************************************/ +FSTATIC void setFromCaseByte( + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + FLMUINT uiCaseLen, + FLMBOOL bIsDBCS, + FLMBOOL bAscending, + FLMBOOL bExcl) +{ + + // Subtract off all but the case marker. + // Remember that for DBCS (Asian) the case marker is two bytes. + + *puiFromKeyLen -= (uiCaseLen - + ((FLMUINT)(bIsDBCS + ? (FLMUINT)2 + : (FLMUINT)1))); + if (bExcl) + { + if (bAscending) + { + // Keys are in ascending order: + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // Thus, to exclude all "abc"s on "from" side we need the + // following key: + // key == abc+6 (COLL_MARKER | SC_UPPER) + 1 + + pucFromKey[ *puiFromKeyLen - 1] = (COLL_MARKER | SC_UPPER); + } + else + { + + // Keys are in descending order: + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // Thus, to exclude "abc"s on "from" side we need the + // following key: + // key == abc+4 (COLL_MARKER | SC_LOWER) + + pucFromKey[ *puiFromKeyLen - 1] = (COLL_MARKER | SC_LOWER); + } + } + else // Inclusive + { + if (bAscending) + { + // Keys are in ascending order: + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // Thus, to include all "abc"s on "from" side, + // we need the following key: + // key == abc+4 (COLL_MARKER | SC_LOWER) + + pucFromKey [*puiFromKeyLen - 1] = COLL_MARKER | SC_LOWER; + } + else + { + + // Keys are in descending order: + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // Thus, to include all "abc"s on "from" side we need the + // following key: + // key == abc+6 (COLL_MARKER | SC_UPPER) + + pucFromKey [*puiFromKeyLen - 1] = COLL_MARKER | SC_UPPER; + } + } +} + +/**************************************************************************** +Desc: Set the case byte on the until key for a case-insensitive search + that is using a case sensitive index. +****************************************************************************/ +FSTATIC void setUntilCaseByte( + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMUINT uiCaseLen, + FLMBOOL bIsDBCS, + FLMBOOL bAscending, + FLMBOOL bExcl) +{ + + // Subtract off all but the case marker. + // Remember that for DBCS (Asian) the case marker is two bytes. + + *puiUntilKeyLen -= (uiCaseLen - + ((FLMUINT)(bIsDBCS + ? (FLMUINT)2 + : (FLMUINT)1))); + if (bExcl) + { + if (bAscending) + { + // Keys are in ascending order: + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // Thus, to exclude all "abc"s on the "until" side we need + // the following key: + // key == abc+4 (COLL_MARKER | SC_LOWER) + + pucUntilKey[ *puiUntilKeyLen - 1] = (COLL_MARKER | SC_LOWER); + } + else + { + + // Keys are in descending order: + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // Thus, to exclude all "abc"s on the "until" side we need + // the following key: + // key == abc+6 (COLL_MARKER | SC_UPPER) + 1 + + pucUntilKey[ *puiUntilKeyLen - 1] = (COLL_MARKER | SC_UPPER); + } + } + else + { + if (bAscending) + { + // Keys are in ascending order: + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // Thus, to get include all "abc"s on the "until" side we need + // the following key: + // key == abc+6 (COLL_MARKER | SC_UPPER) + + pucUntilKey [*puiUntilKeyLen - 1] = (COLL_MARKER | SC_UPPER); + } + else + { + + // Keys are in descending order: + // "ABC" key == abc+6 (6 is COLL_MARKER | SC_UPPER) + // "abc" key == abc+4 (4 is COLL_MARKER | SC_LOWER) + // Thus, to include all "abc"s on the "until side we need + // the following key: + // key == abc+4 (COLL_MARKER | SC_LOWER) + + pucUntilKey [*puiUntilKeyLen - 1] = (COLL_MARKER | SC_LOWER); + } + } +} + +/**************************************************************************** +Desc: Build a text key. +****************************************************************************/ +FSTATIC RCODE flmAddTextKeyPiece( + SQL_PRED * pPred, + F_INDEX * pIndex, + FLMUINT uiKeyComponent, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbCanCompareOnKey) +{ + RCODE rc = NE_SFLM_OK; + ICD * pIcd = pIndex->pKeyIcds + uiKeyComponent - 1; + FLMUINT uiFromKeyLen = 0; + FLMUINT uiUntilKeyLen = 0; + FLMBYTE * pucFromKeyLenPos = pucFromKey; + FLMBYTE * pucUntilKeyLenPos = pucUntilKey; + FLMUINT uiLanguage = pIndex->uiLanguage; + FLMUINT uiCollationLen = 0; + FLMUINT uiCharCount; + FLMUINT uiCaseLen; + FLMBOOL bOriginalCharsLost = FALSE; + FLMBOOL bIsDBCS = (uiLanguage >= FLM_FIRST_DBCS_LANG && + uiLanguage <= FLM_LAST_DBCS_LANG) + ? TRUE + : FALSE; + + FLMBOOL bCaseInsensitive = (FLMBOOL)((pPred->uiCompareRules & + FLM_COMP_CASE_INSENSITIVE) + ? TRUE + : FALSE); + FLMBOOL bDoFirstSubstring = (FLMBOOL)((pIcd->uiFlags & ICD_SUBSTRING) + ? TRUE + : FALSE); + FLMBOOL bDoMatchBegin = FALSE; + FLMBOOL bTrailingWildcard = FALSE; + const FLMBYTE * pucFromUTF8Buf = NULL; + FLMUINT uiFromBufLen = 0; + const FLMBYTE * pucUntilUTF8Buf = NULL; + FLMUINT uiUntilBufLen = 0; + FLMUINT uiWildcardPos; + FLMBOOL bDataTruncated; + FLMUINT uiFromFlags = 0; + FLMUINT uiUntilFlags = 0; + FLMUINT uiCompareRules; + FLMBOOL bAscending = (pIcd->uiFlags & ICD_DESCENDING) ? FALSE: TRUE; + F_BufferIStream bufferIStream; + + switch (pPred->eOperator) + { + + // The difference between MATCH and EQ_OP is that EQ does + // not support wildcards embedded in the search key. + + case SQL_MATCH_OP: + flmAssert( pPred->pFromValue->eValType == SQL_UTF8_VAL); + pucFromUTF8Buf = pPred->pFromValue->val.pucBuf; + uiFromBufLen = pPred->pFromValue->uiDataLen; + uiCompareRules = pIcd->uiCompareRules; + + if (RC_BAD( rc = flmUTF8FindWildcard( pucFromUTF8Buf, &uiWildcardPos, + &uiCompareRules))) + { + goto Exit; + } + + // If there is no wildcard, uiWildcardPos will be FLM_MAX_UINT + + if (uiWildcardPos != FLM_MAX_UINT) + { + + // If wildcard is in position 0, it is NOT + // a match begin. + + if (uiWildcardPos) + { + bDoMatchBegin = TRUE; + uiFromBufLen = uiWildcardPos; + } + } + if (!(pIcd->uiFlags & ICD_SUBSTRING)) + { + + // Index is NOT a substring index + + if (!bDoMatchBegin) + { + + // Wildcard was at the beginning, will have + // to search the index from first to last + + pucFromUTF8Buf = NULL; + } + else + { + bTrailingWildcard = TRUE; + } + } + else + { + FLMBOOL bNotUsingFirstOfString; + + // If this is a substring index look for a + // better 'contains' string to search for. + // We don't like "A*BCDEFG" searches. + + bTrailingWildcard = bDoMatchBegin; + + uiCompareRules = pIcd->uiCompareRules; + if (RC_BAD( rc = flmSelectBestSubstr( &pucFromUTF8Buf, + &uiFromBufLen, + &uiCompareRules, &bTrailingWildcard, + &bNotUsingFirstOfString))) + { + goto Exit; + } + + if (bNotUsingFirstOfString) + { + bDoMatchBegin = bTrailingWildcard; + *pbCanCompareOnKey = FALSE; + bDoFirstSubstring = FALSE; + } + else if (bTrailingWildcard) + { + bDoMatchBegin = TRUE; + } + + if (RC_BAD( rc = flmCountCharacters( pucFromUTF8Buf, + uiFromBufLen, 2, + &uiCompareRules, &uiCharCount))) + { + goto Exit; + } + + // Special case: Single character contains/MEnd in a substr ix. + + if (!bIsDBCS && uiCharCount < 2) + { + pucFromUTF8Buf = NULL; + } + } + + pucUntilUTF8Buf = pucFromUTF8Buf; + uiUntilBufLen = uiFromBufLen; + break; + + case SQL_RANGE_OP: + if (bAscending) + { + if (pPred->pFromValue) + { + flmAssert( pPred->pFromValue->eValType == SQL_UTF8_VAL); + pucFromUTF8Buf = pPred->pFromValue->val.pucBuf; + uiFromBufLen = pPred->pFromValue->uiDataLen; + } + else + { + // Should have been done up above + + // pucFromUTF8Buf = NULL; + // uiFromBufLen = 0; + } + if (pPred->pUntilValue) + { + flmAssert( pPred->pUntilValue->eValType == SQL_UTF8_VAL); + pucUntilUTF8Buf = pPred->pUntilValue->val.pucBuf; + uiUntilBufLen = pPred->pUntilValue->uiDataLen; + } + else + { + // Should have been done up above. + + // pucUntilUTF8Buf = NULL; + // uiUntilBufLen = 0; + } + if (!pPred->bInclFrom) + { + uiFromFlags |= EXCLUSIVE_GT_FLAG; + } + if (!pPred->bInclUntil) + { + uiUntilFlags |= EXCLUSIVE_LT_FLAG; + } + } + else + { + if (pPred->pUntilValue) + { + flmAssert( pPred->pUntilValue->eValType == SQL_UTF8_VAL); + pucFromUTF8Buf = pPred->pUntilValue->val.pucBuf; + uiFromBufLen = pPred->pUntilValue->uiDataLen; + } + else + { + // Should have been done up above + + // pucFromUTF8Buf = NULL; + // uiFromBufLen = 0; + } + + if (pPred->pFromValue) + { + flmAssert( pPred->pFromValue->eValType == SQL_UTF8_VAL); + pucUntilUTF8Buf = pPred->pFromValue->val.pucBuf; + uiUntilBufLen = pPred->pFromValue->uiDataLen; + } + else + { + // Should have been done up above. + + // pucUntilUTF8Buf = NULL; + // uiUntilBufLen = 0; + } + if (!pPred->bInclUntil) + { + uiFromFlags |= EXCLUSIVE_GT_FLAG; + } + if (!pPred->bInclFrom) + { + uiUntilFlags |= EXCLUSIVE_LT_FLAG; + } + } + break; + + case SQL_NE_OP: + + // Set up to do full index scan. + + // Buffers should already be NULL. + + // pucFromUTF8Buf = NULL; + // pucUntilUTF8Buf = NULL; + break; + + case SQL_APPROX_EQ_OP: + + // Set up to do full index scan. + + // Buffers should already be NULL + + // pucFromUTF8Buf = NULL; + // pucUntilUTF8Buf = NULL; + + // Cannot compare on the key if index is upper case, + // even if the bCaseInsensitive flag is set. + + if (pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) + { + *pbCanCompareOnKey = FALSE; + } + break; + + default: + + // Every predicate should have been converted to one of the above + // cases, or should be handled by another routine. + + rc = RC_SET_AND_ASSERT( NE_SFLM_QUERY_SYNTAX); + goto Exit; + } + + // If index is case insensitive, but search is case sensitive + // we must NOT do a key match - we would fail things we should + // not be failing. + + if ((pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && !bCaseInsensitive) + { + *pbCanCompareOnKey = FALSE; + } + + if (!pucFromUTF8Buf && !pucUntilUTF8Buf) + { + + // setup a first-to-last key + + flmSetupFirstToLastKey( pucFromKeyLenPos, puiFromKeyLen, + pucUntilKeyLenPos, puiUntilKeyLen); + goto Exit; + } + + pucFromKey += 2; + pucUntilKey += 2; + if (!pucFromUTF8Buf) + { + uiFromKeyLen = KEY_LOW_VALUE; + } + else + { + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucFromUTF8Buf, uiFromBufLen))) + { + goto Exit; + } + + // Add ICD_ESC_CHAR to the icd flags because + // the search string must have BACKSLASHES and '*' escaped. + + uiFromKeyLen = SFLM_MAX_KEY_SIZE - 2; + bDataTruncated = FALSE; + if (RC_BAD( rc = KYCollateValue( pucFromKey, &uiFromKeyLen, + &bufferIStream, SFLM_STRING_TYPE, + pIcd->uiFlags | ICD_ESC_CHAR, pIcd->uiCompareRules, + pIcd->uiLimit, + &uiCollationLen, &uiCaseLen, + uiLanguage, bDoFirstSubstring, + FALSE, &bDataTruncated, + &bOriginalCharsLost))) + { + goto Exit; + } + + bufferIStream.close(); + + if (bDataTruncated) + { + *pbCanCompareOnKey = FALSE; + + // Save the original data into pFromSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + if (RC_BAD( rc = pFromSearchKey->setUTF8( uiKeyComponent - 1, + pucFromUTF8Buf, uiFromBufLen))) + { + goto Exit; + } + uiFromFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + else if (bOriginalCharsLost) + { + *pbCanCompareOnKey = FALSE; + } + + if (pucFromUTF8Buf != pucUntilUTF8Buf) + { + + // Handle scenario of a case-sensitive index, but search is + // case-insensitive. + + if (uiFromKeyLen && + (bIsDBCS || + (!(pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && + bCaseInsensitive))) + { + setFromCaseByte( pucFromKey, &uiFromKeyLen, uiCaseLen, + bIsDBCS, bAscending, + (uiFromFlags & EXCLUSIVE_GT_FLAG) + ? TRUE + : FALSE); + } + } + else + { + // Handle case where from and until buffers are the same. + // This should only be possible in the equality case or match begin + // case, in which cases neither the EXCLUSIVE_LT_FLAG or the + // EXCLUSIVE_GT_FLAG should be set. + + flmAssert( uiFromBufLen == uiUntilBufLen); + flmAssert( !(uiFromFlags & (EXCLUSIVE_GT_FLAG | EXCLUSIVE_LT_FLAG))); + flmAssert( !(uiUntilFlags & (EXCLUSIVE_GT_FLAG | EXCLUSIVE_LT_FLAG))); + + if (uiFromFlags & SEARCH_KEY_FLAG) + { + + // Save the original data into pUntilSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + if (RC_BAD( rc = pUntilSearchKey->setUTF8( uiKeyComponent - 1, + pucUntilUTF8Buf, uiUntilBufLen))) + { + goto Exit; + } + uiUntilFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + + if (bDoMatchBegin) + { + if (bAscending) + { + + // Handle scenario of a case-sensitive index, but search is + // case-insensitive. + + if (uiFromKeyLen && + (bIsDBCS || + (!(pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && + bCaseInsensitive))) + { + setFromCaseByte( pucFromKey, &uiFromKeyLen, uiCaseLen, + bIsDBCS, bAscending, FALSE); + } + + // From key is set up properly, setup until key. + + if (uiCollationLen) + { + f_memcpy( pucUntilKey, pucFromKey, uiCollationLen); + } + + // Fill the rest of the until key with high values. + + f_memset( &pucUntilKey[ uiCollationLen], 0xFF, + SFLM_MAX_KEY_SIZE - uiCollationLen - 2); + uiUntilKeyLen = SFLM_MAX_KEY_SIZE - 2; + } + else + { + + // Copy from key into until key. + + if (uiFromKeyLen) + { + f_memcpy( pucUntilKey, pucFromKey, uiFromKeyLen); + } + uiUntilKeyLen = uiFromKeyLen; + + if (uiUntilKeyLen && + (bIsDBCS || + (!(pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && + bCaseInsensitive))) + { + // NOTE: Always inclusive because this is a matchbegin. + + setUntilCaseByte( pucUntilKey, &uiUntilKeyLen, uiCaseLen, + bIsDBCS, bAscending, FALSE); + } + + // Fill rest of from key with high values after collation values. + + f_memset( &pucFromKey[ uiCollationLen], 0xFF, + SFLM_MAX_KEY_SIZE - uiCollationLen - 2); + uiFromKeyLen = SFLM_MAX_KEY_SIZE - 2; + } + } + else + { + + // Copy from key into until key. + + if (!uiFromKeyLen) + { + uiUntilKeyLen = 0; + } + else + { + if (!bDoFirstSubstring) + { + f_memcpy( pucUntilKey, pucFromKey, uiFromKeyLen); + uiUntilKeyLen = uiFromKeyLen; + } + else if (bAscending) + { + + // Do two copies so that the first substring byte is gone + // in the until key. + + f_memcpy( pucUntilKey, pucFromKey, uiCollationLen); + uiUntilKeyLen = uiCollationLen; + if (bIsDBCS) + { + uiCollationLen++; + } + uiCollationLen++; + f_memcpy( &pucUntilKey [uiUntilKeyLen], + pucFromKey + uiCollationLen, + uiFromKeyLen - uiCollationLen); + uiUntilKeyLen += (uiFromKeyLen - uiCollationLen); + } + else + { + + // Descending order - put the string without the + // first-substring-marker into the from key instead of + // the until key. + + f_memcpy( pucUntilKey, pucFromKey, uiFromKeyLen); + uiUntilKeyLen = uiFromKeyLen; + + // Modify from key to NOT have first-substring-marker. + + f_memcpy( pucUntilKey, pucFromKey, uiCollationLen); + uiFromKeyLen = uiCollationLen; + if (bIsDBCS) + { + uiCollationLen++; + } + uiCollationLen++; + f_memcpy( &pucFromKey [uiFromKeyLen], + pucUntilKey + uiCollationLen, + uiUntilKeyLen - uiCollationLen); + uiFromKeyLen += (uiUntilKeyLen - uiCollationLen); + } + + // Handle scenario of a case-sensitive index, but search is + // case-insensitive. + + if (bIsDBCS || + (!(pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && + bCaseInsensitive)) + { + setFromCaseByte( pucFromKey, &uiFromKeyLen, uiCaseLen, + bIsDBCS, bAscending, FALSE); + setUntilCaseByte( pucUntilKey, &uiUntilKeyLen, uiCaseLen, + bIsDBCS, bAscending, FALSE); + } + } + } + } + } + + // Do the until key now + + if (!pucUntilUTF8Buf) + { + uiUntilKeyLen = KEY_HIGH_VALUE; + } + else if (pucFromUTF8Buf != pucUntilUTF8Buf) + { + if (RC_BAD( rc = bufferIStream.open( + (const char *)pucUntilUTF8Buf, uiUntilBufLen))) + { + goto Exit; + } + + // Add ICD_ESC_CHAR to the icd flags because + // the search string must have BACKSLASHES and '*' escaped. + + uiUntilKeyLen = SFLM_MAX_KEY_SIZE - 2; + bDataTruncated = FALSE; + if (RC_BAD( rc = KYCollateValue( pucUntilKey, &uiUntilKeyLen, + &bufferIStream, SFLM_STRING_TYPE, + pIcd->uiFlags | ICD_ESC_CHAR, pIcd->uiCompareRules, + pIcd->uiLimit, + &uiCollationLen, &uiCaseLen, + uiLanguage, bDoFirstSubstring, + FALSE, &bDataTruncated, + &bOriginalCharsLost))) + { + goto Exit; + } + + bufferIStream.close(); + + if (bDataTruncated) + { + + // Save the original data into pUntilSearchKey so the comparison + // routines can do a comparison on the full value if + // necessary. + + if (RC_BAD( rc = pUntilSearchKey->setUTF8( uiKeyComponent - 1, + pucUntilUTF8Buf, uiUntilBufLen))) + { + goto Exit; + } + *pbCanCompareOnKey = FALSE; + uiUntilFlags |= (SEARCH_KEY_FLAG | TRUNCATED_FLAG); + } + else if (bOriginalCharsLost) + { + *pbCanCompareOnKey = FALSE; + } + + if (uiUntilKeyLen && + (bIsDBCS || + (!(pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE) && + bCaseInsensitive))) + { + setUntilCaseByte( pucUntilKey, &uiUntilKeyLen, uiCaseLen, + bIsDBCS, bAscending, + (uiUntilFlags & EXCLUSIVE_LT_FLAG) + ? TRUE + : FALSE); + } + } + + UW2FBA( (FLMUINT16)(uiFromKeyLen | uiFromFlags), pucFromKeyLenPos); + UW2FBA( (FLMUINT16)(uiUntilKeyLen | uiUntilFlags), pucUntilKeyLenPos); + + if (!(uiFromFlags & EXCLUSIVE_GT_FLAG) && uiFromKeyLen < SFLM_MAX_KEY_SIZE - 2) + { + uiFromKeyLen += kyAddInclComponent( pIndex->uiNumKeyComponents, uiKeyComponent, + &pucFromKey [uiFromKeyLen], + TRUE, SFLM_MAX_KEY_SIZE - 2 - uiFromKeyLen); + } + if (!(uiUntilFlags & EXCLUSIVE_LT_FLAG) && uiUntilKeyLen < SFLM_MAX_KEY_SIZE - 2) + { + uiUntilKeyLen += kyAddInclComponent( pIndex->uiNumKeyComponents, uiKeyComponent, + &pucUntilKey [uiUntilKeyLen], + FALSE, SFLM_MAX_KEY_SIZE - 2 - uiUntilKeyLen); + } + + // Set the FROM and UNTIL key lengths + + if (uiFromKeyLen != KEY_HIGH_VALUE && uiFromKeyLen != KEY_LOW_VALUE) + { + *puiFromKeyLen = uiFromKeyLen + 2; + } + else + { + *puiFromKeyLen = 2; + } + if (uiUntilKeyLen != KEY_HIGH_VALUE && uiUntilKeyLen != KEY_LOW_VALUE) + { + *puiUntilKeyLen = uiUntilKeyLen + 2; + } + else + { + *puiUntilKeyLen = 2; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Build the from and until keys given a field list with operators and + values and an index. +Notes: The knowledge of query definitions is limited in these routines. +****************************************************************************/ +RCODE flmBuildFromAndUntilKeys( + F_Dict * pDict, + F_INDEX * pIndex, + F_TABLE * pTable, + SQL_PRED * pPred, + F_DataVector * pFromSearchKey, + FLMBYTE * pucFromKey, + FLMUINT * puiFromKeyLen, + F_DataVector * pUntilSearchKey, + FLMBYTE * pucUntilKey, + FLMUINT * puiUntilKeyLen, + FLMBOOL * pbDoRowMatch, + FLMBOOL * pbCanCompareOnKey) +{ + RCODE rc = NE_SFLM_OK; + + *puiFromKeyLen = *puiUntilKeyLen = 0; + *pbDoRowMatch = FALSE; + *pbCanCompareOnKey = TRUE; + + if (!pPred) + { + + // Setup a first-to-last key + + flmSetupFirstToLastKey( pucFromKey, puiFromKeyLen, + pucUntilKey, puiUntilKeyLen); + *pbDoRowMatch = TRUE; + *pbCanCompareOnKey = FALSE; + } + else + { + F_COLUMN * pColumn; + ICD * pIcd; + + // Predicates we are looking at should NEVER be notted. They + // will have been weeded out earlier. + + flmAssert( !pPred->bNotted); + + // Handle special cases for indexing presence and/or exists predicate. + + pIcd = pIndex->pKeyIcds; + pColumn = pDict->getColumn( pTable, pIcd->uiColumnNum); + if (pColumn->eDataTyp == SFLM_STRING_TYPE && + !(pIcd->uiFlags & (ICD_PRESENCE | ICD_METAPHONE)) && + pPred->eOperator != SQL_EXISTS_OP) + { + if (RC_BAD( rc = flmAddTextKeyPiece( pPred, pIndex, 1, + pFromSearchKey, pucFromKey, puiFromKeyLen, + pUntilSearchKey, pucUntilKey, puiUntilKeyLen, + pbCanCompareOnKey))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = flmAddNonTextKeyPiece( pPred, pIndex, 1, pColumn, + pFromSearchKey, pucFromKey, puiFromKeyLen, + pUntilSearchKey, pucUntilKey, puiUntilKeyLen, + pbCanCompareOnKey))) + { + goto Exit; + } + } + } + +Exit: + + if (!(*pbCanCompareOnKey)) + { + *pbDoRowMatch = TRUE; + } + + return( rc); +} + diff --git a/sql/src/kybuild.cpp b/sql/src/kybuild.cpp new file mode 100644 index 0000000..6fc4e48 --- /dev/null +++ b/sql/src/kybuild.cpp @@ -0,0 +1,887 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the main routines for building of index keys, +// and adding them to the database. +// +// Tabs: 3 +// +// Copyright (c) 1990-1992, 1994-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define STACK_DATA_BUF_SIZE 64 + +// Local function prototypes + +FSTATIC RCODE kyAddRowIdToKey( + FLMUINT64 ui64RowId, + FLMBYTE * pucKeyBuf, + FLMUINT uiIDBufSize, + FLMUINT * puiIDLen); + +/**************************************************************************** +Desc: Append row ID to the key buffer. +****************************************************************************/ +FSTATIC RCODE kyAddRowIdToKey( + FLMUINT64 ui64RowId, + FLMBYTE * pucKeyBuf, + FLMUINT uiIDBufSize, + FLMUINT * puiIDLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucTmpSen [FLM_MAX_NUM_BUF_SIZE]; + FLMBYTE * pucTmpSen; + + // Put row ID into buffer. If there is room for at least nine + // bytes, we can encode the ID right into the buffer safely. Otherwise, + // we have to use a temporary buffer and see if there is room. + + if (uiIDBufSize >= 9) + { + *puiIDLen = f_encodeSEN( ui64RowId, &pucKeyBuf); + } + else + { + pucTmpSen = &ucTmpSen [0]; + *puiIDLen = f_encodeSEN( ui64RowId, &pucTmpSen); + if (*puiIDLen > uiIDBufSize) + { + rc = RC_SET( NE_SFLM_KEY_OVERFLOW); + goto Exit; + } + f_memcpy( pucKeyBuf, ucTmpSen, *puiIDLen); + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Add an index key to the buffers +****************************************************************************/ +RCODE F_Db::addToKrefTbl( + FLMUINT uiKeyLen, + FLMUINT uiDataLen) +{ + RCODE rc = NE_SFLM_OK; + KREF_ENTRY * pKref; + FLMUINT uiSizeNeeded; + FLMBYTE * pucDest; + + // If the table is FULL, expand the table + + if (m_uiKrefCount == m_uiKrefTblSize) + { + FLMUINT uiAllocSize; + FLMUINT uiOrigKrefTblSize = m_uiKrefTblSize; + + if (m_uiKrefTblSize > 0x8000 / sizeof( KREF_ENTRY *)) + { + m_uiKrefTblSize += 4096; + } + else + { + m_uiKrefTblSize *= 2; + } + + uiAllocSize = m_uiKrefTblSize * sizeof( KREF_ENTRY *); + + rc = f_realloc( uiAllocSize, &m_pKrefTbl); + if (RC_BAD(rc)) + { + m_uiKrefTblSize = uiOrigKrefTblSize; + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + } + + // Allocate memory for the key's KREF and the key itself. + // We allocate one extra byte so we can zero terminate the key + // below. The extra zero character is to ensure that the compare + // in the qsort routine will work. + + uiSizeNeeded = sizeof( KREF_ENTRY) + uiKeyLen + 1 + uiDataLen; + + if (RC_BAD( rc = m_pKrefPool->poolAlloc( uiSizeNeeded, + (void **)&pKref))) + { + goto Exit; + } + + m_pKrefTbl [ m_uiKrefCount++] = pKref; + m_uiTotalKrefBytes += uiSizeNeeded; + + // Fill in all of the fields in the KREF structure. + + pKref->ui16IxNum = (FLMUINT16)m_keyGenInfo.pIndex->uiIndexNum; + pKref->bDelete = m_keyGenInfo.bAddKeys ? FALSE : TRUE; + pKref->ui16KeyLen = (FLMUINT16)uiKeyLen; + pKref->uiSequence = m_uiKrefCount; + pKref->uiDataLen = uiDataLen; + pKref->pRow = m_keyGenInfo.pRow; + + // Copy the key to just after the KREF structure. + + pucDest = (FLMBYTE *)(&pKref [1]); + f_memcpy( pucDest, m_keyGenInfo.pucKeyBuf, uiKeyLen); + + // Null terminate the key so compare in qsort will work. + + pucDest [uiKeyLen] = 0; + if (uiDataLen) + { + f_memcpy( pucDest + uiKeyLen + 1, m_keyGenInfo.pucData, uiDataLen); + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Build the data part for a key. +Notes: This routine is recursive in nature. Will recurse the number of + data components defined in the index. +****************************************************************************/ +RCODE F_Db::buildData( + ICD * pIcd, + FLMUINT uiDataComponent, + FLMUINT uiKeyLen, + FLMUINT uiDataLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDataComponentLen; + FLMBYTE ucTmpSen [FLM_MAX_NUM_BUF_SIZE]; + FLMBYTE * pucTmpSen; + FLMUINT uiSENLen; + FLMUINT uiIDLen; + F_COLUMN_ITEM * pColumnItem; + + if ((pColumnItem = m_keyGenInfo.pRow->getColumn( pIcd->uiColumnNum)) != NULL) + { + uiDataComponentLen = pColumnItem->uiDataLen; + } + else + { + uiDataComponentLen = 0; + } + + // Output the length of the data as a SEN value + + pucTmpSen = &ucTmpSen [0]; + uiSENLen = f_encodeSEN( uiDataComponentLen, &pucTmpSen); + if (uiDataComponentLen + uiSENLen + uiDataLen > m_keyGenInfo.uiDataBufSize) + { + FLMUINT uiNewSize = uiDataComponentLen + uiSENLen + uiDataLen + 512; + + // Allocate the data buffer if it has not been allocated. Otherwise, + // realloc it. + + if (!m_keyGenInfo.bDataBufAllocated) + { + FLMBYTE * pucNewData; + + if (RC_BAD( rc = f_alloc( uiNewSize, &pucNewData))) + { + goto Exit; + } + + if( uiDataLen) + { + f_memcpy( pucNewData, m_keyGenInfo.pucData, uiDataLen); + } + m_keyGenInfo.pucData = pucNewData; + m_keyGenInfo.bDataBufAllocated = TRUE; + } + else + { + + // Reallocate the buffer. + + if (RC_BAD( rc = f_realloc( uiNewSize, &m_keyGenInfo.pucData))) + { + goto Exit; + } + } + + m_keyGenInfo.uiDataBufSize = uiNewSize; + } + f_memcpy( m_keyGenInfo.pucData + uiDataLen, ucTmpSen, uiSENLen); + if (uiDataComponentLen) + { + f_memcpy( m_keyGenInfo.pucData + uiDataLen + uiSENLen, + m_keyGenInfo.pRow->getColumnDataPtr( pIcd->uiColumnNum), + uiDataComponentLen); + } + + // If this is the last data CDL, append IDs to the + // key and output the key and data to the KREF. + // Otherwise, recurse down. + + if (uiDataComponent < m_keyGenInfo.pIndex->uiNumDataComponents) + { + if (RC_BAD( rc = buildData( pIcd + 1, uiDataComponent + 1, uiKeyLen, + uiDataLen + uiDataComponentLen + uiSENLen))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = kyAddRowIdToKey( m_keyGenInfo.pRow->m_ui64RowId, + &m_keyGenInfo.pucKeyBuf [uiKeyLen], + SFLM_MAX_KEY_SIZE - uiKeyLen, &uiIDLen))) + { + goto Exit; + } + if (RC_BAD( rc = addToKrefTbl( uiKeyLen + uiIDLen, + uiDataLen + uiDataComponentLen + uiSENLen))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Finish the current key component. If there is a next one call + build keys. Otherwise, go on to doing data and context pieces. +****************************************************************************/ +RCODE F_Db::finishKeyComponent( + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiIDLen; + + if (uiKeyComponent < m_keyGenInfo.pIndex->uiNumKeyComponents) + { + flmAssert( m_keyGenInfo.bIsCompound); + if (RC_BAD( rc = buildKeys( pIcd + 1, uiKeyComponent + 1, uiKeyLen))) + { + goto Exit; + } + } + else + { + if (m_keyGenInfo.pIndex->pDataIcds) + { + if (RC_BAD( rc = buildData( m_keyGenInfo.pIndex->pDataIcds, 1, uiKeyLen, 0))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = kyAddRowIdToKey( m_keyGenInfo.pRow->m_ui64RowId, + &m_keyGenInfo.pucKeyBuf [uiKeyLen], + SFLM_MAX_KEY_SIZE - uiKeyLen, &uiIDLen))) + { + goto Exit; + } + if (RC_BAD( rc = addToKrefTbl( uiKeyLen + uiIDLen, 0))) + { + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Generate the keys for a text component. +****************************************************************************/ +RCODE F_Db::genTextKeyComponents( + F_COLUMN * pColumn, + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen, + FLMBYTE ** ppucTmpBuf, + FLMUINT * puiTmpBufSize, + void ** ppvMark) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNumChars; + FLMUINT uiStrBytes; + FLMUINT uiSubstrChars; + FLMUINT uiMeta; + FLMBOOL bEachWord = FALSE; + FLMBOOL bMetaphone = FALSE; + FLMBOOL bSubstring = FALSE; + FLMBOOL bWholeString = FALSE; + FLMBOOL bHadAtLeastOneString = FALSE; + FLMBOOL bDataTruncated; + FLMUINT uiSaveKeyLen; + FLMUINT uiElmLen; + FLMUINT uiKeyLenPos = uiKeyLen; + FLMUINT uiCompareRules = pIcd->uiCompareRules; + F_BufferIStream columnBufferIStream; + F_BufferIStream bufferIStream; + FLMBOOL bIsNull; + + uiKeyLen += 2; + uiSaveKeyLen = uiKeyLen; + + if (RC_BAD( rc = m_keyGenInfo.pRow->getTextIStream( this, pIcd->uiColumnNum, + &bIsNull, &columnBufferIStream, &uiNumChars))) + { + goto Exit; + } + + if (bIsNull || !uiNumChars) + { +No_Strings: + + // Save the key component length + + UW2FBA( 0, &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + + rc = finishKeyComponent( pIcd, uiKeyComponent, uiKeyLen); + goto Exit; + } + + if (pIcd->uiFlags & ICD_EACHWORD) + { + bEachWord = TRUE; + + // OR in the compressing of spaces, because we only want to treat + // spaces as delimiters between words. + + uiCompareRules |= FLM_COMP_COMPRESS_WHITESPACE; + } + else if (pIcd->uiFlags & ICD_METAPHONE) + { + bMetaphone = TRUE; + } + else if (pIcd->uiFlags & ICD_SUBSTRING) + { + bSubstring = TRUE; + } + else + { + bWholeString = TRUE; + } + + // Loop on each word or substring in the value + + for (;;) + { + uiKeyLen = uiSaveKeyLen; + bDataTruncated = FALSE; + + if (bWholeString) + { + uiElmLen = SFLM_MAX_KEY_SIZE - uiKeyLen; + if( RC_BAD( rc = KYCollateValue( &m_keyGenInfo.pucKeyBuf [uiKeyLen], + &uiElmLen, + &columnBufferIStream, SFLM_STRING_TYPE, + pIcd->uiFlags, pIcd->uiCompareRules, + pIcd->uiLimit, NULL, NULL, + m_keyGenInfo.pIndex->uiLanguage, + FALSE, FALSE, + &bDataTruncated, NULL))) + { + goto Exit; + } + } + else if (bEachWord) + { + if (*ppucTmpBuf == NULL) + { + *ppvMark = m_tempPool.poolMark(); + *puiTmpBufSize = (FLMUINT)SFLM_MAX_KEY_SIZE + 8; + if (RC_BAD( rc = m_tempPool.poolAlloc( *puiTmpBufSize, + (void **)ppucTmpBuf))) + { + goto Exit; + } + } + + uiStrBytes = *puiTmpBufSize; + if( RC_BAD( rc = KYEachWordParse( &columnBufferIStream, &uiCompareRules, + pIcd->uiLimit, *ppucTmpBuf, &uiStrBytes))) + { + goto Exit; + } + + if (!uiStrBytes) + { + if (!bHadAtLeastOneString) + { + goto No_Strings; + } + break; + } + + if (RC_BAD( rc = bufferIStream.open( + (const char *)*ppucTmpBuf, uiStrBytes))) + { + goto Exit; + } + + // Pass 0 for compare rules because KYEachWordParse will already + // have taken care of them - except for FLM_COMP_CASE_INSENSITIVE. + + uiElmLen = SFLM_MAX_KEY_SIZE - uiKeyLen; + rc = KYCollateValue( &m_keyGenInfo.pucKeyBuf [uiKeyLen], + &uiElmLen, + &bufferIStream, SFLM_STRING_TYPE, + pIcd->uiFlags, + pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE, + pIcd->uiLimit, + NULL, NULL, + m_keyGenInfo.pIndex->uiLanguage, + FALSE, FALSE, &bDataTruncated, NULL); + bufferIStream.close(); + + if( RC_BAD( rc)) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + bHadAtLeastOneString = TRUE; + } + else if (bMetaphone) + { + FLMBYTE ucStorageBuf[ FLM_MAX_NUM_BUF_SIZE]; + FLMUINT uiStorageLen; + + if( RC_BAD( rc = f_getNextMetaphone( &columnBufferIStream, &uiMeta))) + { + if( rc != NE_SFLM_EOF_HIT) + { + goto Exit; + } + rc = NE_SFLM_OK; + if (!bHadAtLeastOneString) + { + goto No_Strings; + } + break; + } + + uiStorageLen = FLM_MAX_NUM_BUF_SIZE; + if( RC_BAD( rc = flmNumber64ToStorage( uiMeta, + &uiStorageLen, ucStorageBuf, FALSE, FALSE))) + { + goto Exit; + } + + if (RC_BAD( rc = bufferIStream.open( + (const char *)ucStorageBuf, uiStorageLen))) + { + goto Exit; + } + + // Pass 0 for compare rules - only applies to strings. + + uiElmLen = SFLM_MAX_KEY_SIZE - uiKeyLen; + rc = KYCollateValue( &m_keyGenInfo.pucKeyBuf [uiKeyLen], + &uiElmLen, + &bufferIStream, SFLM_NUMBER_TYPE, + pIcd->uiFlags, 0, + pIcd->uiLimit, + NULL, NULL, + m_keyGenInfo.pIndex->uiLanguage, + FALSE, FALSE, NULL, NULL); + bufferIStream.close(); + + if( RC_BAD( rc)) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + bHadAtLeastOneString = TRUE; + } + else + { + flmAssert( bSubstring); + if (*ppucTmpBuf == NULL) + { + *ppvMark = m_tempPool.poolMark(); + *puiTmpBufSize = (FLMUINT)SFLM_MAX_KEY_SIZE + 8; + if (RC_BAD( rc = m_tempPool.poolAlloc( *puiTmpBufSize, + (void **)ppucTmpBuf))) + { + goto Exit; + } + } + uiStrBytes = *puiTmpBufSize; + + if( RC_BAD( rc = KYSubstringParse( &columnBufferIStream, &uiCompareRules, + pIcd->uiLimit, *ppucTmpBuf, &uiStrBytes, &uiSubstrChars))) + { + goto Exit; + } + + if (!uiStrBytes) + { + if (!bHadAtLeastOneString) + { + goto No_Strings; + } + break; + } + + if (bHadAtLeastOneString && uiSubstrChars == 1 && !m_keyGenInfo.bIsAsia) + { + break; + } + + if (RC_BAD( rc = bufferIStream.open( + (const char *)*ppucTmpBuf, uiStrBytes))) + { + goto Exit; + } + + // Pass 0 for compare rules, because KYSubstringParse has already + // taken care of them, except for FLM_COMP_CASE_INSENSITIVE + + uiElmLen = SFLM_MAX_KEY_SIZE - uiKeyLen; + rc = KYCollateValue( &m_keyGenInfo.pucKeyBuf [uiKeyLen], + &uiElmLen, + &bufferIStream, SFLM_STRING_TYPE, + pIcd->uiFlags, + pIcd->uiCompareRules & FLM_COMP_CASE_INSENSITIVE, + pIcd->uiLimit, + NULL, NULL, + m_keyGenInfo.pIndex->uiLanguage, + bHadAtLeastOneString ? FALSE : TRUE, FALSE, + &bDataTruncated, NULL); + + bufferIStream.close(); + + if( RC_BAD( rc)) + { + RC_UNEXPECTED_ASSERT( rc); + goto Exit; + } + bHadAtLeastOneString = TRUE; + } + + uiKeyLen += uiElmLen; + + // Save the key component length + + if (!bDataTruncated) + { + UW2FBA( (FLMUINT16)(uiElmLen), + &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + } + else + { + UW2FBA( (FLMUINT16)(uiElmLen | TRUNCATED_FLAG), + &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + } + + if (RC_BAD( rc = finishKeyComponent( pIcd, uiKeyComponent, uiKeyLen))) + { + goto Exit; + } + + if (bWholeString) + { + break; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Generate the keys for other data types besides text. +****************************************************************************/ +RCODE F_Db::genOtherKeyComponent( + F_COLUMN * pColumn, + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiElmLen; + FLMUINT uiKeyLenPos = uiKeyLen; + FLMBOOL bDataTruncated; + FLMBOOL bIsNull; + F_BufferIStream columnBufferIStream; + eDataType eDataTyp; + FLMUINT uiDataLength; + + uiKeyLen += 2; + + if (pIcd->uiFlags & ICD_PRESENCE) + { + f_UINT32ToBigEndian( (FLMUINT32)pIcd->uiColumnNum, &m_keyGenInfo.pucKeyBuf [uiKeyLen]); + uiKeyLen += 4; + + // Save the key component length. + + UW2FBA( 4, &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + } + else + { + if (RC_BAD( rc = m_keyGenInfo.pRow->getIStream( this, + pIcd->uiColumnNum, + &bIsNull, &columnBufferIStream, + &eDataTyp, &uiDataLength))) + { + goto Exit; + } + + if (bIsNull || !columnBufferIStream.remainingSize()) + { + // Save the key component length + + UW2FBA( 0, &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + + rc = finishKeyComponent( pIcd, uiKeyComponent, uiKeyLen); + goto Exit; + } + + // Compute number of bytes left + + uiElmLen = SFLM_MAX_KEY_SIZE - uiKeyLen; + bDataTruncated = FALSE; + + // Pass zero for compare rules - these are not strings. + + if( RC_BAD( rc = KYCollateValue( &m_keyGenInfo.pucKeyBuf [uiKeyLen], + &uiElmLen, &columnBufferIStream, + eDataTyp, pIcd->uiFlags, + 0, pIcd->uiLimit, NULL, NULL, + m_keyGenInfo.pIndex->uiLanguage, + FALSE, FALSE, + &bDataTruncated, NULL))) + { + goto Exit; + } + uiKeyLen += uiElmLen; + + // Save the key component length. + + if (!bDataTruncated) + { + UW2FBA( (FLMUINT16)(uiElmLen), + &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + } + else + { + UW2FBA( (FLMUINT16)(uiElmLen | TRUNCATED_FLAG), + &m_keyGenInfo.pucKeyBuf [uiKeyLenPos]); + } + } + + if (RC_BAD( rc = finishKeyComponent( pIcd, uiKeyComponent, uiKeyLen))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Build all compound keys from the CDL table. +Notes: This routine is recursive in nature. Will recurse the number of + key components defined in the index. +****************************************************************************/ +RCODE F_Db::buildKeys( + ICD * pIcd, + FLMUINT uiKeyComponent, + FLMUINT uiKeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucTmpBuf = NULL; + void * pvMark = NULL; + FLMUINT uiTmpBufSize = 0; + F_COLUMN * pColumn = m_pDict->getColumn( m_keyGenInfo.pTable, + pIcd->uiColumnNum); + + flmAssert( m_keyGenInfo.bIsCompound || uiKeyComponent == 1); + + // Generate the key component + + if (pColumn->eDataTyp == SFLM_STRING_TYPE && !(pIcd->uiFlags & ICD_PRESENCE)) + { + if (RC_BAD( rc = genTextKeyComponents( pColumn, pIcd, uiKeyComponent, uiKeyLen, + &pucTmpBuf, &uiTmpBufSize, &pvMark))) + { + goto Exit; + } + } + else + { + if (RC_BAD( rc = genOtherKeyComponent( pColumn, pIcd, uiKeyComponent, uiKeyLen))) + { + goto Exit; + } + } + +Exit: + + if (pvMark) + { + m_tempPool.poolReset( pvMark); + } + return( rc); +} + +/**************************************************************************** +Desc: Build all keys from combinations of CDLs. Add keys to KREF table. +****************************************************************************/ +RCODE F_Db::buildKeys( + F_INDEX * pIndex, + F_TABLE * pTable, + F_Row * pOldRow, + F_Row * pNewRow) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucDataBuf [STACK_DATA_BUF_SIZE]; + + if (RC_BAD( rc = krefCntrlCheck())) + { + goto Exit; + } + + // Build all of the keys + + m_keyGenInfo.pTable = pTable; + m_keyGenInfo.pIndex = pIndex; + m_keyGenInfo.bIsAsia = (FLMBOOL)(pIndex->uiLanguage >= FLM_FIRST_DBCS_LANG && + pIndex->uiLanguage <= FLM_LAST_DBCS_LANG) + ? TRUE + : FALSE; + m_keyGenInfo.bIsCompound = pIndex->uiNumKeyComponents > 1 ? TRUE : FALSE; + m_keyGenInfo.pucKeyBuf = m_pucKrefKeyBuf; + m_keyGenInfo.pucData = &ucDataBuf [0]; + m_keyGenInfo.uiDataBufSize = sizeof( ucDataBuf); + m_keyGenInfo.bDataBufAllocated = FALSE; + + // Build the keys for the old row. + + if (pOldRow) + { + m_keyGenInfo.pRow = pOldRow; + m_keyGenInfo.bAddKeys = FALSE; + if (RC_BAD( rc = buildKeys( pIndex->pKeyIcds, 1, 0))) + { + goto Exit; + } + } + + // Build all of the keys for the new row + + if (pNewRow) + { + m_keyGenInfo.pRow = pNewRow; + m_keyGenInfo.bAddKeys = TRUE; + if (RC_BAD( rc = buildKeys( pIndex->pKeyIcds, 1, 0))) + { + goto Exit; + } + } + + // Commit keys if we are over the limit. + + if( isKrefOverThreshold()) + { + processDupKeys( pIndex); + if (RC_BAD( rc = keysCommit( FALSE, FALSE))) + { + goto Exit; + } + } + +Exit: + + if (m_keyGenInfo.bDataBufAllocated) + { + f_free( &m_keyGenInfo.pucData); + m_keyGenInfo.bDataBufAllocated = FALSE; + } + + return( rc); +} + +/**************************************************************************** +Desc: Routine that is called when inserting, modifying, or removing a row + from a table. +****************************************************************************/ +RCODE F_Db::updateIndexKeys( + FLMUINT uiTableNum, + F_Row * pOldRow, + F_Row * pNewRow) +{ + RCODE rc = NE_SFLM_OK; + F_TABLE * pTable = m_pDict->getTable( uiTableNum); + F_INDEX * pIndex; + FLMUINT uiIndexNum; + FLMUINT64 ui64RowId; + + if (pOldRow) + { + ui64RowId = pOldRow->getRowId(); + + // New row, if any, should have same row ID as old row. + + flmAssert( !pNewRow || pNewRow->getRowId() == ui64RowId); + } + else + { + flmAssert( pNewRow); + ui64RowId = pNewRow->getRowId(); + } + +//VISIT - Don't call this routine on a modify operation if none of the columns +//that were modified are indexed. The higher level routine can make this check. + + // Go through each index on this table. + + uiIndexNum = pTable->uiFirstIndexNum; + while (uiIndexNum) + { + pIndex = m_pDict->getIndex( uiIndexNum); + + if (!(pIndex->uiFlags & (IXD_OFFLINE | IXD_SUSPENDED)) || + ui64RowId <= pIndex->ui64LastRowIndexed) + { + if (RC_BAD( rc = buildKeys( pIndex, pTable, pOldRow, pNewRow))) + { + goto Exit; + } + } + uiIndexNum = pIndex->uiNextIndexNum; + } + +Exit: + + return( rc); +} + diff --git a/sql/src/kycollat.cpp b/sql/src/kycollat.cpp new file mode 100644 index 0000000..6c3e10a --- /dev/null +++ b/sql/src/kycollat.cpp @@ -0,0 +1,435 @@ +//------------------------------------------------------------------------------ +// Desc: Index collation routines +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: kycollat.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC RCODE KYFormatUTF8Text( + IF_PosIStream * pIStream, + FLMUINT uiFlags, + FLMUINT uiCompareRules, + F_DynaBuf * pDynaBuf); + +/**************************************************************************** +Desc: Build a collated key value piece. +****************************************************************************/ +RCODE KYCollateValue( + FLMBYTE * pucDest, + FLMUINT * puiDestLen, + IF_PosIStream * pIStream, + FLMUINT uiDataType, + FLMUINT uiFlags, + FLMUINT uiCompareRules, + FLMUINT uiLimit, + FLMUINT * puiCollationLen, + FLMUINT * puiLuLen, + FLMUINT uiLanguage, + FLMBOOL bFirstSubstring, + FLMBOOL bDataTruncated, + FLMBOOL * pbDataTruncated, + FLMBOOL * pbOriginalCharsLost) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDestLen; + IF_BufferIStream * pBufferIStream = NULL; + FLMUINT uiCharLimit; + FLMUINT uiLength; + FLMBYTE * pucTmpDest; + FLMUINT uiBytesRead; + FLMBOOL bHaveData = TRUE; + FLMUNICODE uChar; + FLMBYTE ucDynaBuf[ 64]; + F_DynaBuf dynaBuf( ucDynaBuf, sizeof( ucDynaBuf)); + + if (puiLuLen) + { + *puiLuLen = 0; + } + + if ((uiDestLen = *puiDestLen) == 0) + { + rc = RC_SET( NE_SFLM_KEY_OVERFLOW); + goto Exit; + } + + if (uiDataType != SFLM_STRING_TYPE) + { + if( !pIStream->remainingSize()) + { + bHaveData = FALSE; + } + } + else + { + FLMUINT64 ui64SavePosition = pIStream->getCurrPosition(); + + if( RC_BAD( rc = f_readUTF8CharAsUnicode( + pIStream, &uChar))) + { + if (rc == NE_SFLM_EOF_HIT) + { + bHaveData = FALSE; + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + + if( RC_BAD( rc = pIStream->positionTo( ui64SavePosition))) + { + goto Exit; + } + + // The text is expected to be 0-terminated UTF-8 + + if ((uiFlags & ICD_ESC_CHAR) || + (uiCompareRules & + (FLM_COMP_COMPRESS_WHITESPACE | + FLM_COMP_NO_WHITESPACE | + FLM_COMP_NO_UNDERSCORES | + FLM_COMP_NO_DASHES | + FLM_COMP_WHITESPACE_AS_SPACE | + FLM_COMP_IGNORE_LEADING_SPACE | + FLM_COMP_IGNORE_TRAILING_SPACE))) + { + dynaBuf.truncateData( 0); + if (RC_BAD( rc = KYFormatUTF8Text( pIStream, + uiFlags, uiCompareRules, &dynaBuf))) + { + goto Exit; + } + + if( RC_BAD( rc = FlmAllocBufferIStream( &pBufferIStream))) + { + goto Exit; + } + + if (RC_BAD( rc = pBufferIStream->open( + (const char *)dynaBuf.getBufferPtr(), dynaBuf.getDataLength()))) + { + goto Exit; + } + pIStream = pBufferIStream; + } + + uiCharLimit = uiLimit ? uiLimit : ICD_DEFAULT_LIMIT; + + if( (uiLanguage >= FLM_FIRST_DBCS_LANG ) && + (uiLanguage <= FLM_LAST_DBCS_LANG)) + { + if( RC_BAD( rc = f_asiaUTF8ToColText( pIStream, pucDest, &uiDestLen, + (uiCompareRules & FLM_COMP_CASE_INSENSITIVE) + ? TRUE + : FALSE, + puiCollationLen, puiLuLen, + uiCharLimit, bFirstSubstring, + bDataTruncated, pbDataTruncated))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = flmUTF8ToColText( pIStream, pucDest, &uiDestLen, + (uiCompareRules & FLM_COMP_CASE_INSENSITIVE) + ? TRUE + : FALSE, + puiCollationLen, puiLuLen, + uiLanguage, uiCharLimit, bFirstSubstring, + bDataTruncated, + pbOriginalCharsLost, pbDataTruncated))) + { + goto Exit; + } + } + } + + // TRICKY: uiDestLen could be set to zero if text and no value. + + if (!bHaveData || !uiDestLen) + { + uiDestLen = 0; + goto Exit; + } + + switch (uiDataType) + { + case SFLM_STRING_TYPE: + break; + + case SFLM_NUMBER_TYPE: + { + FLMBYTE ucTmpBuf [FLM_MAX_NUM_BUF_SIZE]; + + uiLength = (FLMUINT)pIStream->remainingSize(); + + flmAssert( uiLength <= sizeof( ucTmpBuf)); + + if (RC_BAD( rc = pIStream->read( ucTmpBuf, uiLength, &uiBytesRead))) + { + goto Exit; + } + flmAssert( uiBytesRead == uiLength); + if (RC_BAD( rc = flmStorageNum2CollationNum( ucTmpBuf, + uiBytesRead, pucDest, &uiDestLen))) + { + goto Exit; + } + break; + } + + case SFLM_BINARY_TYPE: + { + uiLength = (FLMUINT)pIStream->remainingSize(); + pucTmpDest = pucDest; + + if (uiLength >= uiLimit) + { + uiLength = uiLimit; + bDataTruncated = TRUE; + } + + // We don't want any single key piece to "pig out" more + // than 256 bytes of the key + + if (uiDestLen > 256) + { + uiDestLen = 256; + } + + if (uiLength > uiDestLen) + { + + // Compute length so will not overflow + + uiLength = uiDestLen; + bDataTruncated = TRUE; + } + else + { + uiDestLen = uiLength; + } + + // Store as is. + + if (RC_BAD( rc = pIStream->read( pucTmpDest, uiDestLen, &uiBytesRead))) + { + goto Exit; + } + + if (bDataTruncated && pbDataTruncated) + { + *pbDataTruncated = TRUE; + } + break; + } + + default: + { + rc = RC_SET( NE_SFLM_CANNOT_INDEX_DATA_TYPE); + break; + } + } + +Exit: + + if( pBufferIStream) + { + pBufferIStream->Release(); + } + + *puiDestLen = uiDestLen; + return( rc); +} + +/**************************************************************************** +Desc: Format text removing leading and trailing spaces. Treat + underscores as spaces. As options, remove all spaces and dashes. +Ret: NE_SFLM_OK always. WIll truncate so text will fill SFLM_MAX_KEY_SIZE. + Allocate 8 more than SFLM_MAX_KEY_SIZE for psDestBuf. +Visit: Pass in uiLimit and pass back a truncated flag when the + string is truncated. This was not done because we will have + to get the exact truncated count that is done in f_tocoll.cpp + and that could introduce some bugs. +****************************************************************************/ +FSTATIC RCODE KYFormatUTF8Text( + IF_PosIStream * pIStream, + FLMUINT uiFlags, // ICD flags + FLMUINT uiCompareRules, // ICD compare rules + F_DynaBuf * pDynaBuf) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFirstSpaceCharPos = FLM_MAX_UINT; + FLMUNICODE uChar; + FLMUINT uiSize; + FLMUINT uiStrSize = 0; + FLMBYTE * pucTmp; + + if( !pIStream->remainingSize()) + { + pDynaBuf->truncateData( 0); + goto Exit; + } + + for (;;) + { + if (RC_BAD( rc = f_readUTF8CharAsUnicode( pIStream, &uChar))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + break; + } + goto Exit; + } + if ((uChar = f_convertChar( uChar, uiCompareRules)) == 0) + { + continue; + } + + if (uChar == ASCII_SPACE) + { + if (uiCompareRules & + (FLM_COMP_COMPRESS_WHITESPACE | + FLM_COMP_IGNORE_TRAILING_SPACE)) + { + + // Remember the position of the first space. + // When we come to the end of the spaces, we may reset + // the size to compress out spaces if necessary. Or, + // we may opt to get rid of all of them. + + if (uiFirstSpaceCharPos == FLM_MAX_UINT) + { + uiFirstSpaceCharPos = uiStrSize; + } + } + } + else + { + + // Once we hit a non-space character, we can turn off the + // ignore leading spaces flag. + + uiCompareRules &= (~(FLM_COMP_IGNORE_LEADING_SPACE)); + + // See if we need to compress spaces. + + if (uiFirstSpaceCharPos != FLM_MAX_UINT) + { + + // Output exactly one ASCII_SPACE character if we are compressing + // spaces. If we are not compressing spaces, then the only other + // way uiFirstSpaceCharPos would have been set is if we were + // ignoring trailing spaces. In that case, since the spaces + // were not trailing spaces, we need to leave them as is. + + if (uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE) + { + + // A space will already have been encoded into the string. + // Since we know a space takes exactly one byte in the UTF8 + // space, we can simply set our pointer one byte past where + // the last non-space character was found. + + uiStrSize = uiFirstSpaceCharPos + 1; + pDynaBuf->truncateData( uiStrSize); + } + uiFirstSpaceCharPos = FLM_MAX_UINT; + } + + // If we are allowing escaped characters, backslash is treated + // always as an escape character. Whatever follows the + // backslash is the character we need to process. + + if (uChar == ASCII_BACKSLASH && (uiFlags & ICD_ESC_CHAR)) + { + if (RC_BAD( rc = f_readUTF8CharAsUnicode( pIStream, &uChar))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + } + } + + // Output the character - need at most three bytes + + if (RC_BAD( rc = pDynaBuf->allocSpace( 3, (void **)&pucTmp))) + { + goto Exit; + } + uiSize = 3; + if (RC_BAD( rc = f_uni2UTF8( uChar, pucTmp, &uiSize))) + { + goto Exit; + } + uiStrSize += uiSize; + pDynaBuf->truncateData( uiStrSize); + } + + // If uiFirstSpaceCharPos != FLM_MAX_UINT, it means that all of the + // characters at the end of the string were spaces. If we + // are ignoring trailing spaces, we need to truncate the string so + // they will be ignored. Otherwise, we need to compress them into + // a single space. + + if (uiFirstSpaceCharPos != FLM_MAX_UINT) + { + if (uiCompareRules & FLM_COMP_IGNORE_TRAILING_SPACE) + { + uiStrSize = uiFirstSpaceCharPos; + } + else + { + flmAssert( uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE); + + // A space will already have been encoded into the string. + // Since we know a space takes exactly one byte in the UTF8 + // space, we can simply set our pointer one byte past where + // the last non-space character was found. + + uiStrSize = uiFirstSpaceCharPos + 1; + } + pDynaBuf->truncateData( uiStrSize); + } + + // Terminate the UTF-8 string + + if (RC_BAD( rc = pDynaBuf->appendByte( 0))) + { + goto Exit; + } + +Exit: + + return( rc); +} diff --git a/sql/src/kyeword.cpp b/sql/src/kyeword.cpp new file mode 100644 index 0000000..de40cd0 --- /dev/null +++ b/sql/src/kyeword.cpp @@ -0,0 +1,425 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the code to parse out individual words and +// substrings in a text string. +// +// Tabs: 3 +// +// Copyright (c) 1990-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: kyeword.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC RCODE flmGetCharacter( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUINT16 * pui16WPValue, + FLMUNICODE * puUniValue); + +FSTATIC RCODE flmTextGetCharType( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUNICODE * puUniValue, + FLMUINT * puiType); + +/***************************************************************************** +Desc: +*****************************************************************************/ +FINLINE FLMUINT flmCharTypeAnsi7( + FLMUINT16 ui16Char) +{ + if( (ui16Char >= ASCII_LOWER_A && ui16Char <= ASCII_LOWER_Z) || + (ui16Char >= ASCII_UPPER_A && ui16Char <= ASCII_UPPER_Z) || + (ui16Char >= ASCII_ZERO && ui16Char <= ASCII_NINE)) + { + return SDWD_CHR; + } + + if( ui16Char == 0x27) + { + return WDJN_CHR; + } + + if( ui16Char <= 0x2B) + { + return DELI_CHR; + } + + if( ui16Char == ASCII_COMMA || + ui16Char == ASCII_DASH || + ui16Char == ASCII_DOT || + ui16Char == ASCII_SLASH || + ui16Char == ASCII_COLON || + ui16Char == ASCII_AT || + ui16Char == ASCII_BACKSLASH || + ui16Char == ASCII_UNDERSCORE) + { + return WDJN_CHR; + } + + return DELI_CHR; +} + +/***************************************************************************** +Desc: Return the next WP or unicode character value. +Return: Number of bytes formatted to return the character value. +*****************************************************************************/ +FSTATIC RCODE flmGetCharacter( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUINT16 * pui16WPValue, + FLMUNICODE * puUniValue) +{ + RCODE rc = NE_SFLM_OK; + FLMUNICODE uChar = 0; + FLMUINT64 ui64AfterLastSpacePos = 0; + FLMBOOL bLastCharWasSpace = FALSE; + FLMUINT uiCompareRules = *puiCompareRules; + + for( ;;) + { + if (RC_BAD( rc = f_readUTF8CharAsUnicode( pIStream, &uChar))) + { + if (rc != NE_SFLM_EOF_HIT) + { + goto Exit; + } + rc = NE_SFLM_OK; + if (bLastCharWasSpace && + !(uiCompareRules & FLM_COMP_IGNORE_TRAILING_SPACE)) + { + // bLastCharWasSpace flag can only be TRUE if either + // FLM_COMP_IGNORE_TRAILING_SPACE is set or + // FLM_COMP_COMPRESS_WHITESPACE is set. + + flmAssert( uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE); + uChar = ASCII_SPACE; + } + else + { + uChar = 0; + } + break; + } + + if ((uChar = f_convertChar( uChar, uiCompareRules)) == 0) + { + continue; + } + + if (uChar == ASCII_SPACE) + { + if (uiCompareRules & FLM_COMP_COMPRESS_WHITESPACE) + { + bLastCharWasSpace = TRUE; + ui64AfterLastSpacePos = pIStream->getCurrPosition(); + } + else if (uiCompareRules & FLM_COMP_IGNORE_TRAILING_SPACE) + { + + // If the ignore trailing space flag is set, but the compress + // space flag is not set, remember the position of the + // first space character. If we hit a non-space character, + // we will reposition to after this space character. + + if (!bLastCharWasSpace) + { + bLastCharWasSpace = TRUE; + ui64AfterLastSpacePos = pIStream->getCurrPosition(); + } + } + else + { + break; + } + } + else + { + + // Disable the ignore leading space flag, because we are now + // past all leading space, and we don't want spaces ignored + // now on account of that flag. + + uiCompareRules &= (~(FLM_COMP_IGNORE_LEADING_SPACE)); + if (bLastCharWasSpace) + { + + // Position to after the last space + + if (RC_BAD( rc = pIStream->positionTo( ui64AfterLastSpacePos))) + { + goto Exit; + } + uChar = ASCII_SPACE; + bLastCharWasSpace = FALSE; + } + break; + } + } + + if (pui16WPValue) + { + if (!f_unicodeToWP( uChar, pui16WPValue)) + { + *pui16WPValue = 0; + } + } + + if (puUniValue) + { + *puUniValue = uChar; + } + +Exit: + + *puiCompareRules = uiCompareRules; + + return( rc); +} + +/**************************************************************************** +Desc: Substring-ize the string in a node. Normalize spaces and hyphens if + told to. Example: ABC DEF + ABC DEF + BC DEF + C DEF + DEF +****************************************************************************/ +RCODE KYSubstringParse( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, // [in/out] comparison rules + FLMUINT uiLimitParm, // [in] Max characters + FLMBYTE * pucSubstrBuf, // [out] buffer to fill + FLMUINT * puiSubstrBytes, // [out] returns length + FLMUINT * puiSubstrChars) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDestOffset = 0; + FLMUINT uiDestSize = *puiSubstrBytes; + FLMUINT uiLimit = uiLimitParm ? uiLimitParm : ICD_DEFAULT_SUBSTRING_LIMIT; + FLMUINT uiCharCnt = 0; + FLMUINT uiSize; + FLMBOOL bFirstCharacter = TRUE; + FLMUINT64 ui64SavePosition = pIStream->getCurrPosition(); + + // The limit must return one more than requested in order + // for the text to collation routine to set the truncated flag. + + uiLimit++; + + while (uiLimit--) + { + FLMUNICODE uChar; + + if( RC_BAD( rc = flmGetCharacter( pIStream, puiCompareRules, NULL, &uChar))) + { + goto Exit; + } + + if (!uChar) + { + break; + } + + uiCharCnt++; + + uiSize = uiDestSize - uiDestOffset; + if (RC_BAD( rc = f_uni2UTF8( uChar, &pucSubstrBuf[ uiDestOffset], &uiSize))) + { + goto Exit; + } + uiDestOffset += uiSize; + + // If on the first word, position to start on next character + // for the next call. + + if (bFirstCharacter) + { + bFirstCharacter = FALSE; + + // First character - save position so we can restore it + // upon leaving the routine. + + ui64SavePosition = pIStream->getCurrPosition(); + } + } + + if (uiDestOffset) + { + pucSubstrBuf[ uiDestOffset++] = 0; + } + + *puiSubstrBytes = (FLMUINT)uiDestOffset; + *puiSubstrChars = uiCharCnt; + + // Restore position of stream to first character after the first + // character we found - to ready for next call. + + if (RC_BAD( rc = pIStream->positionTo( ui64SavePosition))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE KYEachWordParse( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUINT uiLimit, // [in] Max characters + FLMBYTE * pucWordBuf, // [out] Buffer of at least SFLM_MAX_KEY_SIZE + FLMUINT * puiWordLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bSkippingDelim = TRUE; + FLMUINT uiWordLen = 0; + FLMUINT uiWordBufSize = *puiWordLen; + FLMUNICODE uChar; + FLMUINT uiType = 0; + FLMUINT uiSize; + + if (!uiLimit) + { + uiLimit = ICD_DEFAULT_SUBSTRING_LIMIT; + } + + while (uiLimit) + { + if (RC_BAD( rc = flmTextGetCharType( pIStream, puiCompareRules, &uChar, &uiType))) + { + goto Exit; + } + if (!uChar) + { + break; + } + + // Determine how to handle what we got. + + if (bSkippingDelim) + { + // If we were skipping delimiters, and we run into a non-delimiter + // character, set the bSkippingDelim flag to FALSE to indicate the + // beginning of a word. + + if (uiType & SDWD_CHR) + { + bSkippingDelim = FALSE; + uiLimit--; + uiSize = uiWordBufSize - uiWordLen; + if (RC_BAD( rc = f_uni2UTF8( uChar, &pucWordBuf [uiWordLen], + &uiSize))) + { + goto Exit; + } + uiWordLen += uiSize; + } + } + else + { + + // If we were NOT skipping delimiters, and we run into a delimiter + // output the word. + + if (uiType & (DELI_CHR | WDJN_CHR)) + { + break; + } + uiSize = uiWordBufSize - uiWordLen; + if (RC_BAD( rc = f_uni2UTF8( uChar, &pucWordBuf [uiWordLen], + &uiSize))) + { + goto Exit; + } + uiWordLen += uiSize; + } + } + + // Return the word, if any + + if (uiWordLen) + { + pucWordBuf [uiWordLen++] = 0; + } + *puiWordLen = uiWordLen; + +Exit: + + return( rc); +} + +/***************************************************************************** +Desc: Return the next WP or unicode character value and parsing type. +*****************************************************************************/ +FSTATIC RCODE flmTextGetCharType( + IF_PosIStream * pIStream, + FLMUINT * puiCompareRules, + FLMUNICODE * puUniValue, // [out] Unicode value + FLMUINT * puiType // Char attribute type. + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT16 ui16WPValue; + FLMUINT uiCharSet; + + // We add on compress white space flag because we really want to ignore + // spaces anyway - we are trying to get the "words" from this stream. + + if( RC_BAD( rc = flmGetCharacter( pIStream, puiCompareRules, + &ui16WPValue, puUniValue))) + { + goto Exit; + } + + if (ui16WPValue) + { + if (ui16WPValue < 0x080) + { + *puiType = flmCharTypeAnsi7( ui16WPValue); + goto Exit; + } + uiCharSet = (FLMUINT)(ui16WPValue >> 8); + + if (uiCharSet == 1 || uiCharSet == 2 || + (uiCharSet >= 8 && uiCharSet <= 11)) + { + *puiType = SDWD_CHR; + goto Exit; + } + + *puiType = DELI_CHR; + } + else + { + + // For now all unmapped unicode characters are treated + // as delimeters + + *puiType = DELI_CHR; + } + +Exit: + + return( rc); +} diff --git a/sql/src/kyqsort.cpp b/sql/src/kyqsort.cpp new file mode 100644 index 0000000..8cbbf27 --- /dev/null +++ b/sql/src/kyqsort.cpp @@ -0,0 +1,1215 @@ +//------------------------------------------------------------------------------ +// Desc: Contains specific q-sort code to sort FLAIM's KREF structures. +// +// Tabs: 3 +// +// Copyright (c) 1990-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: kyqsort.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define KY_SWAP( pKrefTbl, leftP, rightP) \ + pTempKref = pKrefTbl [leftP]; \ + pKrefTbl [leftP] = pKrefTbl [rightP]; \ + pKrefTbl [rightP] = pTempKref + +FSTATIC RCODE ixKeyGetRowId( + F_INDEX * pIndex, + const FLMBYTE * pucKey, + const FLMBYTE * pucKeyEnd, + FLMUINT uiKeyComponent, + FLMUINT64 * pui64RowId); + +FSTATIC RCODE ixKeyGetUTF8( + F_Db * pDb, + FLMUINT uiTableNum, + ICD * pIcd, + F_Row * pOldRow, + FLMUINT64 ui64RowId, + F_DataVector * pSearchKey, + FLMUINT uiKeyComponent, + F_DynaBuf * pDynaBuf); + +FSTATIC RCODE ixKeyGetBinary( + F_Db * pDb, + FLMUINT uiTableNum, + ICD * pIcd, + F_Row * pOldRow, + FLMUINT64 ui64RowId, + F_DataVector * pSearchKey, + FLMUINT uiKeyComponent, + F_DynaBuf * pDynaBuf); + +FSTATIC RCODE krefQuickSort( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY ** pEntryTbl, + FLMUINT uiLowerBounds, + FLMUINT uiUpperBounds); + +FSTATIC RCODE krefKillDups( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY ** pKrefTbl, + FLMUINT * puiKrefTotal); + +/*************************************************************************** +Desc: Get the row ID from a key. +*****************************************************************************/ +FSTATIC RCODE ixKeyGetRowId( + F_INDEX * pIndex, + const FLMBYTE * pucKey, + const FLMBYTE * pucKeyEnd, + FLMUINT uiKeyComponent, + FLMUINT64 * pui64RowId) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiComponent; + FLMUINT uiComponentLen; + + // Skip past all of the remaining key components so we can get to the + // row ID. We are currently positioned on the key component + // specified in uiKeyComponent. NOTE: uiKeyComponent is zero-based, + // 0=1st component, 1=2nd component, etc. + + uiComponent = uiKeyComponent; + while (pucKey < pucKeyEnd && uiComponent < pIndex->uiNumKeyComponents) + { + uiComponentLen = getKeyComponentLength( pucKey); + if (uiComponentLen != KEY_HIGH_VALUE && uiComponentLen != KEY_LOW_VALUE) + { + pucKey += (uiComponentLen + 2); + } + else + { + pucKey += 2; + } + uiComponent++; + } + + // See if there is a row ID in the key. A 0xFF could be present if + // we have set a "high" row ID. + + if (pucKey >= pucKeyEnd || *pucKey == 0xFF) + { + *pui64RowId = 0; + goto Exit; + } + + // At this point, we better have a row ID. + + if (RC_BAD( rc = f_decodeSEN64( &pucKey, pucKeyEnd, pui64RowId))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Do binary comparison. +*****************************************************************************/ +FINLINE FLMINT ixKeyCompareBinary( + const void * pvData1, + FLMUINT uiLen1, + const void * pvData2, + FLMUINT uiLen2, + FLMBOOL bSortAscending) +{ + FLMINT iCompare; + + if (uiLen1 > uiLen2) + { + if ((iCompare = f_memcmp( pvData1, pvData2, uiLen2)) >= 0) + { + return( bSortAscending ? 1 : -1); + } + else + { + return( bSortAscending ? -1 : 1); + } + } + else if (uiLen1 < uiLen2) + { + if ((iCompare = f_memcmp( pvData1, pvData2, uiLen1)) <= 0) + { + return( bSortAscending ? -1 : 1); + } + else + { + return( bSortAscending ? 1 : -1); + } + } + else + { + if ((iCompare = f_memcmp( pvData1, pvData2, uiLen1)) != 0) + { + if (iCompare < 0) + { + return( bSortAscending ? -1 : 1); + } + else + { + return( bSortAscending ? 1 : -1); + } + } + } + + return( 0); +} + +/*************************************************************************** +Desc: Get the UTF8 value for a particular key component. +*****************************************************************************/ +FSTATIC RCODE ixKeyGetUTF8( + F_Db * pDb, + FLMUINT uiTableNum, + ICD * pIcd, + F_Row * pOldRow, + FLMUINT64 ui64RowId, + F_DataVector * pSearchKey, + FLMUINT uiKeyComponent, + F_DynaBuf * pDynaBuf) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBufSize; + FLMBOOL bIsNull; + char * pszDestBuffer; + F_Row * pRow = NULL; + + if (ui64RowId) + { + if (!pOldRow || pOldRow->getRowId() != ui64RowId) + { + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( pDb, + uiTableNum, ui64RowId, &pRow))) + { + goto Exit; + } + } + else + { + pRow = pOldRow; + } + + if (RC_BAD( rc = pRow->getUTF8( pDb, pIcd->uiColumnNum, NULL, + 0, &bIsNull, NULL, &uiBufSize))) + { + goto Exit; + } + if (bIsNull || !uiBufSize) + { + pDynaBuf->truncateData( 0); + } + else + { + if( RC_BAD( rc = pDynaBuf->allocSpace( uiBufSize, + (void **)&pszDestBuffer))) + { + goto Exit; + } + if (RC_BAD( rc = pRow->getUTF8( pDb, pIcd->uiColumnNum, + pszDestBuffer, uiBufSize, + &bIsNull, NULL, NULL))) + { + goto Exit; + } + } + } + else + { + if (RC_BAD( rc = pSearchKey->getUTF8( uiKeyComponent, pDynaBuf))) + { + goto Exit; + } + } + +Exit: + + if (pRow && pRow != pOldRow) + { + pRow->ReleaseRow(); + } + + return( rc); +} + +/*************************************************************************** +Desc: Get the binary value for a particular key component. +*****************************************************************************/ +FSTATIC RCODE ixKeyGetBinary( + F_Db * pDb, + FLMUINT uiTableNum, + ICD * pIcd, + F_Row * pOldRow, + FLMUINT64 ui64RowId, + F_DataVector * pSearchKey, + FLMUINT uiKeyComponent, + F_DynaBuf * pDynaBuf) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBufSize; + FLMBOOL bIsNull; + FLMBYTE * pucDestBuffer; + F_Row * pRow = NULL; + + if (ui64RowId) + { + if (!pOldRow || pOldRow->getRowId() != ui64RowId) + { + if( RC_BAD( rc = gv_SFlmSysData.pRowCacheMgr->retrieveRow( pDb, + uiTableNum, ui64RowId, &pRow))) + { + goto Exit; + } + } + else + { + pRow = pOldRow; + } + + pRow->getDataLen( pDb, pIcd->uiColumnNum, &uiBufSize, &bIsNull); + if (bIsNull || !uiBufSize) + { + pDynaBuf->truncateData( 0); + } + else + { + if( RC_BAD( rc = pDynaBuf->allocSpace( uiBufSize, + (void **)&pucDestBuffer))) + { + goto Exit; + } + if (RC_BAD( rc = pRow->getBinary( pDb, pIcd->uiColumnNum, + pucDestBuffer, uiBufSize, NULL, &bIsNull))) + { + goto Exit; + } + } + } + else + { + if (RC_BAD( rc = pSearchKey->getBinary( uiKeyComponent, pDynaBuf))) + { + goto Exit; + } + } + +Exit: + + if (pRow && pRow != pOldRow) + { + pRow->ReleaseRow(); + } + + return( rc); +} + +/*************************************************************************** +Desc: Compares result set entries during the finalization stage to allow + the result set to be sorted and to remove duplicates. +*****************************************************************************/ +RCODE ixKeyCompare( + F_Db * pDb, + F_INDEX * pIndex, + F_DataVector * pSearchKey, + F_Row * pRow1, + F_Row * pRow2, + FLMBOOL bCompareRowId, + const void * pvKey1, + FLMUINT uiKeyLen1, + const void * pvKey2, + FLMUINT uiKeyLen2, + FLMINT * piCompare) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiKeyComponent; + ICD * pIcd; + FLMUINT uiComponentLen1; + FLMUINT uiComponentLen2; + FLMBOOL bTruncated1; + FLMBOOL bTruncated2; + const FLMBYTE * pucKey1 = (const FLMBYTE *)pvKey1; + const FLMBYTE * pucKey2 = (const FLMBYTE *)pvKey2; + const FLMBYTE * pucKeyEnd1 = pucKey1 + uiKeyLen1; + const FLMBYTE * pucKeyEnd2 = pucKey2 + uiKeyLen2; + FLMBOOL bSortAscending; + FLMBOOL bSortMissingHigh; + FLMUINT64 ui64RowId1; + FLMUINT64 ui64RowId2; + + flmAssert( uiKeyLen1 && uiKeyLen2); + + // Loop for each compound piece of key + + uiKeyComponent = 0; + pIcd = pIndex->pKeyIcds; + for (;;) + { + bSortAscending = (pIcd->uiFlags & ICD_DESCENDING) ? FALSE : TRUE; + bSortMissingHigh = (pIcd->uiFlags & ICD_MISSING_HIGH) ? TRUE : FALSE; + uiComponentLen1 = getKeyComponentLength( pucKey1); + uiComponentLen2 = getKeyComponentLength( pucKey2); + + // See if either component is a "high" key + // NOTE: KEY_HIGH_VALUE always sorts highest, regardless of + // ascending or descending. It is never actually stored. It is + // only passed in for searching. + + if (uiComponentLen1 == KEY_HIGH_VALUE) + { + if (uiComponentLen2 == KEY_HIGH_VALUE) + { + uiComponentLen1 = uiComponentLen2 = 0; + goto Test_Exclusive; + } + else + { + *piCompare = 1; + goto Exit; + } + } + else if (uiComponentLen2 == KEY_HIGH_VALUE) + { + *piCompare = -1; + goto Exit; + } + + // See if either component is a "low" key + // NOTE: KEY_LOW_VALUE always sorts lowest, regardless of + // ascending or descending. It is never actually stored. It is + // only passed in for searching. + + if (uiComponentLen1 == KEY_LOW_VALUE) + { + if (uiComponentLen2 == KEY_LOW_VALUE) + { + uiComponentLen1 = uiComponentLen2 = 0; + goto Test_Exclusive; + } + else + { + *piCompare = -1; + goto Exit; + } + } + else if (uiComponentLen2 == KEY_LOW_VALUE) + { + *piCompare = 1; + goto Exit; + } + + // See if either component is missing. Need to apply the rules for + // sorting missing components in that case. + + if (!uiComponentLen1) + { + if (uiComponentLen2) + { + if (bSortMissingHigh) + { + *piCompare = bSortAscending ? 1 : -1; + } + else + { + *piCompare = bSortAscending ? -1 : 1; + } + goto Exit; + } + else + { + goto Test_Exclusive; + } + } + else if (!uiComponentLen2) + { + if (bSortMissingHigh) + { + *piCompare = bSortAscending ? -1 : 1; + } + else + { + *piCompare = bSortAscending ? 1 : -1; + } + goto Exit; + } + else + { + + // Component length must not exceed remaining length of key. + + flmAssert( pucKey1 + 2 + uiComponentLen1 <= pucKeyEnd1 && + pucKey2 + 2 + uiComponentLen2 <= pucKeyEnd2); + + if ((*piCompare = ixKeyCompareBinary( pucKey1 + 2, uiComponentLen1, + pucKey2 + 2, uiComponentLen2, bSortAscending)) != 0) + { + goto Exit; + } + + // Data is equal, see if one or the other is truncated. + + bTruncated1 = isKeyComponentTruncated( pucKey1); + bTruncated2 = isKeyComponentTruncated( pucKey2); + + if (bTruncated1 || bTruncated2) + { + if (!bTruncated2) + { + *piCompare = bSortAscending ? 1 : -1; + goto Exit; + } + else if (!bTruncated1) + { + *piCompare = bSortAscending ? -1 : 1; + goto Exit; + } + + // Need to get the row that holds the data for the 1st key. + + if (isSearchKeyComponent( pucKey1)) + { + flmAssert( pSearchKey); + ui64RowId1 = pSearchKey->getRowId(); + + // The search key better have a row ID or the untruncated + // value. + + flmAssert( ui64RowId1 || + !pSearchKey->isRightTruncated( uiKeyComponent)); + } + else + { + if (RC_BAD( rc = ixKeyGetRowId( pIndex, pucKey1, pucKeyEnd1, + uiKeyComponent, &ui64RowId1))) + { + goto Exit; + } + flmAssert( ui64RowId1); + } + + // Get the row that holds the data for the 2nd key. + + if (isSearchKeyComponent( pucKey2)) + { + flmAssert( pSearchKey); + ui64RowId2 = pSearchKey->getRowId(); + + // The search key better have a row ID or the untruncated + // value. + + flmAssert( ui64RowId2 || + !pSearchKey->isRightTruncated( uiKeyComponent)); + } + else + { + if (RC_BAD( rc = ixKeyGetRowId( pIndex, pucKey2, pucKeyEnd2, + uiKeyComponent, &ui64RowId2))) + { + goto Exit; + } + flmAssert( ui64RowId2); + } + + // If the row IDs are equal, we can skip fetching the data, because + // it will be the same. + + if (ui64RowId1 != ui64RowId2) + { + FLMBYTE ucDynaBuf1[ 64]; + FLMBYTE ucDynaBuf2[ 64]; + F_DynaBuf dynaBuf1( ucDynaBuf1, sizeof( ucDynaBuf1)); + F_DynaBuf dynaBuf2( ucDynaBuf2, sizeof( ucDynaBuf2)); + F_TABLE * pTable = pDb->getDict()->getTable( pIndex->uiTableNum); + F_COLUMN * pColumn = pDb->getDict()->getColumn( pTable, pIcd->uiColumnNum); + + // Better be binary data or text data. + + switch (pColumn->eDataTyp) + { + case SFLM_STRING_TYPE: + { + if (RC_BAD( rc = ixKeyGetUTF8( pDb, pIndex->uiTableNum, + pIcd, pRow1, ui64RowId1, + pSearchKey, uiKeyComponent, &dynaBuf1))) + { + goto Exit; + } + if (RC_BAD( rc = ixKeyGetUTF8( pDb, pIndex->uiTableNum, + pIcd, pRow2, ui64RowId2, + pSearchKey, uiKeyComponent, &dynaBuf2))) + { + goto Exit; + } + + if (RC_BAD( rc = f_compareUTF8Strings( + dynaBuf1.getBufferPtr(), + dynaBuf1.getDataLength(), + FALSE, + dynaBuf2.getBufferPtr(), + dynaBuf2.getDataLength(), + FALSE, pIcd->uiCompareRules, + pIndex->uiLanguage, piCompare))) + { + goto Exit; + } + if (*piCompare < 0) + { + *piCompare = bSortAscending ? -1 : 1; + goto Exit; + } + else if (*piCompare > 0) + { + *piCompare = bSortAscending ? 1 : -1; + goto Exit; + } + break; + } + + case SFLM_BINARY_TYPE: + { + if (RC_BAD( rc = ixKeyGetBinary( pDb, pIndex->uiTableNum, + pIcd, pRow1, ui64RowId1, + pSearchKey, uiKeyComponent, &dynaBuf1))) + { + goto Exit; + } + if (RC_BAD( rc = ixKeyGetBinary( pDb, pIndex->uiTableNum, + pIcd, pRow2, ui64RowId2, + pSearchKey, uiKeyComponent, &dynaBuf2))) + { + goto Exit; + } + + if ((*piCompare = ixKeyCompareBinary( + dynaBuf1.getBufferPtr(), + dynaBuf1.getDataLength(), + dynaBuf2.getBufferPtr(), + dynaBuf2.getDataLength(), + bSortAscending)) != 0) + { + goto Exit; + } + break; + } + + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + } + } + } + +Test_Exclusive: + + // See if either component is exclusive - everything else is + // equal up to this point. + + if (isKeyComponentLTExclusive( pucKey1)) + { + if (!isKeyComponentLTExclusive( pucKey2)) + { + *piCompare = bSortAscending ? -1 : 1; + goto Exit; + } + } + else if (isKeyComponentGTExclusive( pucKey1)) + { + if (!isKeyComponentGTExclusive( pucKey2)) + { + *piCompare = bSortAscending ? 1 : -1; + goto Exit; + } + } + else if (isKeyComponentLTExclusive( pucKey2)) + { + *piCompare = bSortAscending ? 1 : -1; + goto Exit; + } + else if (isKeyComponentGTExclusive( pucKey2)) + { + *piCompare = bSortAscending ? -1 : 1; + goto Exit; + } + + // Position to the end of this component + + pucKey1 += (2 + uiComponentLen1); + pucKey2 += (2 + uiComponentLen2); + + // If there are no more ICDs, we are done with the key + // components. + + if (uiKeyComponent < pIndex->uiNumKeyComponents) + { + break; + } + pIcd++; + uiKeyComponent++; + + // See if we are out of key components - this may be a search that + // passed in only a partial key. + + if (pucKey1 >= pucKeyEnd1) + { + *piCompare = (pucKey2 >= pucKeyEnd2) ? 0 : -1; + goto Exit; + } + else if (pucKey2 >= pucKeyEnd2) + { + *piCompare = 1; + goto Exit; + } + } + + // Compare the row ID, if being requested to. Includes comparing of the + // last byte, which is the total number of bytes in the row ID. + + if (bCompareRowId) + { + + // See if we have a row ID - this may be a search that + // passed in only a partial key and there is now ROW id on it. + + if (pucKey1 >= pucKeyEnd1) + { + *piCompare = (pucKey2 >= pucKeyEnd2) ? 0 : -1; + goto Exit; + } + else if (pucKey2 >= pucKeyEnd2) + { + *piCompare = 1; + goto Exit; + } + + // See if either one has an ID buffer of "high" + + if (*pucKey1 == 0xFF) + { + + // Key1 has a "high" set of node IDs, see what key2 has. + + *piCompare = (*pucKey2 == 0xFF) ? 0 : 1; + goto Exit; + } + else if (*pucKey2 == 0xFF) + { + // Key2 has a "high" set of node IDs, key1 does not. + + *piCompare = -1; + goto Exit; + } + else + { + FLMUINT64 ui64RowId1; + FLMUINT64 ui64RowId2; + + // Get the document ID and compare it, and only it. + // At this point, both keys should be positioned to + // get the document ID. + + if (RC_BAD( rc = f_decodeSEN64( &pucKey1, pucKeyEnd1, &ui64RowId1))) + { + goto Exit; + } + if (RC_BAD( rc = f_decodeSEN64( &pucKey2, pucKeyEnd2, &ui64RowId2))) + { + goto Exit; + } + if (ui64RowId1 == ui64RowId2) + { + *piCompare = 0; + } + else if (ui64RowId1 < ui64RowId2) + { + *piCompare = -1; + } + else + { + *piCompare = 1; + } + } + } + else + { + *piCompare = 0; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Compare function used to compare index number and key +****************************************************************************/ +FINLINE RCODE krefCompareIxAndKey( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY * pKrefA, + KREF_ENTRY * pKrefB, + FLMINT * piCompare) +{ + RCODE rc = NE_SFLM_OK; + + // Compare index numbers + + if ((*piCompare = ((FLMINT) pKrefA->ui16IxNum) - + ((FLMINT) pKrefB->ui16IxNum)) != 0) + { + goto Exit; + } + + if (!pIndex || pIndex->uiIndexNum != (FLMUINT)pKrefA->ui16IxNum) + { + pIndex = pDb->getDict()->getIndex( (FLMUINT)pKrefA->ui16IxNum); + } + + if (RC_BAD( rc = ixKeyCompare( pDb, pIndex, NULL, + pKrefA->pRow, pKrefB->pRow, TRUE, + &pKrefA [1], (FLMUINT)pKrefA->ui16KeyLen, + &pKrefB [1], (FLMUINT)pKrefB->ui16KeyLen, piCompare))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Compare function used to compare key data +****************************************************************************/ +FINLINE FLMBOOL krefIsKeyDataEqual( + KREF_ENTRY * pKrefA, + KREF_ENTRY * pKrefB) +{ + if( pKrefA->uiDataLen != pKrefB->uiDataLen) + { + return( FALSE); + } + + if( pKrefA->uiDataLen) + { + if( f_memcmp( (FLMBYTE *)(&pKrefA [1]) + + pKrefA->ui16KeyLen + 1, + (FLMBYTE *)(&pKrefB [1]) + + pKrefB->ui16KeyLen + 1, + pKrefA->uiDataLen) != 0) + { + return( FALSE); + } + } + + return( TRUE); +} + +/**************************************************************************** +Desc: Compare function used to compare two keys. +****************************************************************************/ +FINLINE RCODE krefSortCompare( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY * pKrefA, + KREF_ENTRY * pKrefB, + FLMINT * piCompare) +{ + RCODE rc = NE_SFLM_OK; + + if (RC_BAD( rc = krefCompareIxAndKey( pDb, pIndex, pKrefA, pKrefB, piCompare))) + { + goto Exit; + } + + if (*piCompare == 0) + { + *piCompare = (pKrefA->uiSequence < pKrefB->uiSequence) ? -1 : 1; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Checks if the current database has any UNIQUE indexes that need + to checked. Also does duplicate processing for the record. +****************************************************************************/ +RCODE F_Db::processDupKeys( + F_INDEX * pIndex) +{ + RCODE rc = NE_SFLM_OK; + + // Sort and remove duplicates + + if (m_uiKrefCount > 1) + { + if (RC_BAD( rc = krefQuickSort( this, pIndex, m_pKrefTbl, + 0, m_uiKrefCount - 1))) + { + goto Exit; + } + if (RC_BAD( rc = krefKillDups( this, pIndex, + m_pKrefTbl, &m_uiKrefCount))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Commit (write out) all keys that have built up in the KREF table. +****************************************************************************/ +RCODE F_Db::keysCommit( + FLMBOOL bCommittingTrans, + FLMBOOL bSortKeys) +{ + RCODE rc = NE_SFLM_OK; + + // If the Kref has not been initialized, there is no + // work to do. + + if (m_bKrefSetup) + { + F_INDEX * pIndex = NULL; + FLMUINT uiTotal = m_uiKrefCount; + KREF_ENTRY * pKref; + KREF_ENTRY ** pKrefTbl = m_pKrefTbl; + FLMUINT uiKrefNum; + FLMUINT uiLastIxNum; + + // We should not have reached this point if bAbortTrans is TRUE + + if( RC_BAD( m_AbortRc)) + { + rc = RC_SET_AND_ASSERT( m_AbortRc); + goto Exit; + } + + // Sort the KREF table, if it contains more than one key. + // This will sort all keys from the same index the same. + + if (uiTotal > 1 && bSortKeys) + { + processDupKeys( NULL); + uiTotal = m_uiKrefCount; + } + + // Loop through the KREF table outputting all keys + + uiLastIxNum = 0; + for (uiKrefNum = 0; uiKrefNum < uiTotal; uiKrefNum++) + { + pKref = pKrefTbl [uiKrefNum]; + + // See if the LFILE changed + + flmAssert( pKref->ui16IxNum); + + if (pKref->ui16IxNum != uiLastIxNum) + { + uiLastIxNum = pKref->ui16IxNum; + pIndex = m_pDict->getIndex( uiLastIxNum); + } + + // Flush the key to the index + if (m_pKeyColl) + { + m_pKeyColl->addKey( this, pIndex, pKref); + } + else + { + if (RC_BAD(rc = refUpdate( pIndex, pKref, TRUE))) + { + if (rc != NE_SFLM_NOT_UNIQUE) + { + RC_UNEXPECTED_ASSERT( rc); + } + goto Exit; + } + } + } + + if (bCommittingTrans) + { + krefCntrlFree(); + } + else + { + // Empty the table out so we can add more keys in this trans. + + m_pKrefPool->poolReset( NULL, TRUE); + m_uiKrefCount = 0; + m_uiTotalKrefBytes = 0; + } + } + +Exit: + + if( RC_BAD( rc)) + { + setMustAbortTrans( rc); + } + + return( rc); +} + +/*************************************************************************** +Desc: Quick sort an array of KREF_ENTRY * values. +****************************************************************************/ +FSTATIC RCODE krefQuickSort( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY ** pEntryTbl, + FLMUINT uiLowerBounds, + FLMUINT uiUpperBounds) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLBPos; + FLMUINT uiUBPos; + FLMUINT uiMIDPos; + FLMUINT uiLeftItems; + FLMUINT uiRightItems; + KREF_ENTRY * pCurEntry; + KREF_ENTRY * pTempKref; + FLMINT iCompare; + +Iterate_Larger_Half: + + uiUBPos = uiUpperBounds; + uiLBPos = uiLowerBounds; + uiMIDPos = (uiUpperBounds + uiLowerBounds + 1) / 2; + pCurEntry = pEntryTbl[ uiMIDPos ]; + for( ;;) + { + for (;;) + { + if (uiLBPos != uiMIDPos) + { + if (RC_BAD( rc = krefSortCompare( pDb, pIndex, + pEntryTbl[ uiLBPos], pCurEntry, &iCompare))) + { + goto Exit; + } + if (iCompare >= 0) + { + break; + } + } + if (uiLBPos >= uiUpperBounds) + { + break; + } + uiLBPos++; + } + + for (;;) + { + if (uiUBPos != uiMIDPos) + { + if (RC_BAD( rc = krefSortCompare( pDb, pIndex, pCurEntry, + pEntryTbl[ uiUBPos], &iCompare))) + { + goto Exit; + } + if (iCompare >= 0) + { + break; + } + } + if (!uiUBPos) + { + break; + } + uiUBPos--; + } + + if (uiLBPos < uiUBPos) // Interchange and continue loop. + { + + // Interchange [uiLBPos] with [uiUBPos]. + + KY_SWAP( pEntryTbl, uiLBPos, uiUBPos ); + uiLBPos++; // Scan from left to right. + uiUBPos--; // Scan from right to left. + } + else // Past each other - done + { + break; + } + } + + // Check for swap( LB, MID ) - cases 3 and 4 + + if (uiLBPos < uiMIDPos) + { + + // Interchange [uiLBPos] with [uiMIDPos] + + KY_SWAP( pEntryTbl, uiMIDPos, uiLBPos ); + uiMIDPos = uiLBPos; + } + else if (uiMIDPos < uiUBPos) + { + + // Interchange [uUBPos] with [uiMIDPos] + + KY_SWAP( pEntryTbl, uiMIDPos, uiUBPos ); + uiMIDPos = uiUBPos; + } + + // Check the left piece. + + uiLeftItems = (uiLowerBounds + 1 < uiMIDPos ) + ? uiMIDPos - uiLowerBounds // 2 or more + : 0; + uiRightItems = (uiMIDPos + 1 < uiUpperBounds ) + ? uiUpperBounds - uiMIDPos // 2 or more + : 0; + + if (uiLeftItems < uiRightItems) + { + + // Recurse on the LEFT side and goto the top on the RIGHT side. + + if (uiLeftItems) + { + if (RC_BAD( rc = krefQuickSort( pDb, pIndex, pEntryTbl, + uiLowerBounds, uiMIDPos - 1))) + { + goto Exit; + } + } + uiLowerBounds = uiMIDPos + 1; + goto Iterate_Larger_Half; + } + else if (uiLeftItems) // Compute a truth table to figure out this check. + { + + // Recurse on the RIGHT side and goto the top for the LEFT side. + + if (uiRightItems) + { + if (RC_BAD( rc = krefQuickSort( pDb, pIndex, pEntryTbl, + uiMIDPos + 1, uiUpperBounds))) + { + goto Exit; + } + } + uiUpperBounds = uiMIDPos - 1; + goto Iterate_Larger_Half; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Kill all duplicate keys in the KREF table. +****************************************************************************/ +FSTATIC RCODE krefKillDups( + F_Db * pDb, + F_INDEX * pIndex, + KREF_ENTRY ** pKrefTbl, + FLMUINT * puiKrefTotal) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCurKref = 0; + FLMUINT uiLastKref = *puiKrefTotal; + FLMUINT uiFirstForKey; + FLMUINT uiLastForKey; + FLMUINT uiNewPosOffset = 0; + FLMINT iCompare; + + while( uiCurKref < uiLastKref) + { + uiFirstForKey = uiLastForKey = uiCurKref; + uiCurKref = uiFirstForKey + 1; + + while( uiCurKref < uiLastKref) + { + if (RC_BAD( rc = krefCompareIxAndKey( pDb, pIndex, pKrefTbl[ uiFirstForKey], + pKrefTbl[ uiCurKref], &iCompare))) + { + goto Exit; + } + + if (iCompare) + { + break; + } + + uiLastForKey = uiCurKref++; + } + + if( uiFirstForKey == uiLastForKey) + { + pKrefTbl[ uiNewPosOffset++] = pKrefTbl[ uiFirstForKey]; + continue; + } + + if( pKrefTbl[ uiFirstForKey]->bDelete) + { + if( pKrefTbl[ uiLastForKey]->bDelete) + { + pKrefTbl[ uiNewPosOffset++] = pKrefTbl[ uiFirstForKey]; + } + else + { +TestCancel: + // See if the operations cancel each other. If they don't, we + // need to keep both operations + + if( !krefIsKeyDataEqual( pKrefTbl[ uiFirstForKey], pKrefTbl[ uiLastForKey])) + { + pKrefTbl[ uiNewPosOffset++] = pKrefTbl[ uiFirstForKey]; + pKrefTbl[ uiNewPosOffset++] = pKrefTbl[ uiLastForKey]; + } + } + } + else + { + if( pKrefTbl[ uiLastForKey]->bDelete) + { + goto TestCancel; + } + else + { + pKrefTbl[ uiNewPosOffset++] = pKrefTbl[ uiLastForKey]; + } + } + } + + *puiKrefTotal = uiNewPosOffset; + +Exit: + + return( rc); +} + diff --git a/sql/src/kyunlock.cpp b/sql/src/kyunlock.cpp new file mode 100644 index 0000000..3a8da17 --- /dev/null +++ b/sql/src/kyunlock.cpp @@ -0,0 +1,139 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains the routines to initialize and set up +// structures for indexing. +// +// Tabs: 3 +// +// Copyright (c) 1992-2000, 2002-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: kyunlock.cpp 3115 2006-01-19 13:24:39 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: Setup routine for the KREF_CNTRL structure for record updates. +****************************************************************************/ +RCODE F_Db::krefCntrlCheck( void) +{ + RCODE rc = NE_SFLM_OK; + + // Check if we need to flush keys between updates, but not during the + // processing of an update. + + if( m_bKrefSetup) + { + if (isKrefOverThreshold()) + { + if (RC_BAD( rc = keysCommit( FALSE))) + { + goto Exit; + } + } + } + else + { + m_uiKrefCount = 0; + m_uiTotalKrefBytes = 0; + m_pKrefPool = NULL; + m_bReuseKrefPool = FALSE; + m_bKrefCompoundKey = FALSE; + m_pKrefReset = NULL; + m_bKrefSetup = TRUE; + + if (m_eTransType == SFLM_UPDATE_TRANS) + { + m_pKrefPool = &m_pDatabase->m_krefPool; + + m_bReuseKrefPool = TRUE; + m_pKrefPool->poolReset( NULL, TRUE); + } + else + { + m_tmpKrefPool.poolFree(); + m_tmpKrefPool.poolInit( DEFAULT_KREF_POOL_BLOCK_SIZE); + m_pKrefPool = &m_tmpKrefPool; + + m_bReuseKrefPool = FALSE; + } + + if( !m_pKrefTbl) + { + if( RC_BAD( rc = f_alloc( + DEFAULT_KREF_TBL_SIZE * sizeof( KREF_ENTRY *), &m_pKrefTbl))) + { + goto Exit; + } + + m_uiKrefTblSize = DEFAULT_KREF_TBL_SIZE; + } + + if( !m_pucKrefKeyBuf) + { + if (RC_BAD( rc = f_alloc( SFLM_MAX_KEY_SIZE, &m_pucKrefKeyBuf))) + { + goto Exit; + } + } + } + + m_pKrefReset = m_pKrefPool->poolMark(); + flmAssert( m_pucKrefKeyBuf); + +Exit: + + if (RC_BAD( rc)) + { + krefCntrlFree(); + } + + return( rc); +} + +/**************************************************************************** +Desc: Frees the memory associated with the KREF +****************************************************************************/ +void F_Db::krefCntrlFree( void) +{ + if( m_bKrefSetup) + { + if( m_bReuseKrefPool) + { + m_pKrefPool->poolReset( NULL, TRUE); + } + else + { + m_pKrefPool->poolFree(); + m_pKrefPool->poolInit( DEFAULT_KREF_POOL_BLOCK_SIZE); + } + + m_pKrefPool = NULL; + if( m_pKrefTbl && m_uiKrefTblSize != DEFAULT_KREF_TBL_SIZE) + { + f_free( &m_pKrefTbl); + m_uiKrefTblSize = 0; + } + + m_uiKrefCount = 0; + m_uiTotalKrefBytes = 0; + m_bReuseKrefPool = FALSE; + m_bKrefCompoundKey = FALSE; + m_pKrefReset = NULL; + m_bKrefSetup = FALSE; + } +} diff --git a/sql/src/recover.cpp b/sql/src/recover.cpp new file mode 100644 index 0000000..691c2d9 --- /dev/null +++ b/sql/src/recover.cpp @@ -0,0 +1,469 @@ +//------------------------------------------------------------------------------ +// Desc: Contains routines for recovering a database after +// a failure. +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: recover.cpp 3116 2006-01-19 13:31:53 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +/**************************************************************************** +Desc: This routine reads the next before-image block from the database. +****************************************************************************/ +RCODE F_Db::readRollbackLog( + FLMUINT uiLogEOF, // Address of end of rollback log. + FLMUINT * puiCurrAddr, // This is the current address we are + // reading in the log file. It + // will be updated after reading the + // data. + F_BLK_HDR * pBlkHdr, // This is the buffer that is to hold + // the data that is read from the + // log file. + FLMBOOL * pbIsBeforeImageBlk// Is block a before-image block? + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFilePos; + FLMUINT uiBlkSize = m_pDatabase->m_uiBlockSize; + FLMUINT uiBytesRead; + F_TMSTAMP StartTime; + + uiFilePos = *puiCurrAddr; + + // Verify that we are not going to read beyond the log EOF + + if (!FSAddrIsAtOrBelow( uiFilePos + uiBlkSize, uiLogEOF)) + { + rc = RC_SET( NE_SFLM_INCOMPLETE_LOG); + goto Exit; + } + + // Position to the appropriate place and read the data + + if (m_pDbStats) + { + m_pDbStats->bHaveStats = TRUE; + m_pDbStats->LogBlockReads.ui64Count++; + m_pDbStats->LogBlockReads.ui64TotalBytes += uiBlkSize; + f_timeGetTimeStamp( &StartTime); + } + + if (RC_BAD( rc = m_pSFileHdl->readBlock( uiFilePos, + uiBlkSize, (FLMBYTE *)pBlkHdr, &uiBytesRead))) + { + if (rc == NE_FLM_IO_END_OF_FILE) + { + rc = RC_SET( NE_SFLM_INCOMPLETE_LOG); + } + + if (m_pDbStats) + { + m_pDbStats->uiReadErrors++; + } + goto Exit; + } + + if (m_pDbStats) + { + flmAddElapTime( &StartTime, &m_pDbStats->LogBlockReads.ui64ElapMilli); + } + + if (uiBytesRead != uiBlkSize) + { + if (m_pDbStats) + { + m_pDbStats->uiLogBlockChkErrs++; + } + + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + if (RC_BAD( rc = flmPrepareBlockForUse( uiBlkSize, pBlkHdr))) + { + if (m_pDbStats && rc == NE_SFLM_BLOCK_CRC) + { + m_pDbStats->uiLogBlockChkErrs++; + } + goto Exit; + } + + // See if before image bit is set. Need to unset it if it is. + + *pbIsBeforeImageBlk = (FLMBOOL)((pBlkHdr->ui8BlkFlags & + BLK_IS_BEFORE_IMAGE) + ? (FLMBOOL)TRUE + : (FLMBOOL)FALSE); + pBlkHdr->ui8BlkFlags &= ~(BLK_IS_BEFORE_IMAGE); + + // Adjust the current address for the next read + + uiFilePos += uiBlkSize; + if (FSGetFileOffset( uiFilePos) >= m_pDatabase->m_uiMaxFileSize) + { + FLMUINT uiFileNumber = FSGetFileNumber( uiFilePos); + + if (!uiFileNumber) + { + uiFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + } + else + { + uiFileNumber++; + } + + if (uiFileNumber > MAX_LOG_BLOCK_FILE_NUMBER) + { + rc = RC_SET( NE_SFLM_DB_FULL); + goto Exit; + } + uiFilePos = FSBlkAddress( uiFileNumber, 0 ); + } + + *puiCurrAddr = uiFilePos; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine reads and processes a before-image block record + in the log file. The reapply flag indicates whether the + block should be written back to the database file. +****************************************************************************/ +RCODE F_Db::processBeforeImage( + FLMUINT uiLogEOF, // Address of the end of the rollback + // log. + FLMUINT * puiCurrAddrRV, // This is the current offset we are + // reading in the log file. + // It will be updated after reading the + // data. + F_BLK_HDR * pBlkHdr, // This is a pointer to a buffer that + // will be used to hold the block that + // is read. + FLMBOOL bDoingRecovery, // Are we doing a recovery as opposed to + // rolling back a transaction? + FLMUINT64 ui64MaxTransID // Maximum transaction ID to recover to when + // bDoingRecovery is TRUE. This parameter + // is ignored when bDoingRecover is FALSE. + + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkAddress; + FLMUINT uiBlkLength; +#ifdef FLM_DBG_LOG + FLMUINT64 ui64TransID; +#endif + FLMUINT uiBytesWritten; + FLMBOOL bIsBeforeImageBlk; + F_TMSTAMP StartTime; + + // Read the block from the log + + if (RC_BAD( rc = readRollbackLog( uiLogEOF, puiCurrAddrRV, pBlkHdr, + &bIsBeforeImageBlk))) + { + goto Exit; + } + + // Determine if we want to restore the block. + // If we are doing a recovery, restore the block only if + // its checkpoint is <= ui64MaxTransID. If we are + // rolling back a transaction, restore the block only if + // it is marked as a before-image block. + + // For the recovery process, multiple versions + // of the same block may be restored if there are + // multiple versions in the log. However, because + // the versions will be in ascending order in the + // file, ultimately, the one with the highest + // checkpoint that does not exceed ui64MaxTransID + // will be restored - which is precisely the one + // we want to be restored for a recovery. + + // For a transaction rollback, it is impossible for us + // to see more than one version of a block that is + // marked as the before-image version, because we + // started from a point in the log where the last + // update transaction logged its first block. All + // blocks after that point that have the BI bits + // set should be restored. Any that do not have + // the BI bit set should NOT be restored. + + if (bDoingRecovery) + { + if (pBlkHdr->ui64TransID > ui64MaxTransID) + { + goto Exit; + } + } + else if (!bIsBeforeImageBlk) + { + goto Exit; + } + + // Determine the block address before setting the checksum. + + uiBlkAddress = (FLMUINT)pBlkHdr->ui32BlkAddr; + uiBlkLength = blkGetEnd( m_pDatabase->m_uiBlockSize, blkHdrSize( pBlkHdr), + pBlkHdr); +#ifdef FLM_DBG_LOG + ui64TransID = pBlkHdr->ui64TransID; +#endif + + if (RC_BAD( rc = flmPrepareBlockToWrite( m_pDatabase->m_uiBlockSize, + pBlkHdr))) + { + goto Exit; + } + + if (m_pDbStats) + { + m_pDbStats->bHaveStats = TRUE; + m_pDbStats->LogBlockRestores.ui64Count++; + m_pDbStats->LogBlockRestores.ui64TotalBytes += uiBlkLength; + f_timeGetTimeStamp( &StartTime); + } + + m_pSFileHdl->setMaxAutoExtendSize( m_pDatabase->m_uiMaxFileSize); + m_pSFileHdl->setExtendSize( m_pDatabase->m_uiFileExtendSize); + rc = m_pSFileHdl->writeBlock( uiBlkAddress, uiBlkLength, pBlkHdr, + m_pDatabase->m_uiBlockSize, NULL, &uiBytesWritten); +#ifdef FLM_DBG_LOG + flmDbgLogWrite( m_pDatabase, uiBlkAddress, 0, ui64TransID, + "ROLLBACK"); +#endif + + if (m_pDbStats) + { + flmAddElapTime( &StartTime, &m_pDbStats->LogBlockRestores.ui64ElapMilli); + if (RC_BAD( rc)) + { + m_pDbStats->uiWriteErrors++; + } + } + +Exit: + + return( rc); +} + +/*************************************************************************** +Desc: Writes the log header to disk. The checksum is calculated before + writing the log header to disk. +*****************************************************************************/ +RCODE F_Database::writeDbHdr( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + SFLM_DB_HDR * pDbHdr, // DB header to be written out. + SFLM_DB_HDR * pCPDbHdr, // DB header as it was at the time + // of the checkpoint. + FLMBOOL bIsCheckpoint // Are we writing a checkpoint? If we + // we are, we may write the DB header + // as is. Otherwise, we need to make + // sure we don't write out certain + // parts of the DB header - they must + // not be updated on disk until a + // checkpoint actually occurs. + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesWritten; + IF_FileHdl * pCFileHdl = NULL; + SFLM_DB_HDR * pTmpDbHdr; + F_TMSTAMP StartTime; + + // Force any recent writes to disk before modifying the DB + // header. This routine is generally called after having + // written out data blocks or rollback blocks. It is + // critial that any previous writes be flushed before the + // header is updated because the header will generally have + // been modified to point to the new things that were added. + + if (RC_BAD( rc = pSFileHdl->flush())) + { + goto Exit; + } + + // No need to ever actually write the header to disk if this is + // a temporary database + + if (m_bTempDb) + { + goto Exit; + } + + pTmpDbHdr = m_pDbHdrWriteBuf; + + uiBytesWritten = sizeof( SFLM_DB_HDR); + f_memcpy( pTmpDbHdr, pDbHdr, sizeof( SFLM_DB_HDR)); + + // If we are not doing a checkpoint, we don't really want + // to write out certain items, so we restore them from + // the database header as it was at the time of the last + // checkpoint. + + if (!bIsCheckpoint && pCPDbHdr) + { + pTmpDbHdr->ui32RflLastCPFileNum = pCPDbHdr->ui32RflLastCPFileNum; + pTmpDbHdr->ui32RflLastCPOffset = pCPDbHdr->ui32RflLastCPOffset; + pTmpDbHdr->ui64CurrTransID = pCPDbHdr->ui64CurrTransID; + pTmpDbHdr->ui64TransCommitCnt = pCPDbHdr->ui64TransCommitCnt; + pTmpDbHdr->ui32FirstAvailBlkAddr = pCPDbHdr->ui32FirstAvailBlkAddr; + pTmpDbHdr->ui32LogicalEOF = pCPDbHdr->ui32LogicalEOF; + pTmpDbHdr->ui32BlksChangedSinceBackup = + pCPDbHdr->ui32BlksChangedSinceBackup; + pTmpDbHdr->ui64LastRflCommitID = pCPDbHdr->ui64LastRflCommitID; + } + + // Header is always written out in native format. Set the CRC + + flmAssert( !hdrIsNonNativeFormat( pTmpDbHdr)); + + pTmpDbHdr->ui32HdrCRC = calcDbHdrCRC( pTmpDbHdr); + + // Now update the log header record on disk. + + if (pDbStats) + { + pDbStats->bHaveStats = TRUE; + pDbStats->DbHdrWrites.ui64Count++; + pDbStats->DbHdrWrites.ui64TotalBytes += + uiBytesWritten; + f_timeGetTimeStamp( &StartTime); + } + + if( RC_BAD( rc = pSFileHdl->getFileHdl( 0, TRUE, &pCFileHdl))) + { + goto Exit; + } + + if( RC_BAD( rc = pCFileHdl->sectorWrite( 0, + uiBytesWritten, pTmpDbHdr, + pCFileHdl->getSectorSize(), + NULL, &uiBytesWritten, FALSE))) + { + if (pDbStats) + { + pDbStats->uiWriteErrors++; + } + + goto Exit; + } + + if (pDbStats) + { + flmAddElapTime( &StartTime, &pDbStats->DbHdrWrites.ui64ElapMilli); + } + + // Finally, force the header to disk. + + if (RC_BAD( rc = pCFileHdl->flush())) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine recovers the database to a physically consistent + state. +Ret: NE_SFLM_OK - Indicates the database has been recovered. + other - other FLAIM error codes +****************************************************************************/ +RCODE F_Db::physRollback( + FLMUINT uiLogEOF, + FLMUINT uiFirstLogBlkAddr, // Address of first log block + FLMBOOL bDoingRecovery, // Doing recovery? If so, we will + // ignore blocks whose transaction + // ID is higher than ui64MaxTransID. + // Also, we will not check the BI + // bits in the logged blocks, because + // we are not rolling back a + // transaction. + FLMUINT64 ui64MaxTransID // Ignored when bDoingRecovery is + // FALSE + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCurrAddr; + FLMBYTE * pucBlk = NULL; + + // If the log is empty, no need to do anything. + // A uiFirstLogBlkAddr of zero indicates that there + // is nothing in the log to rollback. This will be true + // if we are rolling back a transaction, and the transaction + // has not logged anything or if we are doing a recovery and + // nothing was logged since the last checkpoint. + + if (uiLogEOF == m_pDatabase->m_uiBlockSize || !uiFirstLogBlkAddr) + { + goto Exit; // Will return NE_SFLM_OK + } + + // Allocate a buffer to be used for reading. + + if( RC_BAD( rc = f_allocAlignedBuffer( m_pDatabase->m_uiBlockSize, + (void **)&pucBlk))) + { + goto Exit; + } + + // Start from beginning of log and read to EOF restoring before-image + // blocks along the way. + + uiCurrAddr = uiFirstLogBlkAddr; + m_pSFileHdl->enableFlushMinimize(); + while (FSAddrIsBelow( uiCurrAddr, uiLogEOF)) + { + if (RC_BAD( rc = processBeforeImage( uiLogEOF, &uiCurrAddr, + (F_BLK_HDR *)pucBlk, bDoingRecovery, + ui64MaxTransID))) + { + goto Exit; + } + } + + // Force the writes to the file. + + if (RC_BAD( rc = m_pSFileHdl->flush())) + { + goto Exit; + } + +Exit: + m_pSFileHdl->disableFlushMinimize(); + + // Free the memory handle, if one was allocated. + + if (pucBlk) + { + f_freeAlignedBuffer( (void **)&pucBlk); + } + + return( rc); +} diff --git a/sql/src/rfl.cpp b/sql/src/rfl.cpp new file mode 100644 index 0000000..9c4b4bb --- /dev/null +++ b/sql/src/rfl.cpp @@ -0,0 +1,5537 @@ +// Desc: This module contains routine for roll forward logging. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: rfl.cpp 3116 2006-01-19 13:31:53 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define MOD_512( uiNum) (FLMUINT)((uiNum) & 511) +#define ON_512_BYTE_BOUNDARY( uiNum) (!MOD_512(uiNum)) +#define ROUND_DOWN_TO_NEAREST_512( uiNum) \ + (FLMUINT)((uiNum) & (~((FLMUINT)511))) + +/******************************************************************** +Desc: +*********************************************************************/ +FINLINE FLMBOOL F_Rfl::useDataOnlyBlocks( + F_Db * pDb, + FLMUINT uiDataLen) +{ + if( uiDataLen > (pDb->m_pDatabase->m_uiBlockSize * 8) / 5) + { + return( TRUE); + } + else + { + return( FALSE); + } +} + +/******************************************************************** +Desc: +*********************************************************************/ +class F_RflOStream : public IF_OStream +{ +public: + + F_RflOStream( + F_Rfl * pRfl, + F_Db * pDb) + { + m_pRfl = pRfl; + m_pRfl->AddRef(); + m_pDb = pDb; + } + + virtual ~F_RflOStream() + { + if( m_pRfl) + { + m_pRfl->Release(); + } + } + + RCODE write( + const void * pvBuffer, + FLMUINT uiBytesToWrite, + FLMUINT * puiBytesWritten = NULL); + + RCODE write( + IF_PosIStream * pIStream); + + FINLINE RCODE close( void) + { + if( m_pRfl) + { + m_pRfl->Release(); + m_pRfl = NULL; + } + + return( NE_SFLM_OK); + } + +private: + + F_Rfl * m_pRfl; + F_Db * m_pDb; +}; + +/******************************************************************** +Desc: +*********************************************************************/ +F_Rfl::F_Rfl() +{ + m_pDatabase = NULL; + m_hBufMutex = F_MUTEX_NULL; + m_pCommitBuf = NULL; + m_pCurrentBuf = NULL; + m_uiRflWriteBufs = DEFAULT_RFL_WRITE_BUFFERS; + m_uiBufferSize = DEFAULT_RFL_BUFFER_SIZE; + f_memset( &m_Buf1, 0, sizeof( m_Buf1)); + f_memset( &m_Buf2, 0, sizeof( m_Buf2)); + m_bKeepRflFiles = FALSE; + m_uiRflMinFileSize = SFLM_DEFAULT_MIN_RFL_FILE_SIZE; + m_uiRflMaxFileSize = SFLM_DEFAULT_MAX_RFL_FILE_SIZE; + m_pFileHdl = NULL; + m_uiLastRecoverFileNum = 0; + f_memset( m_ucCurrSerialNum, 0, sizeof( m_ucCurrSerialNum)); + m_uiTransStartFile = 0; + m_uiTransStartAddr = 0; + m_ui64CurrTransID = 0; + m_ui64LastTransID = 0; + m_ui64LastLoggedCommitTransID = 0; + m_uiOperCount = 0; + m_uiRflReadOffset = 0; + m_uiFileEOF = 0; + m_pRestore = NULL; + m_pRestoreStatus = NULL; + f_memset( m_szRflDir, 0, sizeof( m_szRflDir)); + m_bRflDirSameAsDb = FALSE; + m_bCreateRflDir = FALSE; + f_memset( m_ucNextSerialNum, 0, sizeof( m_ucNextSerialNum)); + m_bRflVolumeOk = TRUE; + m_bRflVolumeFull = FALSE; + m_uiLastLfNum = 0; + m_eLastLfType = SFLM_LF_INVALID; + m_pIxCompareObject = NULL; + m_pCompareObject = NULL; + m_uiDisabledCount = 0; +} + +/******************************************************************** +Desc: Destructor +*********************************************************************/ +F_Rfl::~F_Rfl() +{ + if (m_Buf1.pIOBuffer) + { + m_Buf1.pIOBuffer->Release(); + m_Buf1.pIOBuffer = NULL; + } + if (m_Buf2.pIOBuffer) + { + m_Buf2.pIOBuffer->Release(); + m_Buf2.pIOBuffer = NULL; + } + + if( m_Buf1.pBufferMgr) + { + flmAssert( !m_Buf1.pBufferMgr->havePendingIO() && + !m_Buf1.pBufferMgr->haveUsed()); + m_Buf1.pBufferMgr->Release(); + m_Buf1.pBufferMgr = NULL; + } + + if( m_Buf2.pBufferMgr) + { + flmAssert( !m_Buf2.pBufferMgr->havePendingIO() && + !m_Buf2.pBufferMgr->haveUsed()); + m_Buf2.pBufferMgr->Release(); + m_Buf2.pBufferMgr = NULL; + } + + if (m_hBufMutex != F_MUTEX_NULL) + { + f_mutexDestroy( &m_hBufMutex); + } + + if (m_pFileHdl) + { + m_pFileHdl->close(); + m_pFileHdl->Release(); + m_pFileHdl = NULL; + m_pDatabase = NULL; + } + + if (m_pIxCompareObject) + { + m_pIxCompareObject->Release(); + } +} + +/******************************************************************** +Desc: Returns a boolean indicating whether or not we are at + the end of the RFL log - will only be TRUE when we are + doing recovery. +*********************************************************************/ +FLMBOOL F_Rfl::atEndOfLog( void) +{ + return( (!m_pRestore && + m_uiFileEOF && + m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes >= m_uiFileEOF && + m_uiRflReadOffset == m_pCurrentBuf->uiRflBufBytes && + m_pCurrentBuf->uiCurrFileNum == m_uiLastRecoverFileNum) + ? TRUE + : FALSE); +} + +/******************************************************************** +Desc: Gets the base RFL file name - does not have directory part. + This needs to be separate from the F_Rfl object so it can + be called without having to instantiate and set up an F_Rfl + object. +*********************************************************************/ +void rflGetBaseFileName( + FLMUINT uiFileNum, + char * pszBaseNameOut, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated) +{ + FLMUINT uiCnt; + FLMUINT uiDigit; + char szTmpBuf [14]; + char * pszTmp = &szTmpBuf [0]; + + // Output as eight digit hex number + + uiCnt = 0; + pszTmp += 7; + while (uiCnt < 8) + { + uiDigit = (FLMUINT)(uiFileNum & 0xF); + uiFileNum >>= 4; + if (uiDigit <= 9) + { + uiDigit += NATIVE_ZERO; + } + else + { + uiDigit += (NATIVE_LOWER_A - 10); + } + *pszTmp = (FLMBYTE)uiDigit; + pszTmp--; + uiCnt++; + } + + // Skip to end of digits and append ".log" to name + + f_strcpy( pszTmp + 9, ".log"); + if (*puiFileNameBufSize >= 13) + { + *puiFileNameBufSize = 12; + f_strcpy( pszBaseNameOut, szTmpBuf); + if (pbNameTruncated) + { + *pbNameTruncated = FALSE; + } + } + else + { + flmAssert( *puiFileNameBufSize); + (*puiFileNameBufSize)--; + if (*puiFileNameBufSize) + { + f_memcpy( pszBaseNameOut, szTmpBuf, *puiFileNameBufSize); + } + pszBaseNameOut [*puiFileNameBufSize] = 0; + if (pbNameTruncated) + { + *pbNameTruncated = TRUE; + } + } +} + +/******************************************************************** +Desc: Generates the full roll forward log file name. +*********************************************************************/ +void F_Rfl::getFullRflFileName( + FLMUINT uiFileNum, + char * pszRflFileName, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated) +{ + FLMUINT uiBaseNameSize; + FLMUINT uiLen = f_strlen( m_szRflDir); + FLMBOOL bNameTruncated = FALSE; + + // Must at least be room for a null byte to terminate the string. + + flmAssert( *puiFileNameBufSize); + if (uiLen > *puiFileNameBufSize - 1) + { + uiLen = *puiFileNameBufSize - 1; + if (uiLen) + { + f_memcpy( pszRflFileName, m_szRflDir, uiLen); + } + bNameTruncated = TRUE; + goto Exit; + } + + // Get the directory name. + + if (uiLen) + { + f_memcpy( pszRflFileName, m_szRflDir, uiLen); + + // See if we need to append a slash. + +#ifdef FLM_UNIX + if (m_szRflDir [uiLen - 1] != '/') +#else + if (m_szRflDir [uiLen - 1] != '/' && + m_szRflDir [uiLen - 1] != '\\') +#endif + { + + // See if we have room for one more character, plus null + + if (uiLen == *puiFileNameBufSize - 1) + { + bNameTruncated = TRUE; + goto Exit; + } +#ifdef FLM_UNIX + pszRflFileName [uiLen] = '/'; +#else + pszRflFileName [uiLen] = '\\'; +#endif + uiLen++; + } + } + + // See if there is room for at least one more byte plus a + // null byte. + + if (uiLen == *puiFileNameBufSize - 1) + { + bNameTruncated = TRUE; + goto Exit; + } + + // Get the base RFL file name. + + uiBaseNameSize = *puiFileNameBufSize - uiLen; + rflGetBaseFileName( uiFileNum, pszRflFileName + uiLen, + &uiBaseNameSize, &bNameTruncated); + uiLen += uiBaseNameSize; + +Exit: + + pszRflFileName [uiLen] = 0; + *puiFileNameBufSize = uiLen; + if (pbNameTruncated) + { + *pbNameTruncated = bNameTruncated; + } +} + +/******************************************************************** +Desc: Positions to the offset specified in the RFL file. +*********************************************************************/ +RCODE F_Rfl::positionTo( + FLMUINT uiFileOffset) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesToRead; + FLMUINT uiBytesRead; + + // Should never be attempting to position to something less + // than 512 - the header is stored in the first 512 bytes. + + flmAssert( uiFileOffset >= 512); + + // If the position is within our current buffer, see if we + // can adjust things without having to go back and re-read + // the buffer from disk. + + if (m_pCurrentBuf->uiRflBufBytes && + uiFileOffset >= m_pCurrentBuf->uiRflFileOffset && + uiFileOffset <= m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes) + { + + // Whatever is in the buffer beyond uiFileOffset is irrelevant + // and can be discarded. + + m_pCurrentBuf->uiRflBufBytes = uiFileOffset - + m_pCurrentBuf->uiRflFileOffset; + } + else + { + + // Populate the buffer from the 512 byte boundary that is just + // before the offset we are trying to position to. + + uiBytesToRead = MOD_512( uiFileOffset); + m_pCurrentBuf->uiRflFileOffset = ROUND_DOWN_TO_NEAREST_512( uiFileOffset); + m_pCurrentBuf->uiRflBufBytes = MOD_512( uiFileOffset); + if (m_pCurrentBuf->uiRflBufBytes) + { + if (RC_BAD( rc = m_pFileHdl->sectorRead( + m_pCurrentBuf->uiRflFileOffset, m_pCurrentBuf->uiRflBufBytes, + m_pCurrentBuf->pIOBuffer->getBuffer(), &uiBytesRead))) + { + if (rc == NE_FLM_IO_END_OF_FILE) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + } + else + { + m_bRflVolumeOk = FALSE; + } + goto Exit; + } + else if (uiBytesRead < m_pCurrentBuf->uiRflBufBytes) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Get the ACTUAL RFL directory, using as input parameters the + database version, the name of the database, and the + user specified RFL directory. Also return the database + prefix. +*********************************************************************/ +RCODE rflGetDirAndPrefix( + const char * pszDbFileName, + const char * pszRflDirIn, + char * pszRflDirOut) +{ + RCODE rc = NE_SFLM_OK; + char szDbPath [F_PATH_MAX_SIZE]; + char szBaseName [F_FILENAME_SIZE]; + char szPrefix [F_FILENAME_SIZE]; + F_DbSystem dbSystem; + + // Parse the database name into directory and base name + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathReduce( pszDbFileName, + szDbPath, szBaseName))) + { + goto Exit; + } + + // Get the base path + + F_DbSystem::getDbBasePath( szPrefix, szBaseName, NULL); + + // Determine the RFL directory. If one was + // specified, it is whatever was specified. + // Otherwise, it is relative to the database + // directory. + + if (pszRflDirIn && *pszRflDirIn) + { + f_strcpy( pszRflDirOut, pszRflDirIn); + } + else + { + f_strcpy( pszRflDirOut, szDbPath); + } + + f_strcpy( szBaseName, szPrefix); + f_strcat( szBaseName, ".rfl"); + gv_SFlmSysData.pFileSystem->pathAppend( pszRflDirOut, szBaseName); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Set the RFL directory. If pszRflDir is NULL or empty string, + the RFL directory is set to the same directory as the + database. +*********************************************************************/ +RCODE F_Rfl::setRflDir( + const char * pszRflDir) +{ + // Better have set up the F_Database pointer. + + flmAssert( m_pDatabase != NULL); + + m_bRflDirSameAsDb = (!pszRflDir || !(*pszRflDir)) + ? TRUE + : FALSE; + + flmAssert( m_pDatabase->m_lastCommittedDbHdr.ui32DbVersion); + + m_bCreateRflDir = TRUE; + return( rflGetDirAndPrefix( m_pDatabase->m_pszDbPath, + pszRflDir, m_szRflDir)); +} + +/******************************************************************** +Desc: Gets an RFL file name - based on DB name and RFL directory. +*********************************************************************/ +RCODE rflGetFileName( + const char * pszDbName, + const char * pszRflDir, + FLMUINT uiFileNum, + char * pszRflFileName) +{ + RCODE rc = NE_SFLM_OK; + char szBaseName [F_FILENAME_SIZE]; + FLMUINT uiBaseNameSize; + + // Get the full RFL file name. + + if (RC_BAD( rc = rflGetDirAndPrefix( pszDbName, pszRflDir, + pszRflFileName))) + { + goto Exit; + } + + uiBaseNameSize = sizeof( szBaseName); + rflGetBaseFileName( uiFileNum, szBaseName, &uiBaseNameSize, NULL); + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->pathAppend( + pszRflFileName, szBaseName))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Gets an RFL file number from the RFL file name. +*********************************************************************/ +FLMBOOL rflGetFileNum( + const char * pszRflFileName, + FLMUINT * puiFileNum) +{ + FLMBOOL bGotNum = FALSE; + char szDir[F_PATH_MAX_SIZE]; + char szBaseName[F_FILENAME_SIZE]; + char * pszTmp; + FLMUINT uiCharCnt; + + if( RC_BAD( gv_SFlmSysData.pFileSystem->pathReduce( + pszRflFileName, szDir, szBaseName))) + { + goto Exit; + } + + // See if it has a .log extension. + + pszTmp = &szBaseName [0]; + while (*pszTmp && *pszTmp != '.') + { + pszTmp++; + } + + // If we do not have a .log extension, it is not a legitimate + // RFL file. + + if (f_stricmp( pszTmp, ".log") != 0) + { + goto Exit; + } + + // Parse out the name according to the rules for this DB version. + + *pszTmp = 0; // Set period to zero + pszTmp = &szBaseName [0]; + *puiFileNum = 0; + uiCharCnt = 0; + + // Name up to the period should be a hex number + + while (*pszTmp) + { + (*puiFileNum) <<= 4; + if (*pszTmp >= NATIVE_ZERO && *pszTmp <= NATIVE_NINE) + { + *puiFileNum += (FLMUINT)(*pszTmp - NATIVE_ZERO); + } + else if (*pszTmp >= NATIVE_LOWER_A && *pszTmp <= NATIVE_LOWER_F) + { + *puiFileNum += ((FLMUINT)(*pszTmp - NATIVE_LOWER_A) + 10); + } + else if (*pszTmp >= NATIVE_UPPER_A && *pszTmp <= NATIVE_UPPER_F) + { + *puiFileNum += ((FLMUINT)(*pszTmp - NATIVE_UPPER_A) + 10); + } + else + { + goto Exit; // Not a hex number + } + uiCharCnt++; + pszTmp++; + } + + // Better have been exactly 8 hex digits. + + bGotNum = (FLMBOOL)((uiCharCnt == 8) + ? TRUE + : FALSE); +Exit: + + return( bGotNum); +} + +/******************************************************************** +Desc: Sets up the RFL object - associating with a file, etc. +*********************************************************************/ +RCODE F_Rfl::setup( + F_Database * pDatabase, + const char * pszRflDir) +{ + RCODE rc = NE_SFLM_OK; + + // Better not already be associated with an F_Database object + + flmAssert( m_pDatabase == NULL); + m_pDatabase = pDatabase; + + // Allocate memory for the RFL buffers + +#ifndef FLM_UNIX + if (!gv_SFlmSysData.bOkToDoAsyncWrites) +#endif + { + m_uiRflWriteBufs = 1; + m_uiBufferSize = + DEFAULT_RFL_WRITE_BUFFERS * DEFAULT_RFL_BUFFER_SIZE; + } + + if (RC_BAD( rc = f_mutexCreate( &m_hBufMutex))) + { + goto Exit; + } + + if( RC_BAD( rc = FlmAllocIOBufferMgr( &m_Buf1.pBufferMgr))) + { + goto Exit; + } + + if( RC_BAD( rc = FlmAllocIOBufferMgr( &m_Buf2.pBufferMgr))) + { + goto Exit; + } + + m_Buf1.pBufferMgr->enableKeepBuffer(); + m_Buf1.pBufferMgr->setMaxBuffers( m_uiRflWriteBufs); + m_Buf1.pBufferMgr->setMaxBytes( m_uiRflWriteBufs * m_uiBufferSize); + + if( RC_BAD( rc = m_Buf1.pBufferMgr->getBuffer( &m_Buf1.pIOBuffer, + m_uiBufferSize, m_uiBufferSize))) + { + goto Exit; + } + + m_Buf2.pBufferMgr->enableKeepBuffer(); + m_Buf2.pBufferMgr->setMaxBuffers( m_uiRflWriteBufs); + m_Buf2.pBufferMgr->setMaxBytes( m_uiRflWriteBufs * m_uiBufferSize); + + if( RC_BAD( rc = m_Buf2.pBufferMgr->getBuffer( &m_Buf2.pIOBuffer, + m_uiBufferSize, m_uiBufferSize))) + { + goto Exit; + } + + m_pCurrentBuf = &m_Buf1; + m_pCurrentBuf->uiRflBufBytes = 0; + + // Set the RFL directory and prefix if necessary. + + if (RC_BAD( rc = setRflDir( pszRflDir))) + { + goto Exit; + } + + // Set up the compare object for comparing index keys. + + if ((m_pIxCompareObject = f_new IXKeyCompare) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Wait for the writes of a buffer to finish. This routine assumes + that the m_hBufMutex is locked when coming in. It will ALWAYS + unlock the mutex before exiting. +*********************************************************************/ +RCODE F_Rfl::waitForWrites( + F_SEM hWaitSem, + RFL_BUFFER * pBuffer, + FLMBOOL bIsWriter) +{ + RCODE rc = NE_SFLM_OK; + RCODE TempRc; + RFL_WAITER Waiter; + FLMBOOL bMutexLocked = TRUE; + + // Put self on the wait queue for the buffer. + + Waiter.uiThreadId = f_threadId(); + Waiter.bIsWriter = bIsWriter; + Waiter.hESem = hWaitSem; + + // Note: rc better be changed to success or write error + // by the process that signals us. + + rc = RC_SET( NE_SFLM_FAILURE); + + Waiter.pRc = &rc; + Waiter.pNext = NULL; + + if (pBuffer->pLastWaiter) + { + pBuffer->pLastWaiter->pNext = &Waiter; + } + else + { + pBuffer->pFirstWaiter = &Waiter; + + } + pBuffer->pLastWaiter = &Waiter; + + f_mutexUnlock( m_hBufMutex); + bMutexLocked = FALSE; + + // Now just wait to be signaled. + + if (RC_BAD( TempRc = f_semWait( Waiter.hESem, F_SEM_WAITFOREVER))) + { + RC_UNEXPECTED_ASSERT( TempRc); + rc = TempRc; + } + else + { + // Process that signaled us better set the rc to something + // besides NE_SFLM_FAILURE. + + if (rc == NE_SFLM_FAILURE) + { + RC_UNEXPECTED_ASSERT( rc); + } + } + + if (bMutexLocked) + { + f_mutexUnlock( m_hBufMutex); + } + + return( rc); +} + +/******************************************************************** +Desc: If a commit is in progress, wait for it to finish. +*********************************************************************/ +RCODE F_Rfl::waitForCommit( + F_SEM hWaitSem) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + + // NOTE: If m_pCommitBuf is NULL it cannot be set to something + // non-NULL except by this thread when this thread ends the + // transaction. So, there is no need to lock the mutex and + // re-check if it is NULL. + + if (m_pCommitBuf) + { + f_mutexLock( m_hBufMutex); + bMutexLocked = TRUE; + + // Check m_pCommitBuf again after locking mutex - may have + // finished. + + if (m_pCommitBuf) + { + bMutexLocked = FALSE; + rc = waitForWrites( hWaitSem, m_pCommitBuf, FALSE); + } + } + + if (bMutexLocked) + { + f_mutexUnlock( m_hBufMutex); + } + + return( rc); +} + +/******************************************************************** +Desc: Write out the header information for an RFL file. +*********************************************************************/ +RCODE F_Rfl::writeHeader( + FLMUINT uiFileNum, + FLMUINT uiEof, + FLMBYTE * pucSerialNum, + FLMBYTE * pucNextSerialNum, + FLMBOOL bKeepSignature) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucBuf[ 512]; + FLMUINT uiBytesWritten; + + flmAssert( m_pDatabase); + flmAssert( m_pFileHdl); + + f_memset( ucBuf, 0, sizeof( ucBuf)); + f_memcpy( &ucBuf [RFL_NAME_POS], RFL_NAME, RFL_NAME_LEN); + f_memcpy( &ucBuf [RFL_VERSION_POS], RFL_VERSION, RFL_VERSION_LEN); + UD2FBA( (FLMUINT32)uiFileNum, &ucBuf [RFL_FILE_NUMBER_POS]); + UD2FBA( (FLMUINT32)uiEof, &ucBuf [RFL_EOF_POS]); + + f_memcpy( &ucBuf [RFL_DB_SERIAL_NUM_POS], + m_pDatabase->m_lastCommittedDbHdr.ucDbSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_memcpy( &ucBuf [RFL_SERIAL_NUM_POS], pucSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_memcpy( &ucBuf [RFL_NEXT_FILE_SERIAL_NUM_POS], pucNextSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_strcpy( (char *)&ucBuf [RFL_KEEP_SIGNATURE_POS], + (char *)((bKeepSignature) + ? RFL_KEEP_SIGNATURE + : RFL_NOKEEP_SIGNATURE)); + + // Write out the header + + if (RC_BAD( rc = m_pFileHdl->sectorWrite( 0L, 512, + ucBuf, sizeof( ucBuf), + NULL, &uiBytesWritten))) + { + // Remap disk full error + + if (rc == NE_FLM_IO_DISK_FULL) + { + rc = RC_SET( NE_SFLM_RFL_DISK_FULL); + m_bRflVolumeFull = TRUE; + } + m_bRflVolumeOk = FALSE; + goto Exit; + } + + // Flush the file handle to ensure it is forced to disk. + + if (RC_BAD( rc = m_pFileHdl->flush())) + { + // Remap disk full error + + if (rc == NE_FLM_IO_DISK_FULL) + { + rc = RC_SET( NE_SFLM_RFL_DISK_FULL); + m_bRflVolumeFull = TRUE; + } + m_bRflVolumeOk = FALSE; + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Verifies the header of an RFL file. +*********************************************************************/ +RCODE F_Rfl::verifyHeader( + FLMBYTE * pucHeader, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum) +{ + RCODE rc = NE_SFLM_OK; + + flmAssert( m_pDatabase); + + // Check the RFL name and version number + + if( f_memcmp( &pucHeader [RFL_NAME_POS], RFL_NAME, + RFL_NAME_LEN) != 0) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + + if( f_memcmp( &pucHeader [RFL_VERSION_POS], RFL_VERSION, + RFL_VERSION_LEN) != 0) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + + // Verify the database serial number + + if( f_memcmp( &pucHeader [RFL_DB_SERIAL_NUM_POS], + m_pDatabase->m_lastCommittedDbHdr.ucDbSerialNum, + SFLM_SERIAL_NUM_SIZE) != 0) + { + rc = RC_SET( NE_SFLM_BAD_RFL_DB_SERIAL_NUM); + goto Exit; + } + + // Verify the serial number that is expected to be on the + // RFL file. If pucSerialNum is NULL, we will not verify + // it. This is generally only done during recovery or restore + // when we are reading through multiple RFL files and we need + // to verify their serial numbers. + + if( pucSerialNum && + f_memcmp( &pucHeader [RFL_SERIAL_NUM_POS], + pucSerialNum, SFLM_SERIAL_NUM_SIZE) != 0) + { + rc = RC_SET( NE_SFLM_BAD_RFL_SERIAL_NUM); + goto Exit; + } + + // Verify the file number. + + if( uiFileNum != (FLMUINT)FB2UD( &pucHeader [RFL_FILE_NUMBER_POS])) + { + rc = RC_SET( NE_SFLM_BAD_RFL_FILE_NUMBER); + goto Exit; + } + + // Save serial numbers from the header. + + f_memcpy( m_ucCurrSerialNum, &pucHeader [RFL_SERIAL_NUM_POS], + SFLM_SERIAL_NUM_SIZE); + f_memcpy( m_ucNextSerialNum, &pucHeader [RFL_NEXT_FILE_SERIAL_NUM_POS], + SFLM_SERIAL_NUM_SIZE); + + // Save some things from the header. + + m_uiFileEOF = (FLMUINT)FB2UD( &pucHeader [RFL_EOF_POS]); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Opens an RFL file. Verifies the serial number for 4.3 dbs. +*********************************************************************/ +RCODE F_Rfl::openFile( + F_SEM hWaitSem, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum) +{ + RCODE rc = NE_SFLM_OK; + char szRflFileName [F_PATH_MAX_SIZE]; + FLMUINT uiFileNameSize; + FLMBYTE ucBuf [512]; + FLMUINT uiBytesRead; + + flmAssert( m_pDatabase); + + // If we have a file open and it is not the file number + // passed in, close it. + + if (m_pFileHdl) + { + if (m_pCurrentBuf->uiCurrFileNum != uiFileNum) + { + if (RC_BAD( rc = waitForCommit( hWaitSem))) + { + goto Exit; + } + closeFile(); + } + else + { + goto Exit; // Will return NE_SFLM_OK + } + } + else + { + // Should not be able to be in the middle of a commit + // if we don't have a file open! + + flmAssert( !m_pCommitBuf); + } + + // Generate the log file name. + + uiFileNameSize = sizeof( szRflFileName); + getFullRflFileName( uiFileNum, szRflFileName, &uiFileNameSize, NULL); + + // Open the file. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->openBlockFile( szRflFileName, + FLM_IO_RDWR | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + 512, &m_pFileHdl))) + { + goto Exit; + } + + // Read the header. + + if (RC_BAD( rc = m_pFileHdl->sectorRead( 0, 512, ucBuf, + &uiBytesRead))) + { + if (rc == NE_FLM_IO_END_OF_FILE) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + } + else + { + m_bRflVolumeOk = FALSE; + } + goto Exit; + } + + // If there is not enough data in the buffer, it is not an + // RFL file. + + if (uiBytesRead < 512) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + + // Verify the header information + + if (RC_BAD( rc = verifyHeader( ucBuf, uiFileNum, pucSerialNum))) + { + goto Exit; + } + + m_pCurrentBuf->uiRflBufBytes = 0; + m_pCurrentBuf->uiRflFileOffset = 0; + m_pCurrentBuf->uiCurrFileNum = uiFileNum; + +Exit: + + if( RC_BAD( rc)) + { + waitForCommit( hWaitSem); + closeFile(); + } + + return( rc); +} + +/******************************************************************** +Desc: Creates a new roll forward log file. +*********************************************************************/ +RCODE F_Rfl::createFile( + F_Db * pDb, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum, + FLMBYTE * pucNextSerialNum, + FLMBOOL bKeepSignature) +{ + RCODE rc = NE_SFLM_OK; + char szRflFileName [F_PATH_MAX_SIZE]; + FLMUINT uiFileNameSize; + + flmAssert( m_pDatabase); + + // Better not be trying to create the current file + + flmAssert( uiFileNum != m_pCurrentBuf->uiCurrFileNum); + + // If we have a file open, close it. + + if (RC_BAD( rc = waitForCommit( pDb->m_hWaitSem))) + { + goto Exit; + } + closeFile(); + + // Generate the log file name. + + uiFileNameSize = sizeof( szRflFileName); + getFullRflFileName( uiFileNum, szRflFileName, &uiFileNameSize, NULL); + + // Delete the file if it already exists - don't care + // about return code here + + (void)gv_SFlmSysData.pFileSystem->deleteFile( szRflFileName); + + // If DB is 4.3 or greater and we are in the same directory as + // our database files, see if we need to create the + // subdirectory. Otherwise, the RFL directory should already + // have been created. If the directory already exists, it is + // OK - we only try this the first time after setRflDir is + // called - to either verify that the directory exists, or if + // it doesn't, to create it. + + if (m_bCreateRflDir) + { + + // If it already exists, don't attempt to create it. + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->doesFileExist( m_szRflDir))) + { + if (rc != NE_FLM_IO_PATH_NOT_FOUND && + rc != NE_FLM_IO_INVALID_FILENAME) + { + goto Exit; + } + else + { + if (RC_BAD( rc = + gv_SFlmSysData.pFileSystem->createDir( m_szRflDir))) + { + goto Exit; + } + } + } + m_bCreateRflDir = FALSE; + } + + // Create the file + + if (RC_BAD( rc = gv_SFlmSysData.pFileSystem->createBlockFile( szRflFileName, + FLM_IO_RDWR | FLM_IO_EXCL | FLM_IO_SH_DENYNONE | FLM_IO_DIRECT, + 512, &m_pFileHdl))) + { + goto Exit; + } + + // Initialize the header. + + if (RC_BAD( rc = writeHeader( uiFileNum, 0, + pucSerialNum, pucNextSerialNum, bKeepSignature))) + { + goto Exit; + } + + m_pCurrentBuf->uiRflBufBytes = 0; + m_pCurrentBuf->uiRflFileOffset = 512; + m_pCurrentBuf->uiCurrFileNum = uiFileNum; +Exit: + + // Close the RFL log file AND delete it if we were not successful. + + if (RC_BAD( rc)) + { + closeFile(); + (void)gv_SFlmSysData.pFileSystem->deleteFile( szRflFileName); + } + return( rc); +} + +/******************************************************************** +Desc: Copy last partial sector of last buffer written (or to be + written) into a new buffer. +*********************************************************************/ +void F_Rfl::copyLastSector( + RFL_BUFFER * pBuffer, + FLMBYTE * pucOldBuffer, + FLMBYTE * pucNewBuffer, + FLMUINT uiCurrPacketLen, + FLMBOOL bStartingNewFile) +{ + FLMUINT uiOldBufBytes = pBuffer->uiRflBufBytes; + + // If we will be starting a new file, no need to keep any of + // what is in the buffer. Only the current packet needs to + // be copied - at the beginning of the buffer. + + // OTHERWISE: + + // If there are fewer than 512 bytes in the buffer, we simply + // keep them and keep appending to the buffer the next time + // we output stuff. The beginning of the buffer must ALWAYS be + // a 512 byte boundary in the file, because we want to always + // do our writing on 512 byte boundaries - because of direct IO. + + // If the number of bytes in the buffer is over 512 and it is + // evenly divisible by 512, we can clear the buffer. Otherwise, + // we want to move the extra bytes over the last 512 byte boundary + // down to the beginning of the buffer and adjust the buffer bytes + // to reflect just these left-over bytes. + + if (bStartingNewFile) + { + pBuffer->uiRflBufBytes = 0; + pBuffer->uiRflFileOffset = 512; + } + else if (pBuffer->uiRflBufBytes >= 512) + { + + // See if the number of bytes in the buffer is an exact + // multiple of 512. + + if (pBuffer->uiRflBufBytes & 511) // Not exact multiple + { + + // Round m_uiRflBufBytes down to next 512 byte boundary + + FLMUINT ui512Offset = ROUND_DOWN_TO_NEAREST_512( + pBuffer->uiRflBufBytes); + + // Move all bytes above the nearest 512 byte boundary + // down to the beginning of the buffer and adjust + // pBuffer->uiRflBufBytes and + // pBuffer->uiRflFileOffset accordingly. + + f_memcpy( pucNewBuffer, &pucOldBuffer[ui512Offset], + pBuffer->uiRflBufBytes - ui512Offset); + pBuffer->uiRflBufBytes -= ui512Offset; + pBuffer->uiRflFileOffset += ui512Offset; + } + else + { + pBuffer->uiRflFileOffset += pBuffer->uiRflBufBytes; + pBuffer->uiRflBufBytes = 0; + } + } + else if (pucNewBuffer != pucOldBuffer) + { + f_memcpy( pucNewBuffer, pucOldBuffer, pBuffer->uiRflBufBytes); + } + + if (uiCurrPacketLen) + { + flmAssert( uiOldBufBytes + uiCurrPacketLen <= m_uiBufferSize); + f_memmove( &pucNewBuffer [pBuffer->uiRflBufBytes], + &pucOldBuffer [uiOldBufBytes], + uiCurrPacketLen); + } +} + +/******************************************************************** +Desc: Flush the RFL data from the buffer to disk. +*********************************************************************/ +RCODE F_Rfl::flush( + F_Db * pDb, + RFL_BUFFER * pBuffer, + FLMBOOL bFinalWrite, + FLMUINT uiCurrPacketLen, + FLMBOOL bStartingNewFile) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesWritten; + IF_IOBuffer * pNewBuffer = NULL; + IF_IOBuffer * pAsyncBuf = NULL; + FLMBYTE * pucOldBuffer; + FLMUINT uiFileOffset; + FLMUINT uiBufBytes; + + if (m_pFileHdl && pBuffer->uiRflBufBytes) + { + // Must wait for stuff in committing buffer, if any, before + // going ahead here. + + if (pBuffer != m_pCommitBuf) + { + if (RC_BAD( rc = waitForCommit( pDb->m_hWaitSem))) + { + goto Exit; + } + } + + if (m_uiRflWriteBufs > 1 && m_pFileHdl->canDoAsync()) + { + pAsyncBuf = pBuffer->pIOBuffer; + } + + if ((FLMUINT)(-1) - pBuffer->uiRflFileOffset <= + pBuffer->uiRflBufBytes) + { + rc = RC_SET( NE_SFLM_DB_FULL); + goto Exit; + } + + pucOldBuffer = pBuffer->pIOBuffer->getBuffer(); + uiFileOffset = pBuffer->uiRflFileOffset; + uiBufBytes = pBuffer->uiRflBufBytes; + if (m_uiRflWriteBufs > 1) + { + if( RC_BAD( rc = pBuffer->pBufferMgr->getBuffer( + &pNewBuffer, + m_uiBufferSize, m_uiBufferSize))) + { + goto Exit; + } + + // No need to copy data if it is the final write, + // because it won't be reused anyway - the data for + // the next transaction has already been copied to + // another buffer. + + if (!bFinalWrite) + { + copyLastSector( pBuffer, pucOldBuffer, pNewBuffer->getBuffer(), + uiCurrPacketLen, bStartingNewFile); + } + } + + rc = m_pFileHdl->sectorWrite( uiFileOffset, uiBufBytes, + pucOldBuffer, + m_uiBufferSize, pAsyncBuf, + &uiBytesWritten, FALSE); + if( m_uiRflWriteBufs == 1) + { + + // We are counting on the fact that the write completed. + // When we only have one buffer, we cannot do async + // writes. + + flmAssert( !pAsyncBuf); + if (RC_OK( rc) && !bFinalWrite) + { + copyLastSector( pBuffer, pucOldBuffer, pucOldBuffer, + uiCurrPacketLen, bStartingNewFile); + } + + // DO NOT call notifyComplete - that would put + // pBuffer->pIOBuffer into the avail list, and we + // don't want that. We simply want to keep + // reusing it. + + } + else + { + + // No need to call copyLastSector, because it was called + // above before calling SectorWrite. The part of the + // old buffer that needs to be transferred to the new + // buffer has already been transferred. + + if (!pAsyncBuf) + { + pBuffer->pIOBuffer->notifyComplete( rc); + } + pBuffer->pIOBuffer = pNewBuffer; + } + + if (RC_BAD( rc)) + { + // Remap disk full error + + if (rc == NE_FLM_IO_DISK_FULL) + { + rc = RC_SET( NE_SFLM_RFL_DISK_FULL); + m_bRflVolumeFull = TRUE; + } + m_bRflVolumeOk = FALSE; + goto Exit; + } + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Switch buffers. This routine assumes the m_hBufMutex is locked. +*********************************************************************/ +void F_Rfl::switchBuffers( void) +{ + RFL_BUFFER * pOldBuffer = m_pCurrentBuf; + + if (m_pCurrentBuf == &m_Buf1) + { + m_pCurrentBuf = &m_Buf2; + } + else + { + m_pCurrentBuf = &m_Buf1; + } + m_pCurrentBuf->bTransInProgress = pOldBuffer->bTransInProgress; + m_pCurrentBuf->uiCurrFileNum = pOldBuffer->uiCurrFileNum; + m_pCurrentBuf->uiRflBufBytes = pOldBuffer->uiRflBufBytes; + m_pCurrentBuf->uiRflFileOffset = pOldBuffer->uiRflFileOffset; + + if (pOldBuffer->uiRflBufBytes) + { + copyLastSector( m_pCurrentBuf, pOldBuffer->pIOBuffer->getBuffer(), + m_pCurrentBuf->pIOBuffer->getBuffer(), 0, FALSE); + } +} + +/******************************************************************** +Desc: Wait for all RFL transaction writes to be finished. The caller + has the write lock on the database, which will prevent further + writes to the RFL. +*********************************************************************/ +FLMBOOL F_Rfl::seeIfRflWritesDone( + F_SEM hWaitSem, + FLMBOOL bForceWait) +{ + FLMBOOL bWritesDone; + + f_mutexLock( m_hBufMutex); + + if (!bForceWait) + { + bWritesDone = (FLMBOOL)((m_pCurrentBuf->pFirstWaiter || m_pCommitBuf) + ? FALSE + : TRUE); + + if( bWritesDone) + { + m_pCurrentBuf->uiRflBufBytes = 0; + } + + f_mutexUnlock( m_hBufMutex); + } + else + { + // If the current buffer has a waiter, add self to that list + // to wait, because it will be notified after the commit buffer + // has been notified. Otherwise, if there is a commit in + // progress, add self to that list to wait. + + if (m_pCurrentBuf->pFirstWaiter) + { + + // If bTransInProgress is TRUE and m_pCommitBuf is NULL + // then this thread is the current transaction, and + // nobody is going to wake up the first waiter until we + // are done! Hence, we must wake him up. + + if (!m_pCommitBuf) + { + + // If m_pCommitBuf is NULL, this could only be possible if + // there is a transaction in progress. Otherwise, there + // would not have been a pFirstWaiter, because when + // the commit buffer finishes writing, if there is a + // waiter, it will set commitbuf=currentbuf if there + // is no transaction active. + + flmAssert( m_pCurrentBuf->bTransInProgress); + + m_pCommitBuf = m_pCurrentBuf; + switchBuffers(); + wakeUpWaiter( NE_SFLM_OK, TRUE); + (void)waitForWrites( hWaitSem, m_pCommitBuf, FALSE); + } + else + { + FLMBOOL bSaveTransInProgress = m_pCurrentBuf->bTransInProgress; + + // Must set bTransInProgress to FALSE so that when the writer + // of m_pCommitBuf finishes, it will signal the first waiter + // on m_pCurrentBuf. If we don't do this, m_pCommitBuf will + // simply be set to NULL, and the first waiter will never + // be woke up. + + m_pCurrentBuf->bTransInProgress = FALSE; + (void)waitForWrites( hWaitSem, m_pCurrentBuf, FALSE); + + // It is OK to restore the trans in progress flag to what it + // was before, because whoever called this routine has a lock + // on the database, and it is his trans-in-progress state + // that should be preserved. No other thread will have been + // able to change that state because the database is locked. + + f_mutexLock( m_hBufMutex); + m_pCurrentBuf->bTransInProgress = bSaveTransInProgress; + f_mutexUnlock( m_hBufMutex); + } + + m_pCurrentBuf->uiRflBufBytes = 0; + } + else if (m_pCommitBuf) + { + (void)waitForWrites( hWaitSem, m_pCommitBuf, FALSE); + } + else + { + f_mutexUnlock( m_hBufMutex); + } + bWritesDone = TRUE; + } + return( bWritesDone); +} + +/******************************************************************** +Desc: Wake up the first thread that is waiting on the commit buffer. +*********************************************************************/ +void F_Rfl::wakeUpWaiter( + RCODE rc, + FLMBOOL bIsWriter) +{ + F_SEM hESem; + +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( bIsWriter); +#else + if (bIsWriter) + { + flmAssert( m_pCommitBuf->pFirstWaiter->bIsWriter); + } + else + { + flmAssert( !m_pCommitBuf->pFirstWaiter->bIsWriter); + } +#endif + + *(m_pCommitBuf->pFirstWaiter->pRc) = rc; + hESem = m_pCommitBuf->pFirstWaiter->hESem; + if ((m_pCommitBuf->pFirstWaiter = + m_pCommitBuf->pFirstWaiter->pNext) == NULL) + { + m_pCommitBuf->pLastWaiter = NULL; + } + f_semSignal( hESem); +} + +/******************************************************************** +Desc: Wait for the transaction writes to be finished. +*********************************************************************/ +RCODE F_Rfl::completeTransWrites( + F_Db * pDb, + FLMBOOL bCommitting, + FLMBOOL bOkToUnlock) +{ + RCODE rc = NE_SFLM_OK; + RCODE tmpRc; + FLMBOOL bMutexLocked = FALSE; + FLMBOOL bNotifyWaiters = FALSE; + FLMBOOL bDbUnlocked = FALSE; + SFLM_DB_STATS * pDbStats = NULL; + F_TMSTAMP StartTime; + + f_mutexLock( m_hBufMutex); + bMutexLocked = TRUE; + m_pCurrentBuf->bTransInProgress = FALSE; + + flmAssert( pDb->m_uiFlags & FDB_HAS_WRITE_LOCK); + + // When recovering or restoring all we need to do is write out + // the database header. + + if (pDb->m_uiFlags & FDB_REPLAYING_RFL) + { + if (pDb->m_bHadUpdOper && + m_pCurrentBuf->bOkToWriteHdrs) + { + f_mutexUnlock( m_hBufMutex); + bMutexLocked = FALSE; + if (RC_BAD( rc = m_pDatabase->writeDbHdr( pDb->m_pDbStats, + pDb->m_pSFileHdl, + &m_pCurrentBuf->dbHdr, + &m_pCurrentBuf->cpHdr, FALSE))) + { + m_pDatabase->setMustCloseFlags( rc, FALSE); + } + } + goto Exit; + } + + // Handle empty transactions differently. + // These transactions should not do any writing and do not need to + // wait for all writes to complete, unless the bOkToUnlock flag + // is set to FALSE. In that case they must wait for all writes + // to complete before unlocking. + + if (!pDb->m_bHadUpdOper) + { + // If the current buffer has a waiter, add self to that list + // to wait, because it will be notified after the commit buffer + // has been notified. Otherwise, if there is a commit in + // progress, add self to that list to wait. + + if (m_pCurrentBuf->pFirstWaiter) + { + // If m_pCommitBuf is NULL then nobody is going to wake up + // the first waiter - we must do it. + + if (!m_pCommitBuf) + { + if (bOkToUnlock) + { + pDb->unlinkFromTransList( bCommitting); + bDbUnlocked = TRUE; + } + m_pCommitBuf = m_pCurrentBuf; + switchBuffers(); + wakeUpWaiter( NE_SFLM_OK, TRUE); + if (!bOkToUnlock) + { + bMutexLocked = FALSE; + (void)waitForWrites( pDb->m_hWaitSem, m_pCommitBuf, FALSE); + } + } + else if (!bOkToUnlock) + { + bMutexLocked = FALSE; + (void)waitForWrites( pDb->m_hWaitSem, m_pCurrentBuf, FALSE); + } + } + else if (m_pCommitBuf) + { + if (!bOkToUnlock) + { + bMutexLocked = FALSE; + rc = waitForWrites( pDb->m_hWaitSem, m_pCommitBuf, FALSE); + } + } + goto Exit; + } + + // If there is a transaction committing, put self into + // the wait list on the current buffer. When the committer + // finishes, he will wake up the first thread in the list + // and that thread will commit the buffer. + + if (m_pCommitBuf) + { + FLMBOOL bIsWriter; + + // Another thread has to be doing the writes to m_pCommitBuf, + // which means that m_pCurrentBuf better not be equal to + // m_pCommitBuf. + + flmAssert( m_pCommitBuf != m_pCurrentBuf); + + // If there are no waiters, we are the first one, so when + // we get signaled, we should proceed and do the write. + + bIsWriter = m_pCurrentBuf->pFirstWaiter ? FALSE : TRUE; + if (bOkToUnlock) + { + pDb->unlinkFromTransList( bCommitting); + bDbUnlocked = TRUE; + } + bMutexLocked = FALSE; + rc = waitForWrites( pDb->m_hWaitSem, m_pCurrentBuf, bIsWriter); + + // If we were the first one in the queue, we must now + // do the write. + + if (!bIsWriter) + { + goto Exit; + } + + // First one in the queue, fall through to do the write. + + // The thread that woke me up better have set m_pCommitBuf + // See below. + + flmAssert( m_pCommitBuf); + } + else if (m_pCurrentBuf->pFirstWaiter) + { + + // Another thread is ready to commit the next set of + // buffers, but just needs to be woke up. + + if (bOkToUnlock) + { + pDb->unlinkFromTransList( bCommitting); + bDbUnlocked = TRUE; + } + + // Need to set things up for that first waiter and get him + // going. + + m_pCommitBuf = m_pCurrentBuf; + switchBuffers(); + wakeUpWaiter( rc, TRUE); + + // Wait for the write to be completed. + + bMutexLocked = FALSE; + rc = waitForWrites( pDb->m_hWaitSem, m_pCommitBuf, FALSE); + goto Exit; + } + else + { + m_pCommitBuf = m_pCurrentBuf; + switchBuffers(); + if (bOkToUnlock) + { + pDb->unlinkFromTransList( bCommitting); + bDbUnlocked = TRUE; + } + f_mutexUnlock( m_hBufMutex); + bMutexLocked = FALSE; + } + + // NOTE: From this point on we use tmpRc because we don't want to + // lose the rc that may have been set above in the call to + // waitForWrites + + // At this point the mutex better not be locked. + + flmAssert( !bMutexLocked); + bNotifyWaiters = TRUE; + + if( (pDbStats = pDb->m_pDbStats) != NULL) + { + f_timeGetTimeStamp( &StartTime); + } + + // Must write out whatever we have in the commit buffer before + // unlocking the database. + + if (RC_BAD( tmpRc = flush( pDb, m_pCommitBuf, TRUE))) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + goto Exit; + } + + // Wait for any pending IO off of the log buffer + + if (RC_BAD( tmpRc = m_pCommitBuf->pBufferMgr->waitForAllPendingIO())) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + goto Exit; + } + + // Force the RFL writes to disk if necessary. + // NOTE: It is possible for m_pFileHdl to be NULL at this point if + // there were no operations actually logged. This happens in + // FlmDbUpgrade (see flconvrt.cpp). Even though nothing was logged + // the transaction is not an empty transaction, because it still needs + // to write out the log header. + + if (m_pFileHdl) + { + if (RC_BAD( tmpRc = m_pFileHdl->flush())) + { + + // Remap disk full error + + if (tmpRc == NE_FLM_IO_DISK_FULL) + { + rc = RC_SET( NE_SFLM_RFL_DISK_FULL); + m_bRflVolumeFull = TRUE; + } + else if (RC_OK( rc)) + { + rc = tmpRc; + } + m_bRflVolumeOk = FALSE; + goto Exit; + } + } + + // Write the log header + + if (m_pCommitBuf->bOkToWriteHdrs) + { + if (RC_BAD( tmpRc = m_pDatabase->writeDbHdr( pDb->m_pDbStats, + pDb->m_pSFileHdl, + &m_pCommitBuf->dbHdr, + &m_pCommitBuf->cpHdr, FALSE))) + { + if (RC_OK( rc)) + { + rc = tmpRc; + } + m_pDatabase->setMustCloseFlags( tmpRc, FALSE); + goto Exit; + } + } + +Exit: + + if (!bDbUnlocked && bOkToUnlock) + { + pDb->unlinkFromTransList( bCommitting); + } + + if (bNotifyWaiters) + { + FLMUINT uiNumFinished = 1; // For self + + flmAssert( !bMutexLocked); + f_mutexLock( m_hBufMutex); + bMutexLocked = TRUE; + + // Wake up any waiters + + while (m_pCommitBuf->pFirstWaiter) + { + uiNumFinished++; + wakeUpWaiter( rc, FALSE); + } + + // If there are waiters on the current buffer, the first one + // should be woke up so it can start the next set of writes. + + if (m_pCurrentBuf->pFirstWaiter && !m_pCurrentBuf->bTransInProgress) + { + flmAssert( m_pCurrentBuf != m_pCommitBuf); + m_pCommitBuf = m_pCurrentBuf; + switchBuffers(); + wakeUpWaiter( rc, TRUE); + } + else + { + m_pCommitBuf = NULL; + } + if (pDbStats) + { + flmAddElapTime( &StartTime, + &pDbStats->UpdateTransStats.GroupCompletes.ui64ElapMilli); + pDbStats->UpdateTransStats.GroupCompletes.ui64Count++; + pDbStats->bHaveStats = TRUE; + pDbStats->UpdateTransStats.ui64GroupFinished += uiNumFinished; + } + } + + if (bMutexLocked) + { + f_mutexUnlock( m_hBufMutex); + } + return( rc); +} + +/******************************************************************** +Desc: Flush all completed packets out of the RFL buffer, and shift + the new partial packet down. This guarantees that there is + now room in the buffer for the maximum packet size. +*********************************************************************/ +RCODE F_Rfl::shiftPacketsDown( + F_Db * pDb, + FLMUINT uiCurrPacketLen, + FLMBOOL bStartingNewFile) +{ + RCODE rc = NE_SFLM_OK; + + // The call to flush will move whatever needs to be moved from + // the current buffer into a new buffer if multiple buffers are + // being used. If only one buffer is being used, it will move + // the part of the packet that needs to be moved down to the + // beginning of the buffer - AFTER writing out the buffer. + + if (RC_BAD( rc = flush( pDb, m_pCurrentBuf, FALSE, + uiCurrPacketLen, bStartingNewFile))) + { + goto Exit; + } + + // NOTE: If multiple buffers are being used, whatever was moved + // to the new buffer has not yet been written out + + if (bStartingNewFile) + { + if( RC_BAD( rc = waitPendingWrites())) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Determine if we should start a new file. If we are over the + low limit, and the bDoNewIfOverLowLimit flag is set, we + will start a new log file. Or, if this packet size would + put us over the upper limit, we will start a new log file. +*********************************************************************/ +RCODE F_Rfl::seeIfNeedNewFile( + F_Db * pDb, + FLMUINT uiPacketLen, + FLMBOOL bDoNewIfOverLowLimit) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE ucNextSerialNum [SFLM_SERIAL_NUM_SIZE]; + + flmAssert( m_pDatabase); + + // If the keep files flag is FALSE, we won't start + // a new file. NOTE: This should ALWAYS be false + // for pre 4.3 databases. + + if( !m_bKeepRflFiles) + { + goto Exit; + } + + // VERY IMPORTANT NOTE: It is preferrable that we keep transactions + // entirely contained in the same RFL file if at all possible. Note + // that it is NOT a hard and fast requirement. The system will work + // just fine if we don't. However, it would be nice if RFL files + // always ended with a commit or abort packet. This preferences is + // due to what happens after a restore operation. After a restore + // operation, we always need to start a new RFL file, but if possible, + // we would like that new RFL file to be the next one in the sequence + // after the last RFL file that was restored. We can only do this if + // we were able to restore EVERY transaction that was in the last + // restored RFL file - which we can only do if the last restored RFL + // file ended with a commit or abort packet. + // To accomplish this end, we try to roll to new files on the first + // transaction begin packet that occurs after we have exceeded our + // low threshold - which is why bDoNewIfOverLowLimit is only set to + // TRUE on transaction begin packets. It is set to FALSE on other + // packets so that we will continue logging the transaction in the + // same file that we started the transaction in - if possible. The + // only thing that will cause a non-transaction-begin packet to roll + // to a new file is if we would exceed the high limit. + + if ((bDoNewIfOverLowLimit && + m_pCurrentBuf->uiRflFileOffset + m_pCurrentBuf->uiRflBufBytes >= + m_uiRflMinFileSize) || + (m_pCurrentBuf->uiRflFileOffset + m_pCurrentBuf->uiRflBufBytes + + uiPacketLen >= m_uiRflMaxFileSize)) + { + FLMUINT uiCurrFileEOF = m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes; + + // Shift the current packet to the beginning of the buffer. + // Any packets in the buffer before that one will be written + // out to the current file. + + if (RC_BAD( rc = shiftPacketsDown( pDb, uiPacketLen, TRUE))) + { + goto Exit; + } + + // Update the header of the current file and close it. + + if (RC_BAD( rc = writeHeader( m_pCurrentBuf->uiCurrFileNum, + uiCurrFileEOF, + m_ucCurrSerialNum, m_ucNextSerialNum, TRUE))) + { + goto Exit; + } + + // Truncate the file. + + if (!ON_512_BYTE_BOUNDARY( uiCurrFileEOF)) + { + uiCurrFileEOF = ROUND_DOWN_TO_NEAREST_512( uiCurrFileEOF) + 512; + } + + if (RC_BAD( rc = m_pFileHdl->truncate( uiCurrFileEOF))) + { + goto Exit; + } + + // Close the file handle. + + m_pFileHdl->close(); + m_pFileHdl->Release(); + m_pFileHdl = NULL; + + // Get the next serial number that will be used for the RFL + // file after this one. + + if (RC_BAD( rc = f_createSerialNumber( ucNextSerialNum))) + { + goto Exit; + } + + // Create next file in the sequence. + + // Use the next serial number stored in the FDB's log header + // for the serial number on this RFL file. Use the serial + // number we just generated as the next RFL serial number. + + if (RC_BAD( rc = createFile( pDb, m_pCurrentBuf->uiCurrFileNum + 1, + m_ucNextSerialNum, ucNextSerialNum, TRUE))) + { + goto Exit; + } + + // Move the next serial number to the current serial number + // and the serial number we generated above into the next + // serial number. + + f_memcpy( m_ucCurrSerialNum, m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + f_memcpy( m_ucNextSerialNum, ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + } + +Exit: + return( rc); +} + +/******************************************************************** +Desc: Finish the current RFL file - set up so that next + transaction will begin a new RFL file. +*********************************************************************/ +RCODE F_Rfl::finishCurrFile( + F_Db * pDb, + FLMBOOL bNewKeepState) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bDbLocked = FALSE; + FLMUINT uiTransFileNum; + FLMUINT uiTransOffset; + FLMUINT uiTruncateSize; + SFLM_DB_HDR * pUncommittedDbHdr; + SFLM_DB_HDR checkpointDbHdr; + + // Make sure we don't have a transaction going + + if (pDb->m_eTransType != SFLM_NO_TRANS) + { + rc = RC_SET( NE_SFLM_TRANS_ACTIVE); + goto Exit; + } + + // Make sure there is no active backup running + + m_pDatabase->lockMutex(); + if (m_pDatabase->m_bBackupActive) + { + m_pDatabase->unlockMutex(); + rc = RC_SET( NE_SFLM_BACKUP_ACTIVE); + goto Exit; + } + m_pDatabase->unlockMutex(); + + // Lock the database - need to prevent update + // transactions and checkpoint thread from running. + + if (RC_BAD( rc = pDb->lockExclusive( SFLM_NO_TIMEOUT))) + { + goto Exit; + } + bDbLocked = TRUE; + + // Must wait for all RFL writes before switching files. + + (void)seeIfRflWritesDone( pDb->m_hWaitSem, TRUE); + + // Better not be in the middle of a transaction. + + flmAssert( !m_ui64CurrTransID); + + pUncommittedDbHdr = &m_pDatabase->m_uncommittedDbHdr; + + // Don't want to copy last committed log header into + // uncommitted log header if bNewKeepState is TRUE because + // the caller has already done it, and has made modifications + // to the uncommitted log header that we don't want to lose. + + if (!bNewKeepState) + { + f_memcpy( pUncommittedDbHdr, &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + + // If we are in a no-keep state, but we were not told that + // we have a new keep state, we cannot roll to the next + // RFL file, because a checkpoint has not been done. This is + // not an error - it is just the case where FlmDbConfig was + // asked to roll to the next RFL file when the keep flag was + // still FALSE. + + if (!pUncommittedDbHdr->ui8RflKeepFiles) + { + goto Exit; + } + } + + // Get the last committed serial numbers from the file's log header + // buffer. + + f_memcpy( m_ucCurrSerialNum, + pUncommittedDbHdr->ucLastTransRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_memcpy( m_ucNextSerialNum, + pUncommittedDbHdr->ucNextRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + uiTransFileNum = (FLMUINT)pUncommittedDbHdr->ui32RflCurrFileNum; + uiTransOffset = (FLMUINT)pUncommittedDbHdr->ui32RflLastTransOffset; + + // If ui32RflLastTransOffset is zero, there is no need to go set + // up to go to the next file, because we are already poised to do so at + // the beginning of the next transaction. Just return if this is the + // case. Same for if the file does not exist. + + if (!uiTransOffset) + { + if (!bNewKeepState) + { + goto Exit; + } + } + else if (RC_BAD( rc = openFile( pDb->m_hWaitSem, uiTransFileNum, + m_ucCurrSerialNum))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + if (!bNewKeepState) + { + goto Exit; + } + } + else + { + goto Exit; + } + } + else + { + + // At this point, we know the file exists, so we will update + // its header and then update the log header. Note that we + // use the keep RFL state from the last committed log header, + // not the uncommitted log header - because it will contain + // the correct keep-state for the current RFL file. + + if (RC_BAD( rc = writeHeader( m_pCurrentBuf->uiCurrFileNum, uiTransOffset, + m_ucCurrSerialNum, m_ucNextSerialNum, + m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles + ? TRUE + : FALSE))) + { + goto Exit; + } + + // Truncate the file down to its EOF size - the nearest 512 byte boundary. + + uiTruncateSize = uiTransOffset; + if (!ON_512_BYTE_BOUNDARY( uiTruncateSize)) + { + uiTruncateSize = ROUND_DOWN_TO_NEAREST_512( uiTruncateSize) + 512; + } + if (RC_BAD( rc = m_pFileHdl->truncate( uiTruncateSize))) + { + goto Exit; + } + + // Close the file handle. + + m_pFileHdl->close(); + m_pFileHdl->Release(); + m_pFileHdl = NULL; + + // Set things up in the log header to go to the next file when + // we begin the next transaction. NOTE: NO need to lock the + // mutex, because nobody but an update transaction looks at + // the uncommitted log header. + + uiTransFileNum++; + pUncommittedDbHdr->ui32RflCurrFileNum = (FLMUINT32)uiTransFileNum; + } + + // Generate a new current serial number if bNewKeepState is + // TRUE. Otherwise, move the next serial number into the current + // serial number. + + if (bNewKeepState) + { + if (RC_BAD( rc = f_createSerialNumber( m_ucCurrSerialNum))) + { + goto Exit; + } + } + else + { + f_memcpy( m_ucCurrSerialNum, m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + } + + // Always generate a new next serial number. + + if (RC_BAD( rc = f_createSerialNumber( m_ucNextSerialNum))) + { + goto Exit; + } + + // Set transaction offset to zero. This will force the + // next RFL file to be created on the next transaction + // begin. It will be created even if it is already + // there. + + pUncommittedDbHdr->ui32RflLastTransOffset = 0; + f_memcpy( pUncommittedDbHdr->ucLastTransRflSerialNum, + m_ucCurrSerialNum, SFLM_SERIAL_NUM_SIZE); + f_memcpy( pUncommittedDbHdr->ucNextRflSerialNum, + m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + + // Set the CP file number and CP offset to point into the new file. + // The outer code (FlmDbConfig) has done a checkpoint and the database + // is still locked. We need to set these values here, otherwise + // if we crash before the next checkpoint, recovery will start in the + // old RFL file, causing an NE_SFLM_BAD_RFL_SERIAL_NUM to be returned when + // traversing from the old RFL file to the new RFL file. + // NOTE: These changes must be made to the uncommitted log header AND + // the CP log header (so that they will be written out even + // though we are not forcing a checkpoint). + + if (bNewKeepState) + { + // Do a quick check to see if it looks like we are in a + // checkpointed state + + if( !m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles && + m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset > 512) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + + f_memcpy( &checkpointDbHdr, + &m_pDatabase->m_checkpointDbHdr, sizeof( SFLM_DB_HDR)); + checkpointDbHdr.ui32RflLastCPFileNum = (FLMUINT32)uiTransFileNum; + pUncommittedDbHdr->ui32RflLastCPFileNum = (FLMUINT32)uiTransFileNum; + checkpointDbHdr.ui32RflLastCPOffset = 512; + pUncommittedDbHdr->ui32RflLastCPOffset = 512; + } + + // Write out the db header to disk. + + if (RC_BAD( rc = m_pDatabase->writeDbHdr( pDb->m_pDbStats, pDb->m_pSFileHdl, + pUncommittedDbHdr, + bNewKeepState + ? &checkpointDbHdr + : &m_pDatabase->m_checkpointDbHdr, FALSE))) + { + goto Exit; + } + + // Copy the uncommitted log header back to the committed log header and + // copy the CP log header (if changed above). + + m_pDatabase->lockMutex(); + + f_memcpy( &m_pDatabase->m_lastCommittedDbHdr, pUncommittedDbHdr, + sizeof( SFLM_DB_HDR)); + + if( bNewKeepState) + { + f_memcpy( &m_pDatabase->m_checkpointDbHdr, &checkpointDbHdr, + sizeof( SFLM_DB_HDR)); + } + + m_pDatabase->unlockMutex(); + +Exit: + + if (bDbLocked) + { + (void)pDb->unlockExclusive(); + } + return( rc); +} + +/******************************************************************** +Desc: Finish packet by outputting header information for it. +*********************************************************************/ +RCODE F_Rfl::finishPacket( + F_Db * pDb, + FLMUINT uiPacketType, + FLMUINT uiPacketBodyLen, + FLMBOOL bDoNewIfOverLowLimit) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPacketLen; + FLMBYTE * pucPacket; + + uiPacketLen = uiPacketBodyLen + RFL_PACKET_OVERHEAD; + + // See if this packet will cause us to overflow the limits on + // the current file. If so, create a new file. + + if( RC_BAD( rc = seeIfNeedNewFile( pDb, uiPacketLen, bDoNewIfOverLowLimit))) + { + goto Exit; + } + + // Get a pointer to packet header. + + pucPacket = &(m_pCurrentBuf->pIOBuffer->getBuffer()[ + m_pCurrentBuf->uiRflBufBytes]); + + // Set the packet address in the packet header. + + m_uiPacketAddress = m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes; + UD2FBA( (FLMUINT32)m_uiPacketAddress, &pucPacket [RFL_PACKET_ADDRESS_OFFSET]); + + // Set the packet type and packet body length. + + pucPacket [RFL_PACKET_TYPE_OFFSET] = (FLMBYTE)uiPacketType; + + UW2FBA( (FLMUINT16)uiPacketBodyLen, + &pucPacket [RFL_PACKET_BODY_LENGTH_OFFSET]); + + // Set the checksum for the packet. + + pucPacket [RFL_PACKET_CHECKSUM_OFFSET] = RflCalcChecksum( pucPacket, + uiPacketBodyLen); + + // Increment bytes in the buffer to reflect the fact that this packet + // is now complete. + + m_pCurrentBuf->uiRflBufBytes += uiPacketLen; + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Truncate roll-forward log file to a certain size - only + do if not keeping RFL files. +*********************************************************************/ +RCODE F_Rfl::truncate( + F_SEM hWaitSem, + FLMUINT uiTruncateSize) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNum; + + flmAssert( uiTruncateSize >= 512); + + // Keeping of log files better not be enabled. + + flmAssert( !m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles); + + // Better not be in the middle of a transaction. + + flmAssert( !m_ui64CurrTransID); + + // Open the current RFL file. If it does not exist, it is OK - there + // is nothing to truncate. + + uiFileNum = (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflCurrFileNum; + if (RC_BAD( rc = openFile( hWaitSem, uiFileNum, + m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum))) + { + if (rc == NE_FLM_IO_PATH_NOT_FOUND || rc == NE_FLM_IO_INVALID_FILENAME) + { + rc = NE_SFLM_OK; + } + goto Exit; + } + + if (RC_BAD( rc = m_pFileHdl->truncate( uiTruncateSize))) + { + m_bRflVolumeOk = FALSE; + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Setup to begin a transaction +*********************************************************************/ +RCODE F_Rfl::setupTransaction( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFileNum; + FLMUINT uiLastTransOffset; + FLMBOOL bCreateFile; + + f_mutexLock( m_hBufMutex); + m_pCurrentBuf->bTransInProgress = TRUE; + f_mutexUnlock( m_hBufMutex); + + // Get the last committed serial numbers from the file's log header + // buffer. + + f_memcpy( m_ucCurrSerialNum, + m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + f_memcpy( m_ucNextSerialNum, + m_pDatabase->m_lastCommittedDbHdr.ucNextRflSerialNum, + SFLM_SERIAL_NUM_SIZE); + uiFileNum = (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflCurrFileNum; + uiLastTransOffset = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset; + + // If the LOG_RFL_LAST_TRANS_OFFSET is zero, we need to create the + // next RFL file number no matter what. There are two cases where + // this happens: 1) when the database is first created, and 2) after + // a restore operation. + + if (!uiLastTransOffset) + { + bCreateFile = TRUE; + + // Close the current file, just in case we had opened it before. + // At this point, it doesn't matter because we are going to + // overwrite it. + + if (RC_BAD( rc = waitForCommit( pDb->m_hWaitSem))) + { + goto Exit; + } + closeFile(); + } + else if (RC_BAD( rc = openFile( pDb->m_hWaitSem, + uiFileNum, m_ucCurrSerialNum))) + { + if (rc != NE_FLM_IO_PATH_NOT_FOUND && rc != NE_FLM_IO_INVALID_FILENAME) + { + goto Exit; + } + bCreateFile = TRUE; + } + else + { + bCreateFile = FALSE; + } + + if (bCreateFile) + { + + // If the log header indicates that data has already been logged + // to the file, we need to return the I/O error rather than just + // re-creating the file. This may mean that someone changed the + // RFL directory without moving the RFL files properly. + + if (uiLastTransOffset > 512) + { + rc = RC_SET( NE_SFLM_RFL_FILE_NOT_FOUND); + goto Exit; + } + + // Create the RFL file if not found. + + // Use the next serial number stored in the FDB's log header + // for the serial number on this RFL file. Use the serial + // number we just generated as the next RFL serial number. + + if (RC_BAD( rc = createFile( pDb, uiFileNum, + m_ucCurrSerialNum, m_ucNextSerialNum, + m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles + ? TRUE + : FALSE))) + { + goto Exit; + } + } + else + { + // Read in enough of the buffer from the RFL file so that + // we are positioned on a 512 byte boundary. + + if (RC_BAD( positionTo( uiLastTransOffset))) + { + goto Exit; + } + } + + // These can only be changed when starting a transaction. + + m_bKeepRflFiles = m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles + ? TRUE + : FALSE; + + m_uiRflMaxFileSize = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflMaxFileSize; + + // Round maximum down to nearest 512 boundary. This is necessary + // because we always write a minimum of 512 byte units in direct IO + // mode. If we did not round the maximum down, our last packet could + // end at an offset that is less than the maximum, but greater than + // the nearest 512 byte boundary - technically within the user-specified + // size limit. However, because we always write a full 512 bytes of data + // to fill out the last sector when we are in direct IO mode, we would + // end up with a file that was slightly larger than the user-specified + // limit. The EOF in the header of the file would be below the limit, + // but the actual file size would not be. Thus, the need to round down. + + m_uiRflMaxFileSize = ROUND_DOWN_TO_NEAREST_512( m_uiRflMaxFileSize); + + // The maximum cannot go below a certain threshold - must have room for + // least one packet plus the header. + + if (m_uiRflMaxFileSize < RFL_MAX_PACKET_SIZE + 512) + { + m_uiRflMaxFileSize = RFL_MAX_PACKET_SIZE + 512; + } + else if (m_uiRflMaxFileSize > gv_SFlmSysData.uiMaxFileSize) + { + m_uiRflMaxFileSize = gv_SFlmSysData.uiMaxFileSize; + } + + m_uiRflMinFileSize = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflMinFileSize; + + // Minimum RFL file size should not be allowed to be larger than + // maximum! + + if (m_uiRflMinFileSize > m_uiRflMaxFileSize) + { + m_uiRflMinFileSize = m_uiRflMaxFileSize; + } + + // Set the operation count to zero. + + m_uiOperCount = 0; + + m_pFileHdl->setMaxAutoExtendSize( m_uiRflMaxFileSize); + m_pFileHdl->setExtendSize( m_pDatabase->m_uiFileExtendSize); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Log transaction begin. This routine will also make sure + we have opened an RFL file. + NOTE: The prior version of FLAIM (before 4.3) would log + a time and set the RFL_TIME_LOGGED_FLAG bit in the packet + type. This is no longer done. Old code should be + compatible because it reads the flag. +*********************************************************************/ +RCODE F_Rfl::logBeginTransaction( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if( !isLoggingEnabled()) + { + goto Exit; + } + + flmAssert( !(pDb->m_uiFlags & FDB_REPLAYING_RFL)); + + // Better not be in the middle of a transaction. + + flmAssert( !m_ui64CurrTransID); + + if( RC_BAD( rc = setupTransaction( pDb))) + { + goto Exit; + } + + uiMaxPacketBodyLen = FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if (!haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if (RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID. + + f_encodeSEN( pDb->m_ui64CurrTransID, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if (RC_BAD( rc = finishPacket( + pDb, RFL_TRNS_BEGIN_PACKET, uiPacketBodyLen, TRUE))) + { + goto Exit; + } + + // Save the file offset for the start transaction packet. + + m_uiTransStartFile = m_pCurrentBuf->uiCurrFileNum; + m_uiTransStartAddr = m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes - + uiPacketBodyLen - RFL_PACKET_OVERHEAD; + m_ui64CurrTransID = pDb->m_ui64CurrTransID; + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Do a transaction begin operation during restore or recovery. +*********************************************************************/ +RCODE F_Rfl::recovTransBegin( + F_Db * pDb, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportBeginTrans( + peAction, m_ui64CurrTransID))) + { + goto Exit; + } + + if (*peAction == SFLM_RESTORE_ACTION_STOP) + { + // Need to set m_ui64CurrTransID to 0 since it was + // set by getPacket(). We are not going to + // start a transaction because of the user's request + // to exit. + + m_ui64CurrTransID = 0; + goto Exit; + } + } + + if (RC_BAD( rc = pDb->transBegin( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Flushes the RFL and sets some things in the log header. +*********************************************************************/ +void F_Rfl::finalizeTransaction( void) +{ + FLMUINT uiRflTransEndOffset; + SFLM_DB_HDR * pDbHdr = &m_pDatabase->m_uncommittedDbHdr; + + // Save the serial numbers and file numbers into the file's + // uncommitted log header. + + pDbHdr->ui32RflCurrFileNum = (FLMUINT32)m_pCurrentBuf->uiCurrFileNum; + + uiRflTransEndOffset = getCurrWriteOffset(); + pDbHdr->ui32RflLastTransOffset = (FLMUINT32)uiRflTransEndOffset; + + f_memcpy( pDbHdr->ucLastTransRflSerialNum, + m_ucCurrSerialNum, SFLM_SERIAL_NUM_SIZE); + + f_memcpy( pDbHdr->ucNextRflSerialNum, + m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); +} + +/******************************************************************** +Desc: Handles the commit and abort log operations. If aborting + the transaction, or if the transaction was empty, we will + simply throw away the entire transaction and not bother + to log it. In that case we will reset transaction pointers, + etc. back to the file and offset where the transaction began. + We will also delete RFL files that were created during the + transaction if necessary. NOTE: It is not essential that + the RFL files be deleted. If they are not successfully + deleted, they will be overwritten if need be when creating + new ones. + NOTE: The prior version of FLAIM (before 4.3) would log + a time and set the RFL_TIME_LOGGED_FLAG bit in the packet + type. This is no longer done. Old code should be + compatible because it reads the flag. +*********************************************************************/ +RCODE F_Rfl::logEndTransaction( + F_Db * pDb, + FLMUINT uiPacketType, + FLMBOOL bThrowLogAway, + FLMBOOL * pbLoggedTransEnd) +{ + RCODE rc = NE_SFLM_OK; + RCODE rc2 = NE_SFLM_OK; + FLMUINT uiLowFileNum; + FLMUINT uiHighFileNum; + char szRflFileName [F_PATH_MAX_SIZE]; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Initialize the "logged trans end" flag + + if( pbLoggedTransEnd) + { + *pbLoggedTransEnd = FALSE; + } + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + flmAssert( m_pFileHdl); + flmAssert( m_pDatabase); + + // If the transaction had no operations, throw it away - don't + // even log the packet. An abort operation may also + // elect to throw the log away even if there were + // operations. That is determined by the bThrowLogAway flag. + // The bThrowLogAway flag may be TRUE when doing a commit if + // the caller knows that nothing happened during the transction. + + if (bThrowLogAway || !m_uiOperCount) + { +Throw_Away_Transaction: + + // If we have switched files, delete all but the file we + // started in. + + if (m_pCurrentBuf->uiCurrFileNum != m_uiTransStartFile) + { + flmAssert( m_pCurrentBuf->uiCurrFileNum > m_uiTransStartFile); + + // File number in uncommitted log header better not + // have been changed yet. It is only supposed to + // be changed when the transaction finishes - i.e., in + // this routine. Up until this point, it should only + // be changed in m_pCurrentBuf->uiCurrFileNum. + + flmAssert( m_uiTransStartFile == + (FLMUINT)m_pDatabase->m_uncommittedDbHdr.ui32RflCurrFileNum); + + uiLowFileNum = m_uiTransStartFile + 1; + uiHighFileNum = m_pCurrentBuf->uiCurrFileNum; + + // Close the current file so it can be deleted. + + if (RC_BAD( rc = waitForCommit( pDb->m_hWaitSem))) + { + goto Exit; + } + closeFile(); + + // Delete as many of the files as possible. Don't worry + // about errors here. + + while (uiLowFileNum <= uiHighFileNum) + { + FLMUINT uiFileNameSize = sizeof( szRflFileName); + FLMBOOL bNameTruncated; + + getFullRflFileName( uiLowFileNum, szRflFileName, + &uiFileNameSize, &bNameTruncated); + if (!bNameTruncated) + { + (void)gv_SFlmSysData.pFileSystem->deleteFile( szRflFileName); + } + uiLowFileNum++; + } + } + else + { + // If we are in the file the transaction started in, simply + // reset to where the transaction started. + + if (RC_BAD( rc2 = positionTo( m_uiTransStartAddr))) + { + // If we got to this point because of a + // "goto Throw_Away_Transaction", we don't want to + // clobber the original error code. So, we use rc2 + // temporarily and then determine if its value should + // be set into rc. + + if( RC_OK( rc)) + { + rc = rc2; + } + rc2 = NE_SFLM_OK; + goto Exit; + } + } + } + else + { + // Log a commit or abort packet. + + uiMaxPacketBodyLen = FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if (RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Throw_Away_Transaction; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID. + + f_encodeSEN( m_ui64CurrTransID, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if (RC_BAD( rc = finishPacket( pDb, uiPacketType, + uiPacketBodyLen, FALSE))) + { + goto Throw_Away_Transaction; + } + + finalizeTransaction(); + + if( pbLoggedTransEnd) + { + *pbLoggedTransEnd = TRUE; + } + } + +Exit: + + m_ui64CurrTransID = 0; + return( RC_BAD( rc) ? rc : rc2); +} + +/******************************************************************** +Desc: Log a block chain free packet +*********************************************************************/ +RCODE F_Rfl::logBlockChainFree( + F_Db * pDb, + FLMUINT64 ui64MaintDocID, + FLMUINT uiStartBlkAddr, + FLMUINT uiEndBlkAddr, + FLMUINT uiCount) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + // Better be in the middle of a transaction. + + flmAssert( m_ui64CurrTransID); + + // Increment the operation count + + m_uiOperCount++; + + // Determine the maximum packet size + + uiMaxPacketBodyLen = FLM_MAX_SEN_LEN * 4; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the maintenance document ID + + f_encodeSEN( ui64MaintDocID, &pucPacketBody); + + // Output the starting block address + + f_encodeSEN( uiStartBlkAddr, &pucPacketBody); + + // Output the ending block address + + f_encodeSEN( uiEndBlkAddr, &pucPacketBody); + + // Output the block count + + f_encodeSEN( uiCount, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if( RC_BAD( rc = finishPacket( pDb, RFL_BLK_CHAIN_FREE_PACKET, + uiPacketBodyLen, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Free a chain of blocks +*********************************************************************/ +RCODE F_Rfl::recovBlockChainFree( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64MaintRowId; + FLMUINT uiStartBlkAddr; + FLMUINT uiEndBlkAddr; + FLMUINT uiCount; + FLMUINT uiBlocksFreed; + const FLMBYTE * pucEnd = pucPacketBody + uiPacketBodyLen; + + if( RC_BAD( rc = f_decodeSEN64( &pucPacketBody, pucEnd, &ui64MaintRowId))) + { + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiStartBlkAddr))) + { + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiEndBlkAddr))) + { + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiCount))) + { + goto Exit; + } + + if( pucPacketBody != pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportBlockChainFree( + peAction, m_ui64CurrTransID, ui64MaintRowId, uiStartBlkAddr, + uiEndBlkAddr, uiCount))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + goto Exit; + } + } + + if( RC_BAD( rc = pDb->maintBlockChainFree( + ui64MaintRowId, uiStartBlkAddr, uiCount, uiEndBlkAddr, &uiBlocksFreed))) + { + goto Exit; + } + + if( uiCount != uiBlocksFreed) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_DATA_ERROR); + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Log index suspend and resume packets +*********************************************************************/ +RCODE F_Rfl::logIndexSuspendOrResume( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMUINT uiPacketType) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + // Better be in the middle of a transaction. + + flmAssert( m_ui64CurrTransID); + + // Increment the operation count + + m_uiOperCount++; + + // Determine the maximum packet size + + uiMaxPacketBodyLen = FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the index number. + + flmAssert( uiIndexNum <= FLM_MAX_UINT16); + f_encodeSEN( uiIndexNum, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if (RC_BAD( rc = finishPacket( pDb, uiPacketType, uiPacketBodyLen, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Suspend or resume an index during restore or recovery. +*********************************************************************/ +RCODE F_Rfl::recovIndexSuspendResume( + F_Db * pDb, + FLMUINT uiPacketType, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiIndexNum; + const FLMBYTE * pucEnd = pucPacketBody + uiPacketBodyLen; + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiIndexNum))) + { + goto Exit; + } + + if( pucPacketBody != pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( m_pRestoreStatus) + { + if( uiPacketType == RFL_INDEX_SUSPEND_PACKET) + { + if( RC_BAD( rc = m_pRestoreStatus->reportIndexSuspend( + peAction, m_ui64CurrTransID, uiIndexNum))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = m_pRestoreStatus->reportIndexResume( + peAction, m_ui64CurrTransID, uiIndexNum))) + { + goto Exit; + } + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + goto Exit; + } + } + + if( uiPacketType == RFL_INDEX_SUSPEND_PACKET) + { + if( RC_BAD( rc = pDb->indexSuspend( uiIndexNum))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = pDb->indexResume( uiIndexNum))) + { + goto Exit; + } + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Log a reduce packet +*********************************************************************/ +RCODE F_Rfl::logReduce( + F_Db * pDb, + FLMUINT uiCount) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + // We need to set up to log this packet as if we + // were logging a transaction. The only difference + // is that we don't log the begin transaction packet. + + if( RC_BAD( rc = setupTransaction( pDb))) + { + goto Exit; + } + + uiMaxPacketBodyLen = 2 * FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID. + + f_encodeSEN( pDb->m_ui64CurrTransID, &pucPacketBody); + + // Output the count + + f_encodeSEN( uiCount, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if( RC_BAD( rc = finishPacket( pDb, RFL_REDUCE_PACKET, + uiPacketBodyLen, TRUE))) + { + goto Exit; + } + + // Finalize the transaction (as if we were committing a transaction) + + finalizeTransaction(); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Reduce the database during restore or recovery. +*********************************************************************/ +RCODE F_Rfl::recovReduce( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiCount; + const FLMBYTE * pucEnd = pucPacketBody + uiPacketBodyLen; + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiCount))) + { + goto Exit; + } + + if( pucPacketBody != pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportReduce( + peAction, m_ui64CurrTransID, uiCount))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + // Need to set m_ui64CurrTransID to 0 since it was + // set by getPacket(). We are not going to + // start a transaction because of the user's request + // to exit. + + m_ui64CurrTransID = 0; + goto Exit; + } + } + + if( RC_BAD( rc = pDb->reduceSize( uiCount, &uiCount))) + { + goto Exit; + } + +Exit: + + m_ui64CurrTransID = 0; + return( rc); +} + +/******************************************************************** +Desc: Log a database conversion packet +Note: This routine performs most of the setup for logging a full + transaction, but it does not cause begin and commit packets + to be logged. It is a "standalone" transaction. +*********************************************************************/ +RCODE F_Rfl::logUpgrade( + F_Db * pDb, + FLMUINT uiOldVersion) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + // We need to set up to log this packet as if we + // were logging a transaction. The only difference + // is that we don't log the begin transaction packet. + + if( RC_BAD( rc = setupTransaction( pDb))) + { + goto Exit; + } + + uiMaxPacketBodyLen = 3 * FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if (RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID + + f_encodeSEN( pDb->m_ui64CurrTransID, &pucPacketBody); + + // Output the old database version + + f_encodeSEN( uiOldVersion, &pucPacketBody); + + // Output the new database version + + f_encodeSEN( SFLM_CURRENT_VERSION_NUM, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if( RC_BAD( rc = finishPacket( pDb, RFL_UPGRADE_PACKET, + uiPacketBodyLen, TRUE))) + { + goto Exit; + } + + // Finalize the transaction (as if we were committing a transaction) + + finalizeTransaction(); + +Exit: + + m_ui64CurrTransID = 0; + return( rc); +} + +/******************************************************************** +Desc: Upgrade the database during restore or recovery. +*********************************************************************/ +RCODE F_Rfl::recovUpgrade( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOldDbVersion; + FLMUINT uiNewDbVersion; + + if( uiPacketBodyLen != 8) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + uiOldDbVersion = (FLMUINT)FB2UD( pucPacketBody); + pucPacketBody += 4; + + uiNewDbVersion = (FLMUINT)FB2UD( pucPacketBody); + pucPacketBody += 4; + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportUpgrade( + peAction, m_ui64CurrTransID, uiOldDbVersion, uiNewDbVersion))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + // Need to set m_ui64CurrTransID to 0 since it was + // set by getPacket(). We are not going to + // start a transaction because of the user's request + // to exit. + + m_ui64CurrTransID = 0; + goto Exit; + } + } + + // Attempt the conversion if the current version is less + // than the target version and the target version is + // less than or equal to the highest version supported + // by this code. + + if( uiNewDbVersion > SFLM_CURRENT_VERSION_NUM) + { + rc = RC_SET( NE_SFLM_UNALLOWED_UPGRADE); + goto Exit; + } + else if( (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32DbVersion < + uiNewDbVersion) + { + // The logged "new" version may be a lesser version + // than SFLM_CURRENT_VERSION_NUM, which is what FlmDbUpgrade + // upgrades to. This is O.K. because the current version + // should support all packets in the RFL for versions + // that are less than it. Otherwise, the RFL chain + // would have been broken by the original upgrade and it + // would not have logged an upgrade packet. + + if( RC_BAD( rc = pDb->upgrade( NULL))) + { + goto Exit; + } + } + +Exit: + + m_ui64CurrTransID = 0; + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::logEncryptionKey( + F_Db * pDb, + FLMUINT uiPacketType, + FLMBYTE * pucKey, + FLMUINT32 ui32KeyLen) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + flmAssert( uiPacketType == RFL_WRAP_KEY_PACKET || + uiPacketType == RFL_ENABLE_ENCRYPTION_PACKET); + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + if( RC_BAD( rc = setupTransaction( pDb))) + { + goto Exit; + } + + uiMaxPacketBodyLen = (2 * FLM_MAX_SEN_LEN) + ui32KeyLen; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID + + f_encodeSEN( pDb->m_ui64CurrTransID, &pucPacketBody); + + // Store the length of the key + + f_encodeSEN( ui32KeyLen, &pucPacketBody); + + // If we were built without encryption, the key length will be zero, + // so no need to store the key. + + if( ui32KeyLen) + { + f_memcpy( pucPacketBody, pucKey, ui32KeyLen); + pucPacketBody += ui32KeyLen; + } + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if( RC_BAD( rc = finishPacket( pDb, uiPacketType, + uiPacketBodyLen, TRUE))) + { + goto Exit; + } + + finalizeTransaction(); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::recovEncryptionKey( + F_Db * pDb, + FLMUINT uiPacketType, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiDBKeyLen; + FLMBOOL bTransStarted = FALSE; + SFLM_DB_HDR * pUncommittedLogHdr = &m_pDatabase->m_uncommittedDbHdr; + const FLMBYTE * pucEnd = pucPacketBody + uiPacketBodyLen; + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiDBKeyLen))) + { + goto Exit; + } + + if( pucPacketBody + uiDBKeyLen != pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( m_pRestoreStatus) + { + if( uiPacketType == RFL_WRAP_KEY_PACKET) + { + if( RC_BAD( rc = m_pRestoreStatus->reportWrapKey( + peAction, m_ui64CurrTransID))) + { + goto Exit; + } + } + else if( uiPacketType == RFL_ENABLE_ENCRYPTION_PACKET) + { + if( RC_BAD( rc = m_pRestoreStatus->reportEnableEncryption( + peAction, m_ui64CurrTransID))) + { + goto Exit; + } + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + m_ui64CurrTransID = 0; + goto Exit; + } + } + + if( uiDBKeyLen) + { + // We cannot directly set the key at this point as it may be + // encrypted using a password, which we do not have here. We will + // write the key out to the log header and trust the user to know whether + // or not a password is needed to open the database. + + if( RC_BAD(rc = pDb->transBegin( SFLM_UPDATE_TRANS))) + { + goto Exit; + } + + bTransStarted = TRUE; + + if( uiDBKeyLen > SFLM_MAX_ENC_KEY_SIZE) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_ENCKEY_SIZE); + goto Exit; + } + + f_memcpy( pUncommittedLogHdr->ucDbKey, pucPacketBody, uiDBKeyLen); + pUncommittedLogHdr->ui32DbKeyLen = (FLMUINT32)uiDBKeyLen; + + pDb->m_bHadUpdOper = TRUE; + + if( RC_BAD( rc = pDb->commitTrans( 0, TRUE))) + { + goto Exit; + } + + bTransStarted = FALSE; + } + +Exit: + + if( bTransStarted) + { + RCODE tmpRc; + + if( RC_BAD( tmpRc = pDb->commitTrans( 0, TRUE))) + { + pDb->abortTrans(); + + if (RC_OK( rc)) + { + rc = tmpRc; + } + } + } + + m_ui64CurrTransID = 0; + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::logEncDefKey( + F_Db * pDb, + FLMUINT uiEncDefId, + FLMBYTE * pucKeyValue, + FLMUINT uiKeyValueLen, + FLMUINT uiKeySize) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + // Better be in the middle of a transaction. + + flmAssert( m_ui64CurrTransID); + + // Increment the operation count + + m_uiOperCount++; + + // Determine the maximum packet size + + uiMaxPacketBodyLen = (3 * FLM_MAX_SEN_LEN) + uiKeyValueLen; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the encryption definition ID + + f_encodeSEN( uiEncDefId, &pucPacketBody); + + // Output the key size (number of bits) + + f_encodeSEN( uiKeySize, &pucPacketBody); + + // Output the key value length + + f_encodeSEN( uiKeyValueLen, &pucPacketBody); + + // Output the key + + f_memcpy( pucPacketBody, pucKeyValue, uiKeyValueLen); + pucPacketBody += uiKeyValueLen; + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if (RC_BAD( rc = finishPacket( pDb, RFL_ENC_DEF_KEY_PACKET, + uiPacketBodyLen, FALSE))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::recovEncDefKey( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction *) // peAction +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64RowId; + FLMUINT uiKeySize; + FLMUINT uiKeyValueLen; + const FLMBYTE * pucEnd = pucPacketBody + uiPacketBodyLen; + + if( RC_BAD( rc = f_decodeSEN64( &pucPacketBody, pucEnd, &ui64RowId))) + { + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiKeySize))) + { + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN( &pucPacketBody, pucEnd, &uiKeyValueLen))) + { + goto Exit; + } + + if( pucPacketBody + uiKeyValueLen != pucEnd) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + // Retrieve the encryption definition + +//VISIT: NEED TO FLESH THIS OUT + +#if 0 + if( RC_BAD( rc = pDb->getNode( XFLM_DICT_COLLECTION, + ui64RootId, FLM_EXACT, &pNode))) + { + if( rc == NE_SFLM_ROW_NOT_FOUND) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + } + + goto Exit; + } + + // Set the key in the DOM node as a binary string. + + if( RC_BAD( rc = pNode->createAttribute( pDb, ATTR_ENCRYPTION_KEY_TAG, + (IF_DOMNode **)&pAttr))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->removeModeFlags( pDb, + FDOM_READ_ONLY | FDOM_CANNOT_DELETE))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->setBinary( pDb, pucPacketBody, uiKeyValueLen))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->addModeFlags( pDb, + FDOM_READ_ONLY | FDOM_CANNOT_DELETE))) + { + goto Exit; + } + + // Set the key size + + if( RC_BAD( rc = pNode->createAttribute( pDb, ATTR_ENCRYPTION_KEY_SIZE_TAG, + (IF_DOMNode **)&pAttr))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->removeModeFlags( pDb, + FDOM_READ_ONLY | FDOM_CANNOT_DELETE))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->setUINT( pDb, uiKeySize))) + { + goto Exit; + } + + if( RC_BAD( rc = pAttr->addModeFlags( pDb, + FDOM_READ_ONLY | FDOM_CANNOT_DELETE))) + { + goto Exit; + } +#endif + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::logRollOverDbKey( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacketStart; + FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen; + FLMUINT uiMaxPacketBodyLen; + + flmAssert( pDb->m_uiFlags & FDB_HAS_FILE_LOCK); +#ifndef FLM_DEBUG + F_UNREFERENCED_PARM( pDb); +#endif + + // Do nothing if logging is disabled. + + if (!isLoggingEnabled()) + { + goto Exit; + } + + if( RC_BAD( rc = setupTransaction( pDb))) + { + goto Exit; + } + + uiMaxPacketBodyLen = FLM_MAX_SEN_LEN; + + // Make sure we have space in the RFL buffer for a complete packet. + + if( !haveBuffSpace( uiMaxPacketBodyLen + RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + } + + // Get a pointer to where we will be laying down the packet body. + + pucPacketBody = pucPacketStart = getPacketBodyPtr(); + + // Output the transaction ID + + f_encodeSEN( pDb->m_ui64CurrTransID, &pucPacketBody); + + // Finish the packet + + uiPacketBodyLen = (FLMUINT)(pucPacketBody - pucPacketStart); + flmAssert( uiPacketBodyLen <= uiMaxPacketBodyLen); + + if( RC_BAD( rc = finishPacket( pDb, RFL_ROLL_OVER_DB_KEY_PACKET, + uiPacketBodyLen, TRUE))) + { + goto Exit; + } + + finalizeTransaction(); + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_Rfl::recovRollOverDbKey( + F_Db * pDb, + const FLMBYTE *, // pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction) +{ + RCODE rc = NE_SFLM_OK; + + if( uiPacketBodyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportRollOverDbKey( + peAction, m_ui64CurrTransID))) + { + goto Exit; + } + + if( *peAction == SFLM_RESTORE_ACTION_STOP) + { + m_ui64CurrTransID = 0; + goto Exit; + } + } + + if( RC_BAD( rc = pDb->rollOverDbKey())) + { + goto Exit; + } + +Exit: + + m_ui64CurrTransID = 0; + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_RflOStream::write( + const void * pvBuffer, + FLMUINT uiBytesToWrite, + FLMUINT * puiBytesWritten) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPacketLen = RFL_PACKET_OVERHEAD; + FLMUINT uiBytesAvail; + FLMBYTE * pucBuffer = (FLMBYTE *)pvBuffer; + + flmAssert( m_pRfl->isLoggingEnabled()); + + if( !m_pRfl->haveBuffSpace( RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = m_pRfl->flush( m_pDb, m_pRfl->m_pCurrentBuf))) + { + goto Exit; + } + } + + while( uiBytesToWrite) + { + if( RC_BAD( rc = m_pRfl->makeRoom( m_pDb, uiBytesToWrite, + &uiPacketLen, RFL_DATA_PACKET, &uiBytesAvail, NULL))) + { + goto Exit; + } + + f_memcpy( m_pRfl->getPacketPtr() + uiPacketLen, pucBuffer, uiBytesAvail); + + pucBuffer += uiBytesAvail; + uiBytesToWrite -= uiBytesAvail; + uiPacketLen += uiBytesAvail; + + if( RC_BAD( rc = m_pRfl->finishPacket( m_pDb, RFL_DATA_PACKET, + uiPacketLen - RFL_PACKET_OVERHEAD, FALSE))) + { + goto Exit; + } + + uiPacketLen = RFL_PACKET_OVERHEAD; + } + + if( puiBytesWritten) + { + *puiBytesWritten = (FLMUINT)(pucBuffer - ((FLMBYTE *)pvBuffer)); + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: +*********************************************************************/ +RCODE F_RflOStream::write( + IF_PosIStream * pIStream) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiPacketLen = RFL_PACKET_OVERHEAD; + FLMUINT uiBytesToWrite = (FLMUINT)pIStream->remainingSize(); + FLMUINT uiBytesAvail; + + flmAssert( m_pRfl->isLoggingEnabled()); + + if( !m_pRfl->haveBuffSpace( RFL_PACKET_OVERHEAD)) + { + if( RC_BAD( rc = m_pRfl->flush( m_pDb, m_pRfl->m_pCurrentBuf))) + { + goto Exit; + } + } + + while( uiBytesToWrite) + { + if( RC_BAD( rc = m_pRfl->makeRoom( m_pDb, uiBytesToWrite, + &uiPacketLen, RFL_DATA_PACKET, &uiBytesAvail, NULL))) + { + goto Exit; + } + + if( RC_BAD( rc = pIStream->read( m_pRfl->getPacketPtr()+ uiPacketLen, + uiBytesAvail))) + { + goto Exit; + } + + uiBytesToWrite -= uiBytesAvail; + uiPacketLen += uiBytesAvail; + + if( RC_BAD( rc = m_pRfl->finishPacket( m_pDb, RFL_DATA_PACKET, + uiPacketLen - RFL_PACKET_OVERHEAD, FALSE))) + { + goto Exit; + } + + uiPacketLen = RFL_PACKET_OVERHEAD; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Make room in the RFL buffer for the additional bytes. + This is done by flushing the log buffer and shifting down + the bytes already used in the current packet. If that + doesn't make room, the current packet will be finished and + a new one started. +*********************************************************************/ +RCODE F_Rfl::makeRoom( + F_Db * pDb, + FLMUINT uiAdditionalBytesNeeded, + FLMUINT * puiCurrPacketLenRV, + FLMUINT uiPacketType, + FLMUINT * puiBytesAvailableRV, + FLMUINT * puiPacketCountRV) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesNeeded; + + uiBytesNeeded = *puiCurrPacketLenRV + uiAdditionalBytesNeeded; + if( uiBytesNeeded <= (FLMUINT)RFL_MAX_PACKET_SIZE) + { + FLMUINT uiTmp = uiBytesNeeded; + + if( haveBuffSpace( uiTmp)) + { + if( puiBytesAvailableRV) + { + *puiBytesAvailableRV = uiAdditionalBytesNeeded; + } + } + else + { + // Bytes requested will fit into a packet, but not the + // buffer, so we need to shift the packets in the buffer + // down. The shiftPacketsDown guarantees that there + // is room in the buffer for a full size packet. + + if( RC_BAD( rc = shiftPacketsDown( pDb, *puiCurrPacketLenRV, FALSE))) + { + goto Exit; + } + + // If a non-NULL puwBytesAvailableRV is passed in it means that we + // are to return the number of bytes that we can actually output. + // Since we know there is enough for the bytes needed, we simply return + // the number of bytes that were requested. + + if( puiBytesAvailableRV) + { + *puiBytesAvailableRV = uiAdditionalBytesNeeded; + } + } + } + else // (uiBytesNeeded > RFL_MAX_PACKET_SIZE) + { + // This is the case where the bytes needed would overflow the + // maximum packet size. + // If puwBytesAvailableRV is NULL, it means that all of the + // requested additional bytes must fit into the packet. In that + // case, since the requested bytes would put us over the packet + // size limit, we must finish the current packet and then + // flush the packets out of the buffer so we can start a + // new packet. + + if( !puiBytesAvailableRV) + { + // Finish the current packet and start a new one. + + if( puiPacketCountRV) + { + (*puiPacketCountRV)++; + } + + if( RC_BAD( rc = finishPacket( pDb, uiPacketType, + *puiCurrPacketLenRV - RFL_PACKET_OVERHEAD, + FALSE))) + { + goto Exit; + } + + if( RC_BAD( rc = flush( pDb, m_pCurrentBuf))) + { + goto Exit; + } + + *puiCurrPacketLenRV = RFL_PACKET_OVERHEAD; + } + else + { + // When puiBytesAvailableRV is non-NULL, it means we can fill up + // the rest of the packet with part of the bytes. In this case + // we return the number of bytes available and then shift the + // packets down in the buffer to make sure there is room for + // a full-size packet. + + *puiBytesAvailableRV = RFL_MAX_PACKET_SIZE - *puiCurrPacketLenRV; + if( RC_BAD( rc = shiftPacketsDown( pDb, *puiCurrPacketLenRV, FALSE))) + { + goto Exit; + } + } + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Reads a full packet, based on what file offset and read + offset are currently set to. +*********************************************************************/ +RCODE F_Rfl::readPacket( + FLMUINT uiMinBytesNeeded) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiTmpOffset; + FLMUINT uiReadLen; + FLMUINT uiBytesRead; + + // If we have enough bytes in the buffer for the minimum bytes + // needed, we don't need to retrieve any more bytes. + + if( m_pCurrentBuf->uiRflBufBytes && + m_pCurrentBuf->uiRflBufBytes >= m_uiRflReadOffset && + m_pCurrentBuf->uiRflBufBytes - m_uiRflReadOffset >= uiMinBytesNeeded) + { + goto Exit; + } + + // If we are doing restore, we have to do only sequential + // reads - cannot depend on doing reads on 512 byte boundaries. + // Otherwise, we read directly from disk on 512 byte boundaries. + + if( m_pRestore) + { + FLMUINT uiCurrFilePos = m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes; + + if( m_uiRflReadOffset > 0 && + m_pCurrentBuf->uiRflBufBytes >= m_uiRflReadOffset) + { + // Move the bytes left in the buffer down to the beginning + // of the buffer. + + f_memmove( m_pCurrentBuf->pIOBuffer->getBuffer(), + &(m_pCurrentBuf->pIOBuffer->getBuffer()[ m_uiRflReadOffset]), + m_pCurrentBuf->uiRflBufBytes - m_uiRflReadOffset); + m_pCurrentBuf->uiRflBufBytes -= m_uiRflReadOffset; + m_pCurrentBuf->uiRflFileOffset += m_uiRflReadOffset; + m_uiRflReadOffset = 0; + } + uiReadLen = m_uiBufferSize - m_pCurrentBuf->uiRflBufBytes; + + // Read enough to fill the rest of the buffer, which is + // guaranteed to hold at least one full packet. + + if( !m_uiFileEOF) + { + if( uiCurrFilePos > (FLMUINT)(-1) - uiReadLen) + { + uiReadLen = (FLMUINT)(-1) - uiCurrFilePos; + } + } + else + { + if( uiCurrFilePos + uiReadLen > m_uiFileEOF) + { + uiReadLen = m_uiFileEOF - uiCurrFilePos; + } + } + + // If reading will not give us the minimum bytes needed, + // we cannot satisfy this request from the current file. + + if( uiReadLen + m_pCurrentBuf->uiRflBufBytes < uiMinBytesNeeded) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + // Read enough to get the entire packet. + + if( RC_BAD( rc = m_pRestore->read( uiReadLen, + &(m_pCurrentBuf->pIOBuffer->getBuffer()[ + m_pCurrentBuf->uiRflBufBytes]), &uiBytesRead))) + { + if( rc == NE_FLM_IO_END_OF_FILE) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + + if( m_pRestoreStatus) + { + eRestoreAction eAction; + + if( RC_BAD( rc = m_pRestoreStatus->reportRflRead( + &eAction, m_pCurrentBuf->uiCurrFileNum, uiBytesRead))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + // If we didn't read enough to satisfy the minimum bytes needed, + // we cannot satisfy this request from the current file. + + if( uiBytesRead + m_pCurrentBuf->uiRflBufBytes < uiMinBytesNeeded) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + m_pCurrentBuf->uiRflBufBytes += uiBytesRead; + } + else + { + // Set offsets so we are on a 512 byte boundary for our + // next read. No need to move data, since we will be + // re-reading it anyway. + + if( m_uiRflReadOffset > 0) + { + uiTmpOffset = m_uiRflReadOffset - (m_uiRflReadOffset & 511); + m_pCurrentBuf->uiRflFileOffset += uiTmpOffset; + m_uiRflReadOffset -= uiTmpOffset; + } + else if( m_pCurrentBuf->uiRflFileOffset & 511) + { + m_uiRflReadOffset = m_pCurrentBuf->uiRflFileOffset & 511; + m_pCurrentBuf->uiRflFileOffset -= m_uiRflReadOffset; + } + + m_pCurrentBuf->uiRflBufBytes = 0; + + // Read enough to fill the rest of the buffer, which is + // guaranteed to hold at least one full packet. + + uiReadLen = m_uiBufferSize; + + // m_uiFileEOF better not be zero at this point - we should + // always know precisely where the RFL file ends when we + // are doing recovery as opposed to doing a restore. + + flmAssert( m_uiFileEOF >= 512); + if( m_pCurrentBuf->uiRflFileOffset + uiReadLen > m_uiFileEOF) + { + uiReadLen = m_uiFileEOF - m_pCurrentBuf->uiRflFileOffset; + } + + // If reading will not give us the minimum number of bytes + // needed, we have a bad packet. + + if( uiReadLen < m_uiRflReadOffset || + uiReadLen - m_uiRflReadOffset < uiMinBytesNeeded) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + // Read to get the entire packet. + + if( RC_BAD( rc = m_pFileHdl->sectorRead( m_pCurrentBuf->uiRflFileOffset, + uiReadLen, m_pCurrentBuf->pIOBuffer->getBuffer(), + &uiBytesRead))) + { + if( rc == NE_FLM_IO_END_OF_FILE) + { + rc = NE_SFLM_OK; + } + else + { + m_bRflVolumeOk = FALSE; + goto Exit; + } + } + + if( uiBytesRead < uiReadLen) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + m_pCurrentBuf->uiRflBufBytes = uiReadLen; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Gets and verifies the next packet from the roll-forward + log file. Packet checksum will be verified. +*********************************************************************/ +RCODE F_Rfl::getPacket( + F_Db * pDb, + FLMBOOL bForceNextFile, + FLMUINT * puiPacketTypeRV, + const FLMBYTE ** ppucPacketBodyRV, + FLMUINT * puiPacketBodyLenRV, + FLMBOOL * pbHitEnd) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucPacket; + const FLMBYTE * pucPacketBody; + const FLMBYTE * pucPacketBodyEnd; + FLMUINT uiOrigPacketBodyLen; + FLMUINT uiPacketBodyLen; + FLMUINT uiPacketType; + FLMBYTE ucHdr [512]; + FLMUINT uiBytesRead; + + // See if we need to go to the next file. Note that we only + // check for this exactly on packet boundaries. We do not expect + // packets to be split across files. If we are not at the end + // of processing what is in the buffer, we should be able to + // read the rest of the packet from the current file. + +Get_Next_File: + + if( bForceNextFile || + (m_uiFileEOF && m_uiRflReadOffset == m_pCurrentBuf->uiRflBufBytes && + m_pCurrentBuf->uiRflFileOffset + m_pCurrentBuf->uiRflBufBytes == + m_uiFileEOF)) + { + if( m_bKeepRflFiles) + { + if( !m_pRestore) + { + // Only doing recovery after a failure, see if we are at + // the last file already. + + if( m_pCurrentBuf->uiCurrFileNum == m_uiLastRecoverFileNum) + { + *pbHitEnd = TRUE; + goto Exit; + } + else if( (m_pCurrentBuf->uiCurrFileNum + 1 ) == + m_uiLastRecoverFileNum && + !m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset) + { + // We are going to try to open the last file. Since the log + // header shows a current offset of 0, the file may have been + // created but nothing was logged to it. We don't want to try + // to open it here because it may not have been initialized + // fully at the time of the server crash. + + m_pCurrentBuf->uiCurrFileNum = m_uiLastRecoverFileNum; + *pbHitEnd = TRUE; + goto Exit; + } + + // Open the next file in the sequence. + + if( RC_BAD( rc = openFile( pDb->m_hWaitSem, + m_pCurrentBuf->uiCurrFileNum + 1, m_ucNextSerialNum))) + { + if( rc == NE_FLM_IO_PATH_NOT_FOUND) + { + rc = NE_SFLM_OK; + *pbHitEnd = TRUE; + } + + goto Exit; + } + + // If this is the last RFL file, the EOF is contained + // in the log header. Otherwise, it will be in the RFL + // file's header, and openFile will already have retrieved it. + + if( m_pCurrentBuf->uiCurrFileNum == m_uiLastRecoverFileNum) + { + m_uiFileEOF = + m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset; + + // Could be zero if RFL file wasn't created yet. + + if( !m_uiFileEOF) + { + m_uiFileEOF = 512; + } + } + + // By this point, the EOF better be greater than or equal to 512. + + flmAssert( m_uiFileEOF >= 512); + } + else + { + if( RC_BAD( rc = m_pRestore->close())) + { + goto Exit; + } + + // Ask the recovery object to open the file. + + if( RC_BAD( rc = m_pRestore->openRflFile( + m_pCurrentBuf->uiCurrFileNum + 1))) + { + if( rc == NE_FLM_IO_PATH_NOT_FOUND) + { + rc = NE_SFLM_OK; + *pbHitEnd = TRUE; + } + + goto Exit; + } + + if( m_pRestoreStatus) + { + eRestoreAction eAction; + + if( RC_BAD( rc = m_pRestoreStatus->reportOpenRflFile( + &eAction, m_pCurrentBuf->uiCurrFileNum + 1))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + // Get the first 512 bytes from the file and verify the header. + + if( RC_BAD( rc = m_pRestore->read( 512, ucHdr, &uiBytesRead))) + { + goto Exit; + } + + if( m_pRestoreStatus) + { + eRestoreAction eAction; + + if( RC_BAD( rc = m_pRestoreStatus->reportRflRead( + &eAction, m_pCurrentBuf->uiCurrFileNum + 1, uiBytesRead))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + if( uiBytesRead < 512) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + + if( RC_BAD( rc = verifyHeader( ucHdr, + m_pCurrentBuf->uiCurrFileNum + 1, m_ucNextSerialNum))) + { + goto Exit; + } + + // We may not know the actual EOF of files during restore + // operations. m_uiFileEOF could be zero here. + + m_uiFileEOF = (FLMUINT)FB2UD( &ucHdr[ RFL_EOF_POS]); + + // File EOF may be zero or >= 512 at this point. + + flmAssert( !m_uiFileEOF || m_uiFileEOF >= 512); + + // Need to increment current file number. + + m_pCurrentBuf->uiCurrFileNum++; + } + + m_pCurrentBuf->uiRflFileOffset = 512; + m_uiRflReadOffset = 0; + m_pCurrentBuf->uiRflBufBytes = 0; + + // Get the next packet from the new file. + + if( RC_BAD( rc = readPacket( RFL_PACKET_OVERHEAD))) + { + if( m_uiFileEOF == 512 && m_bKeepRflFiles) + { + // File was empty, try to go to the next file. + + bForceNextFile = TRUE; + goto Get_Next_File; + } + goto Exit; + } + } + else + { + // This is the case where we are not keeping the RFL files. + // So, there is no next file to go to. If we get to this + // point, we had better not be doing a restore. + + flmAssert( m_pRestore == NULL && !bForceNextFile); + *pbHitEnd = TRUE; + goto Exit; + } + } + + // Make sure we at least have a packet header in the buffer. + + if (RC_BAD( rc = readPacket( RFL_PACKET_OVERHEAD))) + { + goto Exit; + } + + // Verify the packet address. + + m_uiPacketAddress = m_pCurrentBuf->uiRflFileOffset + m_uiRflReadOffset; + pucPacket = &(m_pCurrentBuf->pIOBuffer->getBuffer()[m_uiRflReadOffset]); + if ((FLMUINT)FB2UD( &pucPacket [RFL_PACKET_ADDRESS_OFFSET]) != + m_uiPacketAddress) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + // Get packet type, time flag, and packet body length + + *puiPacketTypeRV = uiPacketType = + RFL_GET_PACKET_TYPE( pucPacket [RFL_PACKET_TYPE_OFFSET]); + + uiPacketBodyLen = uiOrigPacketBodyLen = + (FLMUINT)FB2UW( &pucPacket [RFL_PACKET_BODY_LENGTH_OFFSET]); + + // Make sure we have the entire packet in the buffer. + + if (RC_BAD( rc = readPacket( uiPacketBodyLen + RFL_PACKET_OVERHEAD))) + { + goto Exit; + } + pucPacket = &(m_pCurrentBuf->pIOBuffer->getBuffer()[m_uiRflReadOffset]); + + // At this point, we are guaranteed to have the entire packet + // in the buffer. + + pucPacketBody = &pucPacket [RFL_PACKET_OVERHEAD]; + pucPacketBodyEnd = pucPacketBody + uiPacketBodyLen; + + // Validate the packet checksum + + if (RflCalcChecksum( pucPacket, uiPacketBodyLen) != + pucPacket [RFL_PACKET_CHECKSUM_OFFSET]) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if (uiPacketType == RFL_TRNS_BEGIN_PACKET || + uiPacketType == RFL_UPGRADE_PACKET || + uiPacketType == RFL_REDUCE_PACKET || + uiPacketType == RFL_WRAP_KEY_PACKET || + uiPacketType == RFL_ENABLE_ENCRYPTION_PACKET || + uiPacketType == RFL_ROLL_OVER_DB_KEY_PACKET) + { + // Current transaction ID better be zero, otherwise, we + // have two or more begin packets in a row. + + if( m_ui64CurrTransID) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( RC_BAD( rc = f_decodeSEN64( &pucPacketBody, + pucPacketBodyEnd, &m_ui64CurrTransID))) + { + goto Exit; + } + + uiPacketBodyLen = (FLMUINT)(pucPacketBodyEnd - pucPacketBody); + + // Make sure the transaction numbers are ascending + + if( m_ui64CurrTransID <= m_ui64LastTransID) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + } + else + { + // If transaction ID is not zero, we are not inside a + // transaction, and it is likely that we have a corrupt + // packet. + + if( !m_ui64CurrTransID) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( uiPacketType == RFL_TRNS_COMMIT_PACKET || + uiPacketType == RFL_TRNS_ABORT_PACKET) + { + FLMUINT64 ui64Tmp; + + if( RC_BAD( rc = f_decodeSEN64( &pucPacketBody, + pucPacketBodyEnd, &ui64Tmp))) + { + goto Exit; + } + + if( ui64Tmp != m_ui64CurrTransID) + { + rc = RC_SET( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + uiPacketBodyLen = (FLMUINT)(pucPacketBodyEnd - pucPacketBody); + } + } + + // Set read offset to beginning of next packet. + + m_uiRflReadOffset += (RFL_PACKET_OVERHEAD + uiOrigPacketBodyLen); + *puiPacketBodyLenRV = uiPacketBodyLen; + *ppucPacketBodyRV = pucPacketBody; + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: Restore transactions from the roll-forward log to the + database. +*********************************************************************/ +RCODE F_Rfl::recover( + F_Db * pDb, + IF_RestoreClient * pRestore, + IF_RestoreStatus * pRestoreStatus) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiStartFileNum; + FLMUINT uiStartOffset; + FLMUINT uiOffset; + FLMUINT uiReadLen; + FLMUINT uiBytesRead; + FLMBYTE ucHdr[ 512]; + FLMUINT uiPacketType; + const FLMBYTE * pucPacketBody; + FLMUINT uiPacketBodyLen = 0; + eRestoreAction eAction; + FLMBOOL bTransActive = FALSE; + FLMBOOL bHadOperations = FALSE; + FLMBOOL bLastTransEndedAtFileEOF = FALSE; + FLMBOOL bForceNextFile; + FLMBOOL bHitEnd; + + flmAssert( m_pDatabase); + + m_pCurrentBuf = &m_Buf1; + m_ui64LastLoggedCommitTransID = 0; + + // We need to allow all updates logged in the RFL + // (including dictionary updates). + + pDb->m_bItemStateUpdOk = TRUE; + + // Turn off logging. + + disableLogging(); + + // Set the replay flag on the database. + + pDb->m_uiFlags |= FDB_REPLAYING_RFL; + + // Set the flag as to whether or not we are using multiple RFL files. + + m_bKeepRflFiles = m_pDatabase->m_lastCommittedDbHdr.ui8RflKeepFiles + ? TRUE + : FALSE; + + // If pRestore is NULL, we are doing a database recovery after + // open. In that case, we start from the last checkpoint offset + // and only run until the last transaction offset. + + m_pRestoreStatus = pRestoreStatus; + + if( (m_pRestore = pRestore) == NULL) + { + FLMBYTE * pucCheckSerialNum; + FLMUINT uiEndOffset; + + // Clear the restore status pointer. Assert that the + // pointer is NULL so we can catch the improper use + // of this object. + + if( m_pRestoreStatus) + { + flmAssert( 0); + m_pRestoreStatus = NULL; + } + + uiStartFileNum = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastCPFileNum; + m_uiLastRecoverFileNum = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflCurrFileNum; + uiStartOffset = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastCPOffset; + uiEndOffset = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset; + + // Could be zero if the file was created, but no transactions were + // ever committed to it. + + if( !uiEndOffset) + { + uiEndOffset = 512; + } + + // Start offset better not be less than 512. + + flmAssert( uiStartOffset >= 512); + flmAssert( uiEndOffset >= 512); + + // If start and end are at the same place, there is nothing + // to recover. + + if( uiStartFileNum == m_uiLastRecoverFileNum && + uiStartOffset == uiEndOffset) + { + goto Finish_Recovery; + } + + // We have not recorded the serial number of the last checkpoint file + // number, so we pass in NULL, unless it happens to be the same as the + // last transaction file number, in which case we can pass in the + // serial number we have stored in the log header. + + pucCheckSerialNum = + (uiStartFileNum == m_uiLastRecoverFileNum) + ? &m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum [0] + : NULL; + + if( RC_BAD( rc = openFile( pDb->m_hWaitSem, + uiStartFileNum, pucCheckSerialNum))) + { + goto Exit; + } + + // If this is the last RFL file, the EOF is contained + // in the log header. Otherwise, it will be in the RFL + // file's header, and openFile will already have retrieved it. + + if( uiStartFileNum == m_uiLastRecoverFileNum) + { + m_uiFileEOF = uiEndOffset; + } + + // At this point, file EOF better be greater than or equal to 512. + + flmAssert( m_uiFileEOF >= 512); + } + else if( !m_bKeepRflFiles) + { + // FlmDbRestore should be checking the "keep" flag and not + // attempting to do a restore of the RFL. + + rc = RC_SET_AND_ASSERT( NE_SFLM_CANNOT_RESTORE_RFL_FILES); + goto Exit; + } + else + { + uiStartFileNum = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflCurrFileNum; + uiStartOffset = + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset; + + // Could be zero if the RFL file had never been created. + + if( !uiStartOffset) + { + uiStartOffset = 512; + } + + // Ask the recovery object to open the file. + +Retry_Open: + + flmAssert( uiStartFileNum); + if( RC_BAD( rc = m_pRestore->openRflFile( uiStartFileNum))) + { + if( rc == NE_FLM_IO_PATH_NOT_FOUND) + { + // Need to set m_pCurrentBuf->uiCurrFileNum in case the first + // call to openRflFile fails. This will cause the code at the + // Finish_Recovery label to correctly set up the log + // header. + + if( !uiStartOffset) + { + m_pCurrentBuf->uiCurrFileNum = uiStartFileNum - 1; + } + else + { + m_pCurrentBuf->uiCurrFileNum = uiStartFileNum; + } + + rc = NE_SFLM_OK; + goto Finish_Recovery; + } + else + { + goto Exit; + } + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportOpenRflFile( + &eAction, uiStartFileNum))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + // Get the first 512 bytes from the file and verify the header. + + if( RC_BAD( rc = m_pRestore->read( 512, ucHdr, &uiBytesRead))) + { + goto Exit; + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportRflRead( + &eAction, uiStartFileNum, uiBytesRead))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + if( uiBytesRead < 512) + { + rc = RC_SET( NE_SFLM_NOT_RFL); + goto Exit; + } + + if( RC_BAD( rc = verifyHeader( ucHdr, uiStartFileNum, + m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum))) + { + RCODE tmpRc; + + if( m_pRestoreStatus) + { + if( RC_BAD( tmpRc = m_pRestoreStatus->reportError( &eAction, rc))) + { + rc = tmpRc; + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_RETRY) + { + if( RC_BAD( rc = m_pRestore->close())) + { + goto Exit; + } + + goto Retry_Open; + } + } + + goto Exit; + } + + // We may not know the actual EOF of files during restore operations. + + if( (m_uiFileEOF = (FLMUINT)FB2UD( &ucHdr [RFL_EOF_POS])) == 0) + { + bLastTransEndedAtFileEOF = TRUE; + } + else + { + bLastTransEndedAtFileEOF = (m_uiFileEOF == uiStartOffset) + ? TRUE + : FALSE; + } + + // Position to the start offset. Unfortunately, this means reading + // through the data and discarding it. + + uiOffset = 512; + while( uiOffset < uiStartOffset) + { + uiReadLen = (uiStartOffset - uiOffset); + if( uiReadLen > m_uiBufferSize) + { + uiReadLen = m_uiBufferSize; + } + + if( RC_BAD( rc = m_pRestore->read( uiReadLen, + m_pCurrentBuf->pIOBuffer->getBuffer(), &uiBytesRead))) + { + goto Exit; + } + + if( m_pRestoreStatus) + { + if( RC_BAD( rc = m_pRestoreStatus->reportRflRead( + &eAction, uiStartFileNum, uiBytesRead))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + rc = RC_SET( NE_SFLM_USER_ABORT); + goto Exit; + } + } + + // RFL file is incomplete if we could not read up to the last + // committed transaction. + + if( uiBytesRead < uiReadLen) + { + rc = RC_SET( NE_SFLM_RFL_INCOMPLETE); + goto Exit; + } + + uiOffset += uiBytesRead; + } + + // Need to set current file number + + m_pCurrentBuf->uiCurrFileNum = uiStartFileNum; + + // Better not be any transactions to recover - last database + // state needs to be a completed checkpoint. + + flmAssert( + m_pDatabase->m_lastCommittedDbHdr.ui64RflLastCPTransID == + m_pDatabase->m_lastCommittedDbHdr.ui64CurrTransID); + + // Use uiStartOffset here instead of ui32RflLastTransOffset, + // because ui32RflLastTransOffset may be zero, but we in that + // case we should be comparing to 512, and uiStartOffset will have + // been adjusted to 512 if that is the case. + + flmAssert( + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastCPOffset == + uiStartOffset); + + flmAssert( + (FLMUINT)m_pDatabase->m_lastCommittedDbHdr.ui32RflLastCPFileNum == + uiStartFileNum); + } + + // Set last transaction ID to the last transaction + // that was checkpointed - transaction numbers should ascend + // from here. + + m_ui64LastTransID = + m_pDatabase->m_lastCommittedDbHdr.ui64RflLastCPTransID; + + // Set the last committed trans ID + + m_ui64LastLoggedCommitTransID = + m_pDatabase->m_lastCommittedDbHdr.ui64LastRflCommitID; + + m_pCurrentBuf->uiRflFileOffset = uiStartOffset; + m_uiRflReadOffset = 0; + m_pCurrentBuf->uiRflBufBytes = 0; + + // Now, read until we are done. + + bForceNextFile = FALSE; + bHitEnd = FALSE; + for (;;) + { + flmAssert( pDb->m_uiFlags & FDB_REPLAYING_RFL); + + // Get the next operation from the file. + + rc = getPacket( pDb, bForceNextFile, &uiPacketType, &pucPacketBody, + &uiPacketBodyLen, &bHitEnd); + bForceNextFile = FALSE; + + if( RC_BAD( rc)) + { + if( rc == NE_SFLM_BAD_RFL_PACKET) + { + // If we don't know the current file size, and we + // are doing a restore, it is OK to end on a bad + // packet - we will simply abort the current + // transaction, if any. Then, try to go to the + // next file, because we really don't know where + // this file ends. + + if( m_pRestore && !m_uiFileEOF) + { + if( bTransActive) + { + pDb->transAbort(); + bTransActive = FALSE; + } + + // Set current transaction ID to zero - as if we had encountered + // an abort packet. + + m_ui64CurrTransID = 0; + bLastTransEndedAtFileEOF = TRUE; + + // Force to go to the next file + + bForceNextFile = TRUE; + rc = NE_SFLM_OK; + continue; + } + } + + goto Exit; + } + if (bHitEnd) + { + if (!m_pRestore) + { + // If we didn't end exactly where we should have, we have + // an incomplete log. The same is true if we are in the + // middle of a transaction. + + if( m_pCurrentBuf->uiCurrFileNum != m_uiLastRecoverFileNum || + bTransActive) + { + rc = RC_SET( NE_SFLM_RFL_INCOMPLETE); + } + else + { + rc = NE_SFLM_OK; + goto Finish_Recovery; + } + } + else + { + // If we are doing a restore, and we get to the end of the + // log, it is OK - even if we are in the middle of a + // transaction - the transaction will simply be aborted. + + rc = NE_SFLM_OK; + goto Finish_Recovery; + } + goto Exit; + } + + + // At this point, we know we have a good packet, see what it + // is and handle it. + + bHadOperations = TRUE; + eAction = SFLM_RESTORE_ACTION_CONTINUE; + + switch (uiPacketType) + { + case RFL_TRNS_BEGIN_PACKET: + { + // If we already have a transaction active, we have + // a problem. + + flmAssert( !bTransActive); + + if( uiPacketBodyLen) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + goto Exit; + } + + if( RC_BAD( rc = recovTransBegin( pDb, &eAction))) + { + goto Exit; + } + + if (eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + bTransActive = TRUE; + break; + } + + case RFL_TRNS_COMMIT_PACKET: + { + // Commit the current transaction. + + if (m_pRestoreStatus) + { + if (RC_BAD( rc = m_pRestoreStatus->reportCommitTrans( + &eAction, m_ui64CurrTransID))) + { + goto Exit; + } + + if (eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + } + + flmAssert( bTransActive); + pDb->m_uiFlags |= FDB_REPLAYING_COMMIT; + rc = pDb->transCommit(); + pDb->m_uiFlags &= ~FDB_REPLAYING_COMMIT; + bTransActive = FALSE; + + if (RC_BAD( rc)) + { + goto Exit; + } + + m_ui64LastLoggedCommitTransID = m_ui64CurrTransID; + +Finish_Transaction: + + if (!m_uiFileEOF) + { + bLastTransEndedAtFileEOF = TRUE; + } + else + { + bLastTransEndedAtFileEOF = + (m_uiRflReadOffset == m_pCurrentBuf->uiRflBufBytes && + m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes == m_uiFileEOF) + ? TRUE + : FALSE; + } + + m_ui64CurrTransID = 0; + break; + } + + case RFL_TRNS_ABORT_PACKET: + { + // Abort the current transaction. + + if (m_pRestoreStatus) + { + if (RC_BAD( rc = m_pRestoreStatus->reportAbortTrans( + &eAction, m_ui64CurrTransID))) + { + goto Exit; + } + + if (eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + } + + flmAssert( bTransActive); + + m_uiLastLfNum = 0; + m_eLastLfType = SFLM_LF_INVALID; + + rc = pDb->transAbort(); + bTransActive = FALSE; + + if (RC_BAD( rc)) + { + goto Exit; + } + + goto Finish_Transaction; + } + + case RFL_REDUCE_PACKET: + { + if( RC_BAD( rc = recovReduce( pDb, pucPacketBody, + uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + goto Finish_Transaction; + } + + case RFL_UPGRADE_PACKET: + { + if( RC_BAD( rc = recovUpgrade( pDb, + pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + goto Finish_Transaction; + } + + case RFL_INDEX_SUSPEND_PACKET: + case RFL_INDEX_RESUME_PACKET: + { + if( RC_BAD( rc = recovIndexSuspendResume( pDb, + uiPacketType, pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + break; + } + + case RFL_BLK_CHAIN_FREE_PACKET: + { + if( RC_BAD( rc = recovBlockChainFree( pDb, + pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + break; + } + + case RFL_ENABLE_ENCRYPTION_PACKET: + case RFL_WRAP_KEY_PACKET: + { + if( RC_BAD( rc = recovEncryptionKey( pDb, + uiPacketType, pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + goto Finish_Transaction; + } + + case RFL_ENC_DEF_KEY_PACKET: + { + if( RC_BAD( rc = recovEncDefKey( pDb, + pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + break; + } + + case RFL_ROLL_OVER_DB_KEY_PACKET: + { + if( RC_BAD( rc = recovRollOverDbKey( pDb, + pucPacketBody, uiPacketBodyLen, &eAction))) + { + goto Exit; + } + + if( eAction == SFLM_RESTORE_ACTION_STOP) + { + bLastTransEndedAtFileEOF = FALSE; + goto Finish_Recovery; + } + + goto Finish_Transaction; + } + + default: + { + // Should not be getting other packet types at this + // point. + + // If we don't know the current file size, and we + // are doing a restore, it is OK to end on a bad + // packet - we will simply abort the current + // transaction, if any. + + if( m_pRestore && !m_uiFileEOF) + { + rc = NE_SFLM_OK; + goto Finish_Recovery; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BAD_RFL_PACKET); + } + + goto Exit; + } + } + } + +Finish_Recovery: + + if( bTransActive) + { + pDb->transAbort(); + bTransActive = FALSE; + } + + if( m_pRestore) + { + FLMUINT uiNextRflFileNum = m_pCurrentBuf->uiCurrFileNum + 1; + + // At the end of the restore operation, we need to set things + // up so that the next transaction will begin a new RFL file. + // If we ended the restore in the middle of an RFL file, we + // need to set it up so that the new RFL file will have a new + // serial number. If we ended at the end of an RFL file, we + // can set it up so that the new RFL file will have the next + // serial number. + + // Set up the next RFL file number and offset. + + m_pDatabase->m_lastCommittedDbHdr.ui32RflCurrFileNum = + (FLMINT32)uiNextRflFileNum; + + // Set a zero into the offset, this is a special case which tells + // us that we should create the file no matter what - even if + // it already exists - it should be overwritten. + + m_pDatabase->m_lastCommittedDbHdr.ui32RflLastTransOffset = 0; + + if( bLastTransEndedAtFileEOF) + { + // Move the next serial number of the last RFL file processed into + // into the current RFL serial number so that the log header + // will be correct + + f_memcpy( + m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum, + m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + } + else + { + // Must create a new serial number so that when the new RFL + // file is created, it will have that next serial number. + + if( RC_BAD( rc = f_createSerialNumber( + m_pDatabase->m_lastCommittedDbHdr.ucLastTransRflSerialNum))) + { + goto Exit; + } + } + + // Save the last logged commit transaction ID. + + if( m_ui64LastLoggedCommitTransID) + { + m_pDatabase->m_lastCommittedDbHdr.ui64LastRflCommitID = + m_ui64LastLoggedCommitTransID; + } + + // No matter what, we must generate a new next serial number. This + // is what will be written to the new RFL file's header when it is + // created. + + if( RC_BAD( rc = f_createSerialNumber( + m_pDatabase->m_lastCommittedDbHdr.ucNextRflSerialNum))) + { + goto Exit; + } + } + + if( !bHadOperations) + { + // No transactions were recovered, but still need to + // setup a few things. + + m_pDatabase->m_uiFirstLogCPBlkAddress = 0; + m_pDatabase->m_uiLastCheckpointTime = (FLMUINT)FLM_GET_TIMER(); + + // Save the state of the log header into the checkpointDbHdr buffer. + + f_memcpy( &m_pDatabase->m_checkpointDbHdr, + &m_pDatabase->m_lastCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + } + + // Force a checkpoint to force the log files to be truncated and + // everything to be reset. This is done because during recovery + // the checkpoints that are executed do NOT truncate the RFL file - + // because it is using the log file to recover! + + closeFile(); + + m_pRestore = NULL; + m_pRestoreStatus = NULL; + enableLogging(); + + pDb->m_uiFlags &= ~FDB_REPLAYING_RFL; + + if( RC_BAD( rc = pDb->doCheckpoint( 0))) + { + goto Exit; + } + +Exit: + + if( bTransActive) + { + pDb->transAbort(); + } + + pDb->m_bItemStateUpdOk = FALSE; + pDb->m_uiFlags &= ~FDB_REPLAYING_RFL; + + m_uiLastLfNum = 0; + return( rc); +} + +/**************************************************************************** +Desc: Returns the name of an RFL file given its number +****************************************************************************/ +void F_Db::getRflFileName( + FLMUINT uiFileNum, + FLMBOOL bBaseOnly, + char * pszFileName, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated) +{ + if (bBaseOnly) + { + rflGetBaseFileName( uiFileNum, pszFileName, puiFileNameBufSize, + pbNameTruncated); + } + else + { + m_pDatabase->m_pRfl->getFullRflFileName( uiFileNum, + pszFileName, puiFileNameBufSize, + pbNameTruncated); + } +} + diff --git a/sql/src/rfl.h b/sql/src/rfl.h new file mode 100644 index 0000000..07bbb8e --- /dev/null +++ b/sql/src/rfl.h @@ -0,0 +1,664 @@ +//------------------------------------------------------------------------------ +// Desc: This file contains structures and definitions used for roll +// forward logging. +// +// Tabs: 3 +// +// Copyright (c) 1998-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: rfl.h 3109 2006-01-19 13:07:07 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#ifndef RFL_H +#define RFL_H + +class IXKeyCompare; + +// Packet types for roll forward logging + +#define RFL_TRNS_BEGIN_PACKET 1 +#define RFL_TRNS_COMMIT_PACKET 2 +#define RFL_TRNS_ABORT_PACKET 3 +#define RFL_REDUCE_PACKET 4 +#define RFL_UPGRADE_PACKET 5 +#define RFL_INDEX_SUSPEND_PACKET 6 +#define RFL_INDEX_RESUME_PACKET 7 +#define RFL_BLK_CHAIN_FREE_PACKET 8 +#define RFL_ENABLE_ENCRYPTION_PACKET 9 +#define RFL_WRAP_KEY_PACKET 10 +#define RFL_DATA_PACKET 11 +#define RFL_ROLL_OVER_DB_KEY_PACKET 12 +#define RFL_ENC_DEF_KEY_PACKET 13 + +#define RFL_PACKET_TYPE_MASK 0x7F + +// Flags for all packets + +#define RFL_HAVE_COUNTS_FLAG 0x01 +#define RFL_HAVE_DATA_FLAG 0x02 +#define RFL_FIRST_FLAG 0x04 +#define RFL_LAST_FLAG 0x08 +#define RFL_TRUNCATE_FLAG 0x10 + +#define RFL_GET_PACKET_TYPE(uiPacketType) \ + ((FLMUINT)((uiPacketType) & RFL_PACKET_TYPE_MASK)) + +// Definitions for ROLL FORWARD LOG file header format + +#define RFL_NAME_POS 0 +#define RFL_NAME "RFL6" +#define RFL_NAME_LEN 4 +#define RFL_VERSION_POS RFL_NAME_LEN +#define RFL_VERSION "6.00" +#define RFL_VERSION_LEN 4 +#define RFL_FILE_NUMBER_POS 8 +#define RFL_EOF_POS 12 +#define RFL_DB_SERIAL_NUM_POS 16 +#define RFL_SERIAL_NUM_POS (RFL_DB_SERIAL_NUM_POS + SFLM_SERIAL_NUM_SIZE) +#define RFL_NEXT_FILE_SERIAL_NUM_POS (RFL_SERIAL_NUM_POS + SFLM_SERIAL_NUM_SIZE) +#define RFL_KEEP_SIGNATURE_POS (RFL_NEXT_FILE_SERIAL_NUM_POS + SFLM_SERIAL_NUM_SIZE) + +#define RFL_KEEP_SIGNATURE "----KeepLog----" +#define RFL_NOKEEP_SIGNATURE "--DontKeepLog--" + +// Buffer size needs to be a multiple of 512 for direct IO writes. + +#define DEFAULT_RFL_WRITE_BUFFERS 8 +#define DEFAULT_RFL_BUFFER_SIZE (65536) + +// Definitions for packet format and sizes. + +#define RFL_PACKET_ADDRESS_OFFSET 0 +#define RFL_PACKET_CHECKSUM_OFFSET 4 +#define RFL_PACKET_TYPE_OFFSET 5 +#define RFL_PACKET_BODY_LENGTH_OFFSET 6 +#define RFL_PACKET_OVERHEAD 8 + +// Direct IO requires that we always write on 512 byte boundaries. +// This means that whenever we write out a packet, we may also +// have to write out up to the last 511 bytes of the prior packet +// in order to be on a 512 byte boundary. Thus, the buffer must +// be able to hold a full packet plus up to 512 bytes of the prior +// packet. + +#define RFL_MAX_PACKET_SIZE (65536 - 1024) +#define RFL_MAX_PACKET_BODY_SIZE (RFL_MAX_PACKET_SIZE - RFL_PACKET_OVERHEAD) + +typedef struct RflWaiterTag * RFL_WAITER_p; + +typedef struct RflWaiterTag +{ + FLMUINT uiThreadId; + FLMBOOL bIsWriter; + F_SEM hESem; + RCODE * pRc; + RFL_WAITER_p pNext; +} RFL_WAITER; + +typedef struct +{ + IF_IOBufferMgr * pBufferMgr; // Write buffer manager + IF_IOBuffer * pIOBuffer; + FLMUINT uiCurrFileNum; // Current file number. + FLMUINT uiRflBufBytes; // Number of bytes currently in the + // pIOBuffer. Always points to + // where the last packet ends when + // writing to the log file. + FLMUINT uiRflFileOffset; // Current offset in file that the + // zeroeth byte in the buffer + // represents. + FLMBOOL bTransInProgress; // Transaction in progress using + // these buffers. + FLMBOOL bOkToWriteHdrs; // Is it OK to update the DB + // headers with this information + SFLM_DB_HDR dbHdr; // DB header to be written with + // this buffer. + SFLM_DB_HDR cpHdr; // Checkpoint header for this + // buffer. + RFL_WAITER * pFirstWaiter; + RFL_WAITER * pLastWaiter; +} RFL_BUFFER; + +/************************************************************************** +Desc: This class handles all of the roll-forward logging for FLAIM. There + is one of these objects allocated per F_Database. +**************************************************************************/ +class F_Rfl : public F_Object +{ +public: + + F_Rfl(); + + ~F_Rfl(); + + RCODE setup( + F_Database * pDatabase, + const char * pszRflDir); + + RCODE finishCurrFile( + F_Db * pDb, + FLMBOOL bNewKeepState); + + RCODE logBeginTransaction( + F_Db * pDb); + + RCODE logEndTransaction( + F_Db * pDb, + FLMUINT uiPacketType, + FLMBOOL bThrowLogAway, + FLMBOOL * pbLoggedTransEnd = NULL); + + RCODE logReduce( + F_Db * pDb, + FLMUINT uiCount); + + RCODE logIndexSuspendOrResume( + F_Db * pDb, + FLMUINT uiIndexNum, + FLMUINT uiPacketType); + + RCODE logUpgrade( + F_Db * pDb, + FLMUINT uiOldVersion); + + RCODE logBlockChainFree( + F_Db * pDb, + FLMUINT64 ui64MaintDocID, + FLMUINT uiStartBlkAddr, + FLMUINT uiEndBlkAddr, + FLMUINT uiCount); + + RCODE logEncryptionKey( + F_Db * pDb, + FLMUINT uiPacketType, + FLMBYTE * pucKey, + FLMUINT32 ui32KeyLen); + + RCODE logEncDefKey( + F_Db * pDb, + FLMUINT uiEncDefNum, + FLMBYTE * pucKeyValue, + FLMUINT uiKeyValueLen, + FLMUINT uiKeySize); + + RCODE logRollOverDbKey( + F_Db * pDb); + + RCODE recover( + F_Db * pDb, + IF_RestoreClient * pRestore, + IF_RestoreStatus * pRestoreStatus); + + FLMBOOL atEndOfLog( void); + + // Returns full log file name associated with number. + // Will return the full path name. + + void getFullRflFileName( + FLMUINT uiFileNum, + char * pszFullRflFileName, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated = NULL); + + // Set the RFL directory. Passing in a NULL or empty + // string will cause the directory to be to the same + // directory where the database is located. + + RCODE setRflDir( + const char * pszRflDir); + + FINLINE const char * getRflDirPtr( void) + { + return &m_szRflDir [0]; + } + + FINLINE FLMBOOL isRflDirSameAsDbDir( void) + { + return m_bRflDirSameAsDb; + } + + FINLINE FLMUINT getCurrFileNum( void) + { + return m_pCurrentBuf->uiCurrFileNum; + } + + FINLINE FLMUINT getCurrWriteOffset( void) + { + return( m_pCurrentBuf->uiRflFileOffset + + m_pCurrentBuf->uiRflBufBytes); + } + + FINLINE FLMUINT getCurrReadOffset( void) + { + return( m_uiRflReadOffset + m_pCurrentBuf->uiRflFileOffset); + } + + FINLINE FLMUINT getCurrPacketAddress( void) + { + return( m_uiPacketAddress); + } + + FINLINE void getCurrSerialNum( + FLMBYTE * pucSerialNum) + { + f_memcpy( pucSerialNum, m_ucCurrSerialNum, SFLM_SERIAL_NUM_SIZE); + } + + FINLINE void setCurrSerialNum( + FLMBYTE * pucSerialNum) + { + f_memcpy( m_ucCurrSerialNum, pucSerialNum, SFLM_SERIAL_NUM_SIZE); + } + + FINLINE void getNextSerialNum( + FLMBYTE * pucSerialNum) + { + f_memcpy( pucSerialNum, m_ucNextSerialNum, SFLM_SERIAL_NUM_SIZE); + } + + FINLINE void setNextSerialNum( + FLMBYTE * pucSerialNum) + { + f_memcpy( m_ucNextSerialNum, pucSerialNum, SFLM_SERIAL_NUM_SIZE); + } + + FINLINE FLMUINT64 getCurrTransID( void) + { + return( m_ui64CurrTransID); + } + + RCODE truncate( + F_SEM hWaitSem, + FLMUINT uiTruncateSize); + + // Public functions, but only intended for internal use + + RCODE makeRoom( + F_Db * pDb, + FLMUINT uiAdditionalBytesNeeded, + FLMUINT * puiCurrPacketLenRV, + FLMUINT uiPacketType, + FLMUINT * puiBytesAvailableRV, + FLMUINT * puiPacketCountRV); + + FINLINE FLMBYTE * getPacketPtr( void) + { + return( &(m_pCurrentBuf->pIOBuffer->getBuffer()[ + m_pCurrentBuf->uiRflBufBytes])); + } + + // Close the current RFL file + + FINLINE void closeFile( void) + { + if( m_pCurrentBuf->pBufferMgr) + { + flmAssert( !m_pCurrentBuf->pBufferMgr->havePendingIO()); + } + if (m_pFileHdl) + { + m_pFileHdl->close(); + m_pFileHdl->Release(); + m_pFileHdl = NULL; + m_pCurrentBuf->uiCurrFileNum = 0; + m_pCurrentBuf->uiRflBufBytes = 0; + m_pCurrentBuf->uiRflFileOffset = 0; + } + } + + FINLINE FLMBOOL seeIfRflVolumeOk( void) + { + return m_bRflVolumeOk; + } + + FINLINE void setRflVolumeOk( void) + { + m_bRflVolumeOk = TRUE; + m_bRflVolumeFull = FALSE; + } + + FINLINE FLMBOOL isRflVolumeFull( void) + { + return m_bRflVolumeFull; + } + + FINLINE RCODE waitPendingWrites( void) + { + if (m_uiRflWriteBufs > 1) + { + return( m_pCurrentBuf->pBufferMgr->waitForAllPendingIO()); + } + else + { + return( NE_SFLM_OK); + } + } + + RCODE waitForWrites( + F_SEM hWaitSem, + RFL_BUFFER * pBuffer, + FLMBOOL bIsWriter); + + RCODE waitForCommit( + F_SEM hWaitSem); + + FINLINE void commitDbHdrs( + SFLM_DB_HDR * pDbHdr, + SFLM_DB_HDR * pCPHdr) + { + f_memcpy( &m_pCurrentBuf->dbHdr, pDbHdr, sizeof( SFLM_DB_HDR)); + f_memcpy( &m_pCurrentBuf->cpHdr, pCPHdr, sizeof( SFLM_DB_HDR)); + m_pCurrentBuf->bOkToWriteHdrs = TRUE; + } + + FINLINE void clearDbHdrs( void) + { + m_pCurrentBuf->bOkToWriteHdrs = FALSE; + } + + FLMBOOL seeIfRflWritesDone( + F_SEM hWaitSem, + FLMBOOL bForceWait); + + void wakeUpWaiter( + RCODE rc, + FLMBOOL bIsWriter); + + RCODE completeTransWrites( + F_Db * pDb, + FLMBOOL bCommitting, + FLMBOOL bOkToUnlock); + + FINLINE FLMBOOL isLoggingEnabled( void) + { + return( m_uiDisabledCount ? FALSE : TRUE); + } + + FINLINE FLMUINT disableLogging( void) + { + m_uiDisabledCount++; + return( m_uiDisabledCount); + } + + FINLINE FLMUINT enableLogging( void) + { + if (m_uiDisabledCount) + { + m_uiDisabledCount--; + } + return( m_uiDisabledCount); + } + +private: + + FINLINE FLMBYTE * getPacketBodyPtr( void) + { + return( &(m_pCurrentBuf->pIOBuffer->getBuffer()[ + m_pCurrentBuf->uiRflBufBytes + RFL_PACKET_OVERHEAD])); + } + + FINLINE FLMBOOL haveBuffSpace( + FLMUINT uiSpaceNeeded + ) + { + return( (FLMBOOL)((m_uiBufferSize - m_pCurrentBuf->uiRflBufBytes >= + uiSpaceNeeded) + ? (FLMBOOL)TRUE + : (FLMBOOL)FALSE) ); + } + + // Write the header of an RFL file. + + RCODE writeHeader( + FLMUINT uiFileNum, + FLMUINT uiEof, + FLMBYTE * pucSerialNum, + FLMBYTE * pucNextSerialNum, + FLMBOOL bKeepSignature); + + // Verify the header of an RFL file. + + RCODE verifyHeader( + FLMBYTE * pucHeader, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum); + + // Open a new RFL file. + + RCODE openFile( + F_SEM hWaitSem, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum); + + // Create a new RFL file + + RCODE createFile( + F_Db * pDb, + FLMUINT uiFileNum, + FLMBYTE * pucSerialNum, + FLMBYTE * pucNextSerialNum, + FLMBOOL bKeepSignature); + + void copyLastSector( + RFL_BUFFER * pBuffer, + FLMBYTE * pucOldBuffer, + FLMBYTE * pucNewBuffer, + FLMUINT uiCurrPacketLen, + FLMBOOL bStartingNewFile); + + // Position to an offset in the file. + + RCODE positionTo( + FLMUINT uiFileOffset); + + // Flush data to the current RFL file + + RCODE flush( + F_Db * pDb, + RFL_BUFFER * pBuffer, + FLMBOOL bFinalWrite = FALSE, + FLMUINT uiCurrPacketLen = 0, + FLMBOOL bStartingNewFile = FALSE); + + void switchBuffers( void); + + // Flush all packets except the current one to disk. + // Shift the current one down to close to or at the + // beginning of the buffer. + + RCODE shiftPacketsDown( + F_Db * pDb, + FLMUINT uiCurrPacketLen, + FLMBOOL bStartingNewFile); + + // See if we need to generate a new RFL file. + + RCODE seeIfNeedNewFile( + F_Db * pDb, + FLMUINT uiPacketLen, + FLMBOOL bDoNewIfOverLowLimit); + + // Calculate checksum, etc. on current packet. + + RCODE finishPacket( + F_Db * pDb, + FLMUINT uiPacketType, + FLMUINT uiPacketBodyLen, + FLMBOOL bDoNewIfOverLowLimit); + + // Functions for reading log files + + RCODE readPacket( + FLMUINT uiMinBytesNeeded); + + RCODE getPacket( + F_Db * pDb, + FLMBOOL bForceNextFile, + FLMUINT * puiPacketTypeRV, + const FLMBYTE ** ppucPacketBodyRV, + FLMUINT * puiPacketBodyLenRV, + FLMBOOL * pbHitEnd); + + RCODE setupTransaction( + F_Db * pDb); + + void finalizeTransaction( void); + + RCODE recovTransBegin( + F_Db * pDb, + eRestoreAction * peAction); + + RCODE recovBlockChainFree( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovIndexSuspendResume( + F_Db * pDb, + FLMUINT uiPacketType, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovReduce( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovUpgrade( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovEncryptionKey( + F_Db * pDb, + FLMUINT uiPacketType, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovEncDefKey( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + RCODE recovRollOverDbKey( + F_Db * pDb, + const FLMBYTE * pucPacketBody, + FLMUINT uiPacketBodyLen, + eRestoreAction * peAction); + + FLMBOOL useDataOnlyBlocks( + F_Db * pDb, + FLMUINT uiDataLen); + + // Member variables + + F_Database * m_pDatabase; // Pointer to database + RFL_BUFFER m_Buf1; + RFL_BUFFER m_Buf2; + F_MUTEX m_hBufMutex; + RFL_BUFFER * m_pCommitBuf; // Current buffer being committedout. + // NULL if no buffer is being committed. + RFL_BUFFER * m_pCurrentBuf; // Current write buffer - points to + // m_Buf1 or m_Buf2 + FLMUINT m_uiRflWriteBufs; // Number of RFL buffers + FLMUINT m_uiBufferSize; // Buffer size + FLMBOOL m_bKeepRflFiles; // Keep RFL files after they are + // no longer needed? + FLMUINT m_uiRflMinFileSize; // Minimum RFL file size. + FLMUINT m_uiRflMaxFileSize; // Maximum RFL file size. + IF_FileHdl * m_pFileHdl; // File handle for writing to roll + // forward log file - only need one for + // the writer because we can only have + // one update transaction at a time. + FLMUINT m_uiLastRecoverFileNum; + // Last file number to go to when + // doing recovery. + FLMBYTE m_ucCurrSerialNum [SFLM_SERIAL_NUM_SIZE]; + // Current file's serial number. + FLMUINT m_uiTransStartFile; // File the current transaction started + // in. + FLMUINT m_uiTransStartAddr; // Offset of start transaction packet. + FLMUINT64 m_ui64CurrTransID; // Current transaction ID. + FLMUINT64 m_ui64LastTransID; // Last transaction ID. + FLMUINT64 m_ui64LastLoggedCommitTransID; + // Last committed transaction that + // was logged to the RFL + FLMUINT m_uiOperCount; // Operations that have been logged for + // this transaction. + FLMUINT m_uiRflReadOffset; // Offset we are reading from in the + // buffer - only used when in reading + // mode. + FLMUINT m_uiPacketAddress; // Current packet's address in the file. + FLMUINT m_uiFileEOF; // End of file for current file. + // Only used when reading. + IF_RestoreClient * m_pRestore; // Restore object. + IF_RestoreStatus * m_pRestoreStatus; + // Restore status object + char m_szRflDir [F_PATH_MAX_SIZE]; + // RFL directory + FLMBOOL m_bRflDirSameAsDb; + FLMBOOL m_bCreateRflDir; + FLMBYTE m_ucNextSerialNum [ SFLM_SERIAL_NUM_SIZE]; + // Next file's serial number. + FLMBOOL m_bRflVolumeOk; // Did we have a problem accessing the + // RFL volume? + FLMBOOL m_bRflVolumeFull; // Did we have a problem accessing the + // RFL volume? + FLMUINT m_uiLastLfNum; // Last logical file used in restore/recover + eLFileType m_eLastLfType; // The Last Lfile Type + IXKeyCompare * m_pIxCompareObject; + IF_ResultSetCompare * m_pCompareObject; + FLMUINT m_uiDisabledCount; // Is logging currently disabled + +friend class F_RflOStream; +}; + +// Prototypes for roll forward logging functions. + +FLMBYTE RflCalcChecksum( + FLMBYTE * pucPacket, + FLMUINT uiPacketBodyLen); + +void rflGetBaseFileName( + FLMUINT uiFileNum, + char * pszBaseNameOut, + FLMUINT * puiFileNameBufSize, + FLMBOOL * pbNameTruncated = NULL); + +RCODE rflGetDirAndPrefix( + const char * pszDbFileName, + const char * pszRflDirIn, + char * pszRflDirOut); + +RCODE rflGetFileName( + const char * szDbName, + const char * szRflDir, + FLMUINT uiFileNum, + char * pszRflFileName); + +FLMBOOL rflGetFileNum( + const char * pszRflFileName, + FLMUINT * puiFileNum); + +#endif // ifdef RFL_H diff --git a/sql/src/scache.cpp b/sql/src/scache.cpp new file mode 100644 index 0000000..317fc41 --- /dev/null +++ b/sql/src/scache.cpp @@ -0,0 +1,7894 @@ +//------------------------------------------------------------------------------ +// Desc: Block Cache routines +// +// Tabs: 3 +// +// Copyright (c) 1997-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: scache.cpp 3116 2006-01-19 13:31:53 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +#define MAX_BLOCKS_TO_SORT 500 + +FSTATIC void ScaNotify( + FNOTIFY * pNotify, + F_CachedBlock * pUseSCache, + RCODE NotifyRc); + +FSTATIC void scaWriteComplete( + IF_IOBuffer * pIOBuffer); + +#ifdef SCACHE_LINK_CHECKING +FSTATIC void scaVerify( + int iPlace); +#else +#define scaVerify(iPlace) +#endif + +/*************************************************************************** +Desc: Compare two cache blocks to determine which one has lower address. +*****************************************************************************/ +FINLINE FLMINT scaCompare( + F_CachedBlock * pSCache1, + F_CachedBlock * pSCache2) +{ + if (FSAddrIsAtOrBelow( pSCache1->blkAddress(), pSCache2->blkAddress())) + { + flmAssert( pSCache1->blkAddress() != pSCache2->blkAddress()); + return( -1); + } + else + { + return( 1); + } +} + +/*************************************************************************** +Desc: Compare two cache blocks during a sort to determine which + one has lower address. +*****************************************************************************/ +FINLINE FLMINT scaSortCompare( + void * pvBuffer, + FLMUINT uiPos1, + FLMUINT uiPos2) +{ + return( scaCompare( ((F_CachedBlock **)pvBuffer)[ uiPos1], + ((F_CachedBlock **)pvBuffer)[ uiPos2])); +} + +/*************************************************************************** +Desc: Swap two entries in cache table during sort. +*****************************************************************************/ +FINLINE void scaSortSwap( + void * pvBuffer, + FLMUINT uiPos1, + FLMUINT uiPos2) +{ + F_CachedBlock ** ppSCacheTbl = (F_CachedBlock **)pvBuffer; + F_CachedBlock * pTmpSCache = ppSCacheTbl[ uiPos1]; + + ppSCacheTbl[ uiPos1] = ppSCacheTbl[ uiPos2]; + ppSCacheTbl[ uiPos2] = pTmpSCache; +} + +/**************************************************************************** +Desc: Link a cache block into the list of F_Database blocks that need one or + more versions of the block to be logged. This routine assumes that + the block cache mutex is locked. +*****************************************************************************/ +void F_CachedBlock::linkToLogList( void) +{ + FLMUINT uiPrevBlkAddress; + + flmAssert( m_pDatabase); + flmAssert( m_ui16Flags & CA_DIRTY); + flmAssert( !(m_ui16Flags & (CA_IN_FILE_LOG_LIST | CA_IN_NEW_LIST))); + flmAssert( !m_pPrevInReplaceList); + flmAssert( !m_pNextInReplaceList); + + uiPrevBlkAddress = getPriorImageAddress(); + if( uiPrevBlkAddress || !m_pNextInVersionList) + { + goto Exit; + } + + if ((m_pNextInReplaceList = m_pDatabase->m_pFirstInLogList) != NULL) + { + m_pNextInReplaceList->m_pPrevInReplaceList = this; + } + else + { + m_pDatabase->m_pLastInLogList = this; + } + + setFlags( CA_IN_FILE_LOG_LIST); + m_pPrevInReplaceList = NULL; + m_pDatabase->m_pFirstInLogList = this; + m_pDatabase->m_uiLogListCount++; + +Exit: + + return; +} + +/**************************************************************************** +Desc: Unlinks a cache block from the F_Database's log list + NOTE: This function assumes that the block cache mutex is locked. +****************************************************************************/ +void F_CachedBlock::unlinkFromLogList( void) +{ + flmAssert( m_pDatabase); + flmAssert( m_ui16Flags & CA_IN_FILE_LOG_LIST); + flmAssert( m_pDatabase->m_uiLogListCount); + + if (m_pNextInReplaceList) + { + m_pNextInReplaceList->m_pPrevInReplaceList = m_pPrevInReplaceList; + } + else + { + m_pDatabase->m_pLastInLogList = m_pPrevInReplaceList; + } + + if (m_pPrevInReplaceList) + { + m_pPrevInReplaceList->m_pNextInReplaceList = m_pNextInReplaceList; + } + else + { + m_pDatabase->m_pFirstInLogList = m_pNextInReplaceList; + } + + m_pNextInReplaceList = NULL; + m_pPrevInReplaceList = NULL; + clearFlags( CA_IN_FILE_LOG_LIST); + m_pDatabase->m_uiLogListCount--; +} + +/**************************************************************************** +Desc: Link a cache block into the list of F_Database blocks that are beyond + the EOF. The blocks are linked to the end of the list so that they + are kept in ascending order. + NOTE: This function assumes that the block cache mutex is locked. +*****************************************************************************/ +void F_CachedBlock::linkToNewList( void) +{ + flmAssert( m_pDatabase); + + flmAssert( m_ui64HighTransID == ~((FLMUINT64)0)); + flmAssert( m_ui16Flags & CA_DIRTY); + flmAssert( !(m_ui16Flags & (CA_IN_FILE_LOG_LIST | CA_IN_NEW_LIST))); + flmAssert( !m_pPrevInReplaceList); + flmAssert( !m_pNextInReplaceList); + + if ((m_pPrevInReplaceList = m_pDatabase->m_pLastInNewList) != NULL) + { + flmAssert( scaCompare( m_pDatabase->m_pLastInNewList, this) < 0); + + m_pPrevInReplaceList->m_pNextInReplaceList = this; + } + else + { + m_pDatabase->m_pFirstInNewList = this; + } + + m_pNextInReplaceList = NULL; + m_pDatabase->m_pLastInNewList = this; + setFlags( CA_IN_NEW_LIST); + m_pDatabase->m_uiNewCount++; +} + +/**************************************************************************** +Desc: Unlinks a cache block from the F_Database's new block list + NOTE: This function assumes that the block cache mutex is locked. +****************************************************************************/ +void F_CachedBlock::unlinkFromNewList( void) +{ + flmAssert( m_pDatabase); + + flmAssert( m_ui16Flags & CA_IN_NEW_LIST); + flmAssert( m_pDatabase->m_uiNewCount); + + if (m_pNextInReplaceList) + { + m_pNextInReplaceList->m_pPrevInReplaceList = m_pPrevInReplaceList; + } + else + { + m_pDatabase->m_pLastInNewList = m_pPrevInReplaceList; + } + + if (m_pPrevInReplaceList) + { + m_pPrevInReplaceList->m_pNextInReplaceList = m_pNextInReplaceList; + } + else + { + m_pDatabase->m_pFirstInNewList = m_pNextInReplaceList; + } + + m_pNextInReplaceList = NULL; + m_pPrevInReplaceList = NULL; + clearFlags( CA_IN_NEW_LIST); + m_pDatabase->m_uiNewCount--; +} + +/**************************************************************************** +Desc: Unlinks a cache block from the replace list + NOTE: This function assumes that the block cache mutex is locked. +****************************************************************************/ +void F_CachedBlock::unlinkFromReplaceList( void) +{ + FLMUINT uiSize = memSize(); + + flmAssert( !m_ui16Flags); + + if( m_pNextInReplaceList) + { + m_pNextInReplaceList->m_pPrevInReplaceList = m_pPrevInReplaceList; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pLRUReplace = m_pPrevInReplaceList; + } + + if( m_pPrevInReplaceList) + { + m_pPrevInReplaceList->m_pNextInReplaceList = m_pNextInReplaceList; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pMRUReplace = m_pNextInReplaceList; + } + + m_pNextInReplaceList = NULL; + m_pPrevInReplaceList = NULL; + + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableCount); + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableCount--; + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableBytes >= uiSize); + gv_SFlmSysData.pBlockCacheMgr->m_uiReplaceableBytes -= uiSize; +} + +/**************************************************************************** +Desc: Check hash links. + This routine assumes that the block cache mutex has already been locked. +****************************************************************************/ +#ifdef SCACHE_LINK_CHECKING +void F_CachedBlock::checkHashLinks( + F_CachedBlock ** ppSCacheBucket) +{ + F_CachedBlock * pBlock; + + if (!m_pDatabase) + { + f_breakpoint( 1); + } + + if (m_pPrevInVersionList) + { + f_breakpoint( 2); + } + + if (m_pNextInVersionList == this) + { + f_breakpoint( 3); + } + + if (m_pPrevInVersionList == this) + { + f_breakpoint( 4); + } + + // Make sure that the block isn't added into the list a second time. + + for (pBlock = *ppSCacheBucket; + pBlock; + pBlock = pBlock->m_pNextInHashBucket) + { + if (this == pBlock) + { + f_breakpoint( 5); + } + } + + // Make sure the block is not in the transaction + // log list. + + for (pBlock = m_pDatabase->getTransLogList(); + pBlock; + pBlock = pBlock->m_pNextInHashBucket) + { + if (this == pBlock) + { + f_breakpoint( 6); + } + } +} +#endif + +/**************************************************************************** +Desc: Unlink a cache block from its hash bucket. This routine assumes + that the block cache mutex has already been locked. +****************************************************************************/ +#ifdef SCACHE_LINK_CHECKING +void F_CachedBlock::checkHashUnlinks( + F_CachedBlock ** ppSCacheBucket) +{ + F_CachedBlock * pTmpSCache; + + // Make sure the cache is actually in this bucket + + pTmpSCache = *ppSCacheBucket; + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if (!pTmpSCache) + { + f_breakpoint( 333); + } + + for (pTmpSCache = m_pDatabase->getTransLogList(); + pTmpSCache; + pTmpSCache = pTmpSCache->m_pNextInHashBucket) + { + if (this == pTmpSCache) + { + f_breakpoint( 334); + } + } +} +#endif + +/**************************************************************************** +Desc: Link a cache block to its F_Database structure. This routine assumes + that the block cache mutex has already been locked. +****************************************************************************/ +void F_CachedBlock::linkToDatabase( + F_Database * pDatabase) +{ + flmAssert( !m_pDatabase); + + if (m_ui16Flags & CA_WRITE_PENDING) + { + if ((m_pNextInDatabase = pDatabase->m_pPendingWriteList) != NULL) + { + m_pNextInDatabase->m_pPrevInDatabase = this; + } + + pDatabase->m_pPendingWriteList = this; + setFlags( CA_IN_WRITE_PENDING_LIST); + } + else + { + F_CachedBlock * pPrevSCache; + F_CachedBlock * pNextSCache; + + // Link at end of dirty blocks. + + if (pDatabase->m_pLastDirtyBlk) + { + pPrevSCache = pDatabase->m_pLastDirtyBlk; + pNextSCache = pPrevSCache->m_pNextInDatabase; + } + else + { + // No dirty blocks, so link to head of list. + + pPrevSCache = NULL; + pNextSCache = pDatabase->m_pSCacheList; + } + + // If the block is dirty, change the last dirty block pointer. + + if (m_ui16Flags & CA_DIRTY) + { + pDatabase->m_pLastDirtyBlk = this; + } + + if ((m_pNextInDatabase = pNextSCache) != NULL) + { + pNextSCache->m_pPrevInDatabase = this; + } + + if ((m_pPrevInDatabase = pPrevSCache) != NULL) + { + pPrevSCache->m_pNextInDatabase = this; + } + else + { + pDatabase->m_pSCacheList = this; + } + } + + m_pDatabase = pDatabase; +} + +/**************************************************************************** +Desc: Unlink a cache block from its F_Database object. This routine assumes + that the block cache mutex has already been locked. +****************************************************************************/ +void F_CachedBlock::unlinkFromDatabase( void) +{ + flmAssert( m_pDatabase); + + if (m_ui16Flags & CA_IN_WRITE_PENDING_LIST) + { + if (m_pPrevInDatabase) + { + m_pPrevInDatabase->m_pNextInDatabase = m_pNextInDatabase; + } + else + { + m_pDatabase->m_pPendingWriteList = m_pNextInDatabase; + } + + if (m_pNextInDatabase) + { + m_pNextInDatabase->m_pPrevInDatabase = m_pPrevInDatabase; + } + + clearFlags( CA_IN_WRITE_PENDING_LIST); + } + else + { + if (this == m_pDatabase->m_pLastDirtyBlk) + { + m_pDatabase->m_pLastDirtyBlk = m_pDatabase->m_pLastDirtyBlk->m_pPrevInDatabase; +#ifdef FLM_DEBUG + + // If m_pLastDirtyBlk is non-NULL, it had better be pointing + // to a dirty block. + + if (m_pDatabase->m_pLastDirtyBlk) + { + flmAssert( m_pDatabase->m_pLastDirtyBlk->m_ui16Flags & CA_DIRTY); + } +#endif + } + + if (m_pNextInDatabase) + { + m_pNextInDatabase->m_pPrevInDatabase = m_pPrevInDatabase; + } + + if (m_pPrevInDatabase) + { + m_pPrevInDatabase->m_pNextInDatabase = m_pNextInDatabase; + } + else + { + m_pDatabase->m_pSCacheList = m_pNextInDatabase; + } + + m_pNextInDatabase = NULL; + m_pPrevInDatabase = NULL; + } + + m_pDatabase = NULL; +} + +/**************************************************************************** +Desc: Link a cache block to the free list. This routine assumes that the + block cache mutex is locked. +****************************************************************************/ +void F_CachedBlock::linkToFreeList( + FLMUINT uiFreeTime) +{ + flmAssert( !m_ui16Flags); + flmAssert( !m_pDatabase); + flmAssert( !m_pPrevInReplaceList); + flmAssert( !m_pNextInReplaceList); + + if (m_ui64HighTransID != ~((FLMUINT64)0)) + { + // Set the transaction ID to ~((FLMUINT64)0) so that the old version + // counts will be decremented if this is an old version + // of the block. Also, we want the transaction ID to be + // ~((FLMUINT64)0) so that when the block is re-used in allocBlock() + // the old version counts won't be decremented again. + + setTransID( ~((FLMUINT64)0)); + } + + if ((m_pNextInDatabase = gv_SFlmSysData.pBlockCacheMgr->m_pFirstFree) != NULL) + { + m_pNextInDatabase->m_pPrevInDatabase = this; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pLastFree = this; + } + + m_pPrevInDatabase = NULL; + m_uiBlkAddress = uiFreeTime; + m_ui16Flags = CA_FREE; + + gv_SFlmSysData.pBlockCacheMgr->m_pFirstFree = this; + gv_SFlmSysData.pBlockCacheMgr->m_uiFreeBytes += memSize(); + gv_SFlmSysData.pBlockCacheMgr->m_uiFreeCount++; +} + +/**************************************************************************** +Desc: Unlink a cache block from the free list. This routine assumes + that the block cache mutex has already been locked. +****************************************************************************/ +void F_CachedBlock::unlinkFromFreeList( void) +{ + FLMUINT uiSize = memSize(); + + flmAssert( !m_uiUseCount); + flmAssert( m_ui16Flags == CA_FREE); + + if( m_pNextInDatabase) + { + m_pNextInDatabase->m_pPrevInDatabase = m_pPrevInDatabase; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pLastFree = m_pPrevInDatabase; + } + + if( m_pPrevInDatabase) + { + m_pPrevInDatabase->m_pNextInDatabase = m_pNextInDatabase; + } + else + { + gv_SFlmSysData.pBlockCacheMgr->m_pFirstFree = m_pNextInDatabase; + } + + m_pNextInDatabase = NULL; + m_pPrevInDatabase = NULL; + m_ui16Flags &= ~CA_FREE; + flmAssert( !m_ui16Flags); + + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_uiFreeBytes >= uiSize); + gv_SFlmSysData.pBlockCacheMgr->m_uiFreeBytes -= uiSize; + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_uiFreeCount); + gv_SFlmSysData.pBlockCacheMgr->m_uiFreeCount--; +} + +/**************************************************************************** +Desc: This routine notifies threads waiting for a pending read or write + to complete. + NOTE: This routine assumes that the block cache mutex is already + locked. +****************************************************************************/ +FSTATIC void ScaNotify( + FNOTIFY * pNotify, + F_CachedBlock * pUseSCache, + RCODE NotifyRc) +{ + while (pNotify) + { + F_SEM hSem; + + *(pNotify->pRc) = NotifyRc; + if (RC_OK( NotifyRc)) + { + if (pNotify->pvUserData) + { + *((F_CachedBlock **)pNotify->pvUserData) = pUseSCache; + } + if (pUseSCache) + { + pUseSCache->useForThread( pNotify->uiThreadId); + } + } + hSem = pNotify->hSem; + pNotify = pNotify->pNext; + f_semSignal( hSem); + } +} + +/**************************************************************************** +Desc: This routine logs information about changes to a cache block's flags +****************************************************************************/ +#ifdef FLM_DBG_LOG +void F_CachedBlock::logFlgChange( + FLMUINT16 ui16OldFlags, + char cPlace + ) +{ + char szBuf [60]; + char * pszTmp; + FLMUINT16 ui16NewFlags = m_ui16Flags; + + szBuf [0] = cPlace; + szBuf [1] = '-'; + f_strcpy( &szBuf[2], "FLG:"); + pszTmp = &szBuf [6]; + + if (ui16OldFlags & CA_DIRTY) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_DIRTY)) + { + *pszTmp++ = '-'; + } + *pszTmp++ = 'D'; + *pszTmp = 0; + } + else if (ui16NewFlags & CA_DIRTY) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+D"); + pszTmp += 2; + } + + if (ui16OldFlags & CA_WRITE_INHIBIT) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_WRITE_INHIBIT)) + { + *pszTmp++ = '-'; + } + f_strcpy( pszTmp, "WI"); + pszTmp += 2; + } + else if (ui16NewFlags & CA_WRITE_INHIBIT) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+WI"); + pszTmp += 3; + } + + if (ui16OldFlags & CA_READ_PENDING) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_READ_PENDING)) + { + *pszTmp++ = '-'; + } + f_strcpy( pszTmp, "RD"); + pszTmp += 2; + } + else if (ui16NewFlags & CA_READ_PENDING) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+RD"); + pszTmp += 3; + } + + if (ui16OldFlags & CA_WRITE_TO_LOG) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_WRITE_TO_LOG)) + { + *pszTmp++ = '-'; + } + f_strcpy( pszTmp, "WL"); + pszTmp += 2; + } + else if (ui16NewFlags & CA_WRITE_TO_LOG) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+WL"); + pszTmp += 3; + } + + if (ui16OldFlags & CA_LOG_FOR_CP) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_LOG_FOR_CP)) + { + *pszTmp++ = '-'; + } + f_strcpy( pszTmp, "CP"); + pszTmp += 2; + } + else if (ui16NewFlags & CA_LOG_FOR_CP) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+CP"); + pszTmp += 3; + } + + if (ui16OldFlags & CA_WAS_DIRTY) + { + *pszTmp++ = ' '; + if (!(ui16NewFlags & CA_WAS_DIRTY)) + { + *pszTmp++ = '-'; + } + f_strcpy( pszTmp, "WD"); + pszTmp += 2; + } + else if (ui16NewFlags & CA_WAS_DIRTY) + { + *pszTmp++ = ' '; + f_strcpy( pszTmp, "+WD"); + pszTmp += 3; + } + + if (pszTmp != &szBuf [6]) + { + flmDbgLogWrite( m_pDatabase, m_uiBlkAddress, 0, getLowTransID(), szBuf); + } +} +#endif + +/**************************************************************************** +Desc: This routine frees the memory for a cache block and decrements the + necessary counters in the cache manager. + NOTE: This routine assumes that the block cache mutex is already locked. +****************************************************************************/ +F_CachedBlock::~F_CachedBlock() +{ + FLMUINT uiSize = memSize(); + + if (m_ui64HighTransID != ~((FLMUINT64)0)) + { + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerBytes >= uiSize); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerBytes -= uiSize; + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerCount); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiOldVerCount--; + } + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiByteCount >= uiSize); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiByteCount -= uiSize; + flmAssert( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCount); + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCount--; + if (shouldRehash( gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCount, + gv_SFlmSysData.pBlockCacheMgr->m_uiNumBuckets)) + { + if (checkHashFailTime( &gv_SFlmSysData.pBlockCacheMgr->m_uiHashFailTime)) + { + (void)gv_SFlmSysData.pBlockCacheMgr->rehash(); + } + } +} + +/**************************************************************************** +Desc: This routine unlinks a cache block from all of its lists and then + optionally frees it. NOTE: This routine assumes that the block cache + mutex is already locked. +****************************************************************************/ +void F_CachedBlock::unlinkCache( + FLMBOOL bFreeIt, + RCODE NotifyRc) +{ +#ifdef FLM_DEBUG + SCACHE_USE * pUse; +#endif + + // Cache block better not be dirty and better not need to be written + // to the log. + +#ifdef FLM_DEBUG + if( RC_OK( NotifyRc)) + { + flmAssert (!(m_ui16Flags & + (CA_DIRTY | CA_WRITE_TO_LOG | CA_LOG_FOR_CP | + CA_WAS_DIRTY | CA_IN_FILE_LOG_LIST | CA_IN_NEW_LIST))); + } +#endif + + unlinkFromGlobalList(); + +#ifdef FLM_DBG_LOG + flmDbgLogWrite( m_pDatabase, m_uiBlkAddress, 0, getLowTransID(), "UNLINK"); +#endif + + // If cache block has no previous versions linked to it, it + // is in the hash bucket and needs to be unlinked from it. + // Otherwise, it only needs to be unlinked from the version list. + + if (m_pDatabase) + { + if (!m_pPrevInVersionList) + { + F_CachedBlock ** ppSCacheBucket; + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_pDatabase->getSigBitsInBlkSize(), + (FLMUINT)m_uiBlkAddress); + unlinkFromHashBucket( ppSCacheBucket); + if (m_pNextInVersionList) + { + // Older version better not be needing to be logged + +#ifdef FLM_DEBUG + if( RC_OK( NotifyRc)) + { + flmAssert( !(m_pNextInVersionList->m_ui16Flags & + (CA_WRITE_TO_LOG | CA_DIRTY | + CA_WAS_DIRTY | CA_IN_FILE_LOG_LIST | CA_IN_NEW_LIST))); + } +#endif + m_pNextInVersionList->m_pPrevInVersionList = NULL; + m_pNextInVersionList->linkToHashBucket( ppSCacheBucket); + m_pNextInVersionList->verifyCache( 2100); + m_pNextInVersionList = NULL; + } + } + else + { + verifyCache( 2000); + savePrevBlkAddress(); + + m_pPrevInVersionList->m_pNextInVersionList = m_pNextInVersionList; + m_pPrevInVersionList->verifyCache( 2200); + + if (m_pNextInVersionList) + { + // Older version better not be dirty or not yet logged. + +#ifdef FLM_DEBUG + if( RC_OK( NotifyRc)) + { + flmAssert( !(m_pNextInVersionList->m_ui16Flags & + (CA_WRITE_TO_LOG | CA_DIRTY | CA_WAS_DIRTY))); + } +#endif + m_pNextInVersionList->m_pPrevInVersionList = m_pPrevInVersionList; + m_pNextInVersionList->verifyCache( 2300); + } + + m_pNextInVersionList = NULL; + m_pPrevInVersionList = NULL; + } +#ifdef SCACHE_LINK_CHECKING + + // Verify that the thing is not in a hash bucket. + { + F_CachedBlock ** ppSCacheBucket; + F_CachedBlock * pTmpSCache; + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_pDatabase->getSigBitsInBlkSize(), + m_uiBlkAddress); + pTmpSCache = *ppSCacheBucket; + while (pTmpSCache && this != pTmpSCache) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if (pTmpSCache) + { + f_breakpoint( 4); + } + } +#endif + + unlinkFromDatabase(); + } + + if (bFreeIt) + { + + // Free the notify list associated with the cache block. + // NOTE: If there is actually a notify list, NotifyRc WILL ALWAYS + // be something other than NE_SFLM_OK. If there is a notify list, + // the notified threads will thus get a non-OK return code + // in every case. + +#ifdef FLM_DEBUG + if( NotifyRc == NE_SFLM_OK) + { + flmAssert( m_pNotifyList == NULL); + } +#endif + + ScaNotify( m_pNotifyList, NULL, NotifyRc); + m_pNotifyList = NULL; + +#ifdef FLM_DEBUG + + // Free the use list associated with the cache block + + pUse = m_pUseList; + while (pUse) + { + SCACHE_USE * pTmp; + + pTmp = pUse; + pUse = pUse->pNext; + f_free( &pTmp); + } +#endif + + delete this; + } +} + +/**************************************************************************** +Desc: Unlink all log blocks for a file that were logged during the transaction. + NOTE: This is only called when a transaction is aborted. + WHEN A TRANSACTION IS ABORTED, THIS FUNCTION SHOULD BE CALLED BEFORE + FREEING DIRTY BLOCKS. OTHERWISE, THE m_pPrevInVersionList POINTER + WILL BE NULL AND WILL CAUSE AN ABEND WHEN IT IS ACCESSED. + NOTE: This routine assumes that the block cache mutex has been + locked. +****************************************************************************/ +void F_Database::unlinkTransLogBlocks( void) +{ + F_CachedBlock * pSCache; + F_CachedBlock * pNextSCache; + + pSCache = m_pTransLogList; + while (pSCache) + { +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags = pSCache->m_ui16Flags; +#endif + + if (pSCache->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP)) + { + flmAssert( m_uiLogCacheCount); + m_uiLogCacheCount--; + } + + pSCache->clearFlags( CA_WRITE_TO_LOG | CA_LOG_FOR_CP); + pNextSCache = pSCache->m_pNextInHashBucket; + + if (pSCache->m_ui16Flags & CA_WAS_DIRTY) + { + flmAssert( this == pSCache->m_pDatabase); + pSCache->setDirtyFlag( this); + pSCache->clearFlags( CA_WAS_DIRTY); + + // Move the block into the dirty blocks. + + pSCache->unlinkFromDatabase(); + pSCache->linkToDatabase( this); + } + +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'A'); +#endif + + // Perhaps we don't really need to set these pointers to NULL, + // but it helps keep things clean. + + pSCache->m_pNextInHashBucket = NULL; + pSCache->m_pPrevInHashBucket = NULL; + pSCache = pNextSCache; + } + m_pTransLogList = NULL; +} + +/**************************************************************************** +Desc: Unlink a cache block from the list of cache blocks that are in the log + list for the current transaction. +****************************************************************************/ +void F_CachedBlock::unlinkFromTransLogList( void) +{ + +#ifdef SCACHE_LINK_CHECKING + + // Make sure the block is not in a hash bucket + + { + F_CachedBlock ** ppSCacheBucket; + F_CachedBlock * pTmpSCache; + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_pDatabase->m_uiSigBitsInBlkSize, + m_uiBlkAddress); + pTmpSCache = *ppSCacheBucket; + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if (pTmpSCache) + { + f_breakpoint( 1001); + } + + // Make sure the block is in the log list. + + pTmpSCache = m_pDatabase->m_pTransLogList; + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if (!pTmpSCache) + { + f_breakpoint( 1002); + } + } +#endif + + if (m_pPrevInHashBucket) + { + m_pPrevInHashBucket->m_pNextInHashBucket = m_pNextInHashBucket; + } + else + { + m_pDatabase->m_pTransLogList = m_pNextInHashBucket; + } + + if (m_pNextInHashBucket) + { + m_pNextInHashBucket->m_pPrevInHashBucket = m_pPrevInHashBucket; + } + + m_pNextInHashBucket = NULL; + m_pPrevInHashBucket = NULL; +} + +/**************************************************************************** +Desc: The block pointed to by pSCache is about to be removed from from the + version list for a particular block address because it is no longer + needed. Before doing that, the previous block address should be + moved to the next newer version's block header so that it will not be + lost, but only if the next newer version's block header is not already + pointing to a prior version of the block. + This method assumes the block cache mutex is locked. +****************************************************************************/ +void F_CachedBlock::savePrevBlkAddress( void) +{ + FLMUINT uiPrevBlkAddress = getPriorImageAddress(); + F_CachedBlock * pNewerSCache; + FLMUINT uiNewerBlkPrevAddress; + + // NOTE: If a block is being read in from disk, it has to have a + // previous block address in its header. Otherwise, it could never + // have been written out to disk and removed from cache in the first + // place. This is obvious for versions being read from the rollback + // log - it would be impossible to retrieve them from the rollback + // log if they weren't already part of a version chain! It is also + // true for the most current version of a block. The most current + // version of a block can never be written out and removed from + // cache without having a pointer to the chain of older versions that + // may still be needed by read transactions - or to rollback the + // transaction - or to recover a checkpoint. + + if ((uiPrevBlkAddress) && + ((pNewerSCache = m_pPrevInVersionList) != NULL) && + (!(pNewerSCache->m_ui16Flags & CA_READ_PENDING))) + { + + // Only move the older version's previous block address to the + // newer version, if the newer version doesn't already have a + // previous block address. Also need to set the previous + // transaction ID. + // + // NOTE: The newer block may or may not be dirty. It is OK + // to change the prior version address in the header of a + // non-dirty block in this case. This is because the block + // may or may not be written out to the roll-back log. If it + // is, we want to make sure it has the correct prior version + // address. If it isn't ever written out to the log, it + // will eventually fall out of cache because it is no longer + // needed. + + uiNewerBlkPrevAddress = pNewerSCache->getPriorImageAddress(); + if (!uiNewerBlkPrevAddress) + { + + // Need to temporarily use the newer version of the block + // before changing its prior image block address. + + pNewerSCache->useForThread( 0); + flmAssert( uiPrevBlkAddress); + + pNewerSCache->m_pBlkHdr->ui32PriorBlkImgAddr = + (FLMUINT32)uiPrevBlkAddress; + + // Need to remove the newer block from the file log + // list, since it no longer needs to be logged + + if( pNewerSCache->m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + pNewerSCache->unlinkFromLogList(); + } + + pNewerSCache->releaseForThread(); + } + } +} + +/**************************************************************************** +Desc: See if we should force a checkpoint. +****************************************************************************/ +FINLINE FLMBOOL scaSeeIfForceCheckpoint( + FLMUINT uiCurrTime, + FLMUINT uiLastCheckpointTime, + CP_INFO * pCPInfo) +{ + if (FLM_ELAPSED_TIME( uiCurrTime, uiLastCheckpointTime) >= + gv_SFlmSysData.uiMaxCPInterval) + { + if (pCPInfo) + { + pCPInfo->bForcingCheckpoint = TRUE; + pCPInfo->eForceCheckpointReason = SFLM_CP_TIME_INTERVAL_REASON; + pCPInfo->uiForceCheckpointStartTime = (FLMUINT)FLM_GET_TIMER(); + } + + return( TRUE); + } + + return( FALSE); +} + +/**************************************************************************** +Desc: Allocate the array that keeps track of blocks written or logged. +****************************************************************************/ +RCODE F_Database::allocBlocksArray( + FLMUINT uiNewSize, + FLMBOOL bOneArray) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOldSize = m_uiBlocksDoneArraySize; + + if (!uiNewSize) + { + uiNewSize = uiOldSize + 500; + } + + // Re-alloc the array + + if (RC_BAD( rc = f_realloc( + (FLMUINT)(uiNewSize * + (sizeof( F_CachedBlock *) + + sizeof( F_CachedBlock *))), + &m_ppBlocksDone))) + { + goto Exit; + } + + // Copy the old stuff into the two new areas of the new array. + + if (uiOldSize && !bOneArray) + { + f_memmove( &m_ppBlocksDone [uiNewSize], + &m_ppBlocksDone [uiOldSize], + uiOldSize * sizeof( F_CachedBlock *)); + } + + // Set the new array size + + m_uiBlocksDoneArraySize = uiNewSize; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Write out log blocks to the rollback log for a database. +****************************************************************************/ +RCODE F_Database::flushLogBlocks( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bIsCPThread, + FLMUINT uiMaxDirtyCache, + FLMBOOL * pbForceCheckpoint, + FLMBOOL * pbWroteAll) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiLogEof; + SFLM_DB_HDR * pDbHdr; + F_CachedBlock * pTmpSCache; + F_CachedBlock * pLastBlockToLog; + F_CachedBlock * pFirstBlockToLog; + F_CachedBlock * pDirtySCache; + F_CachedBlock * pSavedSCache = NULL; + FLMUINT uiDirtyCacheLeft; + FLMUINT uiPrevBlkAddress; + FLMBOOL bMutexLocked = TRUE; + FLMBOOL bLoggedFirstBlk = FALSE; + FLMBOOL bLoggedFirstCPBlk = FALSE; + FLMUINT uiCurrTime; + FLMUINT uiSaveEOFAddr; + FLMUINT uiSaveFirstCPBlkAddr = 0; + FLMBOOL bDone = FALSE; + F_CachedBlock * pUsedSCache; + F_CachedBlock * pNextSCache = NULL; + F_CachedBlock ** ppUsedBlocks = (F_CachedBlock **)((m_ppBlocksDone) + ? &m_ppBlocksDone [m_uiBlocksDoneArraySize] + : (F_CachedBlock **)NULL); + FLMUINT uiTotalLoggedBlocks = 0; + FLMBOOL bForceCheckpoint = *pbForceCheckpoint; + FLMBOOL bDoAsync; +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags; +#endif + + m_uiCurrLogWriteOffset = 0; + bDoAsync = (gv_SFlmSysData.bOkToDoAsyncWrites && pSFileHdl->canDoAsync()) + ? TRUE + : FALSE; + + // Get the correct log header. If we are in an update transaction, + // need to use the uncommitted log header. Otherwise, use the last + // committed log header. + + pDbHdr = bIsCPThread + ? &m_lastCommittedDbHdr + : &m_uncommittedDbHdr; + + uiLogEof = (FLMUINT)pDbHdr->ui32RblEOF; + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + pDirtySCache = m_pFirstInLogList; + uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + flmAssert( m_pCurrLogBuffer == NULL); + + uiDirtyCacheLeft = (m_uiDirtyCacheCount + m_uiLogCacheCount) * + m_uiBlockSize; + + for (;;) + { + if( !pDirtySCache) + { + bDone = TRUE; + goto Write_Log_Blocks; + } + + flmAssert( pDirtySCache->m_ui16Flags & CA_DIRTY); + flmAssert( pDirtySCache->m_ui16Flags & CA_IN_FILE_LOG_LIST); + + // See if we should give up our write lock. Will do so if we + // are not forcing a checkpoint and we have not exceeded the + // maximum time since the last checkpoint AND the dirty cache + // left is below the maximum. + + if (!bForceCheckpoint && bIsCPThread) + { + if (scaSeeIfForceCheckpoint( uiCurrTime, m_uiLastCheckpointTime, + m_pCPInfo)) + { + bForceCheckpoint = TRUE; + } + else + { + if (m_pWriteLockObj->ThreadWaitingLock() && + uiDirtyCacheLeft <= uiMaxDirtyCache) + { + bDone = TRUE; + *pbWroteAll = FALSE; + goto Write_Log_Blocks; + } + } + } + + uiPrevBlkAddress = pDirtySCache->getPriorImageAddress(); + if (uiPrevBlkAddress) + { + // We shouldn't find anything in the log list that has + // already been logged. However, if we do find something, + // we will deal with it rather than returning an error. + + flmAssert( 0); + pTmpSCache = pDirtySCache->m_pNextInReplaceList; + pDirtySCache->unlinkFromLogList(); + pDirtySCache = pTmpSCache; + continue; + } + + // The replace list pointers are used to maintain links + // between items in the file log list + + pTmpSCache = pDirtySCache->m_pNextInVersionList; + pLastBlockToLog = NULL; + pFirstBlockToLog = NULL; + + // Grab the next block in the chain and see if we are done. + // NOTE: pDirtySCache should not be accessed in the loop + // below, because it has been changed to point to the + // next cache block in the log list. If you need to access + // the current block, use pSavedSCache. + + pSavedSCache = pDirtySCache; + if ((pDirtySCache = pDirtySCache->m_pNextInReplaceList) == NULL) + { + bDone = TRUE; + } +#ifdef FLM_DEBUG + else + { + flmAssert( pDirtySCache->m_ui16Flags & CA_DIRTY); + flmAssert( pDirtySCache->m_ui16Flags & CA_IN_FILE_LOG_LIST); + } +#endif + + // Traverse down the list of prior versions of the block until + // we hit one that has a prior version on disk. Throw out + // any not marked as CA_WRITE_TO_LOG, CA_LOG_FOR_CP, and + // not needed by a read transaction. + + while (pTmpSCache) + { + pNextSCache = pTmpSCache->m_pNextInVersionList; + FLMBOOL bWillLog; + + uiPrevBlkAddress = pTmpSCache->getPriorImageAddress(); + + // If we determine that we need to log a block, put a use on the + // newer version of the block to prevent other threads from verifying + // their checksums while we are writing the older versions to + // the log. This is because lgOutputBlock may modify information + // in the newer block's header area. + + if (pTmpSCache->m_ui16Flags & CA_READ_PENDING) + { + + // No need to go further down the list if this block is + // being read in. If it is being read in, every older + // version has a path to it - otherwise, it would never + // have been written out so that it would need to be + // read back in. + + break; + } + else if (pTmpSCache->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP)) + { + bWillLog = TRUE; + } + + // Even if the block is not needed by a read transaction, if it + // has a use count, we need to log it so that all blocks between + // pFirstBlockToLog and pLastBlockToLog are logged. This is + // necessary to ensure that previous block addresses carry all + // the way up the version chain. Also, the loop that does the + // actual logging below assumes that the links from pLastBlockToLog + // to pFirstBlockToLog will NOT be altered - even though the mutex + // is not locked. This can only be ensured if every block between + // the two points is guaranteed to be logged - which also guarantees + // that it will not be moved out of the list - because of the fact + // that some sort of logging bit has been set. + // Note that a block can have a use count even though it is no + // longer needed by a read transaction because another thread + // may have temporarily put a use on it while traversing down + // the chain - or for any number of other reasons. + + else if (pTmpSCache->neededByReadTrans() || + pTmpSCache->m_uiUseCount) + { + bWillLog = TRUE; + } + else + { + bWillLog = FALSE; + + // Since the block is no longer needed by a read transaction, + // and it is not in use, free it + + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + + // Add this block to the list of those we will be logging if the + // bWillLog flag got set above. + + if (bWillLog) + { + if (uiTotalLoggedBlocks >= m_uiBlocksDoneArraySize) + { + if (RC_BAD( rc = allocBlocksArray( 0, FALSE))) + { + goto Exit; + } + ppUsedBlocks = &m_ppBlocksDone [m_uiBlocksDoneArraySize]; + } + + pLastBlockToLog = pTmpSCache; + if (!pFirstBlockToLog) + { + pFirstBlockToLog = pLastBlockToLog; + } + + pTmpSCache->m_pPrevInVersionList->useForThread( 0); + pTmpSCache->useForThread( 0); + m_ppBlocksDone [uiTotalLoggedBlocks] = pTmpSCache; + ppUsedBlocks [uiTotalLoggedBlocks] = pTmpSCache->m_pPrevInVersionList; + uiTotalLoggedBlocks++; + } + + // No need to go further down the list if this block has + // has a previous block address. + + if (uiPrevBlkAddress) + { + break; + } + pTmpSCache = pNextSCache; + } + +#ifdef FLM_DEBUG + while (pNextSCache) + { + flmAssert( !(pNextSCache->m_ui16Flags & + (CA_WRITE_TO_LOG | CA_LOG_FOR_CP))); + pNextSCache = pNextSCache->m_pNextInVersionList; + + } +#endif + + // If nothing to log for the block, unlink it from the + // log list. We check CA_IN_FILE_LOG_LIST again, because + // savePrevBlkAddress may have been called during an + // unlink above. savePrevBlkAddress will remove + // the dirty cache block from the log list if it determines + // that there is no need to log prior versions + + if (!pLastBlockToLog) + { + if (pSavedSCache->m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + pSavedSCache->unlinkFromLogList(); + } + continue; + } + + // Don't want the mutex locked while we do the I/O + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + // Write the log blocks to the rollback log. + // Do all of the blocks from oldest to most current. Stop when we + // hit the first log block. + + while (pLastBlockToLog) + { + FLMUINT uiLogPos = uiLogEof; + + if (RC_BAD( rc = lgOutputBlock( pDbStats, pSFileHdl, + pLastBlockToLog, + pLastBlockToLog->m_pPrevInVersionList->m_pBlkHdr, + bDoAsync, &uiLogEof))) + { + goto Exit; + } + + if (pLastBlockToLog->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP)) + { + flmAssert( uiDirtyCacheLeft >= m_uiBlockSize); + uiDirtyCacheLeft -= m_uiBlockSize; + } + + // If we are logging a block for the current update + // transaction, and this is the first block we have logged, + // remember the block address where we logged it. + + if ((pLastBlockToLog->m_ui16Flags & CA_WRITE_TO_LOG) && + !m_uiFirstLogBlkAddress) + { + // This better not EVER happen in the CP thread. + + flmAssert( !bIsCPThread); + bLoggedFirstBlk = TRUE; + m_uiFirstLogBlkAddress = uiLogPos; + } + + // If we are logging the checkpoint version of the + // block, and this is the first block we have logged + // since the last checkpoint, remember its position so + // that we can write it out to the log header when we + // complete the checkpoint. + + if ((pLastBlockToLog->m_ui16Flags & CA_LOG_FOR_CP) && + !m_uiFirstLogCPBlkAddress) + { + bLoggedFirstCPBlk = TRUE; + m_uiFirstLogCPBlkAddress = uiLogPos; + } + + // Break when we hit the first log block. + + if (pLastBlockToLog == pFirstBlockToLog) + { + break; + } + + pLastBlockToLog = pLastBlockToLog->m_pPrevInVersionList; + } + + // If we have logged some blocks, force the log header to be + // updated on one of the following conditions: + + // 1. We have logged over 2000 blocks. We do this to keep + // our array of logged blocks from growing too big. + // 2. We are done logging. + +Write_Log_Blocks: + + if (uiTotalLoggedBlocks && // Must be at least one logged block + (uiTotalLoggedBlocks >= 2000 || bDone)) + { + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Flush the last log buffer, if not already flushed. + + if (m_uiCurrLogWriteOffset) + { + if (RC_BAD( rc = lgFlushLogBuffer( pDbStats, pSFileHdl, + bDoAsync))) + { + goto Exit; + } + } + + // If doing async, wait for pending writes to complete before writing + // the log header. + + if (bDoAsync) + { + if (RC_BAD( rc = m_pBufferMgr->waitForAllPendingIO())) + { + goto Exit; + } + } + + // Must wait for all RFL writes before writing out log header. + + if (!bIsCPThread) + { + (void)m_pRfl->seeIfRflWritesDone( hWaitSem, TRUE); + } + + // Save the EOF address so we can restore it if + // the write fails. + + uiSaveEOFAddr = (FLMUINT)pDbHdr->ui32RblEOF; + pDbHdr->ui32RblEOF = (FLMUINT32)uiLogEof; + + if (bLoggedFirstCPBlk) + { + uiSaveFirstCPBlkAddr = pDbHdr->ui32RblFirstCPBlkAddr; + pDbHdr->ui32RblFirstCPBlkAddr = + (FLMUINT32)m_uiFirstLogCPBlkAddress; + } + + if (RC_BAD( rc = writeDbHdr( pDbStats, pSFileHdl, + pDbHdr, &m_checkpointDbHdr, FALSE))) + { + + // If the write of the log header fails, + // we want to restore the log header to what it was before + // because we always use the log header from memory instead + // of reading it from disk. The one on disk is only + // current for many fields as of the last checkpoint. + + pDbHdr->ui32RblEOF = (FLMUINT32)uiSaveEOFAddr; + if (bLoggedFirstCPBlk) + { + pDbHdr->ui32RblFirstCPBlkAddr = (FLMUINT32)uiSaveFirstCPBlkAddr; + } + goto Exit; + } + + // Need to update the committed log header when we are operating in + // an uncommitted transaction so that if the transaction turns out + // to be empty, we will have the correct values in the committed + // log header for subsequent transactions or the checkpoint thread + // itself. + + if (!bIsCPThread) + { + m_lastCommittedDbHdr.ui32RblEOF = pDbHdr->ui32RblEOF; + + if (bLoggedFirstCPBlk) + { + m_lastCommittedDbHdr.ui32RblFirstCPBlkAddr = + pDbHdr->ui32RblFirstCPBlkAddr; + } + } + + // Once the write is safe, we can reset things to start over. + + bLoggedFirstBlk = FALSE; + bLoggedFirstCPBlk = FALSE; + + // Clean up the log blocks array - releasing blocks, etc. + + if (m_pCPInfo) + { + lockMutex(); + m_pCPInfo->uiLogBlocksWritten += uiTotalLoggedBlocks; + unlockMutex(); + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + + while (uiTotalLoggedBlocks) + { + uiTotalLoggedBlocks--; + pTmpSCache = m_ppBlocksDone [uiTotalLoggedBlocks]; +#ifdef FLM_DBG_LOG + ui16OldFlags = pTmpSCache->m_ui16Flags; +#endif + pUsedSCache = ppUsedBlocks [uiTotalLoggedBlocks]; + + // Newer block should be released, whether we succeeded + // or not - because it will always have been used. + + pUsedSCache->releaseForThread(); + pTmpSCache->releaseForThread(); + + // The current version of the block may have already been removed from + // the file log list if more than one block in the version chain + // needed to be logged. If the block is still in the file log list, + // it will be removed. Otherwise, the prior image address better + // be a non-zero value. + + if( pUsedSCache->m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + pUsedSCache->unlinkFromLogList(); + } + + flmAssert( pUsedSCache->getPriorImageAddress()); + + // Unlink from list of transaction log blocks + + if (pTmpSCache->m_ui16Flags & CA_WRITE_TO_LOG) + { + pTmpSCache->unlinkFromTransLogList(); + } + + // Unset logging flags on logged block. + + if (pTmpSCache->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP)) + { + flmAssert( m_uiLogCacheCount); + m_uiLogCacheCount--; + } + + pTmpSCache->clearFlags( CA_LOG_FOR_CP | CA_WRITE_TO_LOG | CA_WAS_DIRTY); + +#ifdef FLM_DBG_LOG + pTmpSCache->logFlgChange( ui16OldFlags, 'D'); +#endif + + if (!pTmpSCache->m_uiUseCount && + !pTmpSCache->m_ui16Flags && + !pTmpSCache->neededByReadTrans()) + { + flmAssert( pTmpSCache->m_ui64HighTransID != ~((FLMUINT64)0)); + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + } + + uiDirtyCacheLeft = + (m_uiDirtyCacheCount + m_uiLogCacheCount) * + m_uiBlockSize; + + // When the current set of log blocks were flushed, they were + // also unlinked from the file log list. So, we need to + // start at the beginning of the log list to pick up + // where we left off. + + pDirtySCache = m_pFirstInLogList; + } + else if (!bDone) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + // Need to reset pDirtySCache here because the background cache + // cleanup thread may have unlinked it from the log list and + // cleaned up any prior versions if it determined that the blocks + // were no longer needed. + + if( (pDirtySCache = pSavedSCache->m_pNextInReplaceList) == NULL) + { + bDone = TRUE; + goto Write_Log_Blocks; + } + } + + if (bDone) + { + break; + } + + flmAssert( bMutexLocked); + } + +#ifdef FLM_DEBUG + if( bForceCheckpoint || !bIsCPThread || + (!bForceCheckpoint && bIsCPThread && *pbWroteAll)) + { + flmAssert( !m_uiLogListCount); + flmAssert( !m_uiLogCacheCount); + } +#endif + +Exit: + + if (RC_BAD( rc)) + { + // Flush the last log buffer, if not already flushed. + + if (m_uiCurrLogWriteOffset) + { + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Don't care what rc is at this point. Just calling + // lgFlushLogBuffer to clear the buffer. + + (void)lgFlushLogBuffer( pDbStats, pSFileHdl, bDoAsync); + } + + // Need to wait for any async writes to complete. + + if (bDoAsync) + { + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Don't care about rc here, but we don't want to leave + // this routine until all pending IO is taken care of. + + (void)m_pBufferMgr->waitForAllPendingIO(); + } + + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + // Clean up the log blocks array - releasing blocks, etc. + + while (uiTotalLoggedBlocks) + { + uiTotalLoggedBlocks--; + pTmpSCache = m_ppBlocksDone [uiTotalLoggedBlocks]; + pUsedSCache = ppUsedBlocks [uiTotalLoggedBlocks]; + +#ifdef FLM_DEBUG + + // If this is the most current version of the block, it + // should still be in the file log list. + + if( !pUsedSCache->m_pPrevInVersionList) + { + flmAssert( pUsedSCache->m_ui16Flags & CA_IN_FILE_LOG_LIST); + } +#endif + + // Used blocks should be released, whether we succeeded + // or not. + + pUsedSCache->releaseForThread(); + pTmpSCache->releaseForThread(); + + // If we quit before logging the blocks, we don't really + // want to change anything on the block, but we do want + // to set the previous block address back to zero on the + // block that is just newer than this one. + + // Must put a USE on the block so that the memory cache + // verifying code will not barf when we change the + // data in the block - checksum is calculated and set when + // the use count goes from one to zero, and then verified + // when it goes from zero to one. + + pTmpSCache->m_pPrevInVersionList->useForThread( 0); + pTmpSCache->m_pPrevInVersionList->m_pBlkHdr->ui32PriorBlkImgAddr = 0; + pTmpSCache->m_pPrevInVersionList->releaseForThread(); + } + +#ifdef SCACHE_LINK_CHECKING + + // If above logic changes where mutex might not be locked at + // this point, be sure to modify this code to re-lock it. + + flmAssert( bMutexLocked); + scaVerify( 100); +#endif + + // Things to restore to their original state if we had an error. + + if (bLoggedFirstBlk) + { + m_uiFirstLogBlkAddress = 0; + } + if (bLoggedFirstCPBlk) + { + m_uiFirstLogCPBlkAddress = 0; + } + } + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Better not be any incomplete writes at this point. + + flmAssert( !m_pBufferMgr->havePendingIO()); + flmAssert( m_pCurrLogBuffer == NULL); + + *pbForceCheckpoint = bForceCheckpoint; + return( rc); +} + +/**************************************************************************** +Desc: This routine is called whenever a write of a dirty block completes. +****************************************************************************/ +FSTATIC void scaWriteComplete( + IF_IOBuffer * pIOBuffer) +{ + RCODE rc = pIOBuffer->getCompletionCode(); + FLMUINT uiNumBlocks = pIOBuffer->getBufferSize() / + pIOBuffer->getBlockSize(); + F_CachedBlock * pSCache; + F_Database * pDatabase; + SFLM_DB_STATS * pDbStats = (SFLM_DB_STATS *)pIOBuffer->getStats(); + FLMUINT uiMilliPerBlock = 0; + FLMUINT uiExtraMilli = 0; + +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags; +#endif + + if (pDbStats) + { + FLMUINT64 ui64ElapMilli = pIOBuffer->getElapTime(); + + uiMilliPerBlock = (FLMUINT)(ui64ElapMilli / (FLMUINT64)uiNumBlocks); + uiExtraMilli = (FLMUINT)(ui64ElapMilli % (FLMUINT64)uiNumBlocks); + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + while (uiNumBlocks) + { + uiNumBlocks--; + pSCache = (F_CachedBlock *)pIOBuffer->getCompletionCallbackData( uiNumBlocks); + pDatabase = pSCache->getDatabase(); + + if (pDbStats) + { + F_BLK_HDR * pBlkHdr = pSCache->getBlockPtr(); + SFLM_LFILE_STATS * pLFileStats; + SFLM_BLOCKIO_STATS * pBlockIOStats; + + if (!blkIsBTree( pBlkHdr)) + { + pLFileStats = NULL; + } + else + { + if (RC_BAD( flmStatGetLFile( pDbStats, + (FLMUINT)((F_BTREE_BLK_HDR *)pBlkHdr)->ui16LogicalFile, + getBlkLfType( (F_BTREE_BLK_HDR *)pBlkHdr), + 0, &pLFileStats, NULL, NULL))) + { + pLFileStats = NULL; + } + } + if ((pBlockIOStats = flmGetBlockIOStatPtr( pDbStats, + pLFileStats, (FLMBYTE *)pBlkHdr)) != NULL) + { + pBlockIOStats->BlockWrites.ui64Count++; + pBlockIOStats->BlockWrites.ui64TotalBytes += + pDatabase->getBlockSize(); + if (uiExtraMilli) + { + pBlockIOStats->BlockWrites.ui64ElapMilli += + (uiMilliPerBlock + 1); + uiExtraMilli--; + } + else + { + pBlockIOStats->BlockWrites.ui64ElapMilli += + uiMilliPerBlock; + } + } + } + + pSCache->releaseForThread(); + if (pSCache->getModeFlags() & CA_DIRTY) + { + flmAssert( pSCache->getModeFlags() & CA_WRITE_PENDING); +#ifdef FLM_DBG_LOG + ui16OldFlags = pSCache->getModeFlags(); +#endif + pSCache->clearFlags( CA_WRITE_PENDING); + if (RC_OK( rc)) + { + pSCache->unsetDirtyFlag(); + } + +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'H'); +#endif + + // If there are more dirty blocks after this + // one, move this one out of the dirty + // blocks. + + pSCache->unlinkFromDatabase(); + pSCache->linkToDatabase( pDatabase); + } + else + { + flmAssert( !(pSCache->getModeFlags() & CA_WRITE_PENDING)); + } + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_BlockCacheMgr::cleanupLRUCache( void) +{ + FLMUINT uiByteThreshold; + FLMUINT uiSlabThreshold; + FLMUINT uiSlabSize; + F_CachedBlock * pPrevSCache; + F_CachedBlock * pTmpSCache; + FLMBOOL bDefragNeeded = FALSE; + + // Remove non-dirty blocks from the LRU end of the cache + + uiSlabThreshold = gv_SFlmSysData.pGlobalCacheMgr->m_uiMaxSlabs >> 1; + uiSlabSize = gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->getSlabSize(); + + // If the cache isn't over its slab threshold, we are done + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + + uiByteThreshold = m_Usage.uiByteCount > uiSlabSize + ? m_Usage.uiByteCount - uiSlabSize + : 0; + + pTmpSCache = (F_CachedBlock *)m_MRUList.m_pLRUItem; + while( pTmpSCache) + { + // Save the pointer to the previous entry in the list because + // we may end up unlinking pTmpSCache below, in which case we would + // have lost the next entry. + + pPrevSCache = (F_CachedBlock *)pTmpSCache->m_pPrevInGlobal; + + // Block must not currently be in use, cannot be dirty in any way, + // cannot be in the process of being read in from disk, + // and must not be needed by a read transaction. + + if( !pTmpSCache->m_uiUseCount && !pTmpSCache->m_ui16Flags && + (!pTmpSCache->m_pDatabase || !pTmpSCache->neededByReadTrans())) + { + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + bDefragNeeded = TRUE; + + if( m_Usage.uiByteCount <= uiByteThreshold) + { + if( pPrevSCache) + { + pPrevSCache->useForThread( 0); + } + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + bDefragNeeded = FALSE; + + if( !pPrevSCache) + { + break; + } + + pPrevSCache->releaseForThread(); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + + uiByteThreshold = m_Usage.uiByteCount > uiSlabSize + ? m_Usage.uiByteCount - uiSlabSize + : 0; + + } + } + + pTmpSCache = pPrevSCache; + } + + if( bDefragNeeded) + { + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + } + +Exit: + + return; +} + +/**************************************************************************** +Desc: Cleanup old blocks in cache that are no longer needed by any + transaction. This routine assumes that the block cache mutex has + been locked. +****************************************************************************/ +void F_BlockCacheMgr::cleanupReplaceList( void) +{ + F_CachedBlock * pTmpSCache; + F_CachedBlock * pPrevSCache; + + pTmpSCache = m_pLRUReplace; + + for (;;) + { + // Stop when we reach end of list or all old blocks have + // been freed. + + if (!pTmpSCache || !m_Usage.uiOldVerBytes) + { + break; + } + + // Shouldn't encounter anything with CA_FREE set + + flmAssert( !(pTmpSCache->m_ui16Flags & CA_FREE)); + + // Save the pointer to the previous entry in the list because + // we may end up unlinking pTmpSCache below, in which case we would + // have lost the next entry. + + pPrevSCache = pTmpSCache->m_pPrevInReplaceList; + + // Block must not currently be in use, + // Must not be the most current version of a block, + // Cannot be dirty in any way, + // Cannot be in the process of being read in from disk, + // And must not be needed by a read transaction. + + if (!pTmpSCache->m_uiUseCount && + pTmpSCache->m_ui64HighTransID != ~((FLMUINT64)0) && + !pTmpSCache->m_ui16Flags && + (!pTmpSCache->m_pDatabase || + !pTmpSCache->neededByReadTrans())) + { + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + pTmpSCache = pPrevSCache; + } +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_BlockCacheMgr::cleanupFreeCache( void) +{ + F_CachedBlock * pSCache = m_pLastFree; + F_CachedBlock * pPrevSCache; + + while( pSCache) + { + pPrevSCache = pSCache->m_pPrevInDatabase; + if( !pSCache->m_uiUseCount) + { + pSCache->unlinkFromFreeList(); + delete pSCache; + } + pSCache = pPrevSCache; + } +} + +/**************************************************************************** +Desc: This routine will reduce the number of blocks in the reuse list + until cache is below its limit. + NOTE: This routine assumes that the block cache mutex is already locked. +****************************************************************************/ +void F_BlockCacheMgr::reduceReuseList( void) +{ + F_CachedBlock * pTmpSCache; + F_CachedBlock * pPrevSCache; + FLMUINT uiSlabThreshold; + FLMUINT uiSlabSize; + FLMUINT uiByteThreshold; + + // Determine if the block limit for block cache been exceeded + + uiSlabThreshold = gv_SFlmSysData.pGlobalCacheMgr->m_uiMaxSlabs >> 1; + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold) + { + goto Exit; + } + + // Remove items from cache starting from the LRU + + pTmpSCache = m_pLRUReplace; + uiSlabSize = gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->getSlabSize(); + uiByteThreshold = m_Usage.uiByteCount > uiSlabSize + ? m_Usage.uiByteCount - uiSlabSize + : 0; + + while( pTmpSCache) + { + // Need to save the pointer to the previous entry in the list because + // we may end up freeing pTmpNode below. + + pPrevSCache = pTmpSCache->m_pPrevInReplaceList; + + // See if the item can be freed. + + if( pTmpSCache->canBeFreed()) + { + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + + if( m_Usage.uiByteCount <= uiByteThreshold) + { + if( pPrevSCache) + { + pPrevSCache->useForThread( 0); + } + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + if( !pPrevSCache) + { + break; + } + + pPrevSCache->releaseForThread(); + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold) + { + goto Exit; + } + + uiByteThreshold = m_Usage.uiByteCount > uiSlabSize + ? m_Usage.uiByteCount - uiSlabSize + : 0; + } + } + + pTmpSCache = pPrevSCache; + } + +Exit: + + return; +} + +/**************************************************************************** +Desc: Reduce cache to below the cache limit. NOTE: This routine assumes + that the block cache mutex is locked. It may temporarily unlock the mutex + to write out dirty blocks, but it will always return with the mutex + still locked. +****************************************************************************/ +RCODE F_BlockCacheMgr::reduceCache( + F_Db * pDb) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = pDb ? pDb->m_pDatabase : NULL; + FLMBOOL bForceCheckpoint; + FLMBOOL bWroteAll; + FLMUINT uiSlabSize; + FLMUINT uiSlabThreshold; + FLMBOOL bDoingReduce = FALSE; + + // If cache is not full, we are done. + + if( !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit() || m_bReduceInProgress) + { + goto Exit; + } + + m_bReduceInProgress = TRUE; + bDoingReduce = TRUE; + + // Determine the cache threshold + + uiSlabThreshold = gv_SFlmSysData.pGlobalCacheMgr->m_uiMaxSlabs >> 1; + uiSlabSize = gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager->getSlabSize(); + + // Are we over the threshold? + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold) + { + goto Exit; + } + + // Try cleaning up the replace list + + if( m_Usage.uiOldVerBytes) + { + cleanupReplaceList(); + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + } + + // Clean up the free list + + if( m_uiFreeBytes) + { + cleanupFreeCache(); + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + } + + // Clean up the LRU list + + cleanupLRUCache(); + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + + // If this isn't an update transaction, there isn't anything else + // that can be done to reduce cache. + + if( !pDb || + (pDb->m_eTransType != SFLM_UPDATE_TRANS && !pDatabase->m_bTempDb)) + { + goto Exit; + } + + // Flush log blocks + + if( pDatabase->m_pFirstInLogList) + { + bForceCheckpoint = FALSE; + bWroteAll = TRUE; + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + if( RC_BAD( rc = pDatabase->flushLogBlocks( + pDb->m_hWaitSem, pDb->m_pDbStats, pDb->m_pSFileHdl, FALSE, 0, + &bForceCheckpoint, &bWroteAll))) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + goto Exit; + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + cleanupFreeCache(); + reduceReuseList(); + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + } + + // Flush new blocks + + for( ;;) + { + FLMUINT uiNewBlocks = 0; + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + if( RC_BAD( rc = pDatabase->reduceNewBlocks( + pDb->m_pDbStats, pDb->m_pSFileHdl, &uiNewBlocks))) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + goto Exit; + } + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + if( !uiNewBlocks) + { + break; + } + + cleanupFreeCache(); + reduceReuseList(); + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + } + + // Flush dirty blocks + + flmAssert( !pDatabase->m_pFirstInLogList); + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + if( RC_BAD( rc = pDatabase->reduceDirtyCache( + pDb->m_pDbStats, pDb->m_pSFileHdl))) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + goto Exit; + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + cleanupFreeCache(); + reduceReuseList(); + + gv_SFlmSysData.pBlockCacheMgr->defragmentMemory( TRUE); + + // We're going to quit when we get under 50 percent for block cache + // or we aren't over the global limit. Note that this means we + // may quit reducing before we get under the global limit. We + // don't want to get into a situation where we are starving block + // cache because node cache is over its limit. + + if( m_Usage.slabUsage.ui64Slabs <= uiSlabThreshold || + !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + goto Exit; + } + + // Try cleaning up the LRU again + + cleanupLRUCache(); + +Exit: + + if( RC_BAD( rc) && pDb) + { + pDb->setMustAbortTrans( rc); + } + + if( bDoingReduce) + { + m_bReduceInProgress = FALSE; + } + + return( rc); +} + +/**************************************************************************** +Desc: Constructor for cached block. +****************************************************************************/ +F_CachedBlock::F_CachedBlock( + FLMUINT uiBlockSize) +{ + m_pPrevInDatabase = NULL; + m_pNextInDatabase = NULL; + m_pBlkHdr = (F_BLK_HDR *)((FLMBYTE *)this + sizeof( F_CachedBlock)); + m_pDatabase = NULL; + m_uiBlkAddress = 0; + m_pNextInReplaceList = NULL; + m_pPrevInReplaceList = NULL; + m_pPrevInHashBucket = NULL; + m_pNextInHashBucket = NULL; + m_pPrevInVersionList = NULL; + m_pNextInVersionList = NULL; + m_pNotifyList = NULL; + + // Need to set high transaction ID to 0xFFFFFFFF. This indicates that + // the block is not currently counted in the Usage.uiOldVerBytes tally - + // seeing as how it was just allocated. + // DO NOT USE setTransID routine here because that routine + // will adjust the tally. The caller of this routine should call + // setTransID to ensure that the tally is set appropriately. + // This is the only place in the code where it is legal to set + // ui64HighTransID without calling setTransID. + + m_ui64HighTransID = ~((FLMUINT64)0); + m_uiUseCount = 0; + m_ui16Flags = 0; + m_ui16BlkSize = (FLMUINT16)uiBlockSize; + +#ifdef FLM_DEBUG + m_uiChecksum = 0; + m_pUseList = NULL; +#endif +} + +/**************************************************************************** +Desc: Allocate a cache block. If we are at the cache limit, unused cache + blocks will be replaced. NOTE: This routine assumes that the block + cache mutex is locked. +****************************************************************************/ +RCODE F_BlockCacheMgr::allocBlock( + F_Db * pDb, + F_CachedBlock ** ppSCacheRV) +{ + RCODE rc = NE_SFLM_OK; + F_Database * pDatabase = pDb->getDatabase(); + FLMUINT uiBlockSize = pDatabase->getBlockSize(); + F_CachedBlock * pSCache; + F_CachedBlock * pTmpSCache; + F_CachedBlock * pPrevSCache; + + // Quick check to see if there is a block in the free list that can be + // re-used. Start at the MRU end of the list so that if items in the + // free list are only being used periodically, the items at the LRU end + // will age out and the size of the list will be reduced. + + pSCache = m_pFirstFree; + while (pSCache) + { + if (!pSCache->m_uiUseCount && + pSCache->getBlkSize() == uiBlockSize) + { + pSCache->unlinkFromFreeList(); + goto Reuse_Block; + } + pSCache = pSCache->m_pNextInDatabase; + } + + // The intent of this little loop is to be optimistic and hope that + // there is a block we can cannibalize or free without having to write + // it. If not, we will still allocate a new block and allow ourselves + // to be temporarily over the cache limit. In this case, the cache size + // will be reduced only AFTER this new block is safely linked into cache. + // This is necessary because we don't want two different threads allocating + // memory for the same block. + + pTmpSCache = m_pLRUReplace; + while( pTmpSCache && gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + // Need to save the pointer to the previous entry in the list because + // we may end up unlinking it below, in which case we would have lost + // the previous entry. + + pPrevSCache = pTmpSCache->m_pPrevInReplaceList; + + // See if the cache block can be replaced or freed. + + flmAssert( !pTmpSCache->m_ui16Flags); + if (pTmpSCache->canBeFreed()) + { + if (pTmpSCache->getBlkSize() == uiBlockSize) + { + pSCache = pTmpSCache; + flmAssert( !pSCache->m_ui16Flags); + pTmpSCache->unlinkCache( FALSE, NE_SFLM_OK); + + // We use a goto instead of a break because then + // we don't have to do the additional test + // down below. We already know that pSCache + // will be non-NULL. + + goto Reuse_Block; + } + else + { + // NOTE: This call will free the memory pointed to by + // pTmpSCache. Hence, pTmpSCache should NOT be used after + // this point. + + pTmpSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + } + pTmpSCache = pPrevSCache; + } + + // If we were not able to cannibalize an F_CachedBlock object, + // allocate one. + + if (pSCache) + { +Reuse_Block: + + flmAssert( !pSCache->m_pPrevInReplaceList); + flmAssert( !pSCache->m_pNextInReplaceList); + flmAssert( !pSCache->m_ui16Flags); + flmAssert( !pSCache->m_uiUseCount); + + // If block is an old version, need to decrement the + // Usage.uiOldVerBytes tally. + + if (pSCache->m_ui64HighTransID != ~((FLMUINT64)0)) + { + FLMUINT uiSize = pSCache->memSize(); + flmAssert( m_Usage.uiOldVerBytes >= uiSize); + m_Usage.uiOldVerBytes -= uiSize; + flmAssert( m_Usage.uiOldVerCount); + m_Usage.uiOldVerCount--; + } + + // If we are cannibalizing, be sure to reset certain fields. + + pSCache->m_ui16Flags = 0; + pSCache->m_uiUseCount = 0; +#ifdef FLM_DEBUG + pSCache->m_uiChecksum = 0; +#endif + + // Need to set high transaction ID to 0xFFFFFFFF. This indicates that + // the block is not currently counted in the Usage.uiOldVerBytes tally - + // seeing as how it was just allocated. + // DO NOT USE setTransID routine here because that routine + // will adjust the tally. The caller of this routine should call + // setTransID to ensure that the tally is set appropriately. + // This is the only place in the code where it is legal to set + // ui64HighTransID without calling setTransID. + + pSCache->m_ui64HighTransID = ~((FLMUINT64)0); + } + else + { + if ((pSCache = new( uiBlockSize) F_CachedBlock( uiBlockSize)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + m_Usage.uiCount++; + m_Usage.uiByteCount += pSCache->memSize(); + + if (shouldRehash( m_Usage.uiCount, m_uiNumBuckets)) + { + if (checkHashFailTime( &m_uiHashFailTime)) + { + if (RC_BAD( rc = rehash())) + { + goto Exit; + } + } + } + } + + *ppSCacheRV = pSCache; + + // Set use count to one so the block cannot be replaced. + + pSCache->useForThread( 0); + +Exit: + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/******************************************************************** +Desc: This converts a block header to native format. +*********************************************************************/ +void convertBlkHdr( + F_BLK_HDR * pBlkHdr + ) +{ + + // This routine should only be called on blocks that are NOT + // currently in native format. + + flmAssert( blkIsNonNativeFormat( pBlkHdr)); + + convert32( &pBlkHdr->ui32BlkAddr); + convert32( &pBlkHdr->ui32PrevBlkInChain); + convert32( &pBlkHdr->ui32NextBlkInChain); + convert32( &pBlkHdr->ui32PriorBlkImgAddr); + convert64( &pBlkHdr->ui64TransID); + convert32( &pBlkHdr->ui32BlkCRC); + convert16( &pBlkHdr->ui16BlkBytesAvail); + if (blkIsBTree( pBlkHdr)) + { + convert16( &(((F_BTREE_BLK_HDR *)pBlkHdr)->ui16LogicalFile)); + convert16( &(((F_BTREE_BLK_HDR *)pBlkHdr)->ui16NumKeys)); + } + blkSetNativeFormat( pBlkHdr); +} + +/******************************************************************** +Desc: This converts a logical file header structure +*********************************************************************/ +void convertLfHdr( + F_LF_HDR * pLfHdr) +{ + convert64( &pLfHdr->ui64NextRowId); + convert32( &pLfHdr->ui32LfType); + convert32( &pLfHdr->ui32RootBlkAddr); + convert32( &pLfHdr->ui32LfNum); + convert32( &pLfHdr->ui32EncDefNum); +} + +/******************************************************************** +Desc: This converts a block header to native format. +*********************************************************************/ +void convertBlk( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr) +{ + // This routine should only be called on blocks that are NOT + // currently in native format. + + convertBlkHdr( pBlkHdr); + if (pBlkHdr->ui8BlkType == BT_LFH_BLK) + { + FLMUINT uiPos = SIZEOF_STD_BLK_HDR; + FLMUINT uiEnd = blkGetEnd( uiBlockSize, SIZEOF_STD_BLK_HDR, + pBlkHdr); + F_LF_HDR * pLfHdr = (F_LF_HDR *)((FLMBYTE *)pBlkHdr + + SIZEOF_STD_BLK_HDR); + + // Only one block type requires further conversion. + + while (uiPos + sizeof( F_LF_HDR) <= uiEnd) + { + convertLfHdr( pLfHdr); + pLfHdr++; + uiPos += sizeof( F_LF_HDR); + } + } +} + +/******************************************************************** +Desc: This routine prepares a block for use after reading it in from + disk. It will convert the block to native format if necessary, + and will also verify the CRC on the block. We always want to + convert the block if we can, so even if the CRC is bad or the + block end is bad, we will attempt to do a convert if it is in + non-native format. +*********************************************************************/ +RCODE flmPrepareBlockForUse( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT32 ui32CRC; + FLMUINT16 ui16BlkBytesAvail; + FLMUINT uiBlkEnd; + FLMBOOL bBadBlkEnd; + + // Determine if we should convert the block here. + // Calculation of CRC should be on unconverted block. + + ui16BlkBytesAvail = pBlkHdr->ui16BlkBytesAvail; + if (blkIsNonNativeFormat( pBlkHdr)) + { + convert16( &ui16BlkBytesAvail); + } + + if( (FLMUINT)ui16BlkBytesAvail > uiBlockSize - blkHdrSize( pBlkHdr)) + { + uiBlkEnd = blkHdrSize( pBlkHdr); + bBadBlkEnd = TRUE; + } + else + { + uiBlkEnd = (blkIsNewBTree( pBlkHdr) + ? uiBlockSize + : uiBlockSize - (FLMUINT)ui16BlkBytesAvail); + bBadBlkEnd = FALSE; + } + + // CRC must be calculated BEFORE converting the block. + + ui32CRC = calcBlkCRC( pBlkHdr, uiBlkEnd); + + if( blkIsNonNativeFormat( pBlkHdr)) + { + convertBlk( uiBlockSize, pBlkHdr); + } + + if( ui32CRC != pBlkHdr->ui32BlkCRC || bBadBlkEnd) + { + rc = RC_SET( NE_SFLM_BLOCK_CRC); + goto Exit; + } + +Exit: + + return( rc); +} + +/******************************************************************** +Desc: This routine attempts to read a block from disk. It will + attempt the specified number of times. +*********************************************************************/ +RCODE F_Database::readTheBlock( + F_Db * pDb, + TMP_READ_STATS * pTmpReadStats, // READ statistics. + F_BLK_HDR * pBlkHdr, // Pointer to buffer where block is + // to be read into. + FLMUINT uiFilePos, // File position to be read from. If + // file position != block address, we + // are reading from the log. + FLMUINT uiBlkAddress // Block address that is to be read. + ) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesRead; + F_TMSTAMP StartTime; + FLMUINT64 ui64ElapMilli; + SFLM_DB_STATS * pDbStats = pDb->m_pDbStats; + + flmAssert( this == pDb->m_pDatabase); + + // We should NEVER be attempting to read a block address that is + // beyond the current logical end of file. + + if (!FSAddrIsBelow( uiBlkAddress, pDb->m_uiLogicalEOF)) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Read the block + + if (pDb->m_uiKilledTime) + { + rc = RC_SET( NE_SFLM_OLD_VIEW); + goto Exit; + } + + if (pTmpReadStats) + { + if (uiFilePos != uiBlkAddress) + { + pTmpReadStats->OldViewBlockReads.ui64Count++; + pTmpReadStats->OldViewBlockReads.ui64TotalBytes += + m_uiBlockSize; + } + else + { + pTmpReadStats->BlockReads.ui64Count++; + pTmpReadStats->BlockReads.ui64TotalBytes += + m_uiBlockSize; + } + ui64ElapMilli = 0; + f_timeGetTimeStamp( &StartTime); + } + + if (RC_BAD( rc = pDb->m_pSFileHdl->readBlock( uiFilePos, + m_uiBlockSize, pBlkHdr, &uiBytesRead))) + { + if (pDbStats) + { + pDbStats->uiReadErrors++; + } + + if (rc == NE_FLM_IO_END_OF_FILE) + { + + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); + } + goto Exit; + } + + if (pTmpReadStats) + { + flmAddElapTime( &StartTime, &ui64ElapMilli); + if (uiFilePos != uiBlkAddress) + { + pTmpReadStats->OldViewBlockReads.ui64ElapMilli += ui64ElapMilli; + } + else + { + pTmpReadStats->BlockReads.ui64ElapMilli += ui64ElapMilli; + } + } + + if (uiBytesRead < m_uiBlockSize) + { + + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); +#ifdef FLM_DBG_LOG + // Must make this call so we can be ensured that the + // transaction ID in the block header has been + // converted if need be. + (void)flmPrepareBlockForUse( m_uiBlockSize, pBlkHdr); +#endif + } + else + { + rc = flmPrepareBlockForUse( m_uiBlockSize, pBlkHdr); + } + + // Decrypt the block if it was encrypted + + if (RC_BAD( rc = decryptBlock( pDb->m_pDict, (FLMBYTE *)pBlkHdr))) + { + goto Exit; + } + +#ifdef FLM_DBG_LOG + if (uiFilePos != uiBlkAddress) + { + flmDbgLogWrite( this, uiBlkAddress, uiFilePos, + (FLMUINT)pBlkHdr->ui64TransID, "LGRD"); + } + else + { + flmDbgLogWrite( this, uiBlkAddress, 0, + pBlkHdr->ui64TransID, "READ"); + } +#endif + + if (RC_BAD( rc)) + { + if (pTmpReadStats && + (rc == NE_SFLM_BLOCK_CRC || rc == NE_SFLM_OLD_VIEW)) + { + if (uiFilePos != uiBlkAddress) + { + pTmpReadStats->uiOldViewBlockChkErrs++; + } + else + { + pTmpReadStats->uiBlockChkErrs++; + } + } + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: Read a data block into cache. This routine reads the requested + version of a block into memory. It follows links to previous + versions of the block if necessary in order to do this. +****************************************************************************/ +RCODE F_Database::readBlock( + F_Db * pDb, + LFILE * pLFile, // Pointer to logical file structure + // We are retrieving the block for. + // NULL if there is no logical file. + FLMUINT uiFilePos, // File position where we are to + // start reading from. + FLMUINT uiBlkAddress, // Address of block that is to + // be read into cache. + FLMUINT64 ui64NewerBlkLowTransID, + // Low transaction ID of the last newer + // version of the block. + // NOTE: This has no meaning + // when uiFilePos == uiBlkAddress. + F_CachedBlock * pSCache, // Cache block to read the data + // into. + FLMBOOL * pbFoundVerRV, // Returns a flag to the caller to + // tell it whether it found any + // versions of the requested block + // starting at the file position + // that was passed in. + FLMBOOL * pbDiscardRV // Returns a flag which, if TRUE, + // tells the caller to discard + // the block that was just read + // in and set the high transaction ID + // on the block that comes just + // after it - because they are + // the same version. + ) +{ + RCODE rc = NE_SFLM_OK; + F_BLK_HDR * pBlkHdr = pSCache->m_pBlkHdr; + F_CachedBlock * pNextSCache; + FLMBOOL bMutexLocked = FALSE; + SFLM_LFILE_STATS * pLFileStats; + SFLM_BLOCKIO_STATS * pBlockIOStats; + FLMBOOL bIncrPriorImageCnt = FALSE; + FLMBOOL bIncrOldViewCnt = FALSE; + TMP_READ_STATS TmpReadStats; + TMP_READ_STATS * pTmpReadStats; + SFLM_DB_STATS * pDbStats = pDb->m_pDbStats; + + flmAssert( this == pDb->m_pDatabase); + + *pbFoundVerRV = FALSE; + *pbDiscardRV = FALSE; + + if (pDbStats) + { + f_memset( &TmpReadStats, 0, sizeof( TmpReadStats)); + pTmpReadStats = &TmpReadStats; + } + else + { + pTmpReadStats = NULL; + } + + // Read in the block from the database + + // Stay in a loop reading until we get an error or get the block + + for (;;) + { + if (pDbStats) + { + if (uiFilePos != uiBlkAddress) + { + bIncrPriorImageCnt = TRUE; + } + bIncrOldViewCnt = FALSE; + } + + // Read and verify the block. + + if (RC_BAD( rc = readTheBlock( pDb, pTmpReadStats, + pBlkHdr, uiFilePos, uiBlkAddress))) + { + goto Exit; + } + pBlkHdr->ui8BlkFlags &= ~(BLK_IS_BEFORE_IMAGE); + + // See if we can use the current version of the block, or if we + // must go get a previous version. + + // See if we even got the block we thought we wanted. + + if ((FLMUINT)pBlkHdr->ui32BlkAddr != uiBlkAddress) + { + if (uiFilePos == uiBlkAddress) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + } + else + { + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); + } + + goto Exit; + } + + // This flag is set to true to indicate that we found at least one + // version of the requested block. NOTE: This flag does NOT mean + // that we found the specific version requested, only that we + // found some version starting at the given address. + + *pbFoundVerRV = TRUE; + + // Check to see if the transaction range for the block we just read + // overlaps the transaction range for next older version of the block + // in the version list. If the ranges overlap, the transaction ID on + // each block had better be the same or we have a corruption. If the + // transaction IDs are the same, they are the same version of the block, + // and we can discard the one we just read. + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + +Get_Next_Block: + + if ((pNextSCache = pSCache->m_pNextInVersionList) != NULL) + { + FLMUINT64 ui64TmpTransID1; + FLMUINT64 ui64TmpTransID2; + + // If next block is still being read in, we must wait for + // it to complete before looking at its transaction IDs. + + if (pNextSCache->m_ui16Flags & CA_READ_PENDING) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiIoWaits++; + if (RC_BAD( rc = flmWaitNotifyReq( + gv_SFlmSysData.hBlockCacheMutex, pDb->m_hWaitSem, + &pNextSCache->m_pNotifyList, (void *)&pNextSCache))) + { + goto Exit; + } + + // The thread doing the notify "uses" the cache block + // on behalf of this thread to prevent the cache block + // from being flushed after it unlocks the mutex. + // At this point, since we have locked the mutex, + // we need to release the cache block - because we + // will put a "use" on it below. + + pNextSCache->releaseForThread(); + + // See if we still have the same next block. + + goto Get_Next_Block; + } + + // Check for overlapping trans ID ranges. NOTE: At this + // point, if we have an overlap, we know we have the version + // of the block we need (see comment above). Hence, we will + // either break out of the loop at this point or goto exit + // and return an error. + + ui64TmpTransID1 = pBlkHdr->ui64TransID; + if (ui64TmpTransID1 <= pNextSCache->m_ui64HighTransID) + { + ui64TmpTransID2 = pNextSCache->getLowTransID(); + + // If the low trans IDs on the two blocks are not equal + // we have a corruption. + + if (ui64TmpTransID1 != ui64TmpTransID2) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // The blocks are the same, discard one of them. + + *pbDiscardRV = TRUE; + + // Set the high trans ID on the block we are NOT discarding. + // To find the version of the block we want, we have been + // reading through a chain of blocks, from newer versions to + // progressively older versions. If uiFilePos == uiBlkAddress, + // we are positioned on the most current version of the block. + // In this case, the high trans ID for the block should be + // set to the highest possible value. + + // If uiFilePos != uiBlkAddress, we are positioned on an older + // version of the block. The variable ui64NewerBlkLowTransID + // contains the low transaction ID for a newer version of the + // block we read just prior to reading this block. + + if (pDb->m_eTransType == SFLM_UPDATE_TRANS || m_bTempDb || + uiFilePos == uiBlkAddress) + { + pNextSCache->setTransID( ~((FLMUINT64)0)); + } + else + { + pNextSCache->setTransID( (ui64NewerBlkLowTransID - 1)); + } + + // When discard flag is TRUE, we need to go right to + // exit, because we don't want to decrypt, do sanity + // check, etc. NOTE: mutex is still locked, and + // we want it to remain locked - see code at Exit. + + goto Exit; + } + } + + // See if this version of the block is what we want + + if (pBlkHdr->ui64TransID <= pDb->m_ui64CurrTransID) + { + + // Set the high trans ID on the block. If we are in an + // update transaction, or uiFilePos == uiBlkAddress, we + // are positioned on the most current version of the block. + // In this case, the high trans ID for the block should be + // set to ~((FLMUINT64)0). Otherwise we are positioned on an older + // version of the block, and the block's high transaction ID + // should be set to the newer block's low transaction ID minus one. + + if (pDb->m_eTransType == SFLM_UPDATE_TRANS || m_bTempDb || + uiFilePos == uiBlkAddress) + { + pSCache->setTransID( ~((FLMUINT64)0)); + } + else + { + pSCache->setTransID( (ui64NewerBlkLowTransID - 1)); + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + break; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + // At this point, we know we are going to have to get a prior + // version of the block. In an update transaction, this is + // indicative of a file corruption. + + if (pDb->m_eTransType != SFLM_READ_TRANS) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // At this point, we know we are in a read transaction. Save the + // block's low trans ID. + + ui64NewerBlkLowTransID = pBlkHdr->ui64TransID; + + // See if there is a prior version of the block and determine whether + // it's expected trans ID is in the range we need. + // NOTE: If the prior version address is zero or is the same as our + // current file position, there is no previous version of the block. + + if ((FLMUINT)pBlkHdr->ui32PriorBlkImgAddr == uiFilePos) + { + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); + goto Exit; + } + uiFilePos = (FLMUINT)pBlkHdr->ui32PriorBlkImgAddr; + if (!uiFilePos) + { + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); + goto Exit; + } + } + + // Perform a sanity check on the block header. + + if ((FLMUINT)pBlkHdr->ui16BlkBytesAvail > + m_uiBlockSize - blkHdrSize( pBlkHdr)) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + +Exit: + + // NOTE: When we are discarding the block, we CANNOT unlock + // the mutex, because we have to take care of it on + // the outside. Mutex better be locked if we are discarding. + + if (*pbDiscardRV) + { + flmAssert( bMutexLocked); + } + else if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + // If we got an old view error, it has to be a corruption, unless we + // were killed. + + if (rc == NE_SFLM_OLD_VIEW) + { + if (!pDb->m_uiKilledTime || pDb->m_eTransType == SFLM_UPDATE_TRANS || + m_bTempDb) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + } + } + + // Increment cache fault statistics + + if (pDbStats) + { + if ((pLFileStats = pDb->getLFileStatPtr( pLFile)) == NULL) + { + pBlockIOStats = flmGetBlockIOStatPtr( pDbStats, NULL, (FLMBYTE *)pBlkHdr); + } + else if (RC_BAD( rc)) + { + // Didn't really get a valid block, assign all statistics + // gathered to the leaf block statistics. + + pBlockIOStats = &pLFileStats->LeafBlockStats; + } + else + { + pBlockIOStats = flmGetBlockIOStatPtr( pDbStats, + pLFileStats, (FLMBYTE *)pBlkHdr); + } + + if (pBlockIOStats) + { + pDbStats->bHaveStats = TRUE; + if (pLFileStats) + { + pLFileStats->bHaveStats = TRUE; + } + + flmUpdateDiskIOStats( &pBlockIOStats->BlockReads, + &TmpReadStats.BlockReads); + + flmUpdateDiskIOStats( &pBlockIOStats->OldViewBlockReads, + &TmpReadStats.OldViewBlockReads); + + pBlockIOStats->uiBlockChkErrs += + TmpReadStats.uiBlockChkErrs; + + pBlockIOStats->uiOldViewBlockChkErrs += + TmpReadStats.uiOldViewBlockChkErrs; + if (rc == NE_SFLM_OLD_VIEW || bIncrOldViewCnt) + { + pBlockIOStats->uiOldViewErrors++; + } + } + } + + return( rc); +} + +/**************************************************************************** +Desc: Increment the use count on a cache block for a particular + thread. NOTE: This routine assumes that the block cache mutex + is locked. +****************************************************************************/ +#ifdef FLM_DEBUG +void F_CachedBlock::dbgUseForThread( + FLMUINT uiThreadId) +{ + SCACHE_USE * pUse; + FLMUINT uiMyThreadId = (FLMUINT)(!uiThreadId + ? (FLMUINT)f_threadId() + : uiThreadId); + + // If the use count is 0, make sure there are not entries + // in the use list. + + if (!m_uiUseCount && m_pUseList != NULL) + { + return; + } + + // First increment the overall use count for the block + + m_uiUseCount++; + if (m_uiUseCount == 1) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiBlocksUsed++; + if (m_uiChecksum) + { + flmAssert( m_uiChecksum == computeChecksum()); + } + } + gv_SFlmSysData.pBlockCacheMgr->m_uiTotalUses++; + + // Now add the thread's usage record - or increment it if there + // is already one there for the thread. + + // See if we already have this thread in the use list + + pUse = m_pUseList; + while (pUse && pUse->uiThreadId != uiMyThreadId) + { + pUse = pUse->pNext; + } + + if (!pUse) + { + if (RC_BAD( f_calloc( (FLMUINT)sizeof( SCACHE_USE), + &pUse))) + { + return; + } + + f_memset( pUse, 0, sizeof( SCACHE_USE)); + pUse->uiThreadId = uiMyThreadId; + pUse->pNext = m_pUseList; + m_pUseList = pUse; + } + + pUse->uiUseCount++; +} +#endif + +/**************************************************************************** +Desc: Decrement the use count on a cache block for a particular + thread. NOTE: This routine assumes that the block cache mutex + is locked. +****************************************************************************/ +#ifdef FLM_DEBUG +void F_CachedBlock::dbgReleaseForThread( void) +{ + SCACHE_USE * pUse; + SCACHE_USE * pPrevUse; + FLMUINT uiMyThreadId = (FLMUINT)f_threadId(); + + // Find the thread's use + + pUse = m_pUseList; + pPrevUse = NULL; + while (pUse && pUse->uiThreadId != uiMyThreadId) + { + pPrevUse = pUse; + pUse = pUse->pNext; + } + + if (!pUse) + { + return; + } + + m_uiUseCount--; + gv_SFlmSysData.pBlockCacheMgr->m_uiTotalUses--; + if (!m_uiUseCount) + { + m_uiChecksum = computeChecksum(); + gv_SFlmSysData.pBlockCacheMgr->m_uiBlocksUsed--; + flmAssert( pUse->uiUseCount == 1); + } + + // Free the use record if its count goes to zero + + pUse->uiUseCount--; + if (!pUse->uiUseCount) + { + if (!pPrevUse) + { + m_pUseList = pUse->pNext; + } + else + { + pPrevUse->pNext = pUse->pNext; + } + f_free( &pUse); + } +} +#endif + +/**************************************************************************** +Desc: Read a data block into cache. This routine takes care of allocating + a cache block and reading the block from disk into memory. NOTE: + This routine assumes that the block cache mutex is locked. It may + unlock the block cache mutex long enough to do the read, but the + mutex will still be locked when it exits. +****************************************************************************/ +RCODE F_Database::readIntoCache( + F_Db * pDb, + LFILE * pLFile, // Pointer to logical file structure + // We are retrieving the block for. + // NULL if there is no logical file. + FLMUINT uiBlkAddress, // Address of block that is to + // be read into cache. + F_CachedBlock * pPrevInVerList, // Previous block in version list to + // link the block to. + F_CachedBlock * pNextInVerList, // Next block in version list to link + // the block to. + F_CachedBlock ** ppSCacheRV, // Returns allocated cache block. + FLMBOOL * pbGotFromDisk) // Returns TRUE if block was read + // from disk +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache; + F_CachedBlock * pTmpSCache; + FNOTIFY * pNotify; + FLMUINT uiFilePos; + FLMUINT64 ui64NewerBlkLowTransID = 0; + FLMBOOL bFoundVer; + FLMBOOL bDiscard; + + flmAssert( this == pDb->m_pDatabase); + + *pbGotFromDisk = FALSE; + + // Lock the prev and next in place by incrementing their use + // count. We don't want allocBlock to use them. + + if (pPrevInVerList) + { + pPrevInVerList->useForThread( 0); + } + + if (pNextInVerList) + { + pNextInVerList->useForThread( 0); + } + + // Allocate a cache block - either a new one or by replacing + // an existing one. + + rc = gv_SFlmSysData.pBlockCacheMgr->allocBlock( pDb, &pSCache); + + if (pPrevInVerList) + { + pPrevInVerList->releaseForThread(); + } + + if (pNextInVerList) + { + pNextInVerList->releaseForThread(); + } + + if (RC_BAD( rc)) + { + goto Exit; + } + + pSCache->m_uiBlkAddress = uiBlkAddress; + + // Set the "dummy" flag so that we won't incur the overhead of + // linking the block into the replace list. It would be removed + // from the replace list almost immediately anyway, when the + // "read pending" flag is set below. + + pSCache->m_ui16Flags |= CA_DUMMY_FLAG; + + // Link block into various lists + + if( pDb->m_uiFlags & FDB_DONT_POISON_CACHE) + { + if( !(pDb->m_uiFlags & FDB_BACKGROUND_INDEXING) || + (pLFile && pLFile->eLfType != SFLM_LF_INDEX)) + { + pSCache->linkToGlobalListAsLRU(); + } + else + { + pSCache->linkToGlobalListAsMRU(); + } + } + else + { + pSCache->linkToGlobalListAsMRU(); + } + + pSCache->linkToDatabase( this); + if (!pPrevInVerList) + { + F_CachedBlock ** ppSCacheBucket; + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_uiSigBitsInBlkSize, uiBlkAddress); + uiFilePos = uiBlkAddress; + if (pNextInVerList) + { + pNextInVerList->unlinkFromHashBucket( ppSCacheBucket); + } + pSCache->linkToHashBucket( ppSCacheBucket); + } + else + { + uiFilePos = pPrevInVerList->getPriorImageAddress(); + ui64NewerBlkLowTransID = pPrevInVerList->getLowTransID(); + pPrevInVerList->m_pNextInVersionList = pSCache; + pPrevInVerList->verifyCache( 2400); + } + + if (pNextInVerList) + { + pNextInVerList->m_pPrevInVersionList = pSCache; + pNextInVerList->verifyCache( 2500); + } + + pSCache->m_pPrevInVersionList = pPrevInVerList; + pSCache->m_pNextInVersionList = pNextInVerList; + pSCache->verifyCache( 2600); + + // Set the read-pending flag for this block. This will force other + // threads that need to read this block to wait for the I/O to + // complete. + + pSCache->setFlags( CA_READ_PENDING); + pSCache->m_ui16Flags &= ~CA_DUMMY_FLAG; + gv_SFlmSysData.pBlockCacheMgr->m_uiPendingReads++; + + // Unlock the mutex and attempt to read the block into memory + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + rc = readBlock( pDb, pLFile, uiFilePos, uiBlkAddress, + ui64NewerBlkLowTransID, + pSCache, &bFoundVer, &bDiscard); + + // NOTE: If the bDiscard flag is TRUE, the mutex will still be + // locked. If FALSE, we need to relock it. + + if (!bDiscard) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + } + + // Get a pointer to the notify list BEFORE discarding the cache + // block - if we are going to discard - because pSCache can + // change if we discard. + + pNotify = pSCache->m_pNotifyList; + pSCache->m_pNotifyList = NULL; + + // Unset the read pending flag and reset the use count to zero. + // Both of these actions should be done before doing a discard, + // if a discard is going to be done. + + pSCache->clearFlags( CA_READ_PENDING); + gv_SFlmSysData.pBlockCacheMgr->m_uiPendingReads--; + pSCache->releaseForThread(); + + // If we had no errors, take care of some other things + + if (RC_OK( rc)) + { + // The bDiscard flag tells us that we should discard the + // block that we just read and use the next block in the + // version list - because they are the same version. + + if (bDiscard) + { + + // NOTE: We are guaranteed that pSCache->m_pNextInVersionList + // is non-NULL at this point, because when we set the + // bDiscard flag to TRUE, it was non-NULL, and we know that + // the mutex was NOT unlocked in that case. + + pTmpSCache = pSCache->m_pNextInVersionList; + pSCache->unlinkCache( TRUE, NE_SFLM_OK); + pSCache = pTmpSCache; + } + else + { + *pbGotFromDisk = TRUE; + } + } + + // Notify all of the waiters of the read result. + // IMPORTANT NOTE: This should be the LAST thing that is + // done except for unlink the block below in the case of + // an error having occurred. + + ScaNotify( pNotify, pSCache, rc); + + // If we had a BAD rc, unlink the block from the lists it is in and + // free the memory. + + if (RC_BAD( rc)) + { + pSCache->unlinkCache( TRUE, NE_SFLM_OK); + goto Exit; + } + + *ppSCacheRV = pSCache; + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine frees all cache blocks that have been modified by + the update transaction. This routine is called whenever a + transaction is to be aborted. +****************************************************************************/ +void F_Database::freeModifiedBlocks( + FLMUINT64 ui64CurrTransId) +{ + F_CachedBlock * pSCache; + F_CachedBlock * pNextSCache; + FLMBOOL bFirstPass = TRUE; + FLMBOOL bFreedAll; + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + // Unlink all log blocks and reset their flags so they + // won't be marked as needing to be written to disk. + + unlinkTransLogBlocks(); + +Do_Free_Pass: + + pSCache = m_pSCacheList; + flmAssert( !m_pPendingWriteList); + bFreedAll = TRUE; + while (pSCache) + { + + // If the high transaction ID on the block is one less than this + // transaction's ID, the block is the most current block. Therefore, + // its high transaction ID should be reset to ~((FLMUINT64)0). + + if (pSCache->m_ui64HighTransID == ui64CurrTransId - 1) + { + pSCache->setTransID( ~((FLMUINT64)0)); + + // Need to link blocks that become the current version again + // into the file log list if they are dirty. linkToLogList + // will check to see if the block has already been logged. If it has, + // it won't be linked into the list. + // NOTE: If the blocks were in the "new" list originally, we don't take + // the time to put them back into that list because they would have to + // be inserted in order. They will still get written out eventually, but + // they won't be written out by the reduceNewBlocks call. + + if (pSCache->m_ui16Flags & CA_DIRTY) + { + pSCache->linkToLogList(); + } + } + else if (pSCache->m_ui64HighTransID == ~((FLMUINT64)0) && + pSCache->getLowTransID() >= ui64CurrTransId && + !(pSCache->m_ui16Flags & CA_READ_PENDING)) + + { + pNextSCache = pSCache->m_pNextInDatabase; + + // Another thread might have a temporary "use" on this + // block. Unlock the mutex long enough to allow the + // other thread(s) to get rid of their "uses". Then start + // from the top of the list again. + + if (pSCache->m_uiUseCount) + { + + // Don't want to unlock the mutex during the first pass + // because it opens the door to the prior version of one of + // these modified blocks being removed from cache before we + // have a chance to reset its ui64HighTransID back to + // ~((FLMUINT64)0). + // During the first pass, we want to get through all of the + // blocks so that the code up above will get exercised for + // each such block. + + if (bFirstPass) + { + bFreedAll = FALSE; + pSCache = pNextSCache; + continue; + } + else + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + f_sleep( 10); + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + pSCache = m_pSCacheList; + continue; + } + } + else + { +#ifdef FLM_DEBUG + F_CachedBlock * pResetDirty = NULL; +#endif + + // Unset dirty flag so we don't get an assert in unlinkCache. + + if (pSCache->m_ui16Flags & CA_DIRTY) + { +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags = pSCache->m_ui16Flags; +#endif + flmAssert( this == pSCache->m_pDatabase); + pSCache->unsetDirtyFlag(); +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'G'); +#endif + } + +#ifdef FLM_DEBUG + // If m_pNextInVersionList is dirty it is because + // ScaUnlinkTransLogBlocks changed the WAS_DIRTY flag to + // DIRTY. If we don't temporarily clear the DIRTY flag, + // unlinkCache will assert. + + if( pSCache->m_pNextInVersionList && + (pSCache->m_pNextInVersionList->m_ui16Flags & CA_DIRTY)) + { + pResetDirty = pSCache->m_pNextInVersionList; + pResetDirty->m_ui16Flags &= ~CA_DIRTY; + } +#endif + + pSCache->unlinkCache( TRUE, NE_SFLM_OK); + +#ifdef FLM_DEBUG + if( pResetDirty) + { + pResetDirty->m_ui16Flags |= CA_DIRTY; + } +#endif + pSCache = pNextSCache; + continue; + } + } + + pSCache = pSCache->m_pNextInDatabase; + } + + if (!bFreedAll && bFirstPass) + { + bFirstPass = FALSE; + goto Do_Free_Pass; + } + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: Write an IO buffer to disk. +****************************************************************************/ +RCODE F_Database::writeContiguousBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + IF_IOBuffer * pIOBuffer, + FLMUINT uiBlkAddress, + FLMBOOL bDoAsync) +{ + RCODE rc = NE_SFLM_OK; + FLMBYTE * pucWriteBuffer; + IF_IOBuffer * pAsyncBuffer; + FLMUINT uiBytesWritten; + FLMUINT uiWriteLen; + + pucWriteBuffer = pIOBuffer->getBuffer(); + + if (!bDoAsync) + { + pAsyncBuffer = NULL; + } + else + { + pAsyncBuffer = pIOBuffer; + } + + // Determine how many bytes to write + + uiWriteLen = pIOBuffer->getBufferSize(); + pSFileHdl->setMaxAutoExtendSize( m_uiMaxFileSize); + pSFileHdl->setExtendSize( m_uiFileExtendSize); + + pIOBuffer->startTimer( pDbStats); + + // NOTE: No guarantee that pIOBuffer will still be around + // after the call to writeBlock, unless we are doing + // non-asynchronous write. + + rc = pSFileHdl->writeBlock( uiBlkAddress, uiWriteLen, + pucWriteBuffer, pIOBuffer->getBufferSize(), + pAsyncBuffer, &uiBytesWritten); + if (!pAsyncBuffer) + { + pIOBuffer->notifyComplete( rc); + } + pIOBuffer = NULL; + + if (RC_BAD( rc)) + { + if (pDbStats) + { + pDbStats->bHaveStats = TRUE; + pDbStats->uiWriteErrors++; + } + + goto Exit; + } + +Exit: + + // If we allocated a write buffer, but did not do a write with it + // still need to do the notify to clean up cache blocks. + + if (pIOBuffer) + { + flmAssert( RC_BAD( rc)); + pIOBuffer->notifyComplete( rc); + } + return( rc); +} + +/**************************************************************************** +Desc: Prepares a block to be written out. Calculates the checksum and + converts the block to native format if not currently in native + format. +****************************************************************************/ +RCODE flmPrepareBlockToWrite( + FLMUINT uiBlockSize, + F_BLK_HDR * pBlkHdr) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkLen; + + if ((FLMUINT)pBlkHdr->ui16BlkBytesAvail > + uiBlockSize - blkHdrSize( pBlkHdr)) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_BLOCK_CRC); + goto Exit; + } + uiBlkLen = (blkIsNewBTree( pBlkHdr) + ? uiBlockSize + : uiBlockSize - (FLMUINT)pBlkHdr->ui16BlkBytesAvail); + + // Block should already be in native format. + + flmAssert( !blkIsNonNativeFormat( pBlkHdr)); + + // Calculate and set the block CRC. + + pBlkHdr->ui32BlkCRC = calcBlkCRC( pBlkHdr, uiBlkLen); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine writes all blocks in the sorted list, or releases them. + It attempts to write as many as it can that are currently + contiguous. + NOTE: This routine assumes that the block cache mutex is NOT locked. +****************************************************************************/ +RCODE F_Database::writeSortedBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT uiMaxDirtyCache, + FLMUINT * puiDirtyCacheLeft, + FLMBOOL * pbForceCheckpoint, + FLMBOOL bIsCPThread, + FLMBOOL bDoAsync, + FLMUINT uiNumSortedBlocks, + FLMBOOL * pbWroteAll) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + FLMUINT uiStartBlkAddr = 0; + FLMUINT uiLastBlkAddr = 0; + FLMUINT uiContiguousBlocks = 0; + FLMUINT uiNumSortedBlocksProcessed; + FLMUINT uiBlockCount; + F_CachedBlock * ppContiguousBlocks[ FLM_MAX_IO_BUFFER_BLOCKS]; + FLMBOOL bBlockDirty[ FLM_MAX_IO_BUFFER_BLOCKS]; + FLMUINT uiOffset; + FLMUINT uiTmpOffset; + FLMUINT uiLoop; + FLMUINT uiStartOffset; + FLMUINT uiCopyLen; + FLMBOOL bForceCheckpoint = *pbForceCheckpoint; + F_CachedBlock * pSCache; + IF_IOBuffer * pIOBuffer = NULL; + FLMBYTE * pucBuffer; + + uiOffset = 0; + for (;;) + { + + // Mutex must be locked to test dirty flags. + + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + // See how many we have that are contiguous + + uiContiguousBlocks = 0; + uiNumSortedBlocksProcessed = 0; + uiStartOffset = uiTmpOffset = uiOffset; + while (uiTmpOffset < uiNumSortedBlocks) + { + pSCache = m_ppBlocksDone [uiTmpOffset]; + + // See if this block is still eligible for writing out. + // If so, mark it as write pending and add to list. + + flmAssert( pSCache->m_ui16Flags & CA_DIRTY); + + // Is it contiguous with last block or the first block? + + if (!uiContiguousBlocks || + (FSGetFileNumber( uiLastBlkAddr) == + FSGetFileNumber( pSCache->m_uiBlkAddress) && + uiLastBlkAddr + m_uiBlockSize == pSCache->m_uiBlkAddress)) + { + + // Block is either first block or contiguous with + // last block. + +Add_Contiguous_Block: + uiLastBlkAddr = pSCache->m_uiBlkAddress; + + // Set first block address if this is the first one. + + if (!uiContiguousBlocks) + { + uiStartBlkAddr = pSCache->m_uiBlkAddress; + } + ppContiguousBlocks [uiContiguousBlocks] = pSCache; + bBlockDirty [uiContiguousBlocks++] = TRUE; + uiNumSortedBlocksProcessed++; + if (uiContiguousBlocks == FLM_MAX_IO_BUFFER_BLOCKS) + { + break; + } + uiTmpOffset++; + } + else + { + FLMUINT uiGap; + FLMUINT uiSaveContiguousBlocks; + FLMUINT uiBlkAddress; + + // Ran into a non-contiguous block. If we are not forcing + // a checkpoint, take what we have and write it out. + // If we are forcing a checkpoint, see if we can fill the + // gap with other blocks in cache. + + if (!bForceCheckpoint) + { + break; + } + + // See if the gap is worth trying to fill. + + // If blocks are in different files, cannot fill gap. + + if (FSGetFileNumber( uiLastBlkAddr) != + FSGetFileNumber( pSCache->m_uiBlkAddress)) + { + break; + } + + // If 32K won't encompass both blocks, not worth it to try + // and fill the gap. + + uiGap = FSGetFileOffset( pSCache->m_uiBlkAddress) - + FSGetFileOffset( uiLastBlkAddr) - m_uiBlockSize; + if (uiGap > 32 * 1024 - (m_uiBlockSize * 2)) + { + break; + } + + // If the gap would run us off the maximum blocks to + // request, don't try to fill it. + + if (uiContiguousBlocks + uiGap / m_uiBlockSize + 1 > + FLM_MAX_IO_BUFFER_BLOCKS) + { + break; + } + + uiSaveContiguousBlocks = uiContiguousBlocks; + uiBlkAddress = uiLastBlkAddr + m_uiBlockSize; + while (uiBlkAddress != pSCache->m_uiBlkAddress) + { + F_CachedBlock ** ppSCacheBucket; + F_CachedBlock * pTmpSCache; + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_uiSigBitsInBlkSize, uiBlkAddress); + pTmpSCache = *ppSCacheBucket; + while (pTmpSCache && + (pTmpSCache->m_uiBlkAddress != uiBlkAddress || + pTmpSCache->m_pDatabase != this)) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + if (!pTmpSCache || + (pTmpSCache->m_ui16Flags & + (CA_READ_PENDING | CA_WRITE_PENDING | CA_WRITE_INHIBIT)) || + pTmpSCache->m_ui64HighTransID != ~((FLMUINT64)0)) + { + break; + } + ppContiguousBlocks [uiContiguousBlocks] = pTmpSCache; + + bBlockDirty [uiContiguousBlocks++] = + (pTmpSCache->m_ui16Flags & CA_DIRTY) + ? TRUE + : FALSE; + + pTmpSCache->useForThread( 0); + uiBlkAddress += m_uiBlockSize; + } + + // If we couldn't fill in the entire gap, we are done. + + if (uiBlkAddress != pSCache->m_uiBlkAddress) + { + + // Release the blocks we obtained in the above loop. + + while (uiContiguousBlocks > uiSaveContiguousBlocks) + { + uiContiguousBlocks--; + ppContiguousBlocks [uiContiguousBlocks]->releaseForThread(); + } + break; + } + else + { + goto Add_Contiguous_Block; + } + } + } + + // At this point, we know how many are contiguous. + + if (!uiContiguousBlocks) + { + flmAssert( uiOffset == uiNumSortedBlocks); + break; + } + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Ask for a buffer of the size needed. + + flmAssert( pIOBuffer == NULL); + if (RC_BAD( rc = m_pBufferMgr->getBuffer( + &pIOBuffer, uiContiguousBlocks * m_uiBlockSize, + m_uiBlockSize))) + { + goto Exit; + } + pIOBuffer->setCompletionCallback( scaWriteComplete); + + // Callback will now take care of everything between + // uiStartOffset and uiStartOffset + uiNumSortedBlocksProcessed - 1 + // inclusive, as well as any non-dirty blocks that were + // put in for filler. + + flmAssert( uiNumSortedBlocksProcessed); + uiOffset = uiStartOffset + uiNumSortedBlocksProcessed; + uiBlockCount = uiContiguousBlocks; + + // Must set to zero so we don't process ppContiguousBlocks + // at exit. + + uiContiguousBlocks = 0; + + // Set write pending on all of the blocks before unlocking + // the mutex. + // Then unlock the mutex and come out and copy + // the blocks. + + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + for (uiLoop = 0; uiLoop < uiBlockCount; uiLoop++) + { + pSCache = ppContiguousBlocks [uiLoop]; + if (bBlockDirty [uiLoop]) + { + flmAssert( pSCache->m_ui16Flags & CA_DIRTY); + flmAssert( !(pSCache->m_ui16Flags & CA_WRITE_INHIBIT)); + pSCache->setFlags( CA_WRITE_PENDING); + flmAssert( *puiDirtyCacheLeft >= m_uiBlockSize); + (*puiDirtyCacheLeft) -= m_uiBlockSize; + pSCache->unlinkFromDatabase(); + pSCache->linkToDatabase( this); + } + else + { + flmAssert( !(pSCache->m_ui16Flags & CA_DIRTY)); + } + + // Set callback data so we will release these and clear + // the pending flag if we don't do the I/O. + + pIOBuffer->setCompletionCallbackData( uiLoop, pSCache); + } + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + // Copy blocks into the IO buffer. + + pucBuffer = pIOBuffer->getBuffer(); + for (uiLoop = 0; + uiLoop < uiBlockCount; + uiLoop++, pucBuffer += m_uiBlockSize) + { + pSCache = ppContiguousBlocks [uiLoop]; + + // Copy data from block to the write buffer + + uiCopyLen = blkGetEnd( m_uiBlockSize, + blkHdrSize( pSCache->m_pBlkHdr), + pSCache->m_pBlkHdr); + f_memcpy( pucBuffer, pSCache->m_pBlkHdr, uiCopyLen); + + // Encrypt the block if needed + + if (RC_BAD( rc = encryptBlock( m_pDictList, + pucBuffer))) + { + goto Exit; + } + + if (RC_BAD( rc = flmPrepareBlockToWrite( m_uiBlockSize, + (F_BLK_HDR *)pucBuffer))) + { + goto Exit; + } + } + + rc = writeContiguousBlocks( pDbStats, pSFileHdl, + pIOBuffer, uiStartBlkAddr, bDoAsync); + pIOBuffer = NULL; + + // See if we should give up our write lock. Will do so if we + // are not forcing a checkpoint and we have not exceeded the + // maximum time since the last checkpoint AND we have gotten + // below the maximum dirty blocks allowed. + + if (!bForceCheckpoint && bIsCPThread) + { + FLMUINT uiCurrTime = (FLMUINT)FLM_GET_TIMER(); + + if (scaSeeIfForceCheckpoint( uiCurrTime, m_uiLastCheckpointTime, + m_pCPInfo)) + { + bForceCheckpoint = TRUE; + } + else + { + if (m_pWriteLockObj->ThreadWaitingLock() && + *puiDirtyCacheLeft <= uiMaxDirtyCache) + { + + // Break out of loop and finish writing whatever + // we have pending. + + *pbWroteAll = FALSE; + goto Exit; + } + } + } + } + +Exit: + + // Unuse any blocks that did not get processed. + + while (uiOffset < uiNumSortedBlocks) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + m_ppBlocksDone[ uiOffset]->releaseForThread(); + uiOffset++; + } + + while (uiContiguousBlocks) + { + uiContiguousBlocks--; + + // Only release the non-dirty blocks, because dirty blocks + // will have been taken care of in the loop above. + + if (!bBlockDirty [uiContiguousBlocks]) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + ppContiguousBlocks [uiContiguousBlocks]->releaseForThread(); + } + } + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + // If we allocated a write buffer, but did not do a write with it, + // still need to call notifyComplete so that our callback function + // will be called and the F_CachedBlock objects will be released. + + if (pIOBuffer) + { + flmAssert( RC_BAD( rc)); + pIOBuffer->notifyComplete( rc); + } + + *pbForceCheckpoint = bForceCheckpoint; + return( rc); +} + +/**************************************************************************** +Desc: This routine writes all dirty cache blocks to disk. This routine + is called when a transaction is committed. +****************************************************************************/ +RCODE F_Database::flushDirtyBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT uiMaxDirtyCache, + FLMBOOL bForceCheckpoint, + FLMBOOL bIsCPThread, + FLMBOOL * pbWroteAll) +{ + RCODE rc = NE_SFLM_OK; + RCODE rc2; + F_CachedBlock * pSCache; + FLMBOOL bDoAsync; + FLMBOOL bMutexLocked = FALSE; + FLMUINT uiSortedBlocks = 0; + FLMUINT uiBlockCount = 0; + FLMBOOL bWasForcing; + FLMBOOL bWriteInhibited; + FLMUINT uiDirtyCacheLeft; + FLMBOOL bAllocatedAll = FALSE; + + flmAssert( !m_uiLogCacheCount); + + if (m_pCPInfo) + { + lockMutex(); + m_pCPInfo->bWritingDataBlocks = TRUE; + unlockMutex(); + } + + // See if we can do async IO. + + bDoAsync = (gv_SFlmSysData.bOkToDoAsyncWrites && pSFileHdl->canDoAsync()) + ? TRUE + : FALSE; + + flmAssert( !m_pPendingWriteList); + + uiDirtyCacheLeft = m_uiDirtyCacheCount * m_uiBlockSize; + + // If we are forcing a checkpoint, pre-allocate an array big enough + // to hold all of the dirty blocks. We do this so we won't end up + // continually re-allocating the array in the loop below. + +Force_Checkpoint: + + if (bForceCheckpoint) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + pSCache = m_pSCacheList; + uiBlockCount = 0; + while (pSCache && (pSCache->m_ui16Flags & CA_DIRTY)) + { + uiBlockCount++; + pSCache = pSCache->m_pNextInDatabase; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + bAllocatedAll = TRUE; + if (uiBlockCount > m_uiBlocksDoneArraySize * 2) + { + if (RC_BAD( rc = allocBlocksArray( + (uiBlockCount + 1) / 2, TRUE))) + { + if (rc == NE_SFLM_MEM) + { + bAllocatedAll = FALSE; + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + } + } + + for (;;) + { + + flmAssert( !bMutexLocked); + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + + // Create a list of blocks to write out - MAX_BLOCKS_TO_SORT at most. + + pSCache = m_pSCacheList; + uiSortedBlocks = 0; + for (;;) + { + FLMUINT uiPrevBlkAddress; + + if (bAllocatedAll) + { + if (uiSortedBlocks == uiBlockCount) + { +#ifdef FLM_DEBUG + // Better not be any dirty blocks after the last one. + + if (uiSortedBlocks) + { + pSCache = m_ppBlocksDone [uiSortedBlocks - 1]; + flmAssert( !pSCache->m_pNextInDatabase || + !(pSCache->m_pNextInDatabase->m_ui16Flags & CA_DIRTY)); + } +#endif + break; + } + flmAssert( pSCache && (pSCache->m_ui16Flags & CA_DIRTY)); + } + else + { + if (!pSCache || !(pSCache->m_ui16Flags & CA_DIRTY) || + uiSortedBlocks == MAX_BLOCKS_TO_SORT) + { + break; + } + } + + flmAssert( !(pSCache->m_ui16Flags & CA_WRITE_PENDING)); + uiPrevBlkAddress = pSCache->getPriorImageAddress(); + + bWriteInhibited = FALSE; + if (pSCache->m_ui16Flags & CA_WRITE_INHIBIT) + { + // When the checkpoint thread is running there is no need to + // inhibit writes - because it is not possible for an updater + // to be making any changes at this point. However, + // the inhibit writes bit may still be set because the + // thread that originally did the update transaction never + // got the use count to go to zero (due to a reader that + // simultaneously had a use) and hence, the inhibit bit + // has never been unset. It is only unset when we see + // the use count go to zero. + + if (bIsCPThread) + { + pSCache->clearFlags( CA_WRITE_INHIBIT); + } + else + { + bWriteInhibited = TRUE; + } + } + + // Skip blocks that are write inhibited or that have + // not been properly logged yet. + + if (bWriteInhibited || + (!uiPrevBlkAddress && pSCache->m_pNextInVersionList)) + { + flmAssert( !bForceCheckpoint); + } + else + { + if (uiSortedBlocks == m_uiBlocksDoneArraySize * 2) + { + if (RC_BAD( rc = allocBlocksArray( 0, TRUE))) + { + goto Exit; + } + } + + // Keep list of blocks to process + + m_ppBlocksDone [uiSortedBlocks++] = pSCache; + + // Must use to keep from going away. + + pSCache->useForThread( 0); + } + pSCache = pSCache->m_pNextInDatabase; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + // Sort the list of blocks by block address. + + if (uiSortedBlocks) + { + if (uiSortedBlocks > 1) + { + f_qsort( m_ppBlocksDone, + 0, uiSortedBlocks - 1, scaSortCompare, scaSortSwap); + } + bWasForcing = bForceCheckpoint; + rc = writeSortedBlocks( pDbStats, pSFileHdl, + uiMaxDirtyCache, &uiDirtyCacheLeft, + &bForceCheckpoint, bIsCPThread, + bDoAsync, uiSortedBlocks, pbWroteAll); + } + else + { + goto Exit; + } + + // Set to zero so won't get released at exit. + + uiSortedBlocks = 0; + + if (!bIsCPThread || RC_BAD( rc) || !(*pbWroteAll)) + { + goto Exit; + } + if (bForceCheckpoint) + { + if (!bWasForcing) + { + + // Needs to be the checkpoint thread that does this + // because all of the log blocks have to have been + // written out - which the checkpoint thread does + // before calling this routine. + + flmAssert( bIsCPThread); + goto Force_Checkpoint; + } + else if (bAllocatedAll) + { + // We did all of the blocks in one pass, so + // break out of the loop. + + goto Exit; + } + } + } + +Exit: + + // Release any blocks that are still used. + + while (uiSortedBlocks) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + uiSortedBlocks--; + + // Release any blocks we didn't process through. + + pSCache = m_ppBlocksDone [uiSortedBlocks]; + pSCache->releaseForThread(); + } + + // Need to finish up any async writes. + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + if (bDoAsync) + { + + // Wait for writes to complete. + + if (RC_BAD( rc2 = m_pBufferMgr->waitForAllPendingIO())) + { + if (RC_OK( rc)) + { + rc = rc2; + } + } + } + + flmAssert( !m_pPendingWriteList); + + // Better not be any incomplete writes at this point. + + flmAssert( !m_pBufferMgr->havePendingIO()); + + // Don't keep around a large block array if we happened to + // allocate one that is bigger than our normal size. It may + // be huge because we were forcing a checkpoint. + + if (m_uiBlocksDoneArraySize > MAX_BLOCKS_TO_SORT) + { + f_free( &m_ppBlocksDone); + m_uiBlocksDoneArraySize = 0; + } + return( rc); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_Database::reduceDirtyCache( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl) +{ + RCODE rc = NE_SFLM_OK; + RCODE rc2; + F_CachedBlock * pSCache; + FLMBOOL bDoAsync; + FLMBOOL bMutexLocked = FALSE; + FLMUINT uiDirtyCacheLeft; + FLMUINT uiSortedBlocks = 0; + FLMUINT uiInhibitCount; + FLMBOOL bForceCheckpoint; + FLMBOOL bWroteAll; + + flmAssert( !m_uiLogCacheCount); + + // See if we can do async IO. + + bDoAsync = (gv_SFlmSysData.bOkToDoAsyncWrites && + pSFileHdl->canDoAsync()) + ? TRUE + : FALSE; + + flmAssert( !m_pPendingWriteList); + + if( m_uiDirtyCacheCount > m_uiBlocksDoneArraySize * 2) + { + if( RC_BAD( rc = allocBlocksArray( + (m_uiDirtyCacheCount + 1) / 2, TRUE))) + { + goto Exit; + } + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + + pSCache = m_pSCacheList; + uiSortedBlocks = 0; + uiInhibitCount = 0; + + while( pSCache && + (pSCache->m_ui16Flags & CA_DIRTY)) + { + if( (pSCache->m_ui16Flags & CA_WRITE_INHIBIT) != 0) + { + uiInhibitCount++; + } + else + { + flmAssert( uiSortedBlocks < m_uiDirtyCacheCount); + m_ppBlocksDone[ uiSortedBlocks++] = pSCache; + pSCache->useForThread( 0); + } + + pSCache = pSCache->m_pNextInDatabase; + } + + flmAssert( uiSortedBlocks + uiInhibitCount == m_uiDirtyCacheCount); + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + if( !uiSortedBlocks) + { + goto Exit; + } + + if( uiSortedBlocks > 1) + { + f_qsort( m_ppBlocksDone, + 0, uiSortedBlocks - 1, scaSortCompare, scaSortSwap); + } + + uiDirtyCacheLeft = m_uiDirtyCacheCount * m_uiBlockSize; + bForceCheckpoint = FALSE; + bWroteAll = TRUE; + + rc = writeSortedBlocks( pDbStats, pSFileHdl, 0, &uiDirtyCacheLeft, + &bForceCheckpoint, FALSE, bDoAsync, uiSortedBlocks, &bWroteAll); + + uiSortedBlocks = 0; + + if( RC_BAD( rc)) + { + goto Exit; + } + +Exit: + + while( uiSortedBlocks) + { + if( !bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + + uiSortedBlocks--; + + // Release any blocks we didn't process through. + + pSCache = m_ppBlocksDone[ uiSortedBlocks]; + pSCache->releaseForThread(); + } + + // Need to finish up any async writes. + + if( bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + if( bDoAsync) + { + // Wait for writes to complete. + + if( RC_BAD( rc2 = m_pBufferMgr->waitForAllPendingIO())) + { + if( RC_OK( rc)) + { + rc = rc2; + } + } + } + + flmAssert( !m_pPendingWriteList); + + // Better not be any incomplete writes at this point. + + flmAssert( !m_pBufferMgr->havePendingIO()); + + // Don't keep around a large block array if we happened to + // allocate one that is bigger than our normal size. It may + // be huge because we were forcing a checkpoint. + + if( m_uiBlocksDoneArraySize > MAX_BLOCKS_TO_SORT) + { + f_free( &m_ppBlocksDone); + m_uiBlocksDoneArraySize = 0; + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine writes new cache blocks to disk. The purpose of this + routine is to allow cache to be reduced as quickly as possible. + This is best accomplished by flushing blocks that are contiguous + before resorting to writing out non-contiguous blocks. The "new" + block list attempts to accomplish this by keeping an ordered list + of most of the blocks that have been created since the last checkpoint. + The list may not contain all of the new blocks if a transaction, which + modified blocks, was aborted since the last checkpoint completed. + In this case, the blocks would have been removed from the new list + when new versions of the blocks were created. Upon aborting, the + blocks that were originally in the new list are not put back into the + list because of the cost associated with finding their correct places + in the list. Even though these blocks aren't in the new list anymore, + they are still marked as being dirty and will written out eventually. +****************************************************************************/ +RCODE F_Database::reduceNewBlocks( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMUINT * puiBlocksFlushed) +{ + RCODE rc = NE_SFLM_OK; + RCODE rc2; + F_CachedBlock * pSCache; + FLMBOOL bDoAsync = FALSE; + FLMBOOL bMutexLocked = FALSE; + FLMUINT uiSortedBlocks = 0; + FLMUINT uiDirtyCacheLeft; + FLMUINT uiBlocksFlushed = 0; + + flmAssert( !m_uiLogCacheCount); + if (m_pCPInfo) + { + lockMutex(); + m_pCPInfo->bWritingDataBlocks = TRUE; + unlockMutex(); + } + + // See if we can do async IO. + + bDoAsync = (gv_SFlmSysData.bOkToDoAsyncWrites && pSFileHdl->canDoAsync()) + ? TRUE + : FALSE; + + flmAssert( !m_pPendingWriteList); + uiDirtyCacheLeft = m_uiDirtyCacheCount * m_uiBlockSize; + + if (m_uiBlocksDoneArraySize < MAX_BLOCKS_TO_SORT) + { + if (RC_BAD( rc = allocBlocksArray( MAX_BLOCKS_TO_SORT, TRUE))) + { + // If the array size is non-zero, but we were unable to allocate + // the size we wanted, we'll just be content to output as many + // blocks as possible with the existing size of the array + + if( rc == NE_SFLM_MEM && m_uiBlocksDoneArraySize) + { + rc = NE_SFLM_OK; + } + else + { + goto Exit; + } + } + } + + // Create a list of blocks to write out + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + pSCache = m_pFirstInNewList; + uiSortedBlocks = 0; + for (;;) + { + FLMUINT uiPrevBlkAddress; + + if (!pSCache || uiSortedBlocks == m_uiBlocksDoneArraySize) + { + break; + } + + flmAssert( !(pSCache->m_ui16Flags & CA_WRITE_PENDING)); + flmAssert( pSCache->m_ui16Flags & (CA_DIRTY | CA_IN_NEW_LIST)); + + uiPrevBlkAddress = pSCache->getPriorImageAddress(); + + // Skip blocks that are write inhibited + + if( pSCache->m_ui16Flags & CA_WRITE_INHIBIT) + { + pSCache = pSCache->m_pNextInReplaceList; + continue; + } + + // Keep list of blocks to process + + m_ppBlocksDone [uiSortedBlocks++] = pSCache; + + // Must use to keep from going away. + + pSCache->useForThread( 0); + pSCache = pSCache->m_pNextInReplaceList; + } + + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + if (uiSortedBlocks) + { + FLMBOOL bForceCheckpoint = FALSE; + FLMBOOL bDummy; + + rc = writeSortedBlocks( pDbStats, pSFileHdl, + ~((FLMUINT)0), &uiDirtyCacheLeft, + &bForceCheckpoint, FALSE, + bDoAsync, uiSortedBlocks, &bDummy); + + if( RC_OK( rc)) + { + uiBlocksFlushed += uiSortedBlocks; + } + } + else + { + goto Exit; + } + + // Set to zero so won't get released at exit. + + uiSortedBlocks = 0; + +Exit: + + // Release any blocks that are still used. + + while (uiSortedBlocks) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + uiSortedBlocks--; + + // Release any blocks we didn't process through. + + pSCache = m_ppBlocksDone [uiSortedBlocks]; + +#ifdef FLM_DEBUG + if( RC_OK( rc)) + { + flmAssert( !(pSCache->m_ui16Flags & CA_IN_NEW_LIST)); + } +#endif + + pSCache->releaseForThread(); + } + + // Need to finish up any async writes. + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + } + + if (bDoAsync) + { + + // Wait for writes to complete. + + if (RC_BAD( rc2 = m_pBufferMgr->waitForAllPendingIO())) + { + if (RC_OK( rc)) + { + rc = rc2; + } + } + } + + flmAssert( !m_pPendingWriteList); + + // Better not be any incomplete writes at this point. + + flmAssert( !m_pBufferMgr->havePendingIO()); + + // Don't keep around a large block array if we happened to + // allocate one that is bigger than our normal size. It may + // be huge because we were forcing a checkpoint. + + if (m_uiBlocksDoneArraySize > MAX_BLOCKS_TO_SORT) + { + f_free( &m_ppBlocksDone); + m_uiBlocksDoneArraySize = 0; + } + + if (puiBlocksFlushed) + { + *puiBlocksFlushed = uiBlocksFlushed; + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine is called to determine if a cache block or cache record + is still needed. +****************************************************************************/ +FLMBOOL F_Database::neededByReadTrans( + FLMUINT64 ui64LowTransId, + FLMUINT64 ui64HighTransId) +{ + FLMBOOL bNeeded = FALSE; + F_Db * pReadTrans; + + lockMutex(); + + // Quick check - so we don't have to traverse all read transactions. + + if (!m_pFirstReadTrans || + ui64HighTransId < m_pFirstReadTrans->m_ui64CurrTransID || + ui64LowTransId > m_pLastReadTrans->m_ui64CurrTransID) + { + goto Exit; + } + + // Traverse all read transactions - this loop assumes that the + // read transactions are in order of when they started - meaning + // that the ui64CurrTransID on each will be ascending order. The + // loop will quit early once it can detect that the block is + // too old for all remaining transactions. + + pReadTrans = m_pFirstReadTrans; + while (pReadTrans) + { + if (pReadTrans->m_ui64CurrTransID >= ui64LowTransId && + pReadTrans->m_ui64CurrTransID <= ui64HighTransId) + { + bNeeded = TRUE; + goto Exit; + } + else if (pReadTrans->m_ui64CurrTransID > ui64HighTransId) + { + // All remaining transaction's transaction IDs will + // also be greater than the block's high trans ID + // Therefore, we can quit here. + + goto Exit; + } + pReadTrans = pReadTrans->m_pNextReadTrans; + } + +Exit: + + unlockMutex(); + return( bNeeded); +} + +/**************************************************************************** +Desc: This routine is called just after a transaction has successfully + committed. It will unset the flags on log blocks + that would cause them to be written to disk. If the block is no longer + needed by a read transaction, it will also put the block in the + LRU list so it will be selected for replacement first. +****************************************************************************/ +void F_Database::releaseLogBlocks( void) +{ + F_CachedBlock * pSCache; + F_CachedBlock * pNextSCache; + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + pSCache = m_pTransLogList; + while (pSCache) + { + +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags = pSCache->m_ui16Flags; +#endif + + // A block in this list should never be dirty. + + flmAssert( !(pSCache->m_ui16Flags & CA_DIRTY)); + if ((pSCache->m_ui16Flags & CA_WRITE_TO_LOG) && + !(pSCache->m_ui16Flags & CA_LOG_FOR_CP)) + { + flmAssert( m_uiLogCacheCount); + m_uiLogCacheCount--; + } + + pSCache->clearFlags( CA_WRITE_TO_LOG | CA_WAS_DIRTY); + +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'I'); +#endif + pNextSCache = pSCache->m_pNextInHashBucket; + + // Perhaps we don't really need to set these pointers to NULL, + // but it helps keep things clean. + + pSCache->m_pNextInHashBucket = NULL; + pSCache->m_pPrevInHashBucket = NULL; + + // If the block is no longer needed by a read transaction, + // and it does not need to be logged for the checkpoint, + // move it to the free list. + + if ((!pSCache->m_uiUseCount) && + (!pSCache->neededByReadTrans()) && + (!(pSCache->m_ui16Flags & CA_LOG_FOR_CP))) + { + F_CachedBlock * pNewerVer = pSCache->m_pPrevInVersionList; + + if( !pSCache->m_pNextInVersionList && pNewerVer && + pNewerVer->m_ui64HighTransID == ~((FLMUINT64)0) && + pNewerVer->m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + pNewerVer->unlinkFromLogList(); + } + + pSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + + pSCache = pNextSCache; + } + m_pTransLogList = NULL; + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: Retrieve a data block. Shared cache is searched first. If the block + is not in shared cache, it will be retrieved from disk and put into + cache. The use count on the block will be incremented. +****************************************************************************/ +RCODE F_Database::getBlock( + F_Db * pDb, + LFILE * pLFile, // Pointer to logical file structure + // We are retrieving the block for. + // NULL if there is no logical file. + FLMUINT uiBlkAddress, // Address of requested block. + FLMUINT * puiNumLooksRV, // Pointer to FLMUINT where number of + // cache lookups is to be returned. + // If pointer is non-NULL it indicates + // that we only want to find the block + // if it is in cache. If it is NOT + // in cache, do NOT read it in from + // disk. -- This capability is needed + // by the FlmDbReduceSize function. + F_CachedBlock ** ppSCacheRV) // Returns pointer to cache block. +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bMutexLocked = FALSE; + FLMUINT64 ui64BlkVersion; + FLMUINT uiNumLooks; + F_CachedBlock ** ppSCacheBucket; + F_CachedBlock * pSBlkVerCache; + F_CachedBlock * pSMoreRecentVerCache; + F_CachedBlock * pSCache; + FLMBOOL bGotFromDisk = FALSE; + + flmAssert( this == pDb->m_pDatabase); + flmAssert( uiBlkAddress != 0); + + *ppSCacheRV = NULL; + + // We should NEVER be attempting to read a block address that is + // beyond the current logical end of file. + + if (!FSAddrIsBelow( uiBlkAddress, pDb->m_uiLogicalEOF)) + { + rc = RC_SET( NE_SFLM_DATA_ERROR); + goto Exit; + } + + // Release CPU to prevent CPU hog + + f_yieldCPU(); + + // Lock the mutex + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + pDb->m_uiInactiveTime = 0; + + // Search shared cache for the desired version of the block. + // First, determine the hash bucket. + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_uiSigBitsInBlkSize, uiBlkAddress); + + // Search down the linked list of F_CachedBlock objects off of the bucket + // looking for the correct cache block. + + pSCache = *ppSCacheBucket; + uiNumLooks = 1; + while ((pSCache) && + (pSCache->m_uiBlkAddress != uiBlkAddress || + pSCache->m_pDatabase != this)) + { + if ((pSCache = pSCache->m_pNextInHashBucket) != NULL) + { + uiNumLooks++; + } + } + + // If there was no block found with the appropriate file/address we need to + // create a dummy block and attempt to read it in. + + if (!pSCache) + { + if (puiNumLooksRV) + { + *puiNumLooksRV = uiNumLooks; + *ppSCacheRV = NULL; + goto Exit; + } + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaults++; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaultLooks += uiNumLooks; + if (RC_BAD( rc = readIntoCache( pDb, pLFile, uiBlkAddress, + NULL, NULL, &pSCache, &bGotFromDisk))) + { + goto Exit; + } + } + else + { + // A block with the appropriate file/address was found. We now + // need to follow the chain until we find the version of the + // block that we need. + + pSMoreRecentVerCache = NULL; + + // Save pointer to block that is newest version. + + pSBlkVerCache = pSCache; + ui64BlkVersion = pDb->m_ui64CurrTransID; + + for (;;) + { + + // If the block is being read into memory, wait for the read + // to complete so we can see what it is. + + if (pSCache && (pSCache->m_ui16Flags & CA_READ_PENDING)) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiIoWaits++; + if (RC_BAD( rc = flmWaitNotifyReq( + gv_SFlmSysData.hBlockCacheMutex, pDb->m_hWaitSem, + &pSCache->m_pNotifyList, (void *)&pSCache))) + { + goto Exit; + } + + // The thread doing the notify "uses" the cache block + // on behalf of this thread to prevent the cache block + // from being flushed after it unlocks the mutex. + // At this point, since we have locked the mutex, + // we need to release the cache block. + + pSCache->releaseForThread(); + + // Start over at the top of the list. + + pSBlkVerCache = pSCache; + while (pSBlkVerCache->m_pPrevInVersionList) + { + pSBlkVerCache = pSBlkVerCache->m_pPrevInVersionList; + } + pSCache = pSBlkVerCache; + pSMoreRecentVerCache = NULL; + continue; + } + + if (!pSCache || ui64BlkVersion > pSCache->m_ui64HighTransID) + { + if (puiNumLooksRV) + { + *puiNumLooksRV = uiNumLooks; + *ppSCacheRV = NULL; + goto Exit; + } + + // The version of the block we want is not in the list, + // either because we are at the end of the list (!pSCache), + // or because the block version we want is higher than + // the high trans ID on the cache block we are looking + // at. See if there is anything on disk that comes after + // that block. If not, simply return an OLD_VIEW + // error. + + if (pSMoreRecentVerCache && + pSMoreRecentVerCache->getPriorImageAddress() == 0) + { + // Should only be possible when reading a root block, + // because the root block address in the LFILE may be + // a block that was just created by an update + // transaction. + + flmAssert( pDb->m_uiKilledTime); + rc = RC_SET( NE_SFLM_OLD_VIEW); + goto Exit; + } + + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaults++; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheFaultLooks += uiNumLooks; + + if (pSMoreRecentVerCache) + { + if( RC_BAD( rc = readIntoCache( pDb, pLFile, uiBlkAddress, + pSMoreRecentVerCache, + pSMoreRecentVerCache->m_pNextInVersionList, + &pSCache, &bGotFromDisk))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = readIntoCache( pDb, pLFile, uiBlkAddress, + NULL, pSBlkVerCache, &pSCache, &bGotFromDisk))) + { + goto Exit; + } + } + + // At this point, if the read was successful, we should + // have the block we want. + + break; + } + else if (ui64BlkVersion >= pSCache->getLowTransID()) + { + + // This is the version of the block that we need. + + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheHits++; + gv_SFlmSysData.pBlockCacheMgr->m_Usage.uiCacheHitLooks += uiNumLooks; + break; + } + else + { + // If we are in an update transaction, the version of the + // block we want should ALWAYS be at the top of the list. + // If not, we have a serious problem! + + flmAssert( pDb->m_eTransType != SFLM_UPDATE_TRANS); + + pSMoreRecentVerCache = pSCache; + pSCache = pSCache->m_pNextInVersionList; + + if (pSCache) + { + uiNumLooks++; + } + } + } + } + + // Increment the use count on the block. + + pSCache->useForThread( 0); + + // Block was found, make it the MRU block or bump it up in the MRU list, + // if it is not already at the top. + + if( pDb->m_uiFlags & FDB_DONT_POISON_CACHE) + { + if (!(pDb->m_uiFlags & FDB_BACKGROUND_INDEXING) || + (pLFile && pLFile->eLfType != SFLM_LF_INDEX)) + { + if (!bGotFromDisk) + { + pSCache->stepUpInGlobalList(); + } + + // If the block was read from disk and FDB_DONT_POISION_CACHE is + // set, we don't need to do anything because the block is + // already linked at the LRU position. + } + else if (pSCache->m_pPrevInGlobal) + { + pSCache->unlinkFromGlobalList(); + pSCache->linkToGlobalListAsMRU(); + } + } + else if (pSCache->m_pPrevInGlobal) + { + pSCache->unlinkFromGlobalList(); + pSCache->linkToGlobalListAsMRU(); + } + + *ppSCacheRV = pSCache; + +Exit: + +#ifdef SCACHE_LINK_CHECKING + if (RC_BAD( rc)) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + scaVerify( 300); + } +#endif + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + return( rc); +} + +/**************************************************************************** +Desc: Create a data block. +****************************************************************************/ +RCODE F_Database::createBlock( + F_Db * pDb, + F_CachedBlock ** ppSCacheRV) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBlkAddress; + F_BLK_HDR * pBlkHdr; + F_CachedBlock * pSCache = NULL; + F_CachedBlock * pOldSCache = NULL; + FLMBOOL bMutexLocked = FALSE; + FLMBOOL bLocalCacheAllocation = FALSE; + FLMUINT uiOldLogicalEOF; + F_CachedBlock ** ppSCacheBucket; + FLMUINT uiBlockSize = pDb->m_pDatabase->getBlockSize(); + + pDb->m_bHadUpdOper = TRUE; + + // First see if there is a free block in the avail list. + + if (pDb->m_uiFirstAvailBlkAddr) + { + rc = blockUseNextAvail( pDb, ppSCacheRV); + goto Exit; + } + + // See if we need to free any cache or write dirty cache + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + if (RC_BAD( rc = gv_SFlmSysData.pBlockCacheMgr->reduceCache( pDb))) + { + goto Exit; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = FALSE; + + // Create a new block at EOF + + uiBlkAddress = pDb->m_uiLogicalEOF; + + // Time for a new block file? + + if (FSGetFileOffset(uiBlkAddress) >= m_uiMaxFileSize) + { + FLMUINT uiFileNumber = FSGetFileNumber( uiBlkAddress) + 1; + + if (uiFileNumber > MAX_DATA_BLOCK_FILE_NUMBER) + { + rc = RC_SET( NE_SFLM_DB_FULL); + goto Exit; + } + + if (RC_BAD( rc = pDb->m_pSFileHdl->createFile( uiFileNumber))) + { + goto Exit; + } + uiBlkAddress = FSBlkAddress( uiFileNumber, 0 ); + } + + // Allocate a cache block for this new block. If we have older + // versions of this block already in cache, we need to link the + // new block above the older version. If reasonable, try to + // allocate the new cache block without locking the mutex. + + if( !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->lockMutex(); + + if( (pSCache = new( uiBlockSize, TRUE) F_CachedBlock( uiBlockSize)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + pSCache->m_uiUseCount++; + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->unlockMutex(); + bLocalCacheAllocation = TRUE; + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + + // Determine the hash bucket the new block should be put into. + + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_uiSigBitsInBlkSize, uiBlkAddress); + + // Search down the linked list of F_CachedBlock objects off of the bucket + // looking for an older version of the block. If there are older + // versions, we should get rid of them. + + pOldSCache = *ppSCacheBucket; + while (pOldSCache && + (pOldSCache->m_uiBlkAddress != uiBlkAddress || + pOldSCache->m_pDatabase != this)) + { + pOldSCache = pOldSCache->m_pNextInHashBucket; + } + + while (pOldSCache) + { + F_CachedBlock * pNextSCache = pOldSCache->m_pNextInVersionList; + + // Older versions of blocks should not be in use or needed + // by anyone because the only we we would have an older + // version of a block beyond the logical EOF is if + // FlmDbReduceSize had been called. But it forces a + // checkpoint that requires any read transactions to be + // non-active, or killed. + + flmAssert( !pOldSCache->m_ui16Flags); + flmAssert( !pOldSCache->m_uiUseCount); + flmAssert( pOldSCache->m_ui64HighTransID == ~((FLMUINT64)0) || + !pOldSCache->neededByReadTrans()); + pOldSCache->unlinkCache( TRUE, NE_SFLM_OK); + pOldSCache = pNextSCache; + } + + // Allocate a cache block - either a new one or by replacing + // an existing one. + + if( bLocalCacheAllocation) + { + F_BlockCacheMgr * pBlockCacheMgr = gv_SFlmSysData.pBlockCacheMgr; + + // Now that the mutex is locked, update stats and do other work + // that couldn't be done when the block was allocated. + + pBlockCacheMgr->m_Usage.uiCount++; + pBlockCacheMgr->m_Usage.uiByteCount += pSCache->memSize(); + + // Set use count to one so the block cannot be replaced. + + pSCache->m_uiUseCount--; + pSCache->useForThread( 0); + } + else + { + if (RC_BAD( rc = gv_SFlmSysData.pBlockCacheMgr->allocBlock( pDb, &pSCache))) + { + goto Exit; + } + } + + pSCache->m_uiBlkAddress = uiBlkAddress; + pSCache->setTransID( ~((FLMUINT64)0)); + + // Initialize the block data, dirty flag is set so that it will be + // flushed as needed. + + pBlkHdr = pSCache->m_pBlkHdr; + f_memset( pBlkHdr, 0, m_uiBlockSize); + pBlkHdr->ui32BlkAddr = (FLMUINT32)uiBlkAddress; + pBlkHdr->ui64TransID = pDb->m_ui64CurrTransID; + pBlkHdr->ui16BlkBytesAvail = + (FLMUINT16)(m_uiBlockSize - SIZEOF_STD_BLK_HDR); + blkSetNativeFormat( pBlkHdr); + +#ifdef FLM_DBG_LOG + flmDbgLogWrite( this, pSCache->m_uiBlkAddress, 0, + pSCache->getLowTransID(), + "CREATE"); +#endif + + // Link block into the global list + + pSCache->m_ui16Flags |= CA_DUMMY_FLAG; + pSCache->linkToGlobalListAsMRU(); + + // Set the dirty flag + + pSCache->setDirtyFlag( this); + pSCache->m_ui16Flags &= ~CA_DUMMY_FLAG; + + // Set write inhibit bit so we will not unset the dirty bit + // until the use count goes to zero. + + pSCache->setFlags( CA_WRITE_INHIBIT); + +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( 0, 'J'); +#endif + + // Now that the dirty flag and write inhibit flag + // have been set, link the block to the file + + pSCache->linkToDatabase( this); + pSCache->linkToHashBucket( ppSCacheBucket); + + uiOldLogicalEOF = pDb->m_uiLogicalEOF; + pDb->m_uiLogicalEOF = uiBlkAddress + m_uiBlockSize; + + // Link the block into the "new" list + + pSCache->linkToNewList(); + + // Return a pointer to the block + + *ppSCacheRV = pSCache; + +Exit: + +#ifdef SCACHE_LINK_CHECKING + if (RC_BAD( rc)) + { + if (!bMutexLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bMutexLocked = TRUE; + } + scaVerify( 400); + } +#endif + + if (bMutexLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + if (RC_BAD( rc)) + { + *ppSCacheRV = NULL; + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/**************************************************************************** +Desc: This routine releases a cache block by decrementing its use count. + If the use count goes to zero, the block will be moved to the MRU + position in the global cache list. +****************************************************************************/ +void ScaReleaseCache( + F_CachedBlock * pSCache, + FLMBOOL bMutexAlreadyLocked) +{ + if (!bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + } + + // Can turn off write inhibit when the use count is about to go to zero, + // because we are guaranteed at that point that nobody is going to still + // update it. + + if (pSCache->getUseCount() == 1) + { + pSCache->clearFlags( CA_WRITE_INHIBIT); + } + + pSCache->releaseForThread(); + + if (!bMutexAlreadyLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } +} + +/**************************************************************************** +Desc: This routine increments the use count on a cache block +****************************************************************************/ +void ScaUseCache( + F_CachedBlock * pSCache, + FLMBOOL bMutexAlreadyLocked) +{ + if (!bMutexAlreadyLocked) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + } + + pSCache->useForThread( 0); + + if (!bMutexAlreadyLocked) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } +} + +/**************************************************************************** +Desc: Test and set the dirty flag on a block if not already set. This is + called on the case where a block didn't need to be logged because + it had already been logged, but it still needs to have its dirty + bit set. +****************************************************************************/ +void F_Database::setBlkDirty( + F_CachedBlock * pSCache) +{ +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags; +#endif + + // If the dirty flag is already set, we will NOT attempt to set it. + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); +#ifdef FLM_DBG_LOG + ui16OldFlags = pSCache->m_ui16Flags; +#endif + + if (!(pSCache->m_ui16Flags & CA_DIRTY)) + { + flmAssert( this == pSCache->m_pDatabase); + pSCache->setDirtyFlag( this); + } + + // Move the block into the dirty blocks. Even if block was + // already dirty, put at the end of the list of dirty blocks. + // This will make it so that when reduceCache hits a dirty + // block, it is likely to also be one that will be written + // out by a call to ScaFlushDirtyBlocks. + + pSCache->unlinkFromDatabase(); + pSCache->linkToDatabase( this); + + // Move the block to the MRU slot in the global list + + if (pSCache->m_pPrevInGlobal) + { + pSCache->unlinkFromGlobalList(); + pSCache->linkToGlobalListAsMRU(); + } + + // Set write inhibit bit so we will not unset the dirty bit + // until the use count goes to zero. + + pSCache->setFlags( CA_WRITE_INHIBIT); +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'O'); +#endif + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + +/**************************************************************************** +Desc: This routine logs a block before it is modified. In the shared cache + system, the block is cloned. The old version of the block is marked + so that it will be written to the rollback log before the new version + of the block can be written to disk. + NOTE: It is assumed that the caller has "used" the block that is passed + in. This routine will release it once we have made a copy of the block. +****************************************************************************/ +RCODE F_Database::logPhysBlk( + F_Db * pDb, + F_CachedBlock ** ppSCacheRV, // This is a pointer to the pointer of the + // cache block that is to be logged. + // If the block has not been logged before + // during the transaction, a new version + // of the block will be created and a + // pointer to that block will be returned. + // Otherwise, the pointer is unchanged. + F_CachedBlock ** ppOldSCache) +{ + RCODE rc = NE_SFLM_OK; + F_CachedBlock * pSCache = *ppSCacheRV; + F_BLK_HDR * pBlkHdr = pSCache->m_pBlkHdr; + F_CachedBlock * pNewSCache = NULL; + FLMBOOL bLockedMutex = FALSE; + FLMBOOL bLocalCacheAllocation = FALSE; + FLMUINT uiBlockSize = getBlockSize(); + F_CachedBlock ** ppSCacheBucket; +#ifdef FLM_DBG_LOG + FLMUINT16 ui16OldFlags; +#endif + + flmAssert( this == pDb->m_pDatabase); + flmAssert( pSCache->m_pPrevInVersionList == NULL); + + if( ppOldSCache) + { + *ppOldSCache = NULL; + } + + // Increment the block change count -- this is not an accurate + // indication of the number of blocks that have actually changed. The + // count is used by the cursor code to determine when to re-position in + // the B-Tree. The value is only used by cursors operating within + // an update transaction. + + pDb->m_uiBlkChangeCnt++; + + // See if the block has already been logged since the last transaction. + // If so, there is no need to log it again. + + if( pBlkHdr->ui64TransID == pDb->m_ui64CurrTransID) + { + flmAssert( pDb->m_bHadUpdOper); + setBlkDirty( pSCache); + goto Exit; + } + pDb->m_bHadUpdOper = TRUE; + + // See if we need to free any cache or write dirty cache + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bLockedMutex = TRUE; + if (RC_BAD( rc = gv_SFlmSysData.pBlockCacheMgr->reduceCache( pDb))) + { + goto Exit; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + bLockedMutex = FALSE; + + // See if the transaction ID is greater than the last backup + // transaction ID. If so, we need to update our block change + // count. + + if (pBlkHdr->ui64TransID < m_uncommittedDbHdr.ui64LastBackupTransID) + { + m_uncommittedDbHdr.ui32BlksChangedSinceBackup++; + } + + // pDb->m_uiTransEOF contains what the EOF address was at + // the beginning of the transaction. There is no need to log the + // block if it's address is beyond that point because it is a + // NEW block. + + if (!FSAddrIsBelow( (FLMUINT)pSCache->m_pBlkHdr->ui32BlkAddr, + pDb->m_uiTransEOF)) + { + pBlkHdr->ui64TransID = pDb->m_ui64CurrTransID; + setBlkDirty( pSCache); + goto Exit; + } + + // Allocate a cache block for this new block. If we have older + // versions of this block already in cache, we need to link the + // new block above the older version. Try to allocate the new + // block outside of the block cache mutex. If not possible, lock + // the mutex and allocate. + + if( !gv_SFlmSysData.pGlobalCacheMgr->cacheOverLimit()) + { + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->lockMutex(); + + if( (pNewSCache = new( uiBlockSize, TRUE) F_CachedBlock( uiBlockSize)) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + + pNewSCache->m_uiUseCount++; + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->unlockMutex(); + bLocalCacheAllocation = TRUE; + + // Copy the old block's data into this one. + + pBlkHdr = pNewSCache->m_pBlkHdr; + f_memcpy( pBlkHdr, pSCache->m_pBlkHdr, m_uiBlockSize); + } + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bLockedMutex = TRUE; + + // Allocate a cache block - either a new one or by replacing + // an existing one. + + if( bLocalCacheAllocation) + { + F_BlockCacheMgr * pBlockCacheMgr = gv_SFlmSysData.pBlockCacheMgr; + + // Now that the mutex is locked, update stats and do other work + // that couldn't be done when the block was allocated. + + pBlockCacheMgr->m_Usage.uiCount++; + pBlockCacheMgr->m_Usage.uiByteCount += pNewSCache->memSize(); + + // Set use count to one so the block cannot be replaced. + + pNewSCache->m_uiUseCount--; + pNewSCache->useForThread( 0); + } + else + { + if (RC_BAD( rc = gv_SFlmSysData.pBlockCacheMgr->allocBlock( + pDb, &pNewSCache))) + { + goto Exit; + } + + // Copy the old block's data into this one. + + pBlkHdr = pNewSCache->m_pBlkHdr; + f_memcpy( pBlkHdr, pSCache->m_pBlkHdr, m_uiBlockSize); + } + +#ifdef FLM_DEBUG + + // Make sure the caller isn't logging one that has already been + // logged. + + if (gv_SFlmSysData.pBlockCacheMgr->m_bDebug) + { + F_CachedBlock * pTmpSCache = m_pTransLogList; + + while (pTmpSCache) + { + flmAssert( pTmpSCache != pSCache); + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + } +#endif + + pNewSCache->m_uiBlkAddress = pSCache->m_uiBlkAddress; + +#ifdef FLM_DBG_LOG + flmDbgLogWrite( this, + (FLMUINT)pNewSCache->m_pBlkHdr->ui32BlkAddr, 0, + pDb->m_ui64CurrTransID, + "NEW-VER"); +#endif + + // Link the block to the global list + + pNewSCache->m_ui16Flags |= CA_DUMMY_FLAG; + pNewSCache->linkToGlobalListAsMRU(); + + // Set flags so that appropriate flushing to log and DB will be done. + + pNewSCache->setDirtyFlag( this); + pNewSCache->m_ui16Flags &= ~CA_DUMMY_FLAG; + + // Set write inhibit bit so we will not unset the dirty bit + // until the use count goes to zero. + + pNewSCache->setFlags( CA_WRITE_INHIBIT); + + // Previous block address should be zero until we actually log the + // prior version of the block. + + pBlkHdr->ui32PriorBlkImgAddr = 0; + + // Set the low and high trans IDs on the newly created block. + + pNewSCache->setTransID( ~((FLMUINT64)0)); + pBlkHdr->ui64TransID = pDb->m_ui64CurrTransID; +#ifdef FLM_DBG_LOG + pNewSCache->logFlgChange( 0, 'L'); +#endif + + // Determine the hash bucket the new block should be put into. + + flmAssert( pSCache->m_uiBlkAddress); + ppSCacheBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + m_uiSigBitsInBlkSize, pSCache->m_uiBlkAddress); + + // Link new block into various lists. + + pSCache->unlinkFromHashBucket( ppSCacheBucket); + pSCache->m_pPrevInVersionList = pNewSCache; + pSCache->verifyCache( 2900); + pNewSCache->m_pNextInVersionList = pSCache; + pNewSCache->linkToDatabase( this); + pNewSCache->linkToHashBucket( ppSCacheBucket); + pNewSCache->verifyCache( 3000); + + // Set the high trans ID on the old block to be one less than + // the current trans ID. Also set the flag indicating that + // the block needs to be written to the rollback log. + + pSCache->setTransID( (pDb->m_ui64CurrTransID - 1)); +#ifdef FLM_DBG_LOG + ui16OldFlags = pSCache->m_ui16Flags; +#endif + + if (!(pSCache->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP))) + { + m_uiLogCacheCount++; + } + + pSCache->setFlags( CA_WRITE_TO_LOG); + + if (pSCache->getLowTransID() <= + m_uncommittedDbHdr.ui64RflLastCPTransID) + { + pSCache->setFlags( CA_LOG_FOR_CP); + } + + if (pSCache->m_ui16Flags & CA_DIRTY) + { + pSCache->setFlags( CA_WAS_DIRTY); + flmAssert( this == pSCache->m_pDatabase); + pSCache->unsetDirtyFlag(); + + // No more need to write inhibit - because the old version of the + // block cannot possibly be changed. + + pSCache->clearFlags( CA_WRITE_INHIBIT); + + // Move the block out of the dirty blocks. + + pSCache->unlinkFromDatabase(); + pSCache->linkToDatabase( this); + } + +#ifdef FLM_DBG_LOG + pSCache->logFlgChange( ui16OldFlags, 'N'); +#endif + + // Put the old block into the list of the transaction's + // log blocks + + pSCache->m_pPrevInHashBucket = NULL; + if ((pSCache->m_pNextInHashBucket = m_pTransLogList) != NULL) + { + pSCache->m_pNextInHashBucket->m_pPrevInHashBucket = pSCache; + } + m_pTransLogList = pSCache; + + // Link the new block to the file log list + + pNewSCache->linkToLogList(); + + // If this is an indexing thread, the old version of the + // block will probably not be needed again so put it at the LRU end + // of the cache. The assumption is that a background indexing thread + // has also set the FDB_DONT_POISON_CACHE flag. + + if (pDb->m_uiFlags & FDB_BACKGROUND_INDEXING) + { + pSCache->unlinkFromGlobalList(); + pSCache->linkToGlobalListAsLRU(); + } + + // Release the old block and return a pointer to the new block. + + if( !ppOldSCache) + { + ScaReleaseCache( pSCache, bLockedMutex); + } + else + { + *ppOldSCache = pSCache; + } + + *ppSCacheRV = pNewSCache; + +Exit: +#ifdef SCACHE_LINK_CHECKING + if (RC_BAD( rc)) + { + if (!bLockedMutex) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + bLockedMutex = TRUE; + } + scaVerify( 500); + } +#endif + + if (bLockedMutex) + { + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + if( RC_BAD( rc)) + { + pDb->setMustAbortTrans( rc); + } + + return( rc); +} + +/**************************************************************************** +Desc: Constructor for block cache manager. +****************************************************************************/ +F_BlockCacheMgr::F_BlockCacheMgr() +{ + m_pBlockAllocator = NULL; + m_pMRUReplace = NULL; + m_pLRUReplace = NULL; + m_pFirstFree = NULL; + m_pLastFree = NULL; + m_ppHashBuckets = NULL; + m_uiNumBuckets = 0; + m_uiHashFailTime = 0; + f_memset( &m_Usage, 0, sizeof( m_Usage)); + m_uiFreeBytes = 0; + m_uiFreeCount = 0; + m_uiReplaceableCount = 0; + m_uiReplaceableBytes = 0; + m_bAutoCalcMaxDirty = FALSE; + m_uiMaxDirtyCache = 0; + m_uiLowDirtyCache = 0; + m_uiTotalUses = 0; + m_uiBlocksUsed = 0; + m_uiPendingReads = 0; + m_uiIoWaits = 0; + m_uiHashMask = 0; + m_bReduceInProgress = FALSE; +#ifdef FLM_DEBUG + m_bDebug = FALSE; +#endif +} + +/**************************************************************************** +Desc: This routine initializes the hash table for block cache. +****************************************************************************/ +RCODE F_BlockCacheMgr::initHashTbl( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiAllocSize; + + // Calculate the number of bits needed to represent values in the + // hash table. + + m_uiNumBuckets = MIN_HASH_BUCKETS; + m_uiHashMask = (m_uiNumBuckets - 1); + uiAllocSize = (FLMUINT)sizeof( F_CachedBlock *) * m_uiNumBuckets; + if (RC_BAD( rc = f_calloc( uiAllocSize, &m_ppHashBuckets))) + { + goto Exit; + } + gv_SFlmSysData.pGlobalCacheMgr->incrTotalBytes( f_msize( m_ppHashBuckets)); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine initializes the block cache manager. +****************************************************************************/ +RCODE F_BlockCacheMgr::initCache( void) +{ + RCODE rc = NE_SFLM_OK; +#define MAX_NUM_BLOCK_SIZES 16 + FLMUINT uiBlockSizes[ MAX_NUM_BLOCK_SIZES]; + FLMUINT uiBlockSize; + FLMUINT uiLoop; + + // Allocate memory for the hash table. + + if (RC_BAD( rc = initHashTbl())) + { + goto Exit; + } + + // Initialize the cache block allocator + + for( uiLoop = 0, uiBlockSize = SFLM_MIN_BLOCK_SIZE; + uiBlockSize <= SFLM_MAX_BLOCK_SIZE; + uiLoop++, uiBlockSize *= 2) + { + if( uiLoop >= MAX_NUM_BLOCK_SIZES) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_MEM); + goto Exit; + } + + uiBlockSizes[ uiLoop] = uiBlockSize + sizeof( F_CachedBlock); + } + + uiBlockSizes[ uiLoop] = 0; + + if( RC_BAD( rc = FlmAllocMultiAllocator( &m_pBlockAllocator))) + { + goto Exit; + } + + if (RC_BAD( rc = m_pBlockAllocator->setup( + gv_SFlmSysData.pGlobalCacheMgr->m_pSlabManager, uiBlockSizes, + &m_Usage.slabUsage))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine resizes the hash table for the block cache manager. + NOTE: This routine assumes that the cache block mutex has been locked. +****************************************************************************/ +RCODE F_BlockCacheMgr::rehash( void) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiNewHashTblSize; + F_CachedBlock ** ppOldHashTbl; + FLMUINT uiOldHashTblSize; + F_CachedBlock ** ppBucket; + F_CachedBlock ** ppSCacheBucket; + FLMUINT uiLoop; + F_CachedBlock * pTmpSCache; + F_CachedBlock * pTmpNextSCache; + FLMUINT uiOldMemSize; + + uiNewHashTblSize = caGetBestHashTblSize( m_Usage.uiCount); + + // At this point we better have a different hash table size + // or something is mucked up! + + flmAssert( uiNewHashTblSize != m_uiNumBuckets); + + // Save the old hash table and its size. + + if ((ppOldHashTbl = m_ppHashBuckets) != NULL) + { + uiOldMemSize = f_msize( ppOldHashTbl); + } + else + { + uiOldMemSize = 0; + } + uiOldHashTblSize = m_uiNumBuckets; + + // Allocate a new hash table. + + if (RC_BAD( rc = f_calloc( (FLMUINT)sizeof( F_CachedBlock *) * + (FLMUINT)uiNewHashTblSize, &m_ppHashBuckets))) + { + m_uiHashFailTime = FLM_GET_TIMER(); + m_ppHashBuckets = ppOldHashTbl; + goto Exit; + } + + // Subtract off old size and add in new size. + + gv_SFlmSysData.pGlobalCacheMgr->decrTotalBytes( uiOldMemSize); + gv_SFlmSysData.pGlobalCacheMgr->incrTotalBytes( f_msize( m_ppHashBuckets)); + + m_uiNumBuckets = uiNewHashTblSize; + m_uiHashMask = uiNewHashTblSize - 1; + + // Relink all of the cache blocks into the new + // hash table. + + for (uiLoop = 0, ppBucket = ppOldHashTbl; + uiLoop < uiOldHashTblSize; + uiLoop++, ppBucket++) + { + pTmpSCache = *ppBucket; + while (pTmpSCache) + { + pTmpNextSCache = pTmpSCache->m_pNextInHashBucket; + + // Should not be anything in a hash bucket that is not + // associated with a database. + + flmAssert( pTmpSCache->m_pDatabase); + flmAssert( pTmpSCache->m_uiBlkAddress); + ppSCacheBucket = blockHash( + pTmpSCache->m_pDatabase->getSigBitsInBlkSize(), + pTmpSCache->m_uiBlkAddress); + + pTmpSCache->linkToHashBucket( ppSCacheBucket); + pTmpSCache = pTmpNextSCache; + } + } + + // Throw away the old hash table. + + f_free( &ppOldHashTbl); + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine performs various checks on an individual cache block + to verify that it is linked into the proper lists, etc. This routine + assumes that the cache block mutex has been locked. +****************************************************************************/ +#ifdef SCACHE_LINK_CHECKING +void F_CachedBlock::verifyCache( + int iPlace) +{ + F_CachedBLock * pTmpSCache; + FLMUINT uiTmp = getBlkSize(); + FLMUINT uiSigBitsInBlkSize; + F_CachedBlock ** ppBucket; + + uiSigBitsInBlkSize = calcSigBits( uiTmp); + ppBucket = gv_SFlmSysData.pBlockCacheMgr->blockHash( + uiSigBitsInBlkSize, m_uiBlkAddress); + pTmpSCache = *ppBucket; + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if( pTmpSCache) + { + if (!m_pDatabase) + { + f_breakpoint( iPlace+3); + } + + if (m_pPrevInVersionList) + { + f_breakpoint( iPlace+4); + } + + // Verify that it is not in the log list. + + if (m_ui16Flags & CA_WRITE_TO_LOG) + { + f_breakpoint( iPlace+5); + } + pTmpSCache = m_pDatabase->getTransLogList(); + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + if (pTmpSCache) + { + f_breakpoint( iPlace+6); + } + } + else + { + if (m_pDatabase && !m_pPrevInVersionList) + { + f_breakpoint( iPlace+7); + } + + // If the block is marked as needing to be logged, verify that + // it is in the log list. + + if (m_ui16Flags & CA_WRITE_TO_LOG) + { + pTmpSCache = m_pDatabase->getTransLogList(); + while (pTmpSCache && pTmpSCache != this) + { + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + + if (!pTmpSCache) + { + // Not in the log list + + f_breakpoint( iPlace+8); + } + + // Better also have a newer version. + + if (!m_pPrevInVersionList) + { + // Not linked to a prior version. + + f_breakpoint( iPlace+9); + } + } + } + + // Verify that the prev and next pointers do not point to itself. + + if (m_pPrevInVersionList == this) + { + f_breakpoint( iPlace+10); + } + + if (m_pNextInVersionList == this) + { + f_breakpoint( iPlace+11); + } +} +#endif + +/**************************************************************************** +Desc: This routine performs various checks on the cache to verify that + things are linked into the proper lists, etc. This routine assumes + that the cache block mutex has been locked. +****************************************************************************/ +#ifdef SCACHE_LINK_CHECKING +FSTATIC void scaVerify( + int iPlace) +{ + FLMUINT uiLoop; + F_CachedBlock ** ppBucket; + F_CachedBlock * pTmpSCache; + + // Verify that everything in buckets has a pFile and does NOT + // have a m_pPrevInVersionList + + for (uiLoop = 0, ppBucket = gv_SFlmSysData.pBlockCacheMgr->m_ppHashBuckets; + uiLoop < gv_SFlmSysData.pBlockCacheMgr->m_uiNumBuckets; + uiLoop++, ppBucket++) + { + pTmpSCache = *ppBucket; + while (pTmpSCache) + { + if (!pTmpSCache->m_pDatabase) + { + f_breakpoint(iPlace+1); + } + if (pTmpSCache->m_pPrevInVersionList) + { + f_breakpoint(iPlace+2); + } + pTmpSCache = pTmpSCache->m_pNextInHashBucket; + } + } + + // Traverse the entire list - make sure that everything + // with a file is hashed and linked properly + // and everything without a file is NOT hashed. + + pTmpSCache = (F_CachedBlock *)gv_SFlmSysData.pBlockCacheMgr->m_MRUList.m_pMRUItem; + while (pTmpSCache) + { + pTmpSCache->verifyCache( 1000 + iPlace); + pTmpSCache = pTmpSCache->m_pNextInGlobal; + } +} +#endif + +/**************************************************************************** +Desc: This routine determines what hash table size best fits the current + item count. It finds the hash bucket size whose midpoint between + the minimum and maximum range is closest to the node count. +****************************************************************************/ +FLMUINT caGetBestHashTblSize( + FLMUINT uiCurrItemCount + ) +{ + FLMUINT uiNumHashBuckets; + FLMUINT uiMaxItemsForNumHashBuckets; + FLMUINT uiMinItemsForNumHashBuckets; + FLMUINT uiClosestNumHashBuckets = 0; + FLMUINT uiDistanceFromMidpoint; + FLMUINT uiLowestDistanceFromMidpoint; + FLMUINT uiHashTblItemsMidpoint; + + uiLowestDistanceFromMidpoint = 0xFFFFFFFF; + for (uiNumHashBuckets = MIN_HASH_BUCKETS; + uiNumHashBuckets <= MAX_HASH_BUCKETS; + uiNumHashBuckets *= 2) + { + + // Maximum desirable record count for a specific hash table size + // we have arbitrarily chosen to be four times the number of buckets. + // Minimum desirable record count we have arbitrarily chosen to be + // the hash table size divided by fourn. + + uiMaxItemsForNumHashBuckets = maxItemCount( uiNumHashBuckets); + uiMinItemsForNumHashBuckets = minItemCount( uiNumHashBuckets); + + // Ignore any hash bucket sizes where the current record count + // is not between the desired minimum and maximum. + + if (uiCurrItemCount >= uiMinItemsForNumHashBuckets && + uiCurrItemCount <= uiMaxItemsForNumHashBuckets) + { + + // Calculate the midpoint between the minimum and maximum + // for this particular hash table size. + + uiHashTblItemsMidpoint = (uiMaxItemsForNumHashBuckets - + uiMinItemsForNumHashBuckets) / 2; + + // See how far our current record count is from this midpoint. + + uiDistanceFromMidpoint = (FLMUINT)((uiHashTblItemsMidpoint > uiCurrItemCount) + ? (uiHashTblItemsMidpoint - uiCurrItemCount) + : (uiCurrItemCount - uiHashTblItemsMidpoint)); + + // If the distance from the midpoint is closer than our previous + // lowest distance, save it. + + if (uiDistanceFromMidpoint < uiLowestDistanceFromMidpoint) + { + uiClosestNumHashBuckets = uiNumHashBuckets; + uiLowestDistanceFromMidpoint = uiDistanceFromMidpoint; + } + } + } + + // Take the number of buckets whose middle was closest to the + // current record count; + + if (uiLowestDistanceFromMidpoint == 0xFFFFFFFF) + { + // If we did not fall between any of the minimums or maximums, + // we are either below the lowest minimum, or higher than the + // highest maximum. + + uiNumHashBuckets = (FLMUINT)((uiCurrItemCount < minItemCount( MIN_HASH_BUCKETS)) + ? (FLMUINT)MIN_HASH_BUCKETS + : (FLMUINT)MAX_HASH_BUCKETS); + + } + else + { + uiNumHashBuckets = uiClosestNumHashBuckets; + } + return( uiNumHashBuckets); +} + +/**************************************************************************** +Desc: This routine shuts down the shared cache manager and frees all + resources allocated by it. +****************************************************************************/ +F_BlockCacheMgr::~F_BlockCacheMgr() +{ + + // Free the hash table + + if (m_ppHashBuckets) + { + gv_SFlmSysData.pGlobalCacheMgr->decrTotalBytes( f_msize( m_ppHashBuckets)); + f_free( &m_ppHashBuckets); + } + + if( m_pBlockAllocator) + { + m_pBlockAllocator->Release(); + } + + flmAssert( !m_MRUList.m_pMRUItem && !m_MRUList.m_pLRUItem); +} + +/**************************************************************************** +Desc: This routine frees all of the cache associated with an F_Database object. +****************************************************************************/ +void F_Database::freeBlockCache( void) +{ + F_CachedBlock * pSCache; + F_CachedBlock * pNextSCache; + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + // First, unlink as many as can be unlinked. + + pSCache = m_pSCacheList; + flmAssert( !m_pPendingWriteList); + while (pSCache) + { + f_yieldCPU(); + pNextSCache = pSCache->m_pNextInDatabase; + + if (!pSCache->m_uiUseCount) + { + // Turn off all bits that would cause an assert - we don't + // care at this point, because we are forcing the file to + // be closed. + + if (pSCache->m_ui16Flags & (CA_WRITE_TO_LOG | CA_LOG_FOR_CP)) + { + flmAssert( m_uiLogCacheCount); + m_uiLogCacheCount--; + } + if (pSCache->m_pNextInVersionList && + (pSCache->m_pNextInVersionList->m_ui16Flags & + (CA_WRITE_TO_LOG | CA_LOG_FOR_CP))) + { + flmAssert( m_uiLogCacheCount); + m_uiLogCacheCount--; + } + +#ifdef FLM_DEBUG + pSCache->clearFlags( + CA_DIRTY | CA_WRITE_TO_LOG | CA_LOG_FOR_CP | CA_WAS_DIRTY); + + if (pSCache->m_pNextInVersionList) + { + pSCache->m_pNextInVersionList->clearFlags( + CA_DIRTY | CA_WRITE_TO_LOG | CA_LOG_FOR_CP | CA_WAS_DIRTY); + } +#endif + + if (pSCache->m_ui16Flags & CA_IN_FILE_LOG_LIST) + { + pSCache->unlinkFromLogList(); + } + else if( pSCache->m_ui16Flags & CA_IN_NEW_LIST) + { + pSCache->unlinkFromNewList(); + } + + pSCache->unlinkCache( TRUE, NE_SFLM_OK); + } + else + { + // Another thread must have a temporary use on this block + // because it is traversing cache for some reason. We + // don't want to free this block until the use count + // is zero, so just put it into the free list so that + // when its use count goes to zero we will either + // re-use or free it. + + pSCache->unlinkCache( FALSE, NE_SFLM_OK); + pSCache->linkToFreeList( FLM_GET_TIMER()); + } + pSCache = pNextSCache; + } + + // Set the F_Database cache list pointer to NULL. Even if we didn't free + // all of the cache blocks right now, we at least unlinked them from + // the F_Database object. + + m_pSCacheList = NULL; + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +} + + +/**************************************************************************** +Desc: This routine computes an in-memory checksum on a cache block. This + is to guard against corrupt cache blocks being written back to disk. + NOTE: This routine assumes that the cache block mutex is already locked. +****************************************************************************/ +#ifdef FLM_DEBUG +FLMUINT F_CachedBlock::computeChecksum( void) +{ + FLMUINT uiChecksum = 0; + + if( gv_SFlmSysData.pBlockCacheMgr->m_bDebug) + { + FLMUINT uiBlkSize = getBlkSize(); + FLMBYTE * pucBlk = (FLMBYTE *)m_pBlkHdr; + FLMUINT uiLoop; + + for( uiLoop = 0; uiLoop < uiBlkSize; uiLoop += 4, pucBlk += 4) + { + uiChecksum = (uiChecksum ^ (FLMUINT)(*((FLMUINT32 *)(pucBlk)))); + } + + if (!uiChecksum) + { + uiChecksum = 1; + } + } + + return( uiChecksum); +} +#endif + +/**************************************************************************** +Desc: This routine finishes a checkpoint. At this point we are guaranteed + to have both the file lock and the write lock, and all dirty blocks + have been written to the database. This is the code that writes out + the log header and truncates the rollback log, roll-forward log, and + database files as needed. +****************************************************************************/ +RCODE F_Database::finishCheckpoint( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoTruncate, + FLMUINT uiCPFileNum, + FLMUINT uiCPOffset, + FLMUINT uiCPStartTime, + FLMUINT uiTotalToWrite) +{ + RCODE rc = NE_SFLM_OK; + SFLM_DB_HDR * pCommittedDbHdr = &m_lastCommittedDbHdr; + SFLM_DB_HDR saveDbHdr; + FLMUINT uiNewCPFileNum; + FLMUINT64 ui64CurrTransID; + FLMUINT uiSaveTransOffset; + FLMUINT uiSaveCPFileNum; + FLMBOOL bTruncateLog = FALSE; + FLMBOOL bTruncateRflFile = FALSE; + FLMUINT uiTruncateRflSize = 0; + FLMUINT uiLogEof; + FLMUINT uiHighLogFileNumber; +#ifdef FLM_DBG_LOG + FLMBOOL bResetRBL = FALSE; +#endif + + // Update the DB header to indicate that we now + // have a new checkpoint. + + f_memcpy( &saveDbHdr, pCommittedDbHdr, sizeof( SFLM_DB_HDR)); + + // Save some of the values we are going to change. These values + // will be needed below. + + ui64CurrTransID = pCommittedDbHdr->ui64CurrTransID; + uiSaveTransOffset = (FLMUINT)pCommittedDbHdr->ui32RflLastTransOffset; + uiSaveCPFileNum = (FLMUINT)pCommittedDbHdr->ui32RflLastCPFileNum; + +#ifdef FLM_DEBUG + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + // If we get to this point, there should be no dirty blocks for + // the file. + + flmAssert( !m_uiDirtyCacheCount && !m_pPendingWriteList && + (!m_pSCacheList || + !(m_pSCacheList->m_ui16Flags & CA_DIRTY))); + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); +#endif + + // Determine if we can reset the physical log. The log can be reset if + // there are no blocks in the log that are needed to preserve a read + // consistent view for a read transaction. By definition, this will + // be the case if there are no read transactions that started before + // the last transaction committed. Thus, that is what we check. + + // If we exceed a VERY large size, we need to wait for the read + // transactions to empty out so we can force a truncation of the + // log. This is also true if we are truncating the database and + // changing the logical EOF. + + uiLogEof = (FLMUINT)pCommittedDbHdr->ui32RblEOF; + uiHighLogFileNumber = FSGetFileNumber( uiLogEof); + + // Lock the database mutex while modifying the last committed header. + + lockMutex(); + if (uiHighLogFileNumber > 0 || bDoTruncate || + FSGetFileOffset( uiLogEof) > LOW_VERY_LARGE_LOG_THRESHOLD_SIZE) + { + FLMINT iWaitCnt = 0; + F_Db * pFirstDb; + FLMUINT ui5MinutesTime; + FLMUINT ui30SecTime; + char szMsgBuf[ 128]; + IF_LogMessageClient * pLogMsg = NULL; + FLMUINT uiFirstDbInactiveSecs; + FLMUINT uiElapTime; + FLMUINT uiLastMsgTime = FLM_GET_TIMER(); + FLMBOOL bMustTruncate = (bDoTruncate || + uiHighLogFileNumber || + FSGetFileOffset( uiLogEof) >= + HIGH_VERY_LARGE_LOG_THRESHOLD_SIZE) + ? TRUE + : FALSE; + + ui5MinutesTime = FLM_SECS_TO_TIMER_UNITS( 300); + ui30SecTime = FLM_SECS_TO_TIMER_UNITS( 30); + + if (m_pCPInfo && bMustTruncate) + { + m_pCPInfo->uiStartWaitTruncateTime = FLM_GET_TIMER(); + } + + pFirstDb = m_pFirstReadTrans; + while ((!m_pCPInfo || !m_pCPInfo->bShuttingDown) && pFirstDb && + pFirstDb->m_ui64CurrTransID < ui64CurrTransID) + { + FLMUINT uiTime; + FLMUINT uiFirstDbInactiveTime = 0; + FLMUINT64 ui64FirstDbCurrTransID = pFirstDb->m_ui64CurrTransID; + FLMUINT uiFirstDbThreadId = pFirstDb->m_uiThreadId; + F_Db * pTmpDb; + + uiTime = (FLMUINT)FLM_GET_TIMER(); + + if( !bMustTruncate) + { + pTmpDb = pFirstDb; + while( pTmpDb && pTmpDb->m_ui64CurrTransID < ui64CurrTransID) + { + if (!pTmpDb->m_uiInactiveTime) + { + pTmpDb->m_uiInactiveTime = uiTime; + } + pTmpDb = pTmpDb->m_pNextReadTrans; + } + uiFirstDbInactiveTime = pFirstDb->m_uiInactiveTime; + } + + // If the read transaction has been inactive for 5 minutes, + // forcibly kill it unless it has been marked as a "don't kill" + // transaction. + + if( !(pFirstDb->m_uiFlags & FDB_DONT_KILL_TRANS) && + (bMustTruncate || (uiFirstDbInactiveTime && + FLM_ELAPSED_TIME( uiTime, uiFirstDbInactiveTime) >= ui5MinutesTime))) + { + pFirstDb->m_uiKilledTime = uiTime; + if ((m_pFirstReadTrans = pFirstDb->m_pNextReadTrans) != NULL) + { + m_pFirstReadTrans->m_pPrevReadTrans = NULL; + } + else + { + m_pLastReadTrans = NULL; + } + pFirstDb->m_pPrevReadTrans = NULL; + + if ((pFirstDb->m_pNextReadTrans = m_pFirstKilledTrans) != NULL) + { + pFirstDb->m_pNextReadTrans->m_pPrevReadTrans = pFirstDb; + } + m_pFirstKilledTrans = pFirstDb; + + unlockMutex(); + + // Log a message indicating that we have killed the transaction + + if ((pLogMsg = flmBeginLogMessage( SFLM_GENERAL_MESSAGE)) != NULL) + { + uiElapTime = FLM_ELAPSED_TIME( uiTime, uiFirstDbInactiveTime); + uiFirstDbInactiveSecs = FLM_TIMER_UNITS_TO_SECS( uiElapTime); + + f_sprintf( szMsgBuf, + "Killed transaction %I64u." + " Thread: %X." + " Inactive time: %u seconds.", + ui64FirstDbCurrTransID, + (unsigned)uiFirstDbThreadId, + (unsigned)uiFirstDbInactiveSecs); + + pLogMsg->changeColor( FLM_YELLOW, FLM_BLACK); + pLogMsg->appendString( szMsgBuf); + flmEndLogMessage( &pLogMsg); + } + + lockMutex(); + pFirstDb = m_pFirstReadTrans; + continue; + } + else if (!bMustTruncate) + { + if (iWaitCnt >= 200) + { + break; + } + } + + unlockMutex(); + + if (!bMustTruncate) + { + iWaitCnt++; + f_sleep( 6); + } + else + { + // Log a message indicating that we are waiting for the + // transaction to complete + + if (FLM_ELAPSED_TIME( uiTime, uiLastMsgTime) >= ui30SecTime) + { + if ((pLogMsg = flmBeginLogMessage( SFLM_GENERAL_MESSAGE)) != NULL) + { + uiElapTime = FLM_ELAPSED_TIME( uiTime, uiFirstDbInactiveTime); + uiFirstDbInactiveSecs = FLM_TIMER_UNITS_TO_SECS( uiElapTime); + + f_sprintf( szMsgBuf, + "Waiting for transaction %I64u to complete." + " Thread: %X." + " Inactive time: %u seconds.", + ui64FirstDbCurrTransID, + (unsigned)uiFirstDbThreadId, + (unsigned)uiFirstDbInactiveSecs); + + pLogMsg->changeColor( FLM_YELLOW, FLM_BLACK); + pLogMsg->appendString( szMsgBuf); + flmEndLogMessage( &pLogMsg); + } + + uiLastMsgTime = FLM_GET_TIMER(); + } + + f_sleep( 100); + } + lockMutex(); + pFirstDb = m_pFirstReadTrans; + } + + if (bMustTruncate && m_pCPInfo) + { + m_pCPInfo->uiStartWaitTruncateTime = 0; + } + } + + if (!m_pFirstReadTrans || + m_pFirstReadTrans->m_ui64CurrTransID >= ui64CurrTransID) + { + + // We may want to truncate the log file if it has grown real big. + + if (uiHighLogFileNumber > 0 || + FSGetFileOffset( uiLogEof) > LOG_THRESHOLD_SIZE) + { + bTruncateLog = TRUE; + } + pCommittedDbHdr->ui32RblEOF = (FLMUINT32)pCommittedDbHdr->ui16BlockSize; +#ifdef FLM_DBG_LOG + bResetRBL = TRUE; +#endif + } + pCommittedDbHdr->ui32RblFirstCPBlkAddr = 0; + + // Set the checkpoint RFL file number and offset to be the same as + // the last transaction's RFL file number and offset if nothing + // is passed in. If a non-zero uiCPFileNum is passed in, it is because + // we are checkpointing the last transaction that has been recovered + // by the recovery process. + // In this case, instead of moving the pointers all the way forward, + // to the last committed transaction, we simply move them forward to + // the last recovered transaction. + + if (uiCPFileNum) + { + pCommittedDbHdr->ui32RflLastCPFileNum = (FLMUINT32)uiCPFileNum; + pCommittedDbHdr->ui32RflLastCPOffset = (FLMUINT32)uiCPOffset; + } + else + { + FLMBOOL bResetRflFile = FALSE; + + // If the RFL volume is full, and the LOG_AUTO_TURN_OFF_KEEP_RFL + // flag is TRUE, change the LOG_KEEP_RFL_FILES to FALSE. + + if (m_pRfl->isRflVolumeFull() && + pCommittedDbHdr->ui8RflKeepFiles && + pCommittedDbHdr->ui8RflAutoTurnOffKeep) + { + pCommittedDbHdr->ui8RflKeepFiles = 0; + bResetRflFile = TRUE; + } + + pCommittedDbHdr->ui32RflLastCPFileNum = + pCommittedDbHdr->ui32RflCurrFileNum; + + if (!pCommittedDbHdr->ui8RflKeepFiles) + { + pCommittedDbHdr->ui32RflLastCPOffset = 512; + if (bResetRflFile) + { + + // This will cause the RFL file to be recreated on the + // next transaction - causing the keep signature to be + // changed. Also need to set up to use new serial + // numbers so restore can't wade into this RFL file and + // attempt to start restoring from it. + + pCommittedDbHdr->ui32RflLastTransOffset = 0; + f_createSerialNumber( pCommittedDbHdr->ucLastTransRflSerialNum); + f_createSerialNumber( pCommittedDbHdr->ucNextRflSerialNum); + } + + // If LOG_RFL_LAST_TRANS_OFFSET is zero, someone has set this up + // intentionally to cause the RFL file to be created at the + // beginning of the next transaction. We don't want to lose + // that, so if it is zero, we don't change it. + + else if (pCommittedDbHdr->ui32RflLastTransOffset != 0) + { + pCommittedDbHdr->ui32RflLastTransOffset = 512; + } + uiTruncateRflSize = (FLMUINT)pCommittedDbHdr->ui32RflMinFileSize; + if ((uiSaveTransOffset >= RFL_TRUNCATE_SIZE) || + (uiSaveTransOffset >= uiTruncateRflSize)) + { + bTruncateRflFile = TRUE; + if (uiTruncateRflSize > RFL_TRUNCATE_SIZE) + { + uiTruncateRflSize = RFL_TRUNCATE_SIZE; + } + else if (uiTruncateRflSize < 512) + { + uiTruncateRflSize = 512; + } + + // Set to nearest 512 byte boundary + + uiTruncateRflSize &= 0xFFFFFE00; + } + } + else + { + FLMUINT uiLastTransOffset = + (FLMUINT)pCommittedDbHdr->ui32RflLastTransOffset; + + // If the RFL volume is not OK, and we are not currently positioned + // at the beginning of an RFL file, we should set things up to roll to + // the next RFL file. That way, if they need to change RFL volumes + // it will be OK, and we can create the new RFL file. + + if (!m_pRfl->seeIfRflVolumeOk() && uiLastTransOffset > 512) + { + pCommittedDbHdr->ui32RflLastTransOffset = 0; + pCommittedDbHdr->ui32RflCurrFileNum++; + pCommittedDbHdr->ui32RflLastCPOffset = 512; + pCommittedDbHdr->ui32RflLastCPFileNum = + pCommittedDbHdr->ui32RflCurrFileNum; + } + else + { + // If the transaction offset is zero, we want the last CP offset + // to be 512 - it should never be set to zero. It is possible + // for the transaction offset to still be zero at this point if + // we haven't done a non-empty transaction yet. + + if (!uiLastTransOffset) + { + uiLastTransOffset = 512; + } + pCommittedDbHdr->ui32RflLastCPOffset = + (FLMUINT32)uiLastTransOffset; + } + } + } + + // Set the checkpoint Trans ID to be the trans ID of the + // last committed transaction. + + pCommittedDbHdr->ui64RflLastCPTransID = pCommittedDbHdr->ui64CurrTransID; + + unlockMutex(); + + // Write the log header - this will complete the checkpoint. + + if (RC_BAD( rc = writeDbHdr( pDbStats, pSFileHdl, + pCommittedDbHdr, + &m_checkpointDbHdr, TRUE))) + { + + // Restore log header. + + lockMutex(); + f_memcpy( pCommittedDbHdr, &saveDbHdr, sizeof( SFLM_DB_HDR)); + unlockMutex(); + goto Exit; + } + else if (bTruncateLog) + { + IF_FileHdl * pCFileHdl; + + if (uiHighLogFileNumber) + { + (void)pSFileHdl->truncateFiles( + FIRST_LOG_BLOCK_FILE_NUMBER, + uiHighLogFileNumber); + } + + if (RC_OK( pSFileHdl->getFileHdl( 0, TRUE, &pCFileHdl))) + { + (void)pCFileHdl->truncate( LOG_THRESHOLD_SIZE); + } + } + +#ifdef FLM_DBG_LOG + if (bResetRBL) + { + char szMsg [80]; + + if (bTruncateLog) + { + f_sprintf( szMsg, "f%u, Reset&TruncRBL, CPTID:%I64u", + (unsigned)((FLMUINT)this), + pCommittedDbHdr->ui64RflLastCPTransID); + } + else + { + f_sprintf( szMsg, "f%u, ResetRBL, CPTID:%I64u", + (unsigned)((FLMUINT)this), + pCommittedDbHdr->ui64RflLastCPTransID); + } + flmDbgLogMsg( szMsg); + } +#endif + + // The checkpoint is now complete. Reset the first checkpoint + // block address to zero. + + m_uiFirstLogCPBlkAddress = 0; + m_uiLastCheckpointTime = (FLMUINT)FLM_GET_TIMER(); + + // Save the state of the log header into the checkpointDbHdr buffer. + + f_memcpy( &m_checkpointDbHdr, pCommittedDbHdr, sizeof( SFLM_DB_HDR)); + + // See if we need to delete RFL files that are no longer in use. + + uiNewCPFileNum = (FLMUINT)pCommittedDbHdr->ui32RflLastCPFileNum; + + if (!pCommittedDbHdr->ui8RflKeepFiles && + uiSaveCPFileNum != uiNewCPFileNum && + uiNewCPFileNum > 1) + { + FLMUINT uiLastRflFileDeleted = + (FLMUINT)pCommittedDbHdr->ui32RflLastFileNumDeleted; + + uiLastRflFileDeleted++; + while (uiLastRflFileDeleted < uiNewCPFileNum) + { + char szLogFilePath [F_PATH_MAX_SIZE]; + RCODE TempRc; + FLMUINT uiFileNameSize = sizeof( szLogFilePath); + FLMBOOL bNameTruncated; + + m_pRfl->getFullRflFileName( uiLastRflFileDeleted, + szLogFilePath, &uiFileNameSize, + &bNameTruncated); + if (bNameTruncated) + { + break; + } + if (RC_BAD( TempRc = gv_SFlmSysData.pFileSystem->deleteFile( + szLogFilePath))) + { + if (TempRc != NE_FLM_IO_PATH_NOT_FOUND && + TempRc != NE_FLM_IO_INVALID_FILENAME) + { + break; + } + } + uiLastRflFileDeleted++; + } + uiLastRflFileDeleted--; + + // If we actually deleted a file, update the log header. + + if (uiLastRflFileDeleted != + (FLMUINT)pCommittedDbHdr->ui32RflLastFileNumDeleted) + { + pCommittedDbHdr->ui32RflLastFileNumDeleted = + (FLMUINT32)uiLastRflFileDeleted; + if (RC_BAD( rc = writeDbHdr( pDbStats, pSFileHdl, + pCommittedDbHdr, + &m_checkpointDbHdr, TRUE))) + { + goto Exit; + } + + // Save the state of the log header into the checkpointDbHdr buffer + // and update the last checkpoint time again. + + f_memcpy( &m_checkpointDbHdr, pCommittedDbHdr, + sizeof( SFLM_DB_HDR)); + m_uiLastCheckpointTime = (FLMUINT)FLM_GET_TIMER(); + } + } + + // Truncate the RFL file, if the truncate flag was set above. + + if (bTruncateRflFile) + { + (void)m_pRfl->truncate( hWaitSem, uiTruncateRflSize); + } + + // Truncate the files, if requested to do so - this would be a request of + // FlmDbReduceSize. + + if (bDoTruncate) + { + if (RC_BAD( rc = pSFileHdl->truncateFile( + (FLMUINT)pCommittedDbHdr->ui32LogicalEOF))) + { + goto Exit; + } + } + + // Re-enable the RFL volume OK flag - in case it was turned off somewhere. + + m_pRfl->setRflVolumeOk(); + + // If we complete a checkpoint successfully, we want to set the + // pFile->CheckpointRc so that new transactions can come in. + // NOTE: CheckpointRc should only be set while we still have the + // lock on the database - which should always be the case at this + // point. This routine can only be called if we have obtained both + // the write lock and the file lock. + + m_CheckpointRc = NE_SFLM_OK; + + // If we were calculating our maximum dirty cache, finish the + // calculation. + + if (uiCPStartTime) + { + FLMUINT uiCPEndTime = FLM_GET_TIMER(); + FLMUINT uiCPElapsedTime = FLM_ELAPSED_TIME( uiCPEndTime, uiCPStartTime); + FLMUINT uiElapsedMilli; + FLMUINT ui15Seconds; + FLMUINT uiMaximum; + FLMUINT uiLow; + + // Get elapsed time in milliseconds - only calculate a new maximum if + // we did at least a half second worth of writing. + + uiElapsedMilli = FLM_TIMER_UNITS_TO_MILLI( uiCPElapsedTime); + + if (uiElapsedMilli >= 500) + { + + // Calculate what could be written in 15 seconds - set maximum + // to that. If calculated maximum is zero, we will not change + // the current maximum. + + ui15Seconds = FLM_SECS_TO_TIMER_UNITS( 15); + + uiMaximum = (FLMUINT)(((FLMUINT64)uiTotalToWrite * + (FLMUINT64)ui15Seconds) / (FLMUINT64)uiCPElapsedTime); + if (uiMaximum) + { + // Low is maximum minus what could be written in roughly + // two seconds. + + uiLow = uiMaximum - (uiMaximum / 7); + + // Only set the maximum if we are still in auto-calculate mode. + + if (gv_SFlmSysData.pBlockCacheMgr->m_bAutoCalcMaxDirty) + { + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + + // Test flag again after locking the mutex + + if (gv_SFlmSysData.pBlockCacheMgr->m_bAutoCalcMaxDirty) + { + gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache = uiMaximum; + gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache = uiLow; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + } + } + } + +Exit: + + return( rc); +} + +/**************************************************************************** +Desc: This routine performs a checkpoint. It will stay in here until + it either finishes, gets interrupted, or gets an error. If we are not + forcing a checkpoint, we periodically check to see if we should switch + to a forced mode. We also periodically check to see if another thread + needs is waiting to obtain the write lock. +****************************************************************************/ +RCODE F_Database::doCheckpoint( + F_SEM hWaitSem, + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoTruncate, + FLMBOOL bForceCheckpoint, + eForceCPReason eForceReason, + FLMUINT uiCPFileNum, + FLMUINT uiCPOffset) +{ + RCODE rc = NE_SFLM_OK; + FLMBOOL bWroteAll; + FLMUINT uiCPStartTime = 0; + FLMUINT uiTotalToWrite; + FLMUINT uiMaxDirtyCache; + F_CachedBlock * pSCache; + FLMUINT uiTimestamp; + + pSFileHdl->enableFlushMinimize(); + + if (m_pCPInfo) + { + lockMutex(); + m_pCPInfo->bDoingCheckpoint = TRUE; + m_pCPInfo->uiStartTime = (FLMUINT)FLM_GET_TIMER(); + m_pCPInfo->bForcingCheckpoint = bForceCheckpoint; + if (bForceCheckpoint) + { + m_pCPInfo->uiForceCheckpointStartTime = m_pCPInfo->uiStartTime; + } + m_pCPInfo->eForceCheckpointReason = eForceReason; + m_pCPInfo->uiDataBlocksWritten = + m_pCPInfo->uiLogBlocksWritten = 0; + unlockMutex(); + } + + uiTotalToWrite = (m_uiDirtyCacheCount + m_uiLogCacheCount) * + m_uiBlockSize; + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + if (bForceCheckpoint) + { + if (gv_SFlmSysData.pBlockCacheMgr->m_bAutoCalcMaxDirty) + { + uiCPStartTime = FLM_GET_TIMER(); + } + } + + // If the amount of dirty cache is over our maximum, we must at least bring + // it down below the low threshhold. Otherwise, we set uiMaxDirtyCache + // to the highest possible value - which will not require us to get + // it below anything - because it is already within limits. + + if (gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache && + uiTotalToWrite > gv_SFlmSysData.pBlockCacheMgr->m_uiMaxDirtyCache) + { + uiMaxDirtyCache = gv_SFlmSysData.pBlockCacheMgr->m_uiLowDirtyCache; + } + else + { + uiMaxDirtyCache = ~((FLMUINT)0); + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + + // Write out log blocks first. + + bWroteAll = TRUE; + if (RC_BAD( rc = flushLogBlocks( hWaitSem, pDbStats, pSFileHdl, + TRUE, uiMaxDirtyCache, + &bForceCheckpoint, &bWroteAll))) + { + goto Exit; + } + + // If we didn't write out all log blocks, we got interrupted. + + if (!bWroteAll) + { + flmAssert( !bForceCheckpoint); + goto Exit; + } + + // Now write out dirty blocks + + if (RC_BAD( rc = flushDirtyBlocks( pDbStats, pSFileHdl, + uiMaxDirtyCache, + bForceCheckpoint, TRUE, &bWroteAll))) + { + goto Exit; + } + + // If we didn't write out all dirty blocks, we got interrupted + + if (!bWroteAll) + { + flmAssert( !bForceCheckpoint); + goto Exit; + } + + // All dirty blocks and log blocks have been written, so we just + // need to finish the checkpoint. + + if (RC_BAD( rc = finishCheckpoint( hWaitSem, pDbStats, pSFileHdl, + bDoTruncate, uiCPFileNum, uiCPOffset, + uiCPStartTime, uiTotalToWrite))) + { + goto Exit; + } + +Exit: + + // If we were attempting to force a checkpoint and it failed, + // we want to set m_CheckpointRc, because we want to + // prevent new transactions from starting until this situation + // is cleared up (see fltrbeg.cpp). Note that setting + // m_CheckpointRc to something besides NE_SFLM_OK will cause + // the checkpoint thread to force checkpoints whenever it is woke + // up until it succeeds (see flopen.cpp). + + if (RC_BAD( rc) && bForceCheckpoint) + { + m_CheckpointRc = rc; + } + + // Timestamp all of the items in the free list + + if (bForceCheckpoint) + { + uiTimestamp = FLM_GET_TIMER(); + + f_mutexLock( gv_SFlmSysData.hBlockCacheMutex); + pSCache = gv_SFlmSysData.pBlockCacheMgr->m_pFirstFree; + while (pSCache) + { + pSCache->m_uiBlkAddress = uiTimestamp; + pSCache = pSCache->m_pNextInDatabase; + } + f_mutexUnlock( gv_SFlmSysData.hBlockCacheMutex); + } + + if (m_pCPInfo) + { + lockMutex(); + m_pCPInfo->bDoingCheckpoint = FALSE; + unlockMutex(); + } + + pSFileHdl->disableFlushMinimize(); + + return( rc); +} + +/**************************************************************************** +Notes: This routine assumes the cache block cache mutex is locked + This is a static method, so there is no "this" pointer to the + F_BlockCacheMgr object. +****************************************************************************/ +FLMBOOL F_BlockRelocator::canRelocate( + void * pvAlloc) +{ + return( ((F_CachedBlock *)pvAlloc)->m_uiUseCount ? FALSE : TRUE); +} + +/**************************************************************************** +Desc: Fixes up all pointers needed to allow an F_CachedBlock object to be + moved to a different location in memory +Notes: This routine assumes the cache block mutex is locked + This is a static method, so there is no "this" pointer to the + F_BlockCacheMgr object. +****************************************************************************/ +void F_BlockRelocator::relocate( + void * pvOldAlloc, + void * pvNewAlloc) +{ + F_CachedBlock * pOldSCache = (F_CachedBlock *)pvOldAlloc; + F_CachedBlock * pNewSCache = (F_CachedBlock *)pvNewAlloc; + F_CachedBlock ** ppBucket; + F_BlockCacheMgr * pBlockCacheMgr = gv_SFlmSysData.pBlockCacheMgr; + F_Database * pDatabase = pOldSCache->m_pDatabase; + + flmAssert( !pOldSCache->m_uiUseCount); + + if( pNewSCache->m_pPrevInDatabase) + { + pNewSCache->m_pPrevInDatabase->m_pNextInDatabase = pNewSCache; + } + + if( pNewSCache->m_pNextInDatabase) + { + pNewSCache->m_pNextInDatabase->m_pPrevInDatabase = pNewSCache; + } + + if( pNewSCache->m_pPrevInGlobal) + { + pNewSCache->m_pPrevInGlobal->m_pNextInGlobal = pNewSCache; + } + + if( pNewSCache->m_pNextInGlobal) + { + pNewSCache->m_pNextInGlobal->m_pPrevInGlobal = pNewSCache; + } + + if( pNewSCache->m_pPrevInReplaceList) + { + pNewSCache->m_pPrevInReplaceList->m_pNextInReplaceList = pNewSCache; + } + + if( pNewSCache->m_pNextInReplaceList) + { + pNewSCache->m_pNextInReplaceList->m_pPrevInReplaceList = pNewSCache; + } + + if( pNewSCache->m_pPrevInHashBucket) + { + pNewSCache->m_pPrevInHashBucket->m_pNextInHashBucket = pNewSCache; + } + + if( pNewSCache->m_pNextInHashBucket) + { + pNewSCache->m_pNextInHashBucket->m_pPrevInHashBucket = pNewSCache; + } + + if( pNewSCache->m_pPrevInVersionList) + { + pNewSCache->m_pPrevInVersionList->m_pNextInVersionList = pNewSCache; + } + + if( pNewSCache->m_pNextInVersionList) + { + pNewSCache->m_pNextInVersionList->m_pPrevInVersionList = pNewSCache; + } + + if( pDatabase) + { + if( pDatabase->m_pSCacheList == pOldSCache) + { + pDatabase->m_pSCacheList = pNewSCache; + } + + if( pDatabase->m_pLastDirtyBlk == pOldSCache) + { + pDatabase->m_pLastDirtyBlk = pNewSCache; + } + + if( pDatabase->m_pFirstInLogList == pOldSCache) + { + pDatabase->m_pFirstInLogList = pNewSCache; + } + + if( pDatabase->m_pLastInLogList == pOldSCache) + { + pDatabase->m_pLastInLogList = pNewSCache; + } + + if( pDatabase->m_pFirstInNewList == pOldSCache) + { + pDatabase->m_pFirstInNewList = pNewSCache; + } + + if( pDatabase->m_pLastInNewList == pOldSCache) + { + pDatabase->m_pLastInNewList = pNewSCache; + } + + if( pDatabase->m_pTransLogList == pOldSCache) + { + pDatabase->m_pTransLogList = pNewSCache; + } + + ppBucket = pBlockCacheMgr->blockHash( pDatabase->getSigBitsInBlkSize(), + pOldSCache->m_uiBlkAddress); + + if( *ppBucket == pOldSCache) + { + *ppBucket = pNewSCache; + } + + flmAssert( pDatabase->m_pPendingWriteList != pOldSCache); + } + + if (pBlockCacheMgr->m_MRUList.m_pMRUItem == (F_CachedItem *)pOldSCache) + { + pBlockCacheMgr->m_MRUList.m_pMRUItem = pNewSCache; + } + + if (pBlockCacheMgr->m_MRUList.m_pLRUItem == (F_CachedItem *)pOldSCache) + { + pBlockCacheMgr->m_MRUList.m_pLRUItem = pNewSCache; + } + + if (pBlockCacheMgr->m_MRUList.m_pLastMRUItem == (F_CachedItem *)pOldSCache) + { + pBlockCacheMgr->m_MRUList.m_pLastMRUItem = pNewSCache; + } + + if (pBlockCacheMgr->m_pMRUReplace == pOldSCache) + { + pBlockCacheMgr->m_pMRUReplace = pNewSCache; + } + + if (pBlockCacheMgr->m_pLRUReplace == pOldSCache) + { + pBlockCacheMgr->m_pLRUReplace = pNewSCache; + } + + if (pBlockCacheMgr->m_pFirstFree == pOldSCache) + { + pBlockCacheMgr->m_pFirstFree = pNewSCache; + } + + if (pBlockCacheMgr->m_pLastFree == pOldSCache) + { + pBlockCacheMgr->m_pLastFree = pNewSCache; + } + + pNewSCache->m_pBlkHdr = (F_BLK_HDR *)&pNewSCache[ 1]; +} + +/**************************************************************************** +Desc: This function will encrypt the block of data passed in. This function + assumes that the buffer passed in includes the block header. +****************************************************************************/ +RCODE F_Database::encryptBlock( + F_Dict * pDict, + FLMBYTE * pucBuffer + ) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + F_TABLE * pTable; + FLMUINT uiLfNum; + F_BTREE_BLK_HDR * pBlkHdr = (F_BTREE_BLK_HDR *)pucBuffer; + F_ENCDEF * pEncDef = NULL; + FLMUINT uiEncDefNum; + FLMUINT uiEncLen = m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr); +#ifdef FLM_USE_NICI + F_CCS * pCcs = NULL; +#endif + + if (!blkIsBTree( (F_BLK_HDR *)pucBuffer)) + { + // Nothing to do. We are only interested in btree blocks. + goto Exit; + } + + if (!isEncryptedBlk( (F_BLK_HDR *)pBlkHdr)) + { + goto Exit; + } + + if (pBlkHdr->stdBlkHdr.ui8BlkType == BT_DATA_ONLY) + { + uiEncLen = m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr); + + if (!m_bTempDb) + { + uiEncDefNum = (FLMUINT)(((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum); + + // Need to get the encryption object. + + if ((pEncDef = pDict->getEncDef( uiEncDefNum)) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + } + } + else if (isTableBlk( pBlkHdr)) + { + if (!m_bTempDb) + { + uiLfNum = pBlkHdr->ui16LogicalFile; + + // Get the table + if ((pTable = pDict->getTable( uiLfNum)) == NULL) + { + goto Exit; + } + + // The collection may not be encrypted. + // We can just exit here. + + if (!pTable->lfInfo.uiEncDefNum) + { + goto Exit; // NE_SFLM_OK; + } + + // Need to get the encryption object. + pEncDef = pDict->getEncDef( pTable->lfInfo.uiEncDefNum); + flmAssert( pEncDef); + } + } + else if (isIndexBlk( pBlkHdr)) + { + if (!m_bTempDb) + { + uiLfNum = pBlkHdr->ui16LogicalFile; + + // Get the index. + if ((pIndex = pDict->getIndex( uiLfNum)) == NULL) + { + goto Exit; + } + + // The index may not be encrypted. + // We can just exit here. + if (!pIndex->lfInfo.uiEncDefNum) + { + goto Exit; // NE_SFLM_OK; + } + + // Need to get the encryption object. + pEncDef = pDict->getEncDef( pIndex->lfInfo.uiEncDefNum); + flmAssert( pEncDef); + } + } + else + { + goto Exit; // NE_SFLM_OK + } + +#ifndef FLM_USE_NICI + rc = RC_SET( NE_SFLM_ENCRYPTION_UNAVAILABLE); + goto Exit; +#else + + if (m_bInLimitedMode) + { + rc = RC_SET( NE_SFLM_ENCRYPTION_UNAVAILABLE); + goto Exit; + } + + if (!m_bTempDb) + { + flmAssert( pEncDef); + pCcs = pEncDef->pCcs; + } + else + { + flmAssert( !pEncDef); + pCcs = m_pWrappingKey; + } + + flmAssert( pCcs); + flmAssert( !(uiEncLen % 16)); + + // Encrypt the buffer in place. + + if (pBlkHdr->stdBlkHdr.ui8BlkType == BT_DATA_ONLY) + { + if (RC_BAD( rc = pCcs->encryptToStore( &pucBuffer[ sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)], + uiEncLen, + &pucBuffer[ sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)], + &uiEncLen))) + { + goto Exit; + } + + flmAssert( uiEncLen == (m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr))); + + } + else + { + if (RC_BAD( rc = pCcs->encryptToStore( &pucBuffer[ sizeofBTreeBlkHdr( pBlkHdr)], + uiEncLen, + &pucBuffer[ sizeofBTreeBlkHdr( pBlkHdr)], + &uiEncLen))) + { + goto Exit; + } + + flmAssert( uiEncLen == (m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr))); + + } +#endif + +Exit: + + return rc; +} + +/**************************************************************************** +Desc: This function will decrypt the block of data passed in. +****************************************************************************/ +RCODE F_Database::decryptBlock( + F_Dict * pDict, + FLMBYTE * pucBuffer) +{ + RCODE rc = NE_SFLM_OK; + F_INDEX * pIndex; + F_TABLE * pTable; + FLMUINT uiLfNum; + F_BTREE_BLK_HDR * pBlkHdr = (F_BTREE_BLK_HDR *)pucBuffer; + FLMUINT uiEncLen; + F_ENCDEF * pEncDef = NULL; +#ifdef FLM_USE_NICI + F_CCS * pCcs = NULL; +#endif + + if (!blkIsBTree( (F_BLK_HDR *)pucBuffer)) + { + // Nothing to do. We are only interested in btree blocks. + goto Exit; + } + + if (!isEncryptedBlk( (F_BLK_HDR *)pBlkHdr)) + { + goto Exit; + } + + if (pBlkHdr->stdBlkHdr.ui8BlkType == BT_DATA_ONLY) + { + + uiEncLen = m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr); + + if (!m_bTempDb) + { + + // Need to get the encryption object. + + if ((pEncDef = pDict->getEncDef( + (FLMUINT)(((F_ENC_DO_BLK_HDR *)pBlkHdr)->ui32EncDefNum))) == NULL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_INVALID_ENCDEF_NUM); + goto Exit; + } + } + } + else if (isTableBlk( pBlkHdr)) + { + uiEncLen = m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr); + + if (!m_bTempDb) + { + uiLfNum = pBlkHdr->ui16LogicalFile; + + // Get the index. + if ((pTable = pDict->getTable( uiLfNum)) == NULL) + { + goto Exit; + } + + // The collection may not be encrypted. + // We can just exit here. + + if (!pTable->lfInfo.uiEncDefNum) + { + goto Exit; // NE_SFLM_OK; + } + + // Need to get the encryption object. + + pEncDef = pDict->getEncDef( pTable->lfInfo.uiEncDefNum); + flmAssert( pEncDef); + } + } + else if (isIndexBlk( pBlkHdr)) + { + uiEncLen = m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr); + + if (!m_bTempDb) + { + uiLfNum = pBlkHdr->ui16LogicalFile; + + // Get the index. + + if ((pIndex = pDict->getIndex( uiLfNum)) == NULL) + { + goto Exit; + } + + // The index may not be encrypted. + // We can just exit here. + + if (!pIndex->lfInfo.uiEncDefNum) + { + goto Exit; // NE_SFLM_OK; + } + + // Need to get the encryption object. + + pEncDef = pDict->getEncDef( pIndex->lfInfo.uiEncDefNum); + flmAssert( pEncDef); + } + } + else + { + goto Exit; // NE_SFLM_OK + } + + +#ifndef FLM_USE_NICI + rc = RC_SET( NE_SFLM_ENCRYPTION_UNAVAILABLE); + goto Exit; +#else + + if (m_bInLimitedMode) + { + rc = RC_SET( NE_SFLM_ENCRYPTION_UNAVAILABLE); + goto Exit; + } + + if (!m_bTempDb) + { + flmAssert( pEncDef); + pCcs = pEncDef->pCcs; + } + else + { + flmAssert( !pEncDef); + pCcs = m_pWrappingKey; + } + + flmAssert( pCcs); + flmAssert( !(uiEncLen % 16)); + + if (pBlkHdr->stdBlkHdr.ui8BlkType == BT_DATA_ONLY) + { + if (RC_BAD( rc = pCcs->decryptFromStore( + &pucBuffer[ sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)], uiEncLen, + &pucBuffer[ sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr)], &uiEncLen))) + { + goto Exit; + } + + flmAssert( uiEncLen == + (m_uiBlockSize - sizeofDOBlkHdr( (F_BLK_HDR *)pBlkHdr))); + } + else + { + if (RC_BAD( rc = pCcs->decryptFromStore( + &pucBuffer[ sizeofBTreeBlkHdr( pBlkHdr)], uiEncLen, + &pucBuffer[ sizeofBTreeBlkHdr( pBlkHdr)], &uiEncLen))) + { + goto Exit; + } + + flmAssert( uiEncLen == (m_uiBlockSize - sizeofBTreeBlkHdr( pBlkHdr))); + } + +#endif + +Exit: + + return rc; +} + +#undef new +#undef delete + +/**************************************************************************** +Desc: +****************************************************************************/ +void * F_CachedBlock::operator new( + FLMSIZET uiSize, + FLMUINT uiBlockSize, + FLMBOOL bAllocMutexLocked) +#ifndef FLM_NLM + throw() +#endif +{ + void * pvPtr; + + flmAssert( uiSize == sizeof( F_CachedBlock)); + if( RC_BAD( gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->allocBuf( + &gv_SFlmSysData.pBlockCacheMgr->m_blockRelocator, + uiSize + uiBlockSize, (FLMBYTE **)&pvPtr, bAllocMutexLocked))) + { + pvPtr = NULL; + } + + return( pvPtr); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void * F_CachedBlock::operator new( + FLMSIZET) //uiSize) +#ifndef FLM_NLM + throw() +#endif +{ + // This new should never be called + flmAssert( 0); + return( NULL); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void * F_CachedBlock::operator new[]( FLMSIZET) +#ifndef FLM_NLM + throw() +#endif +{ + flmAssert( 0); + return( NULL); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +#ifdef FLM_DEBUG +void * F_CachedBlock::operator new( + FLMSIZET, // uiSize, + const char *, // pszFileName, + int) // iLineNum) +#ifndef FLM_NLM + throw() +#endif +{ + // This new should never be called + flmAssert( 0); + return( NULL); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +#ifdef FLM_DEBUG +void * F_CachedBlock::operator new[]( + FLMSIZET, // uiSize, + const char *, // pszFileName, + int) // iLine) +#ifndef FLM_NLM + throw() +#endif +{ + flmAssert( 0); + return( NULL); +} +#endif + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_CachedBlock::operator delete( + void * ptr) +{ + if( !ptr) + { + return; + } + + gv_SFlmSysData.pBlockCacheMgr->m_pBlockAllocator->freeBuf( (FLMBYTE **)&ptr); +} + +/**************************************************************************** +Desc: +****************************************************************************/ +void F_CachedBlock::operator delete[]( + void *) // ptr +{ + flmAssert( 0); +} diff --git a/sql/src/sqleval.cpp b/sql/src/sqleval.cpp new file mode 100644 index 0000000..b664ac4 --- /dev/null +++ b/sql/src/sqleval.cpp @@ -0,0 +1,1486 @@ +//------------------------------------------------------------------------- +// Desc: Parse SQL +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------- + +#include "flaimsys.h" + +// Local function prototypes + +FSTATIC RCODE sqlCompareText( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + FLMUINT uiCompareRules, + FLMBOOL bOpIsMatch, + FLMUINT uiLanguage, + FLMINT * piResult); + +FSTATIC RCODE sqlCompareBinary( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + FLMINT * piResult); + +FSTATIC void sqlArithOpUUBitAND( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUBitOR( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUBitXOR( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUSMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSSMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSUMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUSDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSSDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSUDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUSMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSSMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSUMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUSPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSSPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSUPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUUMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpUSMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSSMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +FSTATIC void sqlArithOpSUMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +typedef void SQL_ARITH_OP( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult); + +SQL_ARITH_OP * SQL_ArithOpTable[ + ((SQL_LAST_ARITH_OP - SQL_FIRST_ARITH_OP) + 1) * 4 ] = +{ +/* U = Unsigned S = Signed + U + U U + S + S + U S + S */ +/* BITAND */ sqlArithOpUUBitAND, sqlArithOpUUBitAND, + sqlArithOpUUBitAND, sqlArithOpUUBitAND, +/* BITOR */ sqlArithOpUUBitOR, sqlArithOpUUBitOR, + sqlArithOpUUBitOR, sqlArithOpUUBitOR, +/* BITXOR */ sqlArithOpUUBitXOR, sqlArithOpUUBitXOR, + sqlArithOpUUBitXOR, sqlArithOpUUBitXOR, +/* MULT */ sqlArithOpUUMult, sqlArithOpUSMult, + sqlArithOpSUMult, sqlArithOpSSMult, +/* DIV */ sqlArithOpUUDiv, sqlArithOpUSDiv, + sqlArithOpSUDiv, sqlArithOpSSDiv, +/* MOD */ sqlArithOpUUMod, sqlArithOpUSMod, + sqlArithOpSUMod, sqlArithOpSSMod, +/* PLUS */ sqlArithOpUUPlus, sqlArithOpUSPlus, + sqlArithOpSUPlus, sqlArithOpSSPlus, +/* MINUS */ sqlArithOpUUMinus, sqlArithOpUSMinus, + sqlArithOpSUMinus, sqlArithOpSSMinus +}; + +//------------------------------------------------------------------------- +// Desc: Compare two entire strings. +//------------------------------------------------------------------------- +FSTATIC RCODE sqlCompareText( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + FLMUINT uiCompareRules, + FLMBOOL bOpIsMatch, + FLMUINT uiLanguage, + FLMINT * piResult) +{ + RCODE rc = NE_SFLM_OK; + F_BufferIStream bufferLStream; + IF_PosIStream * pLStream; + F_BufferIStream bufferRStream; + IF_PosIStream * pRStream; + + // Types must be text + + if (pLValue->eValType != SQL_UTF8_VAL || pRValue->eValType != SQL_UTF8_VAL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + + // Open the streams + + if( !(pLValue->uiFlags & SQL_VAL_IS_STREAM)) + { + if (RC_BAD( rc = bufferLStream.open( (const char *)pLValue->val.pucBuf, + pLValue->uiDataLen))) + { + goto Exit; + } + + pLStream = &bufferLStream; + } + else + { + pLStream = pLValue->val.pIStream; + } + + if( !(pRValue->uiFlags & SQL_VAL_IS_STREAM)) + { + if( RC_BAD( rc = bufferRStream.open( (const char *)pRValue->val.pucBuf, + pRValue->uiDataLen))) + { + goto Exit; + } + pRStream = &bufferRStream; + } + else + { + pRStream = pRValue->val.pIStream; + } + + if( RC_BAD( rc = f_compareUTF8Streams( + pLStream, + (bOpIsMatch && (pLValue->uiFlags & SQL_VAL_IS_CONSTANT)) ? TRUE : FALSE, + pRStream, + (bOpIsMatch && (pRValue->uiFlags & SQL_VAL_IS_CONSTANT)) ? TRUE : FALSE, + uiCompareRules, uiLanguage, piResult))) + { + goto Exit; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Performs binary comparison on two streams - may be text or binary, +// it really doesn't matter. Returns XFLM_TRUE or XFLM_FALSE. +//------------------------------------------------------------------------- +FSTATIC RCODE sqlCompareBinary( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + FLMINT * piResult) +{ + RCODE rc = NE_SFLM_OK; + F_BufferIStream bufferLStream; + IF_PosIStream * pLStream; + F_BufferIStream bufferRStream; + IF_PosIStream * pRStream; + FLMBYTE ucLByte; + FLMBYTE ucRByte; + FLMUINT uiOffset = 0; + FLMBOOL bLEmpty = FALSE; + + *piResult = 0; + + // Types must be binary + + if (pLValue->eValType != SQL_BINARY_VAL || + pRValue->eValType != SQL_BINARY_VAL) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + + // Open the streams + + if( !(pLValue->uiFlags & SQL_VAL_IS_STREAM)) + { + if (RC_BAD( rc = bufferLStream.open( (const char *)pLValue->val.pucBuf, + pLValue->uiDataLen))) + { + goto Exit; + } + + pLStream = &bufferLStream; + } + else + { + pLStream = pLValue->val.pIStream; + } + + if( !(pRValue->uiFlags & SQL_VAL_IS_STREAM)) + { + if( RC_BAD( rc = bufferRStream.open( (const char *)pRValue->val.pucBuf, + pRValue->uiDataLen))) + { + goto Exit; + } + pRStream = &bufferRStream; + } + else + { + pRStream = pRValue->val.pIStream; + } + + for (;;) + { + if (RC_BAD( rc = flmReadStorageAsBinary( + pLStream, &ucLByte, 1, uiOffset, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + bLEmpty = TRUE; + } + else + { + goto Exit; + } + } + + if (RC_BAD( rc = flmReadStorageAsBinary( + pRStream, &ucRByte, 1, uiOffset, NULL))) + { + if (rc == NE_SFLM_EOF_HIT) + { + rc = NE_SFLM_OK; + if( bLEmpty) + { + *piResult = 0; + } + else + { + *piResult = 1; + } + } + goto Exit; + } + else if( bLEmpty) + { + *piResult = -1; + goto Exit; + } + + if( ucLByte != ucRByte) + { + *piResult = ucLByte < ucRByte ? -1 : 1; + goto Exit; + } + + uiOffset++; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Compare two values. This routine assumes that pValue1 and pValue2 +// are non-null. +//------------------------------------------------------------------------- +RCODE sqlCompare( + SQL_VALUE * pValue1, + SQL_VALUE * pValue2, + FLMUINT uiCompareRules, + FLMUINT uiLanguage, + FLMINT * piCmp) +{ + RCODE rc = NE_SFLM_OK; + + // We have already called sqlCanCompare, so no need to do it here + + switch (pValue1->eValType) + { + case SQL_BOOL_VAL: + *piCmp = pValue1->val.eBool > pValue2->val.eBool + ? 1 + : pValue1->val.eBool < pValue2->val.eBool + ? -1 + : 0; + break; + case SQL_UINT_VAL: + switch (pValue2->eValType) + { + case SQL_UINT_VAL: + *piCmp = pValue1->val.uiVal > pValue2->val.uiVal + ? 1 + : pValue1->val.uiVal < pValue2->val.uiVal + ? -1 + : 0; + break; + case SQL_UINT64_VAL: + *piCmp = (FLMUINT64)pValue1->val.uiVal > pValue2->val.ui64Val + ? 1 + : (FLMUINT64)pValue1->val.uiVal < pValue2->val.ui64Val + ? -1 + : 0; + break; + case SQL_INT_VAL: + *piCmp = pValue2->val.iVal < 0 || + pValue1->val.uiVal > (FLMUINT)pValue2->val.iVal + ? 1 + : pValue1->val.uiVal < (FLMUINT)pValue2->val.iVal + ? -1 + : 0; + break; + case SQL_INT64_VAL: + *piCmp = pValue2->val.i64Val < 0 || + (FLMUINT64)pValue1->val.uiVal > + (FLMUINT64)pValue2->val.i64Val + ? 1 + : (FLMUINT64)pValue1->val.uiVal < + (FLMUINT64)pValue2->val.i64Val + ? -1 + : 0; + break; + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + break; + case SQL_UINT64_VAL: + switch (pValue2->eValType) + { + case SQL_UINT_VAL: + *piCmp = pValue1->val.ui64Val > (FLMUINT64)pValue2->val.uiVal + ? 1 + : pValue1->val.ui64Val < (FLMUINT64)pValue2->val.uiVal + ? -1 + : 0; + break; + case SQL_UINT64_VAL: + *piCmp = pValue1->val.ui64Val > pValue2->val.ui64Val + ? 1 + : pValue1->val.ui64Val < pValue2->val.ui64Val + ? -1 + : 0; + break; + case SQL_INT_VAL: + *piCmp = pValue2->val.iVal < 0 || + pValue1->val.ui64Val > (FLMUINT64)pValue2->val.iVal + ? 1 + : pValue1->val.ui64Val < (FLMUINT64)pValue2->val.iVal + ? -1 + : 0; + break; + case SQL_INT64_VAL: + *piCmp = pValue2->val.i64Val < 0 || + pValue1->val.ui64Val > (FLMUINT64)pValue2->val.i64Val + ? 1 + : pValue1->val.ui64Val < (FLMUINT64)pValue2->val.i64Val + ? -1 + : 0; + break; + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + break; + case SQL_INT_VAL: + switch (pValue2->eValType) + { + case SQL_UINT_VAL: + *piCmp = pValue1->val.iVal < 0 || + (FLMUINT)pValue1->val.iVal < pValue2->val.uiVal + ? -1 + : (FLMUINT)pValue1->val.iVal > pValue2->val.uiVal + ? 1 + : 0; + break; + case SQL_UINT64_VAL: + *piCmp = pValue1->val.iVal < 0 || + (FLMUINT64)pValue1->val.iVal < pValue2->val.ui64Val + ? -1 + : (FLMUINT64)pValue1->val.iVal > pValue2->val.ui64Val + ? 1 + : 0; + break; + case SQL_INT_VAL: + *piCmp = pValue1->val.iVal < pValue2->val.iVal + ? -1 + : pValue1->val.iVal > pValue2->val.iVal + ? 1 + : 0; + break; + case SQL_INT64_VAL: + *piCmp = (FLMINT64)pValue1->val.iVal < pValue2->val.i64Val + ? -1 + : (FLMINT64)pValue1->val.iVal > pValue2->val.i64Val + ? 1 + : 0; + break; + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + break; + case SQL_INT64_VAL: + switch (pValue2->eValType) + { + case SQL_UINT_VAL: + *piCmp = pValue1->val.i64Val < 0 || + (FLMUINT64)pValue1->val.i64Val < + (FLMUINT64)pValue2->val.uiVal + ? -1 + : (FLMUINT64)pValue1->val.i64Val > + (FLMUINT64)pValue2->val.uiVal + ? 1 + : 0; + break; + case SQL_UINT64_VAL: + *piCmp = pValue1->val.i64Val < 0 || + (FLMUINT64)pValue1->val.i64Val < pValue2->val.ui64Val + ? -1 + : (FLMUINT64)pValue1->val.i64Val > pValue2->val.ui64Val + ? 1 + : 0; + break; + case SQL_INT_VAL: + *piCmp = pValue1->val.i64Val < (FLMINT64)pValue2->val.iVal + ? -1 + : pValue1->val.i64Val > (FLMINT64)pValue2->val.iVal + ? 1 + : 0; + break; + case SQL_INT64_VAL: + *piCmp = pValue1->val.i64Val < pValue2->val.i64Val + ? -1 + : pValue1->val.i64Val > pValue2->val.i64Val + ? 1 + : 0; + break; + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_COMPARE_OPERAND_TYPE_MISMATCH); + goto Exit; + } + break; + case SQL_BINARY_VAL: + if (RC_BAD( rc = sqlCompareBinary( pValue1, pValue2, piCmp))) + { + goto Exit; + } + break; + case SQL_UTF8_VAL: + if (RC_BAD( rc = sqlCompareText( pValue1, pValue2, + uiCompareRules, FALSE, uiLanguage, piCmp))) + { + goto Exit; + } + break; + default: + break; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Returns a 64-bit unsigned integer +//------------------------------------------------------------------------- +FINLINE FLMUINT64 sqlGetUInt64( + SQL_VALUE * pValue) +{ + if (pValue->eValType == SQL_UINT_VAL) + { + return( (FLMUINT64)pValue->val.uiVal); + } + else if( pValue->eValType == SQL_UINT64_VAL) + { + return( pValue->val.ui64Val); + } + else if( pValue->eValType == SQL_INT64_VAL) + { + if( pValue->val.i64Val >= 0) + { + return( (FLMUINT64)pValue->val.i64Val); + } + } + else if( pValue->eValType == SQL_INT_VAL) + { + if( pValue->val.iVal >= 0) + { + return( (FLMUINT64)pValue->val.iVal); + } + } + + flmAssert( 0); + return( 0); +} + +//------------------------------------------------------------------------- +// Desc: Returns a 64-bit signed integer +//------------------------------------------------------------------------- +FINLINE FLMINT64 sqlGetInt64( + SQL_VALUE * pValue) +{ + if (pValue->eValType == SQL_INT_VAL) + { + return( (FLMINT64)pValue->val.iVal); + } + else if( pValue->eValType == SQL_INT64_VAL) + { + return( pValue->val.i64Val); + } + else if( pValue->eValType == SQL_UINT_VAL) + { + return( (FLMINT64)pValue->val.uiVal); + } + else if( pValue->eValType == SQL_UINT64_VAL) + { + if( pValue->val.ui64Val <= (FLMUINT64)FLM_MAX_INT64) + { + return( (FLMINT64)pValue->val.ui64Val); + } + } + + flmAssert( 0); + return( 0); +} + +//------------------------------------------------------------------------- +// Desc: Performs the bit and operation +//------------------------------------------------------------------------- +FSTATIC void sqlArithOpUUBitAND( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.uiVal = pLValue->val.uiVal & pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.ui64Val = + sqlGetUInt64( pLValue) & sqlGetUInt64( pRValue); + pResult->eValType = SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the bit or operation +***************************************************************************/ +FSTATIC void sqlArithOpUUBitOR( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.uiVal = pLValue->val.uiVal | pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.ui64Val = + sqlGetUInt64( pLValue) | sqlGetUInt64( pRValue); + pResult->eValType = SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the bit xor operation +***************************************************************************/ +FSTATIC void sqlArithOpUUBitXOR( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.uiVal = pLValue->val.uiVal ^ pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.ui64Val = + sqlGetUInt64( pLValue) ^ sqlGetUInt64( pRValue); + pResult->eValType = SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the multiply operation +***************************************************************************/ +FSTATIC void sqlArithOpUUMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.uiVal = pLValue->val.uiVal * pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.ui64Val = + sqlGetUInt64( pLValue) * sqlGetUInt64( pRValue); + pResult->eValType = SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the multiply operation +***************************************************************************/ +FSTATIC void sqlArithOpUSMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.iVal = (FLMINT)pLValue->val.uiVal * pRValue->val.iVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64) + sqlGetUInt64( pLValue) * sqlGetInt64( pRValue); + pResult->eValType = SQL_INT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the multiply operation +***************************************************************************/ +FSTATIC void sqlArithOpSSMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.iVal = pLValue->val.iVal * pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL + : SQL_UINT_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64)(sqlGetInt64( pLValue) * + sqlGetInt64( pRValue)); + + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL + : SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the multiply operation +***************************************************************************/ +FSTATIC void sqlArithOpSUMult( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.iVal = pLValue->val.iVal * + (FLMINT)pRValue->val.uiVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64) + (sqlGetInt64( pLValue) * sqlGetUInt64( pRValue)); + pResult->eValType = SQL_INT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs the divide operation +***************************************************************************/ +FSTATIC void sqlArithOpUUDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.uiVal) + { + pResult->val.uiVal = pLValue->val.uiVal / pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64RValue) + { + pResult->val.ui64Val = ui64LValue / ui64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the divide operation +***************************************************************************/ +FSTATIC void sqlArithOpUSDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.iVal) + { + pResult->val.iVal = pLValue->val.uiVal / pRValue->val.iVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( i64RValue) + { + pResult->val.i64Val = ui64LValue / i64RValue; + pResult->eValType = SQL_INT64_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the divide operation +***************************************************************************/ +FSTATIC void sqlArithOpSSDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.iVal) + { + pResult->val.iVal = pLValue->val.iVal / pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( i64RValue) + { + pResult->val.i64Val = i64LValue / i64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the divide operation +***************************************************************************/ +FSTATIC void sqlArithOpSUDiv( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.uiVal) + { + pResult->val.iVal = pLValue->val.iVal / pRValue->val.uiVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64RValue) + { + pResult->val.i64Val = i64LValue / ui64RValue; + pResult->eValType = SQL_INT64_VAL; + } + else + { + pResult->val.uiVal = 0; // Divide by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the modulo operation +***************************************************************************/ +FSTATIC void sqlArithOpUUMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.uiVal) + { + pResult->val.uiVal = pLValue->val.uiVal % pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64RValue) + { + pResult->val.ui64Val = ui64LValue % ui64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the modulo operation +***************************************************************************/ +FSTATIC void sqlArithOpUSMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.iVal) + { + pResult->val.iVal = pLValue->val.uiVal % pRValue->val.iVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( i64RValue) + { + pResult->val.i64Val = ui64LValue % i64RValue; + pResult->eValType = SQL_INT64_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the modulo operation +***************************************************************************/ +FSTATIC void sqlArithOpSSMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.iVal) + { + pResult->val.iVal = pLValue->val.iVal % pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( i64RValue) + { + pResult->val.i64Val = i64LValue % i64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs the modulo operation +***************************************************************************/ +FSTATIC void sqlArithOpSUMod( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.uiVal) + { + pResult->val.iVal = pLValue->val.iVal % pRValue->val.uiVal; + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64RValue) + { + pResult->val.i64Val = i64LValue % ui64RValue; + pResult->eValType = SQL_INT64_VAL; + } + else + { + pResult->val.uiVal = 0; // MOD by ZERO case. + pResult->eValType = SQL_MISSING_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs an addition operation +***************************************************************************/ +FSTATIC void sqlArithOpUUPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.uiVal = pLValue->val.uiVal + pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.ui64Val = + sqlGetUInt64( pLValue) + sqlGetUInt64( pRValue); + pResult->eValType = SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs an addition operation +***************************************************************************/ +FSTATIC void sqlArithOpUSPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( (pRValue->val.iVal >= 0) || + (pLValue->val.uiVal > gv_uiMaxSignedIntVal)) + { + pResult->val.uiVal = pLValue->val.uiVal + (FLMUINT)pRValue->val.iVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.iVal = (FLMINT)pLValue->val.uiVal + pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( (i64RValue >= 0) || (ui64LValue > gv_ui64MaxSignedIntVal)) + { pResult->val.ui64Val = ui64LValue + (FLMUINT64)i64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64)ui64LValue + i64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs an addition operation +***************************************************************************/ +FSTATIC void sqlArithOpSSPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + pResult->val.iVal = pLValue->val.iVal + pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + else + { + pResult->val.i64Val = + sqlGetInt64( pLValue) + sqlGetInt64( pRValue); + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } +} + +/*************************************************************************** +Desc: Performs an addition operation +***************************************************************************/ +FSTATIC void sqlArithOpSUPlus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( (pLValue->val.iVal >= 0) || + (pRValue->val.uiVal > gv_uiMaxSignedIntVal)) + { + pResult->val.uiVal = (FLMUINT)pLValue->val.iVal + pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.iVal = pLValue->val.iVal + (FLMINT)pRValue->val.uiVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( (i64LValue >= 0) || (ui64RValue > gv_ui64MaxSignedIntVal)) + { + pResult->val.ui64Val = (FLMUINT64)i64LValue + ui64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.i64Val = i64LValue + (FLMINT64)ui64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs a subtraction operation +***************************************************************************/ +FSTATIC void sqlArithOpUUMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pLValue->val.uiVal >= pRValue->val.uiVal) + { + pResult->val.uiVal = pLValue->val.uiVal - pRValue->val.uiVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.iVal = (FLMINT)(pLValue->val.uiVal - pRValue->val.uiVal); + pResult->eValType = SQL_INT_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64LValue >= ui64RValue) + { + pResult->val.ui64Val = ui64LValue - ui64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64)(ui64LValue - ui64RValue); + pResult->eValType = SQL_INT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs a subtraction operation +***************************************************************************/ +FSTATIC void sqlArithOpUSMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.iVal < 0) + { + pResult->val.uiVal = pLValue->val.uiVal - pRValue->val.iVal; + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.iVal = (FLMINT)pLValue->val.uiVal - pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + } + else + { + FLMUINT64 ui64LValue = sqlGetUInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( i64RValue < 0) + { + pResult->val.ui64Val = ui64LValue - i64RValue; + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.i64Val = (FLMINT64)ui64LValue - i64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs a subtraction operation +***************************************************************************/ +FSTATIC void sqlArithOpSSMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if(( pLValue->val.iVal > 0) && ( pRValue->val.iVal < 0)) + { + pResult->val.uiVal = (FLMUINT)(pLValue->val.iVal - pRValue->val.iVal); + pResult->eValType = SQL_UINT_VAL; + } + else + { + pResult->val.iVal = pLValue->val.iVal - pRValue->val.iVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMINT64 i64RValue = sqlGetInt64( pRValue); + + if( (i64LValue > 0) && (i64RValue < 0)) + { + pResult->val.ui64Val = (FLMUINT64)( i64LValue - i64RValue); + pResult->eValType = SQL_UINT64_VAL; + } + else + { + pResult->val.i64Val = i64LValue - i64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Performs a subtraction operation +***************************************************************************/ +FSTATIC void sqlArithOpSUMinus( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + SQL_VALUE * pResult) +{ + if (isSQLValNativeNum( pLValue->eValType) && + isSQLValNativeNum( pRValue->eValType)) + { + if( pRValue->val.uiVal > gv_uiMaxSignedIntVal) + { + pResult->val.iVal = (pLValue->val.iVal - gv_uiMaxSignedIntVal) - + (FLMINT)(pRValue->val.uiVal - gv_uiMaxSignedIntVal); + pResult->eValType = SQL_INT_VAL; + } + else + { + pResult->val.iVal = pLValue->val.iVal - (FLMINT)pRValue->val.uiVal; + pResult->eValType = (pResult->val.iVal < 0) + ? SQL_INT_VAL : SQL_UINT_VAL; + } + } + else + { + FLMINT64 i64LValue = sqlGetInt64( pLValue); + FLMUINT64 ui64RValue = sqlGetUInt64( pRValue); + + if( ui64RValue > gv_ui64MaxSignedIntVal) + { + pResult->val.i64Val = (i64LValue - gv_ui64MaxSignedIntVal) - + (FLMINT64)(ui64RValue - gv_ui64MaxSignedIntVal); + pResult->eValType = SQL_INT64_VAL; + } + else + { + pResult->val.i64Val = i64LValue - (FLMINT64)ui64RValue; + pResult->eValType = (pResult->val.i64Val < 0) + ? SQL_INT64_VAL : SQL_UINT64_VAL; + } + } +} + +/*************************************************************************** +Desc: Do an arithmetic operator. +***************************************************************************/ +RCODE sqlEvalArithOperator( + SQL_VALUE * pLValue, + SQL_VALUE * pRValue, + eSQLQueryOperators eOperator, + SQL_VALUE * pResult) +{ + RCODE rc = NE_SFLM_OK; + SQL_ARITH_OP * fnOp; + FLMUINT uiOffset = 0; + + if (!isSQLArithOp( eOperator)) + { + rc = RC_SET( NE_SFLM_Q_INVALID_OPERATOR); + goto Exit; + } + + if (pLValue->eValType == SQL_MISSING_VAL || + pRValue->eValType == SQL_MISSING_VAL) + { + pResult->eValType = SQL_MISSING_VAL; + goto Exit; + } + + if (isSQLValUnsigned( pLValue->eValType)) + { + if (isSQLValUnsigned( pRValue->eValType)) + { + uiOffset = 0; + } + else if (isSQLValSigned( pRValue->eValType)) + { + uiOffset = 1; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + } + else if (isSQLValSigned( pLValue->eValType)) + { + if (isSQLValUnsigned( pRValue->eValType)) + { + uiOffset = 2; + } + else if (isSQLValSigned( pRValue->eValType)) + { + uiOffset = 3; + } + else + { + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + } + + fnOp = SQL_ArithOpTable[ ((((FLMUINT)eOperator) - + SQL_FIRST_ARITH_OP) * 4) + uiOffset]; + fnOp( pLValue, pRValue, pResult); + +Exit: + + return( rc); +} + diff --git a/sql/src/sqloptimize.cpp b/sql/src/sqloptimize.cpp new file mode 100644 index 0000000..6c07a1d --- /dev/null +++ b/sql/src/sqloptimize.cpp @@ -0,0 +1,2762 @@ +//------------------------------------------------------------------------- +// Desc: Optimize an SQL query +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------- + +#include "flaimsys.h" + +// Local function prototypes + +FSTATIC RCODE sqlGetRowIdValue( + SQL_VALUE * pValue); + +FSTATIC RCODE setupPredicate( + SQL_PRED * pPred, + SQL_TABLE * pTable, + FLMUINT uiColumnNum, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue); + +FSTATIC RCODE sqlCompareValues( + SQL_VALUE * pValue1, + FLMBOOL bInclusive1, + FLMBOOL bNullIsLow1, + SQL_VALUE * pValue2, + FLMBOOL bInclusive2, + FLMBOOL bNullIsLow2, + FLMUINT uiCompareRules, + FLMUINT uiLanguage, + FLMINT * piCmp); + +FSTATIC SQL_NODE * sqlEvalLogicalOperands( + SQL_NODE * pSQLNode); + +FSTATIC SQL_NODE * sqlClipNotNode( + SQL_NODE * pNotNode, + SQL_NODE ** ppExpr); + +FSTATIC RCODE createDNFNode( + F_Pool * pPool, + SQL_DNF_NODE * pParentDNFNode, + SQL_DNF_NODE ** ppDNFNode, + SQL_NODE * pNode); + +FSTATIC RCODE copyAndLinkSubTree( + F_Pool * pPool, + SQL_DNF_NODE * pSrcSubTree, + SQL_DNF_NODE * pParentNode); + +FSTATIC RCODE distributeAndOverOr( + F_Pool * pPool, + SQL_DNF_NODE * pOldOrNode, + SQL_DNF_NODE ** ppDNFTree); + +#if 0 +FSTATIC FLMBOOL predIsForTable( + SQL_NODE * pPredRootNode, + SQL_TABLE * pTable); + +FSTATIC void rankIndexes( + SQL_TABLE * pTable); +#endif + +//------------------------------------------------------------------------- +// Desc: Get the row ID constant from an SQL_VALUE node. +//------------------------------------------------------------------------- +FSTATIC RCODE sqlGetRowIdValue( + SQL_VALUE * pValue) +{ + RCODE rc = NE_SFLM_OK; + + switch (pValue->eValType) + { + case SQL_UINT_VAL: + pValue->eValType = SQL_UINT64_VAL; + pValue->val.ui64Val = (FLMUINT64)pValue->val.uiVal; + break; + case SQL_MISSING_VAL: + case SQL_UINT64_VAL: + break; + case SQL_INT_VAL: + pValue->eValType = SQL_UINT64_VAL; + pValue->val.ui64Val = (FLMUINT64)((FLMINT64)(pValue->val.iVal)); + break; + case SQL_INT64_VAL: + pValue->eValType = SQL_UINT64_VAL; + pValue->val.ui64Val = (FLMUINT64)(pValue->val.i64Val); + break; + default: + rc = RC_SET_AND_ASSERT( NE_Q_INVALID_ROW_ID_VALUE); + goto Exit; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Setup a predicate using the passed in parameters. +//------------------------------------------------------------------------- +FSTATIC RCODE setupPredicate( + SQL_PRED * pPred, + SQL_TABLE * pTable, + FLMUINT uiColumnNum, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue) +{ + RCODE rc = NE_SFLM_OK; + + pPred->pTable = pTable; + pPred->uiColumnNum = uiColumnNum; + if (!pValue || pValue->eValType != SQL_UTF8_VAL) + { + + // Comparison rules don't matter for anything that is + // not text, so we normalize them to zero, so the test + // below to see if the comparison rule is the same as + // the comparison rule of the operator will work. + + pPred->uiCompareRules = 0; + } + else + { + pPred->uiCompareRules = uiCompareRules; + } + pPred->bNotted = bNotted; + switch (eOperator) + { + case SQL_EXISTS_OP: + case SQL_NE_OP: + pPred->eOperator = eOperator; + pPred->pFromValue = pValue; + break; + case SQL_APPROX_EQ_OP: + pPred->eOperator = eOperator; + pPred->pFromValue = pValue; + pPred->bInclFrom = TRUE; + pPred->bInclUntil = TRUE; + break; + case SQL_EQ_OP: + if ((pValue->uiFlags & SQL_VAL_IS_CONSTANT) && + (pValue->uiFlags & SQL_VAL_HAS_WILDCARDS)) + { + pPred->eOperator = SQL_MATCH_OP; + pPred->pFromValue = pValue; + } + else + { + pPred->eOperator = SQL_RANGE_OP; + pPred->pFromValue = pValue; + pPred->pUntilValue = pValue; + pPred->bInclFrom = TRUE; + pPred->bInclUntil = TRUE; + } + break; + case SQL_LE_OP: + pPred->eOperator = SQL_RANGE_OP; + pPred->pFromValue = NULL; + pPred->pUntilValue = pValue; + pPred->bInclUntil = TRUE; + break; + case SQL_LT_OP: + pPred->eOperator = SQL_RANGE_OP; + pPred->pFromValue = NULL; + pPred->pUntilValue = pValue; + pPred->bInclUntil = FALSE; + break; + case SQL_GE_OP: + pPred->eOperator = SQL_RANGE_OP; + pPred->pFromValue = pValue; + pPred->pUntilValue = NULL; + pPred->bInclFrom = TRUE; + break; + case SQL_GT_OP: + pPred->eOperator = SQL_RANGE_OP; + pPred->pFromValue = pValue; + pPred->pUntilValue = NULL; + pPred->bInclFrom = FALSE; + break; + default: + rc = RC_SET_AND_ASSERT( NE_SFLM_NOT_IMPLEMENTED); + goto Exit; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Compare two values +//------------------------------------------------------------------------- +FSTATIC RCODE sqlCompareValues( + SQL_VALUE * pValue1, + FLMBOOL bInclusive1, + FLMBOOL bNullIsLow1, + SQL_VALUE * pValue2, + FLMBOOL bInclusive2, + FLMBOOL bNullIsLow2, + FLMUINT uiCompareRules, + FLMUINT uiLanguage, + FLMINT * piCmp) +{ + RCODE rc = NE_SFLM_OK; + + // We have already called sqlCanCompare, so no need to do it here + + if (!pValue1) + { + if (!pValue2) + { + if (bNullIsLow2) + { + *piCmp = (FLMINT)(bNullIsLow1 ? 0 : 1); + } + else + { + *piCmp = (FLMINT)(bNullIsLow1 ? -1 : 0); + } + } + else + { + *piCmp = (FLMINT)(bNullIsLow1 ? -1 : 1); + } + goto Exit; + } + else if (!pValue2) + { + *piCmp = (FLMINT)(bNullIsLow2 ? 1 : -1); + goto Exit; + } + + if (RC_BAD( rc = sqlCompare( pValue1, pValue2, + uiCompareRules, uiLanguage, piCmp))) + { + goto Exit; + } + + // If everything else is equal, the last distinguisher + // is the inclusive flags and which side of the + // value we are on if we are exclusive which is indicated + // by the bNullIsLow flags + + if (*piCmp == 0) + { + if (bInclusive1 != bInclusive2) + { + if (bNullIsLow1) + { + if (bNullIsLow2) + { + // *--> v1 + // o--> v2 v1 < v2 + + // o--> v1 + // *--> v2 v1 > v2 + + *piCmp = bInclusive1 ? -1 : 1; + } + else + { + // *--> v1 + // v2 <--o v1 > v2 + + // o--> v1 + // v2 <--* v1 > v2 + + *piCmp = 1; + } + } + else + { + if (bNullIsLow2) + { + // v1 <--* + // o--> v2 v1 < v2 + + // v1 <--o + // *--> v2 v1 < v2 + + *piCmp = -1; + } + else + { + // v1 <--* + // v2 <--o v1 > v2 + + // v1 <--o + // v2 <--* v1 < v2 + + *piCmp = bInclusive1 ? 1 : -1; + } + } + } + else if (!bInclusive1) + { + + // bInclusive2 is also FALSE + + if (bNullIsLow1) + { + if (!bNullIsLow2) + { + // o--> v1 + // v2 <--o v1 > v2 + *piCmp = 1; + } +// else +// { + // o--> v1 + // o--> v2 v1 == v2 + // *piCmp = 0; +// } + } + else + { + if (bNullIsLow2) + { + + // v1 <--o + // o--> v2 v1 < v2 + + *piCmp = -1; + } +// else +// { + // v1 <--o + // v2 <--o v1 == v2 + // *piCmp = 0; +// } + } + } +// else +// { + // bInclusive1 == TRUE && bInclusive2 == TRUE + // else case covers the cases where + // both are inclusive, in which case it + // doesn't matter which is low and which + // is high + + // v1 <--* + // *--> v2 v1 == v2 + + // v1 <--* + // v2 <--* v1 == v2 + + // *--> v1 + // v2 <--* v1 == v2 + + // *--> v1 + // *--> v2 v1 == v2 + +// } + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Intersect a predicate into an existing predicate. +//------------------------------------------------------------------------- +RCODE SQLQuery::intersectPredicates( + SQL_PRED * pPred, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue, + FLMBOOL * pbAlwaysFalse, + FLMBOOL * pbIntersected) +{ + RCODE rc = NE_SFLM_OK; + FLMINT iCmp; + FLMBOOL bDoMatch; + + *pbIntersected = FALSE; + if (!pValue || pValue->eValType != SQL_UTF8_VAL) + { + bDoMatch = FALSE; + + // Comparison rules don't matter for anything that is + // not text, so we normalize them to zero, so the test + // below to see if the comparison rule is the same as + // the comparison rule of the operator will work. + + uiCompareRules = 0; + } + else + { + bDoMatch = (eOperator == SQL_EQ_OP && + (pValue->uiFlags & SQL_VAL_IS_CONSTANT) && + (pValue->uiFlags & SQL_VAL_HAS_WILDCARDS)) + ? TRUE + : FALSE; + } + + if (eOperator == SQL_EXISTS_OP) + { + *pbIntersected = TRUE; + + // An exists operator will either merge with an existing predicate or + // cancel the whole thing out as an empty result. + + // If this predicate is not-exists, another predicate ANDed with this + // one can never return a result that will match, unless that predicate + // is also a not-exists, in which case, we simply combine this one + // with that one. + + if (bNotted) + { + if (pPred->eOperator != SQL_EXISTS_OP || !pPred->bNotted) + { + *pbAlwaysFalse = TRUE; + } + } + } + else if (pPred->eOperator == SQL_EXISTS_OP) + { + + *pbIntersected = TRUE; + + // If the first predicate is an exists operator + // it will be the only one, because otherwise + // it will have been merged with another operator + // in the code just above. + + flmAssert( !pPred->pNext); + + // If the predicate is notted, another predicate + // ANDed with this one can never return a result. + + if (pPred->bNotted) + { + *pbAlwaysFalse = TRUE; + } + else + { + + // Change the predicate to the current + // operator. + + if (RC_BAD( rc = setupPredicate( pPred, pPred->pTable, + pPred->uiColumnNum, eOperator, uiCompareRules, + bNotted, pValue))) + { + goto Exit; + } + } + } + + // See if the operator intersects a range operator + + else if (pPred->eOperator == SQL_RANGE_OP && + pPred->uiCompareRules == uiCompareRules && + !bDoMatch && + (eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP || + eOperator == SQL_LT_OP || + eOperator == SQL_GE_OP || + eOperator == SQL_GT_OP)) + { + SQL_VALUE * pFromValue; + SQL_VALUE * pUntilValue; + FLMBOOL bInclFrom; + FLMBOOL bInclUntil; + + *pbIntersected = TRUE; + + pFromValue = (eOperator == SQL_EQ_OP || + eOperator == SQL_GE_OP || + eOperator == SQL_GT_OP) + ? pValue + : NULL; + pUntilValue = (eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP || + eOperator == SQL_LT_OP) + ? pValue + : NULL; + bInclFrom = (FLMBOOL)(eOperator == SQL_EQ_OP || + eOperator == SQL_GE_OP + ? TRUE + : FALSE); + bInclUntil = (FLMBOOL)(eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP + ? TRUE + : FALSE); + + // If the value type is not compatible with the predicate's + // value type, we cannot do the comparison, and there is + // no intersection. + + if (!sqlCanCompare( pValue, pPred->pFromValue) || + !sqlCanCompare( pValue, pPred->pUntilValue)) + { + *pbAlwaysFalse = TRUE; + } + else if (RC_BAD( rc = sqlCompareValues( pFromValue, + bInclFrom, TRUE, + pPred->pFromValue, pPred->bInclFrom, TRUE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + else if (iCmp > 0) + { + + // From value is greater than predicate's from value. + // If the from value is also greater than the predicate's + // until value, we have no intersection. + + if (RC_BAD( rc = sqlCompareValues( pFromValue, + bInclFrom, TRUE, + pPred->pUntilValue, pPred->bInclUntil, FALSE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + if (iCmp > 0) + { + *pbAlwaysFalse = TRUE; + } + else + { + pPred->pFromValue = pFromValue; + pPred->bInclFrom = bInclFrom; + } + } + else if (RC_BAD( rc = sqlCompareValues( pUntilValue, + bInclUntil, FALSE, + pPred->pUntilValue, pPred->bInclUntil, FALSE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + else if (iCmp < 0) + { + + // Until value is less than predicate's until value. If the + // until value is also less than predicate's from value, we + // have no intersection. + + if (RC_BAD( rc = sqlCompareValues( pUntilValue, + bInclUntil, FALSE, + pPred->pFromValue, pPred->bInclFrom, TRUE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + if (iCmp < 0) + { + *pbAlwaysFalse = TRUE; + } + else + { + pPred->pUntilValue = pUntilValue; + pPred->bInclUntil = bInclUntil; + } + } + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: See if a predicate can be unioned with another one. +//------------------------------------------------------------------------- +RCODE SQLQuery::unionPredicates( + SQL_PRED * pPred, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue, + FLMBOOL * pbUnioned) +{ + RCODE rc = NE_SFLM_OK; + FLMINT iCmp; + FLMBOOL bDoMatch; + + *pbUnioned = FALSE; + if (!pValue || pValue->eValType != SQL_UTF8_VAL) + { + bDoMatch = FALSE; + + // Comparison rules don't matter for anything that is + // not text, so we normalize them to zero, so the test + // below to see if the comparison rule is the same as + // the comparison rule of the operator will work. + + uiCompareRules = 0; + } + else + { + bDoMatch = (eOperator == SQL_EQ_OP && + (pValue->uiFlags & SQL_VAL_IS_CONSTANT) && + (pValue->uiFlags & SQL_VAL_HAS_WILDCARDS)) + ? TRUE + : FALSE; + } + + if (eOperator == SQL_EXISTS_OP || eOperator == SQL_NE_OP) + { + + // See if there is another operator that is an exact + // match of this one. + + if (pPred->eOperator == eOperator && + pPred->bNotted == bNotted) + { + + // Perfect match - no need to do any more. + + *pbUnioned = TRUE; + } + } + + // See if the operator overlaps with another range operator + + else if (pPred->eOperator == SQL_RANGE_OP && + pPred->uiCompareRules == uiCompareRules && + !bDoMatch && + (eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP || + eOperator == SQL_LT_OP || + eOperator == SQL_GE_OP || + eOperator == SQL_GT_OP)) + { + SQL_VALUE * pFromValue; + SQL_VALUE * pUntilValue; + FLMBOOL bInclFrom; + FLMBOOL bInclUntil; + + pFromValue = (eOperator == SQL_EQ_OP || + eOperator == SQL_GE_OP || + eOperator == SQL_GT_OP) + ? pValue + : NULL; + pUntilValue = (eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP || + eOperator == SQL_LT_OP) + ? pValue + : NULL; + bInclFrom = (FLMBOOL)(eOperator == SQL_EQ_OP || + eOperator == SQL_GE_OP + ? TRUE + : FALSE); + bInclUntil = (FLMBOOL)(eOperator == SQL_EQ_OP || + eOperator == SQL_LE_OP + ? TRUE + : FALSE); + + // If the value type is not compatible with the predicate's + // value type, we cannot do the comparison, and there is + // no overlap. + + if (!sqlCanCompare( pValue, pPred->pFromValue) || + !sqlCanCompare( pValue, pPred->pUntilValue)) + { + // Nothing to do here + } + else if (RC_BAD( rc = sqlCompareValues( pFromValue, + bInclFrom, TRUE, + pPred->pFromValue, pPred->bInclFrom, TRUE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + else if (iCmp >= 0) + { + + // From value is greater than or equal to the predicate's + // from value. + // If the from value is also less than or equal to the + // predicate's until value, we have an overlap. + + if (RC_BAD( rc = sqlCompareValues( pFromValue, + bInclFrom, TRUE, + pPred->pUntilValue, pPred->bInclUntil, FALSE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + if (iCmp <= 0) + { + + // If the until value is greater than the predicate's + // until value, change the predicate's until value. + + if (RC_BAD( rc = sqlCompareValues( pUntilValue, + bInclUntil, FALSE, + pPred->pUntilValue, pPred->bInclUntil, FALSE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + if (iCmp > 0) + { + pPred->pUntilValue = pUntilValue; + pPred->bInclUntil = bInclUntil; + } + *pbUnioned = TRUE; + goto Exit; + } + } + + // At this point we already know that the from value is + // less than the predicate's from value. + // See if the until value is greater than or equal + // to the from value. If it is we have an overlap. + + else if (RC_BAD( rc = sqlCompareValues( pUntilValue, + bInclUntil, FALSE, + pPred->pFromValue, pPred->bInclFrom, TRUE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + else if (iCmp >= 0) + { + + // Until value is greater than or equal to the predicate's + // from value, so we definitely have an overlap. We + // already know that the from value is less than the + // predicate's from value, so we will change that for sure. + + pPred->pFromValue = pFromValue; + pPred->bInclFrom = bInclFrom; + + // See if the until value is greater than the + // predicate's until value, in which case we need to + // change the predicate's until value. + + if (RC_BAD( rc = sqlCompareValues( pUntilValue, + bInclUntil, FALSE, + pPred->pUntilValue, pPred->bInclUntil, FALSE, + uiCompareRules, m_uiLanguage, &iCmp))) + { + goto Exit; + } + if (iCmp > 0) + { + pPred->pUntilValue = pUntilValue; + pPred->bInclUntil = bInclUntil; + } + *pbUnioned = TRUE; + } + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Convert an operand to a predicate. If it is merged with another +// predicate, remove it and return the next node in the list of +// operands. If it is not merged, still return the next node in +// the list of operands. +//------------------------------------------------------------------------- +RCODE SQLQuery::addPredicate( + SQL_SUBQUERY * pSubQuery, + FLMUINT * puiOperand, + SQL_TABLE * pTable, + FLMUINT uiColumnNum, + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules, + FLMBOOL bNotted, + SQL_VALUE * pValue) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiOperand = *puiOperand; + FLMUINT uiLoop; + SQL_NODE * pCurrNode; + SQL_NODE * pOperandNode = pSubQuery->ppOperands [uiOperand]; + FLMBOOL bAlwaysFalse; + SQL_PRED * pPred; + + // Convert the constant value in a node id predicate to + // a 64 bit unsigned value. + + if (eOperator != SQL_EXISTS_OP && !uiColumnNum) + { + if (RC_BAD( rc = sqlGetRowIdValue( pValue))) + { + goto Exit; + } + } + + // Look at all of the operands up to the one we are processing to + // see if this operand should be merged with a previous one. + + for (uiLoop = 0; uiLoop < uiOperand; uiLoop++) + { + pCurrNode = pSubQuery->ppOperands [uiLoop]; + if (pCurrNode->eNodeType != SQL_PRED_NODE) + { + pCurrNode = pCurrNode->pNextSib; + continue; + } + pPred = &pCurrNode->nd.pred; + if (pPred->pTable == pTable && pPred->uiColumnNum == uiColumnNum) + { + FLMBOOL bIntersected; + + if (RC_BAD( rc = intersectPredicates( pPred, eOperator, + uiCompareRules, bNotted, pValue, + &bAlwaysFalse, &bIntersected))) + { + goto Exit; + } + if (!bIntersected && !bAlwaysFalse) + { + continue; + } + + // If we get a false result, then we know that the + // intersection of predicates is creating a situation where + // it can never be true, so this sub-query can never return + // anything. Therefore, we remove the sub-query. + + if (bAlwaysFalse) + { + + // Remove the sub-query - it will never return anything. + + if (pSubQuery->pPrev) + { + pSubQuery->pPrev->pNext = pSubQuery->pNext; + } + else + { + m_pFirstSubQuery = pSubQuery->pNext; + } + if (pSubQuery->pNext) + { + pSubQuery->pNext->pPrev = pSubQuery->pPrev; + } + else + { + m_pLastSubQuery = pSubQuery->pPrev; + } + + // Setup so that we will quit processing this sub-query's + // operands - it is now unlinked. + + uiOperand = pSubQuery->uiOperandCount; + } + else + { + + flmAssert( bIntersected); + + // We intersected, so we want to remove the current + // operand node out of the list and set up so that + // we will increment to the next one in the list. + + pSubQuery->uiOperandCount--; + if (uiOperand < pSubQuery->uiOperandCount) + { + f_memmove( &pSubQuery->ppOperands [uiOperand], + &pSubQuery->ppOperands [uiOperand + 1], + sizeof( SQL_NODE *) * (pSubQuery->uiOperandCount - uiOperand)); + } + } + goto Exit; + } + } + + // If we didn't find one to intersect with or union with, we need to + // create a new operand node of type SQL_PRED_NODE. Can't just modify + // this node, because other sub-queries may be pointing to it also, and + // they would modify it in a different way. Unlike other nodes, predicate + // nodes are ALWAYS tied to one and only one sub-query. + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE), + (void **)&pOperandNode))) + { + goto Exit; + } + + // Set the stuff that needs to be set for this predicate. + + pOperandNode->eNodeType = SQL_PRED_NODE; + if (RC_BAD( rc = setupPredicate( &pOperandNode->nd.pred, + pTable, uiColumnNum, + eOperator, uiCompareRules, bNotted, pValue))) + { + goto Exit; + } + + // Go to the next operand + + uiOperand++; + +Exit: + + *puiOperand = uiOperand; + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Convert all of the operands underneath an AND or OR operator to +// predicates where possible, except for the operands which are AND +// or OR nodes. +//------------------------------------------------------------------------- +RCODE SQLQuery::convertOperandsToPredicates( void) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pSQLNode; + SQL_VALUE * pValue; + SQL_TABLE * pTable; + FLMUINT uiColumnNum; + eSQLQueryOperators eOperator; + FLMUINT uiOperand; + SQL_SUBQUERY * pSubQuery; + SQL_SUBQUERY * pNextSubQuery; + + pSubQuery = m_pFirstSubQuery; + while (pSubQuery) + { + pNextSubQuery = pSubQuery->pNext; + uiOperand = 0; + while (uiOperand < pSubQuery->uiOperandCount) + { + pSQLNode = pSubQuery->ppOperands [uiOperand]; + if (pSQLNode->eNodeType == SQL_COLUMN_NODE) + { + if (RC_BAD( rc = addPredicate( pSubQuery, &uiOperand, + pSQLNode->nd.column.pTable, + pSQLNode->nd.column.uiColumnNum, + SQL_EXISTS_OP, 0, pSQLNode->bNotted, NULL))) + { + goto Exit; + } + } + else if (pSQLNode->eNodeType == SQL_OPERATOR_NODE && + isSQLCompareOp( pSQLNode->nd.op.eOperator) && + ((pSQLNode->pFirstChild->eNodeType == SQL_COLUMN_NODE && + pSQLNode->pLastChild->eNodeType == SQL_VALUE_NODE) || + (pSQLNode->pFirstChild->eNodeType == SQL_VALUE_NODE && + pSQLNode->pLastChild->eNodeType == SQL_COLUMN_NODE))) + { + + // Have a Column,Op,Value or Value,Op,Column. Convert to a + // predicate node and merge with other predicate nodes that + // have already been created, if possible. + + if (pSQLNode->pFirstChild->eNodeType == SQL_COLUMN_NODE) + { + pTable = pSQLNode->pFirstChild->nd.column.pTable; + uiColumnNum = pSQLNode->pFirstChild->nd.column.uiColumnNum; + pValue = &pSQLNode->pLastChild->nd.value; + eOperator = pSQLNode->nd.op.eOperator; + } + else + { + pTable = pSQLNode->pFirstChild->nd.column.pTable; + uiColumnNum = pSQLNode->pFirstChild->nd.column.uiColumnNum; + pValue = &pSQLNode->pFirstChild->nd.value; + + // Need to invert the operator in this case. + + switch (pSQLNode->nd.op.eOperator) + { + case SQL_EQ_OP: + case SQL_NE_OP: + // No change. + break; + case SQL_LT_OP: + eOperator = SQL_GE_OP; + break; + case SQL_LE_OP: + eOperator = SQL_GT_OP; + break; + case SQL_GT_OP: + eOperator = SQL_LE_OP; + break; + case SQL_GE_OP: + eOperator = SQL_LT_OP; + break; + default: + // Should never get here! + flmAssert( 0); + break; + } + } + + if (RC_BAD( rc = addPredicate( pSubQuery, &uiOperand, + pTable, uiColumnNum, + eOperator, pSQLNode->nd.op.uiCompareRules, + pSQLNode->bNotted, pValue))) + { + goto Exit; + } + } + else + { + + // Can't do anything with this operand, leave it and go to the + // next one. + + uiOperand++; + } + } + pSubQuery = pNextSubQuery; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Evaluate operands of an AND or OR operator to see if we can +// replace one. +// TRUE && P1 will be replaced with P1 +// FALSE && P1 will be replaced with FALSE +// UNKNOWN && P1 will be replaced with UNKNOWN +// TRUE || P1 will be replaced with TRUE +// UNKNOWN || P1 will be replaced with UNKNOWN +// FALSE || P1 will be replaced with P1 +//------------------------------------------------------------------------- +FSTATIC SQL_NODE * sqlEvalLogicalOperands( + SQL_NODE * pSQLNode) +{ + eSQLQueryOperators eOperator = pSQLNode->nd.op.eOperator; + SQL_NODE * pChildNode; + SQLBoolType eChildBoolVal; + SQLBoolType eClipValue = (eOperator == SQL_AND_OP) + ? SQL_TRUE + : SQL_FALSE; + SQL_NODE * pReplacementNode = NULL; + + pChildNode = pSQLNode->pFirstChild; + while (pChildNode) + { + if (isSQLNodeBool( pChildNode)) + { + eChildBoolVal = pChildNode->nd.value.val.eBool; + } + else + { + pChildNode = pChildNode->pNextSib; + continue; + } + + // For AND operators eClipValue will be SQL_TRUE. For OR + // operators, it will be SQL_FALSE. Those nodes should all be + // clipped out. If, after clipping the value, there is only + // one node left, whatever it is should be moved up to replace + // the AND or the OR node. + + if (eChildBoolVal == eClipValue) + { + if (pChildNode->pPrevSib) + { + pChildNode->pPrevSib->pNextSib = pChildNode->pNextSib; + } + else + { + pSQLNode->pFirstChild = pChildNode->pNextSib; + } + if (pChildNode->pNextSib) + { + pChildNode->pNextSib->pPrevSib = pChildNode->pPrevSib; + } + else + { + pSQLNode->pLastChild = pChildNode->pPrevSib; + } + if (pSQLNode->pFirstChild != pSQLNode->pLastChild) + { + pChildNode = pChildNode->pNextSib; + continue; + } + else + { + pReplacementNode = pSQLNode->pFirstChild; + break; + } + } + else + { + + // The child node is a a boolean value that should simply replace + // the AND or OR operator node. This handles the following cases: + // 1. Value is SQL_UNKNOWN and operator is SQL_OR or SQL_AND + // 2. Value is SQL_FALSE and operator is SQL_AND + // 3. Value is SQL_TRUE and operator is SQL_OR. + + pReplacementNode = pChildNode; + break; + } + } + + // If we got a replacement node, link it in where the AND or OR + // node was. + + if (pReplacementNode) + { + SQL_NODE * pParentNode; + + if ((pParentNode = pSQLNode->pParent) == NULL) + { + pReplacementNode->pParent = NULL; + pReplacementNode->pPrevSib = NULL; + pReplacementNode->pNextSib = NULL; + } + else + { + pReplacementNode->pParent = pParentNode; + if ((pReplacementNode->pPrevSib = pSQLNode->pPrevSib) != NULL) + { + pReplacementNode->pPrevSib->pNextSib = pReplacementNode; + } + else + { + pParentNode->pFirstChild = pReplacementNode; + } + + if ((pReplacementNode->pNextSib = pSQLNode->pNextSib) != NULL) + { + pReplacementNode->pNextSib->pPrevSib = pReplacementNode; + } + else + { + pParentNode->pLastChild = pReplacementNode; + } + } + pSQLNode = pReplacementNode; + } + + return( pSQLNode); +} + +//------------------------------------------------------------------------- +// Desc: Clip a NOT node out of the tree. +//------------------------------------------------------------------------- +FSTATIC SQL_NODE * sqlClipNotNode( + SQL_NODE * pNotNode, + SQL_NODE ** ppExpr) +{ + SQL_NODE * pKeepNode; + + // If this NOT node has no parent, the root + // of the tree needs to be set to its child. + + pKeepNode = pNotNode->pFirstChild; + + // Child better not have any siblings - NOT nodes only have + // one operand. + + flmAssert( !pKeepNode->pNextSib && !pKeepNode->pPrevSib); + + // Set child to point to the NOT node's parent. + + if ((pKeepNode->pParent = pNotNode->pParent) == NULL) + { + *ppExpr = pKeepNode; + } + else + { + + // Link child in where the NOT node used to be. + + if ((pKeepNode->pPrevSib = pNotNode->pPrevSib) != NULL) + { + pKeepNode->pPrevSib->pNextSib = pKeepNode; + } + else + { + pKeepNode->pParent->pFirstChild = pKeepNode; + } + if ((pKeepNode->pNextSib = pNotNode->pNextSib) != NULL) + { + pKeepNode->pNextSib->pPrevSib = pKeepNode; + } + else + { + pKeepNode->pParent->pLastChild = pKeepNode; + } + } + return( pKeepNode); +} + +//------------------------------------------------------------------------- +// Desc: Flatten the query tree. This coalesces AND and OR nodes so that +// they can have multiple operands. This will also strip out NOT nodes +// and resolve constant expressions to a single node. +//------------------------------------------------------------------------- +RCODE SQLQuery::flattenTree( void) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pSQLNode = m_pQuery; + SQL_NODE * pTmpNode; + SQL_NODE * pParentNode = NULL; + eSQLNodeTypes eNodeType; + eSQLQueryOperators eOperator; + FLMBOOL bNotted = FALSE; + + for (;;) + { + eNodeType = pSQLNode->eNodeType; + + // Need to save bNotted on each node so that when we traverse + // back up the tree it can be reset properly. If bNotted is + // TRUE and pSQLNode is an operator, we may change the operator in + // some cases. Even if we change the operator, we still want to + // set the bNotted flag because it also implies "for every" when set + // to TRUE, and we need to remember that as well. + + pSQLNode->bNotted = bNotted; + if (eNodeType == SQL_OPERATOR_NODE) + { + eOperator = pSQLNode->nd.op.eOperator; + if (eOperator == SQL_AND_OP || eOperator == SQL_OR_OP) + { + // AND and OR nodes better have child nodes + + if (!pSQLNode->pFirstChild || !pSQLNode->pLastChild) + { + flmAssert( 0); + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + if (bNotted) + { + eOperator = (eOperator == SQL_AND_OP + ? SQL_OR_OP + : SQL_AND_OP); + pSQLNode->nd.op.eOperator = eOperator; + } + if (pParentNode) + { + + // Logical sub-expressions can only be operands of + // AND, OR, or NOT operators. + + if (!isSQLLogicalOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + if (pParentNode->nd.op.eOperator == eOperator) + { + + // Move all of pSQLNode's children become the immediate + // children of pParentNode. + + pTmpNode = pSQLNode->pFirstChild; + while (pTmpNode) + { + pTmpNode->pParent = pParentNode; + pTmpNode = pTmpNode->pNextSib; + } + + if (pSQLNode->pPrevSib) + { + pSQLNode->pPrevSib->pNextSib = pSQLNode->pFirstChild; + pSQLNode->pFirstChild->pPrevSib = pSQLNode->pPrevSib; + } + if (pSQLNode->pNextSib) + { + pSQLNode->pNextSib->pPrevSib = pSQLNode->pLastChild; + pSQLNode->pLastChild->pNextSib = pSQLNode->pNextSib; + } + + // Continue processing from pSQLNode's first child, which + // is the beginning of the list of nodes we just replaced + // pSQLNode with. + + pSQLNode = pSQLNode->pFirstChild; + continue; + } + } + } + else if (eOperator == SQL_NOT_OP) + { + + // Logical sub-expressions can only be operands of + // AND, OR, or NOT operators. + + if (pParentNode) + { + if (!isSQLLogicalOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + } + bNotted = !bNotted; + + // Clip NOT nodes out of the tree. + + pSQLNode = sqlClipNotNode( pSQLNode, &m_pQuery); + pParentNode = pSQLNode->pParent; + continue; + } + else if (isSQLCompareOp( eOperator)) + { + + // Comparison sub-expressions can only be operands of + // AND, OR, or NOT operators. + + if (pParentNode) + { + if (!isSQLLogicalOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + } + if (bNotted) + { + switch (eOperator) + { + case SQL_EQ_OP: + eOperator = SQL_NE_OP; + break; + case SQL_NE_OP: + eOperator = SQL_EQ_OP; + break; + case SQL_LT_OP: + eOperator = SQL_GE_OP; + break; + case SQL_LE_OP: + eOperator = SQL_GT_OP; + break; + case SQL_GT_OP: + eOperator = SQL_LE_OP; + break; + case SQL_GE_OP: + eOperator = SQL_LT_OP; + break; + default: + + // Don't change the other operators. + // Will just use the bNotted flag when + // evaluating. + + break; + } + pSQLNode->nd.op.eOperator = eOperator; + } + } + else + { + + // Better be an arithmetic operator we are dealing with + // at this point. + + flmAssert( isSQLArithOp( eOperator)); + + // Arithmetic sub-expressions can only be operands + // of arithmetic or comparison operators + + if (pParentNode) + { + if (!isSQLCompareOp( pParentNode->nd.op.eOperator) && + !isSQLArithOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + } + } + } + else if (eNodeType == SQL_COLUMN_NODE) + { + flmAssert( !pSQLNode->pFirstChild); + } + else + { + flmAssert( eNodeType == SQL_VALUE_NODE); + + // If bNotted is TRUE and we have a boolean value, change + // the value: FALSE ==> TRUE, TRUE ==> FALSE. + + if (bNotted && pSQLNode->nd.value.eValType == SQL_BOOL_VAL) + { + if (pSQLNode->nd.value.val.eBool == SQL_TRUE) + { + pSQLNode->nd.value.val.eBool = SQL_FALSE; + } + else if (pSQLNode->nd.value.val.eBool == SQL_FALSE) + { + pSQLNode->nd.value.val.eBool = SQL_TRUE; + } + } + + // Values can only be operands of arithmetic or comparison operators, + // unless they are boolean values, in which case they can only be + // operands of logical operators. + + if (pParentNode) + { + if (pSQLNode->nd.value.eValType == SQL_BOOL_VAL) + { + if (!isSQLLogicalOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + } + else + { + if (!isSQLCompareOp( pParentNode->nd.op.eOperator) && + !isSQLArithOp( pParentNode->nd.op.eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERAND); + goto Exit; + } + } + } + + // A value node should not have any children + + flmAssert( !pSQLNode->pFirstChild); + } + + // Do traversal to child node, if any + + if (pSQLNode->pFirstChild) + { + pParentNode = pSQLNode; + pSQLNode = pSQLNode->pFirstChild; + continue; + } + + // Go back up the tree until we hit something that has + // a sibling. + + while (!pSQLNode->pNextSib) + { + + // If there are no more parents, we are done. + + if ((pSQLNode = pSQLNode->pParent) == NULL) + { + goto Exit; + } + + flmAssert( pSQLNode->eNodeType == SQL_OPERATOR_NODE); + + // Evaluate arithmetic expressions if both operands are + // constants. + + if (isSQLArithOp( pSQLNode->nd.op.eOperator) && + pSQLNode->pFirstChild->eNodeType == SQL_VALUE_NODE && + pSQLNode->pLastChild->eNodeType == SQL_VALUE_NODE) + { + if (RC_BAD( rc = sqlEvalArithOperator( + &pSQLNode->pFirstChild->nd.value, + &pSQLNode->pLastChild->nd.value, + pSQLNode->nd.op.eOperator, + &pSQLNode->nd.value))) + { + goto Exit; + } + pSQLNode->eNodeType = SQL_VALUE_NODE; + pSQLNode->nd.value.uiFlags = SQL_VAL_IS_CONSTANT; + pSQLNode->pFirstChild = NULL; + pSQLNode->pLastChild = NULL; + } + else + { + + // For the AND and OR operators, check the operands to + // see if they are boolean values. Boolean values can + // be weeded out of the criteria as we go back up the + // tree. + + if (pSQLNode->nd.op.eOperator == SQL_OR_OP || + pSQLNode->nd.op.eOperator == SQL_AND_OP) + { + pSQLNode = sqlEvalLogicalOperands( pSQLNode); + if (!pSQLNode->pParent) + { + m_pQuery = pSQLNode; + } + } + } + + pParentNode = pSQLNode->pParent; + } + + // pSQLNode will NEVER be NULL if we get here, because we + // will jump to Exit in those cases. + + pSQLNode = pSQLNode->pNextSib; + + // Need to reset the bNotted flag to what it would have + // been as we traverse back up the tree. + + bNotted = pParentNode->bNotted; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Allocate and set up a DNF node. +//------------------------------------------------------------------------- +FSTATIC RCODE createDNFNode( + F_Pool * pPool, + SQL_DNF_NODE * pParentDNFNode, + SQL_DNF_NODE ** ppDNFNode, + SQL_NODE * pNode) +{ + RCODE rc = NE_SFLM_OK; + SQL_DNF_NODE * pDNFNode; + + if (RC_BAD( rc = pPool->poolCalloc( sizeof( SQL_DNF_NODE), + (void **)&pDNFNode))) + { + goto Exit; + } + + // pDNFNode->pNode will be NULL if it is an AND or OR operator. + + if (pNode->eNodeType == SQL_OPERATOR_NODE) + { + if (pNode->nd.op.eOperator == SQL_AND_OP) + { + pDNFNode->bAndOp = TRUE; + } + else if (pNode->nd.op.eOperator == SQL_OR_OP) + { + // No need to really set as it is already 0 from poolCalloc. + // pDNFNode->bAndOp = FALSE; + } + else + { + pDNFNode->pNode = pNode; + } + } + else + { + pDNFNode->pNode = pNode; + } + if ((pDNFNode->pParent = pParentDNFNode) != NULL) + { + if ((pDNFNode->pPrevSib = pParentDNFNode->pLastChild) != NULL) + { + pDNFNode->pPrevSib->pNextSib = pDNFNode; + } + else + { + pParentDNFNode->pFirstChild = pDNFNode; + } + pParentDNFNode->pLastChild = pDNFNode; + } + *ppDNFNode = pDNFNode; + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Copy the sub-tree pointed to by pSrcSubTree and then link the +// new sub-tree as the last child of pParentNode. +//------------------------------------------------------------------------- +FSTATIC RCODE copyAndLinkSubTree( + F_Pool * pPool, + SQL_DNF_NODE * pSrcSubTree, + SQL_DNF_NODE * pParentNode) +{ + RCODE rc = NE_SFLM_OK; + SQL_DNF_NODE * pNewSubTree = NULL; + SQL_DNF_NODE * pCurrDestParentNode = NULL; + SQL_DNF_NODE * pCurrSrcNode = pSrcSubTree; + SQL_DNF_NODE * pNewDestNode = NULL; + + for (;;) + { + if (RC_BAD( rc = pPool->poolCalloc( sizeof( SQL_DNF_NODE), + (void **)&pNewDestNode))) + { + goto Exit; + } + pNewDestNode->pNode = pCurrSrcNode->pNode; + pNewDestNode->bAndOp = pCurrSrcNode->bAndOp; + if (!pNewSubTree) + { + pNewSubTree = pNewDestNode; + } + else + { + pNewDestNode->pParent = pCurrDestParentNode; + if ((pNewDestNode->pPrevSib = pCurrDestParentNode->pLastChild) != NULL) + { + pNewDestNode->pPrevSib->pNextSib = pNewDestNode; + } + else + { + pCurrDestParentNode->pFirstChild = pNewDestNode; + } + pCurrDestParentNode->pLastChild = pNewDestNode; + } + + // Try to go down to a child node + + if (pCurrSrcNode->pFirstChild) + { + pCurrSrcNode = pCurrSrcNode->pFirstChild; + pCurrDestParentNode = pNewDestNode; + continue; + } + + // No child nodes, go back up parent chain until we find one that + // has a sibling. + + for (;;) + { + if (pCurrSrcNode == pSrcSubTree) + { + break; + } + if (pCurrSrcNode->pNextSib) + { + break; + } + pCurrSrcNode = pCurrSrcNode->pParent; + pCurrDestParentNode = pCurrDestParentNode->pParent; + } + if (pCurrSrcNode == pSrcSubTree) + { + break; + } + pCurrSrcNode = pCurrSrcNode->pNextSib; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Distribute an AND operator over an OR operator. The AND operator +// is the parent node of the passed in pOldOrNode. A new list of +// AND nodes is created which will replace the original AND node in +// the tree. +//------------------------------------------------------------------------- +FSTATIC RCODE distributeAndOverOr( + F_Pool * pPool, + SQL_DNF_NODE * pOldOrNode, + SQL_DNF_NODE ** ppDNFTree) +{ + RCODE rc = NE_SFLM_OK; + SQL_DNF_NODE * pOldAndNode; + SQL_DNF_NODE * pOldAndParentNode; + SQL_DNF_NODE * pNewAndNode; + SQL_DNF_NODE * pFirstNewAndNode; + SQL_DNF_NODE * pLastNewAndNode; + SQL_DNF_NODE * pOrChildNode; + SQL_DNF_NODE * pAndChildNode; + + // Parent node to pOldOrNode better be an AND node. + + pOldAndNode = pOldOrNode->pParent; + flmAssert( !pOldAndNode->pNode && pOldAndNode->bAndOp); + + // Distribute ALL of the AND node's children (except this OR node) + // across ALL of the OR node's children + + pFirstNewAndNode = NULL; + pLastNewAndNode = NULL; + pOrChildNode = pOldOrNode->pFirstChild; + while (pOrChildNode) + { + if (RC_BAD( rc = pPool->poolCalloc( sizeof( SQL_DNF_NODE), + (void **)&pNewAndNode))) + { + goto Exit; + } + pNewAndNode->bAndOp = TRUE; + if ((pNewAndNode->pPrevSib = pLastNewAndNode) != NULL) + { + pLastNewAndNode->pNextSib = pNewAndNode; + } + else + { + pFirstNewAndNode = pNewAndNode; + } + pLastNewAndNode = pNewAndNode; + + // Copy all of the old AND node's children, except for this + // OR node as children of the new AND node. + + pAndChildNode = pOldAndNode->pFirstChild; + while (pAndChildNode) + { + if (pAndChildNode != pOldOrNode) + { + + if (RC_BAD( rc = copyAndLinkSubTree( pPool, pAndChildNode, pNewAndNode))) + { + goto Exit; + } + } + pAndChildNode = pAndChildNode->pNextSib; + } + + // Copy the entire sub-tree of pOrChildNode and link it as the last + // child of the new AND node. + + if (RC_BAD( rc = copyAndLinkSubTree( pPool, pOrChildNode, pNewAndNode))) + { + goto Exit; + } + pOrChildNode = pOrChildNode->pNextSib; + } + + // Link the newly created AND list in where the old + // AND node was (pOldAndNode). If it was at the root + // of the tree, we will need to create a new OR root. + + if ((pOldAndParentNode = pOldAndNode->pParent) == NULL) + { + if (RC_BAD( rc = pPool->poolCalloc( sizeof( SQL_DNF_NODE), + (void **)&pOldAndParentNode))) + { + goto Exit; + } + + // NOTE: No need to set anything in this new node, we want it to be + // an OR node, which means that bAndOp is FALSE and pNode is NULL - both + // of which are set by the poolCalloc. + + *ppDNFTree = pOldAndParentNode; + } + + // Point all of the new AND nodes to the parent of the old AND node. + + pAndChildNode = pFirstNewAndNode; + while (pAndChildNode) + { + pAndChildNode->pParent = pOldAndParentNode; + pAndChildNode = pAndChildNode->pNextSib; + } + + // Link the new list of AND nodes where the old AND node was. + // Although the old AND node is still allocated, it is no longer + // pointed to from the tree. + + if ((pFirstNewAndNode->pPrevSib = pOldAndNode->pPrevSib) != NULL) + { + pFirstNewAndNode->pPrevSib->pNextSib = pFirstNewAndNode; + } + else + { + pOldAndParentNode->pFirstChild = pFirstNewAndNode; + } + if ((pLastNewAndNode->pNextSib = pOldAndNode->pNextSib) != NULL) + { + pLastNewAndNode->pNextSib->pPrevSib = pLastNewAndNode; + } + else + { + pOldAndParentNode->pLastChild = pLastNewAndNode; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Convert query tree to disjunctive normal form (DNF). Result is +// a list of sub-queries that are ORed together. +//------------------------------------------------------------------------- +RCODE SQLQuery::convertToDNF( void) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pCurrNode; + SQL_DNF_NODE * pParentDNFNode; + SQL_DNF_NODE * pCurrDNFNode; + SQL_DNF_NODE * pDNFTree; + SQL_DNF_NODE * pAndList; + SQL_DNF_NODE * pExprList; + F_Pool pool; + SQL_SUBQUERY * pSubQuery; + FLMUINT uiLoop; + + pool.poolInit( 1024); + + // If the top node in the tree is not an AND or OR operator, + // create a single subquery that has a single operand. + + if (m_pQuery->eNodeType != SQL_OPERATOR_NODE || + m_pQuery->nd.op.eOperator != SQL_AND_OP || + m_pQuery->nd.op.eOperator != SQL_OR_OP) + { + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_SUBQUERY), + (void **)&m_pFirstSubQuery))) + { + goto Exit; + } + m_pLastSubQuery = m_pFirstSubQuery; + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE *), + (void **)&m_pFirstSubQuery->ppOperands))) + { + goto Exit; + } + m_pFirstSubQuery->uiOperandCount = 1; + m_pFirstSubQuery->ppOperands [0] = m_pQuery; + goto Exit; + } + + // Create the tree of DNF nodes to point to all of the AND and OR nodes + // in the tree and their immediate child nodes. + + pCurrNode = m_pQuery; + pParentDNFNode = NULL; + pDNFTree = NULL; + for (;;) + { + if (RC_BAD( rc = createDNFNode( &pool, pParentDNFNode, + &pCurrDNFNode, pCurrNode))) + { + goto Exit; + } + if (!pDNFTree) + { + pDNFTree = pCurrDNFNode; + } + + // Don't traverse down to child nodes if it is not an AND or OR node. + + if (pCurrNode->eNodeType == SQL_OPERATOR_NODE && + (pCurrNode->nd.op.eOperator == SQL_AND_OP || + pCurrNode->nd.op.eOperator == SQL_OR_OP)) + { + if (pCurrNode->pFirstChild) + { + pCurrNode = pCurrNode->pFirstChild; + pParentDNFNode = pCurrDNFNode; + continue; + } + } + + // Go back up to parent until we find one that has a sibling. + + while (!pCurrNode->pNextSib) + { + if ((pCurrNode = pCurrNode->pParent) == NULL) + { + break; + } + pParentDNFNode = pParentDNFNode->pParent; + } + if (!pCurrNode) + { + break; + } + pCurrNode = pCurrNode->pNextSib; + } + + // Now traverse the DNF tree and move all OR operators to the top. + // When we are done we should have a DNF tree with either a single AND + // node and a list of subordinate expressions, or a single OR node with + // a mix of AND child nodes or non-AND expressions. + + pCurrDNFNode = pDNFTree; + for (;;) + { + + // If we hit an OR node that is not the root node, it's parent should be + // an AND node. Distribute the AND node's operands over all of the + // OR node's operands. + + if (pCurrDNFNode->pNode->eNodeType == SQL_OPERATOR_NODE && + pCurrDNFNode->pNode->nd.op.eOperator == SQL_OR_OP && + pCurrDNFNode->pParent) + { + if (RC_BAD( rc = distributeAndOverOr( &pool, pCurrDNFNode, + &pDNFTree))) + { + goto Exit; + } + + // Start over at the top of the tree. + + pCurrDNFNode = pDNFTree; + continue; + } + + // Go to first child, if there is one. + + if (pCurrDNFNode->pFirstChild) + { + pCurrDNFNode = pCurrDNFNode->pFirstChild; + continue; + } + + // No child nodes, go to sibling nodes. If no sibling nodes, + // traverse back up parent chain until we find one. + + while (!pCurrDNFNode->pNextSib) + { + if ((pCurrDNFNode = pCurrDNFNode->pParent) == NULL) + { + break; + } + } + if (!pCurrDNFNode) + { + break; + } + pCurrDNFNode = pCurrDNFNode->pNextSib; + } + + // If we get to this point, we have created a DNF tree that either + // as an OR at the top or an AND at the top. If it is an OR at the + // top, we have multiple sub-queries. If it is an AND at the top, we + // have a single sub-query. + + if (pDNFTree->bAndOp) + { + pAndList = pDNFTree; + } + else + { + pAndList = pDNFTree->pFirstChild; + flmAssert( pAndList); + } + + while (pAndList) + { + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_SUBQUERY), + (void **)&pSubQuery))) + { + goto Exit; + } + + // Link the subquery as the last sub-query in our sub-query list + + if ((pSubQuery->pPrev = m_pLastSubQuery) != NULL) + { + pSubQuery->pPrev->pNext = pSubQuery; + } + else + { + m_pFirstSubQuery = pSubQuery; + } + m_pLastSubQuery = pSubQuery; + + // The child may be a simple expression, in which case it is its + // own sub-query. + + if (pAndList->pNode) + { + pSubQuery->uiOperandCount = 1; + + // The expression should not have any child nodes. + + flmAssert( !pExprList->pFirstChild && !pExprList->pLastChild); + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE *), + (void **)&pSubQuery->ppOperands))) + { + goto Exit; + } + pSubQuery->ppOperands [0] = pAndList->pNode; + + // NULL out the node's parent pointer and sibling pointers - just + // to keep things tidy. + + pAndList->pNode->pParent = NULL; + pAndList->pNode->pNextSib = NULL; + pAndList->pNode->pPrevSib = NULL; + } + else + { + + // Count the expressions in the list - should be at least one. + + pExprList = pAndList->pFirstChild; + flmAssert( pExprList); + while (pExprList) + { + + // All of the expressions should point to nodes in the query + // tree, and should not be AND or OR nodes. Furthermore, + // they should not have child nodes + + flmAssert( pExprList->pNode && !pExprList->pFirstChild && + !pExprList->pLastChild); + pSubQuery->uiOperandCount++; + pExprList = pExprList->pNextSib; + } + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE *) * pSubQuery->uiOperandCount, + (void **)&pSubQuery->ppOperands))) + { + goto Exit; + } + + // Set the pointers in the operand list for the sub-query. + + for (uiLoop = 0, pExprList = pAndList->pFirstChild; + pExprList; + uiLoop++, pExprList = pExprList->pNextSib) + { + pSubQuery->ppOperands [uiLoop] = pExprList->pNode; + + // NULL out the node's parent pointer and sibling pointers - just + // to keep things tidy. + + pExprList->pNode->pParent = NULL; + pExprList->pNode->pNextSib = NULL; + pExprList->pNode->pPrevSib = NULL; + } + } + flmAssert( uiLoop == pSubQuery->uiOperandCount); + pAndList = pAndList->pNextSib; + } + +Exit: + + return( rc); +} + +#if 0 +//------------------------------------------------------------------------- +// Desc: Determine if a particular predicate is associated with the +// specified table. Only return TRUE if the predicate is associated +// with this table and only with this table. +//------------------------------------------------------------------------- +FSTATIC FLMBOOL predIsForTable( + SQL_NODE * pPredRootNode, + SQL_TABLE * pTable) +{ + FLMBOOL bIsAssociated = FALSE; + SQL_NODE * pCurrNode = pPredRootNode; + + for (;;) + { + if (pCurrNode->eNodeType == SQL_COLUMN_NODE) + { + if (pCurrNode->nd.column.pTable == pTable) + { + bIsAssociated = TRUE; + } + else + { + bIsAssociated = FALSE; + break; + } + } + + if (pCurrNode->pFirstChild) + { + pCurrNode = pCurrNode->pFirstChild; + continue; + } + + // No child nodes, traverse to sibling - or sibling of first node + // in the parent chain that has a next sibling. + + for (;;) + { + if (pCurrNode == pPredRootNode) + { + break; + } + if (pCurrNode->pNextSib) + { + break; + } + pCurrNode = pCurrNode->pParent; + } + if (pCurrNode == pPredRootNode) + { + break; + } + + // If we get to here, there should be a next sibling. + + pCurrNode = pCurrNode->pNextSib; + flmAssert( pCurrNode); + } + + return( bIsAssociated); +} + +//------------------------------------------------------------------------- +// Desc: Associate a predicate with all of the indexes it pertains to +// with respect to a particular table. +//------------------------------------------------------------------------- +RCODE SQLQuery::getPredKeys( + SQL_PRED * pPred, + SQL_TABLE * pTable) +{ + RCODE rc = NE_SFLM_OK; + ICD * pIcd; + SQL_INDEX * pIndex; + SQL_KEY * pKey; + +//visit - notes for reference: column number should be unique for the table. +//visit - notes for reference: no required/non-required pieces for indexes on tables + + + if (RC_BAD( rc = m_pDb->m_pDict->getAttribute( m_pDb, pPred->uiColumnNum, + &defInfo))) + { + goto Exit; + } + + // This ICD chain will only contain ICDs for this particular column on + // the table the column belongs to - because column numbers are globally + // unique. + + for (pIcd = defInfo.m_pFirstIcd; pIcd; pIcd = pIcd->pNextInChain) + { + + // If the table has an index specified for it, skip this ICD if + // it is not that index. + + if (pTable->bIndexSet && pTable->uiIndex != pIcd->pIxd->uiIndexNum) + { + continue; + } + + // Cannot use the index if it is off-line. + + if (pIcd->pIxd->uiFlags & (IXD_OFFLINE | IXD_SUSPENDED)) + { + continue; + } + + // Find the index off of the table. If not there, add it. + + pIndex = pTable->pFirstIndex; + while (pIndex->uiIndexNum != pIcd->pIxd->uiIndexNum) + { + pIndex = pIndex->pNext; + } + if (!pIndex) + { + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_INDEX), + (void **)&pIndex))) + { + goto Exit; + } + pIndex->pTable = pTable; + pIndex->uiIndexNum = pIcd->pIxd->uiIndexNum; + pIndex->uiNumComponents = pIcd->pIxd->uiNumKeyComponents; + if ((pIndex->pPrev = pTable->pLastIndex) != NULL) + { + pIndex->pPrev->pNext = pIndex; + } + else + { + pTable->pFirstIndex = pIndex; + } + pTable->pLastIndex = pIndex; + + // Allocate a single key for the index. + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_KEY), + (void **)&pKey))) + { + goto Exit; + } + pIndex->pLastKey = pIndex->pFirstKey = pKey; + pKey->pIndex = pIndex; + + // Allocate an array of key components for the key. + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_PRED *) * pIndex->uiNumComponents, + (void **)&pKey->ppKeyComponents))) + { + goto Exit; + } + } + else + { + pKey = pIndex->pFirstKey; + } + + // There should not be multiple predicates in a sub-query that + // have the same column, so this key component should NOT already + // be populated. + + flmAssert( !pKey->ppKeyComponents [pIcd->uiKeyComponent - 1]); + pKey->ppKeyComponents [pIcd->uiKeyComponent - 1] = pPred; + + // NOTE: Costs will be calculated later. + + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Determine the order in which to evaluate indexes for a particular +// table and sub-query. Those that have a primary key will be +// give preference over those that don't. +//------------------------------------------------------------------------- +FSTATIC void rankIndexes( + SQL_TABLE * pTable) +{ + SQL_INDEX * pIndex; + SQL_INDEX * pPrevIndex; + SQL_INDEX * pNextIndex; + SQL_KEY * pKey; + FLMUINT uiComponentCount; + FLMUINT uiPrevComponentCount; + + pIndex = pTable->pFirstIndex; + while (pIndex) + { + pNextIndex = pIndex->pNext; + pPrevIndex = pIndex->pPrev; + + // There should only be one key off of the index right now. + pKey = pIndex->pFirstKey; + flmAssert( !pKey->pNext); + + // Determine how many of the key's components point to a + // predicate. This stops at the first NULL pointer. There may + // be pointers after that one, but we really don't care, because + // we won't use those components to generate a key. + + pKey->uiComponentsUsed = 0; + while (pKey->uiComponentsUsed < pIndex->uiNumComponents && + pKey->ppKeyComponents [pKey->uiComponentsUsed]) + { + pKey->uiComponentsUsed++; + } + + // See if this key is using more components that the key for + // prior indexes. + + while (pPrevIndex) + { + if (pKey->uiComponentsUsed > pPrevIndex->pFirstKey->uiComponentsUsed) + { + // Move our current key up in front of the previous key - meaning + // it will be evaluated ahead of that key. + + // First, unlink the index from its current spot. pIndex->pPrev + // must be non-NULL - otherwise, we wouldn't have a pPrevIndex. + + flmAssert( pIndex->pPrev); + pIndex->pPrev->pNext = pIndex->pNext; + if (pIndex->pNext) + { + pIndex->pNext->pPrev = pIndex->pPrev; + } + else + { + pTable->pLastIndex = pIndex->pPrev; + } + + // Now, link it in front of pPrevIndex + + pIndex->pNext = pPrevIndex; + if ((pIndex->pPrev = pPrevIndex->pPrev) != NULL) + { + pIndex->pPrev->pNext = pIndex; + } + else + { + pTable->pFirstIndex = pIndex; + } + pPrevIndex->pPrev = pIndex; + pPrevIndex = pIndex->pPrev; + } + else + { + pPrevIndex = pPrevIndex->pPrev; + } + } + + pIndex = pNextIndex; + } +} + +//------------------------------------------------------------------------- +// Desc: Choose the best index for a table of the indexes for which we have +// generated predicate keys. +//------------------------------------------------------------------------- +RCODE SQLQuery::chooseBestIndex( + SQL_TABLE * pTable, + FLMUINT * puiCost) +{ + RCODE rc = NE_SFLM_OK + SQL_INDEX * pIndex = pTable->pFirstIndex; + + while (pIndex) + { + + // Should only be one key on each index at this point. + + flmAssert( pIndex->pFirstKey && pIndex->pFirstKey == pIndex->pLastKey); + + pIndex = pIndex->pNext; + } + + visit +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Calculate the cost of doing a table scan for a table. +//------------------------------------------------------------------------- +RCODE SQLQuery::calcTableScanCost( + SQL_TABLE * pTable, + FLMUINT64 * pui64Cost, + SQLTableCursor ** ppSQLTableCursor) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT64 ui64LeafBlocksBetween; + FLMUINT64 ui64TotalRefs; + FLMUINT bTotalsEstimated; + + if ((*ppSQLTableCursor = f_new SQLTableCursor) == NULL) + { + rc = RC_SET( NE_SFLM_MEM); + goto Exit; + } + if (RC_BAD( rc = (*ppSQLTableCursor)->setupRange( m_pDb, + pTable->uiTableNum, TRUE, 1, FLM_MAX_UINT64, + &ui64LeafBlocksBetween, &ui64TotalRefs, + &bTotalsEstimated))) + { + (*ppSQLTableCursor)->Release(); + *ppSQLTableCursor = NULL; + goto Exit; + } + if (!ui64LeafBlocksBetween) + { + *pui64Cost = 1; + } + else + { + *pui64Cost = ui64LeafBlocksBetween; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Merge keys from pSrcTable into pDestTable. +//------------------------------------------------------------------------- +RCODE SQLQuery::mergeKeys( + SQL_TABLE * pDestTable, + SQL_TABLE * pSrcTable) +{ + visit +} + +//------------------------------------------------------------------------- +// Desc: Optimize a particular table for a particular sub-query. +//------------------------------------------------------------------------- +RCODE SQLQuery::optimizeTable( + SQL_SUBQUERY * pSubQuery, + SQL_TABLE * pTable) +{ + RCODE rc = NE_SFLM_OK; + SQL_TABLE tmpTable; + FLMUINT uiLoop; + SQL_NODE * pOperand; + SQL_PRED * pPred; + void * pvMark = m_pool.poolMark(); + SQLTableCursor * pSQLTableCursor = NULL; + + // This routine should not be called if the table has already been + // marked to do a table scan. +visit - the caller should handle this case + + flmAssert( !pTable->bScan); + + f_memset( &tmpTable, 0, sizeof( SQL_TABLE)); + tmpTable.uiTableNum = pTable->uiTableNum; + tmpTable.uiIndex = pTable->uiIndex; + tmpTable.bIndexSet = pTable->bIndexSet; + + // Traverse the predicates of the sub-query. If any are found + // that are not predicates, the table must be scanned. + + for (uiLoop = 0, pOperand = pSubQuery->ppOperands [0]; + uiLoop < pSubQuery->uiOperandCount; + uiLoop++, pOperand = pSubQuery->ppOperands [uiLoop]) + { + + // If we hit a predicate that has not been turned into + // an SQL_PRED_NODE, it is not optimizable. + + if (pOperand->eNodeType != SQL_PRED_NODE) + { + + // See if the current table is involved in this predicate. If so, + // and it is the only table involved, the table should be scanned. + // Setting pFirstIndex and pLastIndex to NULL will cause this to + // happen below. + + if (predIsForTable( pOperand, pTable)) + { + m_pool.poolReset( pvMark); + tmpTable.pFirstIndex = NULL; + tmpTable.pLastIndex = NULL; + break; + } + } + else if (pOperand->nd.pred.pTable == pTable) + { + SQL_PRED * pPred = &pOperand->nd.pred; + + // We cannot use from and until keys for not/negative operators. + // We set pFirstIndex and pLastIndex to NULL to indicate that a + // table scan must occur. + + if ((pPred->bNotted && pPred->eOperator == SQL_MATCH_OP) || + pPred->eOperator == SQL_NE_OP) + { + m_pool.poolReset( pvMark); + tmpTable.pFirstIndex = NULL; + tmpTable.pLastIndex = NULL; + break; + } + +visit - before we collect keys for this predicate, we should check to see +if the predicate is a subset of any other predicate in previous sub-queries that +we have already optimized where the predicate in the previous sub-query was used +to optimize that previous sub-query. If so, we should simply merge this sub-query with +that one - it will be a waste of time to get another set of keys for this +sub-query. This, of course, implies that we need to keep track of the +predicates that were selected to optimize a particular sub-query. + + // See if there are any indexes for this predicate's column. + // For now we are just collecting them. We will calculate + // the best one later. + + if (RC_BAD( rc = getPredKeys( pPred, &tmpTable))) + { + goto Exit; + } + } + } + + // If we didn't find indexes for this table, set the bScan flag. + + if (!tmpTable.pFirstIndex) + { + tmpTable.bScan = TRUE; + if (RC_BAD( rc = calcTableScanCost( &tmpTable, &tmpTable.uiCost, + &pSQLTableCursor))) + { + goto Exit; + } + } + else + { + + // Rank the indexes to determine which ones to estimate cost for + // first. + + rankIndexes( &tmpTable); + + // Find the index with the lowest cost, if any. + // If the lowest cost index is still high, estimate the cost of doing + // a table scan. + + if (RC_BAD( rc = chooseBestIndex( &tmpTable, &tmpTable.uiCost))) + { + goto Exit; + } + + // Should be one index left after this. If the cost is high, see if + // a table scan would be cheaper. + + if (tmpTable.uiCost > 8) + { + FLMUINT uiScanCost; + + if (RC_BAD( rc = calcTableScanCost( &tmpTable, &uiScanCost, + &pSQLTableCursor))) + { + goto Exit; + } + if (uiScanCost < tmpTable.uiCost) + { + m_pool.poolReset( pvMark); + tmpTable.uiCost = uiScanCost; + tmpTable.bScan = TRUE; + tmpTable.pFirstIndex = NULL; + tmpTable.pLastIndex = NULL; + } + else + { + pSQLTableCursor->Release(); + pSQLTableCursor = NULL; + } + } + } + + // If we determined that we must do a table scan, set the bScan flag + // for the master table. Otherwise, merge these keys + + if (tmpTable.bScan) + { + pTable->bScan = TRUE; + pTable->uiCost = tmpTable.uiCost; + + // Better have calculated a cost and have a collection + // cursor at this point. + + flmAssert( pSQLTableCursor); + pTable->pSQLTableCursor = pSQLTableCursor; + pSQLTableCursor = NULL; + pTable->pFirstIndex = NULL; + pTable->pLastIndex = NULL; + } + else + { + if (RC_BAD( rc = mergeKeys( pTable, &tmpTable))) + { + goto Exit; + } + pTable->uiCost += tmpTable.uiCost; + } + +Exit: + + if (pSQLTableCursor) + { + pSQLTableCursor->Release(); + } + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Optimize the sub-queries of an SQL query. +//------------------------------------------------------------------------- +RCODE SQLQuery::optimizeSubQueries( void) +{ + RCODE rc = NE_SFLM_OK; + SQL_SUBQUERY * pSubQuery; + SQL_TABLE * pTable; + + // For each table in our expression, attempt to pick an index for each + // subquery. + + for (pTable = m_pFirstTable; pTable; pTable = pTable->pNext) + { + pSubQuery = m_pFirstSubQuery; + while (pSubQuery) + { + if (RC_BAD( rc = optimizeTable( pSubQuery, pTable))) + { + goto Exit; + } + + // If the optimization decided we should scan the table, there + // is no need to look at any more sub-queries for this table. + + if (pTable->bScan) + { + break; + } + pSubQuery = pSubQuery->pNext; + } + + // See if a table scan is going to be cheaper. + + if (!pTable->bScan && pTable->uiCost > 8) + { + SQLTableCursor * pSQLTableCursor = NULL; + FLMUINT uiScanCost; + + if (RC_BAD( rc = calcTableScanCost( pTable, &uiScanCost, + &pSQLTableCursor))) + { + goto Exit; + } + if (uiScanCost < pTable->uiCost) + { + pTable->uiCost = uiScanCost; + pTable->bScan = TRUE; + pTable->pSQLTableCursor = pSQLTableCursor; + pTable->pFirstIndex = NULL; + pTable->pLastIndex = NULL; + } + else + { + pSQLTableCursor->Release(); + } + } + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Optimize an SQL query. +//------------------------------------------------------------------------- +RCODE SQLQuery::optimize( void) +{ + RCODE rc = NE_SFLM_OK; + + if (m_bOptimized) + { + goto Exit; + } + + // We save the F_Database object so that we can always check and make + // sure we are associated with this database on any query operations + // that occur after optimization. -- Link it into the list of queries + // off of the F_Database object. NOTE: We may not always use the + // same F_Db object, but it must always be the same F_Database object. + + m_pDatabase = m_pDb->m_pDatabase; + m_pNext = NULL; + m_pDatabase->lockMutex(); + if ((m_pPrev = m_pDatabase->m_pLastSQLQuery) != NULL) + { + m_pPrev->m_pNext = this; + } + else + { + m_pDatabase->m_pFirstSQLQuery = this; + } + m_pDatabase->m_pLastSQLQuery = this; + m_pDatabase->unlockMutex(); + + // Make sure we have a completed expression + + if (m_pCurrParseState) + { + if (m_pCurrParseState->pPrev || + m_pCurrParseState->uiNestLevel || + (m_pCurrParseState->pLastNode && + m_pCurrParseState->pLastNode->eNodeType == FLM_OPERATOR_NODE)) + { + rc = RC_SET( NE_SFLM_Q_INCOMPLETE_QUERY_EXPR); + goto Exit; + } + m_pQuery = m_pCurrParseState->pRootNode; + } + + m_uiLanguage = m_pDb->getDefaultLanguage(); + + // An empty expression should scan the database and return everything. + + if (!m_pQuery) + { + if (m_bIndexSet && m_uiIndex) + { + rc = setupIndexScan(); + } + else + { + m_bScan = TRUE; + } + goto Exit; + } + + // Handle the case of a value node or arithmetic expression at the root + // These types of expressions do not return results from the database. + + if (m_pQuery->eNodeType == SQL_VALUE_NODE) + { + if (m_pQuery->nd.value.eValType == SQL_BOOL_VAL && + m_pQuery->nd.value.val.eBool == SQL_TRUE) + { + m_bScan = TRUE; + } + else + { + m_bEmpty = TRUE; + } + } + else if (m_pQuery->eNodeType == SQL_OPERATOR_NODE && + isSQLArithOp( pQNode->nd.op.eOperator)) + { + m_bEmpty = TRUE; + goto Exit; + } + + // If the user explicitly said to NOT use an index, we will not + + if (m_bIndexSet && !m_uiIndex) + { + m_bScan = TRUE; + goto Exit; + } + + // Flatten the AND and OR operators in the query tree. + + if (RC_BAD( rc = flattenTree())) + { + goto Exit; + } + + // Convert to DNF + + if (RC_BAD( rc = convertToDNF())) + { + goto Exit; + } + + // Convert all operands of each sub-query to predicates where + // possible. + + if (RC_BAD( rc = convertOperandsToPredicates())) + { + goto Exit; + } + + // Optimize each sub-query. + + if (RC_BAD( rc = optimizeSubQueries())) + { + goto Exit; + } + + m_bOptimized = TRUE; + +Exit: + + return( rc); +} +#endif diff --git a/sql/src/sqlparser.cpp b/sql/src/sqlparser.cpp new file mode 100644 index 0000000..ae9e609 --- /dev/null +++ b/sql/src/sqlparser.cpp @@ -0,0 +1,577 @@ +//------------------------------------------------------------------------- +// Desc: Parse SQL +// Tabs: 3 +// +// Copyright (c) 2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id$ +//------------------------------------------------------------------------- + +#include "flaimsys.h" + +// Local function prototypes + +FSTATIC void sqlUnlinkFromParent( + SQL_NODE * pSQLNode); + +FSTATIC void sqlLinkLastChild( + SQL_NODE * pParent, + SQL_NODE * pChild); + +static FLMUINT uiSQLOpPrecedenceTable[ SQL_NEG_OP - SQL_AND_OP + 1] = +{ + 2, // SQL_AND_OP + 1, // SQL_OR_OP + 10, // SQL_NOT_OP + 6, // SQL_EQ_OP + 6, // SQL_NE_OP + 6, // SQL_APPROX_EQ_OP + 7, // SQL_LT_OP + 7, // SQL_LE_OP + 7, // SQL_GT_OP + 7, // SQL_GE_OP + 5, // SQL_BITAND_OP + 3, // SQL_BITOR_OP + 4, // SQL_BITXOR_OP + 9, // SQL_MULT_OP + 9, // SQL_DIV_OP + 9, // SQL_MOD_OP + 8, // SQL_PLUS_OP + 8, // SQL_MINUS_OP + 10 // SQL_NEG_OP +}; + +FINLINE FLMUINT getSQLOpPrecedence( + eSQLQueryOperators eOperator) +{ + return( uiSQLOpPrecedenceTable [eOperator - SQL_AND_OP]); +} + +//------------------------------------------------------------------------- +// Desc: Constructor +//------------------------------------------------------------------------- +SQLQuery::SQLQuery() +{ + m_uiLanguage = FLM_US_LANG; + m_pool.poolInit( 1024); + m_pFirstSubQuery = NULL; + m_pLastSubQuery = NULL; + m_pFirstTable = NULL; + m_pLastTable = NULL; + m_bOptimized = FALSE; + m_bScan = FALSE; + m_bScanIndex = FALSE; + m_bEmpty = FALSE; + m_uiIndexNum = 0; + m_bIndexSet = FALSE; + m_pQuery = NULL; + m_pDatabase = NULL; + m_pDb = NULL; + m_pNext = NULL; + m_pPrev = NULL; +} + +//------------------------------------------------------------------------- +// Desc: Destructor +//------------------------------------------------------------------------- +SQLQuery::~SQLQuery() +{ + m_pool.poolFree(); + if (m_pDatabase) + { + m_pDatabase->lockMutex(); + + // Unlink the query from the list off of the F_Database object. + + if (m_pPrev) + { + m_pPrev->m_pNext = m_pNext; + } + else + { + m_pDatabase->m_pFirstSQLQuery = m_pNext; + } + if (m_pNext) + { + m_pNext->m_pPrev = m_pPrev; + } + else + { + m_pDatabase->m_pLastSQLQuery = m_pPrev; + } + m_pDatabase->unlockMutex(); + } +} + +//------------------------------------------------------------------------- +// Desc: Allocate a structure for keeping track of the state of the +// current SQL query. +//------------------------------------------------------------------------- +RCODE SQLQuery::allocParseState( void) +{ + RCODE rc = NE_SFLM_OK; + SQL_PARSE_STATE * pParseState; + + if (!m_pCurrParseState || !m_pCurrParseState->pNext) + { + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_PARSE_STATE), + (void **)&pParseState))) + { + goto Exit; + } + if ((pParseState->pPrev = m_pCurrParseState) != NULL) + { + m_pCurrParseState->pNext = pParseState; + } + m_pCurrParseState = pParseState; + } + else + { + SQL_PARSE_STATE * pSaveNext; + SQL_PARSE_STATE * pSavePrev; + + m_pCurrParseState = m_pCurrParseState->pNext; + + // Zero out everything except for the prev and next pointers + + pSaveNext = m_pCurrParseState->pNext; + pSavePrev = m_pCurrParseState->pPrev; + f_memset( m_pCurrParseState, 0, sizeof( SQL_PARSE_STATE)); + m_pCurrParseState->pNext = pSaveNext; + m_pCurrParseState->pPrev = pSavePrev; + } + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Unlinks a node from its parent and siblings. This routine assumes +// that the test has already been made that the node has a parent. +//------------------------------------------------------------------------- +FSTATIC void sqlUnlinkFromParent( + SQL_NODE * pSQLNode) +{ + flmAssert( pSQLNode->pParent); + if (pSQLNode->pPrevSib) + { + pSQLNode->pPrevSib->pNextSib = pSQLNode->pNextSib; + } + else + { + pSQLNode->pParent->pFirstChild = pSQLNode->pNextSib; + } + if (pSQLNode->pNextSib) + { + pSQLNode->pNextSib->pPrevSib = pSQLNode->pPrevSib; + } + else + { + pSQLNode->pParent->pLastChild = pSQLNode->pPrevSib; + } + + pSQLNode->pParent = NULL; + pSQLNode->pPrevSib = NULL; + pSQLNode->pNextSib = NULL; +} + +//------------------------------------------------------------------------- +// Desc: Links one SQL_NODE as the last child of another. Will unlink the +// child node from any parent it may be linked to. +//------------------------------------------------------------------------- +FSTATIC void sqlLinkLastChild( + SQL_NODE * pParent, + SQL_NODE * pChild + ) +{ + + // If necessary, unlink the child from parent and siblings + + if (pChild->pParent) + { + sqlUnlinkFromParent( pChild); + } + + // Link child as the last child to parent + + pChild->pParent = pParent; + pChild->pNextSib = NULL; + if ((pChild->pPrevSib = pParent->pLastChild) != NULL) + { + pChild->pPrevSib->pNextSib = pChild; + } + else + { + pParent->pFirstChild = pChild; + } + pParent->pLastChild = pChild; +} + +//------------------------------------------------------------------------- +// Desc: Allocate a value node. +//------------------------------------------------------------------------- +RCODE SQLQuery::allocValueNode( + FLMUINT uiValLen, + eSQLValTypes eValType, + SQL_NODE ** ppSQLNode + ) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pSQLNode; + + if (!m_pCurrParseState) + { + if (RC_BAD( rc = allocParseState())) + { + goto Exit; + } + } + + if (m_pCurrParseState->bExpectingLParen) + { + rc = RC_SET( NE_SFLM_Q_EXPECTING_LPAREN); + goto Exit; + } + + if (!expectingOperand()) + { + rc = RC_SET( NE_SFLM_Q_UNEXPECTED_VALUE); + goto Exit; + } + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE), + (void **)ppSQLNode))) + { + goto Exit; + } + pSQLNode = *ppSQLNode; + pSQLNode->eNodeType = SQL_VALUE_NODE; + pSQLNode->nd.value.eValType = eValType; + pSQLNode->nd.value.uiDataLen = uiValLen; + pSQLNode->nd.value.uiFlags = SQL_VAL_IS_CONSTANT; + + // For string and binary data, allocate a buffer. + + if (uiValLen && + (eValType == SQL_UTF8_VAL || eValType == SQL_BINARY_VAL)) + { + if (RC_BAD( rc = m_pool.poolAlloc( uiValLen, + (void **)&pSQLNode->nd.value.val.pucBuf))) + { + goto Exit; + } + } + + if (m_pCurrParseState->pRootNode) + { + sqlLinkLastChild( m_pCurrParseState->pCurOperatorNode, pSQLNode); + } + else + { + m_pCurrParseState->pRootNode = pSQLNode; + } + m_pCurrParseState->bExpectingOperator = TRUE; + m_pCurrParseState->pLastNode = pSQLNode; + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Add an operator to the query expression +//------------------------------------------------------------------------- +RCODE SQLQuery::addOperator( + eSQLQueryOperators eOperator, + FLMUINT uiCompareRules) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pSQLNode; + SQL_NODE * pParentNode; + + if (!m_pCurrParseState) + { + if (RC_BAD( rc = allocParseState())) + { + goto Exit; + } + } + + // If we are expecting a left paren (for a function), that is + // the only thing that is acceptable at this point. + + if (m_pCurrParseState->bExpectingLParen && eOperator != SQL_LPAREN_OP) + { + rc = RC_SET( NE_SFLM_Q_EXPECTING_LPAREN); + goto Exit; + } + + switch (eOperator) + { + case SQL_LPAREN_OP: + + // If the operator is a left paren, increment the nesting level + + if (expectingOperator()) + { + rc = RC_SET( NE_SFLM_Q_UNEXPECTED_LPAREN); + goto Exit; + } + m_pCurrParseState->uiNestLevel++; + m_pCurrParseState->bExpectingLParen = FALSE; + goto Exit; + + case SQL_RPAREN_OP: + if (expectingOperand()) + { + rc = RC_SET( NE_SFLM_Q_UNEXPECTED_RPAREN); + goto Exit; + } + if (!m_pCurrParseState->uiNestLevel) + { + rc = RC_SET( NE_SFLM_Q_UNMATCHED_RPAREN); + goto Exit; + } + m_pCurrParseState->uiNestLevel--; + + goto Exit; + + case SQL_NEG_OP: + case SQL_NOT_OP: + if (expectingOperator()) + { + rc = RC_SET( NE_SFLM_Q_EXPECTING_OPERATOR); + goto Exit; + } + break; + + default: + + if (expectingOperand()) + { + rc = RC_SET( NE_SFLM_Q_EXPECTING_OPERAND); + goto Exit; + } + if (!isLegalSQLOperator( eOperator)) + { + rc = RC_SET( NE_SFLM_Q_ILLEGAL_OPERATOR); + goto Exit; + } + + break; + } + + // Cannot set both FLM_COMP_COMPRESS_WHITESPACE and FLM_COMP_NO_WHITESPACE + // in comparison rules. Also, cannot set FLM_COMP_IGNORE_LEADING_SPACE or + // FLM_COMP_IGNORE_TRAILING_SPACE with FLM_COMP_NO_WHITESPACE. + + if ((uiCompareRules & FLM_COMP_NO_WHITESPACE) && + (uiCompareRules & (FLM_COMP_COMPRESS_WHITESPACE | + FLM_COMP_IGNORE_LEADING_SPACE | + FLM_COMP_IGNORE_TRAILING_SPACE))) + { + rc = RC_SET_AND_ASSERT( NE_SFLM_Q_ILLEGAL_COMPARE_RULES); + goto Exit; + } + + // Make a QNODE and find a place for it in the query tree + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE), + (void **)&pSQLNode))) + { + goto Exit; + } + pSQLNode->eNodeType = SQL_OPERATOR_NODE; + pSQLNode->nd.op.eOperator = eOperator; + pSQLNode->nd.op.uiCompareRules = uiCompareRules; + pSQLNode->uiNestLevel = m_pCurrParseState->uiNestLevel; + + // Go up the stack until an operator whose nest level or precedence is < + // this one's is encountered, then link this one in as the last child + + pParentNode = m_pCurrParseState->pCurOperatorNode; + while (pParentNode && + (pParentNode->uiNestLevel > pSQLNode->uiNestLevel || + (pParentNode->uiNestLevel == pSQLNode->uiNestLevel && + getSQLOpPrecedence( pParentNode->nd.op.eOperator) >= + getSQLOpPrecedence( eOperator)))) + { + pParentNode = pParentNode->pParent; + } + if (!pParentNode) + { + if (m_pCurrParseState->pRootNode) + { + sqlLinkLastChild( pSQLNode, m_pCurrParseState->pRootNode); + } + m_pCurrParseState->pRootNode = pSQLNode; + } + else if (eOperator == SQL_NOT_OP || eOperator == SQL_NEG_OP) + { + + // Need to treat NOT and NEG as if they were operands. + + // Parent better be an operator. + + flmAssert( pParentNode->eNodeType == SQL_OPERATOR_NODE); + +#ifdef FLM_DEBUG + if (pParentNode->nd.op.eOperator == SQL_NEG_OP || + pParentNode->nd.op.eOperator == SQL_NOT_OP) + { + + // Must have no children. + + flmAssert( pParentNode->pFirstChild == NULL); + } + else + { + + // Must only have one or zero children. + + flmAssert( pParentNode->pFirstChild == pParentNode->pLastChild); + } +#endif + + sqlLinkLastChild( pParentNode, pSQLNode); + flmAssert( !m_pCurrParseState->bExpectingOperator); + } + else + { + + // Parent better be an operator. + + flmAssert( pParentNode->eNodeType == SQL_OPERATOR_NODE); + + // Unlink last child of parent node and replace with this + // new node. The parent node better already have the correct + // number of children, or we are not parsing correctly. + + flmAssert( pParentNode->pFirstChild); + if (pParentNode->nd.op.eOperator == SQL_NEG_OP || + pParentNode->nd.op.eOperator == SQL_NOT_OP) + { + + // Better only be one child. + + flmAssert( !pParentNode->pFirstChild->pNextSib); + + sqlLinkLastChild( pSQLNode, pParentNode->pFirstChild); + } + else + { + + // Better only be two child nodes + + flmAssert( pParentNode->pFirstChild->pNextSib == + pParentNode->pLastChild); + sqlLinkLastChild( pSQLNode, pParentNode->pLastChild); + } + sqlLinkLastChild( pParentNode, pSQLNode); + } + + m_pCurrParseState->pCurOperatorNode = pSQLNode; + m_pCurrParseState->bExpectingOperator = FALSE; + m_pCurrParseState->pLastNode = pSQLNode; + +Exit: + + return( rc); +} + +//------------------------------------------------------------------------- +// Desc: Add a column name. +//------------------------------------------------------------------------- +RCODE SQLQuery::addColumn( + FLMUINT uiTableNum, + FLMUINT uiColumnNum) +{ + RCODE rc = NE_SFLM_OK; + SQL_NODE * pSQLNode; + SQL_TABLE * pTable; + + if (!m_pCurrParseState) + { + if (RC_BAD( rc = allocParseState())) + { + goto Exit; + } + } + + // Must be expecting an operand + + if (!expectingOperand()) + { + rc = RC_SET( NE_SFLM_Q_UNEXPECTED_COLUMN); + goto Exit; + } + + // Add or find the table structure for the node. + + pTable = m_pFirstTable; + while (pTable && pTable->uiTableNum != uiTableNum) + { + pTable = pTable->pNext; + } + if (!pTable) + { + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_TABLE), + (void **)&pTable))) + { + goto Exit; + } + pTable->uiTableNum = uiTableNum; + if ((pTable->pPrev = m_pLastTable) != NULL) + { + m_pLastTable->pNext = pTable; + } + else + { + m_pFirstTable = pTable; + } + m_pLastTable = pTable; + } + + // Allocate a column node + + if (RC_BAD( rc = m_pool.poolCalloc( sizeof( SQL_NODE), + (void **)&pSQLNode))) + { + goto Exit; + } + pSQLNode->eNodeType = SQL_COLUMN_NODE; + pSQLNode->nd.column.pTable = pTable; + pSQLNode->nd.column.uiColumnNum = uiColumnNum; + + if (m_pCurrParseState->pRootNode) + { + sqlLinkLastChild( m_pCurrParseState->pCurOperatorNode, pSQLNode); + } + else + { + m_pCurrParseState->pRootNode = pSQLNode; + } + m_pCurrParseState->bExpectingOperator = TRUE; + m_pCurrParseState->pLastNode = pSQLNode; + +Exit: + + return( rc); +} + diff --git a/sql/src/translog.cpp b/sql/src/translog.cpp new file mode 100644 index 0000000..9453363 --- /dev/null +++ b/sql/src/translog.cpp @@ -0,0 +1,323 @@ +//------------------------------------------------------------------------------ +// Desc: Routines to handle transaction logging +// +// Tabs: 3 +// +// Copyright (c) 1991-2006 Novell, Inc. All Rights Reserved. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of version 2 of the GNU General Public +// License as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, contact Novell, Inc. +// +// To contact Novell about this file by physical or electronic mail, +// you may find current contact information at www.novell.com +// +// $Id: translog.cpp 3116 2006-01-19 13:31:53 -0700 (Thu, 19 Jan 2006) dsanders $ +//------------------------------------------------------------------------------ + +#include "flaimsys.h" + +FSTATIC void lgWriteComplete( + IF_IOBuffer * pIOBuffer); + +#ifdef FLM_DBG_LOG +/**************************************************************************** +Desc: This routine is used to write out information about logged blocks to + the log file. +****************************************************************************/ +void scaLogWrite( + F_Database * pDatabase, + FLMUINT uiWriteAddress, + FLMBYTE * pucBlkBuf, + FLMUINT uiBufferLen, + FLMUINT uiBlockSize, + char * pszEvent + ) +{ + FLMUINT uiOffset = 0; + FLMUINT32 ui32BlkAddr; + FLMUINT64 ui64TransID; + + while (uiOffset < uiBufferLen) + { + ui32BlkAddr = ((F_BLK_HDR *)pucBlkBuf)->ui32BlkAddr; + ui64TransID = ((F_BLK_HDR *)pucBlkBuf)->ui64TransID; + + // A uiWriteAddress of zero means we are writing exactly at the + // block address - i.e., it is the data block, not the log block. + + flmDbgLogWrite( pDatabase, (FLMUINT)ui32BlkAddr, + (FLMUINT)((uiWriteAddress) + ? uiWriteAddress + uiOffset + : (FLMUINT)ui32BlkAddr), + ui64TransID, pszEvent); + uiOffset += uiBlockSize; + pucBlkBuf += uiBlockSize; + } +} +#endif + +/**************************************************************************** +Desc: This is the callback routine that is called when a disk write is + completed. +****************************************************************************/ +FSTATIC void lgWriteComplete( + IF_IOBuffer * pIOBuffer) +{ + F_Database * pDatabase = + (F_Database *)pIOBuffer->getCompletionCallbackData( 0); +#ifdef FLM_DBG_LOG + FLMUINT uiBlockSize = pDatabase->getBlockSize(); + FLMUINT uiLength = pIOBuffer->getBufferSize(); + char * pszEvent; +#endif + SFLM_DB_STATS * pDbStats = (SFLM_DB_STATS *)pIOBuffer->getStats(); + +#ifdef FLM_DBG_LOG + pszEvent = (char *)(RC_OK( pIOBuffer->getCompletionCode()) + ? (char *)"LGWRT" + : (char *)"LGWRT-FAIL"); + scaLogWrite( pDatabase, 0, pIOBuffer->getBuffer(), uiLength, + uiBlockSize, pszEvent); +#endif + + if (pDbStats) + { + + // Must lock mutex, because this may be called from async write + // completion at any time. + + pDatabase->lockMutex(); + pDbStats->LogBlockWrites.ui64ElapMilli += pIOBuffer->getElapTime(); + pDatabase->unlockMutex(); + } +} + +/**************************************************************************** +Desc: This routine flushes a log buffer to the log file. +****************************************************************************/ +RCODE F_Database::lgFlushLogBuffer( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + FLMBOOL bDoAsync) +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiBytesWritten; + IF_IOBuffer * pAsyncBuffer; + + if (!bDoAsync) + { + pAsyncBuffer = NULL; + } + else + { + pAsyncBuffer = m_pCurrLogBuffer; + } + + if (pDbStats) + { + pDbStats->bHaveStats = TRUE; + pDbStats->LogBlockWrites.ui64Count++; + pDbStats->LogBlockWrites.ui64TotalBytes += m_uiCurrLogWriteOffset; + } + + m_pCurrLogBuffer->setCompletionCallback( lgWriteComplete); + m_pCurrLogBuffer->setCompletionCallbackData( 0, (void *)this); + pSFileHdl->setMaxAutoExtendSize( m_uiMaxFileSize); + pSFileHdl->setExtendSize( m_uiFileExtendSize); + m_pCurrLogBuffer->startTimer( pDbStats); + + // NOTE: No guarantee that m_pCurrLogBuffer will still be around + // after the call to WriteBlock, unless we are doing + // non-asynchronous write. + + rc = pSFileHdl->writeBlock( m_uiCurrLogBlkAddr, + m_uiCurrLogWriteOffset, + m_pCurrLogBuffer->getBuffer(), + m_pCurrLogBuffer->getBufferSize(), + pAsyncBuffer, &uiBytesWritten); + if (!pAsyncBuffer) + { + m_pCurrLogBuffer->notifyComplete( rc); + } + m_pCurrLogBuffer = NULL; + + if (RC_BAD( rc)) + { + if (pDbStats) + { + pDbStats->uiWriteErrors++; + } + goto Exit; + } + +Exit: + + m_uiCurrLogWriteOffset = 0; + m_pCurrLogBuffer = NULL; + return( rc); +} + +/**************************************************************************** +Desc: This routine writes a block to the log file. +****************************************************************************/ +RCODE F_Database::lgOutputBlock( + SFLM_DB_STATS * pDbStats, + F_SuperFileHdl * pSFileHdl, + F_CachedBlock * pLogBlock, // Cached log block. + F_BLK_HDR * pBlkHdr, // Pointer to the corresponding modified + // block in cache. This block will be + // modified to the logged version of + // the block + FLMBOOL bDoAsync, // Do asynchronous writes? + FLMUINT * puiLogEofRV) // Returns log EOF +{ + RCODE rc = NE_SFLM_OK; + FLMUINT uiFilePos = *puiLogEofRV; + FLMBYTE * pucLogBlk; + F_BLK_HDR * pLogBlkHdr; + FLMUINT uiBlkAddress; + FLMUINT uiLogBufferSize; + + // Time for a new block file? + + if (FSGetFileOffset( uiFilePos) >= m_uiMaxFileSize) + { + FLMUINT uiFileNumber; + + // Write out the current buffer, if it has anything in it. + + if (m_uiCurrLogWriteOffset) + { + if (RC_BAD( rc = lgFlushLogBuffer( pDbStats, pSFileHdl, + bDoAsync))) + { + goto Exit; + } + } + + uiFileNumber = FSGetFileNumber( uiFilePos); + + if (!uiFileNumber) + { + uiFileNumber = FIRST_LOG_BLOCK_FILE_NUMBER; + } + else + { + uiFileNumber++; + } + + if (uiFileNumber > MAX_LOG_BLOCK_FILE_NUMBER) + { + rc = RC_SET( NE_SFLM_DB_FULL); + goto Exit; + } + + if (RC_BAD( rc = pSFileHdl->createFile( uiFileNumber ))) + { + goto Exit; + } + uiFilePos = FSBlkAddress( uiFileNumber, 0 ); + } + + // Copy the log block to the log buffer. + + if (!m_uiCurrLogWriteOffset) + { + m_uiCurrLogBlkAddr = uiFilePos; + + // Get a buffer for logging. + // NOTE: Buffers are not kept by the F_Database's buffer manager, + // so once we are done with this buffer, it will be freed. + + uiLogBufferSize = MAX_LOG_BUFFER_SIZE; + + for( ;;) + { + if (RC_BAD( rc = m_pBufferMgr->getBuffer( + &m_pCurrLogBuffer, uiLogBufferSize, uiLogBufferSize))) + { + // If we failed to get a buffer of the requested size, + // reduce the buffer size by half and try again. + + if( rc == NE_SFLM_MEM) + { + uiLogBufferSize /= 2; + if( uiLogBufferSize < m_uiBlockSize) + { + goto Exit; + } + rc = NE_SFLM_OK; + continue; + } + goto Exit; + } + break; + } + } + + // Copy data from log block to the log buffer + + pucLogBlk = m_pCurrLogBuffer->getBuffer() + m_uiCurrLogWriteOffset; + pLogBlkHdr = (F_BLK_HDR *)pucLogBlk; + f_memcpy( pLogBlkHdr, pLogBlock->m_pBlkHdr, m_uiBlockSize); + + // If we are logging this block for the current update + // transaction, set the BEFORE IMAGE (BI) flag in the block header + // so we will know that this block is a before image block that + // needs to be restored when aborting the current update + // transaction. + + if (pLogBlock->m_ui16Flags & CA_WRITE_TO_LOG) + { + pLogBlkHdr->ui8BlkFlags |= BLK_IS_BEFORE_IMAGE; + } + uiBlkAddress = (FLMUINT)pLogBlkHdr->ui32BlkAddr; + + // Encrypt the block if needed + + if (RC_BAD( rc = encryptBlock( m_pDictList, + (FLMBYTE *)pLogBlkHdr))) + { + goto Exit; + } + + if (RC_BAD( rc = flmPrepareBlockToWrite( m_uiBlockSize, pLogBlkHdr))) + { + goto Exit; + } + + // Set up for next log block write + + m_uiCurrLogWriteOffset += m_uiBlockSize; + + // If this log buffer is full, write it out. + + if (m_uiCurrLogWriteOffset == m_pCurrLogBuffer->getBufferSize()) + { + if (RC_BAD( rc = lgFlushLogBuffer( pDbStats, pSFileHdl, + bDoAsync))) + { + goto Exit; + } + } + + // Save the previous block address into the modified block's + // block header area. Also save the transaction id. + + pBlkHdr->ui32PriorBlkImgAddr = (FLMUINT32)uiFilePos; + + *puiLogEofRV = uiFilePos + m_uiBlockSize; + +Exit: + + return( rc); +}