diff --git a/ftk/Makefile b/ftk/Makefile index 7a896f7..bc22666 100644 --- a/ftk/Makefile +++ b/ftk/Makefile @@ -271,6 +271,12 @@ ifndef host_os_family endif endif +ifndef host_os_family + ifneq (,$(findstring HP-UX,$(OSTYPE))) + host_os_family = hpux + endif +endif + ifndef host_os_family $(error Host operating system could not be determined. You may need to export OSTYPE from the environment.) endif @@ -391,19 +397,22 @@ ifeq (,$(HOSTTYPE)) endif ifeq (,$(HOSTTYPE)) - HOSTTYPE := $(shell uname -p) - ifneq (,$(findstring nvalid,$(HOSTTYPE))) - HOSTTYPE := $(shell uname -m) + ifneq ($(host_os_family),hpux) + HOSTTYPE := $(shell uname -p) endif - ifneq (,$(findstring unknown,$(HOSTTYPE))) + ifeq (,$(HOSTTYPE)) HOSTTYPE := $(shell uname -m) + else + ifneq (,$(findstring nvalid,$(HOSTTYPE))) + HOSTTYPE := $(shell uname -m) + else + ifneq (,$(findstring unknown,$(HOSTTYPE))) + HOSTTYPE := $(shell uname -m) + endif + endif endif endif -ifeq (,$(HOSTTYPE)) - HOSTYPE := $(shell uname -p) -endif - ifeq (,$(HOSTTYPE)) $(error HOSTTYPE environment variable has not been set) endif @@ -952,9 +961,19 @@ ifdef unix_target ifeq ($(target_os_family),hpux) ifeq ($(usenativecc),yes) - # Must support 64 bit file sizes - even for 32 bit builds. - - ccdefs += _LARGEFILE_SOURCE _FILE_OFFSET_BITS=64 + + # Disable "Placement operator delete invocation is not yet + # implemented" warning + + ccflags += +W930 + + ifeq ($(target_word_size),64) + ccflags += +DD64 + else + # Must support 64 bit file sizes - even for 32 bit builds. + + ccdefs += _LARGEFILE_SOURCE _FILE_OFFSET_BITS=64 + endif endif endif @@ -1031,6 +1050,9 @@ ifdef unix_target endif ifeq ($(target_os_family),hpux) + ifeq ($(target_word_size),64) + link_flags += +DD64 + endif link_libs += -lm -lc -lrt -lcurses endif diff --git a/ftk/src/ftk.h b/ftk/src/ftk.h index 3f0bad5..44f8e3a 100644 --- a/ftk/src/ftk.h +++ b/ftk/src/ftk.h @@ -148,6 +148,8 @@ #define FLM_UNIX #define FLM_BIG_ENDIAN #define FLM_STRICT_ALIGNMENT + #define FLM_HAS_ASYNC_IO + #define FLM_HAS_DIRECT_IO #elif defined( __APPLE__) #define FLM_OSX #define FLM_OSTYPE_STR "OSX" @@ -178,7 +180,17 @@ #error Cannot define both FLM_32BIT and FLM_64BIT #endif - #ifdef FLM_NLM + #if defined( __x86_64__) || defined( _LP64) || defined( __LP64__) || defined( __sparcv9) + #if !defined( FLM_64BIT) + #error Platform word size is incorrect + #endif + #else + #if !defined( FLM_32BIT) + #error Platform word size is incorrect + #endif + #endif + + #ifdef FLM_NLM #define FSTATIC #else #define FSTATIC static @@ -1639,9 +1651,12 @@ virtual RCODE FLMAPI tell( FLMUINT64 * pui64Offset) = 0; + + virtual RCODE FLMAPI extendFile( + FLMUINT64 ui64FileSize) = 0; virtual RCODE FLMAPI truncateFile( - FLMUINT64 ui64Offset = 0) = 0; + FLMUINT64 ui64FileSize = 0) = 0; virtual RCODE FLMAPI closeFile( void) = 0; @@ -1756,6 +1771,8 @@ virtual RCODE FLMAPI getFilePath( FLMUINT uiFileNumber, char * pszPath) = 0; + + virtual FLMUINT64 FLMAPI getMaxFileSize( void) = 0; }; /**************************************************************************** @@ -1811,6 +1828,10 @@ RCODE FLMAPI releaseFiles( void); + RCODE FLMAPI allocateBlocks( + FLMUINT uiStartAddress, + FLMUINT uiEndAddress); + RCODE FLMAPI truncateFile( FLMUINT uiEOFBlkAddress); @@ -1824,12 +1845,6 @@ RCODE FLMAPI flush( void); - FINLINE void FLMAPI setBlockSize( - FLMUINT uiBlockSize) - { - m_uiBlockSize = uiBlockSize; - } - FINLINE void FLMAPI setExtendSize( FLMUINT uiExtendSize) { @@ -1855,7 +1870,7 @@ FLMBOOL m_bCFileDirty; FLMBOOL m_bBlockFileDirty; FLMUINT m_uiBlockFileNum; - FLMUINT m_uiBlockSize; + FLMUINT m_uiMaxFileSize; FLMUINT m_uiExtendSize; FLMUINT m_uiMaxAutoExtendSize; FLMUINT m_uiFileOpenFlags; diff --git a/ftk/src/ftkfsys.cpp b/ftk/src/ftkfsys.cpp index a99c318..bb3bfd4 100644 --- a/ftk/src/ftkfsys.cpp +++ b/ftk/src/ftkfsys.cpp @@ -1424,10 +1424,37 @@ Exit: return( rc); -#else +#elif defined( DEV_BSIZE) F_UNREFERENCED_PARM( pszFileName); - *puiSectorSize = (FLMUINT)sysconf( _SC_PAGESIZE); + *puiSectorSize = (FLMUINT)DEV_BSIZE; return( NE_FLM_OK); +#else + + int hFile = -1; + struct stat filestats; + + if( (hFile = open( pszFileName, O_RDONLY, 0600)) == -1) + { + rc = f_mapPlatformError( errno, NE_FLM_OPENING_FILE); + goto Exit; + } + + if( fstat( hFile, &filestats) != 0) + { + rc = f_mapPlatformError( errno, NE_FLM_OPENING_FILE); + goto Exit; + } + + *puiSectorSize = (FLMUINT)filestats.st_blksize; + +Exit: + + if( hFile != -1) + { + close( hFile); + } + + return( rc); #endif } @@ -3130,44 +3157,48 @@ RCODE FLMAPI F_FileAsyncClient::waitToComplete( uiBytesDone = (FLMUINT)udBytesDone; #endif -#if defined( FLM_UNIX) && defined( FLM_HAS_ASYNC_IO) - FLMINT iAsyncResult; - const struct aiocb * ppAio[ 1]; - - ppAio[ 0] = &m_aio; - - for( ;;) +#if defined( FLM_UNIX) + #if defined( FLM_HAS_ASYNC_IO) { -#ifdef FLM_AIX - aio_suspend( 1, ppAio); -#else - aio_suspend( ppAio, 1, NULL); -#endif - iAsyncResult = aio_error( &m_aio); - - if( !iAsyncResult) + FLMINT iAsyncResult; + const struct aiocb * ppAio[ 1]; + + ppAio[ 0] = &m_aio; + + for( ;;) { - if( (iAsyncResult = aio_return( &m_aio)) < 0) + #ifdef FLM_AIX + aio_suspend( 1, ppAio); + #else + aio_suspend( ppAio, 1, NULL); + #endif + iAsyncResult = aio_error( &m_aio); + + if( !iAsyncResult) { - f_assert( 0); - completionRc = f_mapPlatformError( errno, NE_FLM_ASYNC_FAILED); - } - else - { - uiBytesDone = (FLMUINT)iAsyncResult; + if( (iAsyncResult = aio_return( &m_aio)) < 0) + { + f_assert( 0); + completionRc = f_mapPlatformError( errno, NE_FLM_ASYNC_FAILED); + } + else + { + uiBytesDone = (FLMUINT)iAsyncResult; + } + + break; } - break; + if( iAsyncResult == EINTR || iAsyncResult == EINPROGRESS) + { + continue; + } + + f_assert( 0); + completionRc = f_mapPlatformError( iAsyncResult, NE_FLM_ASYNC_FAILED); } - - if( iAsyncResult == EINTR || iAsyncResult == EINPROGRESS) - { - continue; - } - - f_assert( 0); - completionRc = f_mapPlatformError( iAsyncResult, NE_FLM_ASYNC_FAILED); } + #endif #endif #ifdef FLM_RING_ZERO_NLM diff --git a/ftk/src/ftkftx.cpp b/ftk/src/ftkftx.cpp index e43172f..66d9b27 100644 --- a/ftk/src/ftkftx.cpp +++ b/ftk/src/ftkftx.cpp @@ -280,7 +280,9 @@ typedef struct FTX_INFO #elif defined( FLM_UNIX) #if defined( FLM_HPUX) || defined( FLM_OSF) - #define _XOPEN_CURSES + #ifndef _XOPEN_CURSES + #define _XOPEN_CURSES + #endif #define _XOPEN_SOURCE_EXTENDED 1 #endif diff --git a/ftk/src/ftkmem.cpp b/ftk/src/ftkmem.cpp index fa615b2..a937872 100644 --- a/ftk/src/ftkmem.cpp +++ b/ftk/src/ftkmem.cpp @@ -3101,7 +3101,7 @@ void * F_FixedAlloc::getCell( #endif if (!m_pDefaultRelocator) { - ((CELLHEADER2 *)pHeader)->pRelocator = pRelocator; + ((CELLHEADER2 *)((void *)pHeader))->pRelocator = pRelocator; } if( m_pUsageStats) @@ -3621,7 +3621,7 @@ void F_FixedAlloc::defragmentMemory( void) (uiLoop * m_uiSizeOfCellAndHeader)); if ((pRelocator = m_pDefaultRelocator) == NULL) { - pRelocator = ((CELLHEADER2 *)pCellHeader)->pRelocator; + pRelocator = ((CELLHEADER2 *)((void *)pCellHeader))->pRelocator; } if( pCellHeader->pContainingSlab) @@ -4790,6 +4790,10 @@ RCODE FLMAPI f_allocAlignedBufferImp( goto Exit; } +#elif defined( FLM_RING_ZERO_NLM) + + pSlab = Alloc( m_uiSlabSize, gv_lAllocRTag) + #elif defined( FLM_SOLARIS) if( (*ppvAlloc = memalign( sysconf( _SC_PAGESIZE), uiMinSize)) == NULL) @@ -4822,7 +4826,7 @@ RCODE FLMAPI f_allocAlignedBufferImp( FLMBYTE * pucAlloc; FLMBYTE * pucStartOfAlloc; - uiAllocSize = f_roundUp( uiMinSize, uiPageSize) + uiPageSize; + uiAllocSize = (FLMUINT)(f_roundUp( uiMinSize, uiPageSize) + uiPageSize); if( RC_BAD( rc = f_alloc( uiAllocSize, &pucAlloc))) { @@ -4861,6 +4865,11 @@ void FLMAPI f_freeAlignedBufferImp( (void)VirtualFree( *ppvAlloc, 0, MEM_RELEASE); *ppvAlloc = NULL; +#elif defined( FLM_RING_ZERO_NLM) + + Free( *ppvAlloc) + *ppvAlloc = NULL; + #elif defined( FLM_UNIX) free( *ppvAlloc); @@ -4932,51 +4941,17 @@ RCODE FLMAPI f_getMemoryInfo( FLMUINT uiProcMemLimit = FLM_MAX_UINT; FLMUINT uiProcVMemLimit = FLM_MAX_UINT; - #if defined( RLIMIT_VMEM) - { - struct rlimit rlim; - - // Bump the process soft virtual limit up to the hard limit - - if( getrlimit( RLIMIT_VMEM, &rlim) != 0) - { - rlim.rlim_cur = RLIM_INFINITY; - rlim.rlim_max = RLIM_INFINITY; - } - - if( rlim.rlim_cur != RLIM_INFINITY) - { - uiProcVMemLimit = (FLMUINT)rlim.rlim_cur; - } - } - #endif - - #if defined( RLIMIT_DATA) - { - struct rlimit rlim; - - // Bump the process soft heap limit up to the hard limit - - if( getrlimit( RLIMIT_DATA, &rlim) != 0) - { - rlim.rlim_cur = RLIM_INFINITY; - rlim.rlim_max = RLIM_INFINITY; - } - - if( rlim.rlim_cur != RLIM_INFINITY) - { - uiProcMemLimit = (FLMUINT)rlim.rlim_cur; - } - } - #endif - - #ifdef FLM_AIX + #if defined( FLM_LINUX) + + f_getLinuxMemInfo( &ui64TotalPhysMem, &ui64AvailPhysMem); + + #elif defined( FLM_AIX) f_getAIXMemInfo( &ui64TotalPhysMem, &ui64AvailPhysMem); - #elif defined( FLM_LINUX) + #elif defined( FLM_HPUX) - f_getLinuxMemInfo( &ui64TotalPhysMem, &ui64AvailPhysMem); + f_getHPUXMemInfo( &ui64TotalPhysMem, &ui64AvailPhysMem); #elif defined( _SC_PAGESIZE) && defined( _SC_AVPHYS_PAGES) @@ -4996,6 +4971,40 @@ RCODE FLMAPI f_getMemoryInfo( // The process might be limited in the amount of memory it // can access. + #if defined( RLIMIT_VMEM) + { + struct rlimit rlim; + + if( getrlimit( RLIMIT_VMEM, &rlim) != 0) + { + rlim.rlim_cur = RLIM_INFINITY; + rlim.rlim_max = RLIM_INFINITY; + } + + if( rlim.rlim_cur != RLIM_INFINITY) + { + uiProcVMemLimit = (FLMUINT)rlim.rlim_cur; + } + } + #endif + + #if defined( RLIMIT_DATA) + { + struct rlimit rlim; + + if( getrlimit( RLIMIT_DATA, &rlim) != 0) + { + rlim.rlim_cur = RLIM_INFINITY; + rlim.rlim_max = RLIM_INFINITY; + } + + if( rlim.rlim_cur != RLIM_INFINITY) + { + uiProcMemLimit = (FLMUINT)rlim.rlim_cur; + } + } + #endif + if( ui64TotalPhysMem > uiProcMemLimit) { ui64TotalPhysMem = uiProcMemLimit; @@ -5227,6 +5236,40 @@ Exit: } #endif +/*************************************************************************** +Desc: +***************************************************************************/ +#ifdef FLM_HPUX +void f_getHPUXMemInfo( + FLMUINT64 * pui64TotalMem, + FLMUINT64 * pui64AvailMem) +{ + FLMUINT uiPageSize; + struct pst_static pst; + struct pst_dynamic dyn; + + if( pstat_getstatic( &pst, sizeof( pst), (size_t)1, 0) == -1) + { + uiPageSize = 4096; + } + else + { + uiPageSize = pst.page_size; + } + + if( pstat_getdynamic( &dyn, sizeof( dyn), 1, 0) != -1) + { + *pui64AvailMem = dyn.psd_free * uiPageSize; + } + else + { + *pui64AvailMem = pst.physical_memory * uiPageSize; + } + + *pui64TotalMem = pst.physical_memory * uiPageSize; +} +#endif + /**************************************************************************** Desc: ****************************************************************************/ diff --git a/ftk/src/ftkmisc.cpp b/ftk/src/ftkmisc.cpp index d743b6a..93ffeb0 100644 --- a/ftk/src/ftkmisc.cpp +++ b/ftk/src/ftkmisc.cpp @@ -107,8 +107,16 @@ RCODE FLMAPI ftkStartup( void) // Sanity check -- make sure we are using the correct // byte-swap macros for this platform - flmAssert( FB2UD( (FLMBYTE *)"\x0A\x0B\x0C\x0D") == 0x0D0C0B0A); - flmAssert( FB2UW( (FLMBYTE *)"\x0A\x0B") == 0x0B0A); + f_assert( FB2UD( (FLMBYTE *)"\x0A\x0B\x0C\x0D") == 0x0D0C0B0A); + f_assert( FB2UW( (FLMBYTE *)"\x0A\x0B") == 0x0B0A); + + // Verify that the platform word size is correct + +#ifdef FLM_64BIT + f_assert( sizeof( FLMUINT) == 8); +#else + f_assert( sizeof( FLMUINT) == 4); +#endif #if defined( FLM_RING_ZERO_NLM) if( RC_BAD( rc = f_netwareStartup())) @@ -1144,7 +1152,7 @@ Notes: This is a base 24 alphanumeric value where FLMBYTE FLMAPI f_getBase24DigitChar( FLMBYTE ucValue) { - flmAssert( ucValue <= 23); + f_assert( ucValue <= 23); if( ucValue <= 9) { @@ -1629,7 +1637,7 @@ RCODE FLMAPI f_decodeSEN64( default: *pui64Value = 0; - flmAssert( 0); + f_assert( 0); break; } } @@ -1717,7 +1725,7 @@ FLMUINT FLMAPI f_encodeSEN( FLMBYTE * pucBuffer = *ppucBuffer; FLMUINT uiSenLen = f_getSENByteCount( ui64Value); - flmAssert( uiSizeWanted <= FLM_MAX_SEN_LEN && + f_assert( uiSizeWanted <= FLM_MAX_SEN_LEN && (!uiSizeWanted || uiSizeWanted >= uiSenLen)); uiSenLen = uiSizeWanted > uiSenLen ? uiSizeWanted : uiSenLen; @@ -1861,7 +1869,7 @@ void F_ListManager::insertFirst( { F_ListNode * pListNode; - flmAssert( uiList < m_uiListNodeCnt); + f_assert( uiList < m_uiListNodeCnt); pNewFirstItem->AddRef(); pListNode = &m_pListNodes[ uiList]; @@ -1894,7 +1902,7 @@ void F_ListManager::insertLast( { F_ListNode * pListNode; - flmAssert( uiList < m_uiListNodeCnt); + f_assert( uiList < m_uiListNodeCnt); pNewLastItem->AddRef(); pListNode = &m_pListNodes[ uiList]; @@ -1930,7 +1938,7 @@ F_ListItem * F_ListManager::getItem( // Check bounds with assert. - flmAssert( uiList < m_uiListNodeCnt ); + f_assert( uiList < m_uiListNodeCnt ); pListNode = &m_pListNodes[ uiList ]; pListItem = pListNode ? pListNode->pNextItem : NULL; @@ -1954,7 +1962,7 @@ void F_ListManager::removeItem( F_ListItem * pPrevItem; F_ListItem * pNextItem; - flmAssert( uiList < m_uiListNodeCnt); + f_assert( uiList < m_uiListNodeCnt); pMgrListNode = &m_pListNodes[ uiList]; @@ -2020,7 +2028,7 @@ void F_ListManager::clearList( FLMUINT uiListCnt; F_ListNode * pListNode; - flmAssert( (FLM_ALL_LISTS == uiList) || (uiList < m_uiListNodeCnt)); + f_assert( (FLM_ALL_LISTS == uiList) || (uiList < m_uiListNodeCnt)); if( uiList == FLM_ALL_LISTS) { @@ -2049,7 +2057,7 @@ void F_ListManager::clearList( // At this point the ListCount should be at 0. - flmAssert( !pListNode->uiListCount); + f_assert( !pListNode->uiListCount); // Clear the managers head and tail list pointers. @@ -2067,7 +2075,7 @@ FLMUINT F_ListManager::getItemCount( FLMUINT uiCount = 0; F_ListNode * pListNode; - flmAssert( (FLM_ALL_LISTS == uiList) || (uiList < m_uiListNodeCnt)); + f_assert( (FLM_ALL_LISTS == uiList) || (uiList < m_uiListNodeCnt)); if( uiList == FLM_ALL_LISTS) { @@ -2097,12 +2105,12 @@ F_ListItem::~F_ListItem() FLMUINT uiLoop; F_ListNode * pTmpNd; - flmAssert( !m_bInList); + f_assert( !m_bInList); for( uiLoop = 0; uiLoop < m_uiListNodeCnt; uiLoop++) { pTmpNd = &m_pListNodes[ uiLoop]; - flmAssert( !pTmpNd->pPrevItem && !pTmpNd->pNextItem); + f_assert( !pTmpNd->pPrevItem && !pTmpNd->pNextItem); } #endif } @@ -2115,9 +2123,9 @@ void F_ListItem::setup( F_ListNode * pListNodes, FLMUINT uiListNodeCnt) { - flmAssert( pListMgr); - flmAssert( pListNodes); - flmAssert( uiListNodeCnt); + f_assert( pListMgr); + f_assert( pListNodes); + f_assert( uiListNodeCnt); m_pListManager = pListMgr; m_uiListNodeCnt = uiListNodeCnt; @@ -2133,7 +2141,7 @@ void F_ListItem::removeFromList( FLMUINT uiList) { - flmAssert( (uiList < m_uiListNodeCnt) || (uiList == FLM_ALL_LISTS)); + f_assert( (uiList < m_uiListNodeCnt) || (uiList == FLM_ALL_LISTS)); if( uiList == FLM_ALL_LISTS) { @@ -2331,7 +2339,7 @@ RCODE FLMAPI F_HashTable::setupHashTable( { RCODE rc = NE_FLM_OK; - flmAssert( uiNumBuckets); + f_assert( uiNumBuckets); // Create the hash table @@ -2433,11 +2441,11 @@ RCODE FLMAPI F_HashTable::addObject( // Calculate and set the objects hash bucket - flmAssert( pObject->getHashBucket() == F_INVALID_HASH_BUCKET); + f_assert( pObject->getHashBucket() == F_INVALID_HASH_BUCKET); pvKey = pObject->getKey(); uiKeyLen = pObject->getKeyLength(); - flmAssert( uiKeyLen); + f_assert( uiKeyLen); uiBucket = getHashBucket( pvKey, uiKeyLen, &ui32CRC); pObject->m_ui32KeyCRC = ui32CRC; @@ -2907,7 +2915,7 @@ void F_HashTable::unlinkObject( // Is the bucket valid? - flmAssert( uiBucket < m_uiBuckets); + f_assert( uiBucket < m_uiBuckets); // Unlink from the hash bucket diff --git a/ftk/src/ftkstrm.cpp b/ftk/src/ftkstrm.cpp index e76c5cd..c1d6250 100644 --- a/ftk/src/ftkstrm.cpp +++ b/ftk/src/ftkstrm.cpp @@ -2782,7 +2782,7 @@ RCODE F_TCPStream::openConnection( for( iTries = 0; iTries < iMaxTries; iTries++ ) { iSockErr = 0; - if( connect( m_iSocket, (struct sockaddr *)&address, + if( connect( m_iSocket, (struct sockaddr *)((void *)&address), (unsigned)sizeof(struct sockaddr)) >= 0) { break; diff --git a/ftk/src/ftksupr.cpp b/ftk/src/ftksupr.cpp index de6969d..74e40a5 100644 --- a/ftk/src/ftksupr.cpp +++ b/ftk/src/ftksupr.cpp @@ -36,9 +36,8 @@ F_SuperFileHdl::F_SuperFileHdl( void) m_uiBlockFileNum = 0; m_bBlockFileDirty = FALSE; m_bCFileDirty = FALSE; - m_uiBlockSize = 0; m_uiExtendSize = (8 * 1024 * 1024); - m_uiMaxAutoExtendSize = f_getMaxFileSize(); + m_uiMaxAutoExtendSize = 0; m_uiFileOpenFlags = 0; m_uiFileCreateFlags = 0; } @@ -111,6 +110,7 @@ RCODE FLMAPI F_SuperFileHdl::setup( m_uiFileOpenFlags = uiFileOpenFlags; m_uiFileCreateFlags = uiFileCreateFlags; + m_uiMaxAutoExtendSize = f_getMaxFileSize(); Exit: @@ -127,8 +127,6 @@ RCODE FLMAPI F_SuperFileHdl::createFile( RCODE rc = NE_FLM_OK; char szFilePath[ F_PATH_MAX_SIZE]; IF_FileHdl * pFileHdl = NULL; - - f_assert( m_uiBlockSize); // If the file creation flags are not set we won't allow this operation // to continue @@ -165,6 +163,14 @@ RCODE FLMAPI F_SuperFileHdl::createFile( { goto Exit; } + + pFileHdl->Release(); + pFileHdl = NULL; + + if( RC_BAD( rc = getFileHdl( uiFileNumber, TRUE, &pFileHdl))) + { + goto Exit; + } } else { @@ -202,8 +208,6 @@ RCODE FLMAPI F_SuperFileHdl::readBlock( RCODE rc = NE_FLM_OK; IF_FileHdl * pFileHdl = NULL; - f_assert( m_uiBlockSize); - if( RC_BAD( rc = getFileHdl( m_pSuperFileClient->getFileNumber( uiBlkAddress), FALSE, &pFileHdl))) { @@ -238,8 +242,6 @@ RCODE F_SuperFileHdl::writeBlock( RCODE rc = NE_FLM_OK; IF_FileHdl * pFileHdl = NULL; - f_assert( m_uiBlockSize); - if( RC_BAD( rc = getFileHdl( m_pSuperFileClient->getFileNumber( uiBlkAddress), TRUE, &pFileHdl))) { @@ -296,8 +298,6 @@ RCODE F_SuperFileHdl::writeBlock( RCODE rc = NE_FLM_OK; IF_FileHdl * pFileHdl = NULL; - f_assert( m_uiBlockSize); - if( RC_BAD( rc = getFileHdl( m_pSuperFileClient->getFileNumber( uiBlkAddress), TRUE, &pFileHdl))) { @@ -416,6 +416,77 @@ Exit: return( rc); } +/**************************************************************************** +Desc: Extends to an end of file block address. +****************************************************************************/ +RCODE FLMAPI F_SuperFileHdl::allocateBlocks( + FLMUINT uiStartAddress, + FLMUINT uiEndAddress) +{ + RCODE rc = NE_FLM_OK; + FLMUINT uiStartFile; + FLMUINT uiEndFile; + FLMUINT uiEndOffset; + FLMUINT uiCurrentFile; + IF_FileHdl * pFileHdl = NULL; + + uiStartFile = m_pSuperFileClient->getFileNumber( uiStartAddress); + uiCurrentFile = uiStartFile; + + uiEndFile = m_pSuperFileClient->getFileNumber( uiEndAddress); + uiEndOffset = m_pSuperFileClient->getFileOffset( uiEndAddress); + + for( ;;) + { + if( uiCurrentFile > uiEndFile) + { + break; + } + + if( RC_BAD( rc = getFileHdl( uiCurrentFile, TRUE, &pFileHdl))) + { + if( rc != NE_FLM_IO_PATH_NOT_FOUND) + { + goto Exit; + } + + if( RC_BAD( rc = createFile( uiCurrentFile, &pFileHdl))) + { + goto Exit; + } + } + + if( uiCurrentFile == uiEndFile) + { + if( RC_BAD( rc = pFileHdl->extendFile( uiEndOffset))) + { + goto Exit; + } + } + else + { + if( RC_BAD( rc = pFileHdl->extendFile( + m_pSuperFileClient->getMaxFileSize()))) + { + goto Exit; + } + } + + pFileHdl->Release(); + pFileHdl = NULL; + uiCurrentFile++; + } + +Exit: + + if( pFileHdl) + { + pFileHdl->Release(); + } + + return( rc); +} + /**************************************************************************** Desc: Truncates back to an end of file block address. ****************************************************************************/ diff --git a/ftk/src/ftksys.h b/ftk/src/ftksys.h index 161fdaa..2ba6dde 100644 --- a/ftk/src/ftksys.h +++ b/ftk/src/ftksys.h @@ -121,15 +121,35 @@ #include "config.h" #endif + #ifdef FLM_OSX + #include + #include + #include + #include + #endif + + #ifdef FLM_SOLARIS + #include + #include + #endif + #ifdef FLM_AIX #ifndef _LARGE_FILES #define _LARGE_FILES #endif #include + #include #include #include #endif + #ifdef FLM_HPUX + #include + #include + #include + #include + #endif + #include #include #include @@ -159,22 +179,6 @@ #include #include - #ifdef FLM_AIX - #include - #endif - - #ifdef FLM_OSX - #include - #include - #include - #include - #endif - - #ifdef FLM_SOLARIS - #include - #include - #endif - typedef int SOCKET; #define INVALID_SOCKET -1 @@ -604,9 +608,12 @@ RCODE FLMAPI tell( FLMUINT64 * pui64Offset); + + RCODE FLMAPI extendFile( + FLMUINT64 ui64FileSize); RCODE FLMAPI truncateFile( - FLMUINT64 ui64Offset = 0); + FLMUINT64 ui64FileSize = 0); RCODE FLMAPI closeFile( void); @@ -738,13 +745,7 @@ FLMUINT64 * pui64CurrFileSize, FLMUINT * puiTotalBytesToExtend); - #if defined( FLM_WIN) - - RCODE extendFile( - FLMUINT64 ui64FileSize, - FLMUINT uiTotalBytesToExtend); - - #elif defined( FLM_RING_ZERO_NLM) + #if defined( FLM_RING_ZERO_NLM) RCODE setup( void); @@ -1197,6 +1198,12 @@ FLMUINT64 * pui64AvailMem); #endif + #if defined( FLM_HPUX) + void f_getHPUXMemInfo( + FLMUINT64 * pui64TotalMem, + FLMUINT64 * pui64AvailMem); + #endif + void f_memoryInit( void); void f_memoryCleanup( void); diff --git a/ftk/src/ftkthrd.cpp b/ftk/src/ftkthrd.cpp index 64f3a9c..559c598 100644 --- a/ftk/src/ftkthrd.cpp +++ b/ftk/src/ftkthrd.cpp @@ -442,12 +442,8 @@ RCODE FLMAPI F_Thread::startThread( unsigned uiThreadId; #endif #if defined( FLM_UNIX) - #if defined( _POSIX_THREADS) - pthread_attr_t thread_attr; - pthread_t uiThreadId; - #else - threadid_p uiThreadId; - #endif + pthread_attr_t thread_attr; + pthread_t uiThreadId; #endif f_assert( fnThread != NULL && m_fnThread == NULL); @@ -574,33 +570,18 @@ RCODE FLMAPI F_Thread::startThread( goto Exit; } #elif defined( FLM_UNIX) - #ifdef _POSIX_THREADS - pthread_attr_init( &thread_attr); - pthread_attr_setdetachstate( &thread_attr, PTHREAD_CREATE_DETACHED); + pthread_attr_init( &thread_attr); + pthread_attr_setdetachstate( &thread_attr, PTHREAD_CREATE_DETACHED); - if (pthread_create( &uiThreadId, &thread_attr, - threadStub, this) != 0) - { - rc = RC_SET( NE_FLM_COULD_NOT_START_THREAD); - goto Exit; - } - #else - m_uiStackSize = f_max( m_uiStackSize, thr_minstack()); - m_uiStackSize = f_max( m_uiStackSize, thr_min_stack()); + if (pthread_create( &uiThreadId, &thread_attr, + threadStub, this) != 0) + { + rc = RC_SET( NE_FLM_COULD_NOT_START_THREAD); + goto Exit; + } - if( thr_create( (void*)NULL, (size_t)uiStackSize, - threadStub, this, (long)0, &uiThreadId) != 0) - { - rc = RC_SET( NE_FLM_COULD_NOT_START_THREAD); - goto Exit; - } - #endif - - m_uiThreadId = (FLMUINT)uiThreadId; - - #ifdef _POSIX_THREADS - pthread_attr_destroy( &thread_attr); - #endif + m_uiThreadId = (FLMUINT)uiThreadId; + pthread_attr_destroy( &thread_attr); #endif // Code is not designed to handle a thread ID of 0 diff --git a/ftk/src/ftkunix.cpp b/ftk/src/ftkunix.cpp index 7311788..892742f 100644 --- a/ftk/src/ftkunix.cpp +++ b/ftk/src/ftkunix.cpp @@ -122,11 +122,10 @@ RCODE F_FileHdl::openOrCreate( bDoDirectIO = (uiIoFlags & FLM_IO_DIRECT) ? TRUE : FALSE; #endif - // HPUX needs this defined to access files larger than 2 GB. The Linux - // man pages *say* it's needed although as of Suse 9.1 it actually - // isn't. Including this flag on Linux anyway just it case... + // The Linux man pages *say* O_LARGEFILE is needed, although as of + // SUSE 9.1 it actually isn't. Including this flag on Linux just it case... -#if defined( FLM_HPUX) || defined( FLM_LINUX) +#if defined( FLM_LINUX) openFlags |= O_LARGEFILE; #endif @@ -168,7 +167,7 @@ RCODE F_FileHdl::openOrCreate( if( bDoDirectIO) { -#if defined( FLM_LINUX) + #if defined( FLM_LINUX) { FLMUINT uiMajor; FLMUINT uiMinor; @@ -189,12 +188,14 @@ RCODE F_FileHdl::openOrCreate( openFlags |= O_NOATIME; } -#elif defined( FLM_AIX) + #elif defined( FLM_AIX) openFlags |= O_DIRECT; bUsingAsync = TRUE; -#elif defined( FLM_SOLARIS) || defined( FLM_OSX) + #elif defined( FLM_HPUX) bUsingAsync = TRUE; -#endif + #elif defined( FLM_SOLARIS) || defined( FLM_OSX) + bUsingAsync = TRUE; + #endif } Retry_Create: @@ -255,6 +256,17 @@ Retry_Create: } #endif +#if defined( FLM_HPUX) + if( bDoDirectIO) + { + if( ioctl( m_fd, VX_SETCACHE, VX_DIRECT) == -1) + { + bDoDirectIO = FALSE; + bUsingAsync = FALSE; + } + } +#endif + #if defined( FLM_OSX) if( bDoDirectIO) { @@ -732,63 +744,10 @@ RCODE F_FileHdl::lowLevelWrite( if( uiTotalBytesToExtend) { - FLMINT iBytesWritten; - FLMUINT uiCurrBytesToExtend; - FLMUINT uiExtendBufferSize; - - uiExtendBufferSize = f_min( uiTotalBytesToExtend, 1024 * 1024); - - for( ;;) + if( RC_BAD( rc = extendFile( ui64CurrFileSize + uiTotalBytesToExtend))) { - if( RC_OK( rc = f_allocAlignedBuffer( - uiExtendBufferSize, &pucExtendBuffer))) - { - break; - } - - if( uiExtendBufferSize <= (32 * 1024)) - { - goto Exit; - } - - uiExtendBufferSize >>= 1; - } - - if( ftruncate( m_fd, ui64CurrFileSize + uiTotalBytesToExtend) == -1) - { - rc = f_mapPlatformError( errno, NE_FLM_WRITING_FILE); goto Exit; } - - while( uiTotalBytesToExtend) - { - uiCurrBytesToExtend = f_min( - uiTotalBytesToExtend, uiExtendBufferSize); - - if( (iBytesWritten = pwrite( m_fd, pucExtendBuffer, - uiCurrBytesToExtend, ui64CurrFileSize)) == -1) - { - if( errno == EINTR) - { - continue; - } - - rc = f_mapPlatformError( errno, NE_FLM_WRITING_FILE); - goto Exit; - } - - uiTotalBytesToExtend -= uiCurrBytesToExtend; - ui64CurrFileSize += uiCurrBytesToExtend; - - if( (FLMUINT)iBytesWritten < uiCurrBytesToExtend) - { - rc = RC_SET_AND_ASSERT( NE_FLM_IO_DISK_FULL); - goto Exit; - } - } - - f_freeAlignedBuffer( &pucExtendBuffer); - m_bFlushRequired = TRUE; } } @@ -979,6 +938,127 @@ Exit: return( rc); } +/**************************************************************************** +Desc: +****************************************************************************/ +RCODE F_FileHdl::extendFile( + FLMUINT64 ui64NewFileSize) +{ + RCODE rc = NE_FLM_OK; + FLMBYTE * pucExtendBuffer = NULL; + FLMUINT uiExtendBufferSize; + FLMUINT uiCurrBytesToExtend; + FLMINT iBytesWritten; + FLMUINT64 ui64FileSize; + FLMUINT64 ui64TotalBytesToExtend = 0; + + // Get the current file size + + if( RC_BAD( rc = size( &ui64FileSize))) + { + goto Exit; + } + + // File is already the requested size + + if( ui64FileSize >= ui64NewFileSize) + { + goto Exit; + } + +#if defined( FLM_HPUX) + { + struct vx_ext extendInfo; + struct stat filestats; + + if( fstat( m_fd, &filestats) != 0) + { + rc = f_mapPlatformError( errno, NE_FLM_WRITING_FILE); + goto Exit; + } + + // If this is a JFS volume, we may be able to use an ioctl command + // to extend the file. If this doesn't work, we'll revert to + // the slow way of extending the file. + + f_memset( &extendInfo, 0, sizeof( extendInfo)); + extendInfo.ext_size = 0; + extendInfo.reserve = ui64NewFileSize / filestats.st_blksize; + extendInfo.a_flags = VX_CHGSIZE; + + if( ioctl( m_fd, VX_SETEXT, &extendInfo) == 0) + { + // The call succeeded. Our work is done. + + goto Exit; + } + } +#endif + + ui64TotalBytesToExtend = ui64NewFileSize - ui64FileSize; + uiExtendBufferSize = (FLMUINT)f_min( ui64TotalBytesToExtend, 1024 * 1024); + + for( ;;) + { + if( RC_OK( rc = f_allocAlignedBuffer( + uiExtendBufferSize, &pucExtendBuffer))) + { + break; + } + + if( uiExtendBufferSize <= (32 * 1024)) + { + goto Exit; + } + + uiExtendBufferSize >>= 1; + } + + if( ftruncate( m_fd, ui64NewFileSize) == -1) + { + rc = f_mapPlatformError( errno, NE_FLM_WRITING_FILE); + goto Exit; + } + + while( ui64TotalBytesToExtend) + { + uiCurrBytesToExtend = (FLMUINT)f_min( + ui64TotalBytesToExtend, uiExtendBufferSize); + + if( (iBytesWritten = pwrite( m_fd, pucExtendBuffer, + uiCurrBytesToExtend, ui64FileSize)) == -1) + { + if( errno == EINTR) + { + continue; + } + + rc = f_mapPlatformError( errno, NE_FLM_WRITING_FILE); + goto Exit; + } + + ui64TotalBytesToExtend -= uiCurrBytesToExtend; + ui64FileSize += uiCurrBytesToExtend; + + if( (FLMUINT)iBytesWritten < uiCurrBytesToExtend) + { + rc = RC_SET_AND_ASSERT( NE_FLM_IO_DISK_FULL); + goto Exit; + } + } + + m_bFlushRequired = TRUE; + +Exit: + + if( pucExtendBuffer) + { + f_freeAlignedBuffer( &pucExtendBuffer); + } + + return( rc); +} + /****************************************************************************** Desc: Attempts to lock byte 0 of the file. This method is used to lock byte 0 of the .lck file to ensure that only one process diff --git a/ftk/src/ftkwin.cpp b/ftk/src/ftkwin.cpp index 4e0dfef..15749ea 100644 --- a/ftk/src/ftkwin.cpp +++ b/ftk/src/ftkwin.cpp @@ -563,7 +563,7 @@ RCODE F_FileHdl::lowLevelWrite( if( uiTotalBytesToExtend) { - if( RC_BAD( rc = extendFile( ui64CurrFileSize, uiTotalBytesToExtend))) + if( RC_BAD( rc = extendFile( ui64CurrFileSize + uiTotalBytesToExtend))) { goto Exit; } @@ -730,9 +730,7 @@ Exit: } /**************************************************************************** -Desc: Truncate the file to the indicated size -WARNING: Direct IO methods are calling this method. Make sure that all changes - to this method work in direct IO mode. +Desc: Truncate the file to the indicated size ****************************************************************************/ RCODE FLMAPI F_FileHdl::truncateFile( FLMUINT64 ui64NewSize) @@ -777,18 +775,32 @@ Exit: Desc: ****************************************************************************/ RCODE F_FileHdl::extendFile( - FLMUINT64 ui64FileSize, - FLMUINT uiTotalBytesToExtend) + FLMUINT64 ui64NewFileSize) { RCODE rc = NE_FLM_OK; FLMUINT uiBytesToWrite; FLMUINT uiBytesWritten; FLMBYTE * pucBuffer = NULL; FLMUINT uiBufferSize; - FLMUINT64 ui64NewFileSize = ui64FileSize + uiTotalBytesToExtend; + FLMUINT64 ui64FileSize; + FLMUINT64 ui64TotalBytesToExtend = 0; LARGE_INTEGER liTmp; F_FileAsyncClient * pAsyncClient = NULL; - + + // Get the current file size + + if( RC_BAD( rc = size( &ui64FileSize))) + { + goto Exit; + } + + // File is already the requested size + + if( ui64FileSize >= ui64NewFileSize) + { + goto Exit; + } + // Try to extend the file using SetFileValidData. This will allocate blocks // without zero-filling them. This call is very fast, but is only available // on WinXP/2003 and newer systems and only if the file was opened while the @@ -816,6 +828,12 @@ RCODE F_FileHdl::extendFile( goto Exit; } + + // Determine the number of bytes to extend + + ui64TotalBytesToExtend = ui64NewFileSize - ui64FileSize; + + // Allocate a zero-filled buffer for extending uiBufferSize = 64 * 1024; if( RC_BAD( rc = f_allocAlignedBuffer( uiBufferSize, &pucBuffer))) @@ -827,11 +845,11 @@ RCODE F_FileHdl::extendFile( // Extend the file until we run out of bytes to write. - while( uiTotalBytesToExtend) + while( ui64TotalBytesToExtend) { - if( (uiBytesToWrite = uiBufferSize) > uiTotalBytesToExtend) + if( (uiBytesToWrite = uiBufferSize) > ui64TotalBytesToExtend) { - uiBytesToWrite = uiTotalBytesToExtend; + uiBytesToWrite = (FLMUINT)ui64TotalBytesToExtend; } if( m_bOpenedInAsyncMode) @@ -902,7 +920,7 @@ RCODE F_FileHdl::extendFile( goto Exit; } - uiTotalBytesToExtend -= uiBytesToWrite; + ui64TotalBytesToExtend -= uiBytesToWrite; ui64FileSize += uiBytesToWrite; }