alexhudson
311 lines
7.2 KiB
C++
311 lines
7.2 KiB
C++
/*------------------------------------------------------------------------------
|
|
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
|
|
*
|
|
* Distributable under the terms of either the Apache License (Version 2.0) or
|
|
* the GNU Lesser General Public License, as specified in the COPYING file.
|
|
------------------------------------------------------------------------------*/
|
|
#include "CLucene/StdHeader.h"
|
|
#include "Analyzers.h"
|
|
#include "CLucene/util/StringBuffer.h"
|
|
|
|
CL_NS_USE(util)
|
|
CL_NS_DEF(analysis)
|
|
|
|
CharTokenizer::CharTokenizer(Reader* in) :
|
|
//input(in), ; input is in tokenizer base class (bug fix thanks to Andy Osipienko)
|
|
offset(0),
|
|
bufferIndex(0),
|
|
dataLen(0)
|
|
|
|
{
|
|
input = in;
|
|
}
|
|
|
|
TCHAR CharTokenizer::normalize(const TCHAR c) const
|
|
{
|
|
return c;
|
|
}
|
|
bool CharTokenizer::next(Token* token){
|
|
int32_t length = 0;
|
|
int32_t start = offset;
|
|
while (true) {
|
|
TCHAR c;
|
|
offset++;
|
|
if (bufferIndex >= dataLen) {
|
|
dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE);
|
|
if (dataLen == -1)
|
|
dataLen = 0;
|
|
bufferIndex = 0;
|
|
}
|
|
if (dataLen <= 0 ) {
|
|
if (length > 0)
|
|
break;
|
|
else
|
|
return false;
|
|
}else
|
|
c = ioBuffer[bufferIndex++];
|
|
if (isTokenChar(c)) { // if it's a token TCHAR
|
|
|
|
if (length == 0) // start of token
|
|
start = offset-1;
|
|
|
|
buffer[length++] = normalize(c); // buffer it, normalized
|
|
|
|
if (length == LUCENE_MAX_WORD_LEN) // buffer overflow!
|
|
break;
|
|
|
|
} else if (length > 0) // at non-Letter w/ chars
|
|
break; // return 'em
|
|
|
|
}
|
|
buffer[length]=0;
|
|
token->set( buffer, start, start+length);
|
|
return true;
|
|
}
|
|
|
|
bool LetterTokenizer::isTokenChar(const TCHAR c) const {
|
|
return _istalpha(c)!=0;
|
|
}
|
|
|
|
|
|
TCHAR LowerCaseTokenizer::normalize(const TCHAR chr) const {
|
|
return _totlower(chr);
|
|
}
|
|
|
|
bool WhitespaceTokenizer::isTokenChar(const TCHAR c) const{
|
|
return _istspace(c)==0; //(return true if NOT a space)
|
|
}
|
|
|
|
TokenStream* WhitespaceAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) {
|
|
return _CLNEW WhitespaceTokenizer(reader);
|
|
}
|
|
|
|
TokenStream* SimpleAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) {
|
|
return _CLNEW LowerCaseTokenizer(reader);
|
|
}
|
|
|
|
bool LowerCaseFilter::next(Token* t){
|
|
if (!input->next(t))
|
|
return false;
|
|
stringCaseFold( t->_termText );
|
|
return true;
|
|
}
|
|
|
|
StopFilter::StopFilter(TokenStream* in, bool deleteTokenStream, TCHAR** stopWords):
|
|
TokenFilter(in, deleteTokenStream),
|
|
table(_CLNEW CLSetList<TCHAR*>(false))
|
|
{
|
|
fillStopTable( table,stopWords );
|
|
}
|
|
|
|
void StopFilter::fillStopTable(CLSetList<TCHAR*>* stopTable,
|
|
TCHAR** stopWords) {
|
|
for (int32_t i = 0; stopWords[i]!=NULL; i++)
|
|
stopTable->insert(stopWords[i]);
|
|
}
|
|
|
|
bool StopFilter::next(Token* token) {
|
|
// return the first non-stop word found
|
|
for (; input->next(token); ){
|
|
if (table->find(token->_termText)==table->end()){
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// reached EOS -- return nothing
|
|
return false;
|
|
}
|
|
|
|
StopAnalyzer::StopAnalyzer():stopTable(false)
|
|
{
|
|
StopFilter::fillStopTable(&stopTable,ENGLISH_STOP_WORDS);
|
|
}
|
|
StopAnalyzer::~StopAnalyzer()
|
|
{
|
|
}
|
|
StopAnalyzer::StopAnalyzer( TCHAR** stopWords) {
|
|
StopFilter::fillStopTable(&stopTable,stopWords);
|
|
}
|
|
TokenStream* StopAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) {
|
|
return _CLNEW StopFilter(_CLNEW LowerCaseTokenizer(reader),true, &stopTable);
|
|
}
|
|
|
|
TCHAR* StopAnalyzer::ENGLISH_STOP_WORDS[] =
|
|
{
|
|
_T("a"), _T("an"), _T("and"), _T("are"), _T("as"), _T("at"), _T("be"), _T("but"), _T("by"),
|
|
_T("for"), _T("if"), _T("in"), _T("into"), _T("is"), _T("it"),
|
|
_T("no"), _T("not"), _T("of"), _T("on"), _T("or"), _T("s"), _T("such"),
|
|
_T("t"), _T("that"), _T("the"), _T("their"), _T("then"), _T("there"), _T("these"),
|
|
_T("they"), _T("this"), _T("to"), _T("was"), _T("will"), _T("with"), NULL
|
|
};
|
|
|
|
PerFieldAnalyzerWrapper::PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer):
|
|
analyzerMap(true,true)
|
|
{
|
|
this->defaultAnalyzer = defaultAnalyzer;
|
|
}
|
|
PerFieldAnalyzerWrapper::~PerFieldAnalyzerWrapper(){
|
|
analyzerMap.clear();
|
|
_CLDELETE(defaultAnalyzer);
|
|
}
|
|
|
|
void PerFieldAnalyzerWrapper::addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer) {
|
|
analyzerMap.put(STRDUP_TtoT(fieldName), analyzer);
|
|
}
|
|
|
|
TokenStream* PerFieldAnalyzerWrapper::tokenStream(const TCHAR* fieldName, Reader* reader) {
|
|
Analyzer* analyzer = analyzerMap.get(fieldName);
|
|
if (analyzer == NULL) {
|
|
analyzer = defaultAnalyzer;
|
|
}
|
|
|
|
return analyzer->tokenStream(fieldName, reader);
|
|
}
|
|
|
|
|
|
|
|
bool ISOLatin1AccentFilter::next(Token* token){
|
|
if ( input->next(token) ){
|
|
StringBuffer output;
|
|
int32_t l = token->termTextLength();
|
|
const TCHAR* chars = token->termText();
|
|
for (int32_t i = 0; i < l; i++) {
|
|
#ifdef _UCS2
|
|
TCHAR c = chars[i];
|
|
#else
|
|
unsigned char c = chars[i];
|
|
#endif
|
|
switch (c) {
|
|
case 0xC0 : // À
|
|
case 0xC1 : // Á
|
|
case 0xC2 : // Â
|
|
case 0xC3 : // Ã
|
|
case 0xC4 : // Ä
|
|
case 0xC5 : // Å
|
|
output.appendChar('A');
|
|
break;
|
|
case 0xC6 : // Æ
|
|
output.append(_T("AE"));
|
|
break;
|
|
case 0xC7 : // Ç
|
|
output.appendChar('C');
|
|
break;
|
|
case 0xC8 : // È
|
|
case 0xC9 : // É
|
|
case 0xCA : // Ê
|
|
case 0xCB : // Ë
|
|
output.appendChar('E');
|
|
break;
|
|
case 0xCC : // Ì
|
|
case 0xCD : // Í
|
|
case 0xCE : // Î
|
|
case 0xCF : // Ï
|
|
output.appendChar('I');
|
|
break;
|
|
case 0xD0 : // Ð
|
|
output.appendChar('D');
|
|
break;
|
|
case 0xD1 : // Ñ
|
|
output.appendChar('N');
|
|
break;
|
|
case 0xD2 : // Ò
|
|
case 0xD3 : // Ó
|
|
case 0xD4 : // Ô
|
|
case 0xD5 : // Õ
|
|
case 0xD6 : // Ö
|
|
case 0xD8 : // Ø
|
|
output.appendChar('O');
|
|
break;
|
|
case 0xDE : // Þ
|
|
output.append(_T("TH"));
|
|
break;
|
|
case 0xD9 : // Ù
|
|
case 0xDA : // Ú
|
|
case 0xDB : // Û
|
|
case 0xDC : // Ü
|
|
output.appendChar('U');
|
|
break;
|
|
case 0xDD : // Ý
|
|
output.appendChar('Y');
|
|
break;
|
|
case 0xE0 : // à
|
|
case 0xE1 : // á
|
|
case 0xE2 : // â
|
|
case 0xE3 : // ã
|
|
case 0xE4 : // ä
|
|
case 0xE5 : // å
|
|
output.appendChar('a');
|
|
break;
|
|
case 0xE6 : // æ
|
|
output.append(_T("ae"));
|
|
break;
|
|
case 0xE7 : // ç
|
|
output.appendChar('c');
|
|
break;
|
|
case 0xE8 : // è
|
|
case 0xE9 : // é
|
|
case 0xEA : // ê
|
|
case 0xEB : // ë
|
|
output.appendChar('e');
|
|
break;
|
|
case 0xEC : // ì
|
|
case 0xED : // í
|
|
case 0xEE : // î
|
|
case 0xEF : // ï
|
|
output.appendChar('i');
|
|
break;
|
|
case 0xF0 : // ð
|
|
output.appendChar('d');
|
|
break;
|
|
case 0xF1 : // ñ
|
|
output.appendChar('n');
|
|
break;
|
|
case 0xF2 : // ò
|
|
case 0xF3 : // ó
|
|
case 0xF4 : // ô
|
|
case 0xF5 : // õ
|
|
case 0xF6 : // ö
|
|
case 0xF8 : // ø
|
|
output.appendChar('o');
|
|
break;
|
|
case 0xDF : // ß
|
|
output.append(_T("ss"));
|
|
break;
|
|
case 0xFE : // þ
|
|
output.append(_T("th"));
|
|
break;
|
|
case 0xF9 : // ù
|
|
case 0xFA : // ú
|
|
case 0xFB : // û
|
|
case 0xFC : // ü
|
|
output.appendChar('u');
|
|
break;
|
|
case 0xFD : // ý
|
|
case 0xFF : // ÿ
|
|
output.appendChar('y');
|
|
break;
|
|
|
|
#ifdef _UCS2
|
|
case 0x152 : // Œ
|
|
output.append(_T("OE"));
|
|
break;
|
|
case 0x153 : // œ
|
|
output.append(_T("oe"));
|
|
break;
|
|
case 0x178 : // Ÿ
|
|
output.appendChar('Y');
|
|
break;
|
|
#endif
|
|
default :
|
|
output.appendChar(chars[i]);
|
|
break;
|
|
}
|
|
}
|
|
token->setText(output.getBuffer());
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
CL_NS_END
|