alexhudson
277 lines
7.8 KiB
C++
277 lines
7.8 KiB
C++
#include "test.h"
|
|
#include "CLucene/util/dirent.h"
|
|
#include "CLucene/util/Reader.h"
|
|
|
|
CL_NS_USE(util)
|
|
|
|
//an extremelly simple analyser. this eliminates differences
|
|
//caused by differences in character classifications functions
|
|
class ReutersTokenizer:public CharTokenizer {
|
|
public:
|
|
// Construct a new LetterTokenizer.
|
|
ReutersTokenizer(CL_NS(util)::Reader* in):
|
|
CharTokenizer(in) {}
|
|
|
|
~ReutersTokenizer(){}
|
|
protected:
|
|
bool isTokenChar(const TCHAR c) const{
|
|
if ( c == ' ' || c == '\t' ||
|
|
c == '-' || c == '.' ||
|
|
c == '\n' || c == '\r' ||
|
|
c == ',' || c == '<' ||
|
|
c == '>' || c<=9){
|
|
return false;
|
|
}else
|
|
return true;
|
|
}
|
|
TCHAR normalize(const TCHAR c) const{
|
|
return c;
|
|
}
|
|
};
|
|
|
|
class ReutersAnalyzer: public Analyzer {
|
|
public:
|
|
TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader){
|
|
return _CLNEW ReutersTokenizer(reader);
|
|
}
|
|
~ReutersAnalyzer(){}
|
|
};
|
|
|
|
|
|
char reuters_fsdirectory[CL_MAX_PATH];
|
|
bool reuters_ready = false;
|
|
|
|
char reuters_srcdirectory[1024];
|
|
char reuters_origdirectory[1024];
|
|
|
|
//indexes the reuters-21578 data.
|
|
void testReuters(CuTest *tc) {
|
|
strcpy(reuters_srcdirectory, clucene_data_location);
|
|
strcat(reuters_srcdirectory, "/reuters-21578");
|
|
CuAssert(tc,_T("Data does not exist"),Misc::dir_Exists(reuters_srcdirectory));
|
|
|
|
strcpy(reuters_origdirectory, clucene_data_location);
|
|
strcat(reuters_origdirectory, "/reuters-21578-index");
|
|
CuAssert(tc,_T("Index does not exist"),Misc::dir_Exists(reuters_origdirectory));
|
|
|
|
FSDirectory* fsdir = FSDirectory::getDirectory(reuters_fsdirectory,true);
|
|
ReutersAnalyzer a;
|
|
|
|
IndexWriter writer(fsdir,&a,true);
|
|
writer.setUseCompoundFile(false);
|
|
writer.setMaxFieldLength(10000);
|
|
DIR* srcdir = opendir(reuters_srcdirectory);
|
|
struct dirent* fl = readdir(srcdir);
|
|
struct fileStat buf;
|
|
char tmppath[CL_MAX_DIR];
|
|
strncpy(tmppath,reuters_srcdirectory,CL_MAX_DIR);
|
|
strcat(tmppath,PATH_DELIMITERA);
|
|
char* tmppathP = tmppath + strlen(tmppath);
|
|
TCHAR tpath[CL_MAX_PATH];
|
|
|
|
while ( fl != NULL ){
|
|
strcpy(tmppathP,fl->d_name);
|
|
STRCPY_AtoT(tpath,fl->d_name,CL_MAX_PATH);
|
|
fileStat(tmppath,&buf);
|
|
if ( buf.st_mode & S_IFREG){
|
|
Document* doc = _CLNEW Document;
|
|
doc->add(*Field::Text(_T("path"),tpath));
|
|
doc->add(*Field::Text(_T("contents"), _CLNEW FileReader(tmppath, "ASCII") ));
|
|
|
|
writer.addDocument( doc );
|
|
_CLDELETE(doc);
|
|
}
|
|
fl = readdir(srcdir);
|
|
}
|
|
closedir(srcdir);
|
|
|
|
writer.optimize();
|
|
writer.close();
|
|
fsdir->close();
|
|
_CLDECDELETE(fsdir);
|
|
|
|
reuters_ready = true;
|
|
}
|
|
|
|
void testByteForByte(CuTest* tc){
|
|
CLUCENE_ASSERT(reuters_ready);
|
|
|
|
char tmppath[CL_MAX_DIR];
|
|
|
|
strcpy(tmppath,reuters_origdirectory);
|
|
strcat(tmppath,"/_z.cfs");
|
|
FILE* f1 = fopen(tmppath,"rb");
|
|
CLUCENE_ASSERT(f1!=NULL);
|
|
|
|
strcpy(tmppath,reuters_fsdirectory);
|
|
strcat(tmppath,"/_z.cfs");
|
|
FILE* f2 = fopen(tmppath,"rb");
|
|
CLUCENE_ASSERT(f2!=NULL);
|
|
|
|
uint8_t buf1[1024];
|
|
uint8_t buf2[1024];
|
|
|
|
int s1,s2;
|
|
|
|
while ( true){
|
|
s1 = fread(buf1,sizeof(uint8_t),1024,f1);
|
|
s2 = fread(buf2,sizeof(uint8_t),1024,f2);
|
|
CuAssert(tc,_T("comparison yielded different lengths"),s1==s2);
|
|
if ( s1 == 0 )
|
|
break;
|
|
|
|
for ( int i=0;i<1024;i++ )
|
|
CuAssert(tc,_T("comparison with original failed"),buf1[i]==buf2[i]);
|
|
}
|
|
|
|
fclose(f1);
|
|
fclose(f2);
|
|
}
|
|
|
|
void testBySection(CuTest* tc){
|
|
IndexReader* reader1 = IndexReader::open(reuters_origdirectory);
|
|
IndexReader* reader2 = IndexReader::open(reuters_fsdirectory);
|
|
|
|
//misc
|
|
CuAssertIntEquals(tc,_T("reader maxDoc not equal"), reader1->maxDoc(), reader2->maxDoc());
|
|
CuAssertIntEquals(tc,_T("reader numDocs not equal"), reader1->numDocs(), reader2->numDocs());
|
|
|
|
|
|
//test field names
|
|
TCHAR** fn1 = reader1->getFieldNames();
|
|
TCHAR** fn2 = reader2->getFieldNames();
|
|
|
|
//make sure field length is the same
|
|
int fn1count = 0;
|
|
int fn2count = 0;
|
|
while(fn1[fn1count++]!=NULL){}
|
|
while(fn2[fn2count++]!=NULL){}
|
|
fn1count--;fn2count--;
|
|
CuAssertIntEquals(tc, _T("reader fieldnames count not equal"), fn1count, fn2count );
|
|
|
|
|
|
for (int n=0;n<fn1count;n++ ){
|
|
//field names aren't always in the same order, so find it.
|
|
int fn2n = 0;
|
|
bool foundField = false;
|
|
while ( fn2[fn2n] != NULL ){
|
|
if ( _tcscmp(fn1[n],fn2[fn2n])==0 ){
|
|
foundField = true;
|
|
break;
|
|
}
|
|
fn2n++;
|
|
}
|
|
CLUCENE_ASSERT( foundField==true );
|
|
|
|
//test field norms
|
|
uint8_t* norms1 = reader1->norms(fn1[n]);
|
|
uint8_t* norms2 = reader2->norms(fn1[n]);
|
|
if ( norms1 != NULL ){
|
|
CLUCENE_ASSERT(norms2 != NULL);
|
|
for ( int i=0;i<reader1->maxDoc();i++ ){
|
|
int diff = norms1[i]-norms2[i];
|
|
if ( diff < 0 )
|
|
diff *= -1;
|
|
if ( diff > 16 ){
|
|
CuAssert(tc,_T("Norms are off by more than the threshold!"),false);
|
|
}
|
|
}
|
|
}else
|
|
CLUCENE_ASSERT(norms2 == NULL);
|
|
////////////////////
|
|
}
|
|
_CLDELETE_CARRAY_ALL(fn1);
|
|
_CLDELETE_CARRAY_ALL(fn2);
|
|
|
|
|
|
//test Terms
|
|
TermEnum* te1 = reader1->terms();
|
|
TermEnum* te2 = reader2->terms();
|
|
while ( te1->next() ){
|
|
CLUCENE_ASSERT(te2->next());
|
|
|
|
CuAssertStrEquals(tc,_T("term enum text not equal"), te1->term(false)->text(), te2->term(false)->text() );
|
|
CuAssertIntEquals(tc,_T("term enum docFreq"), te1->docFreq(), te2->docFreq() );
|
|
|
|
//test term docs
|
|
//todo: this isn't useful until we search the td2 for the doc of td1
|
|
/*
|
|
TermDocs* td1 = reader1->termDocs(te1->term(false));
|
|
TermDocs* td2 = reader2->termDocs(te1->term(false));
|
|
while (td1->next()){
|
|
CLUCENE_ASSERT(td2->next());
|
|
|
|
//CLUCENE_ASSERT(td1->doc()==td2->doc()); todo: doc's aren't always the same, but should check that
|
|
//doc is the same content
|
|
CLUCENE_ASSERT(td1->freq()==td2->freq());
|
|
|
|
//todo: add some data into the index for the termfreqvector.
|
|
//test term freq vector
|
|
TermFreqVector** tfv1 = reader1->getTermFreqVectors(td1->doc());
|
|
TermFreqVector** tfv2 = reader1->getTermFreqVectors(td1->doc());
|
|
if ( tfv1 != NULL ){
|
|
int t=0;
|
|
while ( tfv1[t] != NULL ){
|
|
CLUCENE_ASSERT(tfv2[t] != NULL);
|
|
|
|
TermFreqVector* v1 = tfv1[t];
|
|
TermFreqVector* v2 = tfv2[t];
|
|
|
|
CLUCENE_ASSERT(_tcscmp(v1->getField(),v2->getField())==0);
|
|
CLUCENE_ASSERT(v1->size()==v2->size());
|
|
|
|
//todo: should check a few more things here...
|
|
|
|
t++;
|
|
}
|
|
CLUCENE_ASSERT(tfv2[t] == NULL);
|
|
}else
|
|
CLUCENE_ASSERT(tfv2==NULL);
|
|
|
|
}
|
|
_CLDELETE(td1);
|
|
_CLDELETE(td2);
|
|
*/
|
|
|
|
//test term positions
|
|
//todo: this isn't useful until we search the td2 for the doc of td1
|
|
/*TermPositions* tp1 = reader1->termPositions(te1->term(false));
|
|
TermPositions* tp2 = reader2->termPositions(te1->term(false));
|
|
while ( tp1->next() ){
|
|
CLUCENE_ASSERT(tp2->next());
|
|
|
|
CLUCENE_ASSERT(tp1->doc()==tp2->doc());todo: doc's aren't always the same, but should check that
|
|
//doc is the same content
|
|
CLUCENE_ASSERT(tp1->freq()==tp2->freq());
|
|
CLUCENE_ASSERT(tp1->nextPosition()==tp2->nextPosition());
|
|
}
|
|
_CLDELETE(tp1);
|
|
_CLDELETE(tp2);*/
|
|
|
|
}
|
|
te1->close();
|
|
te2->close();
|
|
_CLDELETE(te1);
|
|
_CLDELETE(te2);
|
|
|
|
reader1->close();
|
|
reader2->close();
|
|
_CLDELETE(reader1);
|
|
_CLDELETE(reader2);
|
|
}
|
|
|
|
CuSuite *testreuters(void)
|
|
{
|
|
CuSuite *suite = CuSuiteNew(_T("CLucene Reuters Test"));
|
|
|
|
//setup some variables
|
|
strcpy(reuters_fsdirectory,cl_tempDir);
|
|
strcat(reuters_fsdirectory,"/reuters-index");
|
|
|
|
SUITE_ADD_TEST(suite, testReuters);
|
|
//SUITE_ADD_TEST(suite, testByteForByte); this test rarely works currently, use more robust by section test...
|
|
SUITE_ADD_TEST(suite, testBySection);
|
|
return suite;
|
|
}
|
|
// EOF
|