Index Class Reference

#include <sqliteInt.h>

Inheritance diagram for Index:

[legend]Collaboration diagram for Index:

[legend]List of all members.

---

Detailed Description

Definition at line 831 of file sqliteInt.h.

Signals
void	indexingProgress (int)
Public Member Functions
	Index (const QString &dp, const QString &hp)
	Index (const QStringList &dl, const QString &hp)
void	writeDict ()
void	readDict ()
int	makeIndex ()
QStringList	query (const QStringList &, const QStringList &, const QStringList &)
QString	getDocumentTitle (const QString &)
void	setDictionaryFile (const QString &)
void	setDocListFile (const QString &)
void	setDocList (const QStringList &)
Public Attributes
char *	zName
int	nColumn
int *	aiColumn
unsigned *	aiRowEst
Table *	pTable
int	tnum
u8	onError
u8	autoIndex
char *	zColAff
Index *	pNext
Schema *	pSchema
u8 *	aSortOrder
char **	azColl
Private Slots
void	setLastWinClosed ()
Private Member Functions
void	setupDocumentList ()
void	parseDocument (const QString &, int)
void	insertInDict (const QString &, int)
void	writeDocumentList ()
void	readDocumentList ()
QStringList	getWildcardTerms (const QString &)
QStringList	split (const QString &)
QVector< Document >	setupDummyTerm (const QStringList &)
bool	searchForPattern (const QStringList &, const QStringList &, const QString &)
void	buildMiniDict (const QString &)
QString	getCharsetForDocument (QFile *)
Private Attributes
QStringList	docList
QHash< QString, Entry * >	dict
QHash< QString, PosEntry * >	miniDict
uint	wordNum
QString	docPath
QString	dictFile
QString	docListFile
bool	alreadyHaveDocList
bool	lastWindowClosed
Classes
struct	Entry
struct	PosEntry

---

Constructor & Destructor Documentation

Index::Index	(	const QString &	dp,
		const QString &	hp
	)

Definition at line 60 of file index.cpp.

References alreadyHaveDocList, QObject::connect(), lastWindowClosed, qApp, setLastWinClosed(), SIGNAL, and SLOT.

    : QObject( 0 ), docPath( dp )
{
    Q_UNUSED(hp);

    alreadyHaveDocList = false;
    lastWindowClosed = false;
    connect( qApp, SIGNAL(lastWindowClosed()),
             this, SLOT(setLastWinClosed()) );
}

Here is the call graph for this function:

Index::Index	(	const QStringList &	dl,
		const QString &	hp
	)

Definition at line 71 of file index.cpp.

References alreadyHaveDocList, QObject::connect(), docList, lastWindowClosed, qApp, setLastWinClosed(), SIGNAL, and SLOT.

    : QObject( 0 )
{
    Q_UNUSED(hp);
    docList = dl;
    alreadyHaveDocList = true;
    lastWindowClosed = false;
    connect( qApp, SIGNAL(lastWindowClosed()),
             this, SLOT(setLastWinClosed()) );
}

Here is the call graph for this function:

---

Member Function Documentation

void Index::writeDict

(

)

Definition at line 234 of file index.cpp.

References dict, dictFile, QFile::open(), s, writeDocumentList(), and QIODevice::WriteOnly.

Referenced by HelpDialog::setupFullTextIndex().

{
    QFile f( dictFile );
    if ( !f.open(QFile::WriteOnly ) )
        return;
    QDataStream s( &f );
    for(QHash<QString, Entry *>::Iterator it = dict.begin(); it != dict.end(); ++it) {
        s << it.key();
        s << it.value()->documents.count();
        s << it.value()->documents;
    }
    f.close();
    writeDocumentList();
}

Here is the call graph for this function:

void Index::readDict

(

)

Definition at line 258 of file index.cpp.

References QFile::close(), dict, dictFile, key, QFile::open(), readDocumentList(), QIODevice::ReadOnly, QVector< T >::resize(), and s.

Referenced by HelpDialog::setupFullTextIndex().

{
    QFile f( dictFile );
    if ( !f.open(QFile::ReadOnly ) )
        return;

    dict.clear();
    QDataStream s( &f );
    QString key;
    int numOfDocs;
    QVector<Document> docs;
    while ( !s.atEnd() ) {
        s >> key;
        s >> numOfDocs;
        docs.resize(numOfDocs);
        s >> docs;
        dict.insert( key, new Entry( docs ) );
    }
    f.close();
    readDocumentList();
}

Here is the call graph for this function:

int Index::makeIndex

(

)

Definition at line 102 of file index.cpp.

References alreadyHaveDocList, QList< T >::begin(), QList< T >::count(), docList, emit, QList< T >::end(), i, indexingProgress(), QList< T >::isEmpty(), lastWindowClosed, parseDocument(), setupDocumentList(), and QUrl::toLocalFile().

Referenced by HelpDialog::setupFullTextIndex().

{
    if ( !alreadyHaveDocList )
        setupDocumentList();
    if ( docList.isEmpty() )
        return 1;
    QStringList::Iterator it = docList.begin();
    int steps = docList.count() / 100;
    if ( !steps )
        steps++;
    int prog = 0;
    for ( int i = 0; it != docList.end(); ++it, ++i ) {
        if ( lastWindowClosed ) {
            return -1;
        }
        QUrl url(*it);
        parseDocument( url.toLocalFile(), i );
        if ( i%steps == 0 ) {
            prog++;
            emit indexingProgress( prog );
        }
    }
    return 0;
}

Here is the call graph for this function:

QStringList Index::query	(	const QStringList &	,
		const QStringList &	,
		const QStringList &
	)

Definition at line 289 of file index.cpp.

References QList< T >::append(), QList< T >::at(), QList< T >::begin(), QVector< T >::begin(), QVector< T >::constBegin(), QVector< T >::constEnd(), QList< T >::count(), dict, docList, Term::documents, QList< T >::end(), QVector< T >::end(), QVector< T >::erase(), getWildcardTerms(), QList< T >::isEmpty(), qSort(), searchForPattern(), setupDummyTerm(), t, and QList< T >::takeFirst().

Referenced by HelpDialog::startSearch().

{
    QList<Term> termList;
    for (QStringList::ConstIterator it = terms.begin(); it != terms.end(); ++it ) {
        Entry *e = 0;
        if ( (*it).contains(QLatin1Char('*')) ) {
            QVector<Document> wcts = setupDummyTerm( getWildcardTerms( *it ) );
            termList.append( Term(QLatin1String("dummy"), wcts.count(), wcts ) );
        } else if ( dict[ *it ] ) {
            e = dict[ *it ];
            termList.append( Term( *it, e->documents.count(), e->documents ) );
        } else {
            return QStringList();
        }
    }
    if ( !termList.count() )
        return QStringList();
    qSort(termList);

    QVector<Document> minDocs = termList.takeFirst().documents;
    for(QList<Term>::Iterator it = termList.begin(); it != termList.end(); ++it) {
        Term *t = &(*it);
        QVector<Document> docs = t->documents;
        for(QVector<Document>::Iterator minDoc_it = minDocs.begin(); minDoc_it != minDocs.end(); ) {
            bool found = false;
            for (QVector<Document>::ConstIterator doc_it = docs.constBegin(); doc_it != docs.constEnd(); ++doc_it ) {
                if ( (*minDoc_it).docNumber == (*doc_it).docNumber ) {
                    (*minDoc_it).frequency += (*doc_it).frequency;
                    found = true;
                    break;
                }
            }
            if ( !found )
                minDoc_it = minDocs.erase( minDoc_it );
            else
                ++minDoc_it;
        }
    }

    QStringList results;
    qSort( minDocs );
    if ( termSeq.isEmpty() ) {
        for(QVector<Document>::Iterator it = minDocs.begin(); it != minDocs.end(); ++it)
            results << docList.at((int)(*it).docNumber);
        return results;
    }

    QString fileName;
    for(QVector<Document>::Iterator it = minDocs.begin(); it != minDocs.end(); ++it) {
        fileName =  docList[ (int)(*it).docNumber ];
        if ( searchForPattern( termSeq, seqWords, fileName ) )
            results << fileName;
    }
    return results;
}

Here is the call graph for this function:

QString Index::getDocumentTitle

(

const QString &

)

Definition at line 345 of file index.cpp.

References Qt::CaseInsensitive, constData(), QString::indexOf(), QString::mid(), QFile::open(), qWarning(), QIODevice::ReadOnly, s, start, and QUrl::toLocalFile().

Referenced by HelpDialog::startSearch().

{
    QUrl url(fullFileName);
    QString fileName = url.toLocalFile();
    QFile file( fileName );
    if ( !file.open( QFile::ReadOnly ) ) {
        qWarning( (QLatin1String("cannot open file ") + fileName).toAscii().constData() );
        return fileName;
    }
    QTextStream s( &file );
    QString text = s.readAll();

    int start = text.indexOf(QLatin1String("<title>"), 0, Qt::CaseInsensitive) + 7;
    int end = text.indexOf(QLatin1String("</title>"), 0, Qt::CaseInsensitive);

    QString title = ( end - start <= 0 ? tr("Untitled") : text.mid( start, end - start ) );
    return title;
}

Here is the call graph for this function:

void Index::setDictionaryFile

(

const QString &

)

Definition at line 87 of file index.cpp.

References dictFile.

Referenced by HelpDialog::setupFullTextIndex().

{
    dictFile = f;
}

void Index::setDocListFile

(

const QString &

)

Definition at line 92 of file index.cpp.

References docListFile.

Referenced by HelpDialog::setupFullTextIndex().

{
    docListFile = f;
}

void Index::setDocList

(

const QStringList &

)

Definition at line 97 of file index.cpp.

References docList.

Referenced by HelpDialog::setupFullTextIndex().

{
    docList = lst;
}

void Index::indexingProgress

(

int

)

[signal]

Referenced by makeIndex().

void Index::setLastWinClosed

(

)

[private, slot]

Definition at line 82 of file index.cpp.

References lastWindowClosed.

Referenced by Index().

{
    lastWindowClosed = true;
}

void Index::setupDocumentList

(

)

[private]

Definition at line 127 of file index.cpp.

References QList< T >::append(), QList< T >::constBegin(), QList< T >::constEnd(), d, docList, docPath, and filters.

Referenced by makeIndex().

{
    QDir d( docPath );
    QStringList filters;
    filters.append(QLatin1String("*.html"));
    QStringList lst = d.entryList(filters);
    QStringList::ConstIterator it = lst.constBegin();
    for ( ; it != lst.constEnd(); ++it )
        docList.append( "file:" + docPath + QLatin1String("/") + *it );
}

Here is the call graph for this function:

void Index::parseDocument	(	const QString &	,
		int
	)			[private]

Definition at line 179 of file index.cpp.

References buf, c, QFile::close(), QTextCodec::codecForName(), constData(), QByteArray::constData(), getCharsetForDocument(), i, insertInDict(), QString::isNull(), j, QString::length(), QFile::open(), qWarning(), QIODevice::ReadOnly, s, QString::toLatin1(), and QString::unicode().

Referenced by makeIndex().

{
    QFile file( filename );
    if ( !file.open(QFile::ReadOnly) ) {
        qWarning( (QLatin1String("can not open file ") + filename).toAscii().constData() );
        return;
    }

    QTextStream s(&file);
    QString en = getCharsetForDocument(&file);
    s.setCodec(QTextCodec::codecForName(en.toLatin1().constData()));

    QString text = s.readAll();
    if (text.isNull())
        return;

    bool valid = true;
    const QChar *buf = text.unicode();
    QChar str[64];
    QChar c = buf[0];
    int j = 0;
    int i = 0;
    while ( j < text.length() ) {
        if ( c == QLatin1Char('<') || c == QLatin1Char('&') ) {
            valid = false;
            if ( i > 1 )
                insertInDict( QString(str,i), docNum );
            i = 0;
            c = buf[++j];
            continue;
        }
        if ( ( c == QLatin1Char('>') || c == QLatin1Char(';') ) && !valid ) {
            valid = true;
            c = buf[++j];
            continue;
        }
        if ( !valid ) {
            c = buf[++j];
            continue;
        }
        if ( ( c.isLetterOrNumber() || c == QLatin1Char('_') ) && i < 63 ) {
            str[i] = c.toLower();
            ++i;
        } else {
            if ( i > 1 )
                insertInDict( QString(str,i), docNum );
            i = 0;
        }
        c = buf[++j];
    }
    if ( i > 1 )
        insertInDict( QString(str,i), docNum );
    file.close();
}

Here is the call graph for this function:

void Index::insertInDict	(	const QString &	,
		int
	)			[private]

Definition at line 138 of file index.cpp.

References dict.

Referenced by parseDocument().

{
    if ( str == QLatin1String("amp") || str == QLatin1String("nbsp"))
        return;
    Entry *e = 0;
    if ( dict.count() )
        e = dict[ str ];

    if ( e ) {
        if ( e->documents.last().docNumber != docNum )
            e->documents.append( Document(docNum, 1 ) );
        else
            e->documents.last().frequency++;
    } else {
        dict.insert( str, new Entry( docNum ) );
    }
}

void Index::writeDocumentList

(

)

[private]

Definition at line 249 of file index.cpp.

References docList, docListFile, QFile::open(), s, and QIODevice::WriteOnly.

Referenced by writeDict().

{
    QFile f( docListFile );
    if ( !f.open(QFile::WriteOnly ) )
        return;
    QDataStream s( &f );
    s << docList;
}

Here is the call graph for this function:

void Index::readDocumentList

(

)

[private]

Definition at line 280 of file index.cpp.

References docList, docListFile, QFile::open(), QIODevice::ReadOnly, and s.

Referenced by readDict().

{
    QFile f( docListFile );
    if ( !f.open(QFile::ReadOnly ) )
        return;
    QDataStream s( &f );
    s >> docList;
}

Here is the call graph for this function:

QStringList Index::getWildcardTerms

(

const QString &

)

[private]

Definition at line 364 of file index.cpp.

References QList< T >::begin(), dict, QList< T >::end(), index, int, QList< T >::last(), and split().

Referenced by query().

{
    QStringList lst;
    QStringList terms = split( term );
    QStringList::Iterator iter;

    for(QHash<QString, Entry*>::Iterator it = dict.begin(); it != dict.end(); ++it) {
        int index = 0;
        bool found = false;
        QString text( it.key() );
        for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
            if ( *iter == QLatin1String("*") ) {
                found = true;
                continue;
            }
            if ( iter == terms.begin() && (*iter)[0] != text[0] ) {
                found = false;
                break;
            }
            index = text.indexOf( *iter, index );
            if ( *iter == terms.last() && index != (int)text.length()-1 ) {
                index = text.lastIndexOf( *iter );
                if ( index != (int)text.length() - (int)(*iter).length() ) {
                    found = false;
                    break;
                }
            }
            if ( index != -1 ) {
                found = true;
                index += (*iter).length();
                continue;
            } else {
                found = false;
                break;
            }
        }
        if ( found )
            lst << text;
    }

    return lst;
}

Here is the call graph for this function:

QStringList Index::split

(

const QString &

)

[private]

Definition at line 407 of file index.cpp.

References i, QString::indexOf(), j, l, QString::length(), QString::mid(), and QString::startsWith().

Referenced by getWildcardTerms().

{
    QStringList lst;
    int j = 0;
    int i = str.indexOf(QLatin1Char('*'), j );

    if (str.startsWith(QLatin1String("*")))
        lst << QLatin1String("*");

    while ( i != -1 ) {
        if ( i > j && i <= (int)str.length() ) {
            lst << str.mid( j, i - j );
            lst << QLatin1String("*");
        }
        j = i + 1;
        i = str.indexOf(QLatin1Char('*'), j );
    }

    int l = str.length() - 1;
    if ( str.mid( j, l - j + 1 ).length() > 0 )
        lst << str.mid( j, l - j + 1 );

    return lst;
}

Here is the call graph for this function:

QVector< Document > Index::setupDummyTerm

(

const QStringList &

)

[private]

Definition at line 432 of file index.cpp.

References QVector< T >::append(), QList< T >::append(), QList< T >::begin(), QVector< T >::begin(), QList< T >::count(), dict, QVector< T >::end(), QList< T >::end(), QVector< T >::indexOf(), qSort(), and t.

Referenced by query().

{
    QList<Term> termList;
    for (QStringList::ConstIterator it = terms.begin(); it != terms.end(); ++it) {
        Entry *e = 0;
        if ( dict[ *it ] ) {
            e = dict[ *it ];
            termList.append( Term( *it, e->documents.count(), e->documents ) );
        }
    }
    QVector<Document> maxList(0);
    if ( !termList.count() )
        return maxList;
    qSort(termList);

    maxList = termList.takeLast().documents;
    for(QList<Term>::Iterator it = termList.begin(); it != termList.end(); ++it) {
        Term *t = &(*it);
        QVector<Document> docs = t->documents;
        for (QVector<Document>::iterator docIt = docs.begin(); docIt != docs.end(); ++docIt ) {
            if ( maxList.indexOf( *docIt ) == -1 )
                maxList.append( *docIt );
        }
    }
    return maxList;
}

Here is the call graph for this function:

bool Index::searchForPattern	(	const QStringList &	,
		const QStringList &	,
		const QString &
	)			[private]

Definition at line 466 of file index.cpp.

References a, b, QList< T >::begin(), buf, buildMiniDict(), c, QFile::close(), constData(), QList< T >::count(), QList< T >::end(), i, int, j, QString::length(), miniDict, QFile::open(), positions, qWarning(), QIODevice::ReadOnly, s, QUrl::toLocalFile(), QString::unicode(), and wordNum.

Referenced by query().

{
    QUrl url(fileName);
    QString fName = url.toLocalFile();
    QFile file( fName );
    if ( !file.open( QFile::ReadOnly ) ) {
        qWarning( (QLatin1String("cannot open file ") + fName).toAscii().constData() );
        return false;
    }

    wordNum = 3;
    miniDict.clear();
    QStringList::ConstIterator cIt = words.begin();
    for ( ; cIt != words.end(); ++cIt )
        miniDict.insert( *cIt, new PosEntry( 0 ) );

    QTextStream s( &file );
    QString text = s.readAll();
    bool valid = true;
    const QChar *buf = text.unicode();
    QChar str[64];
    QChar c = buf[0];
    int j = 0;
    int i = 0;
    while ( j < text.length() ) {
        if ( c == QLatin1Char('<') || c == QLatin1Char('&') ) {
            valid = false;
            if ( i > 1 )
                buildMiniDict( QString(str,i) );
            i = 0;
            c = buf[++j];
            continue;
        }
        if ( ( c == QLatin1Char('>') || c == QLatin1Char(';') ) && !valid ) {
            valid = true;
            c = buf[++j];
            continue;
        }
        if ( !valid ) {
            c = buf[++j];
            continue;
        }
        if ( ( c.isLetterOrNumber() || c == QLatin1Char('_') ) && i < 63 ) {
            str[i] = c.toLower();
            ++i;
        } else {
            if ( i > 1 )
                buildMiniDict( QString(str,i) );
            i = 0;
        }
        c = buf[++j];
    }
    if ( i > 1 )
        buildMiniDict( QString(str,i) );
    file.close();

    QStringList::ConstIterator patIt = patterns.begin();
    QStringList wordLst;
    QList<uint> a, b;
    QList<uint>::iterator aIt;
    for ( ; patIt != patterns.end(); ++patIt ) {
        wordLst = (*patIt).split(QLatin1Char(' '));
        a = miniDict[ wordLst[0] ]->positions;
        for ( int j = 1; j < (int)wordLst.count(); ++j ) {
            b = miniDict[ wordLst[j] ]->positions;
            aIt = a.begin();
            while ( aIt != a.end() ) {
                if ( b.contains( *aIt + 1 )) {
                    (*aIt)++;
                    ++aIt;
                } else {
                    aIt = a.erase( aIt );
                }
            }
        }
    }
    if ( a.count() )
        return true;
    return false;
}

Here is the call graph for this function:

void Index::buildMiniDict

(

const QString &

)

[private]

Definition at line 459 of file index.cpp.

References miniDict, and wordNum.

Referenced by searchForPattern().

{
    if ( miniDict[ str ] )
        miniDict[ str ]->positions.append( wordNum );
    ++wordNum;
}

QString Index::getCharsetForDocument

(

QFile *

)

[private]

Definition at line 156 of file index.cpp.

References QRegExp::cap(), Qt::CaseInsensitive, QRegExp::indexIn(), QString::indexOf(), QString::isEmpty(), QString::mid(), s, QFile::seek(), start, and QString::toLower().

Referenced by parseDocument().

{
    QTextStream s(file);
    QString contents = s.readAll();

    QString encoding;
    int start = contents.indexOf("<meta", 0, Qt::CaseInsensitive);
    if (start > 0) {
        int end = contents.indexOf(">", start);
        QString meta = contents.mid(start+5, end-start);
        meta = meta.toLower();
        QRegExp r("charset=([^\"\\s]+)");
        if (r.indexIn(meta) != -1) {
            encoding = r.cap(1);        
        }
    }

    file->seek(0);
    if (encoding.isEmpty())
        return "utf-8";
    return encoding;
}

Here is the call graph for this function:

---

Member Data Documentation

char* Index::zName

Definition at line 832 of file sqliteInt.h.

int Index::nColumn

Definition at line 833 of file sqliteInt.h.

int* Index::aiColumn

Definition at line 834 of file sqliteInt.h.

unsigned* Index::aiRowEst

Definition at line 835 of file sqliteInt.h.

Table* Index::pTable

Definition at line 836 of file sqliteInt.h.

int Index::tnum

Definition at line 837 of file sqliteInt.h.

u8 Index::onError

Definition at line 838 of file sqliteInt.h.

u8 Index::autoIndex

Definition at line 839 of file sqliteInt.h.

char* Index::zColAff

Definition at line 840 of file sqliteInt.h.

Index* Index::pNext

Definition at line 841 of file sqliteInt.h.

Schema* Index::pSchema

Definition at line 842 of file sqliteInt.h.

u8* Index::aSortOrder

Definition at line 843 of file sqliteInt.h.

char** Index::azColl

Definition at line 844 of file sqliteInt.h.

QStringList Index::docList [private]

Definition at line 100 of file index.h.

Referenced by Index(), makeIndex(), query(), readDocumentList(), setDocList(), setupDocumentList(), and writeDocumentList().

QHash<QString, Entry*> Index::dict [private]

Definition at line 101 of file index.h.

Referenced by getWildcardTerms(), insertInDict(), query(), readDict(), setupDummyTerm(), and writeDict().

QHash<QString, PosEntry*> Index::miniDict [private]

Definition at line 102 of file index.h.

Referenced by buildMiniDict(), and searchForPattern().

uint Index::wordNum [private]

Definition at line 103 of file index.h.

Referenced by buildMiniDict(), and searchForPattern().

QString Index::docPath [private]

Definition at line 104 of file index.h.

Referenced by setupDocumentList().

QString Index::dictFile [private]

Definition at line 105 of file index.h.

Referenced by readDict(), setDictionaryFile(), and writeDict().

QString Index::docListFile [private]

Definition at line 105 of file index.h.

Referenced by readDocumentList(), setDocListFile(), and writeDocumentList().

bool Index::alreadyHaveDocList [private]

Definition at line 106 of file index.h.

Referenced by Index(), and makeIndex().

bool Index::lastWindowClosed [private]

Definition at line 107 of file index.h.

Referenced by Index(), makeIndex(), and setLastWinClosed().

---

The documentation for this class was generated from the following files:

• src/3rdparty/sqlite/sqliteInt.h
• tools/assistant/index.h
• tools/assistant/index.cpp

---