kmfolderindex.cpp
00001 /* -*- mode: C++; c-file-style: "gnu" -*- 00002 This file is part of KMail, the KDE mail client. 00003 Copyright (c) 2000 Don Sanders <sanders@kde.org> 00004 00005 KMail is free software; you can redistribute it and/or modify it 00006 under the terms of the GNU General Public License, version 2, as 00007 published by the Free Software Foundation. 00008 00009 KMail is distributed in the hope that it will be useful, but 00010 WITHOUT ANY WARRANTY; without even the implied warranty of 00011 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00012 General Public License for more details. 00013 00014 You should have received a copy of the GNU General Public License 00015 along with this program; if not, write to the Free Software 00016 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 00017 */ 00018 00019 #include "kmfolderindex.h" 00020 #include "kmfolder.h" 00021 #include "kmfoldertype.h" 00022 #include "kcursorsaver.h" 00023 #include <config.h> 00024 #include <tqfileinfo.h> 00025 #include <tqtimer.h> 00026 #include <kdebug.h> 00027 00028 00029 #define HAVE_MMAP //need to get this into autoconf FIXME --Sam 00030 #include <unistd.h> 00031 #ifdef HAVE_MMAP 00032 #include <sys/mman.h> 00033 #endif 00034 00035 // Current version of the table of contents (index) files 00036 #define INDEX_VERSION 1507 00037 00038 #ifndef MAX_LINE 00039 #define MAX_LINE 4096 00040 #endif 00041 00042 #ifndef INIT_MSGS 00043 #define INIT_MSGS 8 00044 #endif 00045 00046 #include <errno.h> 00047 #include <assert.h> 00048 #include <utime.h> 00049 #include <fcntl.h> 00050 00051 #ifdef HAVE_BYTESWAP_H 00052 #include <byteswap.h> 00053 #endif 00054 #include <kapplication.h> 00055 #include <kcursor.h> 00056 #include <kmessagebox.h> 00057 #include <klocale.h> 00058 #include "kmmsgdict.h" 00059 00060 // We define functions as kmail_swap_NN so that we don't get compile errors 00061 // on platforms where bswap_NN happens to be a function instead of a define. 00062 00063 /* Swap bytes in 32 bit value. */ 00064 #ifdef bswap_32 00065 #define kmail_swap_32(x) bswap_32(x) 00066 #else 00067 #define kmail_swap_32(x) \ 00068 ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >> 8) | \ 00069 (((x) & 0x0000ff00) << 8) | (((x) & 0x000000ff) << 24)) 00070 #endif 00071 00072 #include <stdlib.h> 00073 #include <sys/types.h> 00074 #include <sys/stat.h> 00075 #include <sys/file.h> 00076 00077 KMFolderIndex::KMFolderIndex(KMFolder* folder, const char* name) 00078 : FolderStorage(folder, name), mMsgList(INIT_MSGS) 00079 { 00080 mIndexStream = 0; 00081 mIndexStreamPtr = 0; 00082 mIndexStreamPtrLength = 0; 00083 mIndexSwapByteOrder = false; 00084 mIndexSizeOfLong = sizeof(long); 00085 mIndexId = 0; 00086 mHeaderOffset = 0; 00087 } 00088 00089 00090 KMFolderIndex::~KMFolderIndex() 00091 { 00092 } 00093 00094 00095 TQString KMFolderIndex::indexLocation() const 00096 { 00097 TQString sLocation(folder()->path()); 00098 00099 if ( !sLocation.isEmpty() ) { 00100 sLocation += '/'; 00101 sLocation += '.'; 00102 } 00103 sLocation += dotEscape(fileName()); 00104 sLocation += ".index"; 00105 00106 return sLocation; 00107 } 00108 00109 int KMFolderIndex::updateIndex() 00110 { 00111 if (!mAutoCreateIndex) 00112 return 0; 00113 bool dirty = mDirty; 00114 mDirtyTimer->stop(); 00115 for ( unsigned int i = 0; !dirty && i < mMsgList.high(); i++ ) { 00116 if ( mMsgList.at(i) ) { 00117 if ( !mMsgList.at(i)->syncIndexString() ) { 00118 dirty = true; 00119 } 00120 } 00121 } 00122 if (!dirty) { // Update successful 00123 touchFolderIdsFile(); 00124 return 0; 00125 } 00126 return writeIndex(); 00127 } 00128 00129 int KMFolderIndex::writeIndex( bool createEmptyIndex ) 00130 { 00131 TQString tempName; 00132 TQString indexName; 00133 mode_t old_umask; 00134 int len; 00135 const uchar *buffer = 0; 00136 00137 indexName = indexLocation(); 00138 tempName = indexName + ".temp"; 00139 unlink(TQFile::encodeName(tempName)); 00140 00141 // We touch the folder, otherwise the index is regenerated, if KMail is 00142 // running, while the clock switches from daylight savings time to normal time 00143 utime(TQFile::encodeName(location()), 0); 00144 00145 old_umask = umask(077); 00146 FILE *tmpIndexStream = fopen(TQFile::encodeName(tempName), "w"); 00147 umask(old_umask); 00148 if (!tmpIndexStream) 00149 return errno; 00150 00151 fprintf(tmpIndexStream, "# KMail-Index V%d\n", INDEX_VERSION); 00152 00153 // Header 00154 TQ_UINT32 byteOrder = 0x12345678; 00155 TQ_UINT32 sizeOfLong = sizeof(long); 00156 00157 TQ_UINT32 header_length = sizeof(byteOrder)+sizeof(sizeOfLong); 00158 char pad_char = '\0'; 00159 fwrite(&pad_char, sizeof(pad_char), 1, tmpIndexStream); 00160 fwrite(&header_length, sizeof(header_length), 1, tmpIndexStream); 00161 00162 // Write header 00163 fwrite(&byteOrder, sizeof(byteOrder), 1, tmpIndexStream); 00164 fwrite(&sizeOfLong, sizeof(sizeOfLong), 1, tmpIndexStream); 00165 00166 off_t nho = ftell(tmpIndexStream); 00167 00168 if ( !createEmptyIndex ) { 00169 KMMsgBase* msgBase; 00170 for (unsigned int i=0; i<mMsgList.high(); i++) 00171 { 00172 if (!(msgBase = mMsgList.at(i))) continue; 00173 buffer = msgBase->asIndexString(len); 00174 fwrite(&len,sizeof(len), 1, tmpIndexStream); 00175 00176 off_t tmp = ftell(tmpIndexStream); 00177 msgBase->setIndexOffset(tmp); 00178 msgBase->setIndexLength(len); 00179 if(fwrite(buffer, len, 1, tmpIndexStream) != 1) 00180 kdDebug(5006) << "Whoa! " << __FILE__ << ":" << __LINE__ << endl; 00181 } 00182 } 00183 00184 int fError = ferror( tmpIndexStream ); 00185 if( fError != 0 ) { 00186 fclose( tmpIndexStream ); 00187 return fError; 00188 } 00189 if( ( fflush( tmpIndexStream ) != 0 ) 00190 || ( fsync( fileno( tmpIndexStream ) ) != 0 ) ) { 00191 int errNo = errno; 00192 fclose( tmpIndexStream ); 00193 return errNo; 00194 } 00195 if( fclose( tmpIndexStream ) != 0 ) 00196 return errno; 00197 00198 ::rename(TQFile::encodeName(tempName), TQFile::encodeName(indexName)); 00199 mHeaderOffset = nho; 00200 if (mIndexStream) 00201 fclose(mIndexStream); 00202 00203 if ( createEmptyIndex ) 00204 return 0; 00205 00206 mIndexStream = fopen(TQFile::encodeName(indexName), "r+"); // index file 00207 assert( mIndexStream ); 00208 fcntl(fileno(mIndexStream), F_SETFD, FD_CLOEXEC); 00209 00210 updateIndexStreamPtr(); 00211 00212 writeFolderIdsFile(); 00213 00214 setDirty( false ); 00215 return 0; 00216 } 00217 00218 bool KMFolderIndex::readIndex() 00219 { 00220 if ( contentsType() != KMail::ContentsTypeMail ) { 00221 kdDebug(5006) << k_funcinfo << "Reading index for " << label() << endl; 00222 } 00223 TQ_INT32 len; 00224 KMMsgInfo* mi; 00225 00226 assert(mIndexStream != 0); 00227 rewind(mIndexStream); 00228 00229 clearIndex(); 00230 int version; 00231 00232 setDirty( false ); 00233 00234 if (!readIndexHeader(&version)) return false; 00235 //kdDebug(5006) << "Index version for " << label() << " is " << version << endl; 00236 00237 mUnreadMsgs = 0; 00238 mTotalMsgs = 0; 00239 mHeaderOffset = ftell(mIndexStream); 00240 00241 clearIndex(); 00242 while (!feof(mIndexStream)) 00243 { 00244 mi = 0; 00245 if(version >= 1505) { 00246 if(!fread(&len, sizeof(len), 1, mIndexStream)) { 00247 // Seems to be normal? 00248 // kdDebug(5006) << k_funcinfo << " Unable to read length field!" << endl; 00249 break; 00250 } 00251 00252 if (mIndexSwapByteOrder) 00253 len = kmail_swap_32(len); 00254 00255 off_t offs = ftell(mIndexStream); 00256 if(fseek(mIndexStream, len, SEEK_CUR)) { 00257 kdDebug(5006) << k_funcinfo << " Unable to seek to the end of the message!" << endl; 00258 break; 00259 } 00260 mi = new KMMsgInfo(folder(), offs, len); 00261 } 00262 else 00263 { 00264 TQCString line(MAX_LINE); 00265 fgets(line.data(), MAX_LINE, mIndexStream); 00266 if (feof(mIndexStream)) break; 00267 if (*line.data() == '\0') { 00268 fclose(mIndexStream); 00269 mIndexStream = 0; 00270 clearIndex(); 00271 return false; 00272 } 00273 mi = new KMMsgInfo(folder()); 00274 mi->compat_fromOldIndexString(line, mConvertToUtf8); 00275 } 00276 if(!mi) { 00277 kdDebug(5006) << k_funcinfo << " Unable to create message info object!" << endl; 00278 break; 00279 } 00280 00281 if (mi->isDeleted()) 00282 { 00283 delete mi; // skip messages that are marked as deleted 00284 setDirty( true ); 00285 needsCompact = true; //We have deleted messages - needs to be compacted 00286 continue; 00287 } 00288 #ifdef OBSOLETE 00289 else if (mi->isNew()) 00290 { 00291 mi->setStatus(KMMsgStatusUnread); 00292 mi->setDirty(false); 00293 } 00294 #endif 00295 if ((mi->isNew()) || (mi->isUnread()) || 00296 (folder() == kmkernel->outboxFolder())) 00297 { 00298 ++mUnreadMsgs; 00299 if (mUnreadMsgs == 0) ++mUnreadMsgs; 00300 } 00301 mMsgList.append(mi, false); 00302 } 00303 if( version < 1505) 00304 { 00305 mConvertToUtf8 = false; 00306 setDirty( true ); 00307 writeIndex(); 00308 } 00309 00310 if ( version < 1507 ) { 00311 updateInvitationAndAddressFieldsFromContents(); 00312 setDirty( true ); 00313 writeIndex(); 00314 } 00315 00316 mTotalMsgs = mMsgList.count(); 00317 if ( contentsType() != KMail::ContentsTypeMail ) { 00318 kdDebug(5006) << k_funcinfo << "Done reading the index for " << label() << ", we have " << mTotalMsgs << " messages." << endl; 00319 } 00320 return true; 00321 } 00322 00323 00324 int KMFolderIndex::count(bool cache) const 00325 { 00326 int res = FolderStorage::count(cache); 00327 if (res == -1) 00328 res = mMsgList.count(); 00329 return res; 00330 } 00331 00332 00333 bool KMFolderIndex::readIndexHeader(int *gv) 00334 { 00335 int indexVersion; 00336 assert(mIndexStream != 0); 00337 mIndexSwapByteOrder = false; 00338 mIndexSizeOfLong = sizeof(long); 00339 00340 int ret = fscanf(mIndexStream, "# KMail-Index V%d\n", &indexVersion); 00341 if ( ret == EOF || ret == 0 ) 00342 return false; // index file has invalid header 00343 if(gv) 00344 *gv = indexVersion; 00345 00346 // Check if the index is corrupted ("not compactable") and recreate it if necessary. See 00347 // FolderStorage::getMsg() for the detection code. 00348 if ( !mCompactable ) { 00349 kdWarning(5006) << "Index file " << indexLocation() << " is corrupted!!. Re-creating it." << endl; 00350 recreateIndex( false /* don't call readIndex() afterwards */ ); 00351 return false; 00352 } 00353 00354 if (indexVersion < 1505 ) { 00355 if(indexVersion == 1503) { 00356 kdDebug(5006) << "Converting old index file " << indexLocation() << " to utf-8" << endl; 00357 mConvertToUtf8 = true; 00358 } 00359 return true; 00360 } else if (indexVersion == 1505) { 00361 } else if (indexVersion < INDEX_VERSION && indexVersion != 1506) { 00362 kdDebug(5006) << "Index file " << indexLocation() << " is out of date. Re-creating it." << endl; 00363 createIndexFromContents(); 00364 return false; 00365 } else if(indexVersion > INDEX_VERSION) { 00366 kapp->setOverrideCursor(KCursor::arrowCursor()); 00367 int r = KMessageBox::questionYesNo(0, 00368 i18n( 00369 "The mail index for '%1' is from an unknown version of KMail (%2).\n" 00370 "This index can be regenerated from your mail folder, but some " 00371 "information, including status flags, may be lost. Do you wish " 00372 "to downgrade your index file?") .arg(name()) .arg(indexVersion), TQString(), i18n("Downgrade"), i18n("Do Not Downgrade") ); 00373 kapp->restoreOverrideCursor(); 00374 if (r == KMessageBox::Yes) 00375 createIndexFromContents(); 00376 return false; 00377 } 00378 else { 00379 // Header 00380 TQ_UINT32 byteOrder = 0; 00381 TQ_UINT32 sizeOfLong = sizeof(long); // default 00382 00383 TQ_UINT32 header_length = 0; 00384 fseek(mIndexStream, sizeof(char), SEEK_CUR ); 00385 fread(&header_length, sizeof(header_length), 1, mIndexStream); 00386 if (header_length > 0xFFFF) 00387 header_length = kmail_swap_32(header_length); 00388 00389 off_t endOfHeader = ftell(mIndexStream) + header_length; 00390 00391 bool needs_update = true; 00392 // Process available header parts 00393 if (header_length >= sizeof(byteOrder)) 00394 { 00395 fread(&byteOrder, sizeof(byteOrder), 1, mIndexStream); 00396 mIndexSwapByteOrder = (byteOrder == 0x78563412); 00397 header_length -= sizeof(byteOrder); 00398 00399 if (header_length >= sizeof(sizeOfLong)) 00400 { 00401 fread(&sizeOfLong, sizeof(sizeOfLong), 1, mIndexStream); 00402 if (mIndexSwapByteOrder) 00403 sizeOfLong = kmail_swap_32(sizeOfLong); 00404 mIndexSizeOfLong = sizeOfLong; 00405 header_length -= sizeof(sizeOfLong); 00406 needs_update = false; 00407 } 00408 } 00409 if (needs_update || mIndexSwapByteOrder || (mIndexSizeOfLong != sizeof(long))) 00410 setDirty( true ); 00411 // Seek to end of header 00412 fseek(mIndexStream, endOfHeader, SEEK_SET ); 00413 00414 if (mIndexSwapByteOrder) 00415 kdDebug(5006) << "Index File has byte order swapped!" << endl; 00416 if (mIndexSizeOfLong != sizeof(long)) 00417 kdDebug(5006) << "Index File sizeOfLong is " << mIndexSizeOfLong << " while sizeof(long) is " << sizeof(long) << " !" << endl; 00418 00419 } 00420 return true; 00421 } 00422 00423 00424 #ifdef HAVE_MMAP 00425 bool KMFolderIndex::updateIndexStreamPtr(bool just_close) 00426 #else 00427 bool KMFolderIndex::updateIndexStreamPtr(bool) 00428 #endif 00429 { 00430 // We touch the folder, otherwise the index is regenerated, if KMail is 00431 // running, while the clock switches from daylight savings time to normal time 00432 utime(TQFile::encodeName(location()), 0); 00433 utime(TQFile::encodeName(indexLocation()), 0); 00434 utime(TQFile::encodeName( KMMsgDict::getFolderIdsLocation( *this ) ), 0); 00435 00436 mIndexSwapByteOrder = false; 00437 #ifdef HAVE_MMAP 00438 if(just_close) { 00439 if(mIndexStreamPtr) 00440 munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength); 00441 mIndexStreamPtr = 0; 00442 mIndexStreamPtrLength = 0; 00443 return true; 00444 } 00445 00446 assert(mIndexStream); 00447 struct stat stat_buf; 00448 if(fstat(fileno(mIndexStream), &stat_buf) == -1) { 00449 if(mIndexStreamPtr) 00450 munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength); 00451 mIndexStreamPtr = 0; 00452 mIndexStreamPtrLength = 0; 00453 return false; 00454 } 00455 if(mIndexStreamPtr) 00456 munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength); 00457 mIndexStreamPtrLength = stat_buf.st_size; 00458 mIndexStreamPtr = (uchar *)mmap(0, mIndexStreamPtrLength, PROT_READ, MAP_SHARED, 00459 fileno(mIndexStream), 0); 00460 if(mIndexStreamPtr == MAP_FAILED) { 00461 mIndexStreamPtr = 0; 00462 mIndexStreamPtrLength = 0; 00463 return false; 00464 } 00465 #endif 00466 return true; 00467 } 00468 00469 00470 KMFolderIndex::IndexStatus KMFolderIndex::indexStatus() 00471 { 00472 if ( !mCompactable ) 00473 return IndexCorrupt; 00474 00475 TQFileInfo contInfo(location()); 00476 TQFileInfo indInfo(indexLocation()); 00477 00478 if (!contInfo.exists()) return KMFolderIndex::IndexOk; 00479 if (!indInfo.exists()) return KMFolderIndex::IndexMissing; 00480 00481 return ( contInfo.lastModified() > indInfo.lastModified() ) 00482 ? KMFolderIndex::IndexTooOld 00483 : KMFolderIndex::IndexOk; 00484 } 00485 00486 void KMFolderIndex::clearIndex(bool autoDelete, bool syncDict) 00487 { 00488 mMsgList.clear(autoDelete, syncDict); 00489 } 00490 00491 00492 void KMFolderIndex::truncateIndex() 00493 { 00494 if ( mHeaderOffset ) 00495 truncate(TQFile::encodeName(indexLocation()), mHeaderOffset); 00496 else 00497 // The index file wasn't opened, so we don't know the header offset. 00498 // So let's just create a new empty index. 00499 writeIndex( true ); 00500 } 00501 00502 void KMFolderIndex::fillMessageDict() 00503 { 00504 open("fillDict"); 00505 for (unsigned int idx = 0; idx < mMsgList.high(); idx++) 00506 if ( mMsgList.at( idx ) ) 00507 KMMsgDict::mutableInstance()->insert(0, mMsgList.at( idx ), idx); 00508 close("fillDict"); 00509 } 00510 00511 00512 KMMsgInfo* KMFolderIndex::setIndexEntry( int idx, KMMessage *msg ) 00513 { 00514 KMMsgInfo *msgInfo = msg->msgInfo(); 00515 if ( !msgInfo ) 00516 msgInfo = new KMMsgInfo( folder() ); 00517 00518 *msgInfo = *msg; 00519 mMsgList.set( idx, msgInfo ); 00520 msg->setMsgInfo( 0 ); 00521 delete msg; 00522 return msgInfo; 00523 } 00524 00525 void KMFolderIndex::recreateIndex( bool readIndexAfterwards ) 00526 { 00527 kapp->setOverrideCursor(KCursor::arrowCursor()); 00528 KMessageBox::information(0, 00529 i18n("The mail index for '%1' is corrupted and will be regenerated now, " 00530 "but some information, like status flags, might get lost.").arg(name())); 00531 kapp->restoreOverrideCursor(); 00532 createIndexFromContents(); 00533 if ( readIndexAfterwards ) { 00534 readIndex(); 00535 } 00536 00537 // Clear the corrupted flag 00538 mCompactable = true; 00539 writeConfig(); 00540 } 00541 00542 void KMFolderIndex::silentlyRecreateIndex() 00543 { 00544 Q_ASSERT( !isOpened() ); 00545 open( "silentlyRecreateIndex" ); 00546 KCursorSaver busy( KBusyPtr::busy() ); 00547 createIndexFromContents(); 00548 mCompactable = true; 00549 writeConfig(); 00550 close( "silentlyRecreateIndex" ); 00551 } 00552 00553 void KMFolderIndex::updateInvitationAndAddressFieldsFromContents() 00554 { 00555 kdDebug(5006) << "Updating index for " << label() << ", this might take a while." << endl; 00556 for ( uint i = 0; i < mMsgList.size(); i++ ) { 00557 KMMsgInfo * const msgInfo = dynamic_cast<KMMsgInfo*>( mMsgList[i] ); 00558 if ( msgInfo ) { 00559 DwString msgString( getDwString( i ) ); 00560 if ( msgString.size() > 0 ) { 00561 KMMessage msg; 00562 msg.fromDwString( msgString, false ); 00563 msg.updateInvitationState(); 00564 if ( msg.status() & KMMsgStatusHasInvitation ) { 00565 msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasInvitation ); 00566 } 00567 if ( msg.status() & KMMsgStatusHasNoInvitation ) { 00568 msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasNoInvitation ); 00569 } 00570 msgInfo->setFrom( msg.from() ); 00571 msgInfo->setTo( msg.to() ); 00572 } 00573 } 00574 } 00575 } 00576 00577 #include "kmfolderindex.moc"