kmail

kmfolderindex.cpp
1 /* -*- mode: C++; c-file-style: "gnu" -*-
2  This file is part of KMail, the KDE mail client.
3  Copyright (c) 2000 Don Sanders <sanders@kde.org>
4 
5  KMail is free software; you can redistribute it and/or modify it
6  under the terms of the GNU General Public License, version 2, as
7  published by the Free Software Foundation.
8 
9  KMail is distributed in the hope that it will be useful, but
10  WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  General Public License for more details.
13 
14  You should have received a copy of the GNU General Public License
15  along with this program; if not, write to the Free Software
16  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18 
19 #include "kmfolderindex.h"
20 #include "kmfolder.h"
21 #include "kmfoldertype.h"
22 #include "kcursorsaver.h"
23 #include <config.h>
24 #include <tqfileinfo.h>
25 #include <tqtimer.h>
26 #include <kdebug.h>
27 
28 
29 #define HAVE_MMAP //need to get this into autoconf FIXME --Sam
30 #include <unistd.h>
31 #ifdef HAVE_MMAP
32 #include <sys/mman.h>
33 #endif
34 
35 // Current version of the table of contents (index) files
36 #define INDEX_VERSION 1507
37 
38 #ifndef MAX_LINE
39 #define MAX_LINE 4096
40 #endif
41 
42 #ifndef INIT_MSGS
43 #define INIT_MSGS 8
44 #endif
45 
46 #include <errno.h>
47 #include <assert.h>
48 #include <utime.h>
49 #include <fcntl.h>
50 
51 #ifdef HAVE_BYTESWAP_H
52 #include <byteswap.h>
53 #endif
54 #include <kapplication.h>
55 #include <kcursor.h>
56 #include <kmessagebox.h>
57 #include <klocale.h>
58 #include "kmmsgdict.h"
59 
60 // We define functions as kmail_swap_NN so that we don't get compile errors
61 // on platforms where bswap_NN happens to be a function instead of a define.
62 
63 /* Swap bytes in 32 bit value. */
64 #ifdef bswap_32
65 #define kmail_swap_32(x) bswap_32(x)
66 #else
67 #define kmail_swap_32(x) \
68  ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >> 8) | \
69  (((x) & 0x0000ff00) << 8) | (((x) & 0x000000ff) << 24))
70 #endif
71 
72 #include <stdlib.h>
73 #include <sys/types.h>
74 #include <sys/stat.h>
75 #include <sys/file.h>
76 
77 KMFolderIndex::KMFolderIndex(KMFolder* folder, const char* name)
78  : FolderStorage(folder, name), mMsgList(INIT_MSGS)
79 {
80  mIndexStream = 0;
81  mIndexStreamPtr = 0;
82  mIndexStreamPtrLength = 0;
83  mIndexSwapByteOrder = false;
84  mIndexSizeOfLong = sizeof(long);
85  mIndexId = 0;
86  mHeaderOffset = 0;
87 }
88 
89 
90 KMFolderIndex::~KMFolderIndex()
91 {
92 }
93 
94 
96 {
97  TQString sLocation(folder()->path());
98 
99  if ( !sLocation.isEmpty() ) {
100  sLocation += '/';
101  sLocation += '.';
102  }
103  sLocation += dotEscape(fileName());
104  sLocation += ".index";
105 
106  return sLocation;
107 }
108 
110 {
111  if (!mAutoCreateIndex)
112  return 0;
113  bool dirty = mDirty;
114  mDirtyTimer->stop();
115  for ( unsigned int i = 0; !dirty && i < mMsgList.high(); i++ ) {
116  if ( mMsgList.at(i) ) {
117  if ( !mMsgList.at(i)->syncIndexString() ) {
118  dirty = true;
119  }
120  }
121  }
122  if (!dirty) { // Update successful
124  return 0;
125  }
126  return writeIndex();
127 }
128 
129 int KMFolderIndex::writeIndex( bool createEmptyIndex )
130 {
131  TQString tempName;
132  TQString indexName;
133  mode_t old_umask;
134  int len;
135  const uchar *buffer = 0;
136 
137  indexName = indexLocation();
138  tempName = indexName + ".temp";
139  unlink(TQFile::encodeName(tempName));
140 
141  // We touch the folder, otherwise the index is regenerated, if KMail is
142  // running, while the clock switches from daylight savings time to normal time
143  utime(TQFile::encodeName(location()), 0);
144 
145  old_umask = umask(077);
146  FILE *tmpIndexStream = fopen(TQFile::encodeName(tempName), "w");
147  umask(old_umask);
148  if (!tmpIndexStream)
149  return errno;
150 
151  fprintf(tmpIndexStream, "# KMail-Index V%d\n", INDEX_VERSION);
152 
153  // Header
154  TQ_UINT32 byteOrder = 0x12345678;
155  TQ_UINT32 sizeOfLong = sizeof(long);
156 
157  TQ_UINT32 header_length = sizeof(byteOrder)+sizeof(sizeOfLong);
158  char pad_char = '\0';
159  fwrite(&pad_char, sizeof(pad_char), 1, tmpIndexStream);
160  fwrite(&header_length, sizeof(header_length), 1, tmpIndexStream);
161 
162  // Write header
163  fwrite(&byteOrder, sizeof(byteOrder), 1, tmpIndexStream);
164  fwrite(&sizeOfLong, sizeof(sizeOfLong), 1, tmpIndexStream);
165 
166  off_t nho = ftell(tmpIndexStream);
167 
168  if ( !createEmptyIndex ) {
169  KMMsgBase* msgBase;
170  for (unsigned int i=0; i<mMsgList.high(); i++)
171  {
172  if (!(msgBase = mMsgList.at(i))) continue;
173  buffer = msgBase->asIndexString(len);
174  fwrite(&len,sizeof(len), 1, tmpIndexStream);
175 
176  off_t tmp = ftell(tmpIndexStream);
177  msgBase->setIndexOffset(tmp);
178  msgBase->setIndexLength(len);
179  if(fwrite(buffer, len, 1, tmpIndexStream) != 1)
180  kdDebug(5006) << "Whoa! " << __FILE__ << ":" << __LINE__ << endl;
181  }
182  }
183 
184  int fError = ferror( tmpIndexStream );
185  if( fError != 0 ) {
186  fclose( tmpIndexStream );
187  return fError;
188  }
189  if( ( fflush( tmpIndexStream ) != 0 )
190  || ( fsync( fileno( tmpIndexStream ) ) != 0 ) ) {
191  int errNo = errno;
192  fclose( tmpIndexStream );
193  return errNo;
194  }
195  if( fclose( tmpIndexStream ) != 0 )
196  return errno;
197 
198  ::rename(TQFile::encodeName(tempName), TQFile::encodeName(indexName));
199  mHeaderOffset = nho;
200  if (mIndexStream)
201  fclose(mIndexStream);
202 
203  if ( createEmptyIndex )
204  return 0;
205 
206  mIndexStream = fopen(TQFile::encodeName(indexName), "r+"); // index file
207  assert( mIndexStream );
208  fcntl(fileno(mIndexStream), F_SETFD, FD_CLOEXEC);
209 
210  updateIndexStreamPtr();
211 
213 
214  setDirty( false );
215  return 0;
216 }
217 
219 {
220  if ( contentsType() != KMail::ContentsTypeMail ) {
221  kdDebug(5006) << k_funcinfo << "Reading index for " << label() << endl;
222  }
223  TQ_INT32 len;
224  KMMsgInfo* mi;
225 
226  assert(mIndexStream != 0);
227  rewind(mIndexStream);
228 
229  clearIndex();
230  int version;
231 
232  setDirty( false );
233 
234  if (!readIndexHeader(&version)) return false;
235  //kdDebug(5006) << "Index version for " << label() << " is " << version << endl;
236 
237  mUnreadMsgs = 0;
238  mTotalMsgs = 0;
239  mHeaderOffset = ftell(mIndexStream);
240 
241  clearIndex();
242  while (!feof(mIndexStream))
243  {
244  mi = 0;
245  if(version >= 1505) {
246  if(!fread(&len, sizeof(len), 1, mIndexStream)) {
247  // Seems to be normal?
248  // kdDebug(5006) << k_funcinfo << " Unable to read length field!" << endl;
249  break;
250  }
251 
252  if (mIndexSwapByteOrder)
253  len = kmail_swap_32(len);
254 
255  off_t offs = ftell(mIndexStream);
256  if(fseek(mIndexStream, len, SEEK_CUR)) {
257  kdDebug(5006) << k_funcinfo << " Unable to seek to the end of the message!" << endl;
258  break;
259  }
260  mi = new KMMsgInfo(folder(), offs, len);
261  }
262  else
263  {
264  TQCString line(MAX_LINE);
265  fgets(line.data(), MAX_LINE, mIndexStream);
266  if (feof(mIndexStream)) break;
267  if (*line.data() == '\0') {
268  fclose(mIndexStream);
269  mIndexStream = 0;
270  clearIndex();
271  return false;
272  }
273  mi = new KMMsgInfo(folder());
274  mi->compat_fromOldIndexString(line, mConvertToUtf8);
275  }
276  if(!mi) {
277  kdDebug(5006) << k_funcinfo << " Unable to create message info object!" << endl;
278  break;
279  }
280 
281  if (mi->isDeleted())
282  {
283  delete mi; // skip messages that are marked as deleted
284  setDirty( true );
285  needsCompact = true; //We have deleted messages - needs to be compacted
286  continue;
287  }
288 #ifdef OBSOLETE
289  else if (mi->isNew())
290  {
291  mi->setStatus(KMMsgStatusUnread);
292  mi->setDirty(false);
293  }
294 #endif
295  if ((mi->isNew()) || (mi->isUnread()) ||
296  (folder() == kmkernel->outboxFolder()))
297  {
298  ++mUnreadMsgs;
299  if (mUnreadMsgs == 0) ++mUnreadMsgs;
300  }
301  mMsgList.append(mi, false);
302  }
303  if( version < 1505)
304  {
305  mConvertToUtf8 = false;
306  setDirty( true );
307  writeIndex();
308  }
309 
310  if ( version < 1507 ) {
311  updateInvitationAndAddressFieldsFromContents();
312  setDirty( true );
313  writeIndex();
314  }
315 
316  mTotalMsgs = mMsgList.count();
317  if ( contentsType() != KMail::ContentsTypeMail ) {
318  kdDebug(5006) << k_funcinfo << "Done reading the index for " << label() << ", we have " << mTotalMsgs << " messages." << endl;
319  }
320  return true;
321 }
322 
323 
324 int KMFolderIndex::count(bool cache) const
325 {
326  int res = FolderStorage::count(cache);
327  if (res == -1)
328  res = mMsgList.count();
329  return res;
330 }
331 
332 
334 {
335  int indexVersion;
336  assert(mIndexStream != 0);
337  mIndexSwapByteOrder = false;
338  mIndexSizeOfLong = sizeof(long);
339 
340  int ret = fscanf(mIndexStream, "# KMail-Index V%d\n", &indexVersion);
341  if ( ret == EOF || ret == 0 )
342  return false; // index file has invalid header
343  if(gv)
344  *gv = indexVersion;
345 
346  // Check if the index is corrupted ("not compactable") and recreate it if necessary. See
347  // FolderStorage::getMsg() for the detection code.
348  if ( !mCompactable ) {
349  kdWarning(5006) << "Index file " << indexLocation() << " is corrupted!!. Re-creating it." << endl;
350  recreateIndex( false /* don't call readIndex() afterwards */ );
351  return false;
352  }
353 
354  if (indexVersion < 1505 ) {
355  if(indexVersion == 1503) {
356  kdDebug(5006) << "Converting old index file " << indexLocation() << " to utf-8" << endl;
357  mConvertToUtf8 = true;
358  }
359  return true;
360  } else if (indexVersion == 1505) {
361  } else if (indexVersion < INDEX_VERSION && indexVersion != 1506) {
362  kdDebug(5006) << "Index file " << indexLocation() << " is out of date. Re-creating it." << endl;
364  return false;
365  } else if(indexVersion > INDEX_VERSION) {
366  kapp->setOverrideCursor(KCursor::arrowCursor());
367  int r = KMessageBox::questionYesNo(0,
368  i18n(
369  "The mail index for '%1' is from an unknown version of KMail (%2).\n"
370  "This index can be regenerated from your mail folder, but some "
371  "information, including status flags, may be lost. Do you wish "
372  "to downgrade your index file?") .arg(name()) .arg(indexVersion), TQString(), i18n("Downgrade"), i18n("Do Not Downgrade") );
373  kapp->restoreOverrideCursor();
374  if (r == KMessageBox::Yes)
376  return false;
377  }
378  else {
379  // Header
380  TQ_UINT32 byteOrder = 0;
381  TQ_UINT32 sizeOfLong = sizeof(long); // default
382 
383  TQ_UINT32 header_length = 0;
384  fseek(mIndexStream, sizeof(char), SEEK_CUR );
385  fread(&header_length, sizeof(header_length), 1, mIndexStream);
386  if (header_length > 0xFFFF)
387  header_length = kmail_swap_32(header_length);
388 
389  off_t endOfHeader = ftell(mIndexStream) + header_length;
390 
391  bool needs_update = true;
392  // Process available header parts
393  if (header_length >= sizeof(byteOrder))
394  {
395  fread(&byteOrder, sizeof(byteOrder), 1, mIndexStream);
396  mIndexSwapByteOrder = (byteOrder == 0x78563412);
397  header_length -= sizeof(byteOrder);
398 
399  if (header_length >= sizeof(sizeOfLong))
400  {
401  fread(&sizeOfLong, sizeof(sizeOfLong), 1, mIndexStream);
402  if (mIndexSwapByteOrder)
403  sizeOfLong = kmail_swap_32(sizeOfLong);
404  mIndexSizeOfLong = sizeOfLong;
405  header_length -= sizeof(sizeOfLong);
406  needs_update = false;
407  }
408  }
409  if (needs_update || mIndexSwapByteOrder || (mIndexSizeOfLong != sizeof(long)))
410  setDirty( true );
411  // Seek to end of header
412  fseek(mIndexStream, endOfHeader, SEEK_SET );
413 
414  if (mIndexSwapByteOrder)
415  kdDebug(5006) << "Index File has byte order swapped!" << endl;
416  if (mIndexSizeOfLong != sizeof(long))
417  kdDebug(5006) << "Index File sizeOfLong is " << mIndexSizeOfLong << " while sizeof(long) is " << sizeof(long) << " !" << endl;
418 
419  }
420  return true;
421 }
422 
423 
424 #ifdef HAVE_MMAP
425 bool KMFolderIndex::updateIndexStreamPtr(bool just_close)
426 #else
427 bool KMFolderIndex::updateIndexStreamPtr(bool)
428 #endif
429 {
430  // We touch the folder, otherwise the index is regenerated, if KMail is
431  // running, while the clock switches from daylight savings time to normal time
432  utime(TQFile::encodeName(location()), 0);
433  utime(TQFile::encodeName(indexLocation()), 0);
434  utime(TQFile::encodeName( KMMsgDict::getFolderIdsLocation( *this ) ), 0);
435 
436  mIndexSwapByteOrder = false;
437 #ifdef HAVE_MMAP
438  if(just_close) {
439  if(mIndexStreamPtr)
440  munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
441  mIndexStreamPtr = 0;
442  mIndexStreamPtrLength = 0;
443  return true;
444  }
445 
446  assert(mIndexStream);
447  struct stat stat_buf;
448  if(fstat(fileno(mIndexStream), &stat_buf) == -1) {
449  if(mIndexStreamPtr)
450  munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
451  mIndexStreamPtr = 0;
452  mIndexStreamPtrLength = 0;
453  return false;
454  }
455  if(mIndexStreamPtr)
456  munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
457  mIndexStreamPtrLength = stat_buf.st_size;
458  mIndexStreamPtr = (uchar *)mmap(0, mIndexStreamPtrLength, PROT_READ, MAP_SHARED,
459  fileno(mIndexStream), 0);
460  if(mIndexStreamPtr == MAP_FAILED) {
461  mIndexStreamPtr = 0;
462  mIndexStreamPtrLength = 0;
463  return false;
464  }
465 #endif
466  return true;
467 }
468 
469 
471 {
472  if ( !mCompactable )
473  return IndexCorrupt;
474 
475  TQFileInfo contInfo(location());
476  TQFileInfo indInfo(indexLocation());
477 
478  if (!contInfo.exists()) return KMFolderIndex::IndexOk;
479  if (!indInfo.exists()) return KMFolderIndex::IndexMissing;
480 
481  return ( contInfo.lastModified() > indInfo.lastModified() )
482  ? KMFolderIndex::IndexTooOld
483  : KMFolderIndex::IndexOk;
484 }
485 
486 void KMFolderIndex::clearIndex(bool autoDelete, bool syncDict)
487 {
488  mMsgList.clear(autoDelete, syncDict);
489 }
490 
491 
492 void KMFolderIndex::truncateIndex()
493 {
494  if ( mHeaderOffset )
495  truncate(TQFile::encodeName(indexLocation()), mHeaderOffset);
496  else
497  // The index file wasn't opened, so we don't know the header offset.
498  // So let's just create a new empty index.
499  writeIndex( true );
500 }
501 
503 {
504  open("fillDict");
505  for (unsigned int idx = 0; idx < mMsgList.high(); idx++)
506  if ( mMsgList.at( idx ) )
507  KMMsgDict::mutableInstance()->insert(0, mMsgList.at( idx ), idx);
508  close("fillDict");
509 }
510 
511 
512 KMMsgInfo* KMFolderIndex::setIndexEntry( int idx, KMMessage *msg )
513 {
514  KMMsgInfo *msgInfo = msg->msgInfo();
515  if ( !msgInfo )
516  msgInfo = new KMMsgInfo( folder() );
517 
518  *msgInfo = *msg;
519  mMsgList.set( idx, msgInfo );
520  msg->setMsgInfo( 0 );
521  delete msg;
522  return msgInfo;
523 }
524 
525 void KMFolderIndex::recreateIndex( bool readIndexAfterwards )
526 {
527  kapp->setOverrideCursor(KCursor::arrowCursor());
528  KMessageBox::information(0,
529  i18n("The mail index for '%1' is corrupted and will be regenerated now, "
530  "but some information, like status flags, might get lost.").arg(name()));
531  kapp->restoreOverrideCursor();
533  if ( readIndexAfterwards ) {
534  readIndex();
535  }
536 
537  // Clear the corrupted flag
538  mCompactable = true;
539  writeConfig();
540 }
541 
542 void KMFolderIndex::silentlyRecreateIndex()
543 {
544  Q_ASSERT( !isOpened() );
545  open( "silentlyRecreateIndex" );
546  KCursorSaver busy( KBusyPtr::busy() );
548  mCompactable = true;
549  writeConfig();
550  close( "silentlyRecreateIndex" );
551 }
552 
553 void KMFolderIndex::updateInvitationAndAddressFieldsFromContents()
554 {
555  kdDebug(5006) << "Updating index for " << label() << ", this might take a while." << endl;
556  for ( uint i = 0; i < mMsgList.size(); i++ ) {
557  KMMsgInfo * const msgInfo = dynamic_cast<KMMsgInfo*>( mMsgList[i] );
558  if ( msgInfo ) {
559  DwString msgString( getDwString( i ) );
560  if ( msgString.size() > 0 ) {
561  KMMessage msg;
562  msg.fromDwString( msgString, false );
563  msg.updateInvitationState();
564  if ( msg.status() & KMMsgStatusHasInvitation ) {
565  msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasInvitation );
566  }
567  if ( msg.status() & KMMsgStatusHasNoInvitation ) {
568  msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasNoInvitation );
569  }
570  msgInfo->setFrom( msg.from() );
571  msgInfo->setTo( msg.to() );
572  }
573  }
574  }
575 }
576 
577 #include "kmfolderindex.moc"