maildir-sync.c revision de4288b7369945a31c4001add9445fd0195a358d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Here's a description of how we handle Maildir synchronization and
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen it's problems:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen We want to be as efficient as we can. The most efficient way to
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen check if changes have occured is to stat() the new/ and cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen directories and uidlist file - if their mtimes haven't changed,
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen there's no changes and we don't need to do anything.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Problem 1: Multiple changes can happen within a single second -
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen nothing guarantees that once we synced it, someone else didn't just
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen then make a modification. Such modifications wouldn't get noticed
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen until a new modification occured later.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Problem 2: Syncing cur/ directory is much more costly than syncing
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen new/. Moving mails from new/ to cur/ will always change mtime of
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen cur/ causing us to sync it as well.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Problem 3: We may not be able to move mail from new/ to cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen because we're out of quota, or simply because we're accessing a
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen read-only mailbox.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen MAILDIR_SYNC_SECS
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen -----------------
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Several checks below use MAILDIR_SYNC_SECS, which should be maximum
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen clock drift between all computers accessing the maildir (eg. via
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen NFS), rounded up to next second. Our default is 1 second, since
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen everyone should be using NTP.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Note that setting it to 0 works only if there's only one computer
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen accessing the maildir. It's practically impossible to make two
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen clocks _exactly_ synchronized.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen It might be possible to only use file server's clock by looking at
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen the atime field, but I don't know how well that would actually work.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen cur directory
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen -------------
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen We have dirty_cur_time variable which is set to cur/ directory's
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen synchronized the directory.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen When dirty_cur_time is non-zero, we don't synchronize the cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen directory until
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen a) cur/'s mtime changes
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen b) opening a mail fails with ENOENT
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen This allows us to modify the maildir multiple times without having
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen to sync it at every change. The sync will eventually be done to
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen make sure we didn't miss any external changes.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen The dirty_cur_time is set when:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - we change message flags
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - we expunge messages
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - we move mail from new/ to cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen It's unset when we do the final syncing, ie. when mtime is
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen older than time() - MAILDIR_SYNC_SECS.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen new directory
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen -------------
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen it. dirty_cur_time-like feature might save us a few syncs, but
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen that might break a client which saves a mail in one connection and
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen tries to fetch it in another one. new/ directory is almost always
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen empty, so syncing it should be very fast anyway. Actually this can
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen still happen if we sync only new/ dir while another client is also
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen moving mails from it to cur/ - it takes us a while to see them.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen That's pretty unlikely to happen however, and only way to fix it
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen would be to always synchronize cur/ after new/.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Normally we move all mails from new/ to cur/ whenever we sync it. If
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen it's not possible for some reason, we mark the mail with "probably
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen exists in new/ directory" flag.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen If rename() still fails because of ENOSPC or EDQUOT, we still save
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen the flag changes in index with dirty-flag on. When moving the mail
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen to cur/ directory, or when we notice it's already moved there, we
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen apply the flag changes to the filename, rename it and remove the
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen dirty flag. If there's dirty flags, this should be tried every time
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen after expunge or when closing the mailbox.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen This file contains UID <-> filename mappings. It's updated only when
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen new mail arrives, so it may contain filenames that have already been
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen deleted. Updating is done by getting uidlist.lock file, writing the
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen whole uidlist into it and rename()ing it over the old uidlist. This
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen means there's no need to lock the file for reading.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Whenever uidlist is rewritten, it's mtime must be larger than the old
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen one's. Use utime() before rename() if needed. Note that inode checking
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen wouldn't have been sufficient as inode numbers can be reused.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen This file is usually read the first time you need to know filename for
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen given UID. After that it's not re-read unless new mails come that we
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen don't know about.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen broken clients
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen --------------
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Originally the middle identifier in Maildir filename was specified
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen only as <process id>_<delivery counter>. That however created a
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen problem with randomized PIDs which made it possible that the same
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen PID was reused within one second.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen So if within one second a mail was delivered, MUA moved it to cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen and another mail was delivered by a new process using same PID as
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen the first one, we likely ended up overwriting the first mail when
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen the second mail was moved over it.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Nowadays everyone should be giving a bit more specific identifier,
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen for example include microseconds in it which Dovecot does.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen There's a simple way to prevent this from happening in some cases:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Don't move the mail from new/ to cur/ if it's mtime is >= time() -
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen MAILDIR_SYNC_SECS. The second delivery's link() call then fails
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen because the file is already in new/, and it will then use a
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen different filename. There's a few problems with this however:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - it requires extra stat() call which is unneeded extra I/O
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - another MUA might still move the mail to cur/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen - if first file's flags are modified by either Dovecot or another
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen but that'd be ugly)
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Because this is useful only for very few people and it requires
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen extra I/O, I decided not to implement this. It should be however
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen quite easy to do since we need to be able to deal with files in new/
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen It's also possible to never accidentally overwrite a mail by using
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen link() + unlink() rather than rename(). This however isn't very
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen good idea as it introduces potential race conditions when multiple
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen clients are accessing the mailbox:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen Trying to move the same mail from new/ to cur/ at the same time:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen a) Client 1 uses slightly different filename than client 2,
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen for example one sets read-flag on but the other doesn't.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen You have the same mail duplicated now.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen b) Client 3 sees the mail between Client 1's and 2's link() calls
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen and changes it's flag. You have the same mail duplicated now.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen And it gets worse when they're unlink()ing in cur/ directory:
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen c) Client 1 changes mails's flag and client 2 changes it back
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen between 1's link() and unlink(). The mail is now expunged.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen d) If you try to deal with the duplicates by unlink()ing another
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen one of them, you might end up unlinking both of them.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen So, what should we do then if we notice a duplicate? First of all,
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen it might not be a duplicate at all, readdir() might have just
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen returned it twice because it was just renamed. What we should do is
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen create a completely new base name for it and rename() it to that.
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen If the call fails with ENOENT, it only means that it wasn't a
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen duplicate after all.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_expunge(struct index_mailbox *ibox, const char *path,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen struct maildir_index_sync_context *ctx = context;
d8b77aef97e89f1ccc5cbdaef77be9052279e35fTimo Sirainen (void)maildir_filename_get_flags(path, &flags, keywords);
d8b77aef97e89f1ccc5cbdaef77be9052279e35fTimo Sirainen mail_index_sync_flags_apply(&ctx->sync_rec, &flags8, keywords);
d8b77aef97e89f1ccc5cbdaef77be9052279e35fTimo Sirainen newpath = maildir_filename_set_flags(path, flags8, keywords);
b92813e2f96d4b28f989528ed5dd6115da7d9bdbTimo Sirainen if ((flags8 & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_sync_record(struct index_mailbox *ibox,
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen /* make it go through sequences to avoid looping through huge
659fe5d24825b160cae512538088020d97a60239Timo Sirainen holes in UID range */
659fe5d24825b160cae512538088020d97a60239Timo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_uid(view, seq, &uid) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (maildir_file_do(ibox, uid, maildir_expunge,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
659fe5d24825b160cae512538088020d97a60239Timo Sirainen for (ctx->seq = seq1; ctx->seq <= seq2; ctx->seq++) {
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
b92813e2f96d4b28f989528ed5dd6115da7d9bdbTimo Sirainen /* flag isn't dirty anymore */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint maildir_sync_last_commit(struct index_mailbox *ibox)
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen ret = mail_index_sync_begin(ibox->index, &ctx.sync_ctx, &ctx.view,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen ctx.trans = mail_index_transaction_begin(ctx.view, FALSE);
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen while ((ret = mail_index_sync_next(ctx.sync_ctx,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen if (mail_index_transaction_commit(ctx.trans, &seq, &offset) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmaildir_sync_context_new(struct index_mailbox *ibox)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void maildir_sync_deinit(struct maildir_sync_context *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen old_path = t_strconcat(dir, "/", old_fname, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "rename(%s, %s) failed: %m", old_path, new_path);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_storage *storage = ctx->ibox->box.storage;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *dir;
c282435b57b6f9696fc12d99ea70468b7bdfe24cTimo Sirainen move_new = new_dir && !mailbox_is_readonly(&ctx->ibox->box) &&
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen /* new file and we couldn't lock uidlist, check this
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen later in next sync. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen /* we moved it - it's \Recent for us */
641f0c0900ee6e7cf9667f4b40ed95cec7d0cdcaTimo Sirainen /* someone else moved it already */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* not enough disk space, leave here */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "rename(%s, %s) failed: %m",
641f0c0900ee6e7cf9667f4b40ed95cec7d0cdcaTimo Sirainen } else if (new_dir) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* possibly duplicate - try fixing it */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int maildir_sync_quick_check(struct maildir_sync_context *ctx,
bde6382cf65fba6165dc3603f5419e194d87f404Timo Sirainen /* cur stamp is kept in index, we don't have to sync if
bde6382cf65fba6165dc3603f5419e194d87f404Timo Sirainen someone else has done it and updated the index. make sure
bde6382cf65fba6165dc3603f5419e194d87f404Timo Sirainen we have a fresh index with latest sync_stamp. */
0c17af9d3f9323136a94e66605776ed4462a172dTimo Sirainen new_mtime >= ibox->last_new_sync_time - MAILDIR_SYNC_SECS) {
033557e1c8ebec5ae31f2f24fab90226a1945168Timo Sirainen ioloop_time - ibox->dirty_cur_time > MAILDIR_SYNC_SECS)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* cur/ changed, or delayed cur/ check */
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
e10c0b544b2516e92c5df9ef778e2a826eb02995Timo Sirainenint maildir_sync_index(struct index_mailbox *ibox, int partial)
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen if (mail_index_sync_begin(ibox->index, &sync_ctx.sync_ctx, &view,
de4288b7369945a31c4001add9445fd0195a358dTimo Sirainen /* view is invalidated */
de4288b7369945a31c4001add9445fd0195a358dTimo Sirainen (void)mail_index_sync_rollback(sync_ctx.sync_ctx);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen uid_validity = maildir_uidlist_get_uid_validity(ibox->uidlist);
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen uid_validity != 0 && hdr->uid_validity != 0) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* uidvalidity changed and mailbox isn't being initialized,
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen index must be rebuilt */
6969f13267ad8495a132ece39d34be36b9883f37Timo Sirainen "Maildir %s sync: UIDVALIDITY changed (%u -> %u)",
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen (void)mail_index_sync_rollback(sync_ctx.sync_ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen trans = mail_index_transaction_begin(view, FALSE);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen iter = maildir_uidlist_iter_init(ibox->uidlist);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
d8b77aef97e89f1ccc5cbdaef77be9052279e35fTimo Sirainen maildir_filename_get_flags(filename, &flags, keywords);
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen (uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen (uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen /* mail is recent for next session as well */
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen /* most likely a race condition: we read the
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen maildir, then someone else expunged messages
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen and committed changes to index. so, this
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen message shouldn't actually exist. mark it
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen racy and check in next sync.
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen the difference between this and the later
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen check is that this one happens when messages
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen are expunged from the end */
a33fde36c57839342dafdf3c8d4b5c009e3ef4bcTimo Sirainen /* partial syncing */
6969f13267ad8495a132ece39d34be36b9883f37Timo Sirainen "Maildir %s sync: "
6969f13267ad8495a132ece39d34be36b9883f37Timo Sirainen "UID < next_uid "
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen "(%u < %u, file = %s)",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_update_flags(trans, seq, MODIFY_REPLACE,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* expunged */
a038139a470d2942759b9b86a9852aee7b460996Timo Sirainen /* most likely a race condition: we read the
a038139a470d2942759b9b86a9852aee7b460996Timo Sirainen maildir, then someone else expunged messages and
a038139a470d2942759b9b86a9852aee7b460996Timo Sirainen committed changes to index. so, this message
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen shouldn't actually exist. mark it racy and check
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen in next sync. */
a33fde36c57839342dafdf3c8d4b5c009e3ef4bcTimo Sirainen /* partial syncing */
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
0e2686dfe29a18772fa4026bad53e2c7c560403fTimo Sirainen "Maildir %s sync: "
0e2686dfe29a18772fa4026bad53e2c7c560403fTimo Sirainen "UID inserted in the middle of mailbox "
a038139a470d2942759b9b86a9852aee7b460996Timo Sirainen "(%u > %u, file = %s)",
316689d0ef55e2fa4e2fb4ac5b1ea35ce65688d3Timo Sirainen maildir_uidlist_add_flags(ibox->uidlist, filename,
a33fde36c57839342dafdf3c8d4b5c009e3ef4bcTimo Sirainen if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
a33fde36c57839342dafdf3c8d4b5c009e3ef4bcTimo Sirainen /* partial syncing */
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen /* we haven't been able to update maildir with this
fd3d711f219fd6813492acbe051e04327f0ca0f0Timo Sirainen record's flag changes. don't sync them. */
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen (rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT)) ||
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen /* FIXME: this is wrong if there's pending changes in
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen transaction log already. it gets fixed in next sync
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_update_flags(trans, seq, MODIFY_REPLACE,
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen /* just remove recent flag */
d482b35af87f5fd872bad007da0475813a401a49Timo Sirainen mail_index_update_flags(trans, seq, MODIFY_REMOVE,
b7b81543899e306c71e6152516d8698416162bcbTimo Sirainen /* expunge the rest */
b7b81543899e306c71e6152516d8698416162bcbTimo Sirainen for (seq++; seq <= hdr->messages_count; seq++)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen /* now, sync the index */
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen while ((ret = mail_index_sync_next(sync_ctx.sync_ctx,
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen if (maildir_sync_record(ibox, &sync_ctx) < 0) {
bde6382cf65fba6165dc3603f5419e194d87f404Timo Sirainen ibox->last_cur_mtime != (time_t)hdr->sync_stamp) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen /* get the initial uidvalidity */
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen if (maildir_uidlist_update(ibox->uidlist) < 0)
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen uid_validity = maildir_uidlist_get_uid_validity(ibox->uidlist);
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen maildir_uidlist_set_uid_validity(ibox->uidlist,
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen } else if (uid_validity == 0) {
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen maildir_uidlist_set_uid_validity(ibox->uidlist,
92888ef30960c30ccc9e030fe7eab5d4d04a7d1cTimo Sirainen if (uid_validity != hdr->uid_validity && uid_validity != 0) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen offsetof(struct mail_index_header, uid_validity),
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen next_uid = maildir_uidlist_get_next_uid(ibox->uidlist);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if (next_uid != 0 && hdr->next_uid != next_uid) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
8a3d609fdd84f5938c82e8e7eeb84a24ab41b317Timo Sirainen else if (seq != 0) {
d9de52132072d80b8c268094b879c0ef5a108db3Timo Sirainen if (mail_index_sync_commit(sync_ctx.sync_ctx) < 0)
a38ef15060e45e5060bb24c403a580b9a57a818cTimo Sirainenstatic int maildir_sync_context(struct maildir_sync_context *ctx, int forced)
a38ef15060e45e5060bb24c403a580b9a57a818cTimo Sirainen if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
d77c309fccbc6a7594f8cb08fb01009fa613c568Timo Sirainen /* we have to lock uidlist immediately, otherwise there's race
d77c309fccbc6a7594f8cb08fb01009fa613c568Timo Sirainen conditions with other processes who might write older maildir
d77c309fccbc6a7594f8cb08fb01009fa613c568Timo Sirainen file list into uidlist.
d77c309fccbc6a7594f8cb08fb01009fa613c568Timo Sirainen alternative would be to lock it when new files are found, but
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen the directory scans _must_ be restarted then.
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen if we got here through maildir_sync_last_commit(), we can't sync
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen index as it's already being synced. so, don't try locking uidlist
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen either, we only want to find new filename for some mail.
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen if ((ret = maildir_uidlist_try_lock(ctx->ibox->uidlist)) < 0)
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen /* we didn't get a lock, don't do syncing unless we
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen really want to check for expunges or renames. new
65d6ca3fb5450b81df0190d9e9aa62c00fed5116Timo Sirainen files won't be added. */
73c76fa7340a107229c530196d026aadeae979c7Timo Sirainen maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen /* finish uidlist syncing, but keep it still locked */
73c76fa7340a107229c530196d026aadeae979c7Timo Sirainen maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
e10c0b544b2516e92c5df9ef778e2a826eb02995Timo Sirainen if (maildir_sync_index(ctx->ibox, ctx->partial) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
a38ef15060e45e5060bb24c403a580b9a57a818cTimo Sirainenint maildir_storage_sync_force(struct index_mailbox *ibox)
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainenmaildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct index_mailbox *ibox = (struct index_mailbox *)box;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {