Cross Reference: /dovecot/src/lib-storage/index/maildir/maildir-sync.c
maildir-sync.c revision b20fb5b1df9d604a7541f5118fc5b4b466d211ef
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/*
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Here's a description of how we handle Maildir synchronization and
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it's problems:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e8bdf1be00aec45d0c6dd72ad9c8be02a3dfc778Timo Sirainen We want to be as efficient as we can. The most efficient way to
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen check if changes have occured is to stat() the new/ and cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen directories and uidlist file - if their mtimes haven't changed,
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen there's no changes and we don't need to do anything.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 1: Multiple changes can happen within a single second -
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen nothing guarantees that once we synced it, someone else didn't just
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen then make a modification. Such modifications wouldn't get noticed
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen until a new modification occured later.
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 2: Syncing cur/ directory is much more costly than syncing
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new/. Moving mails from new/ to cur/ will always change mtime of
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cur/ causing us to sync it as well.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Problem 3: We may not be able to move mail from new/ to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen because we're out of quota, or simply because we're accessing a
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen read-only mailbox.
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MAILDIR_SYNC_SECS
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -----------------
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen Several checks below use MAILDIR_SYNC_SECS, which should be maximum
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen clock drift between all computers accessing the maildir (eg. via
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen NFS), rounded up to next second. Our default is 1 second, since
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen everyone should be using NTP.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Note that setting it to 0 works only if there's only one computer
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen accessing the maildir. It's practically impossible to make two
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen clocks _exactly_ synchronized.
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It might be possible to only use file server's clock by looking at
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the atime field, but I don't know how well that would actually work.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cur directory
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen -------------
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen We have dirty_cur_time variable which is set to cur/ directory's
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen synchronized the directory.
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen When dirty_cur_time is non-zero, we don't synchronize the cur/
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen directory until
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen a) cur/'s mtime changes
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen b) opening a mail fails with ENOENT
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen This allows us to modify the maildir multiple times without having
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen to sync it at every change. The sync will eventually be done to
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen make sure we didn't miss any external changes.
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen The dirty_cur_time is set when:
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen - we change message flags
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we expunge messages
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we move mail from new/ to cur/
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
2526d52441ef368215ab6bf04fd0356d3b09d235Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen It's unset when we do the final syncing, ie. when mtime is
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen older than time() - MAILDIR_SYNC_SECS.
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new directory
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen -------------
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it. dirty_cur_time-like feature might save us a few syncs, but
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen that might break a client which saves a mail in one connection and
55773f17bccf6361d6599ffcbe072d7c9fe205bfTimo Sirainen tries to fetch it in another one. new/ directory is almost always
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen empty, so syncing it should be very fast anyway. Actually this can
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen still happen if we sync only new/ dir while another client is also
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen moving mails from it to cur/ - it takes us a while to see them.
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen That's pretty unlikely to happen however, and only way to fix it
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen would be to always synchronize cur/ after new/.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Normally we move all mails from new/ to cur/ whenever we sync it. If
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen it's not possible for some reason, we mark the mail with "probably
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen exists in new/ directory" flag.
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen If rename() still fails because of ENOSPC or EDQUOT, we still save
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen the flag changes in index with dirty-flag on. When moving the mail
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen to cur/ directory, or when we notice it's already moved there, we
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen apply the flag changes to the filename, rename it and remove the
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen dirty flag. If there's dirty flags, this should be tried every time
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen after expunge or when closing the mailbox.
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen uidlist
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen -------
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen This file contains UID <-> filename mappings. It's updated only when
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen new mail arrives, so it may contain filenames that have already been
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen deleted. Updating is done by getting uidlist.lock file, writing the
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen whole uidlist into it and rename()ing it over the old uidlist. This
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen means there's no need to lock the file for reading.
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen Whenever uidlist is rewritten, it's mtime must be larger than the old
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen one's. Use utime() before rename() if needed. Note that inode checking
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen wouldn't have been sufficient as inode numbers can be reused.
c11155a446dbdc9f6cd5b954f09073a9019e27b2Timo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen This file is usually read the first time you need to know filename for
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen given UID. After that it's not re-read unless new mails come that we
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen don't know about.
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen broken clients
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen --------------
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Originally the middle identifier in Maildir filename was specified
8000c86be02008b74acc71fa422444dc432e2c01Timo Sirainen only as <process id>_<delivery counter>. That however created a
8000c86be02008b74acc71fa422444dc432e2c01Timo Sirainen problem with randomized PIDs which made it possible that the same
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen PID was reused within one second.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen So if within one second a mail was delivered, MUA moved it to cur/
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen and another mail was delivered by a new process using same PID as
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen the first one, we likely ended up overwriting the first mail when
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen the second mail was moved over it.
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen Nowadays everyone should be giving a bit more specific identifier,
a63bc30f167a221691ad0a3fe5cd45c57444411eTimo Sirainen for example include microseconds in it which Dovecot does.
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen There's a simple way to prevent this from happening in some cases:
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen Don't move the mail from new/ to cur/ if it's mtime is >= time() -
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen MAILDIR_SYNC_SECS. The second delivery's link() call then fails
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen because the file is already in new/, and it will then use a
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen different filename. There's a few problems with this however:
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen - it requires extra stat() call which is unneeded extra I/O
a63bc30f167a221691ad0a3fe5cd45c57444411eTimo Sirainen - another MUA might still move the mail to cur/
a63bc30f167a221691ad0a3fe5cd45c57444411eTimo Sirainen - if first file's flags are modified by either Dovecot or another
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen but that'd be ugly)
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen Because this is useful only for very few people and it requires
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen extra I/O, I decided not to implement this. It should be however
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen quite easy to do since we need to be able to deal with files in new/
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen in any case.
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen It's also possible to never accidentally overwrite a mail by using
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen link() + unlink() rather than rename(). This however isn't very
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen good idea as it introduces potential race conditions when multiple
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen clients are accessing the mailbox:
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen Trying to move the same mail from new/ to cur/ at the same time:
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen a) Client 1 uses slightly different filename than client 2,
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen for example one sets read-flag on but the other doesn't.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen You have the same mail duplicated now.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen b) Client 3 sees the mail between Client 1's and 2's link() calls
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen and changes it's flag. You have the same mail duplicated now.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen And it gets worse when they're unlink()ing in cur/ directory:
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen c) Client 1 changes mails's flag and client 2 changes it back
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen between 1's link() and unlink(). The mail is now expunged.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen d) If you try to deal with the duplicates by unlink()ing another
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen one of them, you might end up unlinking both of them.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen So, what should we do then if we notice a duplicate? First of all,
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen it might not be a duplicate at all, readdir() might have just
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen returned it twice because it was just renamed. What we should do is
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen create a completely new base name for it and rename() it to that.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen If the call fails with ENOENT, it only means that it wasn't a
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen duplicate after all.
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen*/
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "lib.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "ioloop.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "buffer.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "hash.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "str.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "maildir-storage.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include "maildir-uidlist.h"
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include <stdio.h>
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include <stddef.h>
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include <unistd.h>
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include <dirent.h>
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#include <sys/stat.h>
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#define MAILDIR_SYNC_SECS 1
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen#define MAILDIR_FILENAME_FLAG_FOUND 128
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainenstruct maildir_sync_context {
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct index_mailbox *ibox;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen const char *new_dir, *cur_dir;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen int partial;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct maildir_index_sync_context *index_sync_ctx;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen};
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainenstruct maildir_index_sync_context {
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct index_mailbox *ibox;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_view *view;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_sync_ctx *sync_ctx;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_transaction *trans;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_sync_rec sync_rec;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen uint32_t seq;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen int dirty_state;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainenstatic int maildir_expunge(struct index_mailbox *ibox, const char *path,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen void *context __attr_unused__)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen if (unlink(path) == 0) {
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen ibox->dirty_cur_time = ioloop_time;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return 1;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (errno == ENOENT)
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return 0;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_storage_set_critical(ibox->box.storage,
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen "unlink(%s) failed: %m", path);
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen return -1;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen}
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainenstatic int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen void *context)
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen{
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen struct maildir_index_sync_context *ctx = context;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen const char *newpath;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen enum mail_flags flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint8_t flags8;
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen keywords_mask_t keywords;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen ctx->dirty_state = 0;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen (void)maildir_filename_get_flags(path, &flags, keywords);
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen flags8 = flags;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen mail_index_sync_flags_apply(&ctx->sync_rec, &flags8, keywords);
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen newpath = maildir_filename_set_flags(path, flags8, keywords);
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen if (rename(path, newpath) == 0) {
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen if ((flags8 & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen ctx->dirty_state = -1;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen ibox->dirty_cur_time = ioloop_time;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (errno == ENOENT)
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return 0;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen if (ENOSPACE(errno) || errno == EACCES) {
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen memset(keywords, 0, sizeof(keywords));
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD,
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen MAIL_INDEX_MAIL_FLAG_DIRTY, keywords);
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen ctx->dirty_state = 1;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return 1;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen }
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen mail_storage_set_critical(ibox->box.storage,
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen "rename(%s, %s) failed: %m", path, newpath);
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen return -1;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen}
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainenstatic int maildir_sync_record(struct index_mailbox *ibox,
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct maildir_index_sync_context *ctx)
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen{
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen struct mail_index_view *view = ctx->view;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen uint32_t seq, seq1, seq2, uid;
c36ec256c1bd1abe1c12e792cf64f0b7e3b3135aTimo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen switch (sync_rec->type) {
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen case MAIL_INDEX_SYNC_TYPE_APPEND:
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen break;
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen /* make it go through sequences to avoid looping through huge
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen holes in UID range */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen sync_rec->uid2,
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen &seq1, &seq2) < 0)
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen return -1;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen if (seq1 == 0)
683253c6f556b71d7946452644fc3d633f47153cTimo Sirainen break;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen for (seq = seq1; seq <= seq2; seq++) {
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen if (mail_index_lookup_uid(view, seq, &uid) < 0)
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen return -1;
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen if (maildir_file_do(ibox, uid, maildir_expunge,
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen NULL) < 0)
8754bb7a1f24705ffa5434f9e10d57e0b3b88d6eTimo Sirainen return -1;
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen }
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen break;
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen case MAIL_INDEX_SYNC_TYPE_FLAGS:
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen if (mail_index_lookup_uid_range(view, sync_rec->uid1,
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen sync_rec->uid2,
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainen &seq1, &seq2) < 0)
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen return -1;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (seq1 == 0)
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen break;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen for (ctx->seq = seq1; ctx->seq <= seq2; ctx->seq++) {
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen return -1;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (maildir_file_do(ibox, uid,
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen maildir_sync_flags, ctx) < 0)
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen return -1;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (ctx->dirty_state < 0) {
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen /* flag isn't dirty anymore */
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen keywords_mask_t keywords;
3ec2c1f31631bb5ff86f5fc93a563c33e5cae90dTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen memset(keywords, 0, sizeof(keywords));
944d096df698dd2b476e376277e9afd634b53270Timo Sirainen mail_index_update_flags(ctx->trans, ctx->seq,
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen MODIFY_REMOVE,
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen MAIL_INDEX_MAIL_FLAG_DIRTY, keywords);
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen }
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen }
b142deb9a831c89b1bb9129ada655f3e56b9d4ccTimo Sirainen break;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen }
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen return 0;
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen}
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainenint maildir_sync_last_commit(struct index_mailbox *ibox)
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen{
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen struct maildir_index_sync_context ctx;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen uint32_t seq;
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen uoff_t offset;
944d096df698dd2b476e376277e9afd634b53270Timo Sirainen int ret;
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen if (ibox->commit_log_file_seq == 0)
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen return 0;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen memset(&ctx, 0, sizeof(ctx));
97511ac4d7607e1ba64ce151eda3d9b5f9775519Timo Sirainen ctx.ibox = ibox;
97511ac4d7607e1ba64ce151eda3d9b5f9775519Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen ibox->syncing_commit = TRUE;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen ret = mail_index_sync_begin(ibox->index, &ctx.sync_ctx, &ctx.view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_offset, FALSE, FALSE);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret > 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx.trans = mail_index_transaction_begin(ctx.view, FALSE, TRUE);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen while ((ret = mail_index_sync_next(ctx.sync_ctx,
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen &ctx.sync_rec)) > 0) {
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen if (maildir_sync_record(ibox, &ctx) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ret = -1;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
b6a7e0a7899e7f5d60c23cdaa50e025e4c67d05fTimo Sirainen if (mail_index_transaction_commit(ctx.trans, &seq, &offset) < 0)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ret = -1;
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainen if (mail_index_sync_commit(ctx.sync_ctx) < 0)
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainen ret = -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ibox->syncing_commit = FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_seq = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ibox->commit_log_file_offset = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mail_storage_set_index_error(ibox);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return ret;
a12399903f415a7e14c2816cffa2f7a09dcbb097Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
439980f88f421039dea8335e92d3fa82b3f470a1Timo Sirainenstatic struct maildir_sync_context *
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainenmaildir_sync_context_new(struct index_mailbox *ibox)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct maildir_sync_context *ctx;
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen ctx = t_new(struct maildir_sync_context, 1);
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen ctx->ibox = ibox;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen ctx->new_dir = t_strconcat(ibox->path, "/new", NULL);
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ctx;
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
366d6311c9d5bac6613e3cd64619eb878adce9ecTimo Sirainenstatic void maildir_sync_deinit(struct maildir_sync_context *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen if (ctx->uidlist_sync_ctx != NULL)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->index_sync_ctx != NULL)
maildir_sync_index_abort(ctx->index_sync_ctx);
}
static int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir,
const char *old_fname)
{
const char *new_fname, *old_path, *new_path;
int ret = 0;
t_push();
old_path = t_strconcat(dir, "/", old_fname, NULL);
new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL);
if (rename(old_path, new_path) == 0) {
i_warning("Fixed duplicate in %s: %s -> %s",
ibox->path, old_fname, new_fname);
} else if (errno != ENOENT) {
mail_storage_set_critical(ibox->box.storage,
"rename(%s, %s) failed: %m", old_path, new_path);
ret = -1;
}
t_pop();
return ret;
}
static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
{
struct mail_storage *storage = ctx->ibox->box.storage;
const char *dir;
DIR *dirp;
string_t *src, *dest;
struct dirent *dp;
enum maildir_uidlist_rec_flag flags;
int move_new, ret = 1;
dir = new_dir ? ctx->new_dir : ctx->cur_dir;
dirp = opendir(dir);
if (dirp == NULL) {
mail_storage_set_critical(storage,
"opendir(%s) failed: %m", dir);
return -1;
}
t_push();
src = t_str_new(1024);
dest = t_str_new(1024);
move_new = new_dir && !mailbox_is_readonly(&ctx->ibox->box) &&
!ctx->ibox->keep_recent;
while ((dp = readdir(dirp)) != NULL) {
if (dp->d_name[0] == '.')
continue;
ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
dp->d_name);
if (ret == 0) {
/* new file and we couldn't lock uidlist, check this
later in next sync. */
if (new_dir)
ctx->ibox->last_new_mtime = 0;
else
ctx->ibox->dirty_cur_time = ioloop_time;
continue;
}
if (ret < 0)
break;
flags = 0;
if (move_new) {
str_truncate(src, 0);
str_truncate(dest, 0);
str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
if (strchr(dp->d_name, ':') == NULL)
str_append(dest, ":2,");
if (rename(str_c(src), str_c(dest)) == 0) {
/* we moved it - it's \Recent for us */
ctx->ibox->dirty_cur_time = ioloop_time;
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
} else if (ENOTFOUND(errno)) {
/* someone else moved it already */
flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;
} else if (ENOSPACE(errno)) {
/* not enough disk space, leave here */
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
move_new = FALSE;
} else {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
mail_storage_set_critical(storage,
"rename(%s, %s) failed: %m",
str_c(src), str_c(dest));
}
} else if (new_dir) {
flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
MAILDIR_UIDLIST_REC_FLAG_RECENT;
}
ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
dp->d_name, flags);
if (ret <= 0) {
if (ret < 0)
break;
/* possibly duplicate - try fixing it */
if (maildir_fix_duplicate(ctx->ibox,
dir, dp->d_name) < 0) {
ret = -1;
break;
}
}
}
if (closedir(dirp) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", dir);
}
t_pop();
return ret < 0 ? -1 : 0;
}
static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
int *new_changed_r, int *cur_changed_r)
{
struct index_mailbox *ibox = ctx->ibox;
struct stat st;
time_t new_mtime, cur_mtime;
*new_changed_r = *cur_changed_r = FALSE;
if (stat(ctx->new_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->new_dir);
return -1;
}
new_mtime = st.st_mtime;
if (stat(ctx->cur_dir, &st) < 0) {
mail_storage_set_critical(ibox->box.storage,
"stat(%s) failed: %m", ctx->cur_dir);
return -1;
}
cur_mtime = st.st_mtime;
if (ibox->dirty_cur_time == 0) {
/* cur stamp is kept in index, we don't have to sync if
someone else has done it and updated the index. make sure
we have a fresh index with latest sync_stamp. */
struct mail_index_view *view;
const struct mail_index_header *hdr;
if (mail_index_refresh(ibox->index) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
view = mail_index_view_open(ibox->index);
hdr = mail_index_get_header(view);
ibox->last_cur_mtime = hdr->sync_stamp;
mail_index_view_close(view);
}
if (new_mtime != ibox->last_new_mtime ||
new_mtime >= ibox->last_new_sync_time - MAILDIR_SYNC_SECS) {
*new_changed_r = TRUE;
ibox->last_new_mtime = new_mtime;
ibox->last_new_sync_time = ioloop_time;
}
if (cur_mtime != ibox->last_cur_mtime ||
(ibox->dirty_cur_time != 0 &&
ioloop_time - ibox->dirty_cur_time > MAILDIR_SYNC_SECS)) {
/* cur/ changed, or delayed cur/ check */
*cur_changed_r = TRUE;
ibox->last_cur_mtime = cur_mtime;
ibox->dirty_cur_time =
cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
cur_mtime : 0;
}
return 0;
}
struct maildir_index_sync_context *
maildir_sync_index_begin(struct index_mailbox *ibox)
{
struct maildir_index_sync_context *sync_ctx;
sync_ctx = i_new(struct maildir_index_sync_context, 1);
sync_ctx->ibox = ibox;
if (mail_index_sync_begin(ibox->index, &sync_ctx->sync_ctx,
&sync_ctx->view, (uint32_t)-1, (uoff_t)-1,
FALSE, FALSE) <= 0) {
mail_storage_set_index_error(ibox);
return NULL;
}
return sync_ctx;
}
void maildir_sync_index_abort(struct maildir_index_sync_context *sync_ctx)
{
mail_index_sync_rollback(sync_ctx->sync_ctx);
i_free(sync_ctx);
}
int maildir_sync_index_finish(struct maildir_index_sync_context *sync_ctx,
int partial)
{
struct index_mailbox *ibox = sync_ctx->ibox;
struct mail_index_view *view = sync_ctx->view;
struct maildir_uidlist_iter_ctx *iter;
struct mail_index_transaction *trans;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
uint32_t seq, uid;
enum maildir_uidlist_rec_flag uflags;
const char *filename;
enum mail_flags flags;
keywords_mask_t keywords;
uint32_t uid_validity, next_uid;
int ret;
hdr = mail_index_get_header(view);
uid_validity = maildir_uidlist_get_uid_validity(ibox->uidlist);
if (uid_validity != hdr->uid_validity &&
uid_validity != 0 && hdr->uid_validity != 0) {
/* uidvalidity changed and mailbox isn't being initialized,
index must be rebuilt */
mail_storage_set_critical(ibox->box.storage,
"Maildir %s sync: UIDVALIDITY changed (%u -> %u)",
ibox->path, hdr->uid_validity, uid_validity);
mail_index_mark_corrupted(ibox->index);
maildir_sync_index_abort(sync_ctx);
return -1;
}
trans = mail_index_transaction_begin(view, FALSE, TRUE);
sync_ctx->trans = trans;
seq = 0;
iter = maildir_uidlist_iter_init(ibox->uidlist);
while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
maildir_filename_get_flags(filename, &flags, keywords);
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 &&
(uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 &&
(uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) {
/* mail is recent for next session as well */
flags |= MAIL_RECENT;
}
__again:
seq++;
if (seq > hdr->messages_count) {
if (uid < hdr->next_uid) {
/* most likely a race condition: we read the
maildir, then someone else expunged messages
and committed changes to index. so, this
message shouldn't actually exist. mark it
racy and check in next sync.
the difference between this and the later
check is that this one happens when messages
are expunged from the end */
if ((uflags &
MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
/* partial syncing */
continue;
}
if ((uflags &
MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
mail_storage_set_critical(
ibox->box.storage,
"Maildir %s sync: "
"UID < next_uid "
"(%u < %u, file = %s)",
ibox->path, uid, hdr->next_uid,
filename);
mail_index_mark_corrupted(ibox->index);
ret = -1;
break;
}
ibox->dirty_cur_time = ioloop_time;
maildir_uidlist_add_flags(ibox->uidlist,
filename,
MAILDIR_UIDLIST_REC_FLAG_RACING);
seq--;
continue;
}
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, keywords);
continue;
}
if (mail_index_lookup(view, seq, &rec) < 0) {
ret = -1;
break;
}
if (rec->uid < uid) {
/* expunged */
mail_index_expunge(trans, seq);
goto __again;
}
if (rec->uid > uid) {
/* most likely a race condition: we read the
maildir, then someone else expunged messages and
committed changes to index. so, this message
shouldn't actually exist. mark it racy and check
in next sync. */
if ((uflags &
MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
/* partial syncing */
seq--;
continue;
}
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) {
mail_storage_set_critical(ibox->box.storage,
"Maildir %s sync: "
"UID inserted in the middle of mailbox "
"(%u > %u, file = %s)",
ibox->path, rec->uid, uid, filename);
mail_index_mark_corrupted(ibox->index);
ret = -1;
break;
}
ibox->dirty_cur_time = ioloop_time;
maildir_uidlist_add_flags(ibox->uidlist, filename,
MAILDIR_UIDLIST_REC_FLAG_RACING);
seq--;
continue;
}
if ((rec->flags & MAIL_RECENT) != 0) {
index_mailbox_set_recent(ibox, seq);
if (ibox->keep_recent) {
flags |= MAIL_RECENT;
} else {
mail_index_update_flags(trans, seq,
MODIFY_REMOVE,
MAIL_RECENT, keywords);
}
}
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
/* partial syncing */
continue;
}
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* we haven't been able to update maildir with this
record's flag changes. don't sync them. */
continue;
}
if (((uint8_t)flags & ~MAIL_RECENT) !=
(rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT)) ||
memcmp(keywords, rec->keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
/* FIXME: this is wrong if there's pending changes in
transaction log already. it gets fixed in next sync
however.. */
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
flags, keywords);
} else if ((flags & MAIL_RECENT) == 0 &&
(rec->flags & MAIL_RECENT) != 0) {
/* just remove recent flag */
memset(keywords, 0, sizeof(keywords));
mail_index_update_flags(trans, seq, MODIFY_REMOVE,
MAIL_RECENT, keywords);
}
}
maildir_uidlist_iter_deinit(iter);
if (!partial) {
/* expunge the rest */
for (seq++; seq <= hdr->messages_count; seq++)
mail_index_expunge(trans, seq);
}
/* now, sync the index */
ibox->syncing_commit = TRUE;
while ((ret = mail_index_sync_next(sync_ctx->sync_ctx,
&sync_ctx->sync_rec)) > 0) {
if (maildir_sync_record(ibox, sync_ctx) < 0) {
ret = -1;
break;
}
}
ibox->syncing_commit = FALSE;
if (ibox->dirty_cur_time == 0 &&
ibox->last_cur_mtime != (time_t)hdr->sync_stamp) {
uint32_t sync_stamp = ibox->last_cur_mtime;
mail_index_update_header(trans,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if (hdr->uid_validity == 0) {
/* get the initial uidvalidity */
if (maildir_uidlist_update(ibox->uidlist) < 0)
ret = -1;
uid_validity = maildir_uidlist_get_uid_validity(ibox->uidlist);
if (uid_validity == 0) {
uid_validity = ioloop_time;
maildir_uidlist_set_uid_validity(ibox->uidlist,
uid_validity);
}
} else if (uid_validity == 0) {
maildir_uidlist_set_uid_validity(ibox->uidlist,
hdr->uid_validity);
}
if (uid_validity != hdr->uid_validity && uid_validity != 0) {
mail_index_update_header(trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity));
}
next_uid = maildir_uidlist_get_next_uid(ibox->uidlist);
if (next_uid != 0 && hdr->next_uid != next_uid) {
mail_index_update_header(trans,
offsetof(struct mail_index_header, next_uid),
&next_uid, sizeof(next_uid));
}
if (ret < 0) {
mail_index_transaction_rollback(trans);
mail_index_sync_rollback(sync_ctx->sync_ctx);
} else {
uint32_t seq;
uoff_t offset;
if (mail_index_transaction_commit(trans, &seq, &offset) < 0)
ret = -1;
else if (seq != 0) {
ibox->commit_log_file_seq = seq;
ibox->commit_log_file_offset = offset;
}
if (mail_index_sync_commit(sync_ctx->sync_ctx) < 0)
ret = -1;
}
if (ret == 0) {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
} else {
mail_storage_set_index_error(ibox);
}
i_free(sync_ctx);
return ret;
}
static int maildir_sync_context(struct maildir_sync_context *ctx, int forced)
{
int ret, new_changed, cur_changed;
if (!forced) {
if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
return -1;
if (!new_changed && !cur_changed)
return 0;
} else {
new_changed = cur_changed = TRUE;
}
/*
Locking, locking, locking.. Wasn't maildir supposed to be lockless?
We can get here either as beginning a real maildir sync, or when
committing changes to maildir but a file was lost (maybe renamed).
So, we're going to need two locks. One for index and one for
uidlist. To avoid deadlocking do the index lock first.
uidlist is needed only for figuring out UIDs for newly seen files,
so theoretically we wouldn't need to lock it unless there are new
files. It has a few problems though, assuming the index lock didn't
already protect it (eg. in-memory indexes):
1. Just because you see a new file which doesn't exist in uidlist
file, doesn't mean that the file really exists anymore, or that
your readdir() lists all new files. Meaning that this is possible:
A: opendir(), readdir() -> new file ...
-- new files are written to the maildir --
B: opendir(), readdir() -> new file, lock uidlist,
readdir() -> another new file, rewrite uidlist, unlock
A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist,
unlock
The second time running A didn't see the two new files. To handle
this correctly, it must not remove the new unseen files from
uidlist. This is possible to do, but adds extra complexity.
2. If another process is rename()ing files while we are
readdir()ing, it's possible that readdir() never lists some files,
causing Dovecot to assume they were expunged. In next sync they
would show up again, but client could have already been notified of
that and they would show up under new UIDs, so the damage is
already done.
Both of the problems can be avoided if we simply lock the uidlist
before syncing and keep it until sync is finished. Typically this
would happen in any case, as there is the index lock..
The second case is still a problem with external changes though,
because maildir doesn't require any kind of locking. Luckily this
problem rarely happens except under high amount of modifications.
*/
if (!ctx->ibox->syncing_commit) {
ctx->index_sync_ctx = maildir_sync_index_begin(ctx->ibox);
if (ctx->index_sync_ctx == NULL)
return -1;
}
if ((ret = maildir_uidlist_lock(ctx->ibox->uidlist)) <= 0) {
/* failure / timeout. if forced is TRUE, we could still go
forward and check only for renamed files, but is it worth
the trouble? .. */
return ret;
}
ctx->partial = !cur_changed;
ctx->uidlist_sync_ctx =
maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
if (cur_changed) {
if (maildir_scan_dir(ctx, FALSE) < 0)
return -1;
}
/* finish uidlist syncing, but keep it still locked */
maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
if (!ctx->ibox->syncing_commit) {
if (maildir_sync_index_finish(ctx->index_sync_ctx,
ctx->partial) < 0) {
ctx->index_sync_ctx = NULL;
return -1;
}
ctx->index_sync_ctx = NULL;
}
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
return ret;
}
int maildir_storage_sync_force(struct index_mailbox *ibox)
{
struct maildir_sync_context *ctx;
int ret;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context(ctx, TRUE);
maildir_sync_deinit(ctx);
return ret;
}
struct mailbox_sync_context *
maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
struct maildir_sync_context *ctx;
int ret = 0;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
ctx = maildir_sync_context_new(ibox);
ret = maildir_sync_context(ctx, FALSE);
maildir_sync_deinit(ctx);
}
return index_mailbox_sync_init(box, flags, ret < 0);
}