src/modules/catalog.py

	catalog.py revision 3362


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
1516N/A#!/usr/bin/python
23N/A#
23N/A# CDDL HEADER START
23N/A#
23N/A# The contents of this file are subject to the terms of the
23N/A# Common Development and Distribution License (the "License").
23N/A# You may not use this file except in compliance with the License.
23N/A#
23N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
23N/A# or http://www.opensolaris.org/os/licensing.
23N/A# See the License for the specific language governing permissions
23N/A# and limitations under the License.
23N/A#
23N/A# When distributing Covered Code, include this CDDL HEADER in each
23N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
23N/A# If applicable, add the following below this CDDL HEADER, with the
23N/A# fields enclosed by brackets "[]" replaced with your own identifying
23N/A# information: Portions Copyright [yyyy] [name of copyright owner]
23N/A#
23N/A# CDDL HEADER END
23N/A#
3339N/A# Copyright (c) 2007, 2016, Oracle and/or its affiliates. All rights reserved.
23N/A
221N/A"""Interfaces and implementation for the Catalog object, as well as functions
221N/Athat operate on lists of package FMRIs."""
221N/A
3339N/Afrom __future__ import  print_function
1352N/Aimport copy
1352N/Aimport calendar
2510N/Aimport collections
429N/Aimport datetime
1352N/Aimport errno
2510N/Aimport fnmatch
1516N/Aimport hashlib
1352N/Aimport os
1352N/Aimport simplejson as json
3234N/Aimport six
635N/Aimport stat
1352N/Aimport threading
2810N/Aimport types
23N/A
3339N/Afrom collections import OrderedDict
3234N/Afrom operator import itemgetter
3234N/A
1369N/Aimport pkg.actions
1352N/Aimport pkg.client.api_errors as api_errors
2616N/Aimport pkg.client.pkgdefs as pkgdefs
23N/Aimport pkg.fmri as fmri
1463N/Aimport pkg.misc as misc
429N/Aimport pkg.portable as portable
1352N/Aimport pkg.version
1352N/A
2510N/Afrom pkg.misc import EmptyDict, EmptyI
1352N/A
1352N/Aclass _JSONWriter(object):
1352N/A        """Private helper class used to serialize catalog data and generate
1352N/A        signatures."""
1352N/A
1833N/A        def __init__(self, data, single_pass=False, pathname=None, sign=True):
1352N/A                self.__data = data
1352N/A                self.__fileobj = None
1358N/A
1833N/A                # Determines whether data is encoded in a single pass (uses
1833N/A                # more memory) or iteratively.
1833N/A                self.__single_pass = single_pass
1833N/A
1463N/A                # Default to a 32K buffer.
2810N/A                self.__bufsz = 32 * 1024
1463N/A
2962N/A                # catalog signatures *must* use sha-1 only since clients
2962N/A                # compare entire dictionaries against the reported hash from
2962N/A                # the catalog in the various <CatalogPartBase>.validate()
2962N/A                # methods rather than just attributes within those dictionaries.
2962N/A                # If old clients are to interoperate with new repositories, the
2962N/A                # computed and expected dictionaries must be identical at
2962N/A                # present, so we must use sha-1.
1358N/A                if sign:
1463N/A                        if not pathname:
1463N/A                                # Only needed if not writing to __fileobj.
1516N/A                                self.__sha_1 = hashlib.sha1()
1463N/A                        self.__sha_1_value = None
1463N/A
1358N/A                self.__sign = sign
1431N/A                self.pathname = pathname
1431N/A
1431N/A                if not pathname:
1431N/A                        return
1431N/A
1463N/A                # Call statvfs to find optimal blocksize for destination.
1463N/A                dest_dir = os.path.dirname(self.pathname)
1431N/A                try:
1463N/A                        destvfs = os.statvfs(dest_dir)
1463N/A                        # Set the file buffer size to the blocksize of our
1463N/A                        # filesystem.
3245N/A                        self.__bufsz = destvfs.f_bsize
3171N/A                except EnvironmentError as e:
1463N/A                        if e.errno == errno.EACCES:
1463N/A                                raise api_errors.PermissionsException(
1463N/A                                    e.filename)
3171N/A                except AttributeError as e:
1463N/A                        # os.statvfs is not available on some platforms.
1463N/A                        pass
1463N/A
1463N/A                try:
1463N/A                        tfile = open(pathname, "wb", self.__bufsz)
3171N/A                except EnvironmentError as e:
1431N/A                        if e.errno == errno.EACCES:
1431N/A                                raise api_errors.PermissionsException(
1431N/A                                    e.filename)
1431N/A                        if e.errno == errno.EROFS:
1431N/A                                raise api_errors.ReadOnlyFileSystemException(
1431N/A                                    e.filename)
1431N/A                        raise
1431N/A                self.__fileobj = tfile
1352N/A
1352N/A        def signatures(self):
1352N/A                """Returns a dictionary mapping digest algorithms to the
1352N/A                hex-encoded digest values of the text of the catalog."""
1352N/A
1358N/A                if not self.__sign:
1358N/A                        return {}
1463N/A                return { "sha-1": self.__sha_1_value }
1352N/A
1833N/A        def _dump(self, obj, fp, skipkeys=False, ensure_ascii=True,
1833N/A            check_circular=True, allow_nan=True, cls=json.JSONEncoder,
1833N/A            indent=None, separators=None, encoding='utf-8', default=None, **kw):
1833N/A                iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
1833N/A                    check_circular=check_circular, allow_nan=allow_nan,
1833N/A                    indent=indent, separators=separators, encoding=encoding,
1833N/A                    default=default, **kw).iterencode(obj,
1833N/A                    _one_shot=self.__single_pass)
3339N/A                fp.writelines(misc.force_bytes(i) for i in iterable)
1833N/A
1352N/A        def save(self):
1352N/A                """Serializes and stores the provided data in JSON format."""
1358N/A
1352N/A                # sort_keys is necessary to ensure consistent signature
1352N/A                # generation.  It has a minimal performance cost as well (on
1358N/A                # on SPARC and x86), so shouldn't be an issue.  However, it
1358N/A                # is only needed if the caller has indicated that the content
1358N/A                # should be signed.
1463N/A
1463N/A                # Whenever possible, avoid using the write wrapper (self) as
1463N/A                # this can greatly increase write times.
1463N/A                out = self.__fileobj
1463N/A                if not out:
1463N/A                        out = self
1463N/A
1833N/A                self._dump(self.__data, out, check_circular=False,
1463N/A                    separators=(",", ":"), sort_keys=self.__sign)
3339N/A                out.write(b"\n")
1352N/A
1352N/A                if self.__fileobj:
1352N/A                        self.__fileobj.close()
1463N/A
1463N/A                if not self.__sign or not self.__fileobj:
1463N/A                        # Can't sign unless a file object is provided.  And if
1463N/A                        # one is provided, but no signing is to be done, then
1463N/A                        # ensure the fileobject is discarded.
1352N/A                        self.__fileobj = None
1463N/A                        if self.__sign:
1463N/A                                self.__sha_1_value = self.__sha_1.hexdigest()
1463N/A                        return
1463N/A
1463N/A                # Ensure file object goes out of scope.
1463N/A                self.__fileobj = None
1463N/A
1463N/A                # Calculating sha-1 this way is much faster than intercepting
1463N/A                # write calls because of the excessive number of write calls
1463N/A                # that json.dump() triggers (1M+ for /dev catalog files).
2962N/A                self.__sha_1_value = misc.get_data_digest(self.pathname,
2962N/A                    hash_func=hashlib.sha1)[0]
1463N/A
1463N/A                # Open the JSON file so that the signature data can be added.
3339N/A                with open(self.pathname, "rb+", self.__bufsz) as sfile:
3339N/A                        # The last bytes should be "}\n", which is where the
3339N/A                        # signature data structure needs to be appended.
3339N/A                        sfile.seek(-2, os.SEEK_END)
3339N/A
3339N/A                        # Add the signature data and close.
3339N/A                        sfoffset = sfile.tell()
3339N/A                        if sfoffset > 1:
3339N/A                                # Catalog is not empty, so a separator is needed.
3339N/A                                sfile.write(b",")
3339N/A                        sfile.write(b'"_SIGNATURE":')
3339N/A                        self._dump(self.signatures(), sfile, check_circular=False,
3339N/A                            separators=(",", ":"))
3339N/A                        sfile.write(b"}\n")
1352N/A
1352N/A        def write(self, data):
1352N/A                """Wrapper function that should not be called by external
1352N/A                consumers."""
1358N/A
1358N/A                if self.__sign:
1358N/A                        self.__sha_1.update(data)
1352N/A
1833N/A        def writelines(self, iterable):
1833N/A                """Wrapper function that should not be called by external
1833N/A                consumers."""
1833N/A
1833N/A                for l in iterable:
1833N/A                        self.__sha_1.update(l)
1833N/A
1369N/A
1352N/Aclass CatalogPartBase(object):
1352N/A        """A CatalogPartBase object is an abstract class containing core
1352N/A        functionality shared between CatalogPart and CatalogAttrs."""
1352N/A
1352N/A        # The file mode to be used for all catalog files.
1352N/A        __file_mode = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH
1352N/A
1352N/A        __meta_root = None
3362N/A        __file_root = None
1352N/A        last_modified = None
1352N/A        loaded = False
1352N/A        name = None
1358N/A        sign = True
1352N/A        signatures = None
1352N/A
3362N/A        def __init__(self, name, meta_root=None, sign=True, file_root=None):
1352N/A                """Initializes a CatalogPartBase object."""
1352N/A
1352N/A                self.meta_root = meta_root
3362N/A                self.file_root = file_root
3362N/A                if not self.file_root:
3362N/A                        self.file_root = meta_root
2810N/A                # Sanity check: part names can't be pathname-ish.
2810N/A                if name != os.path.basename(name):
3362N/A                        raise api_errors.UnrecognizedCatalogPart(name)
1352N/A                self.name = name
1358N/A                self.sign = sign
1352N/A                self.signatures = {}
157N/A
1352N/A                if not self.meta_root or not self.exists:
1352N/A                        # Operations shouldn't attempt to load the part data
1352N/A                        # unless meta_root is defined and the data exists.
1352N/A                        self.loaded = True
1352N/A                        self.last_modified = datetime.datetime.utcnow()
1352N/A                else:
1352N/A                        self.last_modified = self.__last_modified()
1352N/A
1352N/A        @staticmethod
1352N/A        def _gen_signatures(data):
1352N/A                f = _JSONWriter(data)
1352N/A                f.save()
1352N/A                return f.signatures()
1352N/A
1352N/A        def __get_meta_root(self):
1352N/A                return self.__meta_root
1352N/A
3362N/A        def __get_file_root(self):
3362N/A                return self.__file_root
3362N/A
1352N/A        def __last_modified(self):
1352N/A                """A UTC datetime object representing the time the file used to
1352N/A                to store object metadata was modified, or None if it does not
1352N/A                exist yet."""
1352N/A
1352N/A                if not self.exists:
1352N/A                        return None
1352N/A
1352N/A                try:
1352N/A                        mod_time = os.stat(self.pathname).st_mtime
3171N/A                except EnvironmentError as e:
1352N/A                        if e.errno == errno.ENOENT:
1352N/A                                return None
1352N/A                        raise
1352N/A                return datetime.datetime.utcfromtimestamp(mod_time)
1352N/A
1352N/A        def __set_meta_root(self, path):
1352N/A                if path:
1352N/A                        path = os.path.abspath(path)
1352N/A                self.__meta_root = path
1352N/A
3362N/A        def __set_file_root(self, path):
3362N/A                if path:
3362N/A                        path = os.path.abspath(path)
3362N/A                self.__file_root = path
3362N/A
1352N/A        def destroy(self):
1431N/A                """Removes any on-disk files that exist for the catalog part and
1352N/A                discards all content."""
1352N/A
1352N/A                if self.pathname:
1352N/A                        if os.path.exists(self.pathname):
1352N/A                                try:
1352N/A                                        portable.remove(self.pathname)
3171N/A                                except EnvironmentError as e:
1431N/A                                        if e.errno == errno.EACCES:
1431N/A                                                raise api_errors.PermissionsException(
1431N/A                                                    e.filename)
1431N/A                                        if e.errno == errno.EROFS:
1431N/A                                                raise api_errors.ReadOnlyFileSystemException(
1431N/A                                                    e.filename)
1431N/A                                        raise
1352N/A                self.signatures = {}
1352N/A                self.loaded = False
1352N/A                self.last_modified = None
1352N/A
1352N/A        @property
1352N/A        def exists(self):
1352N/A                """A boolean value indicating wheher a file for the catalog part
1352N/A                exists at <self.meta_root>/<self.name>."""
1352N/A
1352N/A                if not self.pathname:
1352N/A                        return False
1352N/A                return os.path.exists(self.pathname)
1352N/A
1352N/A        def load(self):
1352N/A                """Load the serialized data for the catalog part and return the
1352N/A                resulting structure."""
1352N/A
1352N/A                location = os.path.join(self.meta_root, self.name)
1352N/A
1352N/A                try:
3362N/A                        fobj = misc.open_image_file(self.file_root, location,
3362N/A                            os.O_RDONLY, misc.PKG_FILE_MODE)
3171N/A                except EnvironmentError as e:
1352N/A                        if e.errno == errno.ENOENT:
1352N/A                                raise api_errors.RetrievalError(e,
1352N/A                                    location=location)
1431N/A                        if e.errno == errno.EROFS:
1431N/A                                raise api_errors.ReadOnlyFileSystemException(
1431N/A                                    e.filename)
1431N/A                        if e.errno == errno.EACCES:
1352N/A                                raise api_errors.PermissionsException(
1352N/A                                    e.filename)
3362N/A                        if e.errno == errno.EREMOTE:
3362N/A                                raise api_errors.UnrecognizedCatalogPart(
3362N/A                                    self.name)
1431N/A                        raise
1352N/A
1352N/A                try:
1352N/A                        struct = json.load(fobj)
3171N/A                except EnvironmentError as e:
1352N/A                        raise api_errors.RetrievalError(e)
3171N/A                except ValueError as e:
1352N/A                        # Not a valid catalog file.
1658N/A                        raise api_errors.InvalidCatalogFile(location)
1352N/A
1352N/A                self.loaded = True
1358N/A                # Signature data, if present, should be removed from the struct
1358N/A                # on load and then stored in the signatures object property.
1358N/A                self.signatures = struct.pop("_SIGNATURE", {})
3339N/A                fobj.close()
1352N/A                return struct
1352N/A
1352N/A        @property
1352N/A        def pathname(self):
1352N/A                """The absolute path of the file used to store the data for
1352N/A                this part or None if meta_root or name is not set."""
1352N/A
1352N/A                if not self.meta_root or not self.name:
1352N/A                        return None
1352N/A                return os.path.join(self.meta_root, self.name)
1352N/A
1833N/A        def save(self, data, single_pass=False):
1352N/A                """Serialize and store the transformed catalog part's 'data' in
1352N/A                a file using the pathname <self.meta_root>/<self.name>.
1352N/A
1833N/A                'data' must be a dict.
1833N/A
1833N/A                'single_pass' is an optional boolean indicating whether the data
1833N/A                should be serialized in a single pass.  This is significantly
1833N/A                faster, but requires that the entire set of data be serialized
1833N/A                in-memory instead of iteratively writing it to the target
1833N/A                storage object."""
1833N/A
1833N/A                f = _JSONWriter(data, single_pass=single_pass,
1833N/A                    pathname=self.pathname, sign=self.sign)
1352N/A                f.save()
1352N/A
1352N/A                # Update in-memory copy to reflect stored data.
1352N/A                self.signatures = f.signatures()
1352N/A
1431N/A                # Ensure the permissions on the new file are correct.
1352N/A                try:
1431N/A                        os.chmod(self.pathname, self.__file_mode)
3171N/A                except EnvironmentError as e:
1431N/A                        if e.errno == errno.EACCES:
1431N/A                                raise api_errors.PermissionsException(
1431N/A                                    e.filename)
1431N/A                        if e.errno == errno.EROFS:
1431N/A                                raise api_errors.ReadOnlyFileSystemException(
1431N/A                                    e.filename)
1352N/A                        raise
1352N/A
1352N/A                # Finally, set the file times to match the last catalog change.
1352N/A                if self.last_modified:
1352N/A                        mtime = calendar.timegm(
1352N/A                            self.last_modified.utctimetuple())
1431N/A                        os.utime(self.pathname, (mtime, mtime))
1352N/A
1352N/A        meta_root = property(__get_meta_root, __set_meta_root)
3362N/A        file_root = property(__get_file_root, __set_file_root)
23N/A
258N/A
1352N/Aclass CatalogPart(CatalogPartBase):
1352N/A        """A CatalogPart object is the representation of a subset of the package
1352N/A        FMRIs available from a package repository."""
1352N/A
1352N/A        __data = None
1352N/A        ordered = None
1352N/A
3362N/A        def __init__(self, name, meta_root=None, ordered=True, sign=True,
3362N/A            file_root=None):
1352N/A                """Initializes a CatalogPart object."""
1352N/A
1352N/A                self.__data = {}
1352N/A                self.ordered = ordered
2810N/A                if not name.startswith("catalog."):
3362N/A                        raise api_errors.UnrecognizedCatalogPart(name)
1358N/A                CatalogPartBase.__init__(self, name, meta_root=meta_root,
3362N/A                    sign=sign, file_root=file_root)
1352N/A
1537N/A        def __iter_entries(self, last=False, ordered=False, pubs=EmptyI):
1537N/A                """Private generator function to iterate over catalog entries.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the CatalogPart has been saved since the last
1537N/A                modifying operation, or sort() has has been called, this will
1537N/A                also be the newest version of the package.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1352N/A                self.load()
1537N/A                if ordered:
1537N/A                        stems = self.pkg_names(pubs=pubs)
1537N/A                else:
1537N/A                        stems = (
1537N/A                            (pub, stem)
1537N/A                            for pub in self.publishers(pubs=pubs)
1537N/A                            for stem in self.__data[pub]
1537N/A                        )
1537N/A
1537N/A                if last:
1537N/A                        return (
1537N/A                            (pub, stem, self.__data[pub][stem][-1])
1537N/A                            for pub, stem in stems
1537N/A                        )
1537N/A
1537N/A                if ordered:
1537N/A                        return (
1537N/A                            (pub, stem, entry)
1537N/A                            for pub, stem in stems
1537N/A                            for entry in reversed(self.__data[pub][stem])
1537N/A                        )
1537N/A                return (
1537N/A                    (pub, stem, entry)
1537N/A                    for pub, stem in stems
1537N/A                    for entry in self.__data[pub][stem]
1537N/A                )
1352N/A
1549N/A        def add(self, pfmri=None, metadata=None, op_time=None, pub=None,
1549N/A            stem=None, ver=None):
1549N/A                """Add a catalog entry for a given FMRI or FMRI components.
1352N/A
1352N/A                'metadata' is an optional dict containing the catalog
1352N/A                metadata that should be stored for the specified FMRI.
1352N/A
1352N/A                The dict representing the entry is returned to callers,
1352N/A                but should not be modified.
1352N/A                """
1352N/A
1549N/A                assert pfmri or (pub and stem and ver)
1549N/A                if pfmri and not pfmri.publisher:
1549N/A                        raise api_errors.AnarchicalCatalogFMRI(str(pfmri))
1549N/A
1549N/A                if not self.loaded:
1549N/A                        # Hot path, so avoid calling load unless necessary, even
1549N/A                        # though it performs this check already.
1549N/A                        self.load()
1549N/A
1549N/A                if pfmri:
1549N/A                        pub, stem, ver = pfmri.tuple()
1549N/A                        ver = str(ver)
1549N/A
1549N/A                pkg_list = self.__data.setdefault(pub, {})
1549N/A                ver_list = pkg_list.setdefault(stem, [])
1352N/A                for entry in ver_list:
1352N/A                        if entry["version"] == ver:
1549N/A                                if not pfmri:
3158N/A                                        pfmri = "pkg://{0}/{1}@{2}".format(pub,
3158N/A                                            stem, ver)
1352N/A                                raise api_errors.DuplicateCatalogEntry(
1352N/A                                    pfmri, operation="add",
1352N/A                                    catalog_name=self.pathname)
1352N/A
1549N/A                if metadata is not None:
1352N/A                        entry = metadata
1352N/A                else:
1352N/A                        entry = {}
1352N/A                entry["version"] = ver
1352N/A
1352N/A                ver_list.append(entry)
1352N/A                if self.ordered:
1463N/A                        self.sort(pfmris=set([pfmri]))
1352N/A
1352N/A                if not op_time:
1352N/A                        op_time = datetime.datetime.utcnow()
1352N/A                self.last_modified = op_time
1352N/A                self.signatures = {}
1352N/A                return entry
1352N/A
1606N/A        def destroy(self):
1606N/A                """Removes any on-disk files that exist for the catalog part and
1606N/A                discards all content."""
1606N/A
1606N/A                self.__data = {}
1606N/A                return CatalogPartBase.destroy(self)
1606N/A
1537N/A        def entries(self, cb=None, last=False, ordered=False, pubs=EmptyI):
1352N/A                """A generator function that produces tuples of the form
1352N/A                (fmri, entry) as it iterates over the contents of the catalog
1352N/A                part (where entry is the related catalog entry for the fmri).
1352N/A                Callers should not modify any of the data that is returned.
1352N/A
1537N/A                'cb' is an optional callback function that will be executed for
1537N/A                each package. It must accept two arguments: 'pkg' and 'entry'.
1537N/A                'pkg' is an FMRI object and 'entry' is the dictionary structure
1537N/A                of the catalog entry for the package.  If the callback returns
1537N/A                False, then the entry will not be included in the results.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the CatalogPart has been saved since the last
1537N/A                modifying operation, or sort() has has been called, this will
1537N/A                also be the newest version of the package.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to.
1537N/A
1352N/A                Results are always in catalog version order on a per-
1352N/A                publisher, per-stem basis.
1352N/A                """
1352N/A
1537N/A                for pub, stem, entry in self.__iter_entries(last=last,
1537N/A                    ordered=ordered, pubs=pubs):
2639N/A                        f = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A                            version=entry["version"])
1537N/A                        if cb is None or cb(f, entry):
1537N/A                                yield f, entry
1537N/A
1537N/A        def entries_by_version(self, name, pubs=EmptyI):
1352N/A                """A generator function that produces tuples of (version,
1352N/A                entries), where entries is a list of tuples of the format
1352N/A                (fmri, entry) where entry is the catalog entry for the
1537N/A                FMRI) as it iterates over the CatalogPart contents.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1352N/A                self.load()
1352N/A
1352N/A                versions = {}
1352N/A                entries = {}
1537N/A                for pub in self.publishers(pubs=pubs):
1352N/A                        ver_list = self.__data[pub].get(name, ())
1352N/A                        for entry in ver_list:
1352N/A                                sver = entry["version"]
2639N/A                                pfmri = fmri.PkgFmri(name=name, publisher=pub,
2639N/A                                    version=sver)
1352N/A
1352N/A                                versions[sver] = pfmri.version
1352N/A                                entries.setdefault(sver, [])
1352N/A                                entries[sver].append((pfmri, entry))
1352N/A
3234N/A                for key, ver in sorted(six.iteritems(versions), key=itemgetter(1)):
1352N/A                        yield ver, entries[key]
1352N/A
1537N/A        def fmris(self, last=False, objects=True, ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces FMRIs as it iterates
1537N/A                over the contents of the catalog part.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last fmri
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the CatalogPart has been saved since the last
1537N/A                modifying operation, or sort() has has been called, this will
1537N/A                also be the newest version of the package.
1352N/A
1352N/A                'objects' is an optional boolean value indicating whether
1352N/A                FMRIs should be returned as FMRI objects or as strings.
1352N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to.
1537N/A
1352N/A                Results are always in catalog version order on a per-
1537N/A                publisher, per-stem basis."""
1352N/A
1352N/A                if objects:
1537N/A                        for pub, stem, entry in self.__iter_entries(last=last,
1537N/A                            ordered=ordered, pubs=pubs):
2639N/A                                yield fmri.PkgFmri(name=stem, publisher=pub,
2639N/A                                    version=entry["version"])
1352N/A                        return
1352N/A
1537N/A                for pub, stem, entry in self.__iter_entries(last=last,
1537N/A                    ordered=ordered, pubs=pubs):
3158N/A                        yield "pkg://{0}/{1}@{2}".format(pub,
1352N/A                            stem, entry["version"])
1352N/A                return
1352N/A
1537N/A        def fmris_by_version(self, name, pubs=EmptyI):
1352N/A                """A generator function that produces tuples of (version,
1352N/A                fmris), where fmris is a list of the fmris related to the
1537N/A                version.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1352N/A                self.load()
1352N/A
1352N/A                versions = {}
1352N/A                entries = {}
1537N/A                for pub in self.publishers(pubs=pubs):
1537N/A                        ver_list = self.__data[pub].get(name, None)
1549N/A                        if not ver_list:
1352N/A                                continue
1352N/A
1352N/A                        for entry in ver_list:
1352N/A                                sver = entry["version"]
2639N/A                                pfmri = fmri.PkgFmri(name=name, publisher=pub,
2639N/A                                    version=sver)
1352N/A
1352N/A                                versions[sver] = pfmri.version
1352N/A                                entries.setdefault(sver, [])
1352N/A                                entries[sver].append(pfmri)
1352N/A
3234N/A                for key, ver in sorted(six.iteritems(versions), key=itemgetter(1)):
1352N/A                        yield ver, entries[key]
1352N/A
1537N/A        def get_entry(self, pfmri=None, pub=None, stem=None, ver=None):
1549N/A                """Returns the catalog part entry for the given package FMRI or
1537N/A                FMRI components."""
1537N/A
1549N/A                assert pfmri or (pub and stem and ver)
1549N/A                if pfmri and not pfmri.publisher:
1549N/A                        raise api_errors.AnarchicalCatalogFMRI(str(pfmri))
1352N/A
1537N/A                # Since this is a hot path, this function checks for loaded
1537N/A                # status before attempting to call the load function.
1537N/A                if not self.loaded:
1537N/A                        self.load()
1537N/A
1537N/A                if pfmri:
1537N/A                        pub, stem, ver = pfmri.tuple()
1537N/A                        ver = str(ver)
1537N/A
1537N/A                pkg_list = self.__data.get(pub, None)
1549N/A                if not pkg_list:
1537N/A                        return
1537N/A
1537N/A                ver_list = pkg_list.get(stem, ())
1369N/A                for entry in ver_list:
1369N/A                        if entry["version"] == ver:
1369N/A                                return entry
1352N/A
1352N/A        def get_package_counts(self):
1352N/A                """Returns a tuple of integer values (package_count,
1352N/A                package_version_count).  The first is the number of
1352N/A                unique packages (per-publisher), and the second is the
1352N/A                number of unique package versions (per-publisher and
1352N/A                stem)."""
1352N/A
1352N/A                self.load()
1352N/A                package_count = 0
1352N/A                package_version_count = 0
1431N/A                for pub in self.publishers():
1352N/A                        for stem in self.__data[pub]:
1352N/A                                package_count += 1
1352N/A                                package_version_count += \
1352N/A                                    len(self.__data[pub][stem])
1352N/A                return (package_count, package_version_count)
1352N/A
3110N/A        def get_package_counts_by_pub(self, pubs=EmptyI):
1968N/A                """Returns a generator of tuples of the form (pub,
1968N/A                package_count, package_version_count).  'pub' is the publisher
1968N/A                prefix, 'package_count' is the number of unique packages for the
1968N/A                publisher, and 'package_version_count' is the number of unique
1968N/A                package versions for the publisher.
1968N/A                """
1968N/A
1968N/A                self.load()
3110N/A                for pub in self.publishers(pubs=pubs):
1968N/A                        package_count = 0
1968N/A                        package_version_count = 0
1968N/A                        for stem in self.__data[pub]:
1968N/A                                package_count += 1
1968N/A                                package_version_count += \
1968N/A                                    len(self.__data[pub][stem])
1968N/A                        yield pub, package_count, package_version_count
1968N/A
1352N/A        def load(self):
1352N/A                """Load and transform the catalog part's data, preparing it
1352N/A                for use."""
1352N/A
1352N/A                if self.loaded:
1352N/A                        # Already loaded, or only in-memory.
1352N/A                        return
1352N/A                self.__data = CatalogPartBase.load(self)
1352N/A
1537N/A        def names(self, pubs=EmptyI):
1352N/A                """Returns a set containing the names of all the packages in
1537N/A                the CatalogPart.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1352N/A                self.load()
1352N/A                return set((
1352N/A                    stem
1537N/A                    for pub in self.publishers(pubs=pubs)
1352N/A                    for stem in self.__data[pub]
1352N/A                ))
1352N/A
1537N/A        def pkg_names(self, pubs=EmptyI):
1537N/A                """A generator function that produces package tuples of the form
1537N/A                (pub, stem) as it iterates over the contents of the CatalogPart.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
2547N/A                the results to.  If specified, publishers will be sorted in
2547N/A                the order given.
1537N/A
1537N/A                Results are always returned sorted by stem and then by
1537N/A                publisher."""
1537N/A
1537N/A                self.load()
1537N/A
1537N/A                # Results have to be sorted by stem first, and by
1537N/A                # publisher prefix second.
1537N/A                pkg_list = [
3158N/A                        "{0}!{1}".format(stem, pub)
1537N/A                        for pub in self.publishers(pubs=pubs)
1537N/A                        for stem in self.__data[pub]
1537N/A                ]
1537N/A
2547N/A                pub_sort = None
2547N/A                if pubs:
2547N/A                        pos = dict((p, i) for (i, p) in enumerate(pubs))
3245N/A                        def pub_key(a):
2547N/A                                astem, apub = a.split("!", 1)
3245N/A                                return (astem, pos[apub])
3245N/A                        pub_sort = pub_key
3245N/A
3245N/A                for entry in sorted(pkg_list, key=pub_sort):
1537N/A                        stem, pub = entry.split("!", 1)
1537N/A                        yield pub, stem
1537N/A
1463N/A        def publishers(self, pubs=EmptyI):
1431N/A                """A generator function that returns publisher prefixes as it
1463N/A                iterates over the package data in the CatalogPart.
1463N/A
1463N/A                'pubs' is an optional list that contains the prefixes of the
1463N/A                publishers to restrict the results to."""
1431N/A
1431N/A                self.load()
1431N/A                for pub in self.__data:
1463N/A                        # Any entries starting with "_" are part of the
1431N/A                        # reserved catalog namespace.
1463N/A                        if not pub[0] == "_" and (not pubs or pub in pubs):
1431N/A                                yield pub
1431N/A
1352N/A        def remove(self, pfmri, op_time=None):
1352N/A                """Remove a package and its metadata."""
1352N/A
1352N/A                if not pfmri.publisher:
1352N/A                        raise api_errors.AnarchicalCatalogFMRI(pfmri.get_fmri())
1352N/A
1352N/A                self.load()
1537N/A                pkg_list = self.__data.get(pfmri.publisher, None)
1549N/A                if not pkg_list:
1352N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1352N/A
1352N/A                ver = str(pfmri.version)
1352N/A                ver_list = pkg_list.get(pfmri.pkg_name, [])
1352N/A                for i, entry in enumerate(ver_list):
1352N/A                        if entry["version"] == ver:
1352N/A                                # Safe to do this since a 'break' is done
1352N/A                                # immediately after removals are performed.
1352N/A                                del ver_list[i]
1352N/A                                if not ver_list:
1352N/A                                        # When all version entries for a
1352N/A                                        # package are removed, its stem
1352N/A                                        # should be also.
1352N/A                                        del pkg_list[pfmri.pkg_name]
1352N/A                                if not pkg_list:
1352N/A                                        # When all package stems for a
1352N/A                                        # publisher have been removed,
1352N/A                                        # it should be also.
1352N/A                                        del self.__data[pfmri.publisher]
1352N/A                                break
1352N/A                else:
1352N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1352N/A
1352N/A                if not op_time:
1352N/A                        op_time = datetime.datetime.utcnow()
1352N/A                self.last_modified = op_time
1352N/A                self.signatures = {}
1352N/A
1833N/A        def save(self, single_pass=False):
1352N/A                """Transform and store the catalog part's data in a file using
1833N/A                the pathname <self.meta_root>/<self.name>.
1833N/A
1833N/A                'single_pass' is an optional boolean indicating whether the data
1833N/A                should be serialized in a single pass.  This is significantly
1833N/A                faster, but requires that the entire set of data be serialized
1833N/A                in-memory instead of iteratively writing it to the target
1833N/A                storage object."""
1352N/A
1352N/A                if not self.meta_root:
1352N/A                        # Assume this is in-memory only.
1352N/A                        return
1833N/A
2022N/A                # Ensure content is loaded before attempting save.
2022N/A                self.load()
2022N/A
1833N/A                CatalogPartBase.save(self, self.__data, single_pass=single_pass)
1352N/A
1463N/A        def sort(self, pfmris=None, pubs=None):
1352N/A                """Re-sorts the contents of the CatalogPart such that version
1352N/A                entries for each package stem are in ascending order.
1352N/A
1463N/A                'pfmris' is an optional set of FMRIs to restrict the sort to.
1463N/A                This is useful during catalog operations as only entries for
1463N/A                the corresponding package stem(s) need to be sorted.
1463N/A
1463N/A                'pubs' is an optional set of publisher prefixes to restrict
1463N/A                the sort to.  This is useful during catalog operations as only
1463N/A                entries for the corresponding publisher stem(s) need to be
1463N/A                sorted.  This option has no effect if 'pfmris' is also
1463N/A                provided.
1463N/A
1463N/A                If neither 'pfmris' or 'pubs' is provided, all entries will be
1463N/A                sorted."""
1352N/A
3245N/A                def key_func(item):
3245N/A                        return pkg.version.Version(item["version"])
1352N/A
1352N/A                self.load()
1463N/A                if pfmris is not None:
1463N/A                        processed = set()
1463N/A                        for f in pfmris:
1463N/A                                pkg_stem = f.get_pkg_stem()
1463N/A                                if pkg_stem in processed:
1463N/A                                        continue
1463N/A                                processed.add(pkg_stem)
1463N/A
1463N/A                                # The specified FMRI may not exist in this
1463N/A                                # CatalogPart, so continue if it does not
1463N/A                                # exist.
1463N/A                                pkg_list = self.__data.get(f.publisher, None)
1549N/A                                if pkg_list:
1463N/A                                        ver_list = pkg_list.get(f.pkg_name,
1463N/A                                            None)
1549N/A                                        if ver_list:
3245N/A                                                ver_list.sort(key=key_func)
1352N/A                        return
1352N/A
1463N/A                for pub in self.publishers(pubs=pubs):
1352N/A                        for stem in self.__data[pub]:
3245N/A                                self.__data[pub][stem].sort(key=key_func)
1352N/A
1537N/A        def tuples(self, last=False, ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces FMRI tuples as it
1537N/A                iterates over the contents of the catalog part.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last FMRI
1537N/A                tuple for each package on a per-publisher basis should be
1537N/A                returned.  As long as the CatalogPart has been saved since
1537N/A                the last modifying operation, or sort() has has been called,
1537N/A                this will also be the newest version of the package.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                return (
1537N/A                    (pub, stem, entry["version"])
1537N/A                    for pub, stem, entry in self.__iter_entries(last=last,
1537N/A                        ordered=ordered, pubs=pubs)
1537N/A                )
1537N/A
1537N/A        def tuple_entries(self, cb=None, last=False, ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces tuples of the form ((pub,
1537N/A                stem, version), entry) as it iterates over the contents of the
1537N/A                catalog part (where entry is the related catalog entry for the
1537N/A                fmri).  Callers should not modify any of the data that is
1537N/A                returned.
1537N/A
1537N/A                'cb' is an optional callback function that will be executed for
1537N/A                each package. It must accept two arguments: 'pkg' and 'entry'.
1537N/A                'pkg' is an FMRI tuple and 'entry' is the dictionary structure
1537N/A                of the catalog entry for the package.  If the callback returns
1537N/A                False, then the entry will not be included in the results.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the CatalogPart has been saved since the last
1537N/A                modifying operation, or sort() has has been called, this will
1537N/A                also be the newest version of the package.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to.
1537N/A
1537N/A                Results are always in catalog version order on a per-publisher,
1537N/A                per-stem basis."""
1537N/A
1537N/A                for pub, stem, entry in self.__iter_entries(last=last,
1537N/A                    ordered=ordered, pubs=pubs):
1537N/A                        t = (pub, stem, entry["version"])
1537N/A                        if cb is None or cb(t, entry):
1537N/A                                yield t, entry
1431N/A
2810N/A        def validate(self, signatures=None, require_signatures=False):
1352N/A                """Verifies whether the signatures for the contents of the
1352N/A                CatalogPart match the specified signature data, or if not
1352N/A                provided, the current signature data.  Raises the exception
1352N/A                named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A                if not self.signatures and not signatures and \
2810N/A                    not require_signatures:
2810N/A                        # Nothing to validate, and we're not required to.
1352N/A                        return
2022N/A
2022N/A                # Ensure content is loaded before attempting to retrieve
2022N/A                # or generate signature data.
2022N/A                self.load()
1352N/A                if not signatures:
1352N/A                        signatures = self.signatures
1352N/A
1352N/A                new_signatures = self._gen_signatures(self.__data)
1352N/A                if new_signatures != signatures:
1352N/A                        raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
814N/A
1352N/Aclass CatalogUpdate(CatalogPartBase):
1352N/A        """A CatalogUpdate object is an augmented representation of a subset
1352N/A        of the package data contained within a Catalog."""
1352N/A
1352N/A        # Properties.
1352N/A        __data = None
1352N/A        last_modified = None
1352N/A
1352N/A        # Operation constants.
1352N/A        ADD = "add"
1352N/A        REMOVE = "remove"
1352N/A
3362N/A        def __init__(self, name, meta_root=None, sign=True, file_root=None):
1352N/A                """Initializes a CatalogUpdate object."""
1352N/A
1352N/A                self.__data = {}
2810N/A                if not name.startswith("update."):
3362N/A                        raise api_errors.UnrecognizedCatalogPart(name)
1431N/A                CatalogPartBase.__init__(self, name, meta_root=meta_root,
3362N/A                    sign=sign, file_root=file_root)
1352N/A
1352N/A        def add(self, pfmri, operation, op_time, metadata=None):
1352N/A                """Records the specified catalog operation and any related
1352N/A                catalog metadata for the specified package FMRI.
1352N/A
1352N/A                'operation' must be one of the following constant values
1352N/A                provided by the CatalogUpdate class:
1352N/A                    ADD
1352N/A                    REMOVE
1352N/A
1352N/A                'op_time' is a UTC datetime object indicating the time
1352N/A                the catalog operation was performed.
1352N/A
1352N/A                'metadata' is an optional dict containing the catalog
1352N/A                metadata that should be stored for the specified FMRI
1352N/A                indexed by catalog part (e.g. "dependency", "summary",
1352N/A                etc.)."""
1352N/A
1352N/A                if not pfmri.publisher:
1352N/A                        raise api_errors.AnarchicalCatalogFMRI(pfmri.get_fmri())
1352N/A
1352N/A                if operation not in (self.ADD, self.REMOVE):
1352N/A                        raise api_errors.UnknownUpdateType(operation)
1352N/A
1352N/A                self.load()
1537N/A                self.__data.setdefault(pfmri.publisher, {})
1537N/A                pkg_list = self.__data[pfmri.publisher]
1537N/A
1537N/A                pkg_list.setdefault(pfmri.pkg_name, [])
1537N/A                ver_list = pkg_list[pfmri.pkg_name]
1537N/A
1537N/A                if metadata is not None:
1352N/A                        entry = metadata
1352N/A                else:
1352N/A                        entry = {}
1352N/A                entry["op-time"] = datetime_to_basic_ts(op_time)
1352N/A                entry["op-type"] = operation
1352N/A                entry["version"] = str(pfmri.version)
1352N/A                ver_list.append(entry)
1352N/A
1352N/A                # To ensure the update log is viewed as having been updated
1352N/A                # at the exact same time as the catalog, the last_modified
1352N/A                # time of the update log must match the operation time.
1352N/A                self.last_modified = op_time
1352N/A                self.signatures = {}
1352N/A
1352N/A        def load(self):
1352N/A                """Load and transform the catalog update's data, preparing it
1352N/A                for use."""
1352N/A
1352N/A                if self.loaded:
1352N/A                        # Already loaded, or only in-memory.
1352N/A                        return
1352N/A                self.__data = CatalogPartBase.load(self)
1352N/A
1431N/A        def publishers(self):
1431N/A                """A generator function that returns publisher prefixes as it
1431N/A                iterates over the package data in the CatalogUpdate."""
1431N/A
1431N/A                self.load()
1431N/A                for pub in self.__data:
1463N/A                        # Any entries starting with "_" are part of the
1431N/A                        # reserved catalog namespace.
1463N/A                        if not pub[0] == "_":
1431N/A                                yield pub
1431N/A
1352N/A        def save(self):
1352N/A                """Transform and store the catalog update's data in a file using
1352N/A                the pathname <self.meta_root>/<self.name>."""
1352N/A
1352N/A                if not self.meta_root:
1352N/A                        # Assume this is in-memory only.
1352N/A                        return
2022N/A
2022N/A                # Ensure content is loaded before attempting save.
2022N/A                self.load()
2022N/A
1352N/A                CatalogPartBase.save(self, self.__data)
1352N/A
1352N/A        def updates(self):
1352N/A                """A generator function that produces tuples of the format
1352N/A                (fmri, op_type, op_time, metadata).  Where:
1352N/A
1352N/A                    * 'fmri' is a PkgFmri object for the package.
1352N/A
1352N/A                    * 'op_type' is a CatalogUpdate constant indicating
1352N/A                      the catalog operation performed.
1352N/A
1352N/A                    * 'op_time' is a UTC datetime object representing the
1352N/A                      time time the catalog operation was performed.
1352N/A
1352N/A                    * 'metadata' is a dict containing the catalog metadata
1352N/A                      for the FMRI indexed by catalog part name.
1352N/A
1352N/A                Results are always in ascending operation time order on a
1352N/A                per-publisher, per-stem basis.
1352N/A                """
1352N/A
1352N/A                self.load()
1352N/A
1352N/A                def get_update(pub, stem, entry):
1352N/A                        mdata = {}
1352N/A                        for key in entry:
1352N/A                                if key.startswith("catalog."):
1352N/A                                        mdata[key] = entry[key]
1352N/A                        op_time = basic_ts_to_datetime(entry["op-time"])
2639N/A                        pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A                            version=entry["version"])
1352N/A                        return (pfmri, entry["op-type"], op_time, mdata)
1352N/A
1431N/A                for pub in self.publishers():
1352N/A                        for stem in self.__data[pub]:
1352N/A                                for entry in self.__data[pub][stem]:
1352N/A                                        yield get_update(pub, stem, entry)
1352N/A                return
1352N/A
2810N/A        def validate(self, signatures=None, require_signatures=False):
1352N/A                """Verifies whether the signatures for the contents of the
1352N/A                CatalogUpdate match the specified signature data, or if not
1352N/A                provided, the current signature data.  Raises the exception
1352N/A                named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A                if not self.signatures and not signatures and \
2810N/A                    not require_signatures:
2810N/A                        # Nothing to validate, and we're not required to.
1352N/A                        return
1352N/A
2022N/A                # Ensure content is loaded before attempting to retrieve
2022N/A                # or generate signature data.
2022N/A                self.load()
1352N/A                if not signatures:
1352N/A                        signatures = self.signatures
1352N/A
1352N/A                new_signatures = self._gen_signatures(self.__data)
1352N/A                if new_signatures != signatures:
1352N/A                        raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
1352N/A
1352N/Aclass CatalogAttrs(CatalogPartBase):
1352N/A        """A CatalogAttrs object is the representation of the attributes of a
1352N/A        Catalog object."""
1352N/A
1352N/A        # Properties.
1352N/A        __data = None
1352N/A
2810N/A        # This structure defines defaults (for use in __init__) as well as
2810N/A        # the set of required elements for this catalog part.  See also the
2810N/A        # logic in load().
2810N/A        __DEFAULT_ELEMS = {
2810N/A            "created": None,
2810N/A            "last-modified": None,
2810N/A            "package-count": 0,
2810N/A            "package-version-count": 0,
2810N/A            "parts": {},
2810N/A            "updates": {},
2810N/A            "version": 1,
2810N/A        }
2810N/A
3362N/A        def __init__(self, meta_root=None, sign=True, file_root=None):
1352N/A                """Initializes a CatalogAttrs object."""
1352N/A
1352N/A                self.__data = {}
1352N/A                CatalogPartBase.__init__(self, name="catalog.attrs",
3362N/A                    meta_root=meta_root, sign=sign, file_root=file_root)
814N/A
1352N/A                if self.loaded:
1352N/A                        # If the data is already seen as 'loaded' during init,
1352N/A                        # this is actually a new object, so setup some sane
1352N/A                        # defaults.
1352N/A                        created = self.__data["last-modified"]
2810N/A                        self.__data = copy.deepcopy(self.__DEFAULT_ELEMS)
2810N/A                        self.__data["created"] = created
2810N/A                        self.__data["last-modified"] = created
1352N/A                else:
1352N/A                        # Assume that the attributes of the catalog can be
1352N/A                        # obtained from a file.
1352N/A                        self.load()
1352N/A
1352N/A        def __get_created(self):
1352N/A                return self.__data["created"]
1352N/A
1352N/A        def __get_last_modified(self):
1352N/A                return self.__data["last-modified"]
1352N/A
1352N/A        def __get_package_count(self):
1352N/A                return self.__data["package-count"]
1352N/A
1352N/A        def __get_package_version_count(self):
1352N/A                return self.__data["package-version-count"]
1352N/A
1352N/A        def __get_parts(self):
1352N/A                return self.__data["parts"]
1352N/A
1352N/A        def __get_updates(self):
1352N/A                return self.__data["updates"]
1352N/A
1352N/A        def __get_version(self):
1352N/A                return self.__data["version"]
1352N/A
1352N/A        def __set_created(self, value):
1352N/A                self.__data["created"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_last_modified(self, value):
1352N/A                self.__data["last-modified"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_package_count(self, value):
1352N/A                self.__data["package-count"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_package_version_count(self, value):
1352N/A                self.__data["package-version-count"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_parts(self, value):
1352N/A                self.__data["parts"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_updates(self, value):
1352N/A                self.__data["updates"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __set_version(self, value):
1352N/A                self.__data["version"] = value
1352N/A                self.signatures = {}
1352N/A
1352N/A        def __transform(self):
1352N/A                """Duplicate and transform 'self.__data' for saving."""
1352N/A
1352N/A                # Use a copy to prevent the in-memory version from being
1352N/A                # affected by the transformations.
1352N/A                struct = copy.deepcopy(self.__data)
3234N/A                for key, val in six.iteritems(struct):
1352N/A                        if isinstance(val, datetime.datetime):
1352N/A                                # Convert datetime objects to an ISO-8601
1352N/A                                # basic format string.
1352N/A                                struct[key] = datetime_to_basic_ts(val)
1352N/A                                continue
1352N/A
1352N/A                        if key in ("parts", "updates"):
1352N/A                                for e in val:
1352N/A                                        lm = val[e].get("last-modified", None)
1352N/A                                        if lm:
1352N/A                                                lm = datetime_to_basic_ts(lm)
1352N/A                                                val[e]["last-modified"] = lm
1352N/A                return struct
1352N/A
1352N/A        def load(self):
1352N/A                """Load and transform the catalog attribute data."""
1352N/A
1352N/A                if self.loaded:
1352N/A                        # Already loaded, or only in-memory.
1352N/A                        return
2810N/A                location = os.path.join(self.meta_root, self.name)
1352N/A
1352N/A                struct = CatalogPartBase.load(self)
2810N/A                # Check to see that struct is as we expect: it must be a dict
2810N/A                # and have all of the elements in self.__DEFAULT_ELEMS.
3194N/A                if type(struct) != dict or \
2810N/A                    not (set(self.__DEFAULT_ELEMS.keys()) <= \
2810N/A                    set(struct.keys())):
2810N/A                        raise api_errors.InvalidCatalogFile(location)
2810N/A
2810N/A                def cat_ts_to_datetime(val):
2810N/A                        try:
2810N/A                                return basic_ts_to_datetime(val)
2810N/A                        except ValueError:
2810N/A                                raise api_errors.InvalidCatalogFile(location)
2810N/A
3234N/A                for key, val in six.iteritems(struct):
1352N/A                        if key in ("created", "last-modified"):
1352N/A                                # Convert ISO-8601 basic format strings to
3329N/A                                # datetime objects.
1352N/A                                if val:
2810N/A                                        struct[key] = cat_ts_to_datetime(val)
1352N/A                                continue
1352N/A
1352N/A                        if key in ("parts", "updates"):
3194N/A                                if type(val) != dict:
2810N/A                                        raise api_errors.InvalidCatalogFile(
2810N/A                                            location)
2810N/A
2810N/A                                # 'parts' and 'updates' have a more complex
2810N/A                                # structure.  Check that all of the subparts
2810N/A                                # look sane.
2810N/A                                for subpart in val:
2810N/A                                        if subpart != os.path.basename(subpart):
2810N/A                                                raise api_errors.\
2810N/A                                                    UnrecognizedCatalogPart(
3158N/A                                                    "{0} {{{1}: {2}}}".format(
3158N/A                                                    self.name, key, subpart))
3362N/A                                        # Check if subpart is a symbolic link
3362N/A                                        # that would cause an access to be
3362N/A                                        # redirected outside of 'file_root'.
3362N/A                                        try:
3362N/A                                                misc.open_image_file(
3362N/A                                                    self.file_root,
3362N/A                                                    os.path.join(self.meta_root,
3362N/A                                                    subpart), os.O_RDONLY,
3362N/A                                                    misc.PKG_FILE_MODE)
3362N/A                                        except OSError as e:
3362N/A                                                if e.errno == errno.EREMOTE:
3362N/A                                                        raise api_errors.\
3362N/A                                                            UnrecognizedCatalogPart(
3362N/A                                                            "{0} {{{1}: {2}}}".format(
3362N/A                                                            self.name, key, subpart))
2810N/A
2810N/A                                # Build datetimes from timestamps.
1352N/A                                for e in val:
1352N/A                                        lm = val[e].get("last-modified", None)
1352N/A                                        if lm:
2810N/A                                                lm = cat_ts_to_datetime(lm)
1352N/A                                                val[e]["last-modified"] = lm
2810N/A
1352N/A                self.__data = struct
1352N/A
1352N/A        def save(self):
1352N/A                """Transform and store the catalog attribute data in a file
1352N/A                using the pathname <self.meta_root>/<self.name>."""
1352N/A
1352N/A                if not self.meta_root:
1352N/A                        # Assume this is in-memory only.
1352N/A                        return
2022N/A
2022N/A                # Ensure content is loaded before attempting save.
2022N/A                self.load()
2022N/A
1833N/A                CatalogPartBase.save(self, self.__transform(), single_pass=True)
1352N/A
2810N/A        def validate(self, signatures=None, require_signatures=False):
1352N/A                """Verifies whether the signatures for the contents of the
1352N/A                CatalogAttrs match the specified signature data, or if not
1352N/A                provided, the current signature data.  Raises the exception
1352N/A                named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A                if not self.signatures and not signatures and \
2810N/A                    not require_signatures:
2810N/A                        # Nothing to validate, and we're not required to.
1352N/A                        return
1352N/A
2022N/A                # Ensure content is loaded before attempting to retrieve
2022N/A                # or generate signature data.
2022N/A                self.load()
1352N/A                if not signatures:
1352N/A                        signatures = self.signatures
1352N/A
1352N/A                new_signatures = self._gen_signatures(self.__transform())
1352N/A                if new_signatures != signatures:
1352N/A                        raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
1352N/A        created = property(__get_created, __set_created)
1352N/A
1352N/A        last_modified = property(__get_last_modified, __set_last_modified)
1352N/A
1352N/A        package_count = property(__get_package_count, __set_package_count)
1352N/A
1352N/A        package_version_count = property(__get_package_version_count,
1352N/A            __set_package_version_count)
1352N/A
1352N/A        parts = property(__get_parts, __set_parts)
1352N/A
1352N/A        updates = property(__get_updates, __set_updates)
1352N/A
1352N/A        version = property(__get_version, __set_version)
814N/A
852N/A
34N/Aclass Catalog(object):
1352N/A        """A Catalog is the representation of the package FMRIs available from
1352N/A        a package repository."""
37N/A
1431N/A        __BASE_PART = "catalog.base.C"
1431N/A        __DEPS_PART = "catalog.dependency.C"
1431N/A        __SUMM_PART_PFX = "catalog.summary"
1352N/A
1352N/A        # The file mode to be used for all catalog files.
1352N/A        __file_mode = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH
1352N/A
1352N/A        # These properties are declared here so that they show up in the pydoc
1352N/A        # documentation as private, and for clarity in the property declarations
1352N/A        # found near the end of the class definition.
1352N/A        _attrs = None
1352N/A        __batch_mode = None
1352N/A        __lock = None
1352N/A        __meta_root = None
3362N/A        __file_root = None
1358N/A        __sign = None
23N/A
1352N/A        # These are used to cache or store CatalogPart and CatalogUpdate objects
1352N/A        # as they are used.  It should not be confused with the CatalogPart
1352N/A        # names and CatalogUpdate names stored in the CatalogAttrs object.
1352N/A        __parts = None
1352N/A        __updates = None
1352N/A
1352N/A        # Class Constants
1369N/A        DEPENDENCY, SUMMARY = range(2)
1352N/A
1352N/A        def __init__(self, batch_mode=False, meta_root=None, log_updates=False,
3362N/A            read_only=False, sign=True, file_root=None):
1352N/A                """Initializes a Catalog object.
157N/A
1352N/A                'batch_mode' is an optional boolean value that indicates that
1352N/A                the caller intends to perform multiple modifying operations on
1352N/A                catalog before saving.  This is useful for performance reasons
1352N/A                as the contents of the catalog will not be sorted after each
1352N/A                change, and the package counts will not be updated (except at
1352N/A                save()).  By default this value is False.  If this value is
1352N/A                True, callers are responsible for calling finalize() to ensure
1352N/A                that catalog entries are in the correct order and package counts
1352N/A                accurately reflect the catalog contents.
429N/A
1352N/A                'meta_root' is an optional absolute pathname of a directory
1352N/A                that catalog metadata can be written to and read from, and
1352N/A                must already exist.  If no path is supplied, then it is
1352N/A                assumed that the catalog object will be used for in-memory
1352N/A                operations only.
445N/A
1352N/A                'log_updates' is an optional boolean value indicating whether
1352N/A                updates to the catalog should be logged.  This enables consumers
1352N/A                of the catalog to perform incremental updates.
34N/A
1352N/A                'read_only' is an optional boolean value that indicates if
1352N/A                operations that modify the catalog are allowed (an assertion
1358N/A                error will be raised if one is attempted and this is True).
1358N/A
1358N/A                'sign' is an optional boolean value that indicates that the
1358N/A                the catalog data should have signature data generated and
1358N/A                embedded when serialized.  This option is primarily a matter
1358N/A                of convenience for callers that wish to trade integrity checks
1358N/A                for improved catalog serialization performance."""
1352N/A
1352N/A                self.__batch_mode = batch_mode
1352N/A                self.__parts = {}
1352N/A                self.__updates = {}
157N/A
1352N/A                # Must be set after the above.
1431N/A                self.log_updates = log_updates
1352N/A                self.meta_root = meta_root
1352N/A                self.read_only = read_only
1358N/A                self.sign = sign
3362N/A                self.file_root = file_root
3362N/A                if not self.file_root:
3362N/A                        self.file_root = meta_root
1352N/A
1352N/A                # Must be set after the above.
3362N/A                self._attrs = CatalogAttrs(meta_root=self.meta_root, sign=sign,
3362N/A                    file_root=file_root)
157N/A
1431N/A                # This lock is used to protect the catalog file from multiple
1431N/A                # threads writing to it at the same time.
1431N/A                self.__lock = threading.Lock()
157N/A
1352N/A                # Must be done last.
814N/A                self.__set_perms()
814N/A
1537N/A        def __actions(self, info_needed, excludes=EmptyI, cb=None, locales=None,
1537N/A            last_version=False, ordered=False, pubs=EmptyI):
1537N/A                assert info_needed
1537N/A                if not locales:
1537N/A                        locales = set(("C",))
1537N/A                else:
1537N/A                        locales = set(locales)
1537N/A
1537N/A                for f, entry in self.__entries(cb=cb, info_needed=info_needed,
1537N/A                    locales=locales, last_version=last_version,
1537N/A                    ordered=ordered, pubs=pubs):
2639N/A                        try:
1970N/A                                yield f, self.__gen_actions(f, entry["actions"],
1537N/A                                    excludes)
2639N/A                        except KeyError:
3329N/A                                yield f, EmptyI
1537N/A
1463N/A        def __append(self, src, cb=None, pfmri=None, pubs=EmptyI):
1463N/A                """Private version; caller responsible for locking."""
1463N/A
1463N/A                base = self.get_part(self.__BASE_PART)
1463N/A                src_base = src.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if src_base is None:
1463N/A                        if pfmri:
1463N/A                                raise api_errors.UnknownCatalogEntry(pfmri)
1463N/A                        # Nothing to do
1463N/A                        return
1463N/A
1463N/A                # Use the same operation time and date for all operations so
1463N/A                # that the last modification times will be synchronized.  This
1463N/A                # also has the benefit of avoiding extra datetime object
1463N/A                # instantiations.
1463N/A                op_time = datetime.datetime.utcnow()
1463N/A
1463N/A                # For each entry in the 'src' catalog, add its BASE entry to the
1463N/A                # current catalog along and then add it to the 'd'iscard dict if
1463N/A                # 'cb' is defined and returns False.
1463N/A                if pfmri:
1537N/A                        entry = src_base.get_entry(pfmri)
1537N/A                        if entry is None:
1537N/A                                raise api_errors.UnknownCatalogEntry(
1537N/A                                    pfmri.get_fmri())
1537N/A                        entries = [(pfmri, entry)]
1463N/A                else:
1463N/A                        entries = src_base.entries()
1463N/A
1463N/A                d = {}
1463N/A                for f, entry in entries:
1463N/A                        if pubs and f.publisher not in pubs:
1463N/A                                continue
1463N/A
1463N/A                        nentry = copy.deepcopy(entry)
1537N/A                        if cb is not None:
1463N/A                                merge, mdata = cb(src, f, entry)
1463N/A                                if not merge:
1463N/A                                        pub = d.setdefault(f.publisher, {})
1463N/A                                        plist = pub.setdefault(f.pkg_name,
1463N/A                                            set())
1463N/A                                        plist.add(f.version)
1463N/A                                        continue
1463N/A
1463N/A                                if mdata:
1463N/A                                        if "metadata" in nentry:
1463N/A                                                nentry["metadata"].update(mdata)
1463N/A                                        else:
1463N/A                                                nentry["metadata"] = mdata
1463N/A                        base.add(f, metadata=nentry, op_time=op_time)
1463N/A
1463N/A                if d and pfmri:
1463N/A                        # If the 'd'iscards dict is populated and pfmri is
1463N/A                        # defined, then there is nothing more to do.
1463N/A                        return
1463N/A
1463N/A                # Finally, merge any catalog part entries that exist unless the
1463N/A                # FMRI is found in the 'd'iscard dict.
1463N/A                for name in src.parts.keys():
1463N/A                        if name == self.__BASE_PART:
1463N/A                                continue
1463N/A
1463N/A                        part = src.get_part(name, must_exist=True)
1537N/A                        if part is None:
1463N/A                                # Part doesn't exist in-memory or on-disk, so
1463N/A                                # skip it.
1463N/A                                continue
1463N/A
1463N/A                        if pfmri:
1537N/A                                entry = part.get_entry(pfmri)
1537N/A                                if entry is None:
1463N/A                                        # Package isn't in this part; skip it.
1463N/A                                        continue
1537N/A                                entries = [(pfmri, entry)]
1463N/A                        else:
1463N/A                                entries = part.entries()
1463N/A
1463N/A                        npart = self.get_part(name)
1463N/A                        for f, entry in entries:
1463N/A                                if pubs and f.publisher not in pubs:
1463N/A                                        continue
1463N/A                                if f.publisher in d and \
1463N/A                                    f.pkg_name in d[f.publisher] and \
1463N/A                                    f.version in d[f.publisher][f.pkg_name]:
1463N/A                                        # Skip this package.
1463N/A                                        continue
1463N/A
1463N/A                                nentry = copy.deepcopy(entry)
1463N/A                                npart.add(f, metadata=nentry, op_time=op_time)
1463N/A
1537N/A        def __entries(self, cb=None, info_needed=EmptyI,
1537N/A            last_version=False, locales=None, ordered=False, pubs=EmptyI,
1537N/A            tuples=False):
1537N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1537N/A                        # Catalog contains nothing.
1537N/A                        return
1537N/A
1537N/A                if not locales:
1537N/A                        locales = set(("C",))
1537N/A                else:
1537N/A                        locales = set(locales)
1537N/A
1537N/A                parts = []
1537N/A                if self.DEPENDENCY in info_needed:
1537N/A                        part = self.get_part(self.__DEPS_PART, must_exist=True)
1537N/A                        if part is not None:
1537N/A                                parts.append(part)
1537N/A
1537N/A                if self.SUMMARY in info_needed:
1537N/A                        for locale in locales:
1537N/A                                part = self.get_part(
3158N/A                                    "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A                                    locale), must_exist=True)
1537N/A                                if part is None:
1537N/A                                        # Data not available for this
1537N/A                                        # locale.
1537N/A                                        continue
1537N/A                                parts.append(part)
1537N/A
1537N/A                def merge_entry(src, dest):
3234N/A                        for k, v in six.iteritems(src):
1537N/A                                if k == "actions":
1537N/A                                        dest.setdefault(k, [])
1537N/A                                        dest[k] += v
1537N/A                                elif k != "version":
1537N/A                                        dest[k] = v
1537N/A
1537N/A                if tuples:
1537N/A                        for r, bentry in base.tuple_entries(cb=cb,
1537N/A                            last=last_version, ordered=ordered, pubs=pubs):
1537N/A                                pub, stem, ver = r
1537N/A                                mdata = {}
1537N/A                                merge_entry(bentry, mdata)
1537N/A                                for part in parts:
1537N/A                                        entry = part.get_entry(pub=pub,
1537N/A                                            stem=stem, ver=ver)
1537N/A                                        if entry is None:
1537N/A                                                # Part doesn't have this FMRI,
1537N/A                                                # so skip it.
1537N/A                                                continue
3234N/A                                        for k, v in six.iteritems(entry):
1537N/A                                                if k == "actions":
1537N/A                                                        mdata.setdefault(k, [])
1537N/A                                                        mdata[k] += v
1537N/A                                                elif k != "version":
1537N/A                                                        mdata[k] = v
1537N/A                                yield r, mdata
1537N/A                        return
1537N/A
1537N/A                for f, bentry in base.entries(cb=cb, last=last_version,
1537N/A                    ordered=ordered, pubs=pubs):
1537N/A                        mdata = {}
1537N/A                        merge_entry(bentry, mdata)
1537N/A                        for part in parts:
1537N/A                                entry = part.get_entry(f)
1537N/A                                if entry is None:
1537N/A                                        # Part doesn't have this FMRI,
1537N/A                                        # so skip it.
1537N/A                                        continue
3234N/A                                for k, v in six.iteritems(entry):
1537N/A                                        if k == "actions":
1537N/A                                                mdata.setdefault(k, [])
1537N/A                                                mdata[k] += v
1537N/A                                        elif k != "version":
1537N/A                                                mdata[k] = v
1537N/A                        yield f, mdata
1537N/A
1463N/A        def __finalize(self, pfmris=None, pubs=None, sort=True):
1352N/A                """Private finalize method; exposes additional controls for
1352N/A                internal callers."""
1352N/A
1352N/A                package_count = 0
1352N/A                package_version_count = 0
1352N/A
1431N/A                part = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if part is not None:
1352N/A                        # If the base Catalog didn't exist (in-memory or on-
1352N/A                        # disk) that implies there is nothing to sort and
1352N/A                        # there are no packages (since the base catalog part
1352N/A                        # must always exist for packages to be present).
1352N/A                        package_count, package_version_count = \
1352N/A                            part.get_package_counts()
1352N/A
1352N/A                        if sort:
1352N/A                                # Some operations don't need this, such as
1352N/A                                # remove...
1352N/A                                for part in self.__parts.values():
1463N/A                                        part.sort(pfmris=pfmris, pubs=pubs)
1352N/A
1352N/A                self._attrs.package_count = package_count
1352N/A                self._attrs.package_version_count = \
1352N/A                    package_version_count
1352N/A
1369N/A        @staticmethod
1970N/A        def __gen_actions(pfmri, actions, excludes=EmptyI):
1970N/A                errors = None
3110N/A                if not isinstance(pfmri, fmri.PkgFmri):
3110N/A                        # pfmri is assumed to be a FMRI tuple.
3110N/A                        pub, stem, ver = pfmri
3110N/A                else:
3110N/A                        pub = pfmri.publisher
1369N/A                for astr in actions:
1970N/A                        try:
1970N/A                                a = pkg.actions.fromstr(astr)
3171N/A                        except pkg.actions.ActionError as e:
1970N/A                                # Accumulate errors and continue so that as
1970N/A                                # much of the action data as possible can be
1970N/A                                # parsed.
1970N/A                                if errors is None:
1970N/A                                        # Allocate this here to avoid overhead
1970N/A                                        # of list allocation/deallocation.
1970N/A                                        errors = []
1970N/A                                if not isinstance(pfmri, fmri.PkgFmri):
2639N/A                                        pfmri = fmri.PkgFmri(name=stem,
2639N/A                                            publisher=pub, version=ver)
1970N/A                                e.fmri = pfmri
1970N/A                                errors.append(e)
1970N/A                                continue
1970N/A
1537N/A                        if a.name == "set" and \
1537N/A                            (a.attrs["name"].startswith("facet") or
1537N/A                            a.attrs["name"].startswith("variant")):
1537N/A                                # Don't filter actual facet or variant
1537N/A                                # set actions.
1537N/A                                yield a
3110N/A                        elif a.include_this(excludes,
3110N/A                            publisher=pub):
1369N/A                                yield a
1369N/A
1970N/A                if errors is not None:
1970N/A                        raise api_errors.InvalidPackageErrors(errors)
1970N/A
1369N/A        @staticmethod
1369N/A        def __gen_manifest_actions(m, atypes, excludes):
1369N/A                """Private helper function to iterate over a Manifest's actions
1369N/A                by action type, returning tuples of (action, attr_name)."""
3110N/A                pub = m.publisher
1369N/A                for atype in atypes:
1369N/A                        for a in m.gen_actions_by_type(atype):
3110N/A                                if not a.include_this(excludes,
3110N/A                                    publisher=pub):
1369N/A                                        continue
1369N/A
1369N/A                                if atype == "set":
1369N/A                                        yield a, a.attrs["name"]
1369N/A                                else:
1369N/A                                        yield a, None
1369N/A
1352N/A        def __get_batch_mode(self):
1352N/A                return self.__batch_mode
1352N/A
1369N/A        def __get_last_modified(self):
1369N/A                return self._attrs.last_modified
1369N/A
1352N/A        def __get_meta_root(self):
1352N/A                return self.__meta_root
1352N/A
3362N/A        def __get_file_root(self):
3362N/A                return self.__file_root
3362N/A
1358N/A        def __get_sign(self):
1358N/A                return self.__sign
1358N/A
1352N/A        def __get_update(self, name, cache=True, must_exist=False):
1352N/A                # First, check if the update has already been cached,
1352N/A                # and if so, return it.
1537N/A                ulog = self.__updates.get(name, None)
1537N/A                if ulog is not None:
1537N/A                        return ulog
1537N/A                elif not self.meta_root and must_exist:
1537N/A                        return
1352N/A
1352N/A                # Next, if the update hasn't been cached,
1352N/A                # create an object for it.
1431N/A                ulog = CatalogUpdate(name, meta_root=self.meta_root,
1431N/A                    sign=self.__sign)
1352N/A                if self.meta_root and must_exist and not ulog.exists:
1352N/A                        # Update doesn't exist on-disk,
1352N/A                        # so don't return anything.
1352N/A                        return
1352N/A                if cache:
1352N/A                        self.__updates[name] = ulog
1352N/A                return ulog
1352N/A
1369N/A        def __get_version(self):
1369N/A                return self._attrs.version
1369N/A
1352N/A        def __lock_catalog(self):
1352N/A                """Locks the catalog preventing multiple threads or external
1352N/A                consumers of the catalog from modifying it during operations.
1352N/A                """
1352N/A
1352N/A                # XXX need filesystem lock too?
1352N/A                self.__lock.acquire()
1352N/A
1352N/A        def __log_update(self, pfmri, operation, op_time, entries=None):
1352N/A                """Helper function to log catalog changes."""
1352N/A
1352N/A                if not self.__batch_mode:
1352N/A                        # The catalog.attrs needs to be updated to reflect
1463N/A                        # the changes made.  A sort doesn't need to be done
1463N/A                        # here as the individual parts will automatically do
1463N/A                        # that as needed in this case.
1463N/A                        self.__finalize(sort=False)
1352N/A
1352N/A                # This must be set to exactly the same time as the update logs
1352N/A                # so that the changes in the update logs are not marked as
1352N/A                # being newer than the catalog or vice versa.
1352N/A                attrs = self._attrs
1352N/A                attrs.last_modified = op_time
1352N/A
1431N/A                if not self.log_updates:
1352N/A                        return
814N/A
1352N/A                updates = {}
1352N/A                for pname in entries:
1352N/A                        # The last component of the updatelog filename is the
1352N/A                        # related locale.
1352N/A                        locale = pname.split(".", 2)[2]
1537N/A                        updates.setdefault(locale, {})
1537N/A                        parts = updates[locale]
1352N/A                        parts[pname] = entries[pname]
1352N/A
1352N/A                logdate = datetime_to_update_ts(op_time)
3234N/A                for locale, metadata in six.iteritems(updates):
3158N/A                        name = "update.{0}.{1}".format(logdate, locale)
1352N/A                        ulog = self.__get_update(name)
1352N/A                        ulog.add(pfmri, operation, metadata=metadata,
1352N/A                            op_time=op_time)
1352N/A                        attrs.updates[name] = {
1352N/A                            "last-modified": op_time
1352N/A                        }
1352N/A
3234N/A                for name, part in six.iteritems(self.__parts):
1352N/A                        # Signature data for each part needs to be cleared,
1352N/A                        # and will only be available again after save().
1352N/A                        attrs.parts[name] = {
1352N/A                            "last-modified": part.last_modified
1352N/A                        }
1352N/A
2510N/A        @staticmethod
2510N/A        def __parse_fmri_patterns(patterns):
2510N/A                """A generator function that yields a list of tuples of the form
2510N/A                (pattern, error, fmri, matcher) based on the provided patterns,
2510N/A                where 'error' is any exception encountered while parsing the
2510N/A                pattern, 'fmri' is the resulting FMRI object, and 'matcher' is
2510N/A                one of the following pkg.fmri matching functions:
2510N/A
2510N/A                        pkg.fmri.exact_name_match
2510N/A                                Indicates that the name portion of the pattern
2510N/A                                must match exactly and the version (if provided)
2510N/A                                must be considered a successor or equal to the
2510N/A                                target FMRI.
2510N/A
2510N/A                        pkg.fmri.fmri_match
2510N/A                                Indicates that the name portion of the pattern
2510N/A                                must be a proper subset and the version (if
2510N/A                                provided) must be considered a successor or
2510N/A                                equal to the target FMRI.
2510N/A
2510N/A                        pkg.fmri.glob_match
2510N/A                                Indicates that the name portion of the pattern
2510N/A                                uses fnmatch rules for pattern matching (shell-
2510N/A                                style wildcards) and that the version can either
2510N/A                                match exactly, match partially, or contain
2510N/A                                wildcards.
2510N/A                """
2510N/A
2510N/A                for pat in patterns:
2510N/A                        error = None
2510N/A                        matcher = None
2510N/A                        npat = None
2510N/A                        try:
2510N/A                                parts = pat.split("@", 1)
2510N/A                                pat_stem = parts[0]
2510N/A                                pat_ver = None
2510N/A                                if len(parts) > 1:
2510N/A                                        pat_ver = parts[1]
2510N/A
2510N/A                                if "*" in pat_stem or "?" in pat_stem:
2510N/A                                        matcher = fmri.glob_match
2510N/A                                elif pat_stem.startswith("pkg:/") or \
2510N/A                                    pat_stem.startswith("/"):
2510N/A                                        matcher = fmri.exact_name_match
2510N/A                                else:
2510N/A                                        matcher = fmri.fmri_match
2510N/A
2510N/A                                if matcher == fmri.glob_match:
2958N/A                                        npat = fmri.MatchingPkgFmri(pat_stem)
2510N/A                                else:
2958N/A                                        npat = fmri.PkgFmri(pat_stem)
2510N/A
2510N/A                                if not pat_ver:
2510N/A                                        # Do nothing.
2510N/A                                        pass
2510N/A                                elif "*" in pat_ver or "?" in pat_ver or \
2510N/A                                    pat_ver == "latest":
2510N/A                                        npat.version = \
2958N/A                                            pkg.version.MatchingVersion(pat_ver)
2510N/A                                else:
2510N/A                                        npat.version = \
2958N/A                                            pkg.version.Version(pat_ver)
2510N/A
3171N/A                        except (fmri.FmriError, pkg.version.VersionError) as e:
2510N/A                                # Whatever the error was, return it.
2510N/A                                error = e
2510N/A                        yield (pat, error, npat, matcher)
2510N/A
1352N/A        def __save(self):
1352N/A                """Private save function.  Caller is responsible for locking
1352N/A                the catalog."""
1352N/A
1352N/A                attrs = self._attrs
1431N/A                if self.log_updates:
3234N/A                        for name, ulog in six.iteritems(self.__updates):
1352N/A                                ulog.save()
1352N/A
1352N/A                                # Replace the existing signature data
1352N/A                                # with the new signature data.
1352N/A                                entry = attrs.updates[name] = {
1352N/A                                    "last-modified": ulog.last_modified
1352N/A                                }
3234N/A                                for n, v in six.iteritems(ulog.signatures):
3158N/A                                        entry["signature-{0}".format(n)] = v
1352N/A
1352N/A                # Save any CatalogParts that are currently in-memory,
1352N/A                # updating their related information in catalog.attrs
1352N/A                # as they are saved.
3234N/A                for name, part in six.iteritems(self.__parts):
1352N/A                        # Must save first so that signature data is
1352N/A                        # current.
1833N/A
1833N/A                        # single-pass encoding is not used for summary part as
2810N/A                        # it increases memory usage substantially (30MB at
1833N/A                        # current for /dev).  No significant difference is
1833N/A                        # detectable for other parts though.
1833N/A                        single_pass = name in (self.__BASE_PART,
1833N/A                            self.__DEPS_PART)
1833N/A                        part.save(single_pass=single_pass)
1352N/A
1352N/A                        # Now replace the existing signature data with
1352N/A                        # the new signature data.
1352N/A                        entry = attrs.parts[name] = {
1352N/A                            "last-modified": part.last_modified
1352N/A                        }
3234N/A                        for n, v in six.iteritems(part.signatures):
3158N/A                                entry["signature-{0}".format(n)] = v
1352N/A
1352N/A                # Finally, save the catalog attributes.
1352N/A                attrs.save()
1352N/A
1352N/A        def __set_batch_mode(self, value):
1352N/A                self.__batch_mode = value
1352N/A                for part in self.__parts.values():
1352N/A                        part.ordered = not self.__batch_mode
1352N/A
1369N/A        def __set_last_modified(self, value):
1369N/A                self._attrs.last_modified = value
1369N/A
1352N/A        def __set_meta_root(self, pathname):
1352N/A                if pathname:
1352N/A                        pathname = os.path.abspath(pathname)
1352N/A                self.__meta_root = pathname
1352N/A
1352N/A                # If the Catalog's meta_root changes, the meta_root of all of
1352N/A                # its parts must be changed too.
1352N/A                if self._attrs:
1352N/A                        self._attrs.meta_root = pathname
1352N/A
1352N/A                for part in self.__parts.values():
1352N/A                        part.meta_root = pathname
1352N/A
1352N/A                for ulog in self.__updates.values():
1352N/A                        ulog.meta_root = pathname
1352N/A
3362N/A        def __set_file_root(self, pathname):
3362N/A                if pathname:
3362N/A                        pathname = os.path.abspath(pathname)
3362N/A                self.__file_root = pathname
3362N/A
3362N/A                # If the Catalog's file_root changes, the file_root of all of
3362N/A                # its parts must be changed too.
3362N/A                if self._attrs:
3362N/A                        self._attrs.file_root = pathname
3362N/A
3362N/A                for part in self.__parts.values():
3362N/A                        part.file_root = pathname
3362N/A
3362N/A                for ulog in self.__updates.values():
3362N/A                        ulog.file_root = pathname
3362N/A
1352N/A        def __set_perms(self):
1352N/A                """Sets permissions on attrs and parts if not read_only and if
1352N/A                the current user can do so; raises BadCatalogPermissions if the
1352N/A                permissions are wrong and cannot be corrected."""
1352N/A
1352N/A                if not self.meta_root:
1352N/A                        # Nothing to do.
1352N/A                        return
1352N/A
1352N/A                files = [self._attrs.name]
1352N/A                files.extend(self._attrs.parts.keys())
1352N/A                files.extend(self._attrs.updates.keys())
814N/A
814N/A                # Force file_mode, so that unprivileged users can read these.
814N/A                bad_modes = []
1352N/A                for name in files:
1352N/A                        pathname = os.path.join(self.meta_root, name)
814N/A                        try:
814N/A                                if self.read_only:
1352N/A                                        fmode = stat.S_IMODE(os.stat(
1352N/A                                            pathname).st_mode)
1352N/A                                        if fmode != self.__file_mode:
1352N/A                                                bad_modes.append((pathname,
3158N/A                                                    "{0:o}".format(
3158N/A                                                    self.__file_mode),
3158N/A                                                    "{0:o}".format(fmode)))
814N/A                                else:
1352N/A                                        os.chmod(pathname, self.__file_mode)
3171N/A                        except EnvironmentError as e:
1352N/A                                # If the file doesn't exist yet, move on.
814N/A                                if e.errno == errno.ENOENT:
814N/A                                        continue
814N/A
814N/A                                # If the mode change failed for another reason,
814N/A                                # check to see if we actually needed to change
814N/A                                # it, and if so, add it to bad_modes.
1352N/A                                fmode = stat.S_IMODE(os.stat(
1352N/A                                    pathname).st_mode)
1352N/A                                if fmode != self.__file_mode:
1352N/A                                        bad_modes.append((pathname,
3158N/A                                            "{0:o}".format(self.__file_mode),
3158N/A                                            "{0:o}".format(fmode)))
814N/A
814N/A                if bad_modes:
1352N/A                        raise api_errors.BadCatalogPermissions(bad_modes)
50N/A
1358N/A        def __set_sign(self, value):
1358N/A                self.__sign = value
1358N/A
1358N/A                # If the Catalog's sign property changes, the value of that
1431N/A                # property for its attributes, etc. must be changed too.
1358N/A                if self._attrs:
1358N/A                        self._attrs.sign = value
1358N/A
1358N/A                for part in self.__parts.values():
1358N/A                        part.sign = value
1358N/A
1431N/A                for ulog in self.__updates.values():
1431N/A                        ulog.sign = value
1431N/A
1369N/A        def __set_version(self, value):
1369N/A                self._attrs.version = value
1369N/A
1352N/A        def __unlock_catalog(self):
1352N/A                """Unlocks the catalog allowing other catalog consumers to
1352N/A                modify it."""
258N/A
1352N/A                # XXX need filesystem unlock too?
1352N/A                self.__lock.release()
1066N/A
1537N/A        def actions(self, info_needed, excludes=EmptyI, cb=None,
1537N/A            last=False, locales=None, ordered=False, pubs=EmptyI):
1369N/A                """A generator function that produces tuples of the format
1369N/A                (fmri, actions) as it iterates over the contents of the
1369N/A                catalog (where 'actions' is a generator that returns the
1431N/A                Actions corresponding to the requested information).
1537N/A
1431N/A                If the catalog doesn't contain any action data for the package
3329N/A                entry, it will return an empty iterator.
1369N/A
1369N/A                'excludes' is a list of variants which will be used to determine
1369N/A                what should be allowed by the actions generator in addition to
1537N/A                what is specified by 'info_needed'.
1537N/A
1537N/A                'cb' is an optional callback function that will be executed for
1537N/A                each package before its action data is retrieved. It must accept
1537N/A                two arguments: 'pkg' and 'entry'.  'pkg' is an FMRI object and
1537N/A                'entry' is the dictionary structure of the catalog entry for the
1537N/A                package.  If the callback returns False, then the entry will not
1537N/A                be included in the results.  This can significantly improve
1537N/A                performance by avoiding action data retrieval for results that
1537N/A                will not be used.
1369N/A
1369N/A                'info_needed' is a set of one or more catalog constants
1369N/A                indicating the types of catalog data that will be returned
1369N/A                in 'actions' in addition to the above:
1369N/A
1369N/A                        DEPENDENCY
1369N/A                                Depend and set Actions for package obsoletion,
1369N/A                                renaming, variants.
1369N/A
1369N/A                        SUMMARY
1369N/A                                Any remaining set Actions not listed above, such
1369N/A                                as pkg.summary, pkg.description, etc.
1369N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the catalog has been saved since the last modifying
1537N/A                operation, or finalize() has has been called, this will also be
1537N/A                the newest version of the package.
1537N/A
1369N/A                'locales' is an optional set of locale names for which Actions
1369N/A                should be returned.  The default is set(('C',)) if not provided.
1369N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pfmri' is an optional FMRI to limit the returned results to.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                return self.__actions(info_needed, excludes=excludes,
1537N/A                    cb=cb, last_version=last, locales=locales, ordered=ordered,
1537N/A                    pubs=pubs)
1369N/A
1352N/A        def add_package(self, pfmri, manifest=None, metadata=None):
1352N/A                """Add a package and its related metadata to the catalog and
1352N/A                its parts as needed.
1066N/A
1352N/A                'manifest' is an optional Manifest object that will be used
1352N/A                to retrieve the metadata related to the package.
1066N/A
1352N/A                'metadata' is an optional dict of additional metadata to store
1352N/A                with the package's BASE record."""
1066N/A
1369N/A                assert not self.read_only
1369N/A
1352N/A                def group_actions(actions):
1352N/A                        dep_acts = { "C": [] }
1352N/A                        # Summary actions are grouped by locale, since each
1352N/A                        # goes to a locale-specific catalog part.
1352N/A                        sum_acts = { "C": [] }
1352N/A                        for act in actions:
1352N/A                                if act.name == "depend":
1352N/A                                        dep_acts["C"].append(str(act))
1352N/A                                        continue
429N/A
1352N/A                                name = act.attrs["name"]
1352N/A                                if name.startswith("variant") or \
1352N/A                                    name.startswith("facet") or \
1731N/A                                    name.startswith("pkg.depend.") or \
1461N/A                                    name in ("pkg.obsolete", "pkg.renamed"):
1352N/A                                        # variant and facet data goes to the
1352N/A                                        # dependency catalog part.
1352N/A                                        dep_acts["C"].append(str(act))
1352N/A                                        continue
1369N/A                                elif name in ("fmri", "pkg.fmri"):
1352N/A                                        # Redundant in the case of the catalog.
1352N/A                                        continue
419N/A
1352N/A                                # All other set actions go to the summary
1352N/A                                # catalog parts, grouped by locale.  To
1352N/A                                # determine the locale, the set attribute's
1352N/A                                # name is split by ':' into its field and
1352N/A                                # locale components.  If ':' is not present,
1352N/A                                # then the 'C' locale is assumed.
1352N/A                                comps = name.split(":")
1352N/A                                if len(comps) > 1:
1352N/A                                        locale = comps[1]
1352N/A                                else:
1352N/A                                        locale = "C"
1352N/A                                if locale not in sum_acts:
1352N/A                                        sum_acts[locale] = []
1352N/A                                sum_acts[locale].append(str(act))
419N/A
1352N/A                        return {
1352N/A                            "dependency": dep_acts,
1352N/A                            "summary": sum_acts,
419N/A                        }
988N/A
1352N/A                self.__lock_catalog()
1352N/A                try:
1352N/A                        entries = {}
1352N/A                        # Use the same operation time and date for all
1352N/A                        # operations so that the last modification times
1352N/A                        # of all catalog parts and update logs will be
1352N/A                        # synchronized.
1352N/A                        op_time = datetime.datetime.utcnow()
1352N/A
1352N/A                        # Always add packages to the base catalog.
1352N/A                        entry = {}
1352N/A                        if metadata:
1352N/A                                entry["metadata"] = metadata
1352N/A                        if manifest:
3234N/A                                for k, v in six.iteritems(manifest.signatures):
3158N/A                                        entry["signature-{0}".format(k)] = v
1431N/A                        part = self.get_part(self.__BASE_PART)
1352N/A                        entries[part.name] = part.add(pfmri, metadata=entry,
1352N/A                            op_time=op_time)
1352N/A
1352N/A                        if manifest:
1352N/A                                # Without a manifest, only the base catalog data
1352N/A                                # can be populated.
419N/A
1352N/A                                # Only dependency and set actions are currently
1352N/A                                # used by the remaining catalog parts.
1352N/A                                actions = []
1352N/A                                for atype in "depend", "set":
1352N/A                                        actions += manifest.gen_actions_by_type(
1352N/A                                            atype)
419N/A
1352N/A                                gacts = group_actions(actions)
1352N/A                                for ctype in gacts:
1352N/A                                        for locale in gacts[ctype]:
1352N/A                                                acts = gacts[ctype][locale]
1352N/A                                                if not acts:
1352N/A                                                        # Catalog entries only
1352N/A                                                        # added if actions are
1352N/A                                                        # present for this
1352N/A                                                        # ctype.
1352N/A                                                        continue
419N/A
1369N/A                                                part = self.get_part("catalog"
3158N/A                                                    ".{0}.{1}".format(ctype,
3158N/A                                                    locale))
1352N/A                                                entry = { "actions": acts }
1352N/A                                                entries[part.name] = part.add(
1352N/A                                                    pfmri, metadata=entry,
1352N/A                                                    op_time=op_time)
419N/A
1463N/A                        self.__log_update(pfmri, CatalogUpdate.ADD, op_time,
1463N/A                            entries=entries)
1352N/A                finally:
1352N/A                        self.__unlock_catalog()
157N/A
1431N/A        def append(self, src, cb=None, pfmri=None, pubs=EmptyI):
1369N/A                """Appends the entries in the specified 'src' catalog to that
1369N/A                of the current catalog.  The caller is responsible for ensuring
1463N/A                that no duplicates exist and must call finalize() afterwards to
1463N/A                to ensure consistent catalog state.  This function cannot be
1463N/A                used when log_updates or read_only is enabled.
1369N/A
1369N/A                'cb' is an optional callback function that must accept src,
1369N/A                an FMRI, and entry.  Where 'src' is the source catalog the
1369N/A                FMRI's entry is being copied from, and entry is the source
1369N/A                catalog entry.  It must return a tuple of the form (append,
1369N/A                metadata), where 'append' is a boolean value indicating if
1369N/A                the specified package should be appended, and 'metadata' is
1369N/A                a dict of additional metadata to store with the package's
1369N/A                BASE record.
1369N/A
1369N/A                'pfmri' is an optional FMRI of a package to append.  If not
1431N/A                provided, all FMRIs in the 'src' catalog will be appended.
1431N/A                This filtering is applied before any provided callback.
1431N/A
1431N/A                'pubs' is an optional list of publisher prefixes to restrict
1431N/A                the append operation to.  FRMIs that have a publisher not in
1431N/A                the list will be skipped.  This filtering is applied before
1431N/A                any provided callback.  If not provided, no publisher
1463N/A                filtering will be applied."""
1431N/A
1431N/A                assert not self.log_updates and not self.read_only
1431N/A
1463N/A                self.__lock_catalog()
1463N/A                try:
1463N/A                        # Append operations are much slower if batch mode is
1463N/A                        # not enabled.  This ensures that the current state
1463N/A                        # is stored and then reset on completion or failure.
1463N/A                        # Since append() is never used as part of the
1463N/A                        # publication process (log_updates == True),
1463N/A                        # this is safe.
1463N/A                        old_batch_mode = self.batch_mode
1463N/A                        self.batch_mode = True
1463N/A                        self.__append(src, cb=cb, pfmri=pfmri, pubs=pubs)
1463N/A                finally:
1463N/A                        self.batch_mode = old_batch_mode
1463N/A                        self.__unlock_catalog()
1369N/A
1352N/A        def apply_updates(self, path):
1352N/A                """Apply any CatalogUpdates available to the catalog based on
1352N/A                the list returned by get_updates_needed.  The caller must
1352N/A                retrieve all of the resources indicated by get_updates_needed
1352N/A                and place them in the directory indicated by 'path'."""
220N/A
1352N/A                if not self.meta_root:
1352N/A                        raise api_errors.CatalogUpdateRequirements()
220N/A
1352N/A                # Used to store the original time each part was modified
1352N/A                # as a basis for determining whether to apply specific
1352N/A                # updates.
1352N/A                old_parts = self._attrs.parts
1352N/A                def apply_incremental(name):
1352N/A                        # Load the CatalogUpdate from the path specified.
1352N/A                        # (Which is why __get_update is not used.)
1352N/A                        ulog = CatalogUpdate(name, meta_root=path)
1352N/A                        for pfmri, op_type, op_time, metadata in ulog.updates():
3234N/A                                for pname, pdata in six.iteritems(metadata):
1369N/A                                        part = self.get_part(pname,
1352N/A                                            must_exist=True)
1537N/A                                        if part is None:
1352N/A                                                # Part doesn't exist; skip.
1352N/A                                                continue
50N/A
1352N/A                                        lm = old_parts[pname]["last-modified"]
1431N/A                                        if op_time <= lm:
1352N/A                                                # Only add updates to the part
1352N/A                                                # that occurred after the last
1352N/A                                                # time it was originally
1352N/A                                                # modified.
1431N/A                                                continue
1431N/A
1431N/A                                        if op_type == CatalogUpdate.ADD:
1352N/A                                                part.add(pfmri, metadata=pdata,
1352N/A                                                    op_time=op_time)
1431N/A                                        elif op_type == CatalogUpdate.REMOVE:
1431N/A                                                part.remove(pfmri,
1431N/A                                                    op_time=op_time)
1431N/A                                        else:
1431N/A                                                raise api_errors.UnknownUpdateType(
1431N/A                                                    op_type)
157N/A
1352N/A                def apply_full(name):
1352N/A                        src = os.path.join(path, name)
1352N/A                        dest = os.path.join(self.meta_root, name)
1352N/A                        portable.copyfile(src, dest)
1352N/A
1352N/A                self.__lock_catalog()
1352N/A                try:
1352N/A                        old_batch_mode = self.batch_mode
1352N/A                        self.batch_mode = True
157N/A
1431N/A                        updates = self.get_updates_needed(path)
1431N/A                        if updates == None:
1431N/A                                # Nothing has changed, so nothing to do.
1431N/A                                return
1431N/A
1431N/A                        for name in updates:
1352N/A                                if name.startswith("update."):
1352N/A                                        # The provided update is an incremental.
1352N/A                                        apply_incremental(name)
1352N/A                                else:
1352N/A                                        # The provided update is a full update.
1352N/A                                        apply_full(name)
1352N/A
1352N/A                        # Next, verify that all of the updated parts have a
1352N/A                        # signature that matches the new catalog.attrs file.
1352N/A                        new_attrs = CatalogAttrs(meta_root=path)
1352N/A                        new_sigs = {}
3234N/A                        for name, mdata in six.iteritems(new_attrs.parts):
1352N/A                                new_sigs[name] = {}
1352N/A                                for key in mdata:
1352N/A                                        if not key.startswith("signature-"):
1352N/A                                                continue
1352N/A                                        sig = key.split("signature-")[1]
1352N/A                                        new_sigs[name][sig] = mdata[key]
873N/A
1352N/A                        # This must be done to ensure that the catalog
1352N/A                        # signature matches that of the source.
1352N/A                        self.batch_mode = old_batch_mode
1352N/A                        self.finalize()
1352N/A
3234N/A                        for name, part in six.iteritems(self.__parts):
1352N/A                                part.validate(signatures=new_sigs[name])
1352N/A
1352N/A                        # Finally, save the catalog, and then copy the new
1352N/A                        # catalog attributes file into place and reload it.
1352N/A                        self.__save()
1352N/A                        apply_full(self._attrs.name)
873N/A
1352N/A                        self._attrs = CatalogAttrs(meta_root=self.meta_root)
1352N/A                        self.__set_perms()
1352N/A                finally:
1431N/A                        self.batch_mode = old_batch_mode
1352N/A                        self.__unlock_catalog()
873N/A
1537N/A        def categories(self, excludes=EmptyI, pubs=EmptyI):
1537N/A                """Returns a set of tuples of the form (scheme, category)
1537N/A                containing the names of all categories in use by the last
1537N/A                version of each unique package in the catalog on a per-
1537N/A                publisher basis.
1537N/A
1537N/A                'excludes' is a list of variants which will be used to
1537N/A                determine what category actions will be checked.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                acts = self.__actions([self.SUMMARY], excludes=excludes,
1537N/A                    last_version=True, pubs=pubs)
1537N/A                return set((
1537N/A                    sc
1537N/A                    for f, acts in acts
1537N/A                    for a in acts
1537N/A                    if a.has_category_info()
1537N/A                    for sc in a.parse_category_info()
1537N/A                ))
1537N/A
1352N/A        @property
1352N/A        def created(self):
1352N/A                """A UTC datetime object indicating the time the catalog was
1352N/A                created."""
1352N/A                return self._attrs.created
1352N/A
1352N/A        def destroy(self):
1352N/A                """Removes any on-disk files that exist for the catalog and
1352N/A                discards all content."""
204N/A
1352N/A                for name in self._attrs.parts:
1369N/A                        part = self.get_part(name)
1352N/A                        part.destroy()
1352N/A
1352N/A                for name in self._attrs.updates:
1369N/A                        ulog = self.__get_update(name, cache=False)
1352N/A                        ulog.destroy()
204N/A
1358N/A                self._attrs = CatalogAttrs(meta_root=self.meta_root,
3362N/A                    sign=self.__sign, file_root=self.file_root)
1352N/A                self.__parts = {}
1352N/A                self.__updates = {}
1606N/A                self._attrs.destroy()
1606N/A
1606N/A                if not self.meta_root or not os.path.exists(self.meta_root):
1606N/A                        return
1606N/A
1606N/A                # Finally, ensure that if there are any leftover files from
1606N/A                # an interrupted destroy in the past that they are removed
1606N/A                # as well.
1606N/A                for fname in os.listdir(self.meta_root):
1606N/A                        if not fname.startswith("catalog.") and \
1606N/A                            not fname.startswith("update."):
1606N/A                                continue
1606N/A
1606N/A                        pname = os.path.join(self.meta_root, fname)
1606N/A                        if not os.path.isfile(pname):
1606N/A                                continue
1606N/A
1606N/A                        try:
1606N/A                                portable.remove(pname)
3171N/A                        except EnvironmentError as e:
1606N/A                                if e.errno == errno.EACCES:
1606N/A                                        raise api_errors.PermissionsException(
1606N/A                                            e.filename)
1606N/A                                if e.errno == errno.EROFS:
1606N/A                                        raise api_errors.ReadOnlyFileSystemException(
1606N/A                                            e.filename)
1606N/A                                raise
1352N/A
1537N/A        def entries(self, info_needed=EmptyI, last=False, locales=None,
1537N/A            ordered=False, pubs=EmptyI):
1352N/A                """A generator function that produces tuples of the format
1352N/A                (fmri, metadata) as it iterates over the contents of the
1352N/A                catalog (where 'metadata' is a dict containing the requested
1352N/A                information).
1352N/A
1352N/A                'metadata' always contains the following information at a
1352N/A                 minimum:
220N/A
1352N/A                        BASE
1352N/A                                'metadata' will be populated with Manifest
1352N/A                                signature data, if available, using key-value
1352N/A                                pairs of the form 'signature-<name>': value.
220N/A
1431N/A                'info_needed' is an optional list of one or more catalog
1352N/A                constants indicating the types of catalog data that will
1352N/A                be returned in 'metadata' in addition to the above:
220N/A
1352N/A                        DEPENDENCY
1352N/A                                'metadata' will contain depend and set Actions
1352N/A                                for package obsoletion, renaming, variants,
1352N/A                                and facets stored in a list under the
1352N/A                                key 'actions'.
220N/A
1352N/A                        SUMMARY
1352N/A                                'metadata' will contain any remaining Actions
1352N/A                                not listed above, such as pkg.summary,
1352N/A                                pkg.description, etc. in a list under the key
1352N/A                                'actions'.
1352N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the catalog has been saved since the last modifying
1537N/A                operation, or finalize() has has been called, this will also be
1537N/A                the newest version of the package.
1537N/A
1352N/A                'locales' is an optional set of locale names for which Actions
1352N/A                should be returned.  The default is set(('C',)) if not provided.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                return self.__entries(info_needed=info_needed,
1537N/A                    last_version=last, locales=locales, ordered=ordered,
1537N/A                    pubs=pubs)
1537N/A
1537N/A        def entries_by_version(self, name, info_needed=EmptyI, locales=None,
1537N/A            pubs=EmptyI):
1352N/A                """A generator function that produces tuples of the format
1352N/A                (version, entries) as it iterates over the contents of the
1352N/A                the catalog, where entries is a list of tuples of the format
1352N/A                (fmri, metadata) and metadata is a dict containing the
1352N/A                requested information.
1352N/A
1352N/A                'metadata' always contains the following information at a
1352N/A                 minimum:
1352N/A
1352N/A                        BASE
1352N/A                                'metadata' will be populated with Manifest
1352N/A                                signature data, if available, using key-value
1352N/A                                pairs of the form 'signature-<name>': value.
1352N/A
1431N/A                'info_needed' is an optional list of one or more catalog
1352N/A                constants indicating the types of catalog data that will
1352N/A                be returned in 'metadata' in addition to the above:
1352N/A
1352N/A                        DEPENDENCY
1352N/A                                'metadata' will contain depend and set Actions
1352N/A                                for package obsoletion, renaming, variants,
1352N/A                                and facets stored in a list under the
1352N/A                                key 'actions'.
1352N/A
1352N/A                        SUMMARY
1352N/A                                'metadata' will contain any remaining Actions
1352N/A                                not listed above, such as pkg.summary,
1352N/A                                pkg.description, etc. in a list under the key
1352N/A                                'actions'.
1352N/A
1352N/A                'locales' is an optional set of locale names for which Actions
1352N/A                should be returned.  The default is set(('C',)) if not provided.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1352N/A                        # Catalog contains nothing.
220N/A                        return
220N/A
1352N/A                if not locales:
1352N/A                        locales = set(("C",))
1431N/A                else:
1431N/A                        locales = set(locales)
1352N/A
1352N/A                parts = []
1352N/A                if self.DEPENDENCY in info_needed:
1431N/A                        part = self.get_part(self.__DEPS_PART, must_exist=True)
1537N/A                        if part is not None:
1352N/A                                parts.append(part)
1352N/A
1352N/A                if self.SUMMARY in info_needed:
1352N/A                        for locale in locales:
1369N/A                                part = self.get_part(
3158N/A                                    "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A                                    locale), must_exist=True)
1537N/A                                if part is None:
1352N/A                                        # Data not available for this
1352N/A                                        # locale.
1352N/A                                        continue
1352N/A                                parts.append(part)
220N/A
1352N/A                def merge_entry(src, dest):
3234N/A                        for k, v in six.iteritems(src):
1352N/A                                if k == "actions":
1369N/A                                        dest.setdefault(k, [])
1352N/A                                        dest[k] += v
1352N/A                                elif k != "version":
1352N/A                                        dest[k] = v
1352N/A
1537N/A                for ver, entries in base.entries_by_version(name, pubs=pubs):
1352N/A                        nentries = []
1352N/A                        for f, bentry in entries:
1369N/A                                mdata = {}
1352N/A                                merge_entry(bentry, mdata)
1537N/A                                for part in parts:
1537N/A                                        entry = part.get_entry(f)
1537N/A                                        if entry is None:
1537N/A                                                # Part doesn't have this FMRI,
1537N/A                                                # so skip it.
1537N/A                                                continue
1537N/A                                        merge_entry(entry, mdata)
1352N/A                                nentries.append((f, mdata))
1352N/A                        yield ver, nentries
220N/A
1537N/A        def entry_actions(self, info_needed, excludes=EmptyI, cb=None,
1537N/A            last=False, locales=None, ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces tuples of the format
1537N/A                ((pub, stem, version), entry, actions) as it iterates over
1537N/A                the contents of the catalog (where 'actions' is a generator
1537N/A                that returns the Actions corresponding to the requested
1537N/A                information).
1537N/A
1537N/A                If the catalog doesn't contain any action data for the package
3329N/A                entry, it will return an empty iterator.
1537N/A
1537N/A                'excludes' is a list of variants which will be used to determine
1537N/A                what should be allowed by the actions generator in addition to
1537N/A                what is specified by 'info_needed'.
1537N/A
1537N/A                'cb' is an optional callback function that will be executed for
1537N/A                each package before its action data is retrieved. It must accept
1537N/A                two arguments: 'pkg' and 'entry'.  'pkg' is an FMRI object and
1537N/A                'entry' is the dictionary structure of the catalog entry for the
1537N/A                package.  If the callback returns False, then the entry will not
1537N/A                be included in the results.  This can significantly improve
1537N/A                performance by avoiding action data retrieval for results that
1537N/A                will not be used.
1537N/A
1537N/A                'info_needed' is a set of one or more catalog constants
1537N/A                indicating the types of catalog data that will be returned
1537N/A                in 'actions' in addition to the above:
1537N/A
1537N/A                        DEPENDENCY
1537N/A                                Depend and set Actions for package obsoletion,
1537N/A                                renaming, variants.
1537N/A
1537N/A                        SUMMARY
1537N/A                                Any remaining set Actions not listed above, such
1537N/A                                as pkg.summary, pkg.description, etc.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the catalog has been saved since the last modifying
1537N/A                operation, or finalize() has has been called, this will also be
1537N/A                the newest version of the package.
1537N/A
1537N/A                'locales' is an optional set of locale names for which Actions
1537N/A                should be returned.  The default is set(('C',)) if not provided.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pfmri' is an optional FMRI to limit the returned results to.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                for r, entry in self.__entries(cb=cb, info_needed=info_needed,
1537N/A                    locales=locales, last_version=last, ordered=ordered,
1537N/A                    pubs=pubs, tuples=True):
2639N/A                        try:
1537N/A                                yield (r, entry,
1970N/A                                    self.__gen_actions(r, entry["actions"],
1537N/A                                    excludes))
2639N/A                        except KeyError:
3329N/A                                yield r, entry, EmptyI
1537N/A
1352N/A        @property
1352N/A        def exists(self):
1352N/A                """A boolean value indicating whether the Catalog exists
1352N/A                on-disk."""
1352N/A
1352N/A                # If the Catalog attrs file exists on-disk,
1352N/A                # then the catalog does.
1352N/A                attrs = self._attrs
1352N/A                return attrs.exists
1352N/A
1463N/A        def finalize(self, pfmris=None, pubs=None):
1352N/A                """This function re-sorts the contents of the Catalog so that
1352N/A                version entries are in the correct order and sets the package
1463N/A                counts for the Catalog based on its current contents.
1463N/A
1463N/A                'pfmris' is an optional set of FMRIs that indicate what package
1463N/A                entries have been changed since this function was last called.
1463N/A                It is used to optimize the finalization process.
1463N/A
1463N/A                'pubs' is an optional set of publisher prefixes that indicate
1463N/A                what publisher has had package entries changed.  It is used
1463N/A                to optimize the finalization process.  This option has no effect
1463N/A                if 'pfmris' is also provided."""
1463N/A
1463N/A                return self.__finalize(pfmris=pfmris, pubs=pubs)
1352N/A
1537N/A        def fmris(self, last=False, objects=True, ordered=False, pubs=EmptyI):
1352N/A                """A generator function that produces FMRIs as it iterates
1352N/A                over the contents of the catalog.
1352N/A
1537N/A                'last' is a boolean value that indicates only the last FMRI
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the catalog has been saved since the last modifying
1537N/A                operation, or finalize() has has been called, this will also be
1537N/A                the newest version of the package.
1537N/A
1352N/A                'objects' is an optional boolean value indicating whether
1537N/A                FMRIs should be returned as FMRI objects or as strings.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
220N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1352N/A                        # Catalog contains nothing.
1352N/A
1352N/A                        # This construction is necessary to get python to
1352N/A                        # return no results properly to callers expecting
1352N/A                        # a generator function.
1352N/A                        return iter(())
1537N/A                return base.fmris(last=last, objects=objects, ordered=ordered,
1537N/A                    pubs=pubs)
1537N/A
1537N/A        def fmris_by_version(self, name, pubs=EmptyI):
1352N/A                """A generator function that produces tuples of (version,
1352N/A                fmris), where fmris is a of the fmris related to the
1537N/A                version, for the given package name.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1352N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1352N/A                        # Catalog contains nothing.
1352N/A
1352N/A                        # This construction is necessary to get python to
1352N/A                        # return no results properly to callers expecting
1352N/A                        # a generator function.
1352N/A                        return iter(())
1537N/A                return base.fmris_by_version(name, pubs=pubs)
1352N/A
1431N/A        def get_entry(self, pfmri, info_needed=EmptyI, locales=None):
1352N/A                """Returns a dict containing the metadata for the specified
1537N/A                FMRI containing the requested information.  If the specified
1537N/A                FMRI does not exist in the catalog, a value of None will be
1537N/A                returned.
1352N/A
1352N/A                'metadata' always contains the following information at a
1352N/A                 minimum:
1352N/A
1352N/A                        BASE
1352N/A                                'metadata' will be populated with Manifest
1352N/A                                signature data, if available, using key-value
1352N/A                                pairs of the form 'signature-<name>': value.
1352N/A
1431N/A                'info_needed' is an optional list of one or more catalog
1352N/A                constants indicating the types of catalog data that will
1352N/A                be returned in 'metadata' in addition to the above:
204N/A
1352N/A                        DEPENDENCY
1352N/A                                'metadata' will contain depend and set Actions
1352N/A                                for package obsoletion, renaming, variants,
1352N/A                                and facets stored in a list under the
1352N/A                                key 'actions'.
1352N/A
1352N/A                        SUMMARY
1352N/A                                'metadata' will contain any remaining Actions
1352N/A                                not listed above, such as pkg.summary,
1352N/A                                pkg.description, etc. in a list under the key
1352N/A                                'actions'.
1352N/A
1352N/A                'locales' is an optional set of locale names for which Actions
1352N/A                should be returned.  The default is set(('C',)) if not provided.
1352N/A                """
1352N/A
1352N/A                def merge_entry(src, dest):
3234N/A                        for k, v in six.iteritems(src):
1352N/A                                if k == "actions":
1369N/A                                        dest.setdefault(k, [])
1352N/A                                        dest[k] += v
1352N/A                                elif k != "version":
1352N/A                                        dest[k] = v
157N/A
1352N/A                parts = []
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1537N/A                        return
1352N/A
1352N/A                if not locales:
1352N/A                        locales = set(("C",))
1431N/A                else:
1431N/A                        locales = set(locales)
1352N/A
1352N/A                # Always attempt to retrieve the BASE entry as FMRIs
1352N/A                # must be present in the BASE catalog part.
1369N/A                mdata = {}
1352N/A                bentry = base.get_entry(pfmri)
1537N/A                if bentry is None:
1537N/A                        return
1352N/A                merge_entry(bentry, mdata)
204N/A
1352N/A                if self.DEPENDENCY in info_needed:
1431N/A                        part = self.get_part(self.__DEPS_PART,
1352N/A                            must_exist=True)
1537N/A                        if part is not None:
1352N/A                                parts.append(part)
204N/A
1352N/A                if self.SUMMARY in info_needed:
1352N/A                        for locale in locales:
1369N/A                                part = self.get_part(
3158N/A                                    "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A                                    locale), must_exist=True)
1537N/A                                if part is None:
1352N/A                                        # Data not available for this
1352N/A                                        # locale.
1352N/A                                        continue
1352N/A                                parts.append(part)
270N/A
1537N/A                for part in parts:
1537N/A                        entry = part.get_entry(pfmri)
1537N/A                        if entry is None:
1537N/A                                # Part doesn't have this FMRI,
1537N/A                                # so skip it.
1537N/A                                continue
1537N/A                        merge_entry(entry, mdata)
1352N/A                return mdata
1352N/A
1369N/A        def get_entry_actions(self, pfmri, info_needed, excludes=EmptyI,
1369N/A            locales=None):
1369N/A                """A generator function that produces Actions as it iterates
1369N/A                over the catalog entry of the specified FMRI corresponding to
1369N/A                the requested information).  If the catalog doesn't contain
3329N/A                any action data for the package entry, it will return an empty
3329N/A                iterator.
1369N/A
1369N/A                'excludes' is a list of variants which will be used to determine
1369N/A                what should be allowed by the actions generator in addition to
1369N/A                what is specified by 'info_needed'.  If not provided, only
1369N/A                'info_needed' will determine what actions are returned.
1369N/A
1369N/A                'info_needed' is a set of one or more catalog constants
1369N/A                indicating the types of catalog data that will be returned
1369N/A                in 'actions' in addition to the above:
1369N/A
1369N/A                        DEPENDENCY
1369N/A                                Depend and set Actions for package obsoletion,
1369N/A                                renaming, variants.
1369N/A
1369N/A                        SUMMARY
1369N/A                                Any remaining set Actions not listed above, such
1369N/A                                as pkg.summary, pkg.description, etc.
1369N/A
1369N/A                'locales' is an optional set of locale names for which Actions
1369N/A                should be returned.  The default is set(('C',)) if not provided.
1369N/A                """
1369N/A
1369N/A                assert info_needed
1369N/A                if not locales:
1369N/A                        locales = set(("C",))
1431N/A                else:
1431N/A                        locales = set(locales)
1369N/A
1369N/A                entry = self.get_entry(pfmri, info_needed=info_needed,
1369N/A                    locales=locales)
1537N/A                if entry is None:
1537N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
2639N/A                try:
1970N/A                        return self.__gen_actions(pfmri, entry["actions"],
1970N/A                            excludes)
2639N/A                except KeyError:
3329N/A                        return EmptyI
1369N/A
1369N/A        def get_entry_all_variants(self, pfmri):
1369N/A                """A generator function that yields tuples of the format
1369N/A                (var_name, variants); where var_name is the name of the
1369N/A                variant and variants is a list of the variants for that
1369N/A                name."""
1369N/A
1369N/A                info_needed = [self.DEPENDENCY]
1369N/A                entry = self.get_entry(pfmri, info_needed=info_needed)
1537N/A                if entry is None:
1537N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
2639N/A                try:
1970N/A                        actions = self.__gen_actions(pfmri, entry["actions"])
2639N/A                except KeyError:
3329N/A                        return
1369N/A
1369N/A                for a in actions:
1369N/A                        if a.name != "set":
1369N/A                                continue
1369N/A
1369N/A                        attr_name = a.attrs["name"]
1369N/A                        if not attr_name.startswith("variant"):
1369N/A                                continue
1369N/A                        yield attr_name, a.attrs["value"]
1369N/A
1431N/A        def get_entry_signatures(self, pfmri):
1431N/A                """A generator function that yields tuples of the form (sig,
1431N/A                value) where 'sig' is the name of the signature, and 'value' is
1431N/A                the raw catalog value for the signature.  Please note that the
1431N/A                data type of 'value' is dependent on the signature, so it may
1431N/A                be a string, list, dict, etc."""
1431N/A
1431N/A                entry = self.get_entry(pfmri)
1537N/A                if entry is None:
1537N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1431N/A                return (
1431N/A                    (k.split("signature-")[1], v)
3234N/A                    for k, v in six.iteritems(entry)
1431N/A                    if k.startswith("signature-")
1431N/A                )
1431N/A
1369N/A        def get_entry_variants(self, pfmri, name):
1369N/A                """A generator function that returns the variants for the
1369N/A                specified variant name.  If no variants exist for the
1369N/A                specified name, None will be returned."""
1369N/A
1369N/A                for var_name, values in self.get_entry_all_variants(pfmri):
1369N/A                        if var_name == name:
1369N/A                                # A package can only have one set of values
1369N/A                                # for a single variant name, so return it.
1369N/A                                return values
1369N/A                return None
1369N/A
2510N/A        def gen_packages(self, collect_attrs=False, matched=None,
2510N/A            patterns=EmptyI, pubs=EmptyI, unmatched=None, return_fmris=False):
2510N/A                """A generator function that produces tuples of the form:
2510N/A
2510N/A                    (
2510N/A                        (
2510N/A                            pub,    - (string) the publisher of the package
2510N/A                            stem,   - (string) the name of the package
2510N/A                            version - (string) the version of the package
2510N/A                        ),
2510N/A                        states,     - (list) states
2510N/A                        attributes  - (dict) package attributes
2510N/A                    )
2510N/A
2510N/A                Results are always sorted by stem, publisher, and then in
2510N/A                descending version order.
2510N/A
2510N/A                'collect_attrs' is an optional boolean that indicates whether
2510N/A                all package attributes should be collected and returned in the
2510N/A                fifth element of the return tuple.  If False, that element will
2510N/A                be an empty dictionary.
2510N/A
2510N/A                'matched' is an optional set to add matched patterns to.
2510N/A
2510N/A                'patterns' is an optional list of FMRI wildcard strings to
2510N/A                filter results by.
2510N/A
2510N/A                'pubs' is an optional list of publisher prefixes to restrict
2510N/A                the results to.
2510N/A
2510N/A                'unmatched' is an optional set to add unmatched patterns to.
2510N/A
2510N/A                'return_fmris' is an optional boolean value that indicates that
2510N/A                an FMRI object should be returned in place of the (pub, stem,
2510N/A                ver) tuple that is normally returned."""
2510N/A
2510N/A                # Each pattern in patterns can be a partial or full FMRI, so
2510N/A                # extract the individual components for use in filtering.
2510N/A                newest = False
2510N/A                illegals = []
2510N/A                pat_tuples = {}
2510N/A                latest_pats = set()
2510N/A                seen = set()
2510N/A                npatterns = set()
2510N/A                for pat, error, pfmri, matcher in self.__parse_fmri_patterns(
2510N/A                    patterns):
2510N/A                        if error:
2510N/A                                illegals.append(error)
2510N/A                                continue
2510N/A
2510N/A                        # Duplicate patterns are ignored.
2510N/A                        sfmri = str(pfmri)
2510N/A                        if sfmri in seen:
2510N/A                                # A different form of the same pattern
2510N/A                                # was specified already; ignore this
2510N/A                                # one (e.g. pkg:/network/ping,
2510N/A                                # /network/ping).
2510N/A                                continue
2510N/A
2510N/A                        # Track used patterns.
2510N/A                        seen.add(sfmri)
2510N/A                        npatterns.add(pat)
2510N/A
2510N/A                        if getattr(pfmri.version, "match_latest", None):
2510N/A                                latest_pats.add(pat)
2510N/A                        pat_tuples[pat] = (pfmri.tuple(), matcher)
2510N/A
2510N/A                patterns = npatterns
2510N/A                del npatterns, seen
2510N/A
2510N/A                if illegals:
2510N/A                        raise api_errors.PackageMatchErrors(illegal=illegals)
2510N/A
2510N/A                # Keep track of listed stems for all other packages on a
2510N/A                # per-publisher basis.
2510N/A                nlist = collections.defaultdict(int)
2510N/A
2510N/A                # Track matching patterns.
2510N/A                matched_pats = set()
2510N/A                pkg_matching_pats = None
2510N/A
2510N/A                # Need dependency and summary actions.
2510N/A                cat_info = frozenset([self.DEPENDENCY, self.SUMMARY])
2510N/A
2510N/A                for t, entry, actions in self.entry_actions(cat_info,
2510N/A                    ordered=True, pubs=pubs):
2510N/A                        pub, stem, ver = t
2510N/A
2510N/A                        omit_ver = False
2510N/A                        omit_package = None
2510N/A
2510N/A                        pkg_stem = "!".join((pub, stem))
2510N/A                        if newest and pkg_stem in nlist:
2510N/A                                # A newer version has already been listed, so
2510N/A                                # any additional entries need to be marked for
2510N/A                                # omission before continuing.
2510N/A                                omit_package = True
2510N/A                        else:
2510N/A                                nlist[pkg_stem] += 1
2510N/A
2510N/A                        if matched is not None or unmatched is not None:
2510N/A                                pkg_matching_pats = set()
2510N/A                        if not omit_package:
2510N/A                                ever = None
2510N/A                                for pat in patterns:
2510N/A                                        (pat_pub, pat_stem, pat_ver), matcher = \
2510N/A                                            pat_tuples[pat]
2510N/A
2510N/A                                        if pat_pub is not None and \
2510N/A                                            pub != pat_pub:
2510N/A                                                # Publisher doesn't match.
2510N/A                                                if omit_package is None:
2510N/A                                                        omit_package = True
2510N/A                                                continue
2510N/A
2510N/A                                        if matcher == fmri.exact_name_match:
2510N/A                                                if pat_stem != stem:
2510N/A                                                        # Stem doesn't match.
2510N/A                                                        if omit_package is None:
2510N/A                                                                omit_package = \
2510N/A                                                                    True
2510N/A                                                        continue
2510N/A                                        elif matcher == fmri.fmri_match:
2510N/A                                                if not ("/" + stem).endswith(
2510N/A                                                    "/" + pat_stem):
2510N/A                                                        # Stem doesn't match.
2510N/A                                                        if omit_package is None:
2510N/A                                                                omit_package = \
2510N/A                                                                    True
2510N/A                                                        continue
2510N/A                                        elif matcher == fmri.glob_match:
2510N/A                                                if not fnmatch.fnmatchcase(stem,
2510N/A                                                    pat_stem):
2510N/A                                                        # Stem doesn't match.
2510N/A                                                        if omit_package is None:
2510N/A                                                                omit_package = \
2510N/A                                                                    True
2510N/A                                                        continue
2510N/A
2510N/A                                        if pat_ver is not None:
2510N/A                                                if ever is None:
2510N/A                                                        # Avoid constructing a
2510N/A                                                        # version object more
2510N/A                                                        # than once for each
2510N/A                                                        # entry.
2958N/A                                                        ever = pkg.version.Version(ver)
2510N/A                                                if not ever.is_successor(pat_ver,
2510N/A                                                    pkg.version.CONSTRAINT_AUTO):
2510N/A                                                        if omit_package is None:
2510N/A                                                                omit_package = \
2510N/A                                                                    True
2510N/A                                                        omit_ver = True
2510N/A                                                        continue
2510N/A
2510N/A                                        if pat in latest_pats and \
2510N/A                                            nlist[pkg_stem] > 1:
2510N/A                                                # Package allowed by pattern,
2510N/A                                                # but isn't the "latest"
2510N/A                                                # version.
2510N/A                                                if omit_package is None:
2510N/A                                                        omit_package = True
2510N/A                                                omit_ver = True
2510N/A                                                continue
2510N/A
2510N/A                                        # If this entry matched at least one
2510N/A                                        # pattern, then ensure it is returned.
2510N/A                                        omit_package = False
2510N/A                                        if (matched is None and
2510N/A                                            unmatched is None):
2510N/A                                                # It's faster to stop as soon
2510N/A                                                # as a match is found.
2510N/A                                                break
2510N/A
2510N/A                                        # If caller has requested other match
2510N/A                                        # cases be returned, then all patterns
2510N/A                                        # must be tested for every entry.  This
2510N/A                                        # is slower, so only done if necessary.
2510N/A                                        pkg_matching_pats.add(pat)
2510N/A
2510N/A                        if omit_package:
2510N/A                                # Package didn't match critera; skip it.
2510N/A                                continue
2510N/A
2510N/A                        # Collect attribute data if requested.
2510N/A                        summ = None
2510N/A
2510N/A                        omit_var = False
2510N/A                        states = set()
2510N/A                        if collect_attrs:
3339N/A                                # use OrderedDict to get a deterministic output
3339N/A                                ddm = lambda: OrderedDict(
3339N/A                                    collections.defaultdict(list))
2510N/A                                attrs = collections.defaultdict(ddm)
2510N/A                        else:
2510N/A                                attrs = EmptyDict
2510N/A
2510N/A                        try:
2510N/A                                for a in actions:
2510N/A                                        if a.name != "set":
2510N/A                                                continue
2510N/A
2510N/A                                        atname = a.attrs["name"]
2510N/A                                        atvalue = a.attrs["value"]
2510N/A                                        if collect_attrs:
2510N/A                                                atvlist = a.attrlist("value")
2510N/A
2510N/A                                                # XXX Need to describe this data
2510N/A                                                # structure sanely somewhere.
2510N/A                                                mods = frozenset(
2510N/A                                                    (k, frozenset(a.attrlist(k)))
3234N/A                                                    for k in six.iterkeys(a.attrs)
2510N/A                                                    if k not in ("name", "value")
2510N/A                                                )
3339N/A                                                if mods not in attrs[atname]:
3339N/A                                                        attrs[atname][mods] = atvlist
3339N/A                                                else:
3339N/A                                                        attrs[atname][mods].extend(
3339N/A                                                            atvlist)
2510N/A
2510N/A                                        if atname == "pkg.summary":
2510N/A                                                summ = atvalue
2510N/A                                                continue
2510N/A
2510N/A                                        if atname == "description":
2510N/A                                                if summ is None:
2510N/A                                                        # Historical summary
2510N/A                                                        # field.
2510N/A                                                        summ = atvalue
2510N/A                                                        collect_attrs and \
2510N/A                                                            attrs["pkg.summary"] \
2510N/A                                                            [mods]. \
2510N/A                                                            extend(atvlist)
2510N/A                                                continue
2510N/A
2510N/A                                        if atname == "pkg.renamed":
2510N/A                                                if atvalue == "true":
2510N/A                                                        states.add(
2616N/A                                                            pkgdefs.PKG_STATE_RENAMED)
2510N/A                                                continue
2510N/A                                        if atname == "pkg.obsolete":
2510N/A                                                if atvalue == "true":
2510N/A                                                        states.add(
2616N/A                                                            pkgdefs.PKG_STATE_OBSOLETE)
2510N/A                                                continue
2510N/A                        except api_errors.InvalidPackageErrors:
2510N/A                                # Ignore errors for packages that have invalid
2510N/A                                # or unsupported metadata.
2616N/A                                states.add(pkgdefs.PKG_STATE_UNSUPPORTED)
2510N/A
2510N/A                        if omit_package:
2510N/A                                # Package didn't match criteria; skip it.
2510N/A                                if omit_ver and nlist[pkg_stem] == 1:
2510N/A                                        del nlist[pkg_stem]
2510N/A                                continue
2510N/A
2510N/A                        if matched is not None or unmatched is not None:
2510N/A                                # Only after all other filtering has been
2510N/A                                # applied are the patterns that the package
2510N/A                                # matched considered "matching".
2510N/A                                matched_pats.update(pkg_matching_pats)
2510N/A
2510N/A                        # Return the requested package data.
2510N/A                        if return_fmris:
2958N/A                                pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2958N/A                                    version=ver)
2510N/A                                yield (pfmri, states, attrs)
2510N/A                        else:
2510N/A                                yield (t, states, attrs)
2510N/A
2510N/A                if matched is not None:
2510N/A                        # Caller has requested that matched patterns be
2510N/A                        # returned.
2510N/A                        matched.update(matched_pats)
2510N/A                if unmatched is not None:
2510N/A                        # Caller has requested that unmatched patterns be
2510N/A                        # returned.
2510N/A                        unmatched.update(set(pat_tuples.keys()) - matched_pats)
2510N/A
2591N/A        def get_matching_fmris(self, patterns):
2301N/A                """Given a user-specified list of FMRI pattern strings, return
2301N/A                a tuple of ('matching', 'references', 'unmatched'), where
2301N/A                matching is a dict of matching fmris, references is a dict of
2301N/A                the patterns indexed by matching FMRI, and unmatched is a set of
2301N/A                the patterns that did not match any FMRIs respectively:
2301N/A
2301N/A                {
2301N/A                 pkgname: [fmri1, fmri2, ...],
2301N/A                 pkgname: [fmri1, fmri2, ...],
2301N/A                 ...
2301N/A                }
2301N/A
2301N/A                {
2301N/A                 fmri1: [pat1, pat2, ...],
2301N/A                 fmri2: [pat1, pat2, ...],
2301N/A                 ...
2301N/A                }
2301N/A
2301N/A                set(['unmatched1', 'unmatchedN'])
2301N/A
2301N/A                'patterns' is the list of package patterns to match.
2301N/A
2301N/A                Constraint used is always AUTO as per expected UI behavior when
2301N/A                determining successor versions.
2301N/A
2301N/A                Note that patterns starting w/ pkg:/ require an exact match;
2301N/A                patterns containing '*' will using fnmatch rules; the default
2301N/A                trailing match rules are used for remaining patterns.
2301N/A
2301N/A                Exactly duplicated patterns are ignored.
2301N/A
2301N/A                Routine raises PackageMatchErrors if errors occur: it is
2301N/A                illegal to specify multiple different patterns that match the
2301N/A                same package name.  Only patterns that contain wildcards are
2301N/A                allowed to match multiple packages.
2301N/A                """
2301N/A
2301N/A                # problems we check for
2301N/A                illegals = []
2301N/A                unmatched = set()
2301N/A                multimatch = []
2301N/A                multispec = []
2301N/A                pat_data = []
2301N/A                wildcard_patterns = set()
2301N/A
2510N/A                # Each pattern in patterns can be a partial or full FMRI, so
2510N/A                # extract the individual components for use in filtering.
2301N/A                latest_pats = set()
2510N/A                seen = set()
2510N/A                npatterns = set()
2510N/A                for pat, error, pfmri, matcher in self.__parse_fmri_patterns(
2510N/A                    patterns):
2510N/A                        if error:
2510N/A                                illegals.append(error)
2510N/A                                continue
2510N/A
2510N/A                        # Duplicate patterns are ignored.
2510N/A                        sfmri = str(pfmri)
2510N/A                        if sfmri in seen:
2510N/A                                # A different form of the same pattern
2510N/A                                # was specified already; ignore this
2510N/A                                # one (e.g. pkg:/network/ping,
2510N/A                                # /network/ping).
2510N/A                                continue
2510N/A
2510N/A                        # Track used patterns.
2510N/A                        seen.add(sfmri)
2510N/A                        npatterns.add(pat)
2510N/A                        if "*" in pfmri.pkg_name or "?" in pfmri.pkg_name:
2510N/A                                wildcard_patterns.add(pat)
2510N/A
2510N/A                        if getattr(pfmri.version, "match_latest", None):
2510N/A                                latest_pats.add(pat)
2762N/A                        pat_data.append((pat, matcher, pfmri))
2510N/A
2510N/A                patterns = npatterns
2510N/A                del npatterns, seen
2301N/A
2301N/A                if illegals:
2301N/A                        raise api_errors.PackageMatchErrors(illegal=illegals)
2301N/A
2301N/A                # Create a dictionary of patterns, with each value being a
2301N/A                # dictionary of pkg names & fmris that match that pattern.
2301N/A                ret = dict(zip(patterns, [dict() for i in patterns]))
2301N/A
2301N/A                for name in self.names():
2762N/A                        for pat, matcher, pfmri in pat_data:
2301N/A                                pub = pfmri.publisher
2301N/A                                version = pfmri.version
2301N/A                                if not matcher(name, pfmri.pkg_name):
2301N/A                                        continue # name doesn't match
2762N/A                                for ver, entries in \
2762N/A                                    self.entries_by_version(name):
2301N/A                                        if version and not ver.is_successor(
2762N/A                                            version,
2762N/A                                            pkg.version.CONSTRAINT_AUTO):
2301N/A                                                continue # version doesn't match
2301N/A                                        for f, metadata in entries:
2301N/A                                                fpub = f.publisher
2301N/A                                                if pub and pub != fpub:
2301N/A                                                        # specified pubs
2301N/A                                                        # conflict
2762N/A                                                        continue
2301N/A                                                ret[pat].setdefault(f.pkg_name,
2301N/A                                                    []).append(f)
2301N/A
2301N/A                # Discard all but the newest version of each match.
2301N/A                if latest_pats:
2301N/A                        # Rebuild ret based on latest version of every package.
2301N/A                        latest = {}
2301N/A                        nret = {}
2301N/A                        for p in patterns:
2301N/A                                if p not in latest_pats or not ret[p]:
2301N/A                                        nret[p] = ret[p]
2301N/A                                        continue
2301N/A
2301N/A                                nret[p] = {}
2301N/A                                for pkg_name in ret[p]:
2301N/A                                        nret[p].setdefault(pkg_name, [])
2301N/A                                        for f in ret[p][pkg_name]:
2301N/A                                                nver = latest.get(f.pkg_name,
2301N/A                                                    None)
2301N/A                                                if nver > f.version:
2301N/A                                                        # Not the newest.
2301N/A                                                        continue
2301N/A                                                if nver == f.version:
2301N/A                                                        # Allow for multiple
2301N/A                                                        # FMRIs of the same
2301N/A                                                        # latest version.
2301N/A                                                        nret[p][pkg_name].append(
2301N/A                                                            f)
2301N/A                                                        continue
2301N/A
2301N/A                                                latest[f.pkg_name] = f.version
2301N/A                                                nret[p][pkg_name] = [f]
2301N/A
2301N/A                        # Assign new version of ret and discard latest list.
2301N/A                        ret = nret
2301N/A                        del latest
2301N/A
2301N/A                # Determine match failures.
2301N/A                matchdict = {}
2301N/A                for p in patterns:
2301N/A                        l = len(ret[p])
2301N/A                        if l == 0: # no matches at all
2301N/A                                unmatched.add(p)
2301N/A                        elif l > 1 and p not in wildcard_patterns:
2301N/A                                # multiple matches
2301N/A                                multimatch.append((p, [
2301N/A                                    ret[p][n][0].get_pkg_stem()
2301N/A                                    for n in ret[p]
2301N/A                                ]))
2301N/A                        else:
2301N/A                                # single match or wildcard
2301N/A                                for k in ret[p].keys():
2301N/A                                        # for each matching package name
2301N/A                                        matchdict.setdefault(k, []).append(p)
2301N/A
2591N/A                if multimatch:
2301N/A                        raise api_errors.PackageMatchErrors(
2591N/A                            multiple_matches=multimatch)
2591N/A
2591N/A                # Group the matching patterns by package name and allow multiple
2591N/A                # fmri matches.
2301N/A                proposed_dict = {}
2301N/A                for d in ret.values():
3234N/A                        for k, l in six.iteritems(d):
2591N/A                                proposed_dict.setdefault(k, []).extend(l)
2301N/A
2301N/A                # construct references so that we can know which pattern
2301N/A                # generated which fmris...
2301N/A                references = dict([
2301N/A                    (f, p)
2301N/A                    for p in ret.keys()
2301N/A                    for flist in ret[p].values()
2301N/A                    for f in flist
2301N/A                ])
2301N/A
2301N/A                return proposed_dict, references, unmatched
2301N/A
3110N/A        def get_package_counts_by_pub(self, pubs=EmptyI):
1968N/A                """Returns a generator of tuples of the form (pub,
1968N/A                package_count, package_version_count).  'pub' is the publisher
1968N/A                prefix, 'package_count' is the number of unique packages for the
1968N/A                publisher, and 'package_version_count' is the number of unique
1968N/A                package versions for the publisher.
1968N/A                """
1968N/A
1968N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1968N/A                if base is None:
1968N/A                        # Catalog contains nothing.
1968N/A
1968N/A                        # This construction is necessary to get python to
1968N/A                        # return no results properly to callers expecting
1968N/A                        # a generator function.
1968N/A                        return iter(())
3110N/A                return base.get_package_counts_by_pub(pubs=pubs)
1968N/A
1369N/A        def get_part(self, name, must_exist=False):
1369N/A                """Returns the CatalogPart object for the named catalog part.
1369N/A
1369N/A                'must_exist' is an optional boolean value that indicates that
1369N/A                the catalog part must already exist in-memory or on-disk, if
1369N/A                not a value of None will be returned."""
1369N/A
1369N/A                # First, check if the part has already been cached, and if so,
1369N/A                # return it.
1537N/A                part = self.__parts.get(name, None)
1537N/A                if part is not None:
1537N/A                        return part
1537N/A                elif not self.meta_root and must_exist:
1537N/A                        return
1369N/A
1549N/A                # If the caller said the part must_exist, then it must already
1549N/A                # be part of the catalog attributes to be valid.
1549N/A                aparts = self._attrs.parts
1549N/A                if must_exist and name not in aparts:
1549N/A                        return
1549N/A
1549N/A                # Next, since the part hasn't been cached, create an object
1549N/A                # for it and add it to catalog attributes.
1369N/A                part = CatalogPart(name, meta_root=self.meta_root,
3362N/A                    ordered=not self.__batch_mode, sign=self.__sign,
3362N/A                    file_root=self.file_root)
1549N/A                if must_exist and self.meta_root and not part.exists:
1549N/A                        # This is a double-check for the client case where
1549N/A                        # there is a part that is known to the catalog but
1549N/A                        # that the client has purposefully not retrieved.
1549N/A                        # (Think locale specific data.)
1369N/A                        return
1549N/A
1369N/A                self.__parts[name] = part
1387N/A
1387N/A                if name not in aparts:
1387N/A                        # Add a new entry to the catalog attributes for this new
1387N/A                        # part since it didn't exist previously.
1387N/A                        aparts[name] = {
1387N/A                            "last-modified": part.last_modified
1387N/A                        }
1369N/A                return part
1369N/A
1352N/A        def get_updates_needed(self, path):
1352N/A                """Returns a list of the catalog files needed to update
1352N/A                the existing catalog parts, based on the contents of the
1352N/A                catalog.attrs file in the directory indicated by 'path'.
1431N/A                A value of None will be returned if the the catalog has
1431N/A                not been modified, while an empty list will be returned
1431N/A                if no catalog parts need to be updated, but the catalog
1431N/A                itself has changed."""
1352N/A
1352N/A                new_attrs = CatalogAttrs(meta_root=path)
1352N/A                if not new_attrs.exists:
1431N/A                        # No updates needed (not even to attrs), so return None.
1431N/A                        return None
1431N/A
1431N/A                old_attrs = self._attrs
1431N/A                if old_attrs.created != new_attrs.created:
1431N/A                        # It's very likely that the catalog has been recreated
1431N/A                        # or this is a completely different catalog than was
1431N/A                        # expected.  In either case, an update isn't possible.
1431N/A                        raise api_errors.BadCatalogUpdateIdentity(path)
1431N/A
1431N/A                if new_attrs.last_modified == old_attrs.last_modified:
1431N/A                        # No updates needed (not even to attrs), so return None.
1431N/A                        return None
270N/A
1352N/A                # First, verify that all of the catalog parts the client has
1352N/A                # still exist.  If they no longer exist, the catalog is no
1352N/A                # longer valid and cannot be updated.
1352N/A                parts = {}
1352N/A                incremental = True
1352N/A                for name in old_attrs.parts:
1352N/A                        if name not in new_attrs.parts:
1352N/A                                raise api_errors.BadCatalogUpdateIdentity(path)
221N/A
1352N/A                        old_lm = old_attrs.parts[name]["last-modified"]
1352N/A                        new_lm = new_attrs.parts[name]["last-modified"]
30N/A
1352N/A                        if new_lm == old_lm:
1352N/A                                # Part hasn't changed.
1352N/A                                continue
1352N/A                        elif new_lm < old_lm:
1352N/A                                raise api_errors.ObsoleteCatalogUpdate(path)
1352N/A
1352N/A                        # The last component of the update name is the locale.
1352N/A                        locale = name.split(".", 2)[2]
1352N/A
1352N/A                        # Now check to see if an update log is still offered for
1352N/A                        # the last time this catalog part was updated.  If it
1352N/A                        # does not, then an incremental update cannot be safely
1352N/A                        # performed since updates may be missing.
1352N/A                        logdate = datetime_to_update_ts(old_lm)
3158N/A                        logname = "update.{0}.{1}".format(logdate, locale)
1352N/A
1352N/A                        if logname not in new_attrs.updates:
1352N/A                                incremental = False
157N/A
1431N/A                        parts.setdefault(locale, set())
1352N/A                        parts[locale].add(name)
1352N/A
1431N/A                # XXX in future, add current locale to this.  For now, just
1431N/A                # ensure that all of the locales of parts that were changed
1431N/A                # and exist on-disk are included.
1431N/A                locales = set(("C",))
1431N/A                locales.update(set(parts.keys()))
1431N/A
1431N/A                # Now determine if there are any new parts for this locale that
1431N/A                # this version of the API knows how to use that the client
1431N/A                # doesn't already have.
1431N/A                for name in new_attrs.parts:
1431N/A                        if name in parts or name in old_attrs.parts:
1431N/A                                continue
1431N/A
1431N/A                        # The last component of the name is the locale.
1431N/A                        locale = name.split(".", 2)[2]
1431N/A                        if locale not in locales:
1431N/A                                continue
1431N/A
1431N/A                        # Currently, only these parts are used by the client,
1431N/A                        # so only they need to be retrieved.
1431N/A                        if name == self.__BASE_PART or \
1431N/A                            name == self.__DEPS_PART or \
1431N/A                            name.startswith(self.__SUMM_PART_PFX):
1431N/A                                incremental = False
1431N/A
1431N/A                                # If a new part has been added for the current
1431N/A                                # locale, then incremental updates can't be
1431N/A                                # performed since updates for this locale can
1431N/A                                # only be applied to parts that already exist.
1431N/A                                parts.setdefault(locale, set())
1431N/A                                parts[locale].add(name)
1431N/A
1352N/A                if not parts:
1431N/A                        # No updates needed to catalog parts on-disk, but
1431N/A                        # catalog has changed.
1352N/A                        return []
1352N/A                elif not incremental:
1352N/A                        # Since an incremental update cannot be performed,
1352N/A                        # just return the updated parts for retrieval.
1352N/A                        updates = set()
1352N/A                        for locale in parts:
1352N/A                                updates.update(parts[locale])
1352N/A                        return updates
24N/A
1352N/A                # Finally, determine the update logs needed based on the catalog
1352N/A                # parts that need updating on a per-locale basis.
1352N/A                updates = set()
1352N/A                for locale in parts:
1352N/A                        # Determine the newest catalog part for a given locale,
1352N/A                        # this will be used to determine which update logs are
1352N/A                        # needed for an incremental update.
1352N/A                        last_lm = None
1352N/A                        for name in parts[locale]:
1431N/A                                if name not in old_attrs.parts:
1431N/A                                        continue
1431N/A
1352N/A                                lm = old_attrs.parts[name]["last-modified"]
1352N/A                                if not last_lm or lm > last_lm:
1352N/A                                        last_lm = lm
220N/A
3234N/A                        for name, uattrs in six.iteritems(new_attrs.updates):
1352N/A                                up_lm = uattrs["last-modified"]
1352N/A
1352N/A                                # The last component of the update name is the
1352N/A                                # locale.
1352N/A                                up_locale = name.split(".", 2)[2]
1352N/A
1352N/A                                if not up_locale == locale:
1352N/A                                        # This update log doesn't apply to the
1352N/A                                        # locale being evaluated for updates.
258N/A                                        continue
258N/A
1352N/A                                if up_lm <= last_lm:
1352N/A                                        # Older or same as newest catalog part
1352N/A                                        # for this locale; so skip.
1352N/A                                        continue
59N/A
1352N/A                                # If this updatelog was changed after the
1352N/A                                # newest catalog part for this locale, then
1352N/A                                # it is needed to update one or more catalog
1352N/A                                # parts for this locale.
1352N/A                                updates.add(name)
215N/A
1352N/A                # Ensure updates are in chronological ascending order.
1352N/A                return sorted(updates)
157N/A
1537N/A        def names(self, pubs=EmptyI):
1352N/A                """Returns a set containing the names of all the packages in
1537N/A                the Catalog.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
157N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1352N/A                        # Catalog contains nothing.
1352N/A                        return set()
1537N/A                return base.names(pubs=pubs)
157N/A
1352N/A        @property
1352N/A        def package_count(self):
1352N/A                """The number of unique packages in the catalog."""
1352N/A                return self._attrs.package_count
310N/A
1352N/A        @property
1352N/A        def package_version_count(self):
1352N/A                """The number of unique package versions in the catalog."""
1352N/A                return self._attrs.package_version_count
621N/A
1352N/A        @property
1352N/A        def parts(self):
1352N/A                """A dict containing the list of CatalogParts that the catalog
1352N/A                is composed of along with information about each part."""
157N/A
1352N/A                return self._attrs.parts
157N/A
1537N/A        def pkg_names(self, pubs=EmptyI):
1537N/A                """A generator function that produces package tuples of the form
1537N/A                (pub, stem) as it iterates over the contents of the catalog.
1537N/A
1537N/A                'pubs' is an optional list that contains the prefixes of the
1537N/A                publishers to restrict the results to."""
1537N/A
1537N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1537N/A                        # Catalog contains nothing.
1537N/A
1537N/A                        # This construction is necessary to get python to
1537N/A                        # return no results properly to callers expecting
1537N/A                        # a generator function.
1537N/A                        return iter(())
1537N/A                return base.pkg_names(pubs=pubs)
1537N/A
1431N/A        def publishers(self):
1431N/A                """Returns a set containing the prefixes of all the publishers
1431N/A                in the Catalog."""
1431N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1431N/A                        # Catalog contains nothing.
1431N/A                        return set()
1431N/A                return set(p for p in base.publishers())
1431N/A
1352N/A        def remove_package(self, pfmri):
1352N/A                """Remove a package and its metadata."""
621N/A
1369N/A                assert not self.read_only
1369N/A
1352N/A                self.__lock_catalog()
1352N/A                try:
1352N/A                        # The package has to be removed from every known part.
1352N/A                        entries = {}
621N/A
1352N/A                        # Use the same operation time and date for all
1352N/A                        # operations so that the last modification times
1352N/A                        # of all catalog parts and update logs will be
1352N/A                        # synchronized.
1352N/A                        op_time = datetime.datetime.utcnow()
157N/A
1352N/A                        for name in self._attrs.parts:
1369N/A                                part = self.get_part(name)
1537N/A                                if part is None:
1191N/A                                        continue
1352N/A
1537N/A                                pkg_entry = part.get_entry(pfmri)
1537N/A                                if pkg_entry is None:
1431N/A                                        if name == self.__BASE_PART:
1431N/A                                                # Entry should exist in at least
1431N/A                                                # the base part.
1537N/A                                                raise api_errors.UnknownCatalogEntry(
1537N/A                                                    pfmri.get_fmri())
1431N/A                                        # Skip; package's presence is optional
1431N/A                                        # in other parts.
847N/A                                        continue
157N/A
1352N/A                                part.remove(pfmri, op_time=op_time)
1431N/A                                if self.log_updates:
1352N/A                                        entries[part.name] = pkg_entry
1352N/A
1352N/A                        self.__log_update(pfmri, CatalogUpdate.REMOVE, op_time,
1352N/A                            entries=entries)
1352N/A                finally:
1352N/A                        self.__unlock_catalog()
1352N/A
1352N/A        def save(self):
1352N/A                """Finalize current state and save to file if possible."""
621N/A
1352N/A                self.__lock_catalog()
1352N/A                try:
1352N/A                        self.__save()
1352N/A                finally:
1352N/A                        self.__unlock_catalog()
847N/A
1352N/A        @property
1352N/A        def signatures(self):
1352N/A                """Returns a dict of the files the catalog is composed of along
1352N/A                with the last known signatures of each if they are available."""
621N/A
1352N/A                attrs = self._attrs
1352N/A                sigs = {
1352N/A                    attrs.name: attrs.signatures
1352N/A                }
1066N/A
1352N/A                for items in (attrs.parts, attrs.updates):
1352N/A                        for name in items:
1352N/A                                entry = sigs[name] = {}
1352N/A                                for k in items[name]:
1352N/A                                        try:
1352N/A                                                sig = k.split("signature-")[1]
1352N/A                                                entry[sig] = items[name][k]
1352N/A                                        except IndexError:
1352N/A                                                # Not a signature entry.
1352N/A                                                continue
1352N/A                return sigs
1066N/A
1537N/A        def tuples(self, last=False, ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces FMRI tuples as it
1537N/A                iterates over the contents of the catalog.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last FMRI
1537N/A                tuple for each package on a per-publisher basis should be
1537N/A                returned.  As long as the catalog has been saved since the
1537N/A                last modifying operation, or finalize() has has been called,
1537N/A                this will also be the newest version of the package.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1431N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
1431N/A                        # Catalog contains nothing.
1431N/A
1431N/A                        # This construction is necessary to get python to
1431N/A                        # return no results properly to callers expecting
1431N/A                        # a generator function.
1431N/A                        return iter(())
1537N/A                return base.tuples(last=last, ordered=ordered, pubs=pubs)
1537N/A
1537N/A        def tuple_entries(self, info_needed=EmptyI, last=False, locales=None,
1537N/A            ordered=False, pubs=EmptyI):
1537N/A                """A generator function that produces tuples of the format
1537N/A                ((pub, stem, version), entry, actions) as it iterates over
1537N/A                the contents of the catalog (where 'metadata' is a dict
1537N/A                containing the requested information).
1537N/A
1537N/A                'metadata' always contains the following information at a
1537N/A                 minimum:
1537N/A
1537N/A                        BASE
1537N/A                                'metadata' will be populated with Manifest
1537N/A                                signature data, if available, using key-value
1537N/A                                pairs of the form 'signature-<name>': value.
1537N/A
1537N/A                'info_needed' is an optional list of one or more catalog
1537N/A                constants indicating the types of catalog data that will
1537N/A                be returned in 'metadata' in addition to the above:
1537N/A
1537N/A                        DEPENDENCY
1537N/A                                'metadata' will contain depend and set Actions
1537N/A                                for package obsoletion, renaming, variants,
1537N/A                                and facets stored in a list under the
1537N/A                                key 'actions'.
1537N/A
1537N/A                        SUMMARY
1537N/A                                'metadata' will contain any remaining Actions
1537N/A                                not listed above, such as pkg.summary,
1537N/A                                pkg.description, etc. in a list under the key
1537N/A                                'actions'.
1537N/A
1537N/A                'last' is a boolean value that indicates only the last entry
1537N/A                for each package on a per-publisher basis should be returned.
1537N/A                As long as the catalog has been saved since the last modifying
1537N/A                operation, or finalize() has has been called, this will also be
1537N/A                the newest version of the package.
1537N/A
1537N/A                'locales' is an optional set of locale names for which Actions
1537N/A                should be returned.  The default is set(('C',)) if not provided.
1537N/A
1537N/A                'ordered' is an optional boolean value that indicates that
1537N/A                results should sorted by stem and then by publisher and
1537N/A                be in descending version order.  If False, results will be
1537N/A                in a ascending version order on a per-publisher, per-stem
1537N/A                basis.
1537N/A
1537N/A                'pubs' is an optional list of publisher prefixes to restrict
1537N/A                the results to."""
1537N/A
1537N/A                return self.__entries(info_needed=info_needed,
1537N/A                    locales=locales, last_version=last, ordered=ordered,
1537N/A                    pubs=pubs, tuples=True)
1431N/A
1352N/A        @property
1352N/A        def updates(self):
1352N/A                """A dict containing the list of known updates for the catalog
1352N/A                along with information about each update."""
1066N/A
1352N/A                return self._attrs.updates
286N/A
2352N/A        def update_entry(self, metadata, pfmri=None, pub=None, stem=None,
2352N/A            ver=None):
1369N/A                """Updates the metadata stored in a package's BASE catalog
2352N/A                record for the specified package.  Cannot be used when read_only
1369N/A                or log_updates is enabled; should never be used with a Catalog
1369N/A                intended for incremental update usage.
1369N/A
2352N/A                'metadata' must be a dict of additional metadata to store with
2352N/A                the package's BASE record.
2352N/A
1369N/A                'pfmri' is the FMRI of the package to update the entry for.
1369N/A
2352N/A                'pub' is the publisher of the package.
2352N/A
2352N/A                'stem' is the stem of the package.
2352N/A
2352N/A                'ver' is the version string of the package.
2352N/A
2352N/A                'pfmri' or 'pub', 'stem', and 'ver' must be provided.
2352N/A                """
2352N/A
2352N/A                assert pfmri or (pub and stem and ver)
1431N/A                assert not self.log_updates and not self.read_only
1431N/A
1431N/A                base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A                if base is None:
2352N/A                        if not pfmri:
2639N/A                                pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A                                    version=ver)
1369N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
1369N/A                # get_entry returns the actual catalog entry, so updating it
1369N/A                # simply requires reassignment.
2352N/A                entry = base.get_entry(pfmri=pfmri, pub=pub, stem=stem, ver=ver)
1537N/A                if entry is None:
2352N/A                        if not pfmri:
2639N/A                                pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A                                    version=ver)
1537N/A                        raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A                if metadata is None:
1369N/A                        if "metadata" in entry:
1369N/A                                del entry["metadata"]
1369N/A                        return
1369N/A                entry["metadata"] = metadata
1369N/A
1710N/A                op_time = datetime.datetime.utcnow()
1710N/A                attrs = self._attrs
1710N/A                attrs.last_modified = op_time
1710N/A                attrs.parts[base.name] = {
1710N/A                    "last-modified": op_time
1710N/A                }
1710N/A                base.last_modified = op_time
1710N/A
2810N/A        def validate(self, require_signatures=False):
1352N/A                """Verifies whether the signatures for the contents of the
1352N/A                catalog match the current signature data.  Raises the
1352N/A                exception named 'BadCatalogSignatures' on failure."""
157N/A
2810N/A                self._attrs.validate(require_signatures=require_signatures)
621N/A
2022N/A                def get_sigs(mdata):
2022N/A                        sigs = {}
2022N/A                        for key in mdata:
2022N/A                                if not key.startswith("signature-"):
2022N/A                                        continue
2022N/A                                sig = key.split("signature-")[1]
2022N/A                                sigs[sig] = mdata[key]
2022N/A                        if not sigs:
2022N/A                                # Allow validate() to perform its own fallback
2022N/A                                # logic if signature data isn't available.
2022N/A                                return None
2022N/A                        return sigs
2022N/A
3234N/A                for name, mdata in six.iteritems(self._attrs.parts):
2022N/A                        part = self.get_part(name, must_exist=True)
1537N/A                        if part is None:
1352N/A                                # Part does not exist; no validation needed.
1352N/A                                continue
2810N/A                        part.validate(signatures=get_sigs(mdata),
2810N/A                            require_signatures=require_signatures)
2022N/A
3234N/A                for name, mdata in six.iteritems(self._attrs.updates):
2022N/A                        ulog = self.__get_update(name, cache=False,
2022N/A                            must_exist=True)
1537N/A                        if ulog is None:
1352N/A                                # Update does not exist; no validation needed.
1352N/A                                continue
2810N/A                        ulog.validate(signatures=get_sigs(mdata),
2810N/A                            require_signatures=require_signatures)
157N/A
1352N/A        batch_mode = property(__get_batch_mode, __set_batch_mode)
1369N/A        last_modified = property(__get_last_modified, __set_last_modified,
1369N/A            doc="A UTC datetime object indicating the last time the catalog "
1369N/A            "was modified.")
1352N/A        meta_root = property(__get_meta_root, __set_meta_root)
3362N/A        file_root = property(__get_file_root, __set_file_root)
1358N/A        sign = property(__get_sign, __set_sign)
1369N/A        version = property(__get_version, __set_version)
814N/A
1431N/A
1431N/A# Methods used by external callers
1431N/Adef verify(filename):
1431N/A        """Convert the catalog part named by filename into the correct
1431N/A        type of Catalog object and then call its validate method to ensure
1431N/A        that is contents are self-consistent."""
1431N/A
1431N/A        path, fn = os.path.split(filename)
1431N/A        catobj = None
1431N/A
1431N/A        if fn.startswith("catalog"):
1431N/A                if fn.endswith("attrs"):
1431N/A                        catobj = CatalogAttrs(meta_root=path)
1431N/A                else:
1431N/A                        catobj = CatalogPart(fn, meta_root=path)
1431N/A        elif fn.startswith("update"):
1431N/A                catobj = CatalogUpdate(fn, meta_root=path)
1431N/A        else:
1431N/A                # Unrecognized.
1537N/A                raise api_errors.UnrecognizedCatalogPart(fn)
1431N/A
1431N/A        # With the else case above, this should never be None.
1431N/A        assert catobj
1431N/A
2810N/A        catobj.validate(require_signatures=True)
1431N/A
1352N/A# Methods used by Catalog classes.
1352N/Adef datetime_to_ts(dt):
1352N/A        """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A        format. """
1352N/A
1352N/A        return dt.isoformat()
215N/A
1352N/Adef datetime_to_basic_ts(dt):
1352N/A        """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A        basic format. """
45N/A
1352N/A        val = dt.isoformat()
1352N/A        val = val.replace("-", "")
1352N/A        val = val.replace(":", "")
621N/A
1352N/A        if not dt.tzname():
1352N/A                # Assume UTC.
1352N/A                val += "Z"
1352N/A        return val
621N/A
1352N/Adef datetime_to_update_ts(dt):
1352N/A        """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A        basic partial format. """
621N/A
1352N/A        val = dt.isoformat()
1352N/A        val = val.replace("-", "")
1352N/A        # Drop the minutes and seconds portion.
1352N/A        val = val.rsplit(":", 2)[0]
1352N/A        val = val.replace(":", "")
621N/A
1352N/A        if not dt.tzname():
1352N/A                # Assume UTC.
1352N/A                val += "Z"
1352N/A        return val
106N/A
1352N/Adef now_to_basic_ts():
1352N/A        """Returns the current UTC time as timestamp in ISO-8601 basic
1352N/A        format."""
1352N/A        return datetime_to_basic_ts(datetime.datetime.utcnow())
220N/A
1352N/Adef now_to_update_ts():
1352N/A        """Returns the current UTC time as timestamp in ISO-8601 basic
1352N/A        partial format."""
1352N/A        return datetime_to_update_ts(datetime.datetime.utcnow())
215N/A
215N/Adef ts_to_datetime(ts):
1352N/A        """Take timestamp ts in ISO-8601 format, and convert it to a
1352N/A        datetime object."""
215N/A
215N/A        year = int(ts[0:4])
215N/A        month = int(ts[5:7])
215N/A        day = int(ts[8:10])
215N/A        hour = int(ts[11:13])
382N/A        minutes = int(ts[14:16])
215N/A        sec = int(ts[17:19])
289N/A        # usec is not in the string if 0
289N/A        try:
382N/A                usec = int(ts[20:26])
289N/A        except ValueError:
382N/A                usec = 0
1352N/A        return datetime.datetime(year, month, day, hour, minutes, sec, usec)
215N/A
1352N/Adef basic_ts_to_datetime(ts):
1352N/A        """Take timestamp ts in ISO-8601 basic format, and convert it to a
1352N/A        datetime object."""
215N/A
1352N/A        year = int(ts[0:4])
1352N/A        month = int(ts[4:6])
1352N/A        day = int(ts[6:8])
1352N/A        hour = int(ts[9:11])
1352N/A        minutes = int(ts[11:13])
1352N/A        sec = int(ts[13:15])
1352N/A        # usec is not in the string if 0
1352N/A        try:
1352N/A                usec = int(ts[16:22])
1352N/A        except ValueError:
1352N/A                usec = 0
1352N/A        return datetime.datetime(year, month, day, hour, minutes, sec, usec)