Lines Matching refs:rm
240 static void vdev_raidz_generate_parity(raidz_map_t *rm);
262 vdev_raidz_map_free(raidz_map_t *rm)
267 for (c = 0; c < rm->rm_firstdatacol; c++) {
268 zio_buf_free(rm->rm_col[c].rc_data, rm->rm_col[c].rc_size);
270 if (rm->rm_col[c].rc_gdata != NULL)
271 zio_buf_free(rm->rm_col[c].rc_gdata,
272 rm->rm_col[c].rc_size);
276 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++)
277 size += rm->rm_col[c].rc_size;
279 if (rm->rm_datacopy != NULL)
280 zio_buf_free(rm->rm_datacopy, size);
282 kmem_free(rm, offsetof(raidz_map_t, rm_col[rm->rm_scols]));
288 raidz_map_t *rm = zio->io_vsd;
290 ASSERT0(rm->rm_freed);
291 rm->rm_freed = 1;
293 if (rm->rm_reports == 0)
294 vdev_raidz_map_free(rm);
301 raidz_map_t *rm = arg;
303 ASSERT3U(rm->rm_reports, >, 0);
305 if (--rm->rm_reports == 0 && rm->rm_freed != 0)
306 vdev_raidz_map_free(rm);
312 raidz_map_t *rm = zcr->zcr_cbdata;
317 const char *bad = rm->rm_col[c].rc_data;
324 if (c < rm->rm_firstdatacol) {
330 if (rm->rm_col[0].rc_gdata == NULL) {
339 for (x = 0; x < rm->rm_firstdatacol; x++) {
340 bad_parity[x] = rm->rm_col[x].rc_data;
341 rm->rm_col[x].rc_data = rm->rm_col[x].rc_gdata =
342 zio_buf_alloc(rm->rm_col[x].rc_size);
347 for (; x < rm->rm_cols; x++) {
348 rm->rm_col[x].rc_data = buf;
349 buf += rm->rm_col[x].rc_size;
355 vdev_raidz_generate_parity(rm);
358 for (x = 0; x < rm->rm_firstdatacol; x++)
359 rm->rm_col[x].rc_data = bad_parity[x];
361 buf = rm->rm_datacopy;
362 for (x = rm->rm_firstdatacol; x < rm->rm_cols; x++) {
363 rm->rm_col[x].rc_data = buf;
364 buf += rm->rm_col[x].rc_size;
368 ASSERT3P(rm->rm_col[c].rc_gdata, !=, NULL);
369 good = rm->rm_col[c].rc_gdata;
374 for (x = rm->rm_firstdatacol; x < c; x++)
375 good += rm->rm_col[x].rc_size;
394 raidz_map_t *rm = zio->io_vsd;
398 zcr->zcr_cbdata = rm;
403 rm->rm_reports++;
404 ASSERT3U(rm->rm_reports, >, 0);
406 if (rm->rm_datacopy != NULL)
419 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++)
420 size += rm->rm_col[c].rc_size;
422 buf = rm->rm_datacopy = zio_buf_alloc(size);
424 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
425 raidz_col_t *col = &rm->rm_col[c];
432 ASSERT3P(buf - (caddr_t)rm->rm_datacopy, ==, size);
448 raidz_map_t *rm;
493 rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_SLEEP);
495 rm->rm_cols = acols;
496 rm->rm_scols = scols;
497 rm->rm_bigcols = bc;
498 rm->rm_skipstart = bc;
499 rm->rm_missingdata = 0;
500 rm->rm_missingparity = 0;
501 rm->rm_firstdatacol = nparity;
502 rm->rm_datacopy = NULL;
503 rm->rm_reports = 0;
504 rm->rm_freed = 0;
505 rm->rm_ecksuminjected = 0;
516 rm->rm_col[c].rc_devidx = col;
517 rm->rm_col[c].rc_offset = coff;
518 rm->rm_col[c].rc_data = NULL;
519 rm->rm_col[c].rc_gdata = NULL;
520 rm->rm_col[c].rc_error = 0;
521 rm->rm_col[c].rc_tried = 0;
522 rm->rm_col[c].rc_skipped = 0;
525 rm->rm_col[c].rc_size = 0;
527 rm->rm_col[c].rc_size = (q + 1) << unit_shift;
529 rm->rm_col[c].rc_size = q << unit_shift;
531 asize += rm->rm_col[c].rc_size;
535 rm->rm_asize = roundup(asize, (nparity + 1) << unit_shift);
536 rm->rm_nskip = roundup(tot, nparity + 1) - tot;
537 ASSERT3U(rm->rm_asize - asize, ==, rm->rm_nskip << unit_shift);
538 ASSERT3U(rm->rm_nskip, <=, nparity);
540 for (c = 0; c < rm->rm_firstdatacol; c++)
541 rm->rm_col[c].rc_data = zio_buf_alloc(rm->rm_col[c].rc_size);
543 rm->rm_col[c].rc_data = data;
546 rm->rm_col[c].rc_data = (char *)rm->rm_col[c - 1].rc_data +
547 rm->rm_col[c - 1].rc_size;
569 ASSERT(rm->rm_cols >= 2);
570 ASSERT(rm->rm_col[0].rc_size == rm->rm_col[1].rc_size);
572 if (rm->rm_firstdatacol == 1 && (offset & (1ULL << 20))) {
573 devidx = rm->rm_col[0].rc_devidx;
574 o = rm->rm_col[0].rc_offset;
575 rm->rm_col[0].rc_devidx = rm->rm_col[1].rc_devidx;
576 rm->rm_col[0].rc_offset = rm->rm_col[1].rc_offset;
577 rm->rm_col[1].rc_devidx = devidx;
578 rm->rm_col[1].rc_offset = o;
580 if (rm->rm_skipstart == 0)
581 rm->rm_skipstart = 1;
584 return (rm);
588 vdev_raidz_generate_parity_p(raidz_map_t *rm)
593 pcount = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
595 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
596 src = rm->rm_col[c].rc_data;
597 p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
598 ccount = rm->rm_col[c].rc_size / sizeof (src[0]);
600 if (c == rm->rm_firstdatacol) {
615 vdev_raidz_generate_parity_pq(raidz_map_t *rm)
620 pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
621 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
622 rm->rm_col[VDEV_RAIDZ_Q].rc_size);
624 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
625 src = rm->rm_col[c].rc_data;
626 p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
627 q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
629 ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);
631 if (c == rm->rm_firstdatacol) {
667 vdev_raidz_generate_parity_pqr(raidz_map_t *rm)
672 pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
673 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
674 rm->rm_col[VDEV_RAIDZ_Q].rc_size);
675 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
676 rm->rm_col[VDEV_RAIDZ_R].rc_size);
678 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
679 src = rm->rm_col[c].rc_data;
680 p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
681 q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
682 r = rm->rm_col[VDEV_RAIDZ_R].rc_data;
684 ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);
686 if (c == rm->rm_firstdatacol) {
732 vdev_raidz_generate_parity(raidz_map_t *rm)
734 switch (rm->rm_firstdatacol) {
736 vdev_raidz_generate_parity_p(rm);
739 vdev_raidz_generate_parity_pq(rm);
742 vdev_raidz_generate_parity_pqr(rm);
750 vdev_raidz_reconstruct_p(raidz_map_t *rm, int *tgts, int ntgts)
757 ASSERT(x >= rm->rm_firstdatacol);
758 ASSERT(x < rm->rm_cols);
760 xcount = rm->rm_col[x].rc_size / sizeof (src[0]);
761 ASSERT(xcount <= rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]));
764 src = rm->rm_col[VDEV_RAIDZ_P].rc_data;
765 dst = rm->rm_col[x].rc_data;
770 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
771 src = rm->rm_col[c].rc_data;
772 dst = rm->rm_col[x].rc_data;
777 ccount = rm->rm_col[c].rc_size / sizeof (src[0]);
789 vdev_raidz_reconstruct_q(raidz_map_t *rm, int *tgts, int ntgts)
798 xcount = rm->rm_col[x].rc_size / sizeof (src[0]);
799 ASSERT(xcount <= rm->rm_col[VDEV_RAIDZ_Q].rc_size / sizeof (src[0]));
801 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
802 src = rm->rm_col[c].rc_data;
803 dst = rm->rm_col[x].rc_data;
808 ccount = rm->rm_col[c].rc_size / sizeof (src[0]);
812 if (c == rm->rm_firstdatacol) {
832 src = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
833 dst = rm->rm_col[x].rc_data;
834 exp = 255 - (rm->rm_cols - 1 - x);
847 vdev_raidz_reconstruct_pq(raidz_map_t *rm, int *tgts, int ntgts)
857 ASSERT(x >= rm->rm_firstdatacol);
858 ASSERT(y < rm->rm_cols);
860 ASSERT(rm->rm_col[x].rc_size >= rm->rm_col[y].rc_size);
869 pdata = rm->rm_col[VDEV_RAIDZ_P].rc_data;
870 qdata = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
871 xsize = rm->rm_col[x].rc_size;
872 ysize = rm->rm_col[y].rc_size;
874 rm->rm_col[VDEV_RAIDZ_P].rc_data =
875 zio_buf_alloc(rm->rm_col[VDEV_RAIDZ_P].rc_size);
876 rm->rm_col[VDEV_RAIDZ_Q].rc_data =
877 zio_buf_alloc(rm->rm_col[VDEV_RAIDZ_Q].rc_size);
878 rm->rm_col[x].rc_size = 0;
879 rm->rm_col[y].rc_size = 0;
881 vdev_raidz_generate_parity_pq(rm);
883 rm->rm_col[x].rc_size = xsize;
884 rm->rm_col[y].rc_size = ysize;
888 pxy = rm->rm_col[VDEV_RAIDZ_P].rc_data;
889 qxy = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
890 xd = rm->rm_col[x].rc_data;
891 yd = rm->rm_col[y].rc_data;
909 b = vdev_raidz_pow2[255 - (rm->rm_cols - 1 - x)];
923 zio_buf_free(rm->rm_col[VDEV_RAIDZ_P].rc_data,
924 rm->rm_col[VDEV_RAIDZ_P].rc_size);
925 zio_buf_free(rm->rm_col[VDEV_RAIDZ_Q].rc_data,
926 rm->rm_col[VDEV_RAIDZ_Q].rc_size);
931 rm->rm_col[VDEV_RAIDZ_P].rc_data = pdata;
932 rm->rm_col[VDEV_RAIDZ_Q].rc_data = qdata;
1089 vdev_raidz_matrix_init(raidz_map_t *rm, int n, int nmap, int *map,
1095 ASSERT(n == rm->rm_cols - rm->rm_firstdatacol);
1119 vdev_raidz_matrix_invert(raidz_map_t *rm, int n, int nmissing, int *missing,
1131 ASSERT3S(used[i], <, rm->rm_firstdatacol);
1134 ASSERT3S(used[i], >=, rm->rm_firstdatacol);
1151 ASSERT3U(used[j], >=, rm->rm_firstdatacol);
1152 jj = used[j] - rm->rm_firstdatacol;
1213 vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing,
1245 ASSERT3U(c, <, rm->rm_cols);
1247 src = rm->rm_col[c].rc_data;
1248 ccount = rm->rm_col[c].rc_size;
1250 cc = missing[j] + rm->rm_firstdatacol;
1251 ASSERT3U(cc, >=, rm->rm_firstdatacol);
1252 ASSERT3U(cc, <, rm->rm_cols);
1255 dst[j] = rm->rm_col[cc].rc_data;
1256 dcount[j] = rm->rm_col[cc].rc_size;
1259 ASSERT(ccount >= rm->rm_col[missing[0]].rc_size || i > 0);
1289 vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts)
1306 n = rm->rm_cols - rm->rm_firstdatacol;
1313 if (tgts[t] >= rm->rm_firstdatacol) {
1315 tgts[t] - rm->rm_firstdatacol;
1325 ASSERT(c < rm->rm_firstdatacol);
1360 for (tt = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
1362 c == missing_rows[tt] + rm->rm_firstdatacol) {
1375 vdev_raidz_matrix_init(rm, n, nmissing_rows, parity_map, rows);
1380 vdev_raidz_matrix_invert(rm, n, nmissing_rows, missing_rows, rows,
1386 vdev_raidz_matrix_reconstruct(rm, n, nmissing_rows, missing_rows,
1395 vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt)
1411 nbadparity = rm->rm_firstdatacol;
1412 nbaddata = rm->rm_cols - nbadparity;
1414 for (i = 0, c = 0; c < rm->rm_cols; c++) {
1415 if (c < rm->rm_firstdatacol)
1421 } else if (rm->rm_col[c].rc_error != 0) {
1423 } else if (c >= rm->rm_firstdatacol) {
1444 return (vdev_raidz_reconstruct_p(rm, dt, 1));
1446 ASSERT(rm->rm_firstdatacol > 1);
1449 return (vdev_raidz_reconstruct_q(rm, dt, 1));
1451 ASSERT(rm->rm_firstdatacol > 2);
1455 ASSERT(rm->rm_firstdatacol > 1);
1459 return (vdev_raidz_reconstruct_pq(rm, dt, 2));
1461 ASSERT(rm->rm_firstdatacol > 2);
1467 code = vdev_raidz_reconstruct_general(rm, tgts, ntgts);
1594 raidz_map_t *rm;
1622 rm = vdev_raidz_map_alloc(data - (offset - origoffset),
1628 for (c = rm->rm_firstdatacol; c < rm->rm_cols;
1630 rc = &rm->rm_col[c];
1667 vdev_raidz_map_free(rm);
1721 raidz_map_t *rm;
1725 rm = vdev_raidz_map_alloc(zio->io_data, zio->io_size, zio->io_offset,
1729 zio->io_vsd = rm;
1732 ASSERT3U(rm->rm_asize, ==, vdev_psize_to_asize(vd, zio->io_size));
1735 vdev_raidz_generate_parity(rm);
1737 for (c = 0; c < rm->rm_cols; c++) {
1738 rc = &rm->rm_col[c];
1750 for (c = rm->rm_skipstart, i = 0; i < rm->rm_nskip; c++, i++) {
1751 ASSERT(c <= rm->rm_scols);
1752 if (c == rm->rm_scols)
1754 rc = &rm->rm_col[c];
1773 for (c = rm->rm_cols - 1; c >= 0; c--) {
1774 rc = &rm->rm_col[c];
1777 if (c >= rm->rm_firstdatacol)
1778 rm->rm_missingdata++;
1780 rm->rm_missingparity++;
1787 if (c >= rm->rm_firstdatacol)
1788 rm->rm_missingdata++;
1790 rm->rm_missingparity++;
1795 if (c >= rm->rm_firstdatacol || rm->rm_missingdata > 0 ||
1818 raidz_map_t *rm = zio->io_vsd;
1825 zbc.zbc_injected = rm->rm_ecksuminjected;
1841 raidz_map_t *rm = zio->io_vsd;
1845 rm->rm_ecksuminjected = 1;
1857 raidz_parity_verify(zio_t *zio, raidz_map_t *rm)
1870 for (c = 0; c < rm->rm_firstdatacol; c++) {
1871 rc = &rm->rm_col[c];
1878 vdev_raidz_generate_parity(rm);
1880 for (c = 0; c < rm->rm_firstdatacol; c++) {
1881 rc = &rm->rm_col[c];
1901 vdev_raidz_worst_error(raidz_map_t *rm)
1905 for (int c = 0; c < rm->rm_cols; c++)
1906 error = zio_worst_error(error, rm->rm_col[c].rc_error);
1922 raidz_map_t *rm = zio->io_vsd;
1930 ASSERT(total_errors < rm->rm_firstdatacol);
1937 for (n = 1; n <= rm->rm_firstdatacol - total_errors; n++) {
1949 c < rm->rm_firstdatacol) {
1950 c = rm->rm_firstdatacol;
1953 while (rm->rm_col[c].rc_error != 0) {
1955 ASSERT3S(c, <, rm->rm_cols);
1964 tgts[n] = rm->rm_cols;
1973 orig[n - 1] = zio_buf_alloc(rm->rm_col[0].rc_size);
1990 ASSERT3S(c, <, rm->rm_cols);
1991 rc = &rm->rm_col[c];
1999 code = vdev_raidz_reconstruct(rm, tgts, n);
2005 rc = &rm->rm_col[c];
2022 rc = &rm->rm_col[c];
2032 next < rm->rm_cols &&
2033 rm->rm_col[next].rc_error != 0; next++)
2049 rm->rm_col[c].rc_error != 0; c++)
2061 zio_buf_free(orig[i], rm->rm_col[0].rc_size);
2093 raidz_map_t *rm = zio->io_vsd;
2106 ASSERT(rm->rm_missingparity <= rm->rm_firstdatacol);
2107 ASSERT(rm->rm_missingdata <= rm->rm_cols - rm->rm_firstdatacol);
2109 for (c = 0; c < rm->rm_cols; c++) {
2110 rc = &rm->rm_col[c];
2115 if (c < rm->rm_firstdatacol)
2124 } else if (c < rm->rm_firstdatacol && !rc->rc_tried) {
2141 if (total_errors > rm->rm_firstdatacol)
2142 zio->io_error = vdev_raidz_worst_error(rm);
2165 if (total_errors <= rm->rm_firstdatacol - parity_untried) {
2177 rm->rm_firstdatacol ||
2179 n = raidz_parity_verify(zio, rm);
2182 rm->rm_firstdatacol);
2195 ASSERT(parity_errors < rm->rm_firstdatacol);
2201 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
2202 rc = &rm->rm_col[c];
2209 ASSERT(rm->rm_firstdatacol >= n);
2211 code = vdev_raidz_reconstruct(rm, tgts, n);
2229 if (parity_errors < rm->rm_firstdatacol - n ||
2231 n = raidz_parity_verify(zio, rm);
2234 rm->rm_firstdatacol);
2250 rm->rm_missingdata = 0;
2251 rm->rm_missingparity = 0;
2253 for (c = 0; c < rm->rm_cols; c++) {
2254 if (rm->rm_col[c].rc_tried)
2259 rc = &rm->rm_col[c];
2267 } while (++c < rm->rm_cols);
2282 if (total_errors > rm->rm_firstdatacol) {
2283 zio->io_error = vdev_raidz_worst_error(rm);
2285 } else if (total_errors < rm->rm_firstdatacol &&
2292 if (code != (1 << rm->rm_firstdatacol) - 1)
2293 (void) raidz_parity_verify(zio, rm);
2310 for (c = 0; c < rm->rm_cols; c++) {
2311 rc = &rm->rm_col[c];
2316 rm->rm_ecksuminjected;
2336 for (c = 0; c < rm->rm_cols; c++) {
2337 rc = &rm->rm_col[c];