crypto/nss/nss_sha512crypt.c

2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski/* This file is based on the work of Ulrich Drepper
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski * (http://people.redhat.com/drepper/SHA-crypt.txt). I have replaced the
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder * included SHA512 implementation by calls to NSS
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski * (http://www.mozilla.org/projects/security/pki/nss/).
97018cf5fa25b494adffd7e9b4e87320dae6bf47Christian Maeder *
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski *  Sumit Bose <sbose@redhat.com>
34bff097c14521b5e57ce37279a34256e1f78aa5Klaus Luettich */
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski/* SHA512-based Unix crypt implementation.
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski   Released into the Public Domain by Ulrich Drepper <drepper@redhat.com>.  */
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski
f3a94a197960e548ecd6520bb768cb0d547457bbChristian Maeder#include "config.h"
684ada8af5c3e6da5c1a69edb6f233c9f2db4ebdWiebke Herding
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski#include <errno.h>
e4e1509ff358e739fddf1483ad39467e0e1becc2Christian Maeder#include <limits.h>
684ada8af5c3e6da5c1a69edb6f233c9f2db4ebdWiebke Herding#include <stdbool.h>
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder#include <stdint.h>
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder#include <stdio.h>
e4e1509ff358e739fddf1483ad39467e0e1becc2Christian Maeder#include <stdlib.h>
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder#include <string.h>
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder#include <sys/param.h>
76647324ed70f33b95a881b536d883daccf9568dChristian Maeder#include <sys/types.h>
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder
d183a4514d8a5b6a5d48d15a8dff52d0c96691eaChristian Maeder#include "util/util.h"
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder#include "util/sss_endian.h"
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski#include "util/crypto/nss/nss_util.h"
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski#include <prinit.h>
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder#include <nss.h>
50dce6b011347f92377adb8bbabaeeb80975e86dChristian Maeder#include <sechash.h>
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder#include <pk11func.h>
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder
50dce6b011347f92377adb8bbabaeeb80975e86dChristian Maeder/* Define our magic string to mark salt for SHA512 "encryption" replacement. */
c74040e2ca9d0534d0c4244f69a3e76a01341f05Klaus Luettichconst char sha512_salt_prefix[] = "$6$";
86b1d0c80abdd4ca36491cf7025b718a5fea5080Christian Maeder#define SALT_PREF_SIZE (sizeof(sha512_salt_prefix) - 1)
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder/* Prefix for optional rounds specification. */
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maederconst char sha512_rounds_prefix[] = "rounds=";
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski#define ROUNDS_SIZE (sizeof(sha512_rounds_prefix) - 1)
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski
05ca76b03b6d16bcfb3e7654c31e41a220e85663Till Mossakowski#define SALT_LEN_MAX 16
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maeder#define ROUNDS_DEFAULT 5000
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maeder#define ROUNDS_MIN 1000
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maeder#define ROUNDS_MAX 999999999
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maeder
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maeder/* Table with characters for base64 transformation.  */
7c757dd5b0b027dfc0cd0b9535758c8992cdde2fChristian Maederconst char b64t[64] =
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski/* base64 conversion function */
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maederstatic inline void b64_from_24bit(char **dest, size_t *len, size_t n,
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski                                  uint8_t b2, uint8_t b1, uint8_t b0)
86b1d0c80abdd4ca36491cf7025b718a5fea5080Christian Maeder{
86b1d0c80abdd4ca36491cf7025b718a5fea5080Christian Maeder    uint32_t w;
86b1d0c80abdd4ca36491cf7025b718a5fea5080Christian Maeder    size_t i;
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder    if (*len < n) n = *len;
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder
4c7f058cdd19ce67b2b5d4b7f69703d0f8a21e38Christian Maeder    w = (b2 << 16) | (b1 << 8) | b0;
c2db39a683438b0f3d484519f4c93db26eec9d2eWiebke Herding    for (i = 0; i < n; i++) {
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski        (*dest)[i] = b64t[w & 0x3f];
c2db39a683438b0f3d484519f4c93db26eec9d2eWiebke Herding        w >>= 6;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder    }
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    *len -= i;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder    *dest += i;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder}
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder#define PTR_2_INT(x) ((x) - ((__typeof__ (x)) NULL))
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder#define ALIGN64 __alignof__(uint64_t)
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder
cd6e5706893519bfcf24539afa252fcbed5097ddKlaus Luettichstatic int sha512_crypt_r(const char *key,
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder                          const char *salt,
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski                          char *buffer, size_t buflen)
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski{
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    unsigned char temp_result[64] __attribute__((__aligned__(ALIGN64)));
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    unsigned char alt_result[64] __attribute__((__aligned__(ALIGN64)));
bec7e681b0ba4d085638ec7af0cf7ae5068840caChristian Maeder    size_t rounds = ROUNDS_DEFAULT;
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder    bool rounds_custom = false;
a10ff6125d62484ec5961c8a5d9d1c5a3e14fa66Christian Maeder    HASHContext *alt_ctx = NULL;
7b27b67b1c8516d7ccf1610a17fec93662d6a93fChristian Maeder    HASHContext *ctx = NULL;
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    size_t salt_len;
0d9160e906743b226d4768707f84151ab6c66253Heng Jiang    size_t key_len;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder    size_t cnt;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder    char *copied_salt = NULL;
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    char *copied_key = NULL;
bec7e681b0ba4d085638ec7af0cf7ae5068840caChristian Maeder    char *p_bytes = NULL;
7b27b67b1c8516d7ccf1610a17fec93662d6a93fChristian Maeder    char *s_bytes = NULL;
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    int p1, p2, p3, pt, n;
bec7e681b0ba4d085638ec7af0cf7ae5068840caChristian Maeder    unsigned int part;
c2db39a683438b0f3d484519f4c93db26eec9d2eWiebke Herding    char *cp, *tmp;
f7d2e793728bbb7fd185e027eb9dfd7b9dd11c21Christian Maeder    int ret;
f6c04b8534762854072795add026d4551156a410Heng Jiang
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    /* Find beginning of salt string. The prefix should normally always be
cd6e5706893519bfcf24539afa252fcbed5097ddKlaus Luettich     * present. Just in case it is not. */
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    if (strncmp(salt, sha512_salt_prefix, SALT_PREF_SIZE) == 0) {
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder        /* Skip salt prefix.  */
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder        salt += SALT_PREF_SIZE;
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    }
7b27b67b1c8516d7ccf1610a17fec93662d6a93fChristian Maeder
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    if (strncmp(salt, sha512_rounds_prefix, ROUNDS_SIZE) == 0) {
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski        unsigned long int srounds;
c2db39a683438b0f3d484519f4c93db26eec9d2eWiebke Herding        const char *num;
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski        char *endp;
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder        num = salt + ROUNDS_SIZE;
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder        srounds = strtoul(num, &endp, 10);
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder        if (*endp == '$') {
eca4db63ed0bdbd93b62678feea6e3eb80aa47bbChristian Maeder            salt = endp + 1;
4aa35aadcb28f8a962096efc70d3bdb58ab7d9faChristian Maeder            if (srounds < ROUNDS_MIN) srounds = ROUNDS_MIN;
0e2ae85e2453466d03c1fc5884a3d693235bb9d9Christian Maeder            if (srounds > ROUNDS_MAX) srounds = ROUNDS_MAX;
0e2ae85e2453466d03c1fc5884a3d693235bb9d9Christian Maeder            rounds = srounds;
27b37f8e6b165f7abb653a54b45ffcdb81cec561Christian Maeder            rounds_custom = true;
e58ed9360500f97f6370bebf69f0c2bfee34782cChristian Maeder        }
e58ed9360500f97f6370bebf69f0c2bfee34782cChristian Maeder    }
e58ed9360500f97f6370bebf69f0c2bfee34782cChristian Maeder
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    salt_len = MIN(strcspn(salt, "$"), SALT_LEN_MAX);
df6f4a9e6b3d0542ecc181fbc1bcec2affca1d30Christian Maeder    key_len = strlen(key);
7b27b67b1c8516d7ccf1610a17fec93662d6a93fChristian Maeder
c2db39a683438b0f3d484519f4c93db26eec9d2eWiebke Herding    if ((PTR_2_INT(key) % ALIGN64) != 0) {
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder        tmp = (char *)alloca(key_len + ALIGN64);
3a761fd74f4f3c5587a199553c0ee7383e5d8ff3Christian Maeder        key = copied_key = memcpy(tmp + ALIGN64 - PTR_2_INT(tmp) % ALIGN64, key, key_len);
e85b224577b78d08ba5c39fe9dcc2e53995454a2Christian Maeder    }
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski
2a693c01b154f1e25931ff6c754d2d02096e2662Till Mossakowski    if (PTR_2_INT(salt) % ALIGN64 != 0) {
b65e16b9e5652ff341ab0f49be5da51e2c0e10a5Till Mossakowski        tmp = (char *)alloca(salt_len + ALIGN64);
cd6e5706893519bfcf24539afa252fcbed5097ddKlaus Luettich        salt = copied_salt = memcpy(tmp + ALIGN64 - PTR_2_INT(tmp) % ALIGN64, salt, salt_len);
    }

    ret = nspr_nss_init();
    if (ret != EOK) {
        ret = EIO;
        goto done;
    }

    ctx = HASH_Create(HASH_AlgSHA512);
    if (!ctx) {
        ret = EIO;
        goto done;
    }

    alt_ctx = HASH_Create(HASH_AlgSHA512);
    if (!alt_ctx) {
        ret = EIO;
        goto done;
    }

    /* Prepare for the real work.  */
    HASH_Begin(ctx);

    /* Add the key string.  */
    HASH_Update(ctx, (const unsigned char *)key, key_len);

    /* The last part is the salt string. This must be at most 16
     * characters and it ends at the first `$' character (for
     * compatibility with existing implementations). */
    HASH_Update(ctx, (const unsigned char *)salt, salt_len);


    /* Compute alternate SHA512 sum with input KEY, SALT, and KEY.
     * The final result will be added to the first context. */
    HASH_Begin(alt_ctx);

    /* Add key. */
    HASH_Update(alt_ctx, (const unsigned char *)key, key_len);

    /* Add salt. */
    HASH_Update(alt_ctx, (const unsigned char *)salt, salt_len);

    /* Add key again. */
    HASH_Update(alt_ctx, (const unsigned char *)key, key_len);

    /* Now get result of this (64 bytes) and add it to the other context. */
    HASH_End(alt_ctx, alt_result, &part, HASH_ResultLenContext(alt_ctx));

    /* Add for any character in the key one byte of the alternate sum. */
    for (cnt = key_len; cnt > 64; cnt -= 64) {
        HASH_Update(ctx, alt_result, 64);
    }
    HASH_Update(ctx, alt_result, cnt);

    /* Take the binary representation of the length of the key and for every
     * 1 add the alternate sum, for every 0 the key. */
    for (cnt = key_len; cnt > 0; cnt >>= 1) {
        if ((cnt & 1) != 0) {
            HASH_Update(ctx, alt_result, 64);
        } else {
            HASH_Update(ctx, (const unsigned char *)key, key_len);
        }
    }

    /* Create intermediate result. */
    HASH_End(ctx, alt_result, &part, HASH_ResultLenContext(ctx));

    /* Start computation of P byte sequence. */
    HASH_Begin(alt_ctx);

    /* For every character in the password add the entire password. */
    for (cnt = 0; cnt < key_len; cnt++) {
        HASH_Update(alt_ctx, (const unsigned char *)key, key_len);
    }

    /* Finish the digest. */
    HASH_End(alt_ctx, temp_result, &part, HASH_ResultLenContext(alt_ctx));

    /* Create byte sequence P. */
    cp = p_bytes = alloca(key_len);
    for (cnt = key_len; cnt >= 64; cnt -= 64) {
        cp = mempcpy(cp, temp_result, 64);
    }
    memcpy(cp, temp_result, cnt);

    /* Start computation of S byte sequence. */
    HASH_Begin(alt_ctx);

    /* For every character in the password add the entire salt. */
    for (cnt = 0; cnt < 16 + alt_result[0]; cnt++) {
        HASH_Update(alt_ctx, (const unsigned char *)salt, salt_len);
    }

    /* Finish the digest. */
    HASH_End(alt_ctx, temp_result, &part, HASH_ResultLenContext(alt_ctx));

    /* Create byte sequence S.  */
    cp = s_bytes = alloca(salt_len);
    for (cnt = salt_len; cnt >= 64; cnt -= 64) {
        cp = mempcpy(cp, temp_result, 64);
    }
    memcpy(cp, temp_result, cnt);

    /* Repeatedly run the collected hash value through SHA512 to burn CPU cycles. */
    for (cnt = 0; cnt < rounds; cnt++) {

        HASH_Begin(ctx);

        /* Add key or last result. */
        if ((cnt & 1) != 0) {
            HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
        } else {
            HASH_Update(ctx, alt_result, 64);
        }

        /* Add salt for numbers not divisible by 3. */
        if (cnt % 3 != 0) {
            HASH_Update(ctx, (const unsigned char *)s_bytes, salt_len);
        }

        /* Add key for numbers not divisible by 7. */
        if (cnt % 7 != 0) {
            HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
        }

        /* Add key or last result. */
        if ((cnt & 1) != 0) {
            HASH_Update(ctx, alt_result, 64);
        } else {
            HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
        }

        /* Create intermediate result. */
        HASH_End(ctx, alt_result, &part, HASH_ResultLenContext(ctx));
    }

    /* Now we can construct the result string.
     * It consists of three parts. */
    if (buflen <= SALT_PREF_SIZE) {
        ret = ERANGE;
        goto done;
    }

    cp = stpncpy(buffer, sha512_salt_prefix, SALT_PREF_SIZE);
    buflen -= SALT_PREF_SIZE;

    if (rounds_custom) {
        n = snprintf(cp, buflen, "%s%zu$",
                     sha512_rounds_prefix, rounds);
        if (n < 0 || n >= buflen) {
            ret = ERANGE;
            goto done;
        }
        cp += n;
        buflen -= n;
    }

    if (buflen <= salt_len + 1) {
        ret = ERANGE;
        goto done;
    }
    cp = stpncpy(cp, salt, salt_len);
    *cp++ = '$';
    buflen -= salt_len + 1;

    /* fuzzyfill the base 64 string */
    p1 = 0;
    p2 = 21;
    p3 = 42;
    for (n = 0; n < 21; n++) {
        b64_from_24bit(&cp, &buflen, 4, alt_result[p1], alt_result[p2], alt_result[p3]);
        if (buflen == 0) {
            ret = ERANGE;
            goto done;
        }
        pt = p1;
        p1 = p2 + 1;
        p2 = p3 + 1;
        p3 = pt + 1;
    }
    /* 64th and last byte */
    b64_from_24bit(&cp, &buflen, 2, 0, 0, alt_result[p3]);
    if (buflen == 0) {
        ret = ERANGE;
        goto done;
    }

    *cp = '\0';
    ret = EOK;

done:
    /* Clear the buffer for the intermediate result so that people attaching
     * to processes or reading core dumps cannot get any information. We do it
     * in this way to clear correct_words[] inside the SHA512 implementation
     * as well.  */
    if (ctx) HASH_Destroy(ctx);
    if (alt_ctx) HASH_Destroy(alt_ctx);
    if (p_bytes) memset(p_bytes, '\0', key_len);
    if (s_bytes) memset(s_bytes, '\0', salt_len);
    if (copied_key) memset(copied_key, '\0', key_len);
    if (copied_salt) memset(copied_salt, '\0', salt_len);
    memset(temp_result, '\0', sizeof(temp_result));

    return ret;
}

int s3crypt_sha512(TALLOC_CTX *memctx,
                   const char *key, const char *salt, char **_hash)
{
    char *hash;
    int hlen = (sizeof (sha512_salt_prefix) - 1
                + sizeof (sha512_rounds_prefix) + 9 + 1
                + strlen (salt) + 1 + 86 + 1);
    int ret;

    hash = talloc_size(memctx, hlen);
    if (!hash) return ENOMEM;

    ret = sha512_crypt_r(key, salt, hash, hlen);
    if (ret) return ret;

    *_hash = hash;
    return ret;
}

#define SALT_RAND_LEN 12

int s3crypt_gen_salt(TALLOC_CTX *memctx, char **_salt)
{
    uint8_t rb[SALT_RAND_LEN];
    char *salt, *cp;
    size_t slen;
    int ret;

    ret = nspr_nss_init();
    if (ret != EOK) {
        return EIO;
    }

    salt = talloc_size(memctx, SALT_LEN_MAX + 1);
    if (!salt) {
        return ENOMEM;
    }

    ret = PK11_GenerateRandom(rb, SALT_RAND_LEN);
    if (ret != SECSuccess) {
        return EIO;
    }

    slen = SALT_LEN_MAX;
    cp = salt;
    b64_from_24bit(&cp, &slen, 4, rb[0], rb[1], rb[2]);
    b64_from_24bit(&cp, &slen, 4, rb[3], rb[4], rb[5]);
    b64_from_24bit(&cp, &slen, 4, rb[6], rb[7], rb[8]);
    b64_from_24bit(&cp, &slen, 4, rb[9], rb[10], rb[11]);
    *cp = '\0';

    *_salt = salt;

    return EOK;
}