summaryrefslogtreecommitdiffstats
path: root/rsawrapr.c
diff options
context:
space:
mode:
authorRob Crittenden <rcritten@redhat.com>2009-05-22 17:56:11 -0400
committerRob Crittenden <rcritten@redhat.com>2009-05-22 17:56:11 -0400
commit5409801c7195dc38665ec4e29c233bf1e9ea53e7 (patch)
tree38476d28439a07c4a47bd3e7ba84a9a188e404cb /rsawrapr.c
downloadpemnss-5409801c7195dc38665ec4e29c233bf1e9ea53e7.tar.gz
pemnss-5409801c7195dc38665ec4e29c233bf1e9ea53e7.tar.xz
pemnss-5409801c7195dc38665ec4e29c233bf1e9ea53e7.zip
Initial import of code. This is equivalent to the 20080124 tarball in
the Fedora nss package tree.
Diffstat (limited to 'rsawrapr.c')
-rw-r--r--rsawrapr.c924
1 files changed, 924 insertions, 0 deletions
diff --git a/rsawrapr.c b/rsawrapr.c
new file mode 100644
index 0000000..5ac4f39
--- /dev/null
+++ b/rsawrapr.c
@@ -0,0 +1,924 @@
+/*
+ * PKCS#1 encoding and decoding functions.
+ * This file is believed to contain no code licensed from other parties.
+ *
+ * ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is the Netscape security libraries.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1994-2000
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ * Rob Crittenden (rcritten@redhat.com)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+/* $Id: $ */
+
+#include "ckpem.h"
+#include "blapi.h"
+#include "softoken.h"
+#include "sechash.h"
+#include "base.h"
+
+#include "secerr.h"
+
+#define RSA_BLOCK_MIN_PAD_LEN 8
+#define RSA_BLOCK_FIRST_OCTET 0x00
+#define RSA_BLOCK_PRIVATE0_PAD_OCTET 0x00
+#define RSA_BLOCK_PRIVATE_PAD_OCTET 0xff
+#define RSA_BLOCK_AFTER_PAD_OCTET 0x00
+
+#define OAEP_SALT_LEN 8
+#define OAEP_PAD_LEN 8
+#define OAEP_PAD_OCTET 0x00
+
+#define FLAT_BUFSIZE 512 /* bytes to hold flattened SHA1Context. */
+
+unsigned
+pem_PublicModulusLen(NSSLOWKEYPublicKey *pubk)
+{
+ unsigned char b0;
+
+ /* interpret modulus length as key strength... in
+ * fortezza that's the public key length */
+
+ switch (pubk->keyType) {
+ case pemLOWKEYRSAKey:
+ b0 = pubk->u.rsa.modulus.data[0];
+ return b0 ? pubk->u.rsa.modulus.len : pubk->u.rsa.modulus.len - 1;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static SHA1Context *SHA1_CloneContext(SHA1Context * original)
+{
+ SHA1Context *clone = NULL;
+ unsigned char *pBuf;
+ int sha1ContextSize = SHA1_FlattenSize(original);
+ SECStatus frv;
+ unsigned char buf[FLAT_BUFSIZE];
+
+ PORT_Assert(sizeof buf >= sha1ContextSize);
+ if (sizeof buf >= sha1ContextSize) {
+ pBuf = buf;
+ } else {
+ pBuf = nss_ZAlloc(NULL, sha1ContextSize);
+ if (!pBuf)
+ goto done;
+ }
+
+ frv = SHA1_Flatten(original, pBuf);
+ if (frv == SECSuccess) {
+ clone = SHA1_Resurrect(pBuf, NULL);
+ memset(pBuf, 0, sha1ContextSize);
+ }
+ done:
+ if (pBuf != buf)
+ nss_ZFreeIf(pBuf);
+ return clone;
+}
+
+/*
+ * Modify data by XORing it with a special hash of salt.
+ */
+static SECStatus
+oaep_xor_with_h1(unsigned char *data, unsigned int datalen,
+ unsigned char *salt, unsigned int saltlen)
+{
+ SHA1Context *sha1cx;
+ unsigned char *dp, *dataend;
+ unsigned char end_octet;
+
+ sha1cx = SHA1_NewContext();
+ if (sha1cx == NULL) {
+ return SECFailure;
+ }
+
+ /*
+ * Get a hash of salt started; we will use it several times,
+ * adding in a different end octet (x00, x01, x02, ...).
+ */
+ SHA1_Begin(sha1cx);
+ SHA1_Update(sha1cx, salt, saltlen);
+ end_octet = 0;
+
+ dp = data;
+ dataend = data + datalen;
+
+ while (dp < dataend) {
+ SHA1Context *sha1cx_h1;
+ unsigned int sha1len, sha1off;
+ unsigned char sha1[SHA1_LENGTH];
+
+ /*
+ * Create hash of (salt || end_octet)
+ */
+ sha1cx_h1 = SHA1_CloneContext(sha1cx);
+ SHA1_Update(sha1cx_h1, &end_octet, 1);
+ SHA1_End(sha1cx_h1, sha1, &sha1len, sizeof(sha1));
+ SHA1_DestroyContext(sha1cx_h1, PR_TRUE);
+ PORT_Assert(sha1len == SHA1_LENGTH);
+
+ /*
+ * XOR that hash with the data.
+ * When we have fewer than SHA1_LENGTH octets of data
+ * left to xor, use just the low-order ones of the hash.
+ */
+ sha1off = 0;
+ if ((dataend - dp) < SHA1_LENGTH)
+ sha1off = SHA1_LENGTH - (dataend - dp);
+ while (sha1off < SHA1_LENGTH)
+ *dp++ ^= sha1[sha1off++];
+
+ /*
+ * Bump for next hash chunk.
+ */
+ end_octet++;
+ }
+
+ SHA1_DestroyContext(sha1cx, PR_TRUE);
+ return SECSuccess;
+}
+
+/*
+ * Modify salt by XORing it with a special hash of data.
+ */
+static SECStatus
+oaep_xor_with_h2(unsigned char *salt, unsigned int saltlen,
+ unsigned char *data, unsigned int datalen)
+{
+ unsigned char sha1[SHA1_LENGTH];
+ unsigned char *psalt, *psha1, *saltend;
+ SECStatus rv;
+
+ /*
+ * Create a hash of data.
+ */
+ rv = SHA1_HashBuf(sha1, data, datalen);
+ if (rv != SECSuccess) {
+ return rv;
+ }
+
+ /*
+ * XOR the low-order octets of that hash with salt.
+ */
+ PORT_Assert(saltlen <= SHA1_LENGTH);
+ saltend = salt + saltlen;
+ psalt = salt;
+ psha1 = sha1 + SHA1_LENGTH - saltlen;
+ while (psalt < saltend) {
+ *psalt++ ^= *psha1++;
+ }
+
+ return SECSuccess;
+}
+
+/*
+ * Format one block of data for public/private key encryption using
+ * the rules defined in PKCS #1.
+ */
+static unsigned char *rsa_FormatOneBlock(unsigned modulusLen,
+ RSA_BlockType blockType,
+ SECItem * data)
+{
+ unsigned char *block;
+ unsigned char *bp;
+ int padLen;
+ int i;
+ SECStatus rv;
+
+ block = (unsigned char *) nss_ZAlloc(NULL, modulusLen);
+ if (block == NULL)
+ return NULL;
+
+ bp = block;
+
+ /*
+ * All RSA blocks start with two octets:
+ * 0x00 || BlockType
+ */
+ *bp++ = RSA_BLOCK_FIRST_OCTET;
+ *bp++ = (unsigned char) blockType;
+
+ switch (blockType) {
+
+ /*
+ * Blocks intended for private-key operation.
+ */
+ case RSA_BlockPrivate0: /* essentially unused */
+ case RSA_BlockPrivate: /* preferred method */
+ /*
+ * 0x00 || BT || Pad || 0x00 || ActualData
+ * 1 1 padLen 1 data->len
+ * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
+ */
+ padLen = modulusLen - data->len - 3;
+ PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
+ if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+ nsslibc_memset(bp,
+ blockType == RSA_BlockPrivate0
+ ? RSA_BLOCK_PRIVATE0_PAD_OCTET
+ : RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
+ bp += padLen;
+ *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
+ nsslibc_memcpy(bp, data->data, data->len);
+ break;
+
+ /*
+ * Blocks intended for public-key operation.
+ */
+ case RSA_BlockPublic:
+
+ /*
+ * 0x00 || BT || Pad || 0x00 || ActualData
+ * 1 1 padLen 1 data->len
+ * Pad is all non-zero random bytes.
+ */
+ padLen = modulusLen - data->len - 3;
+ PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
+ if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+ for (i = 0; i < padLen; i++) {
+ /* Pad with non-zero random data. */
+ do {
+ rv = RNG_GenerateGlobalRandomBytes(bp + i, 1);
+ } while (rv == SECSuccess
+ && bp[i] == RSA_BLOCK_AFTER_PAD_OCTET);
+ if (rv != SECSuccess) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+ }
+ bp += padLen;
+ *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
+ nsslibc_memcpy(bp, data->data, data->len);
+
+ break;
+
+ /*
+ * Blocks intended for public-key operation, using
+ * Optimal Asymmetric Encryption Padding (OAEP).
+ */
+ case RSA_BlockOAEP:
+ /*
+ * 0x00 || BT || Modified2(Salt) || Modified1(PaddedData)
+ * 1 1 OAEP_SALT_LEN OAEP_PAD_LEN + data->len [+ N]
+ *
+ * where:
+ * PaddedData is "Pad1 || ActualData [|| Pad2]"
+ * Salt is random data.
+ * Pad1 is all zeros.
+ * Pad2, if present, is random data.
+ * (The "modified" fields are all the same length as the original
+ * unmodified values; they are just xor'd with other values.)
+ *
+ * Modified1 is an XOR of PaddedData with a special octet
+ * string constructed of iterated hashing of Salt (see below).
+ * Modified2 is an XOR of Salt with the low-order octets of
+ * the hash of Modified1 (see farther below ;-).
+ *
+ * Whew!
+ */
+
+
+ /*
+ * Salt
+ */
+ rv = RNG_GenerateGlobalRandomBytes(bp, OAEP_SALT_LEN);
+ if (rv != SECSuccess) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+ bp += OAEP_SALT_LEN;
+
+ /*
+ * Pad1
+ */
+ nsslibc_memset(bp, OAEP_PAD_OCTET, OAEP_PAD_LEN);
+ bp += OAEP_PAD_LEN;
+
+ /*
+ * Data
+ */
+ nsslibc_memcpy(bp, data->data, data->len);
+ bp += data->len;
+
+ /*
+ * Pad2
+ */
+ if (bp < (block + modulusLen)) {
+ rv = RNG_GenerateGlobalRandomBytes(bp,
+ block - bp + modulusLen);
+ if (rv != SECSuccess) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+ }
+
+ /*
+ * Now we have the following:
+ * 0x00 || BT || Salt || PaddedData
+ * (From this point on, "Pad1 || Data [|| Pad2]" is treated
+ * as the one entity PaddedData.)
+ *
+ * We need to turn PaddedData into Modified1.
+ */
+ if (oaep_xor_with_h1(block + 2 + OAEP_SALT_LEN,
+ modulusLen - 2 - OAEP_SALT_LEN,
+ block + 2, OAEP_SALT_LEN) != SECSuccess) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+
+ /*
+ * Now we have:
+ * 0x00 || BT || Salt || Modified1(PaddedData)
+ *
+ * The remaining task is to turn Salt into Modified2.
+ */
+ if (oaep_xor_with_h2(block + 2, OAEP_SALT_LEN,
+ block + 2 + OAEP_SALT_LEN,
+ modulusLen - 2 - OAEP_SALT_LEN) !=
+ SECSuccess) {
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+
+ break;
+
+ default:
+ PORT_Assert(0);
+ nss_ZFreeIf(block);
+ return NULL;
+ }
+
+ return block;
+}
+
+static SECStatus
+rsa_FormatBlock(SECItem * result, unsigned modulusLen,
+ RSA_BlockType blockType, SECItem * data)
+{
+ /*
+ * XXX For now assume that the data length fits in a single
+ * XXX encryption block; the ASSERTs below force this.
+ * XXX To fix it, each case will have to loop over chunks whose
+ * XXX lengths satisfy the assertions, until all data is handled.
+ * XXX (Unless RSA has more to say about how to handle data
+ * XXX which does not fit in a single encryption block?)
+ * XXX And I do not know what the result is supposed to be,
+ * XXX so the interface to this function may need to change
+ * XXX to allow for returning multiple blocks, if they are
+ * XXX not wanted simply concatenated one after the other.
+ */
+
+ switch (blockType) {
+ case RSA_BlockPrivate0:
+ case RSA_BlockPrivate:
+ case RSA_BlockPublic:
+ /*
+ * 0x00 || BT || Pad || 0x00 || ActualData
+ *
+ * The "3" below is the first octet + the second octet + the 0x00
+ * octet that always comes just before the ActualData.
+ */
+ PORT_Assert(data->len <=
+ (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)));
+
+ result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
+ if (result->data == NULL) {
+ result->len = 0;
+ return SECFailure;
+ }
+ result->len = modulusLen;
+
+ break;
+
+ case RSA_BlockOAEP:
+ /*
+ * 0x00 || BT || M1(Salt) || M2(Pad1||ActualData[||Pad2])
+ *
+ * The "2" below is the first octet + the second octet.
+ * (The other fields do not contain the clear values, but are
+ * the same length as the clear values.)
+ */
+ PORT_Assert(data->len <= (modulusLen - (2 + OAEP_SALT_LEN
+ + OAEP_PAD_LEN)));
+
+ result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
+ if (result->data == NULL) {
+ result->len = 0;
+ return SECFailure;
+ }
+ result->len = modulusLen;
+
+ break;
+
+ case RSA_BlockRaw:
+ /*
+ * Pad || ActualData
+ * Pad is zeros. The application is responsible for recovering
+ * the actual data.
+ */
+ if (data->len > modulusLen) {
+ return SECFailure;
+ }
+ result->data = (unsigned char *) nss_ZAlloc(NULL, modulusLen);
+ result->len = modulusLen;
+ nsslibc_memcpy(result->data + (modulusLen - data->len), data->data,
+ data->len);
+ break;
+
+ default:
+ PORT_Assert(0);
+ result->data = NULL;
+ result->len = 0;
+ return SECFailure;
+ }
+
+ return SECSuccess;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+pem_RSA_Sign(pemLOWKEYPrivateKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int maxOutputLen,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv = SECSuccess;
+ unsigned int modulus_len = pem_PrivateModulusLen(key);
+ SECItem formatted;
+ SECItem unformatted;
+
+ if (maxOutputLen < modulus_len)
+ return SECFailure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ return SECFailure;
+
+ unformatted.len = input_len;
+ unformatted.data = input;
+ formatted.data = NULL;
+ rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockPrivate,
+ &unformatted);
+ if (rv != SECSuccess)
+ goto done;
+
+ rv = RSA_PrivateKeyOpDoubleChecked(&key->u.rsa, output,
+ formatted.data);
+ *output_len = modulus_len;
+
+ goto done;
+
+ done:
+ if (formatted.data != NULL)
+ nss_ZFreeIf(formatted.data);
+ return rv;
+}
+
+#if 0
+/* XXX Doesn't set error code */
+SECStatus
+RSA_CheckSign(NSSLOWKEYPublicKey * key,
+ unsigned char *sign,
+ unsigned int sign_len,
+ unsigned char *hash, unsigned int hash_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+ unsigned int i;
+ unsigned char *buffer;
+
+ modulus_len = pem_PublicModulusLen(key);
+ if (sign_len != modulus_len)
+ goto failure;
+ /*
+ * 0x00 || BT || Pad || 0x00 || ActualData
+ *
+ * The "3" below is the first octet + the second octet + the 0x00
+ * octet that always comes just before the ActualData.
+ */
+ if (hash_len > modulus_len - (3 + RSA_BLOCK_MIN_PAD_LEN))
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ buffer = (unsigned char *) nss_ZAlloc(NULL, modulus_len + 1);
+ if (!buffer)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
+ if (rv != SECSuccess)
+ goto loser;
+
+ /*
+ * check the padding that was used
+ */
+ if (buffer[0] != 0 || buffer[1] != 1)
+ goto loser;
+ for (i = 2; i < modulus_len - hash_len - 1; i++) {
+ if (buffer[i] != 0xff)
+ goto loser;
+ }
+ if (buffer[i] != 0)
+ goto loser;
+
+ /*
+ * make sure we get the same results
+ */
+ if (memcmp(buffer + modulus_len - hash_len, hash, hash_len) != 0)
+ goto loser;
+
+ nss_ZFreeIf(buffer);
+ return SECSuccess;
+
+ loser:
+ nss_ZFreeIf(buffer);
+ failure:
+ return SECFailure;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+RSA_CheckSignRecover(NSSLOWKEYPublicKey * key,
+ unsigned char *data,
+ unsigned int *data_len,
+ unsigned int max_output_len,
+ unsigned char *sign, unsigned int sign_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+ unsigned int i;
+ unsigned char *buffer;
+
+ if (sign_len != modulus_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ buffer = (unsigned char *) nss_ZAlloc(NULL, modulus_len + 1);
+ if (!buffer)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
+ if (rv != SECSuccess)
+ goto loser;
+ *data_len = 0;
+
+ /*
+ * check the padding that was used
+ */
+ if (buffer[0] != 0 || buffer[1] != 1)
+ goto loser;
+ for (i = 2; i < modulus_len; i++) {
+ if (buffer[i] == 0) {
+ *data_len = modulus_len - i - 1;
+ break;
+ }
+ if (buffer[i] != 0xff)
+ goto loser;
+ }
+ if (*data_len == 0)
+ goto loser;
+ if (*data_len > max_output_len)
+ goto loser;
+
+ /*
+ * make sure we get the same results
+ */
+ nsslibc_memcpy(data, buffer + modulus_len - *data_len, *data_len);
+
+ nss_ZFreeIf(buffer);
+ return SECSuccess;
+
+ loser:
+ nss_ZFreeIf(buffer);
+ failure:
+ return SECFailure;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+RSA_EncryptBlock(NSSLOWKEYPublicKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int max_output_len,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+ SECItem formatted;
+ SECItem unformatted;
+
+ formatted.data = NULL;
+ if (max_output_len < modulus_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ unformatted.len = input_len;
+ unformatted.data = input;
+ formatted.data = NULL;
+ rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockPublic,
+ &unformatted);
+ if (rv != SECSuccess)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, output, formatted.data);
+ if (rv != SECSuccess)
+ goto failure;
+
+ nss_ZFreeIf(formatted.data);
+ *output_len = modulus_len;
+ return SECSuccess;
+
+ failure:
+ if (formatted.data != NULL)
+ nss_ZFreeIf(formatted.data);
+ return SECFailure;
+}
+#endif
+
+/* XXX Doesn't set error code */
+SECStatus
+pem_RSA_DecryptBlock(pemLOWKEYPrivateKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int max_output_len,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PrivateModulusLen(key);
+ unsigned int i;
+ unsigned char *buffer;
+
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+ if (input_len != modulus_len)
+ goto failure;
+
+ buffer = (unsigned char *) nss_ZAlloc(NULL, modulus_len + 1);
+ if (!buffer)
+ goto failure;
+
+ rv = RSA_PrivateKeyOp(&key->u.rsa, buffer, input);
+ if (rv != SECSuccess) {
+ goto loser;
+ }
+
+ if (buffer[0] != 0 || buffer[1] != 2)
+ goto loser;
+ *output_len = 0;
+ for (i = 2; i < modulus_len; i++) {
+ if (buffer[i] == 0) {
+ *output_len = modulus_len - i - 1;
+ break;
+ }
+ }
+ if (*output_len == 0)
+ goto loser;
+ if (*output_len > max_output_len)
+ goto loser;
+
+ nsslibc_memcpy(output, buffer + modulus_len - *output_len, *output_len);
+
+ nss_ZFreeIf(buffer);
+ return SECSuccess;
+
+ loser:
+ nss_ZFreeIf(buffer);
+ failure:
+ return SECFailure;
+}
+
+#if 0
+/* XXX Doesn't set error code */
+/*
+ * added to make pkcs #11 happy
+ * RAW is RSA_X_509
+ */
+SECStatus
+pem_RSA_SignRaw(pemLOWKEYPrivateKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int maxOutputLen,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv = SECSuccess;
+ unsigned int modulus_len = pem_PrivateModulusLen(key);
+ SECItem formatted;
+ SECItem unformatted;
+
+ if (maxOutputLen < modulus_len)
+ return SECFailure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ return SECFailure;
+
+ unformatted.len = input_len;
+ unformatted.data = input;
+ formatted.data = NULL;
+ rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockRaw,
+ &unformatted);
+ if (rv != SECSuccess)
+ goto done;
+
+ rv = RSA_PrivateKeyOpDoubleChecked(&key->u.rsa, output,
+ formatted.data);
+ *output_len = modulus_len;
+
+ done:
+ if (formatted.data != NULL)
+ nss_ZFreeIf(formatted.data);
+ return rv;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+RSA_CheckSignRaw(NSSLOWKEYPublicKey * key,
+ unsigned char *sign,
+ unsigned int sign_len,
+ unsigned char *hash, unsigned int hash_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+ unsigned char *buffer;
+
+ if (sign_len != modulus_len)
+ goto failure;
+ if (hash_len > modulus_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ buffer = (unsigned char *) nss_ZAlloc(NULL, modulus_len + 1);
+ if (!buffer)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
+ if (rv != SECSuccess)
+ goto loser;
+
+ /*
+ * make sure we get the same results
+ */
+ /* NOTE: should we verify the leading zeros? */
+ if (memcmp(buffer + (modulus_len - hash_len), hash, hash_len) !=
+ 0)
+ goto loser;
+
+ nss_ZFreeIf(buffer);
+ return SECSuccess;
+
+ loser:
+ nss_ZFreeIf(buffer);
+ failure:
+ return SECFailure;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+RSA_CheckSignRecoverRaw(NSSLOWKEYPublicKey * key,
+ unsigned char *data,
+ unsigned int *data_len,
+ unsigned int max_output_len,
+ unsigned char *sign, unsigned int sign_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+
+ if (sign_len != modulus_len)
+ goto failure;
+ if (max_output_len < modulus_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, data, sign);
+ if (rv != SECSuccess)
+ goto failure;
+
+ *data_len = modulus_len;
+ return SECSuccess;
+
+ failure:
+ return SECFailure;
+}
+
+
+/* XXX Doesn't set error code */
+SECStatus
+RSA_EncryptRaw(NSSLOWKEYPublicKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int max_output_len,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PublicModulusLen(key);
+ SECItem formatted;
+ SECItem unformatted;
+
+ formatted.data = NULL;
+ if (max_output_len < modulus_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+
+ unformatted.len = input_len;
+ unformatted.data = input;
+ formatted.data = NULL;
+ rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockRaw,
+ &unformatted);
+ if (rv != SECSuccess)
+ goto failure;
+
+ rv = RSA_PublicKeyOp(&key->u.rsa, output, formatted.data);
+ if (rv != SECSuccess)
+ goto failure;
+
+ nss_ZFreeIf(formatted.data);
+ *output_len = modulus_len;
+ return SECSuccess;
+
+ failure:
+ if (formatted.data != NULL)
+ nss_ZFreeIf(formatted.data);
+ return SECFailure;
+}
+
+/* XXX Doesn't set error code */
+SECStatus
+pem_RSA_DecryptRaw(pemLOWKEYPrivateKey * key,
+ unsigned char *output,
+ unsigned int *output_len,
+ unsigned int max_output_len,
+ unsigned char *input, unsigned int input_len)
+{
+ SECStatus rv;
+ unsigned int modulus_len = pem_PrivateModulusLen(key);
+
+ if (modulus_len <= 0)
+ goto failure;
+ if (modulus_len > max_output_len)
+ goto failure;
+ PORT_Assert(key->keyType == pemLOWKEYRSAKey);
+ if (key->keyType != pemLOWKEYRSAKey)
+ goto failure;
+ if (input_len != modulus_len)
+ goto failure;
+
+ rv = RSA_PrivateKeyOp(&key->u.rsa, output, input);
+ if (rv != SECSuccess) {
+ goto failure;
+ }
+
+ *output_len = modulus_len;
+ return SECSuccess;
+
+ failure:
+ return SECFailure;
+}
+#endif