summaryrefslogtreecommitdiffstats
path: root/security/nss/lib/smime/cmsutil.c
blob: 713b94aacbb6864e4590e4365bf83c6f6d454f10 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*
 * CMS miscellaneous utility functions.
 */

#include "cmslocal.h"

#include "cert.h"
#include "keyhi.h"
#include "secasn1.h"
#include "secitem.h"
#include "secoid.h"
#include "pk11func.h"
#include "secerr.h"
#include "sechash.h"

/*
 * NSS_CMSArray_SortByDER - sort array of objects by objects' DER encoding
 *
 * make sure that the order of the objects guarantees valid DER (which must be
 * in lexigraphically ascending order for a SET OF); if reordering is necessary it
 * will be done in place (in objs).
 */
SECStatus
NSS_CMSArray_SortByDER(void **objs, const SEC_ASN1Template *objtemplate, void **objs2)
{
    PLArenaPool *poolp;
    int num_objs;
    SECItem **enc_objs;
    SECStatus rv = SECFailure;
    int i;

    if (objs == NULL) /* already sorted */
        return SECSuccess;

    num_objs = NSS_CMSArray_Count((void **)objs);
    if (num_objs == 0 || num_objs == 1) /* already sorted. */
        return SECSuccess;

    poolp = PORT_NewArena(1024); /* arena for temporaries */
    if (poolp == NULL)
        return SECFailure; /* no memory; nothing we can do... */

    /*
     * Allocate arrays to hold the individual encodings which we will use
     * for comparisons and the reordered attributes as they are sorted.
     */
    enc_objs = (SECItem **)PORT_ArenaZAlloc(poolp, (num_objs + 1) * sizeof(SECItem *));
    if (enc_objs == NULL)
        goto loser;

    /* DER encode each individual object. */
    for (i = 0; i < num_objs; i++) {
        enc_objs[i] = SEC_ASN1EncodeItem(poolp, NULL, objs[i], objtemplate);
        if (enc_objs[i] == NULL)
            goto loser;
    }
    enc_objs[num_objs] = NULL;

    /* now compare and sort objs by the order of enc_objs */
    NSS_CMSArray_Sort((void **)enc_objs, NSS_CMSUtil_DERCompare, objs, objs2);

    rv = SECSuccess;

loser:
    PORT_FreeArena(poolp, PR_FALSE);
    return rv;
}

/*
 * NSS_CMSUtil_DERCompare - for use with NSS_CMSArray_Sort to
 *  sort arrays of SECItems containing DER
 */
int
NSS_CMSUtil_DERCompare(void *a, void *b)
{
    SECItem *der1 = (SECItem *)a;
    SECItem *der2 = (SECItem *)b;
    unsigned int j;

    /*
     * Find the lowest (lexigraphically) encoding.  One that is
     * shorter than all the rest is known to be "less" because each
     * attribute is of the same type (a SEQUENCE) and so thus the
     * first octet of each is the same, and the second octet is
     * the length (or the length of the length with the high bit
     * set, followed by the length, which also works out to always
     * order the shorter first).  Two (or more) that have the
     * same length need to be compared byte by byte until a mismatch
     * is found.
     */
    if (der1->len != der2->len)
        return (der1->len < der2->len) ? -1 : 1;

    for (j = 0; j < der1->len; j++) {
        if (der1->data[j] == der2->data[j])
            continue;
        return (der1->data[j] < der2->data[j]) ? -1 : 1;
    }
    return 0;
}

/*
 * NSS_CMSAlgArray_GetIndexByAlgID - find a specific algorithm in an array of
 * algorithms.
 *
 * algorithmArray - array of algorithm IDs
 * algid - algorithmid of algorithm to pick
 *
 * Returns:
 *  An integer containing the index of the algorithm in the array or -1 if
 *  algorithm was not found.
 */
int
NSS_CMSAlgArray_GetIndexByAlgID(SECAlgorithmID **algorithmArray, SECAlgorithmID *algid)
{
    int i;

    if (algorithmArray == NULL || algorithmArray[0] == NULL)
        return -1;

    for (i = 0; algorithmArray[i] != NULL; i++) {
        if (SECOID_CompareAlgorithmID(algorithmArray[i], algid) == SECEqual)
            break; /* bingo */
    }

    if (algorithmArray[i] == NULL)
        return -1; /* not found */

    return i;
}

/*
 * NSS_CMSAlgArray_GetIndexByAlgTag - find a specific algorithm in an array of
 * algorithms.
 *
 * algorithmArray - array of algorithm IDs
 * algtag - algorithm tag of algorithm to pick
 *
 * Returns:
 *  An integer containing the index of the algorithm in the array or -1 if
 *  algorithm was not found.
 */
int
NSS_CMSAlgArray_GetIndexByAlgTag(SECAlgorithmID **algorithmArray,
                                 SECOidTag algtag)
{
    SECOidData *algid;
    int i = -1;

    if (algorithmArray == NULL || algorithmArray[0] == NULL)
        return i;

#ifdef ORDER_N_SQUARED
    for (i = 0; algorithmArray[i] != NULL; i++) {
        algid = SECOID_FindOID(&(algorithmArray[i]->algorithm));
        if (algid->offset == algtag)
            break; /* bingo */
    }
#else
    algid = SECOID_FindOIDByTag(algtag);
    if (!algid)
        return i;
    for (i = 0; algorithmArray[i] != NULL; i++) {
        if (SECITEM_ItemsAreEqual(&algorithmArray[i]->algorithm, &algid->oid))
            break; /* bingo */
    }
#endif

    if (algorithmArray[i] == NULL)
        return -1; /* not found */

    return i;
}

/*
 * Map a sign algorithm to a digest algorithm.
 * This is used to handle incorrectly formatted packages sent to us
 * from Windows 2003.
 */
SECOidTag
NSS_CMSUtil_MapSignAlgs(SECOidTag signAlg)
{
    switch (signAlg) {
        case SEC_OID_PKCS1_MD2_WITH_RSA_ENCRYPTION:
            return SEC_OID_MD2;
            break;
        case SEC_OID_PKCS1_MD5_WITH_RSA_ENCRYPTION:
            return SEC_OID_MD5;
            break;
        case SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION:
        case SEC_OID_ANSIX962_ECDSA_SHA1_SIGNATURE:
        case SEC_OID_ANSIX9_DSA_SIGNATURE_WITH_SHA1_DIGEST:
            return SEC_OID_SHA1;
            break;
        case SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION:
        case SEC_OID_ANSIX962_ECDSA_SHA256_SIGNATURE:
            return SEC_OID_SHA256;
            break;
        case SEC_OID_PKCS1_SHA384_WITH_RSA_ENCRYPTION:
        case SEC_OID_ANSIX962_ECDSA_SHA384_SIGNATURE:
            return SEC_OID_SHA384;
            break;
        case SEC_OID_PKCS1_SHA512_WITH_RSA_ENCRYPTION:
        case SEC_OID_ANSIX962_ECDSA_SHA512_SIGNATURE:
            return SEC_OID_SHA512;
            break;
        default:
            break;
    }
    /* not one of the algtags incorrectly sent to us*/
    return signAlg;
}

const SECHashObject *
NSS_CMSUtil_GetHashObjByAlgID(SECAlgorithmID *algid)
{
    SECOidTag oidTag = SECOID_FindOIDTag(&(algid->algorithm));
    const SECHashObject *digobj = HASH_GetHashObjectByOidTag(oidTag);

    return digobj;
}

const SEC_ASN1Template *
NSS_CMSUtil_GetTemplateByTypeTag(SECOidTag type)
{
    const SEC_ASN1Template *template;
    extern const SEC_ASN1Template NSSCMSSignedDataTemplate[];
    extern const SEC_ASN1Template NSSCMSEnvelopedDataTemplate[];
    extern const SEC_ASN1Template NSSCMSEncryptedDataTemplate[];
    extern const SEC_ASN1Template NSSCMSDigestedDataTemplate[];

    switch (type) {
        case SEC_OID_PKCS7_SIGNED_DATA:
            template = NSSCMSSignedDataTemplate;
            break;
        case SEC_OID_PKCS7_ENVELOPED_DATA:
            template = NSSCMSEnvelopedDataTemplate;
            break;
        case SEC_OID_PKCS7_ENCRYPTED_DATA:
            template = NSSCMSEncryptedDataTemplate;
            break;
        case SEC_OID_PKCS7_DIGESTED_DATA:
            template = NSSCMSDigestedDataTemplate;
            break;
        default:
            template = NSS_CMSType_GetTemplate(type);
            break;
    }
    return template;
}

size_t
NSS_CMSUtil_GetSizeByTypeTag(SECOidTag type)
{
    size_t size;

    switch (type) {
        case SEC_OID_PKCS7_SIGNED_DATA:
            size = sizeof(NSSCMSSignedData);
            break;
        case SEC_OID_PKCS7_ENVELOPED_DATA:
            size = sizeof(NSSCMSEnvelopedData);
            break;
        case SEC_OID_PKCS7_ENCRYPTED_DATA:
            size = sizeof(NSSCMSEncryptedData);
            break;
        case SEC_OID_PKCS7_DIGESTED_DATA:
            size = sizeof(NSSCMSDigestedData);
            break;
        default:
            size = NSS_CMSType_GetContentSize(type);
            break;
    }
    return size;
}

NSSCMSContentInfo *
NSS_CMSContent_GetContentInfo(void *msg, SECOidTag type)
{
    NSSCMSContent c;
    NSSCMSContentInfo *cinfo = NULL;

    if (!msg)
        return cinfo;
    c.pointer = msg;
    switch (type) {
        case SEC_OID_PKCS7_SIGNED_DATA:
            cinfo = &(c.signedData->contentInfo);
            break;
        case SEC_OID_PKCS7_ENVELOPED_DATA:
            cinfo = &(c.envelopedData->contentInfo);
            break;
        case SEC_OID_PKCS7_ENCRYPTED_DATA:
            cinfo = &(c.encryptedData->contentInfo);
            break;
        case SEC_OID_PKCS7_DIGESTED_DATA:
            cinfo = &(c.digestedData->contentInfo);
            break;
        default:
            cinfo = NULL;
            if (NSS_CMSType_IsWrapper(type)) {
                cinfo = &(c.genericData->contentInfo);
            }
    }
    return cinfo;
}

const char *
NSS_CMSUtil_VerificationStatusToString(NSSCMSVerificationStatus vs)
{
    switch (vs) {
        case NSSCMSVS_Unverified:
            return "Unverified";
        case NSSCMSVS_GoodSignature:
            return "GoodSignature";
        case NSSCMSVS_BadSignature:
            return "BadSignature";
        case NSSCMSVS_DigestMismatch:
            return "DigestMismatch";
        case NSSCMSVS_SigningCertNotFound:
            return "SigningCertNotFound";
        case NSSCMSVS_SigningCertNotTrusted:
            return "SigningCertNotTrusted";
        case NSSCMSVS_SignatureAlgorithmUnknown:
            return "SignatureAlgorithmUnknown";
        case NSSCMSVS_SignatureAlgorithmUnsupported:
            return "SignatureAlgorithmUnsupported";
        case NSSCMSVS_MalformedSignature:
            return "MalformedSignature";
        case NSSCMSVS_ProcessingError:
            return "ProcessingError";
        default:
            return "Unknown";
    }
}

SECStatus
NSS_CMSDEREncode(NSSCMSMessage *cmsg, SECItem *input, SECItem *derOut,
                 PLArenaPool *arena)
{
    NSSCMSEncoderContext *ecx;
    SECStatus rv = SECSuccess;
    if (!cmsg || !derOut || !arena) {
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
        return SECFailure;
    }
    ecx = NSS_CMSEncoder_Start(cmsg, 0, 0, derOut, arena, 0, 0, 0, 0, 0, 0);
    if (!ecx) {
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
        return SECFailure;
    }
    if (input) {
        rv = NSS_CMSEncoder_Update(ecx, (const char *)input->data, input->len);
        if (rv) {
            PORT_SetError(SEC_ERROR_BAD_DATA);
        }
    }
    rv |= NSS_CMSEncoder_Finish(ecx);
    if (rv) {
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
    }
    return rv;
}