Bug 820695 - Tracker - TPS (ECC with nethsm) configuration failed at key generation

This patch calls with the right flags for each supported HSM to the new
certutil that addressed the following bug:
Bug 820684 - certutil support for EC on HSMs - need to call PK11_GenerateKeyPairWithOpFlags()

1 files changed, 44 insertions, 43 deletions

diff --git a/base/tps/lib/perl/PKI/TPS/NamePanel.pm b/base/tps/lib/perl/PKI/TPS/NamePanel.pm
index a474d80b9..d3ca4c19f 100755
--- a/base/tps/lib/perl/PKI/TPS/NamePanel.pm
+++ b/base/tps/lib/perl/PKI/TPS/NamePanel.pm
@@ -175,7 +175,7 @@ sub update
         if ($keytype eq "rsa") {
             $keysize = 2048;
         } elsif ($keytype eq "ecc") {
-            $keysize = 256;
+            $keysize = "nistp256";
         }
 
         if (($select eq "") || ($select eq "default")) {
@@ -188,14 +188,10 @@ sub update
             if ($size ne "") {
                 $keysize = $size;
             }
-            if (($keytype eq "ecc") && ($keysize ne 256)) {
-                &PKI::TPS::Wizard::debug_log("NamePanel: update got keysize from config= $keysize changing to 256, the only supported ECC strength");
-                $keysize = 256;
-            }
         }
 
         &PKI::TPS::Wizard::debug_log("NamePanel: update got key type $keytype");
-        my $req;
+        my $req = "";
         my $debug_req;
         my $filename = "/tmp/random.$$";
         `dd if\=/dev/urandom of\=\"$filename\" count\=256 bs\=1`;
@@ -207,10 +203,24 @@ sub update
             $req = `cat $tmpfile`;
             system("rm $tmpfile");
         } elsif ($keytype eq "ecc") {
-            #only support curve nistp256 for now
             my $tmpfile = "/tmp/req$$";
-            system("certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -R -s \"$cert_dn\" -k ec -q nistp256 -a -z $filename> $tmpfile");
+            # try first without specific flags
+            system("certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -R -s \"$cert_dn\" -k ec -q $keysize -a -z $filename> $tmpfile");
             $req = `cat $tmpfile`;
+
+            # try the flags that work with nethsm
+            if ($req eq "") {
+                system("certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -R --keyAttrFlags \"token,private,sensitive,unextractable\" --keyOpFlagsOff derive -s \"$cert_dn\" -k ec -q $keysize -a -z $filename> $tmpfile");
+                $req = `cat $tmpfile`;
+            }
+            # try the flags that work with lunasa
+            if ($req eq "") {
+                system("certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -R --keyAttrFlags \"private,unextractable\" --keyOpFlagsOff derive -s \"$cert_dn\" -k ec -q $keysize -a -z $filename> $tmpfile");
+                $req = `cat $tmpfile`;
+            }
+            if ($req eq "") {
+                &PKI::TPS::Wizard::debug_log("NamePanel: key generation failed on $tokenname.  Please check to see if this is a supported hardware.");
+            }
             system("rm $tmpfile");
         } else {
             &PKI::TPS::Wizard::debug_log("NamePanel: update unsupported keytype $keytype");
@@ -294,9 +304,11 @@ GEN_CERT:
                     $https_ee_port = $sdom_url->port;
                 }
                 if ($changed eq "true") {
+                # nickname changed is true, using token passwd for calling sslget
 $req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"$token_pwd\" -v -n \"$sslnickname\" -r \"/ca/ee/ca/profileSubmit\" $host:$https_ee_port";
 $debug_req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"(sensitive)\" -v -n \"$sslnickname\" -r \"/ca/ee/ca/profileSubmit\" $host:$https_ee_port";
                 } else {
+                # nickname changed is false, using internal passwd for calling sslget
 $req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"$db_password\" -v -n \"$sslnickname\" -r \"/ca/ee/ca/profileSubmit\" $host:$https_ee_port";
 $debug_req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"(sensitive)\" -v -n \"$sslnickname\" -r \"/ca/ee/ca/profileSubmit\" $host:$https_ee_port";
                 }
@@ -367,7 +379,12 @@ $debug_req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"(sen
             }
 
             &PKI::TPS::Wizard::debug_log("NamePanel: update: try to import cert from $cert_fn");
-            $tmp = `certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -A -n "$nickname" -t "u,u,u" -a -i $cert_fn`;
+            if ($certtag ne "audit_signing") {
+                $tmp = `certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -A -n "$nickname" -t "u,u,u" -a -i $cert_fn`;
+            } else {
+               $tmp = `certutil -d $instanceDir/alias $hw -f $instanceDir/conf/.pwfile -A -n "$nickname" -t "u,u,Pu" -a -i $cert_fn`;
+            }
+
             # changed the cert, need to change nickname too, if necessary
             if ($hw ne "") {
                 if ($certtag eq "sslserver") {
@@ -375,13 +392,15 @@ $debug_req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"(sen
                         $::config->put("preop.cert.$certtag.nickname", "$tk$nickname");
                     }
                     $changed = "true";
-                }
-                if ($certtag eq "subsystem") {
+                } elsif ($certtag eq "subsystem") {
                     &PKI::TPS::Wizard::debug_log("NamePanel: update: sslnickname changed");
                     $::config->put("preop.cert.$certtag.nickname", "$tk$nickname");
                     $::config->put("conn.ca1.clientNickname", "$tk$nickname");
                     $::config->put("conn.drm1.clientNickname", "$tk$nickname");
                     $::config->put("conn.tks1.clientNickname", "$tk$nickname");
+                } else {
+                    &PKI::TPS::Wizard::debug_log("NamePanel: update: $certtag changed");
+                    $::config->put("preop.cert.$certtag.nickname", "$tk$nickname");
                 }
                 $::config->commit();
              } else {
@@ -405,38 +424,20 @@ $debug_req = "/usr/bin/sslget -e \"$params\" -d \"$instanceDir/alias\" -p \"(sen
     my $selftestNickname = $::config->get( "preop.cert.subsystem.nickname" );
     my $selftestNickname_sslserver = $::config->get( "preop.cert.sslserver.nickname" );
     my $selftestNickname_audit_signing = $::config->get( "preop.cert.audit_signing.nickname" );
-    if ($hw ne "") {
-        $::config->put( "selftests.plugin.TPSPresence.nickname",
-                        "$tk$selftestNickname" );
-        $::config->put( "selftests.plugin.TPSValidity.nickname", 
-                        "$tk$selftestNickname" );
-
-        $::config->put( "tps.cert.sslserver.nickname",
-                        "$tk$selftestNickname_sslserver" );
-        $::config->put( "tps.cert.subsystem.nickname",
-                        "$tk$selftestNickname" );
-        $::config->put( "tps.cert.audit_signing.nickname",
-                        "$tk$selftestNickname_audit_signing" );
-
-        $::config->put( "logging.audit.signedAuditCertNickname",
-                        "$tk$selftestNickname_audit_signing" );
-    } else {
-        $::config->put( "selftests.plugin.TPSPresence.nickname",
-                        "$selftestNickname" );
-        $::config->put( "selftests.plugin.TPSValidity.nickname", 
-                        "$selftestNickname" );
-
-        $::config->put( "tps.cert.sslserver.nickname",
-                        "$selftestNickname_sslserver" );
-        $::config->put( "tps.cert.subsystem.nickname",
-                        "$selftestNickname" );
-        $::config->put( "tps.cert.audit_signing.nickname",
-                        "$selftestNickname_audit_signing" );
-
-        $::config->put( "logging.audit.signedAuditCertNickname",
-                        "$selftestNickname_audit_signing" );
-    }
-    $::config->commit();
+    $::config->put( "selftests.plugin.TPSPresence.nickname",
+                    "$selftestNickname" );
+    $::config->put( "selftests.plugin.TPSValidity.nickname", 
+                    "$selftestNickname" );
+
+    $::config->put( "tps.cert.sslserver.nickname",
+                    "$selftestNickname_sslserver" );
+    $::config->put( "tps.cert.subsystem.nickname",
+                    "$selftestNickname" );
+    $::config->put( "tps.cert.audit_signing.nickname",
+                    "$selftestNickname_audit_signing" );
+
+    $::config->put( "logging.audit.signedAuditCertNickname",
+                    "$selftestNickname_audit_signing" );
 
 DONE:
     $::config->put("preop.namepanel.done", "true");