Port digests to AF_ALG

3 files changed, 331 insertions, 192 deletions

diff --git a/Makefile.am b/Makefile.am
index 26a8d16..5c74dae 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -40,7 +40,8 @@ TESTS = tests/digests tests/rsa tests/symm_ciphers
 ## Rules
 noinst_PROGRAMS = $(TESTS)
 
-lib_libncrypto_la_SOURCES = lib/ncrypto.c lib/ncrypto_local.c lib/ncrypto_nss.c
+lib_libncrypto_la_SOURCES = lib/ncrypto.c lib/ncrypto_alg.c \
+	lib/ncrypto_local.c lib/ncrypto_nss.c
 lib_libncrypto_la_LDFLAGS = -version-info 0:0:0 $(NSS_LIBS) $(OPENSSL_LIBS)
 
 tests_digests_LDADD = lib/libncrypto.la $(GLIB_LIBS)
diff --git a/lib/ncrypto_alg.c b/lib/ncrypto_alg.c
new file mode 100644
index 0000000..20ef567
--- /dev/null
+++ b/lib/ncrypto_alg.c
@@ -0,0 +1,329 @@
+/* AF_ALG crypto implementation.
+
+Copyright 2010 Red Hat, Inc.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+Red Hat author: Miloslav Trmač <mitr@redhat.com> */
+#include <config.h>
+
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <glib.h>
+#include <linux/if_alg.h>
+
+#include <ncrypto/ncrypto.h>
+
+/* This is in <linux/socket.h>, but not exposed by it to user-space. */
+#define AF_ALG          38      /* Algorithm sockets            */
+
+ /* Helpers */
+
+static CK_RV
+ckr_errno (void)
+{
+  /* FIXME: Something better */
+  return CKR_GENERAL_ERROR;
+}
+
+ /* Digest handling */
+
+/* Semantically, md_size_ptr should be "size_t *", but values will fit into
+   "int *" just as fine. */
+static CK_RV
+digest_params_from_mech (CK_MECHANISM_TYPE mech,
+			 const struct sockaddr_alg **sa_ptr, int *md_size_ptr)
+{
+  const struct sockaddr_alg *sa;
+  int md_size;
+
+  switch (mech)
+    {
+#define E(MECH, NAME, SIZE)						\
+    case CKM_##MECH:							\
+      {									\
+	static const struct sockaddr_alg addr =				\
+	  { .salg_family = AF_ALG, .salg_type = "hash", .salg_name = (NAME) }; \
+									\
+	sa = &addr;							\
+	md_size = (SIZE);						\
+	break;								\
+      }									\
+
+      E (MD5, "md5", 16);
+      E (SHA_1, "sha1", 20);
+      E (SHA224, "sha224", 28);
+      E (SHA256, "sha256", 32);
+      E (SHA384, "sha384", 48);
+      E (SHA512, "sha512", 64);
+#undef E
+    default:
+      g_return_val_if_reached (CKR_MECHANISM_INVALID);
+    }
+  *sa_ptr = sa;
+  *md_size_ptr = md_size;
+  return CKR_OK;
+}
+
+struct ncr_digest_session
+{
+  int fd;
+  int md_size;			/* size_t semantically */
+  /* Debugging only */
+  enum { NDS_NEW, NDS_INITIALIZED, NDS_UPDATED, NDS_FINISHED } state;
+};
+
+CK_RV
+ncr_digest_alloc (struct ncr_digest_session **sess, CK_MECHANISM_TYPE mech)
+{
+  struct ncr_digest_session *s;
+  const struct sockaddr_alg *sa;
+  int top_fd;
+  CK_RV res;
+
+  g_return_val_if_fail (sess != NULL, CKR_ARGUMENTS_BAD);
+
+  s = malloc (sizeof (*s));
+  if (s == NULL)
+    return CKR_HOST_MEMORY;
+
+  res = digest_params_from_mech (mech, &sa, &s->md_size);
+  if (res != CKR_OK)
+    goto err_s;
+
+  top_fd = socket (AF_ALG, SOCK_SEQPACKET, 0);
+  if (top_fd == -1)
+    {
+      res = ckr_errno ();
+      goto err_s;
+    }
+  if (bind (top_fd, (struct sockaddr *)sa, sizeof (*sa)) != 0)
+    {
+      res = ckr_errno ();
+      goto err_top_fd;
+    }
+
+  s->fd = accept (top_fd, NULL, 0);
+  if (s->fd == -1)
+    {
+      res = ckr_errno ();
+      goto err_top_fd;
+    }
+  (void)close (top_fd);
+
+  s->state = NDS_NEW;
+  *sess = s;
+  return CKR_OK;
+
+ err_top_fd:
+  (void)close (top_fd);
+ err_s:
+  free (s);
+  return res;
+}
+
+CK_RV
+ncr_digest_free (struct ncr_digest_session *sess)
+{
+  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
+
+  (void)close (sess->fd);
+  free (sess);
+  return CKR_OK;
+}
+
+CK_RV
+ncr_digest_init (struct ncr_digest_session *sess)
+{
+  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
+  g_return_val_if_fail (sess->state == NDS_NEW || sess->state == NDS_FINISHED,
+			CKR_OPERATION_ACTIVE);
+  /* Nothing to do. */
+  sess->state = NDS_INITIALIZED;
+  return CKR_OK;
+}
+
+CK_RV
+ncr_digest_update (struct ncr_digest_session *sess, const void *data,
+		   size_t size)
+{
+  ssize_t res;
+
+  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
+  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
+  g_return_val_if_fail (size <= SSIZE_MAX, CKR_ARGUMENTS_BAD);
+  g_return_val_if_fail (sess->state == NDS_INITIALIZED
+			|| sess->state == NDS_UPDATED,
+			CKR_OPERATION_NOT_INITIALIZED);
+
+  res = sendto (sess->fd, data, size, MSG_MORE, NULL, 0);
+  if (res == -1)
+    return ckr_errno ();
+  if (res != (ssize_t)size) 	/* "size" overflow verified above */
+    return CKR_GENERAL_ERROR; /* What? */
+
+  sess->state = NDS_UPDATED;
+  return CKR_OK;
+}
+
+CK_RV
+ncr_digest_final (struct ncr_digest_session *sess, void *dest,
+		  size_t *size_ptr)
+{
+  ssize_t res;
+
+  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
+  g_return_val_if_fail (sess->state == NDS_INITIALIZED
+			|| sess->state == NDS_UPDATED,
+			CKR_OPERATION_NOT_INITIALIZED);
+  g_return_val_if_fail (size_ptr != NULL, CKR_ARGUMENTS_BAD);
+
+  if (dest == NULL)
+    {
+      *size_ptr = sess->md_size;
+      return CKR_OK;
+    }
+  if (*size_ptr < (size_t)sess->md_size)
+    {
+      *size_ptr = sess->md_size;
+      return CKR_BUFFER_TOO_SMALL;
+    }
+  *size_ptr = sess->md_size;
+
+  g_return_val_if_fail (dest != NULL, CKR_ARGUMENTS_BAD);
+
+  res = read (sess->fd, dest, sess->md_size);
+  sess->state = NDS_FINISHED;
+  if (res == sess->md_size)
+    return CKR_OK;
+  if (res == -1)
+    return ckr_errno ();
+  return CKR_GENERAL_ERROR; /* What? */
+}
+
+CK_RV
+ncr_digest (struct ncr_digest_session *sess, void *dest, size_t *dest_size_ptr,
+	    const void *data, size_t data_size)
+{
+  ssize_t res;
+
+  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
+  g_return_val_if_fail (sess->state == NDS_INITIALIZED,
+			CKR_OPERATION_NOT_INITIALIZED);
+  g_return_val_if_fail (dest_size_ptr != NULL, CKR_ARGUMENTS_BAD);
+
+  if (dest == NULL)
+    {
+      *dest_size_ptr = sess->md_size;
+      return CKR_OK;
+    }
+  if (*dest_size_ptr < (size_t)sess->md_size)
+    {
+      *dest_size_ptr = sess->md_size;
+      return CKR_BUFFER_TOO_SMALL;
+    }
+  *dest_size_ptr = sess->md_size;
+
+  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
+  g_return_val_if_fail (data_size <= SSIZE_MAX, CKR_ARGUMENTS_BAD);
+
+  sess->state = NDS_FINISHED;
+
+  res = write (sess->fd, data, data_size);
+  if (res == (ssize_t)data_size) /* "data_size" overflow verified above */
+    {
+      res = read (sess->fd, dest, sess->md_size);
+      if (res == sess->md_size)
+	return CKR_OK;
+    }
+  if (res == -1)
+    return ckr_errno ();
+  return CKR_GENERAL_ERROR; /* What? */
+}
+
+CK_RV
+ncr_digest_standalone (CK_MECHANISM_TYPE mech, void *dest,
+		       size_t *dest_size_ptr, const void *data,
+		       size_t data_size)
+{
+  const struct sockaddr_alg *sa;
+  int top_fd, fd, md_size;
+  ssize_t res;
+  CK_RV ret;
+
+  g_return_val_if_fail (dest_size_ptr != NULL, CKR_ARGUMENTS_BAD);
+
+  ret = digest_params_from_mech (mech, &sa, &md_size);
+  if (ret != CKR_OK)
+    return ret;
+
+  if (dest == NULL)
+    {
+      *dest_size_ptr = md_size;
+      return CKR_OK;
+    }
+  if (*dest_size_ptr < (size_t)md_size)
+    {
+      *dest_size_ptr = md_size;
+      return CKR_BUFFER_TOO_SMALL;
+    }
+  *dest_size_ptr = md_size;
+
+  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
+  g_return_val_if_fail (data_size <= SSIZE_MAX, CKR_ARGUMENTS_BAD);
+
+  top_fd = socket (AF_ALG, SOCK_SEQPACKET, 0);
+  if (top_fd == -1)
+    return ckr_errno ();
+  if (bind (top_fd, (struct sockaddr *)sa, sizeof (*sa)) != 0)
+    goto err_top_fd;
+  fd = accept (top_fd, NULL, 0);
+  if (fd == -1)
+    goto err_top_fd;
+  (void)close (top_fd);
+
+  res = write (fd, data, data_size);
+  if (res == (ssize_t)data_size) /* "data_size" overflow verified above */
+    {
+      res = read (fd, dest, md_size);
+      if (res == md_size)
+	{
+	  ret = CKR_OK;
+	  goto end;
+	}
+    }
+  if (res == -1)
+    ret = ckr_errno ();
+  else
+    ret = CKR_GENERAL_ERROR; /* What? */
+
+ end:
+  close (fd);
+  return ret;
+
+ err_top_fd:
+  ret = ckr_errno ();
+  (void)close (top_fd);
+  return ret;
+}
diff --git a/lib/ncrypto_local.c b/lib/ncrypto_local.c
index f311d63..72bc6fd 100644
--- a/lib/ncrypto_local.c
+++ b/lib/ncrypto_local.c
@@ -202,197 +202,6 @@ ncr_symm_key_destroy (struct ncr_symm_key *key)
   return CKR_OK;
 }
 
- /* Digest handling */
-
-static const EVP_MD *
-digest_evp_from_mech (CK_MECHANISM_TYPE mech)
-{
-  switch (mech)
-    {
-#define E(M, E)					\
-    case CKM_##M:				\
-      return EVP_##E ();
-      E (MD5, md5);
-      E (SHA_1, sha1);
-      E (SHA224, sha224);
-      E (SHA256, sha256);
-      E (SHA384, sha384);
-      E (SHA512, sha512);
-#undef E
-    default:
-      g_return_val_if_reached (NULL);
-    }
-}
-
-struct ncr_digest_session
-{
-  EVP_MD_CTX ctx;
-  const EVP_MD *md;
-  size_t md_size;
-  /* Debugging only */
-  enum { NDS_NEW, NDS_INITIALIZED, NDS_UPDATED, NDS_FINISHED } state;
-};
-
-CK_RV
-ncr_digest_alloc (struct ncr_digest_session **sess, CK_MECHANISM_TYPE mech)
-{
-  struct ncr_digest_session *s;
-  const EVP_MD *md;
-
-  g_return_val_if_fail (sess != NULL, CKR_ARGUMENTS_BAD);
-
-  md = digest_evp_from_mech (mech);
-  if (md == NULL)
-    return CKR_MECHANISM_INVALID;
-
-  s = malloc (sizeof (*s));
-  if (s == NULL)
-    return CKR_HOST_MEMORY;
-
-  EVP_MD_CTX_init (&s->ctx);
-  s->state = NDS_NEW;
-  s->md = md;
-  s->md_size = EVP_MD_size (md);
-  *sess = s;
-  return CKR_OK;
-}
-
-CK_RV
-ncr_digest_free (struct ncr_digest_session *sess)
-{
-  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
-
-  EVP_MD_CTX_cleanup (&sess->ctx);
-  free (sess);
-  return CKR_OK;
-}
-
-CK_RV
-ncr_digest_init (struct ncr_digest_session *sess)
-{
-  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
-  g_return_val_if_fail (sess->state == NDS_NEW || sess->state == NDS_FINISHED,
-			CKR_OPERATION_ACTIVE);
-
-  if (EVP_DigestInit_ex (&sess->ctx, sess->md, NULL) == 0)
-    return ckr_openssl ();
-
-  sess->state = NDS_INITIALIZED;
-  return CKR_OK;
-}
-
-CK_RV
-ncr_digest_update (struct ncr_digest_session *sess, const void *data,
-		   size_t size)
-{
-  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
-  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
-  g_return_val_if_fail (sess->state == NDS_INITIALIZED
-			|| sess->state == NDS_UPDATED,
-			CKR_OPERATION_NOT_INITIALIZED);
-
-  if (EVP_DigestUpdate (&sess->ctx, data, size) == 0)
-    return ckr_openssl ();
-
-  sess->state = NDS_UPDATED;
-  return CKR_OK;
-}
-
-CK_RV
-ncr_digest_final (struct ncr_digest_session *sess, void *dest,
-		  size_t *size_ptr)
-{
-  int res;
-
-  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
-  g_return_val_if_fail (sess->state == NDS_INITIALIZED
-			|| sess->state == NDS_UPDATED,
-			CKR_OPERATION_NOT_INITIALIZED);
-  g_return_val_if_fail (size_ptr != NULL, CKR_ARGUMENTS_BAD);
-
-  if (dest == NULL)
-    {
-      *size_ptr = sess->md_size;
-      return CKR_OK;
-    }
-  if (*size_ptr < sess->md_size)
-    {
-      *size_ptr = sess->md_size;
-      return CKR_BUFFER_TOO_SMALL;
-    }
-  *size_ptr = sess->md_size;
-
-  g_return_val_if_fail (dest != NULL, CKR_ARGUMENTS_BAD);
-
-  res = EVP_DigestFinal_ex (&sess->ctx, dest, NULL);
-  sess->state = NDS_FINISHED;
-  return res ? CKR_OK : ckr_openssl ();
-}
-
-CK_RV
-ncr_digest (struct ncr_digest_session *sess, void *dest, size_t *dest_size_ptr,
-	    const void *data, size_t data_size)
-{
-  int res;
-
-  g_return_val_if_fail (sess != NULL, CKR_SESSION_HANDLE_INVALID);
-  g_return_val_if_fail (sess->state == NDS_INITIALIZED,
-			CKR_OPERATION_NOT_INITIALIZED);
-  g_return_val_if_fail (dest_size_ptr != NULL, CKR_ARGUMENTS_BAD);
-
-  if (dest == NULL)
-    {
-      *dest_size_ptr = sess->md_size;
-      return CKR_OK;
-    }
-  if (*dest_size_ptr < sess->md_size)
-    {
-      *dest_size_ptr = sess->md_size;
-      return CKR_BUFFER_TOO_SMALL;
-    }
-  *dest_size_ptr = sess->md_size;
-
-  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
-
-  res = (EVP_DigestUpdate(&sess->ctx, data, data_size) != 0
-	 && EVP_DigestFinal_ex(&sess->ctx, dest, NULL) != 0);
-  sess->state = NDS_FINISHED;
-  return res ? CKR_OK : ckr_openssl ();
-}
-
-CK_RV
-ncr_digest_standalone (CK_MECHANISM_TYPE mech, void *dest,
-		       size_t *dest_size_ptr, const void *data,
-		       size_t data_size)
-{
-  const EVP_MD *md;
-  size_t md_size;
-
-  g_return_val_if_fail (dest_size_ptr != NULL, CKR_ARGUMENTS_BAD);
-
-  md = digest_evp_from_mech (mech);
-  if (md == NULL)
-    return CKR_MECHANISM_INVALID;
-
-  md_size = EVP_MD_size (md);
-  if (dest == NULL)
-    {
-      *dest_size_ptr = md_size;
-      return CKR_OK;
-    }
-  if (*dest_size_ptr < md_size)
-    {
-      *dest_size_ptr = md_size;
-      return CKR_BUFFER_TOO_SMALL;
-    }
-  *dest_size_ptr = md_size;
-
-  g_return_val_if_fail (data != NULL, CKR_ARGUMENTS_BAD);
-
-  return EVP_Digest(data, data_size, dest, NULL, md, NULL) != 0
-    ? CKR_OK : ckr_openssl ();
-}
-
  /* Symmetric encryption */
 
 struct ncr_symm_cipher_session