From 176e3ade8c6715ff82212657a3049816e7c0c9a4 Mon Sep 17 00:00:00 2001 From: William Brown Date: Fri, 14 Jul 2017 17:13:10 +1000 Subject: [PATCH] Ticket 49325 - Proof of concept rust tqueue in sds Bug Description: Rust is a modern systems programming language in the style of C and C++. It has strict compile time guarantees to the correctness of applications, and promises the potential to reduce many times of security and stability issues including bounds checking, null dereferences, use-after free and more. This is achieved without a run time, instead using compile time ownership and lifetime checks. Fix Description: This ticket is to add a proof of concept that we can use Rust as an FFI language with existing C components. This adds an optional configure argument to enable a rust thread safe queue which is used by nunc-stans for event dispatch. My tests already show it is safe (it passes all the existing test) and the server when built with this option passes the basic suite. Importantly it shows how we can integrate cargo with autotools, and how to expose and C compatible apis from rust. To use this, at configure time add "--enable-rust". There are no other changes to the server without this flag, and it is not a requirement to build DS, it's optional. https://pagure.io/389-ds-base/issue/49325 Author: wibrown Review by: mreynolds, tbordaz (Thank you!) --- Makefile.am | 63 ++++++++++++++++-- configure.ac | 34 ++++++++++ rpm.mk | 2 + rpm/389-ds-base.spec.in | 25 +++++++- src/libsds/Cargo.toml | 16 +++++ src/libsds/sds/lib.rs | 32 ++++++++++ src/libsds/sds/tqueue.rs | 131 ++++++++++++++++++++++++++++++++++++++ src/libsds/test/test_sds_tqueue.c | 4 ++ 8 files changed, 297 insertions(+), 10 deletions(-) create mode 100644 src/libsds/Cargo.toml create mode 100644 src/libsds/sds/lib.rs create mode 100644 src/libsds/sds/tqueue.rs diff --git a/Makefile.am b/Makefile.am index b8ed7e9..0a5ece4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,8 +29,21 @@ NSPR_INCLUDES = @nspr_inc@ SVRCORE_INCLUDES = @svrcore_inc@ SASL_INCLUDES = @sasl_inc@ EVENT_INCLUDES = @event_inc@ -# Not used currently -# TCMALLOC_INCLUDES = @tcmalloc_inc@ + +# Rust inclusions. +if RUST_ENABLE +RUST_ON = 1 +CARGO_FLAGS = @cargo_defs@ +RUSTC_FLAGS = @asan_rust_defs@ @debug_rust_defs@ +RUST_LDFLAGS = -ldl -lpthread -lgcc_s -lc -lm -lrt -lutil +RUST_DEFINES = -DRUST_ENABLE +else +RUST_ON = 0 +CARGO_FLAGS = +RUSTC_FLAGS = +RUST_LDFLAGS = +RUST_DEFINES = +endif # We can't add the lfds includes all the time as they have a "bomb" in them that # prevents compilation on unsupported hardware arches. @@ -80,7 +93,7 @@ PATH_DEFINES = -DLOCALSTATEDIR="\"$(localstatedir)\"" -DSYSCONFDIR="\"$(sysconfd # Now that we have all our defines in place, setup the CPPFLAGS # These flags are the "must have" for all components -AM_CPPFLAGS = $(DEBUG_DEFINES) $(GCCSEC_DEFINES) $(ASAN_DEFINES) $(PROFILING_DEFINES) +AM_CPPFLAGS = $(DEBUG_DEFINES) $(GCCSEC_DEFINES) $(ASAN_DEFINES) $(PROFILING_DEFINES) $(RUST_DEFINES) # Flags for Directory Server # WARNING: This needs a clean up, because slap.h is a horrible mess and is publically exposed! DSPLUGIN_CPPFLAGS = $(DS_DEFINES) $(DS_INCLUDES) $(PATH_DEFINES) $(SYSTEMD_DEFINES) $(NUNCSTANS_INCLUDES) @openldap_inc@ @ldapsdk_inc@ @nss_inc@ $(NSPR_INCLUDES) @systemd_inc@ @@ -138,7 +151,7 @@ AM_LDFLAGS = -lpthread else #AM_LDFLAGS = -Wl,-z,defs # Provide the tcmalloc links if needed -AM_LDFLAGS = $(ASAN_DEFINES) $(PROFILING_LINKS) $(TCMALLOC_LINK) -latomic +AM_LDFLAGS = $(RUST_LDFLAGS) $(ASAN_DEFINES) $(PROFILING_LINKS) $(TCMALLOC_LINK) -latomic endif #end hpux # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html#Updating-version-info @@ -236,6 +249,7 @@ clean-local: -rm -rf dist -rm -rf $(abs_top_builddir)/html -rm -rf $(abs_top_builddir)/man + CARGO_TARGET_DIR=$(abs_top_builddir)/rs cargo clean --manifest-path=$(srcdir)/src/libsds/Cargo.toml dberrstrs.h: Makefile perl $(srcdir)/ldap/servers/slapd/mkDBErrStrs.pl -i @db_incdir@ -o . @@ -288,7 +302,12 @@ bin_PROGRAMS = dbscan \ pwdhash \ rsearch -server_LTLIBRARIES = libsds.la libnunc-stans.la libldaputil.la libslapd.la libns-dshttpd.la +server_LTLIBRARIES = +if RUST_ENABLE +server_LTLIBRARIES += librsds.la +endif +server_LTLIBRARIES += libsds.la libnunc-stans.la libldaputil.la libslapd.la libns-dshttpd.la + # this is how to add optional plugins if enable_pam_passthru @@ -1047,7 +1066,6 @@ libsds_la_SOURCES = src/libsds/sds/core/utils.c \ src/libsds/sds/bpt_cow/txn.c \ src/libsds/sds/bpt_cow/verify.c \ src/libsds/sds/queue/queue.c \ - src/libsds/sds/queue/tqueue.c \ src/libsds/sds/queue/lqueue.c \ src/libsds/external/csiphash/csiphash.c \ src/libsds/sds/ht/ht.c \ @@ -1070,6 +1088,32 @@ endif libsds_la_CPPFLAGS = $(AM_CPPFLAGS) $(SDS_CPPFLAGS) libsds_la_LDFLAGS = $(AM_LDFLAGS) $(SDS_LDFLAGS) +if RUST_ENABLE +libsds_la_LIBADD = librsds.la + +librsdspatha = $(abs_top_builddir)/rs/@rust_target_dir@/librsds.a +librsdspatho = $(abs_top_builddir)/rs/@rust_target_dir@/librsds.o + +# Remember, these emit to cargo_target_dir/rust_target_dir/emit target +$(librsdspatha): Makefile src/libsds/Cargo.toml src/libsds/sds/lib.rs src/libsds/sds/tqueue.rs + CARGO_TARGET_DIR=$(abs_top_builddir)/rs cargo rustc $(CARGO_FLAGS) --verbose --manifest-path=$(srcdir)/src/libsds/Cargo.toml -- $(RUSTC_FLAGS) --emit link=$(librsdspatha) + +$(librsdspatho): Makefile src/libsds/Cargo.toml src/libsds/sds/lib.rs src/libsds/sds/tqueue.rs + CARGO_TARGET_DIR=$(abs_top_builddir)/rs cargo rustc $(CARGO_FLAGS) --verbose --manifest-path=$(srcdir)/src/libsds/Cargo.toml -- $(RUSTC_FLAGS) --emit obj=$(librsdspatho) + +am_librsds_la_OBJECTS = $(librsdspatho) +librsds_la_LIBADD = $(librsdspatha) +librsds_la_SOURCES = + +dist_noinst_DATA += $(srcdir)/src/libsds/Cargo.toml \ + $(srcdir)/src/libsds/sds/*.rs + +else +# Just build the tqueue in C. +libsds_la_SOURCES += \ + src/libsds/sds/queue/tqueue.c +endif + #------------------------ # libnunc-stans #------------------------ @@ -2242,7 +2286,12 @@ rpmbrprep: dist-bzip2 rpmroot cp $(distdir).tar.bz2 $(RPMBUILD)/SOURCES cp $(srcdir)/rpm/389-ds-base-git.sh $(RPMBUILD)/SOURCES cp $(srcdir)/rpm/389-ds-base-devel.README $(RPMBUILD)/SOURCES - sed -e "s/__VERSION__/$(RPM_VERSION)/" -e "s/__RELEASE__/$(RPM_RELEASE)/" -e "s/__VERSION_PREREL__/$(VERSION_PREREL)/" -e "s/__NUNC_STANS_ON__/$(NUNC_STANS_ON)/" -e "s/__ASAN_ON__/$(ASAN_ON)/" < $(abs_builddir)/rpm/389-ds-base.spec > $(RPMBUILD)/SPECS/389-ds-base.spec + sed -e "s/__VERSION__/$(RPM_VERSION)/" \ + -e "s/__RELEASE__/$(RPM_RELEASE)/" \ + -e "s/__VERSION_PREREL__/$(VERSION_PREREL)/" \ + -e "s/__NUNC_STANS_ON__/$(NUNC_STANS_ON)/" \ + -e "s/__RUST_ON__/$(RUST_ON)/" \ + -e "s/__ASAN_ON__/$(ASAN_ON)/" < $(abs_builddir)/rpm/389-ds-base.spec > $(RPMBUILD)/SPECS/389-ds-base.spec # Requires rpmdevtools. Consider making this a dependancy of rpms. rpmsources: rpmbrprep diff --git a/configure.ac b/configure.ac index 91d6d39..cb6c528 100644 --- a/configure.ac +++ b/configure.ac @@ -24,6 +24,8 @@ AC_SUBST([CONSOLE_VERSION]) AM_MAINTAINER_MODE AC_CANONICAL_HOST +AC_CONFIG_MACRO_DIRS([m4]) + # Checks for programs. AC_PROG_CXX AC_PROG_CC @@ -78,29 +80,61 @@ AC_CHECK_FUNCS([clock_gettime], [], AC_MSG_ERROR([unable to locate required symb # This will detect if we need to add the LIBADD_DL value for us. LT_LIB_DLLOAD +# Optional rust component support. +AC_MSG_CHECKING(for --enable-rust) +AC_ARG_ENABLE(rust, AS_HELP_STRING([--enable-rust], [Enable rust language features (default: no)]), +[ + AC_CHECK_PROG(CARGO, [cargo], [yes], [no]) + AC_CHECK_PROG(RUSTC, [rustc], [yes], [no]) + + AS_IF([test "$CARGO" != "yes" -o "$RUSTC" != "yes"], [ + AC_MSG_FAILURE("Rust based plugins cannot be built cargo=$CARGO rustc=$RUSTC") + ]) + with_rust=yes + AC_MSG_RESULT(yes) +], +[ + AC_MSG_RESULT(no) +]) +AM_CONDITIONAL([RUST_ENABLE],[test -n "$with_rust"]) + AC_MSG_CHECKING(for --enable-debug) AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug], [Enable debug features (default: no)]), [ AC_MSG_RESULT(yes) debug_defs="-g3 -DDEBUG -DMCC_DEBUG -O0" + debug_rust_defs="-C debuginfo=2" + cargo_defs="" + rust_target_dir="debug" + with_debug=yes ], [ AC_MSG_RESULT(no) debug_defs="" + debug_rust_defs="-C debuginfo=2" + cargo_defs="--release" + rust_target_dir="release" ]) AC_SUBST([debug_defs]) +AC_SUBST([debug_rust_defs]) +AC_SUBST([cargo_defs]) +AC_SUBST([rust_target_dir]) +AM_CONDITIONAL([DEBUG],[test -n "$with_debug"]) AC_MSG_CHECKING(for --enable-asan) AC_ARG_ENABLE(asan, AS_HELP_STRING([--enable-asan], [Enable gcc address sanitizer options (default: no)]), [ AC_MSG_RESULT(yes) asan_defs="-fsanitize=address -fno-omit-frame-pointer" + asan_rust_defs="-Z sanitizer=address" ], [ AC_MSG_RESULT(no) asan_defs="" + asan_rust_defs="" ]) AC_SUBST([asan_defs]) +AC_SUBST([asan_rust_defs]) AM_CONDITIONAL(enable_asan,test "$enable_asan" = "yes") if test -z "$enable_perl" ; then diff --git a/rpm.mk b/rpm.mk index 378a469..4254808 100644 --- a/rpm.mk +++ b/rpm.mk @@ -10,6 +10,7 @@ NAME_VERSION = $(PACKAGE)-$(RPM_VERSION)$(VERSION_PREREL) TARBALL = $(NAME_VERSION).tar.bz2 NUNC_STANS_ON = 1 ASAN_ON = 0 +RUST_ON = 0 clean: rm -rf dist @@ -35,6 +36,7 @@ rpmroot: sed -e s/__VERSION__/$(RPM_VERSION)/ -e s/__RELEASE__/$(RPM_RELEASE)/ \ -e s/__VERSION_PREREL__/$(VERSION_PREREL)/ \ -e s/__NUNC_STANS_ON__/$(NUNC_STANS_ON)/ \ + -e s/__RUST_ON__/$(RUST_ON)/ \ -e s/__ASAN_ON__/$(ASAN_ON)/ \ rpm/$(PACKAGE).spec.in > $(RPMBUILD)/SPECS/$(PACKAGE).spec diff --git a/rpm/389-ds-base.spec.in b/rpm/389-ds-base.spec.in index 2fac560..d16ab5a 100644 --- a/rpm/389-ds-base.spec.in +++ b/rpm/389-ds-base.spec.in @@ -16,11 +16,15 @@ # This enables an ASAN build. This should not go to production, so we rename. %global use_asan __ASAN_ON__ + +# This enables rust in the build. +%global use_rust __RUST_ON__ + %if %{use_asan} %global use_tcmalloc 0 %global variant base-asan %else -%if %{_arch} != "s390x" && %{_arch} != "s390" +%if %{_arch} != "s390x" && %{_arch} != "s390" && !%{use_rust} %global use_tcmalloc 1 %else %global use_tcmalloc 0 @@ -84,6 +88,11 @@ BuildRequires: systemd-devel %if %{use_asan} BuildRequires: libasan %endif +# If rust is enabled +%if %{use_rust} +BuildRequires: cargo +BuildRequires: rust +%endif # Needed to support regeneration of the autotool artifacts. BuildRequires: autoconf BuildRequires: automake @@ -263,7 +272,7 @@ export CXX=clang++ # hack hack hack https://bugzilla.redhat.com/show_bug.cgi?id=833529 NSSARGS="--with-svrcore-inc=%{_includedir} --with-svrcore-lib=%{_libdir} --with-nss-lib=%{_libdir} --with-nss-inc=%{_includedir}/nss3" -%if %{use_asan} +%if %{use_asan} && !%{use_rust} ASAN_FLAGS="--enable-asan --enable-debug" %endif @@ -271,6 +280,10 @@ ASAN_FLAGS="--enable-asan --enable-debug" TCMALLOC_FLAGS="--enable-tcmalloc" %endif +%if %{use_rust} +RUST_FLAGS="--enable-rust" +%endif + # Rebuild the autotool artifacts now. autoreconf -fiv @@ -279,7 +292,7 @@ autoreconf -fiv --with-systemdsystemunitdir=%{_unitdir} \ --with-systemdsystemconfdir=%{_sysconfdir}/systemd/system \ --with-systemdgroupname=%{groupname} \ - $NSSARGS $TCMALLOC_FLAGS $ASAN_FLAGS \ + $NSSARGS $TCMALLOC_FLAGS $ASAN_FLAGS $RUST_FLAGS \ --enable-cmocka %if 0%{?rhel} >= 8 || 0%{?fedora} @@ -526,6 +539,9 @@ fi %{_libdir}/%{pkgname}/libnunc-stans.so %{_libdir}/%{pkgname}/libsds.so %{_libdir}/%{pkgname}/libldaputil.so +%if %{use_rust} +%{_libdir}/%{pkgname}/librsds.so +%endif %{_libdir}/pkgconfig/* %files libs @@ -537,6 +553,9 @@ fi %{_libdir}/%{pkgname}/libnunc-stans.so.* %{_libdir}/%{pkgname}/libsds.so.* %{_libdir}/%{pkgname}/libldaputil.so.* +%if %{use_rust} +%{_libdir}/%{pkgname}/librsds.so.* +%endif %files snmp %defattr(-,root,root,-) diff --git a/src/libsds/Cargo.toml b/src/libsds/Cargo.toml new file mode 100644 index 0000000..594326c --- /dev/null +++ b/src/libsds/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rsds" +version = "0.1.0" +authors = ["William Brown "] + +[dependencies] + +[lib] +path = "sds/lib.rs" +name = "rsds" +crate-type = ["dylib"] + +[profile.release] +panic = "abort" +# lto = true + diff --git a/src/libsds/sds/lib.rs b/src/libsds/sds/lib.rs new file mode 100644 index 0000000..aa70c7a --- /dev/null +++ b/src/libsds/sds/lib.rs @@ -0,0 +1,32 @@ +// BEGIN COPYRIGHT BLOCK +// Copyright (c) 2017, Red Hat, Inc +// All rights reserved. +// +// License: GPL (version 3 or any later version). +// See LICENSE for details. +// END COPYRIGHT BLOCK + +#![warn(missing_docs)] + +//! sds is a collection of datastructures used in the slapi api. This contains +//! a thread safe queue and others implemented in C. + +/// Implementation of a thread safe queue. +pub mod tqueue; + +#[repr(C)] +/// Slapi Data Structure Result types. Indicates the status of the operation +/// for C compatability (instead of Result +pub enum sds_result { + /// The operation was a success + Success = 0, + /// An unknown error occured. This indicates a fault in the API. + UnknownError = 1, + /// A null pointer was provided as an argument to a function. This is + /// invalid. + NullPointer = 2, + /// The list is exhausted, no more elements can be returned. + ListExhausted = 16, +} + + diff --git a/src/libsds/sds/tqueue.rs b/src/libsds/sds/tqueue.rs new file mode 100644 index 0000000..b7042e5 --- /dev/null +++ b/src/libsds/sds/tqueue.rs @@ -0,0 +1,131 @@ +// BEGIN COPYRIGHT BLOCK +// Copyright (c) 2017, Red Hat, Inc +// All rights reserved. +// +// License: GPL (version 3 or any later version). +// See LICENSE for details. +// END COPYRIGHT BLOCK + +#![warn(missing_docs)] + +use super::sds_result; +use std::sync::Mutex; +use std::collections::LinkedList; + +// Borrow from libc +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +pub enum c_void { + // Two dummy variants so the #[repr] attribute can be used. + #[doc(hidden)] + __Variant1, + #[doc(hidden)] + __Variant2, +} + +/// A thread safe queue. This is made to be compatible with the tqueue api +/// provided by libsds as a proof of concept. As a result it contains some C-isms +/// like holding a free function pointer (rather than drop trait). +pub struct TQueue { + q: Mutex>, + free_fn: Option, +} + +impl TQueue { + /// Allocate a new thread safe queue. If the free function is provided + /// on drop of the TQueue, this function will be called on all remaining + /// elements of the queue. + pub fn new(free_fn: Option) -> Self { + TQueue { + q: Mutex::new(LinkedList::new()), + free_fn: free_fn, + } + } + + /// Push a pointer into the tail of the queue. + pub fn enqueue(&self, elem: *const c_void) { + let mut q_inner = self.q.lock().unwrap(); + q_inner.push_back(elem); + } + + /// Dequeue the head element of the queue. If not element + /// exists return None. + pub fn dequeue(&self) -> Option<*const c_void> { + let mut q_inner = self.q.lock().unwrap(); + q_inner.pop_front() + } +} + +impl Drop for TQueue { + fn drop(&mut self) { + println!("droping tqueue"); + if let Some(f) = self.free_fn { + let mut q_inner = self.q.lock().unwrap(); + let mut elem = (*q_inner).pop_front(); + while elem.is_some() { + (f)(elem.unwrap()); + elem = (*q_inner).pop_front(); + } + } + } +} + +#[no_mangle] +/// C compatible wrapper around the TQueue. Given a valid point, a TQueue pointer +/// is allocated on the heap and referenced in retq. free_fn_ptr may be NULL +/// but if it references a function, this will be called during drop of the TQueue. +pub extern fn sds_tqueue_init(retq: *mut *mut TQueue, free_fn_ptr: Option) -> sds_result { + // This piece of type signature magic is because in rust types that extern C, + // with option has None resolve to null. What this causes is we can wrap + // our fn ptr with Option in rust, but the C side gives us fn ptr or NULL, and + // it *works*. It makes the result complete safe on the rust side too! + if retq.is_null() { + return sds_result::NullPointer; + } + + let q = Box::new(TQueue::new(free_fn_ptr)); + unsafe { + *retq = Box::into_raw(q); + } + sds_result::Success +} + +#[no_mangle] +/// Push an element to the tail of the queue. The element may be NULL +pub extern fn sds_tqueue_enqueue(q: *const TQueue, elem: *const c_void) -> sds_result { + // Check for null .... + unsafe { (*q).enqueue(elem) }; + sds_result::Success +} + +#[no_mangle] +/// Dequeue from the head of the queue. The result will be placed into elem. +/// if elem is NULL no dequeue is attempted. If there are no more items +/// ListExhausted is returned. +pub extern fn sds_tqueue_dequeue(q: *const TQueue, elem: *mut *const c_void) -> sds_result { + if elem.is_null() { + return sds_result::NullPointer; + } + match unsafe { (*q).dequeue() } { + Some(e) => { + unsafe { *elem = e; }; + sds_result::Success + } + None => { + sds_result::ListExhausted + } + } +} + +#[no_mangle] +/// Free the queue and all remaining elements. After this point it is +/// not safe to access the queue. +pub extern fn sds_tqueue_destroy(q: *mut TQueue) -> sds_result { + // This will drop the queue and free it's content + // mem::drop(q); + let _q = unsafe { Box::from_raw(q) }; + sds_result::Success +} + + diff --git a/src/libsds/test/test_sds_tqueue.c b/src/libsds/test/test_sds_tqueue.c index cd14f6e..69bc0b7 100644 --- a/src/libsds/test/test_sds_tqueue.c +++ b/src/libsds/test/test_sds_tqueue.c @@ -17,6 +17,7 @@ test_1_tqueue_invalid_create(void **state __attribute__((unused))) assert_int_equal(result, SDS_NULL_POINTER); } +#ifndef RUST_ENABLE static void test_2_tqueue_enqueue(void **state) { @@ -113,6 +114,7 @@ test_6_tqueue_dequeue_multiple(void **state) assert_ptr_equal(q->uq->head, NULL); assert_ptr_equal(q->uq->tail, NULL); } +#endif /* RUST_ENABLE */ static void test_7_tqueue_random(void **state) @@ -178,6 +180,7 @@ run_tqueue_tests(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_1_tqueue_invalid_create), +#ifndef RUST_ENABLE cmocka_unit_test_setup_teardown(test_2_tqueue_enqueue, tqueue_test_setup, tqueue_test_teardown), @@ -193,6 +196,7 @@ run_tqueue_tests(void) cmocka_unit_test_setup_teardown(test_6_tqueue_dequeue_multiple, tqueue_test_setup, tqueue_test_teardown), +#endif cmocka_unit_test_setup_teardown(test_7_tqueue_random, tqueue_test_setup, tqueue_test_teardown), -- 1.8.3.1