From fa3fa35c6dba340811c0e3d5afffc045369d2ec8 Mon Sep 17 00:00:00 2001 From: William Brown Date: Fri, 10 Mar 2017 13:46:57 +1000 Subject: [PATCH] Ticket 49154 - Nunc Stans stress should assert it has 95% success rate Bug Description: We should assert that the nunc-stans stress test is able to pass 95% of it's connections during an overload scenario. Fix Description: Assert we pass 95% of connections. Additionally, we were not actually running the tests properly, so fix that. Improve the the work thread function to be slightly faster by better using our atomic shutdown check. https://pagure.io/389-ds-base/issue/49154 Author: wibrown Review by: ??? --- Makefile.am | 2 +- src/nunc-stans/ns/ns_thrpool.c | 13 ++++++++----- src/nunc-stans/test/test_nuncstans_stress.c | 6 +++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Makefile.am b/Makefile.am index d712aba..f26ea03 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2046,7 +2046,7 @@ test_nuncstans_CPPFLAGS = $(AM_CPPFLAGS) $(CMOCKA_INCLUDES) $(NUNCSTANS_CPPFLAGS test_nuncstans_LDADD = libnunc-stans.la libsds.la test_nuncstans_LDFLAGS = $(ASAN_DEFINES) $(PROFILING_LINKS) $(CMOCKA_LINKS) $(EVENT_LINK) -test_nuncstans_stress_SOURCES = src/nunc-stans/test/test_nuncstans.c +test_nuncstans_stress_SOURCES = src/nunc-stans/test/test_nuncstans_stress.c test_nuncstans_stress_CPPFLAGS = $(AM_CPPFLAGS) $(CMOCKA_INCLUDES) $(NUNCSTANS_CPPFLAGS) test_nuncstans_stress_LDADD = libnunc-stans.la libsds.la test_nuncstans_stress_LDFLAGS = $(ASAN_DEFINES) $(PROFILING_LINKS) $(CMOCKA_LINKS) $(EVENT_LINK) diff --git a/src/nunc-stans/ns/ns_thrpool.c b/src/nunc-stans/ns/ns_thrpool.c index 744749b..dd2545e 100644 --- a/src/nunc-stans/ns/ns_thrpool.c +++ b/src/nunc-stans/ns/ns_thrpool.c @@ -360,6 +360,7 @@ worker_thread_func(void *arg) { ns_thread_t *thr = (ns_thread_t *)arg; ns_thrpool_t *tp = thr->tp; + sds_result result = SDS_SUCCESS; /* Get ready to use lock free ds */ sds_lqueue_tprep(tp->work_q); @@ -369,15 +370,16 @@ worker_thread_func(void *arg) */ while (!ns_thrpool_is_shutdown(tp)) { ns_job_t *job = NULL; + result = sds_lqueue_dequeue(tp->work_q, (void **)&job); /* Don't need monitor here, job_dequeue barriers the memory for us. Job will be valid */ - while(sds_lqueue_dequeue(tp->work_q, (void **)&job) == SDS_LIST_EXHAUSTED && !ns_thrpool_is_shutdown(tp)) - { + /* Is it possible for a worker thread to get stuck here during shutdown? */ + if (result == SDS_LIST_EXHAUSTED) { work_q_wait(tp); - } - - if (job) { + } else if (result == SDS_SUCCESS && job != NULL) { work_job_execute(job); /* MUST NOT ACCESS JOB FROM THIS POINT */ + } else { + ns_log(LOG_ERR, "worker_thread_func encountered a recoverable issue during processing of the queue\n"); } } @@ -1494,6 +1496,7 @@ ns_thrpool_shutdown(struct ns_thrpool_t *tp) __atomic_add_fetch(&(tp->shutdown), 1, __ATOMIC_SEQ_CST); /* Wake up the idle worker threads so they can exit. */ + /* Do we need this to be run in conjuction with our thread join loop incase threads are still active? */ pthread_mutex_lock(&(tp->work_q_lock)); pthread_cond_broadcast(&(tp->work_q_cv)); pthread_mutex_unlock(&(tp->work_q_lock)); diff --git a/src/nunc-stans/test/test_nuncstans_stress.c b/src/nunc-stans/test/test_nuncstans_stress.c index 29585d5..f9981bb 100644 --- a/src/nunc-stans/test/test_nuncstans_stress.c +++ b/src/nunc-stans/test/test_nuncstans_stress.c @@ -377,7 +377,9 @@ client_initiate_connection_cb(struct ns_job_t *job) sock = PR_OpenTCPSocket(PR_AF_INET6); if (sock == NULL) { - do_logging(LOG_ERR, "Socket failed\n"); + char *err = NULL; + PR_GetErrorText(err); + do_logging(LOG_ERR, "FAIL: Socket failed, %d -> %s\n", PR_GetError(), err); PR_AtomicAdd(&client_fail_count, 1); goto done; } @@ -542,6 +544,8 @@ ns_stress_test(void **state __attribute__((unused))) assert_int_equal(client_success_count, job_count); */ assert_int_equal(server_success_count, client_success_count); + int32_t job_threshold = (jobs * client_thread_count) * 0.95; + assert_true(client_success_count >= job_threshold); PR_Cleanup(); } -- 1.8.3.1