cd /var/log/dirsrv/slapd-NNIT centaur errors.20130925-071459.tnng [01/Oct/2013:10:38:01 +0200] - userRoot [01/Oct/2013:10:38:01 +0200] - export userRoot: Processed 297 entries (100%). [01/Oct/2013:10:38:01 +0200] - All database threads now stopped [01/Oct/2013:11:00:07 +0200] - Backend Instance(s): [01/Oct/2013:11:00:07 +0200] - userRoot [01/Oct/2013:11:00:07 +0200] - export userRoot: Processed 297 entries (100%). ZPM errors.20130925-115002.tnng [30/Sep/2013:11:50:07 +0200] - Listening on All Interfaces port 636 for LDAPS requests [01/Oct/2013:12:25:39 +0200] - 389-Directory/1.2.10.14 B2012.290.1622 starting up [01/Oct/2013:12:25:39 +0200] - Detected Disorderly Shutdown last time Directory Server was running, recovering database. cat /etc/cron.d/sysstat # run system activity accounting tool every 10 minutes */10 * * * * root /usr/lib64/sa/sa1 1 1 # generate a daily summary of process accounting at 23:53 53 23 * * * root /usr/lib64/sa/sa2 -A centaur: more /var/log/sa/sar01 10:40:01 16.89 Average: 71.54 00:00:06 cswch/s 00:10:04 2936.41 00:20:01 zpm: more /var/log/sa/sar01 10:40:01 0.52 11:20:01 0.52 *********/var/log/cron Oct 1 10:30:01 centaur crond[8996]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 10:30:01 centaur crond[8997]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /etc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null 2>&1; chown root:root /etc/sudoers >/dev/null 2>&1) Oct 1 10:38:01 centaur crond[16991]: (root) CMD (/usr/lib64/dirsrv/slapd-NNIT/db2ldif -s "dc=nnit" -a /tmp/tmp.ldf >/dev/null 2>&1;/bin/chmod 666 /tmp/tmp.ldf >/dev/null 2>&1;sleep 3 >/dev/null 2>&1;/nnit/sysadm/bin/ldap/scripts/pwd_change.pl >/dev/null 2>&1) Oct 1 10:40:01 centaur crond[19211]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 10:40:01 centaur crond[19212]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /etc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null 2>&1; chown root:root /etc/sudoers >/dev/null 2>&1) Oct 1 10:50:07 centaur crond[27594]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:52:07 centaur crond[27594]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:52:07 centaur crond[27594]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:52:07 centaur crond[29451]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:52:07 centaur crond[29452]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:06 centaur crond[27594]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:06 centaur crond[29451]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:06 centaur crond[29451]: nss_ldap: could not search LDAP server - Server is unavailable ... Oct 1 11:54:06 centaur crond[28306]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:54:06 centaur crond[28306]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:57:23 centaur crond[4470]: (CRON) STARTUP (V5.0) Oct 1 11:57:39 centaur anacron[6067]: Anacron 2.3 started on 2013-10-01 Oct 1 11:57:39 centaur anacron[6067]: Normal exit (0 jobs run) Oct 1 11:58:01 centaur crond[7689]: (root) CMD (/usr/lib64/dirsrv/slapd-NNIT/db2ldif -s "dc=nnit" -a /tmp/tmp.ldf >/dev/null 2>&1;/bin/chmod 666 /tmp/tmp.ldf >/dev/null 2>&1;sleep 3 >/dev/null 2>&1;/nnit/sysadm/bin/ldap/scripts/pwd_change.pl >/dev/null 2>&1) Oct 1 12:00:02 centaur crond[10582]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 12:01:01 centaur crond[12367]: (root) CMD (run-parts /etc/cron.hourly) Oct 1 12:08:01 centaur crond[23493]: (root) CMD (/usr/lib64/dirsrv/slapd-NNIT/db2ldif -s "dc=nnit" -a /tmp/tmp.ldf >/dev/null 2>&1;/bin/chmod 666 /tmp/tmp.ldf >/dev/null 2>&1;sleep 3 >/dev/null 2>&1;/nnit/sysadm/bin/ldap/scripts/pwd_change.pl >/dev/null 2>&1) [root@centaur log]# grep "could not search LDAP server" /var/log/cron*|more /var/log/cron:Oct 1 10:52:07 centaur crond[27594]: nss_ldap: could not search LDAP server - Serve r is unavailable /var/log/cron:Oct 1 10:54:06 centaur crond[29451]: nss_ldap: could not search LDAP server - Serve r is unavailable /var/log/cron:Oct 1 10:54:07 centaur crond[29452]: nss_ldap: could not search LDAP server - Serve r is unavailable .. /var/log/cron:Oct 1 11:54:06 centaur crond[28305]: nss_ldap: could not search LDAP server - Server is unavailable /var/log/cron:Oct 1 11:54:06 centaur crond[28306]: nss_ldap: could not search LDAP server - Server is unavailable zpm Oct 1 10:40:01 zpm crond[20131]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /e tc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null 2>&1; chown root:root /etc/sudoe rs >/dev/null 2>&1) Oct 1 10:52:06 zpm crond[20435]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:52:51 zpm crond[20436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:09 zpm crond[20435]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:09 zpm crond[20435]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:54:51 zpm crond[20436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:54:51 zpm crond[20436]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:56:09 zpm crond[20435]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:57:00 zpm crond[20436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:58:12 zpm crond[20435]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:58:12 zpm crond[20435]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:59:09 zpm crond[20436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:59:09 zpm crond[20436]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:00:12 zpm crond[20435]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:01:09 zpm crond[20436]: nss-ldap: do_open: do_start_tls failed:stat=-1 ... Oct 1 11:56:09 zpm crond[22296]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:56:09 zpm crond[22296]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:56:09 zpm crond[22507]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /etc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null 2>&1; chown root:root /etc/sudoers >/dev/null 2>&1) Oct 1 11:56:09 zpm crond[22296]: (root) error: Job execution of per-minute job scheduled for 11:50 delayed into subsequent minute 11:56. Skipping job run. Oct 1 11:56:09 zpm crond[22296]: CRON (root) ERROR: cannot set security context Oct 1 12:00:01 zpm crond[22622]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 12:01:01 zpm crond[22665]: (root) CMD (run-parts /etc/cron.hourly) Oct 1 12:10:01 zpm crond[22933]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 12:10:01 zpm crond[22934]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /etc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null 2>&1; chown root:root /etc/sudoers >/dev/null 2>&1) Oct 1 12:20:01 zpm crond[23254]: (root) CMD (/usr/lib64/sa/sa1 1 1) Oct 1 12:20:01 zpm crond[23255]: (root) CMD (alias cp=cp; cp -f /nnit/data/sys/sudo/sudoers /etc/sudoers >/dev/null 2>&1; chmod 0440 /etc/sudoers >/dev/null [root@zpm sa]# grep "could not search LDAP server" /var/log/cron*|more /var/log/cron:Oct 1 10:54:09 zpm crond[20435]: nss_ldap: could not search LDAP server - Server is unavailable /var/log/cron:Oct 1 10:54:51 zpm crond[20436]: nss_ldap: could not search LDAP server - Server is unavailable .. /var/log/cron:Oct 1 11:56:09 zpm crond[22297]: nss_ldap: could not search LDAP server - Server is unavailable /var/log/cron:Oct 1 11:56:09 zpm crond[22296]: nss_ldap: could not search LDAP server - Server is unavailable ***********/var/log/message centaur Oct 1 10:51:02 centaur kernel: nfs4_cb: server 10.225.222.31 not responding, timed out Oct 1 10:51:40 centaur kernel: nfs4_cb: server 10.224.9.25 not responding, timed out Oct 1 10:51:46 centaur nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:51:46 centaur kernel: nfs4_cb: server 10.224.9.19 not responding, timed out Oct 1 10:52:11 centaur kernel: nfs4_cb: server 10.226.34.7 not responding, ti reboot at: Oct 1 11:14:34 centaur syslogd 1.4.1: restart. AND Oct 1 11:56:57 centaur syslogd 1.4.1: restart. Oct 1 11:24:31 centaur nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:26:40 centaur nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:28:40 centaur nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:28:40 centaur nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:29:16 centaur kernel: nfs4_cb: server 10.224.9.41 not responding, timed out ... Oct 1 11:59:37 centaur kernel: nfs4_cb: server 10.225.222.29 not responding, timed out Oct 1 12:00:36 centaur ntpd[3946]: synchronized to LOCAL(0), stratum 10 Oct 1 12:00:36 centaur ntpd[3946]: kernel time sync enabled 0001 Oct 1 12:01:39 centaur ntpd[3946]: synchronized to 217.16.102.1, stratum 2 Oct 1 12:07:17 centaur kernel: nfs4_cb: server 217.16.100.14 not responding, timed out Oct 1 12:07:17 centaur kernel: nfs4_cb: server 217.16.100.14 not responding, timed out Oct 1 12:10:25 centaur kernel: nfs4_cb: server 10.225.222.20 ZPM Oct 1 10:49:48 zpm rpc.idmapd[2436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:50:33 zpm kernel: INFO: task statvfs:20379 blocked for more than 120 seconds. Oct 1 10:50:33 zpm kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Oct 1 10:50:33 zpm kernel: statvfs D ffff81003ddf5040 0 20379 21643 19783 (NOTLB) Oct 1 10:50:33 zpm kernel: ffff810013405a28 0000000000000082 0000000000000002 ffffffff800fe277 Oct 1 10:50:33 zpm kernel: ffff810034c35930 0000000000000009 ffff8100071fe0c0 ffff81003ddf5040 Oct 1 10:50:33 zpm kernel: 000c8ee73d1fcfc2 00000000000026eb ffff8100071fe2a8 0000000000000001 Oct 1 10:50:33 zpm kernel: Call Trace: Oct 1 10:50:33 zpm kernel: [] ep_poll_callback+0x64/0x8b Oct 1 10:50:33 zpm kernel: [] __wake_up+0x38/0x4f Oct 1 10:50:33 zpm kernel: [] :nfs:nfs_idmap_id+0x189/0x209 Oct 1 10:50:33 zpm kernel: [] default_wake_function+0x0/0xe Oct 1 10:50:33 zpm kernel: [] :nfs:decode_getfattr+0x661/0xc5e Oct 1 10:50:33 zpm kernel: [] :sunrpc:rpc_wait_bit_interruptible+0x0/0x28 Oct 1 10:50:33 zpm kernel: [] :nfs:nfs4_xdr_dec_getattr+0x0/0x4f Oct 1 10:50:33 zpm kernel: [] :nfs:nfs4_xdr_dec_getattr+0x48/0x4f Oct 1 10:50:33 zpm kernel: [] :sunrpc:call_decode+0x4b7/0x509 Oct 1 10:50:33 zpm kernel: [] :sunrpc:__rpc_execute+0x9c/0x2df Oct 1 10:50:33 zpm kernel: [] :sunrpc:rpc_call_sync+0x80/0xaa Oct 1 10:50:33 zpm kernel: [] vsnprintf+0x5d7/0xb54 Oct 1 10:50:33 zpm kernel: [] :nfs:nfs4_proc_getattr+0x77/0x96 Oct 1 10:50:33 zpm kernel: [] :nfs:__nfs_revalidate_inode+0x13e/0x263 Oct 1 10:50:33 zpm kernel: [] mntput_no_expire+0x19/0x89 Oct 1 10:50:33 zpm kernel: [] :nfs:nfs_getattr+0xaf/0xd9 Oct 1 10:50:33 zpm kernel: [] vfs_stat_fd+0x32/0x4a Oct 1 10:50:33 zpm kernel: [] audit_filter_inodes+0xbd/0xf4 Oct 1 10:50:33 zpm kernel: [] sys32_stat64+0x11/0x29 Oct 1 10:50:33 zpm kernel: [] sysenter_tracesys+0x48/0x83 Oct 1 10:50:33 zpm kernel: [] sysenter_do_call+0x1e/0x76 Oct 1 10:50:33 zpm kernel: Oct 1 10:51:43 zpm nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:51:48 zpm rpc.idmapd[2436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:51:48 zpm rpc.idmapd[2436]: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:51:48 zpm rpc.idmapd[2436]: nss_getpwnam: name 'nfsadm' not found in domain 'csn.nnithosting.com' Oct 1 10:53:43 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:55:46 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:55:46 zpm nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 10:57:46 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:59:46 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 10:59:46 zpm nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:02:07 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:03:49 zpm rpc.idmapd[2436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:04:07 zpm nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 11:04:07 zpm nscd: nss_ldap: could not search LDAP server - Server is unavailable Oct 1 11:04:33 zpm kernel: INFO: task statvfs:20810 blocked for more than 120 seconds. Oct 1 11:04:33 zpm kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Oct 1 11:04:33 zpm kernel: statvfs D ffff81003ddf5040 0 20810 21643 19783 (NOTLB) Oct 1 11:04:33 zpm kernel: ffff81001f06da28 0000000000000082 0000000000000006 ffffffff800fe277 Oct 1 11:04:33 zpm kernel: ffff810034c35930 0000000000000009 ffff8100064cb0c0 ffff81003ddf5040 Oct 1 11:04:33 zpm kernel: 000c8fac211493b6 00000000000024ed ffff8100064cb2a8 0000000000000001 Oct 1 11:04:33 zpm kernel: Call Trace: ... Oct 1 11:57:19 zpm rpc.idmapd[2436]: nss_getpwnam: name '701' does not map into domain 'csn.nnithosting.com' Oct 1 12:05:19 zpm rpc.idmapd[2436]: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 1 12:05:19 zpm rpc.idmapd[2436]: nss_ldap: reconnected to LDAP server ldap://centaur.csn.nnithosting.com/ Oct 1 12:05:19 zpm rpc.idmapd[2436]: nss_getpwnam: name 'nfsadm' not found in domain 'csn.nnitho sting.com' KUN en RECONNECTED!! [root@zpm log]# grep mntput_no_expire mess* messages:Oct 1 10:50:33 zpm kernel: [] mntput_no_expire+0x19/0x89 messages:Oct 1 11:04:33 zpm kernel: [] mntput_no_expire+0x19/0x89 messages:Oct 1 11:06:33 zpm kernel: [] mntput_no_expire+0x19/0x89 messages:Oct 1 11:20:33 zpm kernel: [] mntput_no_expire+0x19/0x89 messages:Oct 1 11:34:33 zpm kernel: [] mntput_no_expire+0x19/0x89 messages:Oct 1 11:50:33 zpm kernel: [] mntput_no_expire+0x19/0x89 TEST (tonight) Oct 2 21:14:05 centaur nscd: nss_ldap: reconnected to LDAP server ldap://zpm.csn.nnithosting.com/ stop: ldap on zpm "id" on centaur not work!! it gets "id" from zpm!! try again, stop/start ldap on zpm "id" on centaur work now stop both ldap: id still work on centaur/zpm, becasue nscd running; Oct 2 21:20:01 centaur nscd: nss-ldap: do_open: do_start_tls failed:stat=-1 Oct 2 21:20:01 centaur nscd: nss_ldap: could not search LDAP server - Server is unavailable [root@centaur log]# ldapsearch -LLLx -h centaur -p 389 -D 'cn=directory manager' -W -b "cn=monitor" "(cn=*)" | egrep connections Enter LDAP Password: currentconnections: 116 totalconnections: 2839 connections: 5562 connectionseq: 2839