Differences

This shows you the differences between two versions of the page.

--- monit:monitoring_and_maintain_trisul_process [2018/06/11 14:49] – dhinesh
+++ monit:monitoring_and_maintain_trisul_process [2020/09/18 14:53] (current) – [Monit Trisul Process] veera
@@ Line 1: / Line 1: @@
-You can use Monit to monitor daemon processes or similar programs running on localhost.
 ====== Monit Trisul Process  =====
-==== Install Monit on Ubuntu 16.04 ====
+This document explains how you can use monit to keep your Trisul Probe processes running.
+<note important>Your Trisul Probe processes should never stop. If you are seeing crashes or memory issues , please read "[[admin:debuggingcrash|Debugging Trisul Probe Crashes]]" and help us debug your setup. This guide is useful when Trisul is in production and you want to ensure uptime.
+</note>
+===== Install Monit on Ubuntu 16.04 and 18.04 =====
 <code bash> apt-get install monit </code>
-=== Edit Configuration  ===
+===== Edit Configuration  =====
 . Open the monitrc configuration file
@@ Line 15: / Line 20: @@
 . Add following lines at the end of file
 <code>
-check process trisul_flushd_context0 with pidfile /usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/flushd.pid
-  start program = "/usr/local/bin/trisulctl_hub start context context0"
-  stop program = "/usr/local/bin/trisulctl_hub stop context context0"
 check process trisul_probe_context0 with pidfile /usr/local/var/lib/trisul-probe/domain0/probe0/context0/run/trisul-probe.pid
   start program = "/usr/local/bin/trisulctl_probe start context context0@probe0"
@@ Line 25: / Line 28: @@
 </code>
+This snippet monitors the trisul probe process and restarts if stopped.
 . Restart the monit
@@ Line 36: / Line 42: @@
+====== Advanced configuration - email when monit restarts  ======
+You can have monit restart stopped processes and email you when that happens. Just copy the following snippet into /etc/monit/monitrc
+===== Email configuration for monit =====
+If you want to receive email for trisul process status please add following lines in monitrc file
+<code >
+set mail-format {
+    from: xxx@abc.com
+    reply-to: abc@xxx.com
+    subject: Trisul Monit $SERVICE $EVENT at $DATE
+    message: Monit $ACTION $SERVICE at $DATE on $HOST: $DESCRIPTION.
+          Yours sincerely,
+          Trisul Network Analytics
+  }
+set mailserver smtp.gmail.com port 587
+username "xxx@abc.com" password "xxxxxxxxxxxxx"
+    using tlsv1
+set alert abc@xxx.com on {nonexist, timeout,resource}
+</code>
+===== Sample File =====
+<code>
+###############################################################################
+## Monit control file
+###############################################################################
+##
+## Comments begin with a '#' and extend through the end of the line. Keywords
+## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'.
+##
+## Below you will find examples of some frequently used statements. For
+## information about the control file and a complete list of statements and
+## options, please have a look in the Monit manual.
+##
+##
+###############################################################################
+## Global section
+###############################################################################
+##
+## Start Monit in the background (run as a daemon):
+#
+  set daemon 120            # check services at 2-minute intervals
+#   with start delay 240    # optional: delay the first check by 4-minutes (by
+#                           # default Monit check immediately after Monit start)
+#
+#
+## Set syslog logging. If you want to log to a standalone log file instead,
+## specify the full path to the log file
+#
+  set logfile /var/log/monit.log
+#
+#
+## Set the location of the Monit lock file which stores the process id of the
+## running Monit instance. By default this file is stored in $HOME/.monit.pid
+#
+# set pidfile /var/run/monit.pid
+#
+## Set the location of the Monit id file which stores the unique id for the
+## Monit instance. The id is generated and stored on first Monit start. By
+## default the file is placed in $HOME/.monit.id.
+#
+# set idfile /var/.monit.id
+  set idfile /var/lib/monit/id
+#
+## Set the location of the Monit state file which saves monitoring states
+## on each cycle. By default the file is placed in $HOME/.monit.state. If
+## the state file is stored on a persistent filesystem, Monit will recover
+## the monitoring state across reboots. If it is on temporary filesystem, the
+## state will be lost on reboot which may be convenient in some situations.
+#
+  set statefile /var/lib/monit/state
+#
+#
+## Set limits for various tests. The following example shows the default values:
+##
+# set limits {
+#     programOutput:     512 B,    # check program's output truncate limit
+#     sendExpectBuffer:  256 B,    # limit for send/expect protocol test
+#     fileContentBuffer: 512 B,    # limit for file content test
+#     httpContentBuffer: 1 MB,     # limit for HTTP content test
+#     networkTimeout:    5 seconds # timeout for network I/O
+# }
+## Set global SSL options (just most common options showed, see manual for
+## full list).
+#
+# set ssl {
+#     verify     : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED)
+#     selfsigned : allow   # allow self signed SSL certificates (reject by default)
+# }
+#
+#
+## Set the list of mail servers for alert delivery. Multiple servers may be
+## specified using a comma separator. If the first mail server fails, Monit
+# will use the second mail server in the list and so on. By default Monit uses
+# port 25 - it is possible to override this with the PORT option.
+#
+# set mailserver mail.bar.baz,               # primary mailserver
+#                backup.bar.baz port 10025,  # backup mailserver on port 10025
+#                localhost                   # fallback relay
+#
+#
+## By default Monit will drop alert events if no mail servers are available.
+## If you want to keep the alerts for later delivery retry, you can use the
+## EVENTQUEUE statement. The base directory where undelivered alerts will be
+## stored is specified by the BASEDIR option. You can limit the queue size
+## by using the SLOTS option (if omitted, the queue is limited by space
+## available in the back end filesystem).
+#
+  set eventqueue
+      basedir /var/lib/monit/events # set the base directory where events will be stored
+      slots 100                     # optionally limit the queue size
+#
+#
+## Send status and events to M/Monit (for more informations about M/Monit
+## see http://mmonit.com/). By default Monit registers credentials with
+## M/Monit so M/Monit can smoothly communicate back to Monit and you don't
+## have to register Monit credentials manually in M/Monit. It is possible to
+## disable credential registration using the commented out option below.
+## Though, if safety is a concern we recommend instead using https when
+## communicating with M/Monit and send credentials encrypted.
+#
+# set mmonit http://monit:monit@192.168.1.10:8080/collector
+#     # and register without credentials     # Don't register credentials
+#
+#
+## Monit by default uses the following format for alerts if the the mail-format
+## statement is missing::
+## --8<--
+## set mail-format {
+##   from:    monit@$HOST
+##   subject: monit alert --  $EVENT $SERVICE
+##   message: $EVENT Service $SERVICE
+##                 Date:        $DATE
+##                 Action:      $ACTION
+##                 Host:        $HOST
+##                 Description: $DESCRIPTION
+##
+##            Your faithful employee,
+##            Monit
+## }
+## --8<--
+##
+## You can override this message format or parts of it, such as subject
+## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc.
+## are expanded at runtime. For example, to override the sender, use:
+#
+# set mail-format { from: monit@foo.bar }
+#
+#
+## You can set alert recipients whom will receive alerts if/when a
+## service defined in this file has errors. Alerts may be restricted on
+## events by using a filter as in the second example below.
+#
+# set alert sysadm@foo.bar                       # receive all alerts
+#
+## Do not alert when Monit starts, stops or performs a user initiated action.
+## This filter is recommended to avoid getting alerts for trivial cases.
+#
+# set alert your-name@your.domain not on { instance, action }
+#
+#
+## Monit has an embedded HTTP interface which can be used to view status of
+## services monitored and manage services from a web interface. The HTTP
+## interface is also required if you want to issue Monit commands from the
+## command line, such as 'monit status' or 'monit restart service' The reason
+## for this is that the Monit client uses the HTTP interface to send these
+## commands to a running Monit daemon. See the Monit Wiki if you want to
+## enable SSL for the HTTP interface.
+#
+# set httpd port 2812 and
+#     use address localhost  # only accept connection from localhost
+#     allow localhost        # allow localhost to connect to the server and
+#     allow admin:monit      # require user 'admin' with password 'monit'
+#
+###############################################################################
+## Services
+###############################################################################
+##
+## Check general system resources such as load average, cpu and memory
+## usage. Each test specifies a resource, conditions and the action to be
+## performed should a test fail.
+#
+#  check system $HOST
+#    if loadavg (1min) > 4 then alert
+#    if loadavg (5min) > 2 then alert
+#    if cpu usage > 95% for 10 cycles then alert
+#    if memory usage > 75% then alert
+#    if swap usage > 25% then alert
+#
+#
+## Check if a file exists, checksum, permissions, uid and gid. In addition
+## to alert recipients in the global section, customized alert can be sent to
+## additional recipients by specifying a local alert handler. The service may
+## be grouped using the GROUP option. More than one group can be specified by
+## repeating the 'group name' statement.
+#
+#  check file apache_bin with path /usr/local/apache/bin/httpd
+#    if failed checksum and
+#       expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor
+#    if failed permission 755 then unmonitor
+#    if failed uid root then unmonitor
+#    if failed gid root then unmonitor
+#    alert security@foo.bar on {
+#           checksum, permission, uid, gid, unmonitor
+#        } with the mail-format { subject: Alarm! }
+#    group server
+#
+#
+## Check that a process is running, in this case Apache, and that it respond
+## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory,
+## and number of children. If the process is not running, Monit will restart
+## it by default. In case the service is restarted very often and the
+## problem remains, it is possible to disable monitoring using the TIMEOUT
+## statement. This service depends on another service (apache_bin) which
+## is defined above.
+#
+#  check process apache with pidfile /usr/local/apache/logs/httpd.pid
+#    start program = "/etc/init.d/httpd start" with timeout 60 seconds
+#    stop program  = "/etc/init.d/httpd stop"
+#    if cpu > 60% for 2 cycles then alert
+#    if cpu > 80% for 5 cycles then restart
+#    if totalmem > 200.0 MB for 5 cycles then restart
+#    if children > 250 then restart
+#    if loadavg(5min) greater than 10 for 8 cycles then stop
+#    if failed host www.tildeslash.com port 80 protocol http
+#       and request "/somefile.html"
+#    then restart
+#    if failed port 443 protocol https with timeout 15 seconds then restart
+#    if 3 restarts within 5 cycles then unmonitor
+#    depends on apache_bin
+#    group server
+#
+#
+## Check filesystem permissions, uid, gid, space and inode usage. Other services,
+## such as databases, may depend on this resource and an automatically graceful
+## stop may be cascaded to them before the filesystem will become full and data
+## lost.
+#
+#  check filesystem datafs with path /dev/sdb1
+#    start program  = "/bin/mount /data"
+#    stop program  = "/bin/umount /data"
+#    if failed permission 660 then unmonitor
+#    if failed uid root then unmonitor
+#    if failed gid disk then unmonitor
+#    if space usage > 80% for 5 times within 15 cycles then alert
+#    if space usage > 99% then stop
+#    if inode usage > 30000 then alert
+#    if inode usage > 99% then stop
+#    group server
+#
+#
+## Check a file's timestamp. In this example, we test if a file is older
+## than 15 minutes and assume something is wrong if its not updated. Also,
+## if the file size exceed a given limit, execute a script
+#
+#  check file database with path /data/mydatabase.db
+#    if failed permission 700 then alert
+#    if failed uid data then alert
+#    if failed gid data then alert
+#    if timestamp > 15 minutes then alert
+#    if size > 100 MB then exec "/my/cleanup/script" as uid dba and gid dba
+#
+#
+## Check directory permission, uid and gid.  An event is triggered if the
+## directory does not belong to the user with uid 0 and gid 0.  In addition,
+## the permissions have to match the octal description of 755 (see chmod(1)).
+#
+#  check directory bin with path /bin
+#    if failed permission 755 then unmonitor
+#    if failed uid 0 then unmonitor
+#    if failed gid 0 then unmonitor
+#
+#
+## Check a remote host availability by issuing a ping test and check the
+## content of a response from a web server. Up to three pings are sent and
+## connection to a port and an application level network check is performed.
+#
+#  check host myserver with address 192.168.1.1
+#    if failed ping then alert
+#    if failed port 3306 protocol mysql with timeout 15 seconds then alert
+#    if failed port 80 protocol http
+#       and request /some/path with content = "a string"
+#    then alert
+#
+#
+## Check a network link status (up/down), link capacity changes, saturation
+## and bandwidth usage.
+#
+#  check network public with interface eth0
+#    if failed link then alert
+#    if changed link then alert
+#    if saturation > 90% then alert
+#    if download > 10 MB/s then alert
+#    if total upload > 1 GB in last hour then alert
+#
+#
+## Check custom program status output.
+#
+#  check program myscript with path /usr/local/bin/myscript.sh
+#    if status != 0 then alert
+#
+#
+###############################################################################
+## Includes
+###############################################################################
+##
+## It is possible to include additional configuration parts from other files or
+## directories.
+#
+   include /etc/monit/conf.d/*
+   include /etc/monit/conf-enabled/*
+#
+check process trisul with pidfile /usr/local/var/lib/trisul-probe/domain0/probe0/context0/run/trisul-probe.pid
+  start program = "/usr/local/bin/trisulctl_probe start context default@probe0"
+  stop program = "/usr/local/bin/trisulctl_probe stop context default@probe0"
+check process trisul_flushd with pidfile /usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/flushd.pid
+  start program = "/usr/local/bin/trisulctl_hub start context default@hub0"
+  stop program = "/usr/local/bin/trisulctl_hub stop context default@hub0"
+</code>