monitor.go 3.94 KB
Newer Older
Jakob Borg's avatar
Jakob Borg committed
1 2 3 4
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

5 6 7 8 9 10 11
package main

import (
	"bufio"
	"io"
	"os"
	"os/exec"
12
	"os/signal"
13 14 15
	"path/filepath"
	"strings"
	"sync"
16
	"syscall"
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
	"time"
)

var (
	stdoutFirstLines []string // The first 10 lines of stdout
	stdoutLastLines  []string // The last 50 lines of stdout
	stdoutMut        sync.Mutex
)

const (
	countRestarts = 5
	loopThreshold = 15 * time.Second
)

func monitorMain() {
	os.Setenv("STNORESTART", "yes")
	l.SetPrefix("[monitor] ")

	args := os.Args
	var restarts [countRestarts]time.Time

38 39 40 41
	sign := make(chan os.Signal, 1)
	sigTerm := syscall.Signal(0xf)
	signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
	for {
		if t := time.Since(restarts[0]); t < loopThreshold {
			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
			os.Exit(exitError)
		}

		copy(restarts[0:], restarts[1:])
		restarts[len(restarts)-1] = time.Now()

		cmd := exec.Command(args[0], args[1:]...)

		stderr, err := cmd.StderrPipe()
		if err != nil {
			l.Fatalln(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			l.Fatalln(err)
		}

63
		l.Infoln("Starting pulse")
64 65 66 67 68 69 70 71 72 73 74 75 76
		err = cmd.Start()
		if err != nil {
			l.Fatalln(err)
		}

		stdoutMut.Lock()
		stdoutFirstLines = make([]string, 0, 10)
		stdoutLastLines = make([]string, 0, 50)
		stdoutMut.Unlock()

		go copyStderr(stderr)
		go copyStdout(stdout)

77 78 79 80 81 82 83 84 85 86 87
		exit := make(chan error)

		go func() {
			exit <- cmd.Wait()
		}()

		select {
		case s := <-sign:
			l.Infof("Signal %d received; exiting", s)
			cmd.Process.Kill()
			<-exit
88
			return
89

90
		case err = <-exit:
91 92 93
			if err == nil {
				// Successfull exit indicates an intentional shutdown
				return
94 95 96 97 98 99 100 101
			} else if exiterr, ok := err.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					switch status.ExitStatus() {
					case exitUpgrading:
						// Restart the monitor process to release the .old
						// binary as part of the upgrade process.
						l.Infoln("Restarting monitor...")
						os.Setenv("STNORESTART", "")
102 103 104 105
						err := exec.Command(args[0], args[1:]...).Start()
						if err != nil {
							l.Warnln("restart:", err)
						}
106 107 108
						return
					}
				}
109
			}
110 111
		}

112
		l.Infoln("Pulse exited:", err)
113
		time.Sleep(1 * time.Second)
114 115 116 117

		// Let the next child process know that this is not the first time
		// it's starting up.
		os.Setenv("STRESTART", "yes")
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
	}
}

func copyStderr(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)

	var panicFd *os.File
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		if panicFd == nil {
			os.Stderr.WriteString(line)

			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
				if err != nil {
					l.Warnln("Create panic log:", err)
					continue
				}

				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
142
				l.Warnln("Please create an issue at https://source.ind.ie/project/pulse/issues/ with the panic log attached")
143

Jakob Borg's avatar
Jakob Borg committed
144
				panicFd.WriteString("Panic at " + time.Now().Format(time.RFC1123) + "\n")
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
				stdoutMut.Lock()
				for _, line := range stdoutFirstLines {
					panicFd.WriteString(line)
				}
				panicFd.WriteString("...\n")
				for _, line := range stdoutLastLines {
					panicFd.WriteString(line)
				}
			}
		}

		if panicFd != nil {
			panicFd.WriteString(line)
		}
	}
}

func copyStdout(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		stdoutMut.Lock()
		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
			stdoutFirstLines = append(stdoutFirstLines, line)
		}
		if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
			stdoutLastLines = stdoutLastLines[:l-1]
		}
		stdoutLastLines = append(stdoutLastLines, line)
		stdoutMut.Unlock()

		os.Stdout.WriteString(line)
	}
}