Merge pull request #10132 from bergquist/go_routine_management

Wait for all sub routines to finish before closing Grafana

pkg/api/http_server.go

@@ -95,7 +95,7 @@ func (hs *HttpServer) Start(ctx context.Context) error {
 
 func (hs *HttpServer) Shutdown(ctx context.Context) error {
 	err := hs.httpSrv.Shutdown(ctx)
-	hs.log.Info("stopped http server")
+	hs.log.Info("Stopped HTTP server")
 	return err
 }
 

pkg/cmd/grafana-server/main.go

@@ -14,8 +14,8 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 
+	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/metrics"
-	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 
 	_ "github.com/grafana/grafana/pkg/services/alerting/conditions"
@@ -40,9 +40,6 @@ var homePath = flag.String("homepath", "", "path to grafana install/home path, d
 var pidFile = flag.String("pidfile", "", "path to pid file")
 var exitChan = make(chan int)
 
-func init() {
-}
-
 func main() {
 	v := flag.Bool("v", false, "prints current version and exits")
 	profile := flag.Bool("profile", false, "Turn on pprof profiling")
@@ -82,12 +79,28 @@ func main() {
 	setting.BuildStamp = buildstampInt64
 
 	metrics.M_Grafana_Version.WithLabelValues(version).Set(1)
-
+	shutdownCompleted := make(chan int)
 	server := NewGrafanaServer()
-	server.Start()
+
+	go listenToSystemSignals(server, shutdownCompleted)
+
+	go func() {
+		code := 0
+		if err := server.Start(); err != nil {
+			log.Error2("Startup failed", "error", err)
+			code = 1
+		}
+
+		exitChan <- code
+	}()
+
+	code := <-shutdownCompleted
+	log.Info2("Grafana shutdown completed.", "code", code)
+	log.Close()
+	os.Exit(code)
 }
 
-func listenToSystemSignals(server models.GrafanaServer) {
+func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int) {
 	signalChan := make(chan os.Signal, 1)
 	ignoreChan := make(chan os.Signal, 1)
 	code := 0
@@ -97,10 +110,12 @@ func listenToSystemSignals(server models.GrafanaServer) {
 
 	select {
 	case sig := <-signalChan:
-		// Stops trace if profiling has been enabled
-		trace.Stop()
+		trace.Stop() // Stops trace if profiling has been enabled
 		server.Shutdown(0, fmt.Sprintf("system signal: %s", sig))
+		shutdownCompleted <- 0
 	case code = <-exitChan:
+		trace.Stop() // Stops trace if profiling has been enabled
 		server.Shutdown(code, "startup error")
+		shutdownCompleted <- code
 	}
 }

pkg/cmd/grafana-server/server.go

@@ -11,7 +11,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 
 	"golang.org/x/sync/errgroup"
@@ -20,7 +19,6 @@ import (
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/login"
 	"github.com/grafana/grafana/pkg/metrics"
-	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	"github.com/grafana/grafana/pkg/services/cleanup"
@@ -33,7 +31,7 @@ import (
 	"github.com/grafana/grafana/pkg/tracing"
 )
 
-func NewGrafanaServer() models.GrafanaServer {
+func NewGrafanaServer() *GrafanaServerImpl {
 	rootCtx, shutdownFn := context.WithCancel(context.Background())
 	childRoutines, childCtx := errgroup.WithContext(rootCtx)
 
@@ -54,9 +52,7 @@ type GrafanaServerImpl struct {
 	httpServer *api.HttpServer
 }
 
-func (g *GrafanaServerImpl) Start() {
-	go listenToSystemSignals(g)
-
+func (g *GrafanaServerImpl) Start() error {
 	g.initLogging()
 	g.writePIDFile()
 
@@ -69,16 +65,12 @@ func (g *GrafanaServerImpl) Start() {
 	plugins.Init()
 
 	if err := provisioning.Init(g.context, setting.HomePath, setting.Cfg); err != nil {
-		logger.Error("Failed to provision Grafana from config", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Failed to provision Grafana from config. error: %v", err)
 	}
 
 	closer, err := tracing.Init(setting.Cfg)
 	if err != nil {
-		g.log.Error("Tracing settings is not valid", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Tracing settings is not valid. error: %v", err)
 	}
 	defer closer.Close()
 
@@ -93,13 +85,12 @@ func (g *GrafanaServerImpl) Start() {
 	g.childRoutines.Go(func() error { return cleanUpService.Run(g.context) })
 
 	if err = notifications.Init(); err != nil {
-		g.log.Error("Notification service failed to initialize", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Notification service failed to initialize. error: %v", err)
 	}
 
-	SendSystemdNotification("READY=1")
-	g.startHttpServer()
+	sendSystemdNotification("READY=1")
+
+	return g.startHttpServer()
 }
 
 func initSql() {
@@ -123,16 +114,16 @@ func (g *GrafanaServerImpl) initLogging() {
 	setting.LogConfigurationInfo()
 }
 
-func (g *GrafanaServerImpl) startHttpServer() {
+func (g *GrafanaServerImpl) startHttpServer() error {
 	g.httpServer = api.NewHttpServer()
 
 	err := g.httpServer.Start(g.context)
 
 	if err != nil {
-		g.log.Error("Fail to start server", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Fail to start server. error: %v", err)
 	}
+
+	return nil
 }
 
 func (g *GrafanaServerImpl) Shutdown(code int, reason string) {
@@ -145,10 +136,9 @@ func (g *GrafanaServerImpl) Shutdown(code int, reason string) {
 
 	g.shutdownFn()
 	err = g.childRoutines.Wait()
-
-	g.log.Info("Shutdown completed", "reason", err)
-	log.Close()
-	os.Exit(code)
+	if err != nil && err != context.Canceled {
+		g.log.Error("Server shutdown completed with an error", "error", err)
+	}
 }
 
 func (g *GrafanaServerImpl) writePIDFile() {
@@ -173,7 +163,7 @@ func (g *GrafanaServerImpl) writePIDFile() {
 	g.log.Info("Writing PID file", "path", *pidFile, "pid", pid)
 }
 
-func SendSystemdNotification(state string) error {
+func sendSystemdNotification(state string) error {
 	notifySocket := os.Getenv("NOTIFY_SOCKET")
 
 	if notifySocket == "" {

pkg/models/server.go

-package models
-
-type GrafanaServer interface {
-	Start()
-	Shutdown(code int, reason string)
-}