Add optional net/http/pprof support (#432)

* Add optional net/http/pprof support

Add a command-line flag to enable net/http/pprof. This is the standard golang profiler and having it available is an easy way to collect cpu/mem profile from users that may face such issues.

Opportunistic fix add SIGTERM for proper k8s shutdown.

Update demo to expose pprof endpoints

* Remove ocagent/cli.go and use viper for pprof setup

Author: Paulo Janotti

README.md

@@ -357,6 +357,7 @@ Flags:
       --debug-processor               Flag to add a debug processor (combine with log level DEBUG to log incoming spans)
       --health-check-http-port uint   Port on which to run the healthcheck http server. (default 13133)
   -h, --help                          help for occollector
+      --http-pprof-port uint          Port to be used by golang net/http/pprof (Performance Profiler), the profiler is disabled if no port or 0 is specified.
       --log-level string              Output level of logs (TRACE, DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO")
       --metrics-level string          Output level of telemetry metrics (NONE, BASIC, NORMAL, DETAILED) (default "BASIC")
       --metrics-port uint             Port exposing collector telemetry. (default 8888)

cmd/ocagent/cli.go

@@ -25,7 +25,10 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"syscall"
 
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/zpages"
 	"go.uber.org/zap"
@@ -34,6 +37,8 @@ import (
 	"github.com/census-instrumentation/opencensus-service/internal"
 	"github.com/census-instrumentation/opencensus-service/internal/config"
 	"github.com/census-instrumentation/opencensus-service/internal/config/viperutils"
+	"github.com/census-instrumentation/opencensus-service/internal/pprofserver"
+	"github.com/census-instrumentation/opencensus-service/internal/version"
 	"github.com/census-instrumentation/opencensus-service/processor"
 	"github.com/census-instrumentation/opencensus-service/receiver/jaegerreceiver"
 	"github.com/census-instrumentation/opencensus-service/receiver/opencensusreceiver"
@@ -42,10 +47,31 @@ import (
 	"github.com/census-instrumentation/opencensus-service/receiver/zipkinreceiver/scribe"
 )
 
+var rootCmd = &cobra.Command{
+	Use:   "ocagent",
+	Short: "ocagent runs the OpenCensus service",
+	Run: func(cmd *cobra.Command, args []string) {
+		runOCAgent()
+	},
+}
+
+var viperCfg = viper.New()
+
 var configYAMLFile string
-var ocReceiverPort int
 
-const zipkinRoute = "/api/v2/spans"
+func init() {
+	var versionCmd = &cobra.Command{
+		Use:   "version",
+		Short: "Print the version information for ocagent",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Print(version.Info())
+		},
+	}
+	rootCmd.AddCommand(versionCmd)
+	rootCmd.PersistentFlags().StringVarP(&configYAMLFile, "config", "c", "config.yaml", "The YAML file with the configurations for the agent and various exporters")
+
+	viperutils.AddFlags(viperCfg, rootCmd, pprofserver.AddFlags)
+}
 
 func main() {
 	if err := rootCmd.Execute(); err != nil {
@@ -78,12 +104,19 @@ func runOCAgent() {
 		log.Fatalf("Could not instantiate logger: %v", err)
 	}
 
+	var asyncErrorChan = make(chan error)
+	err = pprofserver.SetupFromViper(asyncErrorChan, viperCfg, logger)
+	if err != nil {
+		log.Fatalf("Failed to start net/http/pprof: %v", err)
+	}
+
 	// TODO(skaris): move the rest of the configs to use viper
-	v, err := viperutils.ViperFromYAMLBytes([]byte(yamlBlob))
+	err = viperutils.LoadYAMLBytes(viperCfg, []byte(yamlBlob))
 	if err != nil {
 		log.Fatalf("Config: failed to create viper from YAML: %v", err)
 	}
-	traceExporters, metricsExporters, closeFns, err := config.ExportersFromViperConfig(logger, v)
+
+	traceExporters, metricsExporters, closeFns, err := config.ExportersFromViperConfig(logger, viperCfg)
 	if err != nil {
 		log.Fatalf("Config: failed to create exporters from YAML: %v", err)
 	}
@@ -150,11 +183,15 @@ func runOCAgent() {
 		}
 	}()
 
-	signalsChan := make(chan os.Signal)
-	signal.Notify(signalsChan, os.Interrupt)
+	signalsChan := make(chan os.Signal, 1)
+	signal.Notify(signalsChan, os.Interrupt, syscall.SIGTERM)
 
-	// Wait for the closing signal
-	<-signalsChan
+	select {
+	case err = <-asyncErrorChan:
+		log.Fatalf("Asynchronous error %q, terminating process", err)
+	case s := <-signalsChan:
+		log.Printf("Received %q signal from OS, terminating process", s)
+	}
 }
 
 func runZPages(port int) func() error {

cmd/ocagent/main.go

@@ -30,6 +30,8 @@ import (
 
 	"github.com/census-instrumentation/opencensus-service/cmd/occollector/app/builder"
 	"github.com/census-instrumentation/opencensus-service/internal/collector/processor"
+	"github.com/census-instrumentation/opencensus-service/internal/config/viperutils"
+	"github.com/census-instrumentation/opencensus-service/internal/pprofserver"
 	"github.com/census-instrumentation/opencensus-service/receiver"
 )
 
@@ -70,13 +72,15 @@ func (app *Application) init() {
 }
 
 func (app *Application) execute() {
-	var asyncErrorChannel = make(chan error)
-	var signalsChannel = make(chan os.Signal)
-	signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
+	asyncErrorChannel := make(chan error)
 
 	app.logger.Info("Starting...", zap.Int("NumCPU", runtime.NumCPU()))
 
-	var err error
+	err := pprofserver.SetupFromViper(asyncErrorChannel, app.v, app.logger)
+	if err != nil {
+		log.Fatalf("Failed to start net/http/pprof: %v", err)
+	}
+
 	app.healthCheck, err = newHealthCheck(app.v, app.logger)
 	if err != nil {
 		log.Fatalf("Failed to start healthcheck server: %v", err)
@@ -93,14 +97,17 @@ func (app *Application) execute() {
 		os.Exit(1)
 	}
 
+	signalsChannel := make(chan os.Signal, 1)
+	signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
+
 	// mark service as ready to receive traffic.
 	app.healthCheck.Ready()
 
 	select {
 	case err = <-asyncErrorChannel:
 		app.logger.Error("Asynchronous error received, terminating process", zap.Error(err))
-	case <-signalsChannel:
-		app.logger.Info("Received kill signal from OS")
+	case s := <-signalsChannel:
+		app.logger.Info("Received signal from OS", zap.String("signal", s.String()))
 	}
 
 	app.healthCheck.Set(healthcheck.Unavailable)
@@ -131,11 +138,12 @@ func (app *Application) Start() error {
 			app.execute()
 		},
 	}
-	addFlags(app.v, rootCmd,
+	viperutils.AddFlags(app.v, rootCmd,
 		telemetryFlags,
 		builder.Flags,
 		healthCheckFlags,
 		loggerFlags,
+		pprofserver.AddFlags,
 	)
 
 	return rootCmd.Execute()

cmd/occollector/app/collector/collector.go

@@ -17,11 +17,12 @@ services:
   # Collector
   oc-collector:
     image: occollector:latest
-    command: ["--config=/etc/oc-collector-config.yaml"]
+    command: ["--config=/etc/oc-collector-config.yaml", "--http-pprof-port=1777"]
     volumes:
       - ./oc-collector-config.yaml:/etc/oc-collector-config.yaml
     ports:
       - "55678"
+      - "1777:1777"
       - "8888:8888" # Prometheus metrics
     depends_on:
       - jaeger-all-in-one
@@ -30,18 +31,19 @@ services:
   # Agent
   oc-agent:
     image: ocagent:latest
-    command: ["--config=/etc/oc-agent-config.yaml"]
+    command: ["--config=/etc/oc-agent-config.yaml", "--http-pprof-port=1888"]
     volumes:
       - ./oc-agent-config.yaml:/etc/oc-agent-config.yaml
     ports:
+      - "1888:1888"
       - "14268"
       - "55678"
     depends_on:
       - oc-collector
 
   # Synthetic load generator
   synthetic-load-generator:
-    image: omnition/synthetic-load-generator:1.0.6
+    image: omnition/synthetic-load-generator:1.0.25
     environment:
       - JAEGER_COLLECTOR_URL=http://oc-agent:14268
     depends_on:

cmd/occollector/app/collector/flags.go

@@ -1,18 +1,60 @@
+// Copyright 2019, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sampling containts the interfaces and data types used to implement
+// the various sampling policies.
+
 package viperutils
 
 import (
 	"bytes"
+	"flag"
+	"strings"
 
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 // ViperFromYAMLBytes unmarshals byte content in the YAML file format into
-// a viper object
+// a viper object.
 func ViperFromYAMLBytes(yamlBlob []byte) (*viper.Viper, error) {
 	v := viper.New()
-	v.SetConfigType("yaml")
-	if err := v.ReadConfig(bytes.NewBuffer(yamlBlob)); err != nil {
+	if err := LoadYAMLBytes(v, yamlBlob); err != nil {
 		return nil, err
 	}
 	return v, nil
 }
+
+// LoadYAMLBytes sets the viper config type to yaml and loads the given blob.
+func LoadYAMLBytes(v *viper.Viper, yamlBlob []byte) error {
+	v.SetConfigType("yaml")
+	if err := v.ReadConfig(bytes.NewBuffer(yamlBlob)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// AddFlags adds the provided flags to the provided viper and cobra command.
+func AddFlags(v *viper.Viper, command *cobra.Command, addFlagsFns ...func(*flag.FlagSet)) (*viper.Viper, *cobra.Command) {
+	flagSet := new(flag.FlagSet)
+	for _, addFlags := range addFlagsFns {
+		addFlags(flagSet)
+	}
+	command.Flags().AddGoFlagSet(flagSet)
+
+	v.AutomaticEnv()
+	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	v.BindPFlags(command.Flags())
+	return v, command
+}