Introducing Stackdriver Exporter for traces (#48)

* Introducing Stackdriver Exporter for Opencensus C# library

- Current implementation can only store string values
- Added the exporter and trace handler only
- The exporter relies on newest Trace API from Stackdriver.

* Updating translation from ISpan to Stackdriver's Span to cover more fields

* Fixing the issue that prevented Stackdriver API call to succeed:
now construction of Span resource is taken care of by SpanName class that
is part of Stackdriver Trace V2 API.

Author: Simon Zeltser

OpenCensus.sln

@@ -36,6 +36,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".vsts", ".vsts", "{61188153
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Exporter.ApplicationInsights", "src\OpenCensus.Exporter.ApplicationInsights\OpenCensus.Exporter.ApplicationInsights.csproj", "{4493F5D9-874E-4FBF-B2F3-37890BD910E0}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Exporter.Stackdriver", "src\OpenCensus.Exporter.Stackdriver\OpenCensus.Exporter.Stackdriver.csproj", "{DE1B4783-C01F-4672-A6EB-695F1717105B}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples", "src\Samples\Samples.csproj", "{C58393EB-32E2-4AC6-9170-697B36306E15}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Abstractions", "src\OpenCensus.Abstractions\OpenCensus.Abstractions.csproj", "{99F8A331-05E9-45A5-89BA-4C54E825E5B2}"
@@ -52,7 +54,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Collector.AspNet
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "testdata", "testdata", "{77C7929A-2EED-4AA6-8705-B5C443C8AA0F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestApp.AspNetCore.2.0", "test\TestApp.AspNetCore.2.0\TestApp.AspNetCore.2.0.csproj", "{F2F81E76-6A0E-466B-B673-EBBF1A9ED075}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestApp.AspNetCore.2.0", "test\TestApp.AspNetCore.2.0\TestApp.AspNetCore.2.0.csproj", "{F2F81E76-6A0E-466B-B673-EBBF1A9ED075}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -76,6 +78,10 @@ Global
 		{4493F5D9-874E-4FBF-B2F3-37890BD910E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4493F5D9-874E-4FBF-B2F3-37890BD910E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4493F5D9-874E-4FBF-B2F3-37890BD910E0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DE1B4783-C01F-4672-A6EB-695F1717105B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DE1B4783-C01F-4672-A6EB-695F1717105B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DE1B4783-C01F-4672-A6EB-695F1717105B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DE1B4783-C01F-4672-A6EB-695F1717105B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C58393EB-32E2-4AC6-9170-697B36306E15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C58393EB-32E2-4AC6-9170-697B36306E15}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C58393EB-32E2-4AC6-9170-697B36306E15}.Release|Any CPU.ActiveCfg = Release|Any CPU

README.md

@@ -79,6 +79,17 @@ finally
 }
 ```
 
+### Using Stackdriver Exporter
+
+1. Enable [Stackdriver Trace][stackdriver-setup] resource.
+2. Instantiate a new instance of `StackdriverExporter` with your Google Cloud's ProjectId
+3. See [sample][stackdriver-sample] for example use.
+
+``` csharp
+var exporter = new StackdriverExporter("YOUR-GOOGLE-PROJECT-ID", Tracing.ExportComponent);
+exporter.Start();
+```
+
 ### Using Application Insights exporter
 
 1. Create [Application Insights][ai-get-started] resource.
@@ -127,8 +138,10 @@ deprecate it for 18 months before removing it, if possible.
 [good-first-issues]: https://github.com/census-instrumentation/opencensus-csharp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
 [zipkin-get-started]: https://zipkin.io/pages/quickstart.html
 [ai-get-started]: https://docs.microsoft.com/azure/application-insights
+[stackdriver-setup]: https://cloud.google.com/trace/docs/setup/
 [semver]: http://semver.org/
 [ai-sample]: https://github.com/census-instrumentation/opencensus-csharp/blob/develop/src/Samples/TestApplicationInsights.cs
+[stackdriver-sample]: https://github.com/census-instrumentation/opencensus-csharp/blob/develop/src/Samples/TestStackdriver.cs
 [zipkin-sample]: https://github.com/census-instrumentation/opencensus-csharp/blob/develop/src/Samples/TestZipkin.cs
 [prometheus-get-started]: https://prometheus.io/docs/introduction/first_steps/
 [prometheus-sample]: https://github.com/census-instrumentation/opencensus-csharp/blob/develop/src/Samples/TestPrometheus.cs

src/OpenCensus.Exporter.Stackdriver/Implementation/StackdriverTraceExporter.cs

@@ -0,0 +1,100 @@
+
+namespace OpenCensus.Exporter.Stackdriver.Implementation
+{
+    using System.Collections.Generic;
+    using System.Linq;
+    using Google.Cloud.Trace.V2;
+    using OpenCensus.Exporter.Stackdriver.Utils;
+    using OpenCensus.Trace;
+    using OpenCensus.Trace.Export;
+
+    static class SpanExtensions
+    {
+        /// <summary>
+        /// Translating <see cref="ISpanData"/> to Stackdriver's Span
+        /// According to <see cref="https://cloud.google.com/trace/docs/reference/v2/rpc/google.devtools.cloudtrace.v2"/> specifications
+        /// </summary>
+        /// <param name="spanData">Span in OpenCensus format</param>
+        /// <param name="projectId">Google Cloud Platform Project Id</param>
+        /// <returns></returns>
+        public static Span ToSpan(this ISpanData spanData, string projectId)
+        {
+            string spanId = spanData.Context.SpanId.ToLowerBase16();
+            var span = new Span
+            {
+                SpanName = new SpanName(projectId, spanData.Context.TraceId.ToLowerBase16(), spanId),
+                SpanId = spanId,
+                DisplayName = new TruncatableString { Value = spanData.Name },
+                StartTime = spanData.StartTimestamp.ToTimestamp(),
+                EndTime = spanData.EndTimestamp.ToTimestamp(),
+                ChildSpanCount = spanData.ChildSpanCount,
+            };
+
+            if (spanData.Attributes != null)
+            {
+                span.Attributes = new Span.Types.Attributes
+                {
+                    DroppedAttributesCount = spanData.Attributes != null ? spanData.Attributes.DroppedAttributesCount : 0,
+
+                    AttributeMap = { spanData.Attributes?.AttributeMap?.ToDictionary(
+                                        s => s.Key,
+                                        s => s.Value?.ToAttributeValue()) },
+                };
+            }
+
+            if (spanData.ParentSpanId != null)
+            {
+                string parentSpanId = spanData.ParentSpanId.ToLowerBase16();
+                if (!string.IsNullOrEmpty(parentSpanId))
+                {
+                    span.ParentSpanId = parentSpanId;
+                }
+            }
+
+            return span;
+        }
+
+        public static Google.Cloud.Trace.V2.AttributeValue ToAttributeValue(this IAttributeValue av)
+        {
+            // TODO Currently we assume we store only strings.
+            return new Google.Cloud.Trace.V2.AttributeValue
+            {
+                StringValue = new TruncatableString
+                {
+                    Value = av.Match(
+                    s => s,
+                    b => b.ToString(),
+                    l => l.ToString(),
+                    obj => obj.ToString(),
+                    obj => obj.ToString())
+                }
+            };
+        }
+    }
+
+    /// <summary>
+    /// Exports a group of spans to Stackdriver
+    /// </summary>
+    class StackdriverTraceExporter : IHandler
+    {
+        private Google.Api.Gax.ResourceNames.ProjectName googleCloudProjectId;
+
+        public StackdriverTraceExporter(string projectId)
+        {
+            googleCloudProjectId = new Google.Api.Gax.ResourceNames.ProjectName(projectId);
+            
+        }
+
+        public void Export(IList<ISpanData> spanDataList)
+        {
+            TraceServiceClient traceWriter = TraceServiceClient.Create();
+            var batchSpansRequest = new BatchWriteSpansRequest
+            {
+                ProjectName = googleCloudProjectId,
+                Spans = { spanDataList.Select(s => s.ToSpan(googleCloudProjectId.ProjectId)) },
+            };
+
+            traceWriter.BatchWriteSpans(batchSpansRequest);
+        }
+    }
+}

src/OpenCensus.Exporter.Stackdriver/OpenCensus.Exporter.Stackdriver.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <!--
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'OpenCensus.sln'))\build\Common.prod.props" />
+  -->
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeSymbols>True</IncludeSymbols>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <Description>Stackdriver Exporter for OpenCensus.</Description>
+    <PackageTags>Tracing;OpenCensus;Management;Monitoring;Stackdriver;Google;GCP;distributed-tracing</PackageTags>
+    <PackageIconUrl>https://opencensus.io/images/opencensus-logo.png</PackageIconUrl>
+    <PackageProjectUrl>https://opencensus.io</PackageProjectUrl>
+    <PackageLicenseUrl>https://www.apache.org/licenses/LICENSE-2.0</PackageLicenseUrl>
+    <Authors>OpenCensus authors</Authors>
+    <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Cloud.Trace.V2" Version="1.0.0-beta02" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\OpenCensus.Abstractions\OpenCensus.Abstractions.csproj" />
+  </ItemGroup>
+
+</Project>

src/OpenCensus.Exporter.Stackdriver/StackdriverExporter.cs

@@ -0,0 +1,70 @@
+
+// <copyright file="StackdriverExporter.cs" company="OpenCensus Authors">
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of theLicense at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+namespace OpenCensus.Exporter.Stackdriver
+{
+    using OpenCensus.Exporter.Stackdriver.Implementation;
+    using OpenCensus.Trace.Export;
+    using System;
+    using System.Threading;
+
+    public class StackdriverExporter
+    {
+        private const string ExporterName = "StackdriverTraceExporter";
+
+        private readonly IExportComponent exportComponent;
+        private readonly string projectId;
+        private object locker = new object();
+        private bool isInitialized = false;
+
+        public StackdriverExporter(string projectId, IExportComponent exportComponent)
+        {
+            this.projectId = projectId;
+            this.exportComponent = exportComponent;
+        }
+
+        public void Start()
+        {
+            lock (locker)
+            {
+                if (isInitialized)
+                {
+                    return;
+                }
+
+                var traceExporter = new StackdriverTraceExporter(projectId);
+                exportComponent.SpanExporter.RegisterHandler(ExporterName, traceExporter);
+
+                isInitialized = true;
+            }
+        }
+
+        public void Stop()
+        {
+            lock (locker)
+            {
+                if (!isInitialized)
+                {
+                    return;
+                }
+
+                exportComponent.SpanExporter.UnregisterHandler(ExporterName);
+                isInitialized = false;
+            }
+        }
+    }
+}

src/OpenCensus.Exporter.Stackdriver/Utils/ProtoExtensions.cs

@@ -0,0 +1,14 @@
+
+namespace OpenCensus.Exporter.Stackdriver.Utils
+{
+    using Google.Protobuf.WellKnownTypes;
+    using OpenCensus.Common;
+
+    public static class ProtoExtensions
+    {
+        public static Timestamp ToTimestamp(this ITimestamp timestamp)
+        {
+            return new Timestamp { Seconds = timestamp.Seconds, Nanos = timestamp.Nanos };
+        }
+    }
+}

src/OpenCensus/Trace/Export/SpanExporterWorker.cs

@@ -137,9 +137,10 @@ private void Export(IList<ISpanData> export)
                 {
                     handler.Export(export);
                 }
-                catch (Exception)
+                catch (Exception ex)
                 {
-                    // Log warning
+                    // TODO Log warning
+                    Console.WriteLine(ex);
                 }
             }
         }

src/OpenCensus/Trace/TraceComponent.cs

@@ -40,7 +40,7 @@ public TraceComponent(IClock clock, IRandomGenerator randomHandler, IEventQueue
             {
                 this.ExportComponent = Export.ExportComponent.CreateWithoutInProcessStores(eventQueue);
             }
-else
+            else
             {
                 this.ExportComponent = Export.ExportComponent.CreateWithInProcessStores(eventQueue);
             }

src/Samples/Samples.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -7,6 +7,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\OpenCensus.Collector.Dependencies\OpenCensus.Collector.Dependencies.csproj" />
+    <ProjectReference Include="..\OpenCensus.Exporter.Stackdriver\OpenCensus.Exporter.Stackdriver.csproj" />
     <ProjectReference Include="..\OpenCensus.Exporter.Prometheus\OpenCensus.Exporter.Prometheus.csproj" />
     <ProjectReference Include="..\OpenCensus\OpenCensus.csproj" />
     <ProjectReference Include="..\OpenCensus.Exporter.Zipkin\OpenCensus.Exporter.Zipkin.csproj" />

src/Samples/TestStackdriver.cs

@@ -0,0 +1,70 @@
+namespace Samples
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Threading;
+    using OpenCensus.Exporter.Stackdriver;
+    using OpenCensus.Stats;
+    using OpenCensus.Stats.Aggregations;
+    using OpenCensus.Stats.Measures;
+    using OpenCensus.Tags;
+    using OpenCensus.Trace;
+    using OpenCensus.Trace.Sampler;
+
+    internal class TestStackdriver
+    {
+        private static ITracer tracer = Tracing.Tracer;
+        private static ITagger tagger = Tags.Tagger;
+
+        private static IStatsRecorder statsRecorder = Stats.StatsRecorder;
+        private static readonly IMeasureLong VideoSize = MeasureLong.Create("my.org/measure/video_size", "size of processed videos", "By");
+        private static readonly ITagKey FrontendKey = TagKey.Create("my.org/keys/frontend");
+
+        private static long MiB = 1 << 20;
+
+        private static readonly IViewName VideoSizeViewName = ViewName.Create("my.org/views/video_size");
+
+        private static readonly IView VideoSizeView = View.Create(
+            VideoSizeViewName,
+            "processed video size over time",
+            VideoSize,
+            Distribution.Create(BucketBoundaries.Create(new List<double>() { 0.0, 16.0 * MiB, 256.0 * MiB })),
+            new List<ITagKey>() { FrontendKey });
+
+        internal static void Run()
+        {
+            Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", @"C:\Users\zeltser\Downloads\aspnetcoreissue-1af8a0ca869b.json");
+            var exporter = new StackdriverExporter("aspnetcoreissue", Tracing.ExportComponent);
+            exporter.Start();
+
+            ITagContextBuilder tagContextBuilder = tagger.CurrentBuilder.Put(FrontendKey, TagValue.Create("mobile-ios9.3.5"));
+
+            var spanBuilder = tracer
+                .SpanBuilder("incoming request")
+                .SetRecordEvents(true)
+                .SetSampler(Samplers.AlwaysSample);
+
+            Stats.ViewManager.RegisterView(VideoSizeView);
+
+            using (var scopedTags = tagContextBuilder.BuildScoped())
+            {
+                using (var scopedSpan = spanBuilder.StartScopedSpan())
+                {
+                    tracer.CurrentSpan.AddAnnotation("Start processing video.");
+                    Thread.Sleep(TimeSpan.FromMilliseconds(10));
+                    statsRecorder.NewMeasureMap().Put(VideoSize, 25 * MiB).Record();
+                    tracer.CurrentSpan.AddAnnotation("Finished processing video.");
+                }
+            }
+
+            Thread.Sleep(TimeSpan.FromMilliseconds(5100));
+
+            var viewData = Stats.ViewManager.GetView(VideoSizeViewName);
+
+            Console.WriteLine(viewData);
+
+            Console.WriteLine("Done... wait for events to arrive to backend!");
+            Console.ReadLine();
+        }
+    }
+}