gitlab-labkit-0.5.2/0000755000175000017500000000000013570141177013256 5ustar pravipravigitlab-labkit-0.5.2/.rubocop.yml0000644000175000017500000000466613570141177015544 0ustar pravipraviAllCops: TargetRubyVersion: 2.4 require: - rubocop-rspec Style/HashSyntax: EnforcedStyle: no_mixed_keys Style/SymbolLiteral: Enabled: No Style/TrailingCommaInArguments: Enabled: No # Delegated to rufo Style/TrailingCommaInHashLiteral: Enabled: No # Delegated to rufo Style/FrozenStringLiteralComment: EnforcedStyle: always Style/TrailingCommaInArrayLiteral: EnforcedStyleForMultiline: comma Style/StringLiterals: EnforcedStyle: double_quotes Style/StringLiteralsInInterpolation: EnforcedStyle: double_quotes Layout/MultilineMethodCallIndentation: Enabled: No Layout/SpaceInLambdaLiteral: Enabled: No Layout/SpaceInsideBlockBraces: Enabled: No Layout/FirstParameterIndentation: Enabled: No Metrics/AbcSize: Enabled: true Max: 54.28 Metrics/BlockLength: Enabled: false Metrics/BlockNesting: Enabled: true Max: 4 Metrics/ClassLength: Enabled: false Metrics/CyclomaticComplexity: Enabled: true Max: 13 Metrics/LineLength: Enabled: false Metrics/MethodLength: Enabled: false Metrics/ModuleLength: Enabled: false Metrics/ParameterLists: Enabled: true Max: 8 Metrics/PerceivedComplexity: Enabled: true Max: 14 RSpec/AnyInstance: Enabled: false RSpec/BeEql: Enabled: true RSpec/BeforeAfterAll: Enabled: false RSpec/DescribeClass: Enabled: false RSpec/DescribeMethod: Enabled: false RSpec/DescribeSymbol: Enabled: true RSpec/DescribedClass: Enabled: true RSpec/EmptyExampleGroup: Enabled: true RSpec/ExampleLength: Enabled: false Max: 5 RSpec/ExampleWording: Enabled: false CustomTransform: be: is have: has not: does not IgnoredWords: [] RSpec/ExpectActual: Enabled: true RSpec/ExpectOutput: Enabled: true RSpec/FilePath: Enabled: true IgnoreMethods: true RSpec/Focus: Enabled: true RSpec/HookArgument: Enabled: true EnforcedStyle: implicit RSpec/ImplicitExpect: Enabled: true EnforcedStyle: is_expected RSpec/InstanceVariable: Enabled: false RSpec/LeadingSubject: Enabled: false RSpec/LetSetup: Enabled: false RSpec/MessageChain: Enabled: false RSpec/MessageSpies: Enabled: false RSpec/MultipleDescribes: Enabled: false RSpec/MultipleExpectations: Enabled: false RSpec/NamedSubject: Enabled: false RSpec/NestedGroups: Enabled: false RSpec/NotToNot: EnforcedStyle: not_to Enabled: true RSpec/RepeatedDescription: Enabled: false RSpec/SubjectStub: Enabled: false RSpec/VerifiedDoubles: Enabled: false gitlab-labkit-0.5.2/LICENSE0000644000175000017500000000207313570141177014265 0ustar pravipraviThe MIT License (MIT) Copyright (c) 2016-2019 GitLab B.V. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. gitlab-labkit-0.5.2/Rakefile0000644000175000017500000000222613570141177014725 0ustar pravipravi# frozen_string_literal: true require "bundler/gem_tasks" require "rufo" begin require "rspec/core/rake_task" RSpec::Core::RakeTask.new(:spec) end require "rubocop/rake_task" RuboCop::RakeTask.new desc "Alias for `rake rufo:run`" task :format => ["rufo:run"] namespace :rufo do require "rufo" def rufo_command(*switches, rake_args) files_or_dirs = rake_args[:files_or_dirs] || "." args = switches + files_or_dirs.split(" ") Rufo::Command.run(args) end desc "Format Ruby code in current directory" task :run, [:files_or_dirs] do |_task, rake_args| rufo_command(rake_args) end desc "Check that no formatting changes are produced" task :check, [:files_or_dirs] do |_task, rake_args| rufo_command("--check", rake_args) end end desc "Generate test protobuf stubs" task :gen_test_proto do system "grpc_tools_ruby_protoc --ruby_out=. --grpc_out=. spec/support/grpc_service/test.proto" Rufo::Command.run(["spec/support/grpc_service/test_pb.rb", "spec/support/grpc_service/test_services_pb.rb"]) end task :fix => %w[rufo:run rubocop:auto_correct] task :verify => %w[spec rufo:check rubocop] task :default => %w[verify build] gitlab-labkit-0.5.2/.gitlab-ci.yml0000644000175000017500000000060413570141177015712 0ustar pravipravi.test_template: &test_definition stage: test script: - gem install bundler - bundle install - bundle exec rake verify build install test:2.6: image: ruby:2.6 <<: *test_definition test:2.5: image: ruby:2.5 <<: *test_definition test:2.4: image: ruby:2.4 <<: *test_definition deploy: stage: deploy script: - tools/deploy-rubygem.sh only: - tags gitlab-labkit-0.5.2/.rufo0000644000175000017500000000010513570141177014226 0ustar pravipravialign_chained_calls true parens_in_def :dynamic trailing_commas true gitlab-labkit-0.5.2/Gemfile0000644000175000017500000000010613570141177014546 0ustar pravipravi# frozen_string_literal: true source "https://rubygems.org" gemspec gitlab-labkit-0.5.2/.gitignore0000644000175000017500000000004013570141177015240 0ustar pravipraviGemfile.lock *.gem node_modules gitlab-labkit-0.5.2/.rspec0000644000175000017500000000004513570141177014372 0ustar pravipravi--color --require ./spec/spec_helper gitlab-labkit-0.5.2/.ruby-version0000644000175000017500000000001313570141177015715 0ustar pravipraviruby-2.5.3 gitlab-labkit-0.5.2/lib/0000755000175000017500000000000013570141177014024 5ustar pravipravigitlab-labkit-0.5.2/lib/gitlab-labkit.rb0000644000175000017500000000060113570141177017054 0ustar pravipravi# rubocop:disable Naming/FileName # frozen_string_literal: true require "active_support/all" # LabKit is a module for handling cross-project # infrastructural concerns, partcularly related to # observability. module Labkit autoload :Correlation, "labkit/correlation" autoload :Tracing, "labkit/tracing" autoload :Logging, "labkit/logging" end # rubocop:enable Naming/FileName gitlab-labkit-0.5.2/lib/labkit/0000755000175000017500000000000013570141177015272 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/0000755000175000017500000000000013570141177016721 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/redis.rb0000644000175000017500000000072513570141177020360 0ustar pravipravi# frozen_string_literal: true require "redis" module Labkit module Tracing # The Redis interceptor will intercept all calls to Redis and instrument them for distributed tracing module Redis autoload :RedisInterceptor, "labkit/tracing/redis/redis_interceptor" autoload :RedisInterceptorHelper, "labkit/tracing/redis/redis_interceptor_helper" def self.instrument ::Redis::Client.prepend RedisInterceptor end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/tracing_utils.rb0000644000175000017500000000411113570141177022112 0ustar pravipravi# frozen_string_literal: true require "opentracing" module Labkit module Tracing # Internal methods for tracing. This is not part of the LabKit public API. # For internal usage only class TracingUtils # Convience method for running a block with a span def self.with_tracing(operation_name:, tags:, child_of: nil) scope = tracer.start_active_span(operation_name, child_of: child_of, tags: tags) span = scope.span # Add correlation details to the span if we have them correlation_id = Labkit::Correlation::CorrelationId.current_id span.set_tag("correlation_id", correlation_id) if correlation_id begin yield span rescue StandardError => e log_exception_on_span(span, e) raise e ensure scope.close end end # Obtain a tracer instance def self.tracer OpenTracing.global_tracer end # Generate a span retrospectively def self.postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil) span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of) log_exception_on_span(span, exception) if exception span.finish(end_time: end_time) end # Add exception logging to a span def self.log_exception_on_span(span, exception) span.set_tag("error", true) span.log_kv(kv_tags_for_exception(exception)) end # Generate key-value tags for an exception def self.kv_tags_for_exception(exception) case exception when Exception { :"event" => "error", :"error.kind" => exception.class.to_s, :"message" => Labkit::Logging::Sanitizer.sanitize_field(exception.message), :"stack" => exception.backtrace&.join('\n'), } else { :"event" => "error", :"error.kind" => exception.class.to_s, :"error.object" => Labkit::Logging::Sanitizer.sanitize_field(exception.to_s) } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/grpc_interceptor.rb0000644000175000017500000000027513570141177022623 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing # GRPCInterceptor is the deprecated name for GRPCClientInterceptor GRPCInterceptor = GRPC::ClientInterceptor end end gitlab-labkit-0.5.2/lib/labkit/tracing/factory.rb0000644000175000017500000000323213570141177020715 0ustar pravipravi# frozen_string_literal: true require "cgi" module Labkit module Tracing # Factory provides tools for setting up and configuring the # distributed tracing system within the process, given the # tracing connection string class Factory OPENTRACING_SCHEME = "opentracing" def self.create_tracer(service_name, connection_string) return unless connection_string.present? begin opentracing_details = parse_connection_string(connection_string) driver_name = opentracing_details[:driver_name] case driver_name when "jaeger" JaegerFactory.create_tracer(service_name, opentracing_details[:options]) else raise "Unknown driver: #{driver_name}" end # Can't create the tracer? Warn and continue sans tracer rescue StandardError => e warn "Unable to instantiate tracer: #{e}" nil end end def self.parse_connection_string(connection_string) parsed = URI.parse(connection_string) raise "Invalid tracing connection string" unless valid_uri?(parsed) { driver_name: parsed.host, options: parse_query(parsed.query) } end private_class_method :parse_connection_string def self.parse_query(query) return {} unless query CGI.parse(query).symbolize_keys.transform_values(&:first) end private_class_method :parse_query def self.valid_uri?(uri) return false unless uri uri.scheme == OPENTRACING_SCHEME && uri.host.to_s =~ /^[a-z0-9_]+$/ && uri.path.empty? end private_class_method :valid_uri? end end end gitlab-labkit-0.5.2/lib/labkit/tracing/redis/0000755000175000017500000000000013570141177020027 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/redis/redis_interceptor_helper.rb0000644000175000017500000000704213570141177025442 0ustar pravipravi# frozen_string_literal: true require "redis" module Labkit module Tracing module Redis # RedisInterceptorHelper is a helper for the RedisInterceptor. This is not a public API class RedisInterceptorHelper # For optimization, compile this once MASK_REDIS_RE = /^([\w-]+(?:\W+[\w-]+(?:\W+[\w-]+)?)?)(.?)/.freeze def self.call_with_tracing(command, client) Labkit::Tracing::TracingUtils.with_tracing(operation_name: "redis.call", tags: tags_from_command(command, client)) do |_span| yield end end def self.call_pipeline_with_tracing(pipeline, client) Labkit::Tracing::TracingUtils.with_tracing(operation_name: "redis.call_pipeline", tags: tags_from_pipeline(pipeline, client)) do |_span| yield end end def self.common_tags_for_client(client) { "component" => "redis", "span.kind" => "client", "redis.scheme" => client.scheme, "redis.host" => client.host, "redis.port" => client.port, "redis.path" => client.path, } end def self.tags_from_command(command, client) tags = common_tags_for_client(client) tags["redis.command"] = command_serialized(command) tags end def self.command_serialized(command) return "" unless command.is_a?(Array) return "" if command.empty? command_name, *arguments = command command_name ||= "nil" info = [command_name] info << sanitize_argument_for_command(command_name, arguments.first) unless arguments.empty? info << "...#{arguments.size - 1} more value(s)" if arguments.size > 1 info.join(" ") end def self.tags_from_pipeline(pipeline, client) tags = common_tags_for_client(client) commands = pipeline.commands # Limit to the first 5 commands commands.first(5).each_with_index do |command, index| tags["redis.command.#{index}"] = command_serialized(command) end tags["redis.pipeline.commands.length"] = commands.length tags end # get_first_argument_for_command returns a masked value representing the first argument # from a redis command, taking care of certain sensitive commands def self.sanitize_argument_for_command(command_name, first_argument) return "*****" if command_is_sensitive?(command_name) return "nil" if first_argument.nil? return first_argument if first_argument.is_a?(Numeric) return "*****" unless first_argument.is_a?(String) mask_redis_arg(first_argument) end # Returns true if the arguments for the command should be masked def self.command_is_sensitive?(command_name) command_is?(command_name, :auth) || command_is?(command_name, :eval) end # Returns true if the command is equivalent to the command_symbol symbol def self.command_is?(command_name, command_symbol) if command_name.is_a?(Symbol) command_name == command_symbol else command_name.to_s.casecmp(command_symbol.to_s).zero? end end def self.mask_redis_arg(argument) return "" if argument.empty? matches = argument.match(MASK_REDIS_RE) matches[2].empty? ? matches[0] : matches[0] + "*****" end private_class_method :mask_redis_arg end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/redis/redis_interceptor.rb0000644000175000017500000000147113570141177024103 0ustar pravipravi# frozen_string_literal: true require "redis" module Labkit module Tracing module Redis # RedisInterceptor is an interceptor for Redis to add distributed tracing. # It should be installed using the `Labkit::Tracing.instrument` method module RedisInterceptor def call(command) RedisInterceptorHelper.call_with_tracing(command, self) do # Note: when used without any arguments super uses the arguments given to the subclass method. super end end def call_pipeline(pipeline) RedisInterceptorHelper.call_pipeline_with_tracing(pipeline, self) do # Note: when used without any arguments super uses the arguments given to the subclass method. super end end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/rails.rb0000644000175000017500000000074413570141177020365 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing # Rails provides classes for instrumenting Rails events module Rails autoload :ActionViewSubscriber, "labkit/tracing/rails/action_view_subscriber" autoload :ActiveRecordSubscriber, "labkit/tracing/rails/active_record_subscriber" autoload :ActiveSupportSubscriber, "labkit/tracing/rails/active_support_subscriber" autoload :RailsCommon, "labkit/tracing/rails/rails_common" end end end gitlab-labkit-0.5.2/lib/labkit/tracing/sidekiq.rb0000644000175000017500000000064313570141177020702 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing # Sidekiq provides classes for instrumenting Sidekiq client and server # functionality module Sidekiq autoload :ClientMiddleware, "labkit/tracing/sidekiq/client_middleware" autoload :ServerMiddleware, "labkit/tracing/sidekiq/server_middleware" autoload :SidekiqCommon, "labkit/tracing/sidekiq/sidekiq_common" end end end gitlab-labkit-0.5.2/lib/labkit/tracing/rack_middleware.rb0000644000175000017500000000243113570141177022363 0ustar pravipravi# frozen_string_literal: true require "opentracing" require "active_support/all" require "action_dispatch" module Labkit module Tracing # RackMiddleware is a rack middleware component for # instrumenting incoming http requests into a Rails/Rack # server class RackMiddleware REQUEST_METHOD = "REQUEST_METHOD" def initialize(app) @app = app end def call(env) method = env[REQUEST_METHOD] context = TracingUtils.tracer.extract(OpenTracing::FORMAT_RACK, env) tags = { "component" => "rack", "span.kind" => "server", "http.method" => method, "http.url" => self.class.build_sanitized_url_from_env(env) } TracingUtils.with_tracing(operation_name: "http:#{method}", child_of: context, tags: tags) do |span| @app.call(env).tap { |status_code, _headers, _body| span.set_tag("http.status_code", status_code) } end end # Generate a sanitized (safe) request URL from the rack environment def self.build_sanitized_url_from_env(env) request = ::ActionDispatch::Request.new(env) original_url = request.original_url uri = URI.parse(original_url) uri.query = request.filtered_parameters.to_query if uri.query.present? uri.to_s end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/sidekiq/0000755000175000017500000000000013570141177020352 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/sidekiq/client_middleware.rb0000644000175000017500000000135513570141177024356 0ustar pravipravi# frozen_string_literal: true require "opentracing" module Labkit module Tracing module Sidekiq # ClientMiddleware provides a sidekiq client middleware for # instrumenting distributed tracing calls made from the client # application class ClientMiddleware include SidekiqCommon SPAN_KIND = "client" def call(_worker_class, job, _queue, _redis_pool) TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", tags: tags_from_job(job, SPAN_KIND)) do |span| # Inject the details directly into the job TracingUtils.tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job) yield end end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/sidekiq/sidekiq_common.rb0000644000175000017500000000103613570141177023700 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing module Sidekiq # SidekiqCommon is a mixin for the sidekiq middleware components module SidekiqCommon def tags_from_job(job, kind) { "component" => "sidekiq", "span.kind" => kind, "sidekiq.queue" => job["queue"], "sidekiq.jid" => job["jid"], "sidekiq.retry" => job["retry"].to_s, "sidekiq.args" => job["args"]&.join(", "), } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/sidekiq/server_middleware.rb0000644000175000017500000000124513570141177024404 0ustar pravipravi# frozen_string_literal: true require "opentracing" module Labkit module Tracing module Sidekiq # ServerMiddleware provides a sidekiq server middleware for # instrumenting distributed tracing calls when they are # executed by the Sidekiq server class ServerMiddleware include SidekiqCommon SPAN_KIND = "server" def call(_worker, job, _queue) context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job) TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/grpc.rb0000644000175000017500000000050213570141177020176 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing # The GRPC module contains functionality for instrumenting GRPC calls module GRPC autoload :ClientInterceptor, "labkit/tracing/grpc/client_interceptor" autoload :ServerInterceptor, "labkit/tracing/grpc/server_interceptor" end end end gitlab-labkit-0.5.2/lib/labkit/tracing/jaeger_factory.rb0000644000175000017500000000544713570141177022244 0ustar pravipravi# frozen_string_literal: true require "jaeger/client" module Labkit module Tracing # JaegerFactory will configure Jaeger distributed tracing class JaegerFactory # When the probabilistic sampler is used, by default 0.1% of requests will be traced DEFAULT_PROBABILISTIC_RATE = 0.001 # The default port for the Jaeger agent UDP listener DEFAULT_UDP_PORT = 6831 # Reduce this from default of 10 seconds as the Ruby jaeger # client doesn't have overflow control, leading to very large # messages which fail to send over UDP (max packet = 64k) # Flush more often, with smaller packets FLUSH_INTERVAL = 5 def self.create_tracer(service_name, options) kwargs = { service_name: service_name, sampler: get_sampler(options[:sampler], options[:sampler_param]), reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]), }.compact extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) if extra_params.present? message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}" raise message if options[:strict_parsing] warn message end Jaeger::Client.build(kwargs) end def self.get_sampler(sampler_type, sampler_param) case sampler_type when "probabilistic" sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE Jaeger::Samplers::Probabilistic.new(rate: sampler_rate) when "const" const_value = sampler_param == "1" Jaeger::Samplers::Const.new(const_value) end end private_class_method :get_sampler def self.get_reporter(service_name, http_endpoint, udp_endpoint) encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name) if http_endpoint.present? sender = get_http_sender(encoder, http_endpoint) elsif udp_endpoint.present? sender = get_udp_sender(encoder, udp_endpoint) else return nil end Jaeger::Reporters::RemoteReporter.new(sender: sender, flush_interval: FLUSH_INTERVAL) end private_class_method :get_reporter def self.get_http_sender(encoder, address) Jaeger::HttpSender.new(url: address, encoder: encoder, logger: Logger.new(STDOUT)) end private_class_method :get_http_sender def self.get_udp_sender(encoder, address) pair = address.split(":", 2) host = pair[0] port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT Jaeger::UdpSender.new(host: host, port: port, encoder: encoder, logger: Logger.new(STDOUT)) end private_class_method :get_udp_sender end end end gitlab-labkit-0.5.2/lib/labkit/tracing/grpc/0000755000175000017500000000000013570141177017654 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/grpc/client_interceptor.rb0000644000175000017500000000306113570141177024075 0ustar pravipravi# frozen_string_literal: true # Disable the UnusedMethodArgument linter, since we need to declare the kwargs # in the methods, but we don't actually use them. # rubocop:disable Lint/UnusedMethodArgument require "opentracing" require "grpc" module Labkit module Tracing module GRPC # GRPCClientInterceptor is a client-side GRPC interceptor # for instrumenting GRPC calls with distributed tracing class ClientInterceptor < ::GRPC::ClientInterceptor include Singleton def request_response(request:, call:, method:, metadata:) wrap_with_tracing(method, "unary", metadata) { yield } end def client_streamer(requests:, call:, method:, metadata:) wrap_with_tracing(method, "client_stream", metadata) { yield } end def server_streamer(request:, call:, method:, metadata:) wrap_with_tracing(method, "server_stream", metadata) { yield } end def bidi_streamer(requests:, call:, method:, metadata:) wrap_with_tracing(method, "bidi_stream", metadata) { yield } end private def wrap_with_tracing(method, grpc_type, metadata) tags = { "component" => "grpc", "span.kind" => "client", "grpc.method" => method, "grpc.type" => grpc_type } TracingUtils.with_tracing(operation_name: "grpc:#{method}", tags: tags) do |span| OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata) yield end end end end end end # rubocop:enable Lint/UnusedMethodArgument gitlab-labkit-0.5.2/lib/labkit/tracing/grpc/server_interceptor.rb0000644000175000017500000000375613570141177024140 0ustar pravipravi# frozen_string_literal: true # Disable the UnusedMethodArgument linter, since we need to declare the kwargs # in the methods, but we don't actually use them. # rubocop:disable Lint/UnusedMethodArgument require "opentracing" require "grpc" module Labkit module Tracing module GRPC # GRPCServerInterceptor is a server-side GRPC interceptor # for instrumenting GRPC calls with distributed tracing # in a GRPC Ruby server class ServerInterceptor < ::GRPC::ServerInterceptor def request_response(request: nil, call: nil, method: nil) wrap_with_tracing(call, method, "unary") do yield end end def client_streamer(call: nil, method: nil) wrap_with_tracing(call, method, "client_stream") do yield end end def server_streamer(request: nil, call: nil, method: nil) wrap_with_tracing(call, method, "server_stream") do yield end end def bidi_streamer(requests: nil, call: nil, method: nil) wrap_with_tracing(call, method, "bidi_stream") do yield end end private def route_from_method(method) service_class = method.owner rpc_method = method.name.to_s.split("_").map(&:capitalize).join("") "/#{service_class.service_name}/#{rpc_method}" end def wrap_with_tracing(call, method, grpc_type) context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, call.metadata) method_name = route_from_method(method) tags = { "component" => "grpc", "span.kind" => "server", "grpc.method" => method_name, "grpc.type" => grpc_type, } TracingUtils.with_tracing(operation_name: "grpc:#{method_name}", child_of: context, tags: tags) do |_span| yield end end end end end end # rubocop:enable Lint/UnusedMethodArgument gitlab-labkit-0.5.2/lib/labkit/tracing/rails/0000755000175000017500000000000013570141177020033 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/tracing/rails/active_record_subscriber.rb0000644000175000017500000000327313570141177025421 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing module Rails # ActiveRecordSubscriber bridges active record notifications to # the distributed tracing subsystem class ActiveRecordSubscriber include RailsCommon ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record" OPERATION_NAME_PREFIX = "active_record:" DEFAULT_OPERATION_NAME = "sqlquery" # Instruments Rails ActiveRecord events for opentracing. # Returns a lambda, which, when called will unsubscribe from the notifications def self.instrument subscriber = new subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| subscriber.notify(start, finish, payload) end create_unsubscriber [subscription] end # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html def notify(start, finish, payload) generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload)) end private def notification_name(payload) OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME) end def tags_for_notification(payload) { "component" => "ActiveRecord", "span.kind" => "client", "db.type" => "sql", "db.connection_id" => payload[:connection_id], "db.cached" => payload[:cached] || false, "db.statement" => payload[:sql], } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/rails/active_support_subscriber.rb0000644000175000017500000000641113570141177025654 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing module Rails # ActiveSupport bridges action active support notifications to # the distributed tracing subsystem class ActiveSupportSubscriber include RailsCommon COMPONENT_TAG = "ActiveSupport" CACHE_READ_TOPIC = "cache_read.active_support" CACHE_GENERATE_TOPIC = "cache_generate.active_support" CACHE_FETCH_HIT_TOPIC = "cache_fetch_hit.active_support" CACHE_WRITE_TOPIC = "cache_write.active_support" CACHE_DELETE_TOPIC = "cache_delete.active_support" # Instruments Rails ActiveSupport events for opentracing. # Returns a lambda, which, when called will unsubscribe from the notifications def self.instrument subscriber = new subscriptions = [ ActiveSupport::Notifications.subscribe(CACHE_READ_TOPIC) do |_, start, finish, _, payload| subscriber.notify_cache_read(start, finish, payload) end, ActiveSupport::Notifications.subscribe(CACHE_GENERATE_TOPIC) do |_, start, finish, _, payload| subscriber.notify_cache_generate(start, finish, payload) end, ActiveSupport::Notifications.subscribe(CACHE_FETCH_HIT_TOPIC) do |_, start, finish, _, payload| subscriber.notify_cache_fetch_hit(start, finish, payload) end, ActiveSupport::Notifications.subscribe(CACHE_WRITE_TOPIC) do |_, start, finish, _, payload| subscriber.notify_cache_write(start, finish, payload) end, ActiveSupport::Notifications.subscribe(CACHE_DELETE_TOPIC) do |_, start, finish, _, payload| subscriber.notify_cache_delete(start, finish, payload) end, ] create_unsubscriber subscriptions end # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html#active-support def notify_cache_read(start, finish, payload) generate_span_for_notification("cache_read", start, finish, payload, tags_for_cache_read(payload)) end def notify_cache_generate(start, finish, payload) generate_span_for_notification("cache_generate", start, finish, payload, tags_for_key(payload)) end def notify_cache_fetch_hit(start, finish, payload) generate_span_for_notification("cache_fetch_hit", start, finish, payload, tags_for_key(payload)) end def notify_cache_write(start, finish, payload) generate_span_for_notification("cache_write", start, finish, payload, tags_for_key(payload)) end def notify_cache_delete(start, finish, payload) generate_span_for_notification("cache_delete", start, finish, payload, tags_for_key(payload)) end private def tags_for_cache_read(payload) { "component" => COMPONENT_TAG, "cache.key" => payload[:key], "cache.hit" => payload[:hit], "cache.super_operation" => payload[:super_operation], } end def tags_for_key(payload) { "component" => COMPONENT_TAG, "cache.key" => payload[:key], } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/rails/action_view_subscriber.rb0000644000175000017500000000536113570141177025117 0ustar pravipravi# frozen_string_literal: true module Labkit module Tracing module Rails # ActionViewSubscriber bridges action view notifications to # the distributed tracing subsystem class ActionViewSubscriber include RailsCommon COMPONENT_TAG = "ActionView" RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view" RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view" RENDER_PARTIAL_NOTIFICATION_TOPIC = "render_partial.action_view" # Instruments Rails ActionView events for opentracing. # Returns a lambda, which, when called will unsubscribe from the notifications def self.instrument subscriber = new subscriptions = [ ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| subscriber.notify_render_template(start, finish, payload) end, ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| subscriber.notify_render_collection(start, finish, payload) end, ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| subscriber.notify_render_partial(start, finish, payload) end, ] create_unsubscriber subscriptions end # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html def notify_render_template(start, finish, payload) generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload)) end def notify_render_collection(start, finish, payload) generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload)) end def notify_render_partial(start, finish, payload) generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload)) end private def tags_for_render_template(payload) { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.layout" => payload[:layout] } end def tags_for_render_collection(payload) { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.count" => payload[:count] || 0, "template.cache.hits" => payload[:cache_hits] || 0, } end def tags_for_render_partial(payload) { "component" => COMPONENT_TAG, "template.id" => payload[:identifier] } end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing/rails/rails_common.rb0000644000175000017500000000141313570141177023041 0ustar pravipravi# frozen_string_literal: true require "active_support/all" module Labkit module Tracing module Rails # RailsCommon is a mixin for providing instrumentation # functionality for the rails instrumentation classes module RailsCommon extend ActiveSupport::Concern class_methods do def create_unsubscriber(subscriptions) -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } } end end def generate_span_for_notification(operation_name, start, finish, payload, tags) exception = payload[:exception] TracingUtils.postnotify_span(operation_name, start, finish, tags: tags, exception: exception) end end end end end gitlab-labkit-0.5.2/lib/labkit/logging.rb0000644000175000017500000000030513570141177017243 0ustar pravipravi# frozen_string_literal: true module Labkit # Logging provides functionality for logging, such as # sanitization module Logging autoload :Sanitizer, "labkit/logging/sanitizer" end end gitlab-labkit-0.5.2/lib/labkit/correlation/0000755000175000017500000000000013570141177017613 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/correlation/correlation_id.rb0000644000175000017500000000150113570141177023132 0ustar pravipravi# frozen_string_literal: true module Labkit module Correlation # CorrelationId module provides access the Correlation-ID # of the current request module CorrelationId LOG_KEY = "correlation_id" class << self def use_id(correlation_id, &_blk) # always generate a id if null is passed correlation_id ||= new_id ids.push(correlation_id || new_id) begin yield(current_id) ensure ids.pop end end def current_id ids.last end def current_or_new_id current_id || new_id end private def ids Thread.current[:correlation_id] ||= [] end def new_id SecureRandom.uuid end end end end end gitlab-labkit-0.5.2/lib/labkit/tracing.rb0000644000175000017500000000377113570141177017256 0ustar pravipravi# frozen_string_literal: true require "active_support/all" module Labkit # Tracing provides distributed tracing functionality module Tracing autoload :Factory, "labkit/tracing/factory" autoload :GRPC, "labkit/tracing/grpc" autoload :GRPCInterceptor, "labkit/tracing/grpc_interceptor" # Deprecated autoload :JaegerFactory, "labkit/tracing/jaeger_factory" autoload :RackMiddleware, "labkit/tracing/rack_middleware" autoload :Rails, "labkit/tracing/rails" autoload :Redis, "labkit/tracing/redis" autoload :Sidekiq, "labkit/tracing/sidekiq" autoload :TracingUtils, "labkit/tracing/tracing_utils" # Tracing is only enabled when the `GITLAB_TRACING` env var is configured. def self.enabled? connection_string.present? end def self.connection_string ENV["GITLAB_TRACING"] end def self.tracing_url_template ENV["GITLAB_TRACING_URL"] end def self.tracing_url_enabled? enabled? && tracing_url_template.present? end # This will provide a link into the distributed tracing for the current trace, # if it has been captured. def self.tracing_url(service_name) return unless tracing_url_enabled? correlation_id = Labkit::Correlation::CorrelationId.current_id.to_s # Avoid using `format` since it can throw TypeErrors # which we want to avoid on unsanitised env var input tracing_url_template.to_s .gsub("{{ correlation_id }}", correlation_id) .gsub("{{ service }}", service_name) end # This will run a block with a span # @param operation_name [String] The operation name for the span # @param tags [Hash] Tags to assign to the span # @param child_of [SpanContext, Span] SpanContext that acts as a parent to # the newly-started span. If a span instance is provided, its # context is automatically substituted. def self.with_tracing(**kwargs, &block) TracingUtils.with_tracing(**kwargs, &block) end end end gitlab-labkit-0.5.2/lib/labkit/correlation.rb0000644000175000017500000000030013570141177020131 0ustar pravipravi# frozen_string_literal: true module Labkit # Correlation provides correlation functionality module Correlation autoload :CorrelationId, "labkit/correlation/correlation_id" end end gitlab-labkit-0.5.2/lib/labkit/logging/0000755000175000017500000000000013570141177016720 5ustar pravipravigitlab-labkit-0.5.2/lib/labkit/logging/sanitizer.rb0000644000175000017500000000411713570141177021260 0ustar pravipravi# frozen_string_literal: true module Labkit module Logging # Sanitizer provides log message sanitization, removing # confidential information from log messages class Sanitizer SCP_URL_REGEXP = %r{ (?:((?:[\-_.!~*()a-zA-Z\d;&=+$,]|%[a-fA-F\d]{2})+)(:(?:(?:[\-_.!~*()a-zA-Z\d;:&=+$,]|%[a-fA-F\d]{2})*))?@) (?# 1: username, 2: password) (?:((?:(?:[a-zA-Z0-9\-._])+|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:(?:[a-fA-F\d]{1,4}:)*[a-fA-F\d]{1,4})?::(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))?)\]))) (?# 3: host) : ((?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*(?:;(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*)*(?:\/(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*(?:;(?:[\-_.!~*'()a-zA-Z\d:@&=+$,]|%[a-fA-F\d]{2})*)*)*)? (?# 4: path) }x.freeze SCP_ANCHORED_URL_REGEXP = /^#{SCP_URL_REGEXP}$/x.freeze ALLOWED_SCHEMES = %w[http https ssh git].freeze URL_REGEXP = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES).freeze def self.sanitize_field(content) content = content.gsub(URL_REGEXP) { |url| mask_url(url) } content = content.gsub(SCP_URL_REGEXP) { |scp_url| mask_scp_url(scp_url) } content end # Ensures that URLS are sanitized to hide credentials def self.mask_url(url) url = url.to_s.strip p = URI::DEFAULT_PARSER.parse(url) p.password = "*****" if p.password.present? p.user = "*****" if p.user.present? p.to_s rescue URI::InvalidURIError "" end # Ensures that URLs of the form user:password@hostname:project.git are # sanitized to hide credentials def self.mask_scp_url(scp_url) scp_url = scp_url.to_s.strip m = SCP_ANCHORED_URL_REGEXP.match(scp_url) return "" unless m password = m[2] host = m[3] path = m[4] return "*****@#{host}:#{path}" unless password.present? "*****:*****@#{host}:#{path}" end end end end gitlab-labkit-0.5.2/README.md0000644000175000017500000000305313570141177014536 0ustar pravipravi# LabKit-Ruby 🔬🔬🔬🔬🔬 LabKit-Ruby is minimalist library to provide functionality for Ruby services at GitLab. LabKit-Ruby is the Ruby companion for [LabKit](https://gitlab.com/gitlab-org/labkit), a minimalist library to provide functionality for Go services at GitLab. LabKit-Ruby and LabKit are intended to provide similar functionality, but use the semantics of their respective languages, so are not intended to provide identical APIS. ## Documentation API Documentation is available at [the Rubydoc site](https://www.rubydoc.info/gems/gitlab-labkit/). ## Changelog The changelog is available via [**tagged release notes**](https://gitlab.com/gitlab-org/labkit-ruby/tags) ## Functionality LabKit-Ruby provides functionality in three areas: 1. `Labkit::Correlation` for handling and propagating Correlation-IDs. 1. `Labkit::Logging` for sanitizing log messages. 1. `Labkit::Tracing` for handling and propagating distributed traces. ## Developing Anyone can contribute! ```console $ git clone git@gitlab.com:gitlab-org/labkit-ruby.git $ cd labkit-ruby $ bundle install $ # Autoformat code and auto-correct linters $ bundle exec rake fix $ # Run tests, linters $ bundle exec rake verify ``` Note that LabKit-Ruby uses the [`rufo`](https://github.com/ruby-formatter/rufo) for auto-formatting. Please run `bundle exec rake fix` to auto-format your code before pushing. Please also review the [development section of the LabKit (go) README](https://gitlab.com/gitlab-org/labkit#developing-labkit) for details of the LabKit architectural philosophy. gitlab-labkit-0.5.2/gitlab-labkit.gemspec0000644000175000017500000000317213570141177017334 0ustar pravipravi# frozen_string_literal: true lib = File.expand_path("lib", __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "gitlab-labkit" spec.version = `git describe --tags`.chomp.gsub(/^v/, "") spec.authors = ["Andrew Newdigate"] spec.email = ["andrew@gitlab.com"] spec.summary = "Instrumentation for GitLab" spec.homepage = "https://gitlab.com/gitlab-org/labkit-ruby" spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/labkit-ruby" } spec.license = "MIT" spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) } spec.require_paths = ["lib"] spec.required_ruby_version = ">= 2.4.0" # Please maintain alphabetical order for dependencies spec.add_runtime_dependency "actionpack", "~> 5" spec.add_runtime_dependency "activesupport", "~> 5" spec.add_runtime_dependency "grpc", "~> 1.19" # Be sure to update the "grpc-tools" dev_depenency too spec.add_runtime_dependency "jaeger-client", "~> 0.10" spec.add_runtime_dependency "opentracing", "~> 0.4" spec.add_runtime_dependency "redis", ">3.0.0", "<5.0.0" # Please maintain alphabetical order for dev dependencies spec.add_development_dependency "grpc-tools", "~> 1.19" spec.add_development_dependency "rack", "~> 2.0" spec.add_development_dependency "rake", "~> 12.3" spec.add_development_dependency "rspec", "~> 3.6.0" spec.add_development_dependency "rspec-parameterized", "~> 0.4" spec.add_development_dependency "rubocop", "~> 0.65.0" spec.add_development_dependency "rubocop-rspec", "~> 1.22.1" spec.add_development_dependency "rufo", "~> 0.6" end