Revert "Extract otel-collector jobs"

This reverts commit 4e42969.

README.md

@@ -35,10 +35,8 @@ Configurable via files in colocated bosh jobs as described in the [docs][prom-sc
 scrapes are avoided in an HA environment.
 
 ### OTel Collector
-This is the Cloud Foundry distribution of the OTel Collector. It can be found in the
-[otel-collector-release](https://github.com/cloudfoundry/otel-collector-release) repo. It accepts metrics in OTLP format
-and forwards them to any configured exporters. More information can be found in the [OpenTelemetry
-docs](https://opentelemetry.io/docs/collector)
+This is the Cloud Foundry distribution of the OTel Collector. It accepts metrics in OTLP format and forwards them to any configured exporters.
+More information can be found in the [OpenTelemetry docs](https://opentelemetry.io/docs/collector)
 
 ## More Resources and Documentation
 

jobs/otel-collector-windows/monit

@@ -0,0 +1,15 @@
+<%
+  process = {
+    "name" => "otel-collector",
+    "executable" => "/var/vcap/packages/otel-collector-windows/otel-collector.exe",
+    "args" => ["--config", "/var/vcap/jobs/otel-collector-windows/config/config.yml"],
+    "env" => {"NO_WINDOWS_SERVICE" => "1"}
+  }
+
+  monit = { "processes" => [] }
+  if p('enabled')
+    monit["processes"] = [process]
+  end
+%>
+
+<%= JSON.pretty_generate(monit) %>

jobs/otel-collector-windows/spec

@@ -0,0 +1,41 @@
+---
+name: otel-collector-windows
+
+templates:
+  config.yml.erb: config/config.yml
+  ingress_port.yml.erb: config/ingress_port.yml
+  otel-collector.crt.erb: config/certs/otel-collector.crt
+  otel-collector.key.erb: config/certs/otel-collector.key
+  otel-collector-ca.crt.erb: config/certs/otel-collector-ca.crt
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+
+packages:
+- otel-collector-windows
+
+properties:
+  enabled:
+    description: "Enable OTel Collector"
+    default: true
+  ingress.grpc.port:
+    description: "Port the collector is listening on to receive OTLP over gRPC"
+    default: 9100
+  ingress.grpc.tls.ca_cert:
+    description: "CA root required for key/cert verification in gRPC ingress"
+  ingress.grpc.tls.cert:
+    description: "TLS server certificate for gRPC ingress"
+  ingress.grpc.tls.key:
+    description: "TLS server key for gRPC ingress"
+  telemetry.metrics.level:
+    description: "Level of metrics the collector exposes about itself"
+    default: "basic"
+  telemetry.metrics.port:
+    description: "Port to serve the collector's internal metrics"
+    default: 14830
+  metric_exporters:
+    description: "Exporter configuration for aggregate metric egress"
+    default: {}
+    example: |
+      otlp:
+        endpoint: otelcol:4317
+      otlp/2:
+        endpoint: otelcol:4318

jobs/otel-collector-windows/templates/config.yml.erb

@@ -0,0 +1,39 @@
+<%=
+metric_exporters = p('metric_exporters')
+unless metric_exporters.respond_to?(:keys)
+  metric_exporters = YAML::load(metric_exporters)
+end
+if metric_exporters.keys.any?{|k| k.include?('/cf-internal')}
+  raise 'Metric exporters cannot be defined under cf-internal namespace'
+end
+
+config = {
+  "receivers"=> {
+    "otlp"=>{
+      "protocols"=>{
+        "grpc"=>{
+          "endpoint"=>"127.0.0.1:#{p('ingress.grpc.port')}",
+          "tls"=>{
+            "client_ca_file"=>"/var/vcap/jobs/otel-collector-windows/config/certs/otel-collector-ca.crt",
+             "cert_file"=>"/var/vcap/jobs/otel-collector-windows/config/certs/otel-collector.crt",
+             "key_file"=>"/var/vcap/jobs/otel-collector-windows/config/certs/otel-collector.key",
+             "min_version"=>"1.3"
+          }
+        }
+      }
+    }
+  },
+  "exporters"=>metric_exporters,
+  "service"=>{
+    "telemetry"=>{
+      "metrics"=>{
+        "level"=>p('telemetry.metrics.level'),
+        "address"=>"127.0.0.1:#{p('telemetry.metrics.port')}"
+      }
+    },
+    "pipelines"=>{"metrics"=>{"receivers"=>["otlp"], "exporters"=>metric_exporters.keys}}
+  }
+}
+
+YAML::dump(config)
+%>

jobs/otel-collector-windows/templates/ingress_port.yml.erb

@@ -0,0 +1,5 @@
+<% if p('enabled') %>
+---
+ingress: <%= p('ingress.grpc.port') %>
+protocol: otelcol
+<% end %>

jobs/otel-collector-windows/templates/otel-collector-ca.crt.erb

@@ -0,0 +1 @@
+<%= p('ingress.grpc.tls.ca_cert') %>

jobs/otel-collector-windows/templates/otel-collector.crt.erb

@@ -0,0 +1 @@
+<%= p("ingress.grpc.tls.cert") %>

jobs/otel-collector-windows/templates/otel-collector.key.erb

@@ -0,0 +1 @@
+<%= p('ingress.grpc.tls.key') %>

jobs/otel-collector-windows/templates/prom_scraper_config.yml.erb

@@ -0,0 +1,9 @@
+<% if p('enabled') %>
+---
+port: <%= p("telemetry.metrics.port") %>
+source_id: "otel-collector"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: http
+labels:
+  origin: otel-collector
+<% end %>

jobs/otel-collector/monit

@@ -0,0 +1,7 @@
+<% if p('enabled') %>
+check process otel-collector
+  with pidfile /var/vcap/sys/run/bpm/otel-collector/otel-collector.pid
+  start program "/var/vcap/jobs/bpm/bin/bpm start otel-collector"
+  stop program "/var/vcap/jobs/bpm/bin/bpm stop otel-collector"
+  group vcap
+<% end %>

jobs/otel-collector/spec

@@ -0,0 +1,42 @@
+---
+name: otel-collector
+
+templates:
+  bpm.yml.erb: config/bpm.yml
+  config.yml.erb: config/config.yml
+  ingress_port.yml.erb: config/ingress_port.yml
+  otel-collector.crt.erb: config/certs/otel-collector.crt
+  otel-collector.key.erb: config/certs/otel-collector.key
+  otel-collector-ca.crt.erb: config/certs/otel-collector-ca.crt
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+
+packages:
+- otel-collector
+
+properties:
+  enabled:
+    description: "Enable OTel Collector"
+    default: true
+  ingress.grpc.port:
+    description: "Port the collector is listening on to receive OTLP over gRPC"
+    default: 9100
+  ingress.grpc.tls.ca_cert:
+    description: "CA root required for key/cert verification in gRPC ingress"
+  ingress.grpc.tls.cert:
+    description: "TLS server certificate for gRPC ingress"
+  ingress.grpc.tls.key:
+    description: "TLS server key for gRPC ingress"
+  telemetry.metrics.level:
+    description: "Level of metrics the collector exposes about itself"
+    default: "basic"
+  telemetry.metrics.port:
+    description: "Port to serve the collector's internal metrics"
+    default: 14830
+  metric_exporters:
+    description: "Exporter configuration for aggregate metric egress"
+    default: {}
+    example: |
+      otlp:
+        endpoint: otelcol:4317
+      otlp/2:
+        endpoint: otelcol:4318

jobs/otel-collector/templates/bpm.yml.erb

@@ -0,0 +1,4 @@
+processes:
+- name: otel-collector
+  executable: /var/vcap/packages/otel-collector/otel-collector
+  args: ["--config", "/var/vcap/jobs/otel-collector/config/config.yml"]

jobs/otel-collector/templates/config.yml.erb

@@ -0,0 +1,42 @@
+<%=
+metric_exporters = p('metric_exporters')
+unless metric_exporters.respond_to?(:keys)
+  metric_exporters = YAML::load(metric_exporters)
+end
+if metric_exporters.keys.any?{|k| k.include?('/cf-internal')}
+  raise 'Metric exporters cannot be defined under cf-internal namespace'
+end
+if metric_exporters.any?{|k, v| k.start_with?('prometheus') && v['endpoint'] && v['endpoint'].end_with?(':8889')}
+  raise 'Cannot define prometheus exporter listening on port 8889 (reserved for BBS API port)'
+end
+
+config = {
+  "receivers"=> {
+    "otlp"=>{
+      "protocols"=>{
+        "grpc"=>{
+          "endpoint"=>"127.0.0.1:#{p('ingress.grpc.port')}",
+          "tls"=>{
+            "client_ca_file"=>"/var/vcap/jobs/otel-collector/config/certs/otel-collector-ca.crt",
+             "cert_file"=>"/var/vcap/jobs/otel-collector/config/certs/otel-collector.crt",
+             "key_file"=>"/var/vcap/jobs/otel-collector/config/certs/otel-collector.key",
+             "min_version"=>"1.3"
+          }
+        }
+      }
+    }
+  },
+  "exporters"=>metric_exporters,
+  "service"=>{
+    "telemetry"=>{
+      "metrics"=>{
+        "level"=>p('telemetry.metrics.level'),
+        "address"=>"127.0.0.1:#{p('telemetry.metrics.port')}"
+      }
+    },
+    "pipelines"=>{"metrics"=>{"receivers"=>["otlp"], "exporters"=>metric_exporters.keys}}
+  }
+}
+
+YAML::dump(config)
+%>

jobs/otel-collector/templates/ingress_port.yml.erb

@@ -0,0 +1,5 @@
+<% if p('enabled') %>
+---
+ingress: <%= p('ingress.grpc.port') %>
+protocol: otelcol
+<% end %>

jobs/otel-collector/templates/otel-collector-ca.crt.erb

@@ -0,0 +1 @@
+<%= p('ingress.grpc.tls.ca_cert') %>

jobs/otel-collector/templates/otel-collector.crt.erb

@@ -0,0 +1 @@
+<%= p("ingress.grpc.tls.cert") %>

jobs/otel-collector/templates/otel-collector.key.erb

@@ -0,0 +1 @@
+<%= p('ingress.grpc.tls.key') %>

jobs/otel-collector/templates/prom_scraper_config.yml.erb

@@ -0,0 +1,9 @@
+<% if p('enabled') %>
+---
+port: <%= p("telemetry.metrics.port") %>
+source_id: "otel-collector"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: http
+labels:
+  origin: otel-collector
+<% end %>

packages/otel-collector-windows/packaging

@@ -0,0 +1,27 @@
+. ./exiter.ps1
+. C:\var\vcap\packages\golang-1.20-windows\bosh\compile.ps1
+$env:CGO_ENABLED="0"
+
+$ErrorActionPreference = "Stop";
+trap { $host.SetShouldExit(1) }
+
+$BOSH_INSTALL_TARGET = Resolve-Path "${env:BOSH_INSTALL_TARGET}"
+if ((Get-Item $BOSH_INSTALL_TARGET).LinkType -ne "") {
+  $BOSH_INSTALL_TARGET = (Get-Item $BOSH_INSTALL_TARGET).Target
+}
+
+Push-Location otel-collector-builder\vendor\go.opentelemetry.io\collector\cmd\builder\
+  go.exe install
+  if ($LASTEXITCODE -ne 0) {
+    Write-Error "Error compiling builder"
+  }
+Pop-Location
+
+& ${env:GOPATH}\bin\builder.exe --config otel-collector\config.yaml --skip-generate --skip-get-modules
+if ($LASTEXITCODE -ne 0) {
+  Write-Error "Error building otel-collector"
+}
+
+Copy-Item "otel-collector\cf-otel-collector" -Destination "${env:BOSH_INSTALL_TARGET}\otel-collector.exe"
+
+Exit 0

packages/otel-collector-windows/spec

@@ -0,0 +1,10 @@
+---
+name: otel-collector-windows
+
+dependencies:
+- golang-1.20-windows
+
+files:
+- exiter.ps1
+- otel-collector-builder/**/*
+- otel-collector/**/*

packages/otel-collector/packaging

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e -x
+
+source /var/vcap/packages/golang-1.20-linux/bosh/compile.env
+
+export CGO_ENABLED=0
+
+pushd otel-collector-builder/vendor/go.opentelemetry.io/collector/cmd/builder/
+  go install
+popd
+
+builder --config otel-collector/config.yaml --skip-generate --skip-get-modules
+
+cp "otel-collector/cf-otel-collector" "${BOSH_INSTALL_TARGET}/otel-collector"

packages/otel-collector/spec

@@ -0,0 +1,9 @@
+---
+name: otel-collector
+
+dependencies:
+- golang-1.20-linux
+
+files:
+- otel-collector-builder/**/*
+- otel-collector/**/*

scripts/regenerate-otel-collector-distribution

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euxo pipefail
+
+release_dir="$(dirname "$0")/.."
+
+pushd "${release_dir}/src/otel-collector-builder/vendor/go.opentelemetry.io/collector/cmd/builder"
+  CGO_ENABLED=0 go install
+popd
+
+pushd "${release_dir}/src/"
+  builder --skip-compilation --config=otel-collector/config.yaml
+  cd otel-collector
+  go mod vendor
+popd

src/otel-collector-builder/go.mod

@@ -0,0 +1,26 @@
+module code.cloudfoundry.org/loggregator-agent-release/src/otel-collector-builder
+
+go 1.20
+
+require go.opentelemetry.io/collector/cmd/builder v0.89.0
+
+require (
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect
+	github.com/knadh/koanf/providers/env v0.1.0 // indirect
+	github.com/knadh/koanf/providers/file v0.1.0 // indirect
+	github.com/knadh/koanf/providers/fs v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)