Add functions to implement Thanos Receive split functionality

.gitignore

@@ -1,3 +1,4 @@
 tmp
 vendor/
 jsonnet/kube-thanos/jsonnetfile.lock.json
+.idea/

CHANGELOG.md

@@ -44,7 +44,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 - [#226](https://github.com/thanos-io/kube-thanos/pull/226) Only schedule thanos components on linux nodes.
 
 ### Added
-
+- [#244](https://github.com/thanos-io/kube-thanos/pull/244) Add functions to implement Thanos Receive split functionality.
 - [#228](https://github.com/thanos-io/kube-thanos/pull/228) Allow configuring `--web.prefix-header` of query.
 
 [Full Changelog](https://github.com/thanos-io/kube-thanos/compare/v0.20.0...v0.21.0)

README.md

@@ -82,6 +82,7 @@ local commonConfig = {
       name: 'thanos-objectstorage',
       key: 'thanos.yaml',
     },
+    hashringConfigMapName: 'hashring-config',
     volumeClaimTemplate: {
       spec: {
         accessModes: ['ReadWriteOnce'],
@@ -95,6 +96,23 @@ local commonConfig = {
   },
 };
 
+local i = t.receiveIngestor(commonConfig.config {
+  replicas: 1,
+  replicaLabels: ['receive_replica'],
+  replicationFactor: 1,
+  // Disable shipping to object storage for the purposes of this example
+  objectStorageConfig: null,
+});
+
+local r = t.receiveRouter(commonConfig.config {
+  replicas: 1,
+  replicaLabels: ['receive_replica'],
+  replicationFactor: 1,
+  // Disable shipping to object storage for the purposes of this example
+  objectStorageConfig: null,
+  endpoints: i.endpoints,
+});
+
 local s = t.store(commonConfig.config {
   replicas: 1,
   serviceMonitor: true,
@@ -104,10 +122,19 @@ local q = t.query(commonConfig.config {
   replicas: 1,
   replicaLabels: ['prometheus_replica', 'rule_replica'],
   serviceMonitor: true,
+  stores: [s.storeEndpoint] + i.storeEndpoints,
 });
 
 { ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } +
-{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) }
+{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) } +
+{ ['thanos-receive-router-' + resource]: r[resource] for resource in std.objectFields(r) } +
+{ ['thanos-receive-ingestor-' + resource]: i[resource] for resource in std.objectFields(i) if resource != 'ingestors' } +
+{
+  ['thanos-receive-ingestor-' + hashring + '-' + resource]: i.ingestors[hashring][resource]
+  for hashring in std.objectFields(i.ingestors)
+  for resource in std.objectFields(i.ingestors[hashring])
+  if i.ingestors[hashring][resource] != null
+}
 ```
 
 And here's the [build.sh](build.sh) script (which uses `vendor/` to render all manifests in a json structure of `{filename: manifest-content}`):

example.jsonnet

@@ -12,6 +12,7 @@ local commonConfig = {
       name: 'thanos-objectstorage',
       key: 'thanos.yaml',
     },
+    hashringConfigMapName: 'hashring-config',
     volumeClaimTemplate: {
       spec: {
         accessModes: ['ReadWriteOnce'],
@@ -25,6 +26,23 @@ local commonConfig = {
   },
 };
 
+local i = t.receiveIngestor(commonConfig.config {
+  replicas: 1,
+  replicaLabels: ['receive_replica'],
+  replicationFactor: 1,
+  // Disable shipping to object storage for the purposes of this example
+  objectStorageConfig: null,
+});
+
+local r = t.receiveRouter(commonConfig.config {
+  replicas: 1,
+  replicaLabels: ['receive_replica'],
+  replicationFactor: 1,
+  // Disable shipping to object storage for the purposes of this example
+  objectStorageConfig: null,
+  endpoints: i.endpoints,
+});
+
 local s = t.store(commonConfig.config {
   replicas: 1,
   serviceMonitor: true,
@@ -34,7 +52,16 @@ local q = t.query(commonConfig.config {
   replicas: 1,
   replicaLabels: ['prometheus_replica', 'rule_replica'],
   serviceMonitor: true,
+  stores: [s.storeEndpoint] + i.storeEndpoints,
 });
 
 { ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } +
-{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) }
+{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) } +
+{ ['thanos-receive-router-' + resource]: r[resource] for resource in std.objectFields(r) } +
+{ ['thanos-receive-ingestor-' + resource]: i[resource] for resource in std.objectFields(i) if resource != 'ingestors' } +
+{
+  ['thanos-receive-ingestor-' + hashring + '-' + resource]: i.ingestors[hashring][resource]
+  for hashring in std.objectFields(i.ingestors)
+  for resource in std.objectFields(i.ingestors[hashring])
+  if i.ingestors[hashring][resource] != null
+}

examples/all/manifests/thanos-receive-default-statefulSet.yaml

@@ -70,12 +70,12 @@ spec:
         - --http-address=0.0.0.0:10902
         - --remote-write.address=0.0.0.0:19291
         - --receive.replication-factor=2
-        - --objstore.config=$(OBJSTORE_CONFIG)
         - --tsdb.path=/var/thanos/receive
         - --tsdb.retention=15d
-        - --receive.local-endpoint=$(NAME).thanos-receive-default.$(NAMESPACE).svc.cluster.local:10901
         - --label=replica="$(NAME)"
         - --label=receive="true"
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --receive.local-endpoint=$(NAME).thanos-receive-default.$(NAMESPACE).svc.cluster.local:10901
         - --receive.hashrings-file=/var/lib/thanos-receive/hashrings.json
         - |-
           --tracing.config="config":
@@ -92,15 +92,15 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: OBJSTORE_CONFIG
           valueFrom:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        - name: HOST_IP_ADDRESS
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
         image: quay.io/thanos/thanos:v0.22.0
         livenessProbe:
           failureThreshold: 8

examples/all/manifests/thanos-receive-region-1-statefulSet.yaml

@@ -70,12 +70,12 @@ spec:
         - --http-address=0.0.0.0:10902
         - --remote-write.address=0.0.0.0:19291
         - --receive.replication-factor=2
-        - --objstore.config=$(OBJSTORE_CONFIG)
         - --tsdb.path=/var/thanos/receive
         - --tsdb.retention=15d
-        - --receive.local-endpoint=$(NAME).thanos-receive-region-1.$(NAMESPACE).svc.cluster.local:10901
         - --label=replica="$(NAME)"
         - --label=receive="true"
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --receive.local-endpoint=$(NAME).thanos-receive-region-1.$(NAMESPACE).svc.cluster.local:10901
         - --receive.hashrings-file=/var/lib/thanos-receive/hashrings.json
         - |-
           --tracing.config="config":
@@ -92,15 +92,15 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: OBJSTORE_CONFIG
           valueFrom:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        - name: HOST_IP_ADDRESS
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
         image: quay.io/thanos/thanos:v0.22.0
         livenessProbe:
           failureThreshold: 8

examples/all/manifests/thanos-receive-statefulSet.yaml

@@ -66,12 +66,12 @@ spec:
         - --http-address=0.0.0.0:10902
         - --remote-write.address=0.0.0.0:19291
         - --receive.replication-factor=1
-        - --objstore.config=$(OBJSTORE_CONFIG)
         - --tsdb.path=/var/thanos/receive
         - --tsdb.retention=15d
-        - --receive.local-endpoint=$(NAME).thanos-receive.$(NAMESPACE).svc.cluster.local:10901
         - --label=replica="$(NAME)"
         - --label=receive="true"
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --receive.local-endpoint=$(NAME).thanos-receive.$(NAMESPACE).svc.cluster.local:10901
         - --receive.hashrings-file=/var/lib/thanos-receive/hashrings.json
         - |-
           --tracing.config="config":
@@ -88,15 +88,15 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: OBJSTORE_CONFIG
           valueFrom:
             secretKeyRef:
               key: thanos.yaml
               name: thanos-objectstorage
-        - name: HOST_IP_ADDRESS
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
         image: quay.io/thanos/thanos:v0.22.0
         livenessProbe:
           failureThreshold: 8

jsonnet/kube-thanos/kube-thanos-receive-default-params.libsonnet

@@ -12,6 +12,7 @@
   objectStorageConfig: error 'must provide objectStorageConfig',
   podDisruptionBudgetMaxUnavailable: (std.floor(defaults.replicationFactor / 2)),
   hashringConfigMapName: '',
+  enableLocalEndpoint: true,
   volumeClaimTemplate: {},
   retention: '15d',
   logLevel: 'info',

jsonnet/kube-thanos/kube-thanos-receive-ingestor.libsonnet

@@ -0,0 +1,45 @@
+local receiveConfigDefaults = import 'kube-thanos/kube-thanos-receive-default-params.libsonnet';
+local receiveHashring = import 'kube-thanos/kube-thanos-receive-hashrings.libsonnet';
+
+local defaults = receiveConfigDefaults {
+  hashrings: [{
+    hashring: 'default',
+    tenants: [],
+  }],
+  hashringConfigmapName: 'hashring-config',
+  routerReplicas: 1,
+};
+
+function(params) {
+  local tr = self,
+  // Combine the defaults and the passed params to make the component's config.
+  config:: defaults + params,
+
+  local ingestors = receiveHashring(tr.config { name: tr.config.name + '-ingestor' }),
+
+  ingestors: {
+    [name]: ingestors.hashrings[name]
+    for name in std.objectFields(ingestors.hashrings)
+  },
+
+  storeEndpoints:: [
+    'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local:%d' % [ingestors.hashrings[name.hashring].service.metadata.name, tr.config.namespace, tr.config.ports.grpc]
+    for name in tr.config.hashrings
+  ],
+
+  endpoints:: {
+    [name.hashring]: [
+      '%s-%d.%s.%s.svc.cluster.local:%d' % [
+        ingestors.hashrings[name.hashring].service.metadata.name,
+        i,
+        ingestors.hashrings[name.hashring].service.metadata.name,
+        tr.config.namespace,
+        tr.config.ports.grpc,
+      ]
+      // Replica specification is 1-based, but statefulSets are named 0-based.
+      for i in std.range(0, tr.config.replicas - 1)
+    ]
+    for name in tr.config.hashrings
+  },
+  serviceAccount: ingestors.serviceAccount,
+}