Connect & Transform Your Data with Substation

Explore our guides to learn about the system or keep reading to get an overview!


Deploy Modular, Cloud Native Data Pipelines

Substation deploys unique data pipelines running on serverless cloud services that are managed using infrastructure as code.

local sub = import 'substation.libsonnet';

{
  transforms: [
    sub.transform.object.insert(
      settings={ object: { target_key: 'a' }, value: 'x' },
    ),
    sub.transform.send.stdout(),
  ],
}
{
   "transforms": [
      {
         "settings": {
            "object": {
               "key": null,
               "set_key": "a"
            },
            "value": "x"
         },
         "type": "object_insert"
      },
      {
         "type": "send_stdout"
      }
   ]
}

Evaluate & Transform Data

Substation applications evaluate and transform data in real-time using configurations as code to create expressive, reusable data transformation patterns.

package main

import (
	"context"
	"fmt"

	"github.com/brexhq/substation/message"
)

// Duplicates a message.
type Duplicate struct {
	// Count is the number of times to duplicate the message.
	Count int `json:"count"`
}

// Transforms a message based on the configuration.
func (t *Duplicate) Transform(ctx context.Context, msg *message.Message) ([]*message.Message, error) {
	// Always return control messages.
	if msg.IsControl() {
		return []*message.Message{msg}, nil
	}

	output := []*message.Message{msg}
	for i := 0; i < t.Count; i++ {
		output = append(output, msg)
	}

	return output, nil
}

func main() {
	// Create a data message.
	b := []byte(`{"a":"b"}`)
	msg := message.New().SetData(b)

	// Create the transform.
	tf := Duplicate{Count: 3}

	// Run the transform.
	ctx := context.Background()
	msgs, err := tf.Transform(ctx, msg)
	if err != nil {
		// Handle error.
		panic(err)
	}

	// Print the output.
	for _, m := range msgs {
		fmt.Println(string(m.Data()))
	}
}

Build Custom Data Processing Apps

Substation packages let developers create custom data processing applications written in Go. Use them to evaluate and transform data, or create your own Substation application.