add utility to inspect gzip'ed payload

.gitignore

@@ -0,0 +1,2 @@
+bin/
+.vscode/

Makefile

@@ -0,0 +1,14 @@
+all: bindir
+	go build -o ./bin ./cmd/...
+
+httpcat: bindir
+	go build -o ./bin ./cmd/httpcat/...
+
+mdfmt: bindir
+	go build -o ./bin ./cmd/md-fmt/...
+
+otelq: bindir
+	go build -o ./bin ./cmd/otelq/...
+
+bindir:
+	mkdir ./bin; exit 0

cmd/httpcat/README.md

@@ -0,0 +1,5 @@
+# HTTP Cat
+
+An HTTP server that returns the HTTP request as text in the response body.
+Useful for debugging proxy configurations.
+

cmd/httpcat/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+)
+
+var (
+	ports = flag.String("ports", "", "comma-separated list of ports to listen on")
+)
+
+type CatHandler struct{}
+
+var _ http.Handler = &CatHandler{}
+
+func (cs *CatHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	log.Printf("[%s] %s %s", req.Host, req.Method, req.URL.Path)
+
+	w.Write([]byte(fmt.Sprintf("%s %s %s\n", req.Proto, req.Method, req.URL.Path)))
+	for k, v := range req.Header {
+		w.Write([]byte(fmt.Sprintf("%s: %s\n", k, strings.Join(v, "; "))))
+	}
+	w.Write([]byte{'\n'})
+
+	if req.Body != nil {
+		io.Copy(w, req.Body)
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	errch := make(chan error)
+	done := make(chan struct{})
+	for _, p := range strings.Split(*ports, ",") {
+		port := strings.TrimSpace(p)
+		if port[0] != ':' {
+			port = ":" + port
+		}
+		go func() {
+			log.Printf("listening on %s\n", port)
+			if err := http.ListenAndServe(port, new(CatHandler)); err != nil {
+				log.Println(err)
+				errch <- err
+			}
+		}()
+	}
+
+	go func() {
+		<-errch
+		done <- struct{}{}
+	}()
+
+	<-done
+}

cmd/md-fmt/README.md

@@ -0,0 +1,9 @@
+# CSV to Markdown Table Formatter
+
+Takes a csv-like file as input and outputs a formatted markdown table with it.
+
+## Usage
+
+Run `md-fmt -help` for the list of command line arguments. For tab-separated
+files, in order for the separator to be passed correcly from the command line to
+the program, use `-sep=$'\t'` to correctly escape the character.

cmd/md-fmt/main.go

@@ -0,0 +1,175 @@
+// md-fmt takes a csv or tsv input file and outputs a formatted markdown table
+// with the data.
+package main
+
+import (
+	"bufio"
+	"encoding/csv"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+var (
+	sourcePath = flag.String("source", "", "path to the input file")
+	separator  = flag.String("sep", ",", "separator character to use when reading the csv file")
+	lazyQuotes = flag.Bool("lazy-quotes", false, "controls the lazy-quotes setting on the csv reader")
+	toCSV      = flag.Bool("to-csv", false, "parses markdown table and encodes it to csv (opposite operation)")
+)
+
+func main() {
+	flag.Parse()
+
+	fd, err := os.Open(*sourcePath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to open source file %s: %s\n", *sourcePath, err)
+	}
+
+	if *toCSV {
+		markdownToCSV(fd, os.Stdout)
+	} else {
+		csvToMarkdown(fd, os.Stdout)
+	}
+}
+
+func csvToMarkdown(r io.ReadSeeker, w io.Writer) {
+	sep := []rune(*separator)[0]
+	read := csv.NewReader(r)
+	read.Comma = sep
+	read.TrimLeadingSpace = true
+	read.LazyQuotes = *lazyQuotes
+
+	rec, err := read.Read()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error reading from csv file: %s\n", err)
+		return
+	}
+
+	widths := make([]int, len(rec))
+	for i, col := range rec {
+		widths[i] = max(widths[i], len(col))
+	}
+	for {
+		rec, err := read.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error reading from csv file: %s", err)
+			return
+		}
+
+		for i, col := range rec {
+			widths[i] = max(widths[i], len(col))
+		}
+	}
+
+	c := make([]string, len(widths))
+	for i := 0; i < len(c); i++ {
+		c[i] = " %-*s "
+	}
+	pattern := fmt.Sprintf("|%s|\n", strings.Join(c, "|"))
+
+	// Reset file descriptor cursor and take new CSV reader from it
+	r.Seek(0, 0)
+	read = csv.NewReader(r)
+	read.Comma = sep
+	read.TrimLeadingSpace = true
+	read.LazyQuotes = *lazyQuotes
+
+	// Format header row
+	rec, err = read.Read()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read next csv record: %s", err)
+		return
+	}
+
+	curr := make([]any, 0, 2*len(widths))
+	for i := range widths {
+		curr = append(curr, widths[i], rec[i])
+	}
+	fmt.Fprintf(w, pattern, curr...)
+
+	// Format header separator row
+	curr = curr[:0] // empty slice but preserve capacity
+	for i := range widths {
+		curr = append(curr, widths[i], strings.Repeat("-", widths[i]))
+	}
+	fmt.Fprintf(w, pattern, curr...)
+
+	for {
+		rec, err := read.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error reading from csv file: %s", err)
+			return
+		}
+
+		curr = curr[:0]
+		for i := range widths {
+			curr = append(curr, widths[i], rec[i])
+		}
+		fmt.Fprintf(w, pattern, curr...)
+	}
+}
+
+func markdownToCSV(r io.ReadSeeker, w io.Writer) {
+	cw := csv.NewWriter(w)
+	defer func() {
+		cw.Flush()
+		if err := cw.Error(); err != nil {
+			fmt.Fprintf(os.Stderr, "error occured during scan: %s", err)
+		}
+	}()
+
+	// for now we're assuming there's only one table in a markdown file, this
+	// flag keeps track of whether or not we've seen a table already, and is
+	// used to exit the loop early once we're done reading the table, ignoring
+	// subsequent lines in the file.
+	var seen bool
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		if err := scanner.Err(); err != nil {
+			fmt.Fprintln(os.Stderr, "reading standard input:", err)
+		}
+
+		line := strings.TrimSpace(scanner.Text())
+		if !(strings.HasPrefix(line, "|") && strings.HasSuffix(line, "|")) {
+			if seen {
+				break // exit early if we've seen a table already
+			}
+			continue // keep going until we see a table
+		}
+
+		// the rest of the body only applies to lines that are part of a table
+		// i.e. that start and end with a pipe character (`|`)
+		seen = true
+
+		line = strings.Trim(line, "|")
+		parts := strings.Split(line, "|")
+		for i := 0; i < len(parts); i++ {
+			parts[i] = strings.TrimSpace(parts[i])
+		}
+
+		// ignore separator line
+		if parts[0] == strings.Repeat("-", len(parts[0])) {
+			continue
+		}
+
+		if err := cw.Write(parts); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to write to output: %s\n", err)
+			return
+		}
+	}
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}

cmd/tempo-to-otlp/README.md

@@ -0,0 +1,3 @@
+# Tempo To OTLP
+
+Converts a Tempo protobuf message to an OTLP protobuf message

cmd/tempo-to-otlp/main.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/golang/protobuf/proto" // NOTE: keep this, it's required to unmarshall the tempopb
+	"github.com/grafana/tempo/pkg/tempopb"
+	tempocommonv1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
+	tempotracev1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
+	commonv1 "go.opentelemetry.io/proto/otlp/common/v1"
+	resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1"
+	tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"
+	newproto "google.golang.org/protobuf/proto"
+)
+
+var (
+	file = flag.String("file", "", "Tempo proto file. if set, will not use stdin.")
+	out  = flag.String("out", "", "Output file destination. Defaults to stdout.")
+	// asJSON = flag.Bool("json", false, "Read input as jsonpb.") // TODO(wperron) implement this
+)
+
+// TODO(wperron) implement reading from stdin
+func main() {
+	flag.Parse()
+
+	var output io.Writer
+	if *out == "" {
+		output = os.Stdout
+	} else {
+		output = must(os.OpenFile(*out, os.O_CREATE+os.O_RDWR, 0o664))
+	}
+
+	h := must(os.Open(*file))
+	bs := must(io.ReadAll(h))
+	var trace tempopb.Trace
+	if err := proto.Unmarshal(bs, &trace); err != nil {
+		log.Fatalln(err)
+	}
+
+	td := must(convert(trace))
+	bs = must(newproto.Marshal(&td))
+	written := must(io.Copy(output, bytes.NewReader(bs)))
+
+	fmt.Fprintf(os.Stderr, "completed successfully, %d bytes written", written)
+}
+
+func convert(trace tempopb.Trace) (tracev1.TracesData, error) {
+	td := make([]*tracev1.ResourceSpans, 0, len(trace.Batches))
+	for _, resourceSpans := range trace.Batches {
+		rs := &tracev1.ResourceSpans{
+			Resource: &resourcev1.Resource{
+				Attributes:             convertAttributes(resourceSpans.Resource.Attributes),
+				DroppedAttributesCount: resourceSpans.Resource.DroppedAttributesCount,
+			},
+			ScopeSpans: make([]*tracev1.ScopeSpans, 0),
+		}
+		for _, ils := range resourceSpans.InstrumentationLibrarySpans {
+			scope := &tracev1.ScopeSpans{
+				Spans: make([]*tracev1.Span, 0),
+			}
+
+			scope.Scope = &commonv1.InstrumentationScope{
+				Name:                   ils.InstrumentationLibrary.Name,
+				Version:                ils.InstrumentationLibrary.Version,
+				Attributes:             convertAttributes(resourceSpans.Resource.Attributes), // TODO(wperron) is this right?
+				DroppedAttributesCount: resourceSpans.Resource.DroppedAttributesCount,        // TODO(wperron) is this right?
+			}
+
+			for _, span := range ils.Spans {
+				otelSpan := &tracev1.Span{
+					TraceId:                span.TraceId,
+					SpanId:                 span.SpanId,
+					TraceState:             span.TraceState,
+					ParentSpanId:           span.ParentSpanId,
+					Name:                   span.Name,
+					Kind:                   tracev1.Span_SpanKind(span.Kind),
+					StartTimeUnixNano:      span.StartTimeUnixNano,
+					EndTimeUnixNano:        span.EndTimeUnixNano,
+					Attributes:             convertAttributes(span.Attributes),
+					DroppedAttributesCount: span.DroppedAttributesCount,
+					Events:                 convertEvents(span.Events),
+					DroppedEventsCount:     span.DroppedEventsCount,
+					Links:                  convertLinks(span.Links),
+					DroppedLinksCount:      span.DroppedLinksCount,
+					Status:                 convertStatus(span.Status),
+				}
+				scope.Spans = append(scope.Spans, otelSpan)
+			}
+
+			rs.ScopeSpans = append(rs.ScopeSpans, scope)
+		}
+		td = append(td, rs)
+	}
+
+	return tracev1.TracesData{
+		ResourceSpans: td,
+	}, nil
+}
+
+func convertAttributes(attrs []*tempocommonv1.KeyValue) []*commonv1.KeyValue {
+	kvs := make([]*commonv1.KeyValue, 0, len(attrs))
+	for _, a := range attrs {
+		kvs = append(kvs, &commonv1.KeyValue{
+			Key:   a.Key,
+			Value: convertAnyValue(a.Value),
+		})
+	}
+	return kvs
+}
+
+func convertAnyValue(av *tempocommonv1.AnyValue) *commonv1.AnyValue {
+	inner := av.GetValue()
+	v := &commonv1.AnyValue{}
+	switch inner.(type) {
+	case *tempocommonv1.AnyValue_StringValue:
+		v.Value = &commonv1.AnyValue_StringValue{StringValue: av.GetStringValue()}
+	case *tempocommonv1.AnyValue_IntValue:
+		v.Value = &commonv1.AnyValue_IntValue{IntValue: av.GetIntValue()}
+	case *tempocommonv1.AnyValue_DoubleValue:
+		v.Value = &commonv1.AnyValue_DoubleValue{DoubleValue: av.GetDoubleValue()}
+	case *tempocommonv1.AnyValue_BoolValue:
+		v.Value = &commonv1.AnyValue_BoolValue{BoolValue: av.GetBoolValue()}
+	case *tempocommonv1.AnyValue_ArrayValue:
+		inner := av.GetArrayValue().Values
+		i := make([]*commonv1.AnyValue, 0, len(inner))
+		for _, val := range inner {
+			i = append(i, convertAnyValue(val))
+		}
+		v.Value = &commonv1.AnyValue_ArrayValue{ArrayValue: &commonv1.ArrayValue{Values: i}}
+	case *tempocommonv1.AnyValue_KvlistValue:
+		inner := av.GetKvlistValue().Values
+		v.Value = &commonv1.AnyValue_KvlistValue{KvlistValue: &commonv1.KeyValueList{Values: convertAttributes(inner)}}
+	}
+	return v
+}
+
+func convertEvents(events []*tempotracev1.Span_Event) []*tracev1.Span_Event {
+	ev := make([]*tracev1.Span_Event, 0, len(events))
+	for _, event := range events {
+		ev = append(ev, &tracev1.Span_Event{
+			TimeUnixNano:           event.TimeUnixNano,
+			Name:                   event.Name,
+			DroppedAttributesCount: event.DroppedAttributesCount,
+			Attributes:             convertAttributes(event.Attributes),
+		})
+	}
+	return ev
+}
+
+func convertLinks(links []*tempotracev1.Span_Link) []*tracev1.Span_Link {
+	ls := make([]*tracev1.Span_Link, 0, len(links))
+	for _, link := range links {
+		ls = append(ls, &tracev1.Span_Link{
+			TraceId:                link.TraceId,
+			SpanId:                 link.SpanId,
+			TraceState:             link.TraceState,
+			DroppedAttributesCount: link.DroppedAttributesCount,
+			Attributes:             convertAttributes(link.Attributes),
+		})
+	}
+	return ls
+}
+
+func convertStatus(status *tempotracev1.Status) *tracev1.Status {
+	return &tracev1.Status{
+		Message: status.Message,
+		Code:    tracev1.Status_StatusCode(status.Code),
+	}
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return v
+}

go.mod

@@ -2,26 +2,34 @@ module go.wperron.io/toolkit
 
 go 1.20
 
-require go.opentelemetry.io/otel/sdk v1.14.0
+require (
+	github.com/grafana/tempo v1.5.0
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/otel/sdk v1.14.0
+	go.opentelemetry.io/proto/otlp v0.19.0
+	google.golang.org/protobuf v1.28.1
+)
 
 require (
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
-	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel v1.14.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -68,7 +70,10 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -108,6 +113,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -121,6 +127,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/grafana/tempo v1.5.0 h1:JSwulLVtXvUw2MyuUPcvRg3MJiwTUs5XWnbG6fOKatc=
+github.com/grafana/tempo v1.5.0/go.mod h1:IB52YU6zkGL+3t0eNrY8kAExx0lLa4LH20wGu3c4wD8=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
@@ -129,10 +137,14 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -142,9 +154,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -165,6 +180,7 @@ go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+go
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
+go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -226,6 +242,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -243,6 +260,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -268,6 +286,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -322,9 +341,11 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -418,11 +439,14 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

gziputil/util.go

@@ -0,0 +1,59 @@
+package gziputil
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type Gzip struct {
+	magic []byte
+	cm    byte
+	flg   byte
+	mtime uint32
+	xfl   byte
+	os    byte
+	xlen  uint16
+	xflds []byte
+	crc32 uint32
+	isize uint32
+
+	rest    []byte
+	trailer []byte
+}
+
+func explainGzip(bs []byte) Gzip {
+	g := Gzip{
+		magic: bs[0:2],
+		cm:    bs[2],
+		flg:   bs[3],
+		mtime: binary.BigEndian.Uint32(bs[4:8]),
+		xfl:   bs[8],
+		os:    bs[9],
+	}
+
+	offset := 10
+	if g.xfl&2 == 1 {
+		g.xlen = binary.BigEndian.Uint16(bs[offset : offset+2])
+		offset += 2
+
+		g.xflds = bs[offset : offset+int(g.xlen)]
+		offset += int(g.xlen)
+	}
+
+	g.crc32 = binary.BigEndian.Uint32(bs[offset : offset+4])
+	offset += 4
+
+	g.isize = binary.BigEndian.Uint32(bs[offset : offset+4])
+	offset += 4
+
+	g.rest = bs[offset : len(bs)-8]
+	offset = len(bs) - 8
+
+	g.trailer = bs[offset:]
+
+	return g
+}
+
+func (g Gzip) String() string {
+	return fmt.Sprintf("magic=%04x cm=%02x flg=%02x mtime=%d xfl=%02x os=%02x xlen=%d crc32=%d isize=%d rest=%x trailer=%08x", g.magic, g.cm, g.flg, g.mtime, g.xfl, g.os, g.xlen, g.crc32, g.isize, g.rest, g.trailer)
+}

http/round_tripper.go

@@ -44,3 +44,14 @@ func (f *FollowRedirect) RoundTrip(req *stdhttp.Request) (*stdhttp.Response, err
 	}
 	return res, nil
 }
+
+type BasicAuthRoundTripper struct {
+	u, p string
+
+	next stdhttp.RoundTripper
+}
+
+func (bart *BasicAuthRoundTripper) RoundTrip(req *stdhttp.Request) (*stdhttp.Response, error) {
+	req.SetBasicAuth(bart.u, bart.p)
+	return bart.next.RoundTrip(req)
+}

integrationtest/README.md

@@ -0,0 +1,5 @@
+# Integration Test Util
+
+Based off of Deno's internal integration test utilities. This is meant to test
+external systems that are hard to mock or can't rely fully on dependency
+injection.

integrationtest/assert_matches_text.go

@@ -0,0 +1,59 @@
+package integrationtest
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const WILDCARD = "[WILDCARD]"
+
+func AssertMatchesText(t *testing.T, expected, actual string) {
+	if !strings.Contains(expected, WILDCARD) {
+		assert.Equal(t, expected, actual)
+	} else if !wildcardMatch(expected, actual) {
+		t.Errorf("texts do not match. expected \"%s\", got \"%s\"", expected, actual)
+	}
+}
+
+func wildcardMatch(expected, actual string) bool {
+	return patternMatch(expected, actual, WILDCARD)
+}
+
+func patternMatch(expected, actual, pattern string) bool {
+	if expected == pattern {
+		return true
+	}
+
+	parts := strings.Split(expected, pattern)
+	if len(parts) == 1 {
+		return expected == actual
+	}
+
+	if !strings.HasPrefix(actual, parts[0]) {
+		return false
+	}
+
+	if lines := strings.Split(expected, "\n"); len(lines) > 0 && lines[0] == pattern {
+		actual = "\n" + actual
+	}
+
+	t := []string{actual[:len(parts[0])], actual[len(parts[0]):]}
+	for i, part := range parts {
+		if i == 0 {
+			continue
+		}
+
+		if i == len(parts)-1 && (part == "" || part == "\n") {
+			return true
+		}
+
+		if found := strings.Index(t[1], part); found != -1 {
+			t = []string{t[1][:found+len(part)], t[1][found+len(part):]}
+		} else {
+			return false
+		}
+	}
+	return t[1] == ""
+}

integrationtest/assert_matches_text_test.go

@@ -0,0 +1,124 @@
+package integrationtest
+
+import (
+	"testing"
+)
+
+func Test_patternMatch(t *testing.T) {
+	type args struct {
+		expected string
+		actual   string
+		pattern  string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "foobarbaz matches",
+			args: args{
+				expected: "foo[BAR]baz",
+				actual:   "foobarbaz",
+				pattern:  "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "foobarbaz does not match",
+			args: args{
+				expected: "foo[BAR]baz",
+				actual:   "foobazbar",
+				pattern:  "[BAR]",
+			},
+			want: false,
+		},
+		{
+			name: "wildcard matches but not rest of string",
+			args: args{
+				expected: "foo[BAR]baz",
+				actual:   "foobarfizz",
+				pattern:  "[BAR]",
+			},
+			want: false,
+		},
+		{
+			name: "just wildcard",
+			args: args{
+				expected: "[BAR]",
+				actual:   "foobarbaz",
+				pattern:  "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "prefix",
+			args: args{
+				expected: "prefix[BAR]",
+				actual:   "prefixbarbaz",
+				pattern:  "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "prefix does not match",
+			args: args{
+				expected: "prefix[BAR]",
+				actual:   "somethingelsebarbaz",
+				pattern:  "[BAR]",
+			},
+			want: false,
+		},
+		{
+			name: "suffix",
+			args: args{
+				expected: "[BAR]suffix",
+				actual:   "foobarbazsuffix",
+				pattern:  "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "no pattern",
+			args: args{
+				expected: "exact match",
+				actual:   "exact match",
+				pattern:  "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "multiline",
+			args: args{
+				expected: `first line
+[BAR]
+last line`,
+				actual: `first line
+second line
+last line`,
+				pattern: "[BAR]",
+			},
+			want: true,
+		},
+		{
+			name: "multiline prefix",
+			args: args{
+				expected: `[BAR]
+first line
+last line`,
+				actual: `[BAR]
+first line
+last line`,
+				pattern: "[BAR]",
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := patternMatch(tt.args.expected, tt.args.actual, tt.args.pattern); got != tt.want {
+				t.Errorf("patternMatch() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}