<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package throttler

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"testing"
	"time"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	primaryTablet   *cluster.Vttablet
	replicaTablet   *cluster.Vttablet
	hostname        = "localhost"
	keyspaceName    = "ks"
	cell            = "zone1"
	sqlSchema       = `
	create table t1(
		id bigint,
		value varchar(16),
		primary key(id)
	) Engine=InnoDB;
`

	vSchema = `
	{
    "sharded": true,
    "vindexes": {
      "hash": {
        "type": "hash"
      }
    },
    "tables": {
      "t1": {
        "column_vindexes": [
          {
            "column": "id",
            "name": "hash"
          }
        ]
      }
    }
	}`

	httpClient       = base.SetupHTTPClient(time.Second)
	checkAPIPath     = "throttler/check"
	checkSelfAPIPath = "throttler/check-self"
	vtParams         mysql.ConnParams
)

const (
	testThreshold     = 5
	throttlerInitWait = 10 * time.Second
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(cell, hostname)
		defer clusterInstance.Teardown()

		// Start topo server
		err := clusterInstance.StartTopo()
		if err != nil {
			return 1
		}

		// Set extra tablet args for lock timeout
		clusterInstance.VtTabletExtraArgs = []string{
			"-lock_tables_timeout", "5s",
			"-watch_replication_stream",
			"-enable_replication_reporter",
			"-enable-lag-throttler",
			"-throttle_metrics_query", "show global status like 'threads_running'",
			"-throttle_metrics_threshold", fmt.Sprintf("%d", testThreshold),
			"-throttle_check_as_check_self",
			"-heartbeat_enable",
			"-heartbeat_interval", "250ms",
		}
		// We do not need semiSync for this test case.
		clusterInstance.EnableSemiSync = false

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name:      keyspaceName,
			SchemaSQL: sqlSchema,
			VSchema:   vSchema,
		}

		if err = clusterInstance.StartUnshardedKeyspace(*keyspace, 0, false); err != nil {
			return 1
		}

		// Collect table paths and ports
		tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets
		for _, tablet := range tablets {
			if tablet.Type == "master" || tablet.Type == "primary" {
				primaryTablet = tablet
			} else if tablet.Type != "rdonly" {
				replicaTablet = tablet
			}
		}

		vtgateInstance := clusterInstance.NewVtgateInstance()
		// set the gateway we want to use
		vtgateInstance.GatewayImplementation = "tabletgateway"
		// Start vtgate
		if err := vtgateInstance.Setup(); err != nil {
			return 1
		}
		// ensure it is torn down during cluster TearDown
		clusterInstance.VtgateProcess = *vtgateInstance
		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}

		return m.Run()
	}()
	os.Exit(exitCode)
}

func throttleCheck(tablet *cluster.Vttablet) (*http.Response, error) {
	return httpClient.Head(fmt.Sprintf("http://localhost:%d/%s", tablet.HTTPPort, checkAPIPath))
}

func throttleCheckSelf(tablet *cluster.Vttablet) (*http.Response, error) {
	return httpClient.Head(fmt.Sprintf("http://localhost:%d/%s", tablet.HTTPPort, checkSelfAPIPath))
}

func TestThrottlerThresholdOK(t *testing.T) {
	defer cluster.PanicHandler(t)

	{
		resp, err := throttleCheck(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
}

func TestThrottlerAfterMetricsCollected(t *testing.T) {
	defer cluster.PanicHandler(t)

	time.Sleep(throttlerInitWait)
	// By this time metrics will have been collected. We expect no lag, and something like:
	// {"StatusCode":200,"Value":0.282278,"Threshold":1,"Message":""}
	{
		resp, err := throttleCheck(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
	{
		resp, err := throttleCheckSelf(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
}

func TestThreadsRunning(t *testing.T) {
	defer cluster.PanicHandler(t)

	sleepSeconds := 6
	for i := 0; i < testThreshold; i++ {
		go vtgateExec(t, fmt.Sprintf("select sleep(%d)", sleepSeconds), "")
	}
	t.Run("exceeds threshold", func(t *testing.T) {
		time.Sleep(3 * time.Second)
		// by this time we will have +1 threads_running, and we should hit the threshold
		// {"StatusCode":429,"Value":2,"Threshold":2,"Message":"Threshold exceeded"}
		{
			resp, err := throttleCheck(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
		}
		{
			resp, err := throttleCheckSelf(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
		}
	})
	t.Run("restored below threshold", func(t *testing.T) {
		time.Sleep(time.Duration(sleepSeconds) * time.Second)
		// Restore
		{
			resp, err := throttleCheck(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusOK, resp.StatusCode)
		}
		{
			resp, err := throttleCheckSelf(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusOK, resp.StatusCode)
		}
	})
}

func vtgateExec(t *testing.T, query string, expectError string) *sqltypes.Result {
	t.Helper()

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()

	qr, err := conn.ExecuteFetch(query, 1000, true)
	if expectError == "" {
		require.NoError(t, err)
	} else {
		require.Error(t, err, "error should not be nil")
		assert.Contains(t, err.Error(), expectError, "Unexpected error")
	}
	return qr
}
<commit_msg>Sleep twice as long for threshold end to end test<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package throttler

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"os"
	"testing"
	"time"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	primaryTablet   *cluster.Vttablet
	replicaTablet   *cluster.Vttablet
	hostname        = "localhost"
	keyspaceName    = "ks"
	cell            = "zone1"
	sqlSchema       = `
	create table t1(
		id bigint,
		value varchar(16),
		primary key(id)
	) Engine=InnoDB;
`

	vSchema = `
	{
    "sharded": true,
    "vindexes": {
      "hash": {
        "type": "hash"
      }
    },
    "tables": {
      "t1": {
        "column_vindexes": [
          {
            "column": "id",
            "name": "hash"
          }
        ]
      }
    }
	}`

	httpClient       = base.SetupHTTPClient(time.Second)
	checkAPIPath     = "throttler/check"
	checkSelfAPIPath = "throttler/check-self"
	vtParams         mysql.ConnParams
)

const (
	testThreshold     = 5
	throttlerInitWait = 10 * time.Second
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(cell, hostname)
		defer clusterInstance.Teardown()

		// Start topo server
		err := clusterInstance.StartTopo()
		if err != nil {
			return 1
		}

		// Set extra tablet args for lock timeout
		clusterInstance.VtTabletExtraArgs = []string{
			"-lock_tables_timeout", "5s",
			"-watch_replication_stream",
			"-enable_replication_reporter",
			"-enable-lag-throttler",
			"-throttle_metrics_query", "show global status like 'threads_running'",
			"-throttle_metrics_threshold", fmt.Sprintf("%d", testThreshold),
			"-throttle_check_as_check_self",
			"-heartbeat_enable",
			"-heartbeat_interval", "250ms",
		}
		// We do not need semiSync for this test case.
		clusterInstance.EnableSemiSync = false

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name:      keyspaceName,
			SchemaSQL: sqlSchema,
			VSchema:   vSchema,
		}

		if err = clusterInstance.StartUnshardedKeyspace(*keyspace, 0, false); err != nil {
			return 1
		}

		// Collect table paths and ports
		tablets := clusterInstance.Keyspaces[0].Shards[0].Vttablets
		for _, tablet := range tablets {
			if tablet.Type == "master" || tablet.Type == "primary" {
				primaryTablet = tablet
			} else if tablet.Type != "rdonly" {
				replicaTablet = tablet
			}
		}

		vtgateInstance := clusterInstance.NewVtgateInstance()
		// set the gateway we want to use
		vtgateInstance.GatewayImplementation = "tabletgateway"
		// Start vtgate
		if err := vtgateInstance.Setup(); err != nil {
			return 1
		}
		// ensure it is torn down during cluster TearDown
		clusterInstance.VtgateProcess = *vtgateInstance
		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}

		return m.Run()
	}()
	os.Exit(exitCode)
}

func throttleCheck(tablet *cluster.Vttablet) (*http.Response, error) {
	return httpClient.Head(fmt.Sprintf("http://localhost:%d/%s", tablet.HTTPPort, checkAPIPath))
}

func throttleCheckSelf(tablet *cluster.Vttablet) (*http.Response, error) {
	return httpClient.Head(fmt.Sprintf("http://localhost:%d/%s", tablet.HTTPPort, checkSelfAPIPath))
}

func TestThrottlerThresholdOK(t *testing.T) {
	defer cluster.PanicHandler(t)

	{
		resp, err := throttleCheck(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
}

func TestThrottlerAfterMetricsCollected(t *testing.T) {
	defer cluster.PanicHandler(t)

	time.Sleep(throttlerInitWait)
	// By this time metrics will have been collected. We expect no lag, and something like:
	// {"StatusCode":200,"Value":0.282278,"Threshold":1,"Message":""}
	{
		resp, err := throttleCheck(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
	{
		resp, err := throttleCheckSelf(primaryTablet)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, resp.StatusCode)
	}
}

func TestThreadsRunning(t *testing.T) {
	defer cluster.PanicHandler(t)

	sleepSeconds := 6
	for i := 0; i < testThreshold; i++ {
		go vtgateExec(t, fmt.Sprintf("select sleep(%d)", sleepSeconds), "")
	}
	t.Run("exceeds threshold", func(t *testing.T) {
		time.Sleep(3 * time.Second)
		// by this time we will have +1 threads_running, and we should hit the threshold
		// {"StatusCode":429,"Value":2,"Threshold":2,"Message":"Threshold exceeded"}
		{
			resp, err := throttleCheck(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
		}
		{
			resp, err := throttleCheckSelf(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
		}
	})
	t.Run("restored below threshold", func(t *testing.T) {
		time.Sleep(time.Duration(sleepSeconds) * time.Second * 2) // * 2 since we have two planner executing the select sleep(6) query
		// Restore
		{
			resp, err := throttleCheck(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusOK, resp.StatusCode)
		}
		{
			resp, err := throttleCheckSelf(primaryTablet)
			assert.NoError(t, err)
			assert.Equal(t, http.StatusOK, resp.StatusCode)
		}
	})
}

func vtgateExec(t *testing.T, query string, expectError string) *sqltypes.Result {
	t.Helper()

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()

	qr, err := conn.ExecuteFetch(query, 1000, true)
	if expectError == "" {
		require.NoError(t, err)
	} else {
		require.Error(t, err, "error should not be nil")
		assert.Contains(t, err.Error(), expectError, "Unexpected error")
	}
	return qr
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/kid0m4n/go-rpi/controller/pca9685"
	"github.com/kid0m4n/go-rpi/i2c"
)

func main() {
	bus, err := i2c.NewBus(1)
	if err != nil {
		log.Panic(err)
	}

	pca9685 := pca9685.New(bus, 0x41, 1000)
	pca9685.Debug = true
	defer pca9685.Close()

	if err := pca9685.SetPwm(15, 0, 1000); err != nil {
		log.Panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)

	timer := time.Tick(2 * time.Second)
	sleeping := false

	for {
		select {
		case <-timer:
			sleeping = !sleeping
			if sleeping {
				pca9685.Sleep()
			} else {
				pca9685.Wake()
			}
		case <-c:
			return
		}
	}
}
<commit_msg>pca9685: higher speed for motor<commit_after>package main

import (
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/kid0m4n/go-rpi/controller/pca9685"
	"github.com/kid0m4n/go-rpi/i2c"
)

func main() {
	bus, err := i2c.NewBus(1)
	if err != nil {
		log.Panic(err)
	}

	pca9685 := pca9685.New(bus, 0x41, 1000)
	pca9685.Debug = true
	defer pca9685.Close()

	if err := pca9685.SetPwm(15, 0, 2000); err != nil {
		log.Panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)

	timer := time.Tick(2 * time.Second)
	sleeping := false

	for {
		select {
		case <-timer:
			sleeping = !sleeping
			if sleeping {
				pca9685.Sleep()
			} else {
				pca9685.Wake()
			}
		case <-c:
			return
		}
	}
}
<|endoftext|>


<commit_before>// The GADGT VM implementation and instruction set
package vm

import (
	"fmt"
	"io"

	"github.com/actgardner/gogen-avro/vm/types"
)

type frame struct {
	Target types.Field

	Boolean bool
	Int     int32
	Long    int64
	Float   float32
	Double  float64
	Bytes   []byte
	String  string

	Condition bool
}

func Eval(r io.Reader, program *Program, target types.Field) (err error) {
	callStack := make([]int, 256)
	callStackDepth := 0

	stack := make([]frame, 256)
	stack[0].Target = target
	depth := 0

	pc := 0

	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Panic at pc %v - %v", pc, r)
		}
	}()

	for pc = 0; pc < len(program.Instructions); pc++ {
		inst := program.Instructions[pc]
		frame := &stack[depth]
		log("PC: %v\tD:%v\tOp: %v", pc, depth, inst)
		log("Frame: %v", frame)
		switch inst.Op {
		case Read:
			switch inst.Operand {
			case Null:
				break
			case Boolean:
				frame.Boolean, err = readBool(r)
				break
			case Int:
				frame.Int, err = readInt(r)
				break
			case Long:
				frame.Long, err = readLong(r)
				break
			case UnusedLong:
				_, err = readLong(r)
				break
			case Float:
				frame.Float, err = readFloat(r)
				break
			case Double:
				frame.Double, err = readDouble(r)
				break
			case Bytes:
				frame.Bytes, err = readBytes(r)
				break
			case String:
				frame.String, err = readString(r)
				break
			default:
				frame.Bytes, err = readFixed(r, inst.Operand-11)
				break
			}
			break
		case Set:
			switch inst.Operand {
			case Null:
				break
			case Boolean:
				frame.Target.SetBoolean(frame.Boolean)
				break
			case Int:
				frame.Target.SetInt(frame.Int)
				break
			case Long:
				frame.Target.SetLong(frame.Long)
				break
			case Float:
				frame.Target.SetFloat(frame.Float)
				break
			case Double:
				frame.Target.SetDouble(frame.Double)
				break
			case Bytes:
				frame.Target.SetBytes(frame.Bytes)
				break
			case String:
				frame.Target.SetString(frame.String)
				break
			}
			break
		case SetDefault:
			stack[depth].Target.SetDefault(inst.Operand)
			break
		case Enter:
			depth += 1
			stack[depth].Target = frame.Target.Get(inst.Operand)
			break
		case Exit:
			stack[depth].Target.Finalize()
			depth -= 1
			break
		case AppendArray:
			depth += 1
			stack[depth].Target = frame.Target.AppendArray()
			break
		case AppendMap:
			depth += 1
			stack[depth].Target = frame.Target.AppendMap(stack[depth-1].String)
			break
		case Call:
			callStack[callStackDepth] = pc
			callStackDepth += 1
			pc = inst.Operand - 1
			break
		case Return:
			pc = callStack[callStackDepth-1]
			callStackDepth -= 1
			break
		case Jump:
			pc = inst.Operand - 1
			break
		case EvalGreater:
			frame.Condition = (frame.Long > int64(inst.Operand))
			break
		case EvalEqual:
			frame.Condition = (frame.Long == int64(inst.Operand))
			break
		case CondJump:
			if frame.Condition {
				pc = inst.Operand - 1
			}
			break
		case AddLong:
			frame.Long += int64(inst.Operand)
			break
		case MultLong:
			frame.Long *= int64(inst.Operand)
			break

		case Halt:
			if inst.Operand == 0 {
				return nil
			}
			return fmt.Errorf("Runtime error: %v, frame: %v, pc: %v", program.Errors[inst.Operand-1], frame, pc)
		default:
			return fmt.Errorf("Unknown instruction %v", program.Instructions[pc])
		}

		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Support growing stack<commit_after>// The GADGT VM implementation and instruction set
package vm

import (
	"fmt"
	"io"

	"github.com/actgardner/gogen-avro/vm/types"
)

type stackFrame struct {
	Target types.Field

	Boolean bool
	Int     int32
	Long    int64
	Float   float32
	Double  float64
	Bytes   []byte
	String  string

	Condition bool
}

func Eval(r io.Reader, program *Program, target types.Field) (err error) {
	callStack := make([]int, 8)
	callStackDepth := 0

	stack := make([]stackFrame, 8)
	stack[0].Target = target
	depth := 0

	pc := 0

	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Panic at pc %v - %v", pc, r)
		}
	}()

	for pc = 0; pc < len(program.Instructions); pc++ {
		inst := program.Instructions[pc]
		frame := &stack[depth]
		switch inst.Op {
		case Read:
			switch inst.Operand {
			case Null:
				break
			case Boolean:
				frame.Boolean, err = readBool(r)
				break
			case Int:
				frame.Int, err = readInt(r)
				break
			case Long:
				frame.Long, err = readLong(r)
				break
			case UnusedLong:
				_, err = readLong(r)
				break
			case Float:
				frame.Float, err = readFloat(r)
				break
			case Double:
				frame.Double, err = readDouble(r)
				break
			case Bytes:
				frame.Bytes, err = readBytes(r)
				break
			case String:
				frame.String, err = readString(r)
				break
			default:
				frame.Bytes, err = readFixed(r, inst.Operand-11)
				break
			}
			break
		case Set:
			switch inst.Operand {
			case Null:
				break
			case Boolean:
				frame.Target.SetBoolean(frame.Boolean)
				break
			case Int:
				frame.Target.SetInt(frame.Int)
				break
			case Long:
				frame.Target.SetLong(frame.Long)
				break
			case Float:
				frame.Target.SetFloat(frame.Float)
				break
			case Double:
				frame.Target.SetDouble(frame.Double)
				break
			case Bytes:
				frame.Target.SetBytes(frame.Bytes)
				break
			case String:
				frame.Target.SetString(frame.String)
				break
			}
			break
		case SetDefault:
			stack[depth].Target.SetDefault(inst.Operand)
			break
		case Enter:
			depth += 1
			if depth >= len(stack) {
				stack = append(stack, make([]stackFrame, len(stack))...)
			}
			stack[depth].Target = frame.Target.Get(inst.Operand)
			break
		case Exit:
			stack[depth].Target.Finalize()
			depth -= 1
			break
		case AppendArray:
			depth += 1
			if depth >= len(stack) {
				stack = append(stack, make([]stackFrame, len(stack))...)
			}
			stack[depth].Target = frame.Target.AppendArray()
			break
		case AppendMap:
			depth += 1
			if depth >= len(stack) {
				stack = append(stack, make([]stackFrame, len(stack))...)
			}
			stack[depth].Target = frame.Target.AppendMap(stack[depth-1].String)
			break
		case Call:
			callStack[callStackDepth] = pc
			callStackDepth += 1
			if callStackDepth >= len(callStack) {
				callStack = append(callStack, make([]int, len(callStack))...)
			}
			pc = inst.Operand - 1
			break
		case Return:
			pc = callStack[callStackDepth-1]
			callStackDepth -= 1
			break
		case Jump:
			pc = inst.Operand - 1
			break
		case EvalGreater:
			frame.Condition = (frame.Long > int64(inst.Operand))
			break
		case EvalEqual:
			frame.Condition = (frame.Long == int64(inst.Operand))
			break
		case CondJump:
			if frame.Condition {
				pc = inst.Operand - 1
			}
			break
		case AddLong:
			frame.Long += int64(inst.Operand)
			break
		case MultLong:
			frame.Long *= int64(inst.Operand)
			break

		case Halt:
			if inst.Operand == 0 {
				return nil
			}
			return fmt.Errorf("Runtime error: %v, frame: %v, pc: %v", program.Errors[inst.Operand-1], frame, pc)
		default:
			return fmt.Errorf("Unknown instruction %v", program.Instructions[pc])
		}

		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>api/classes: add another missing error check<commit_after><|endoftext|>


<commit_before>package queue

import (
	"archive/zip"
	"bytes"
	"cred-alert/githubclient"
	"cred-alert/metrics"
	"cred-alert/mimetype"
	"cred-alert/notifications"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"

	"code.cloudfoundry.org/lager"
)

const initialCommitParentHash = "0000000000000000000000000000000000000000"

type RefScanJob struct {
	RefScanPlan
	client            githubclient.Client
	sniffer           sniff.Sniffer
	notifier          notifications.Notifier
	emitter           metrics.Emitter
	credentialCounter metrics.Counter
	mimetype          mimetype.Decoder
	id                string
}

func NewRefScanJob(
	plan RefScanPlan,
	client githubclient.Client,
	sniffer sniff.Sniffer,
	notifier notifications.Notifier,
	emitter metrics.Emitter,
	mimetype mimetype.Decoder,
	id string,
) *RefScanJob {
	credentialCounter := emitter.Counter("cred_alert.violations")

	job := &RefScanJob{
		RefScanPlan:       plan,
		client:            client,
		sniffer:           sniffer,
		notifier:          notifier,
		emitter:           emitter,
		credentialCounter: credentialCounter,
		mimetype:          mimetype,
		id:                id,
	}

	return job
}

func (j *RefScanJob) Run(logger lager.Logger) error {
	logger = logger.Session("ref-scan", lager.Data{
		"owner":      j.Owner,
		"repository": j.Repository,
		"ref":        j.Ref,
		"task-id":    j.id,
		"private":    j.Private,
	})
	logger.Debug("starting")

	if j.Ref == initialCommitParentHash {
		logger.Info("skipped-initial-nil-ref")
		logger.Debug("done")
		return nil
	}

	downloadURL, err := j.client.ArchiveLink(j.Owner, j.Repository, j.Ref)
	if err != nil {
		logger.Session("archive-link").Error("failed", err)
		if err == githubclient.ErrNotFound {
			return nil
		}
		return err
	}

	archiveFile, err := downloadArchive(logger, downloadURL)
	if err != nil {
		logger.Error("failed", err)
		return err
	}
	defer os.Remove(archiveFile.Name())
	defer archiveFile.Close()

	archiveReader, err := zip.OpenReader(archiveFile.Name())
	if err != nil {
		logger.Session("zip-open-reader").Error("failed", err)
		return err
	}
	defer archiveReader.Close()

	logger.Info("unzipped-archive", lager.Data{
		"file-count": len(archiveReader.File),
	})

	logger.Info("scanning-unzipped-files")

	for _, f := range archiveReader.File {
		l := logger.Session("archive-reader-file", lager.Data{"filename": f.Name})

		if j.shouldSkip(l, f) {
			logger.Info("skipped")
			continue
		}

		unzippedReader, err := f.Open()
		if err != nil {
			logger.Error("failed", err)
			continue
		}
		defer unzippedReader.Close()

		bufioScanner := filescanner.New(unzippedReader, f.Name)
		handleViolation := j.createHandleViolation(j.Ref, j.Owner+"/"+j.Repository)

		err = j.sniffer.Sniff(l, bufioScanner, handleViolation)
		if err != nil {
			l.Error("failed", err)
			return err
		}
	}

	logger.Debug("done")

	return nil
}

func downloadArchive(logger lager.Logger, link *url.URL) (*os.File, error) {
	if link == nil {
		err := errors.New("Archive link was nil")
		logger.Error("failed", err)
		return nil, err
	}

	logger.Info("download-archive", lager.Data{
		"url": link.String(),
	})
	logger.Debug("starting")

	tempFile, err := ioutil.TempFile("", "downloaded-git-archive")
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	resp, err := http.Get(link.String())
	defer resp.Body.Close()
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	_, err = io.Copy(tempFile, resp.Body)
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	logger.Debug("done")
	return tempFile, nil
}

func (j *RefScanJob) createHandleViolation(ref string, repoName string) func(lager.Logger, scanners.Line) error {
	return func(logger lager.Logger, line scanners.Line) error {
		logger = logger.Session("handle-violation", lager.Data{
			"path":        line.Path,
			"line-number": line.LineNumber,
			"ref":         ref,
		})
		logger.Debug("starting")

		err := j.notifier.SendNotification(logger, repoName, ref, line, j.Private)
		if err != nil {
			logger.Error("failed", err)
			return err
		}

		tag := "public"
		if j.Private {
			tag = "private"
		}

		j.credentialCounter.Inc(logger, tag)

		logger.Debug("done")
		return nil
	}
}

func (j *RefScanJob) shouldSkip(logger lager.Logger, f *zip.File) bool {
	logger = logger.Session("consider-skipping")

	unzippedReader, err := f.Open()
	if err != nil {
		logger.Error("failed", err)
		return false
	}
	defer unzippedReader.Close()

	buf := new(bytes.Buffer)
	numBytes, err := buf.ReadFrom(unzippedReader)
	if err != nil {
		logger.Error("failed", err)
		return false
	}
	if numBytes <= 0 {
		logger.Debug("done")
		return true
	}
	bytes := buf.Bytes()

	mime, err := j.mimetype.TypeByBuffer(bytes)
	if err != nil {
		logger.Error("failed", err)
		return false
	}

	if strings.HasPrefix(mime, "text") {
		logger.Debug("done")
		return false
	}

	logger.Debug("done")
	return true
}
<commit_msg>do not log sensitive url<commit_after>package queue

import (
	"archive/zip"
	"bytes"
	"cred-alert/githubclient"
	"cred-alert/metrics"
	"cred-alert/mimetype"
	"cred-alert/notifications"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"

	"code.cloudfoundry.org/lager"
)

const initialCommitParentHash = "0000000000000000000000000000000000000000"

type RefScanJob struct {
	RefScanPlan
	client            githubclient.Client
	sniffer           sniff.Sniffer
	notifier          notifications.Notifier
	emitter           metrics.Emitter
	credentialCounter metrics.Counter
	mimetype          mimetype.Decoder
	id                string
}

func NewRefScanJob(
	plan RefScanPlan,
	client githubclient.Client,
	sniffer sniff.Sniffer,
	notifier notifications.Notifier,
	emitter metrics.Emitter,
	mimetype mimetype.Decoder,
	id string,
) *RefScanJob {
	credentialCounter := emitter.Counter("cred_alert.violations")

	job := &RefScanJob{
		RefScanPlan:       plan,
		client:            client,
		sniffer:           sniffer,
		notifier:          notifier,
		emitter:           emitter,
		credentialCounter: credentialCounter,
		mimetype:          mimetype,
		id:                id,
	}

	return job
}

func (j *RefScanJob) Run(logger lager.Logger) error {
	logger = logger.Session("ref-scan", lager.Data{
		"owner":      j.Owner,
		"repository": j.Repository,
		"ref":        j.Ref,
		"task-id":    j.id,
		"private":    j.Private,
	})
	logger.Debug("starting")

	if j.Ref == initialCommitParentHash {
		logger.Info("skipped-initial-nil-ref")
		logger.Debug("done")
		return nil
	}

	downloadURL, err := j.client.ArchiveLink(j.Owner, j.Repository, j.Ref)
	if err != nil {
		logger.Session("archive-link").Error("failed", err)
		if err == githubclient.ErrNotFound {
			return nil
		}
		return err
	}

	archiveFile, err := downloadArchive(logger, downloadURL)
	if err != nil {
		logger.Error("failed", err)
		return err
	}
	defer os.Remove(archiveFile.Name())
	defer archiveFile.Close()

	archiveReader, err := zip.OpenReader(archiveFile.Name())
	if err != nil {
		logger.Session("zip-open-reader").Error("failed", err)
		return err
	}
	defer archiveReader.Close()

	logger.Info("unzipped-archive", lager.Data{
		"file-count": len(archiveReader.File),
	})

	logger.Info("scanning-unzipped-files")

	for _, f := range archiveReader.File {
		l := logger.Session("archive-reader-file", lager.Data{"filename": f.Name})

		if j.shouldSkip(l, f) {
			logger.Info("skipped")
			continue
		}

		unzippedReader, err := f.Open()
		if err != nil {
			logger.Error("failed", err)
			continue
		}
		defer unzippedReader.Close()

		bufioScanner := filescanner.New(unzippedReader, f.Name)
		handleViolation := j.createHandleViolation(j.Ref, j.Owner+"/"+j.Repository)

		err = j.sniffer.Sniff(l, bufioScanner, handleViolation)
		if err != nil {
			l.Error("failed", err)
			return err
		}
	}

	logger.Debug("done")

	return nil
}

func downloadArchive(logger lager.Logger, link *url.URL) (*os.File, error) {
	logger = logger.Session("downloading-archive")

	if link == nil {
		err := errors.New("Archive link was nil")
		logger.Error("failed", err)
		return nil, err
	}

	logger.Debug("starting")

	tempFile, err := ioutil.TempFile("", "downloaded-git-archive")
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	resp, err := http.Get(link.String())
	defer resp.Body.Close()
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	_, err = io.Copy(tempFile, resp.Body)
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	logger.Debug("done")
	return tempFile, nil
}

func (j *RefScanJob) createHandleViolation(ref string, repoName string) func(lager.Logger, scanners.Line) error {
	return func(logger lager.Logger, line scanners.Line) error {
		logger = logger.Session("handle-violation", lager.Data{
			"path":        line.Path,
			"line-number": line.LineNumber,
			"ref":         ref,
		})
		logger.Debug("starting")

		err := j.notifier.SendNotification(logger, repoName, ref, line, j.Private)
		if err != nil {
			logger.Error("failed", err)
			return err
		}

		tag := "public"
		if j.Private {
			tag = "private"
		}

		j.credentialCounter.Inc(logger, tag)

		logger.Debug("done")
		return nil
	}
}

func (j *RefScanJob) shouldSkip(logger lager.Logger, f *zip.File) bool {
	logger = logger.Session("consider-skipping")

	unzippedReader, err := f.Open()
	if err != nil {
		logger.Error("failed", err)
		return false
	}
	defer unzippedReader.Close()

	buf := new(bytes.Buffer)
	numBytes, err := buf.ReadFrom(unzippedReader)
	if err != nil {
		logger.Error("failed", err)
		return false
	}
	if numBytes <= 0 {
		logger.Debug("done")
		return true
	}
	bytes := buf.Bytes()

	mime, err := j.mimetype.TypeByBuffer(bytes)
	if err != nil {
		logger.Error("failed", err)
		return false
	}

	if strings.HasPrefix(mime, "text") {
		logger.Debug("done")
		return false
	}

	logger.Debug("done")
	return true
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"bytes"
	"crypto/x509/internal/macOS"
	"fmt"
	"os"
	"strings"
)

var debugDarwinRoots = strings.Contains(os.Getenv("GODEBUG"), "x509roots=1")

func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
	return nil, nil
}

// loadSystemRootsWithCgo is set in root_cgo_darwin_amd64.go when cgo is
// available, and is only used for testing.
var loadSystemRootsWithCgo func() (*CertPool, error)

func loadSystemRoots() (*CertPool, error) {
	var trustedRoots []*Certificate
	untrustedRoots := make(map[string]bool)

	// macOS has three trust domains: one for CAs added by users to their
	// "login" keychain, one for CAs added by Admins to the "System" keychain,
	// and one for the CAs that ship with the OS.
	for _, domain := range []macOS.SecTrustSettingsDomain{
		macOS.SecTrustSettingsDomainUser,
		macOS.SecTrustSettingsDomainAdmin,
		macOS.SecTrustSettingsDomainSystem,
	} {
		certs, err := macOS.SecTrustSettingsCopyCertificates(domain)
		if err == macOS.ErrNoTrustSettings {
			continue
		} else if err != nil {
			return nil, err
		}
		defer macOS.CFRelease(certs)

		for i := 0; i < macOS.CFArrayGetCount(certs); i++ {
			c := macOS.CFArrayGetValueAtIndex(certs, i)
			cert, err := exportCertificate(c)
			if err != nil {
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: domain %d, certificate #%d: %v\n", domain, i, err)
				}
				continue
			}

			var result macOS.SecTrustSettingsResult
			if domain == macOS.SecTrustSettingsDomainSystem {
				// Certs found in the system domain are always trusted. If the user
				// configures "Never Trust" on such a cert, it will also be found in the
				// admin or user domain, causing it to be added to untrustedRoots.
				result = macOS.SecTrustSettingsResultTrustRoot
			} else {
				result, err = sslTrustSettingsResult(c)
				if err != nil {
					if debugDarwinRoots {
						fmt.Fprintf(os.Stderr, "crypto/x509: trust settings for %v: %v\n", cert.Subject, err)
					}
					continue
				}
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: trust settings for %v: %d\n", cert.Subject, result)
				}
			}

			switch result {
			// "Note the distinction between the results kSecTrustSettingsResultTrustRoot
			// and kSecTrustSettingsResultTrustAsRoot: The former can only be applied to
			// root (self-signed) certificates; the latter can only be applied to
			// non-root certificates."
			case macOS.SecTrustSettingsResultTrustRoot:
				if isRootCertificate(cert) {
					trustedRoots = append(trustedRoots, cert)
				}
			case macOS.SecTrustSettingsResultTrustAsRoot:
				if !isRootCertificate(cert) {
					trustedRoots = append(trustedRoots, cert)
				}

			case macOS.SecTrustSettingsResultDeny:
				// Add this certificate to untrustedRoots, which are subtracted
				// from trustedRoots, so that we don't have to evaluate policies
				// for every root in the system domain, but still apply user and
				// admin policies that override system roots.
				untrustedRoots[string(cert.Raw)] = true

			case macOS.SecTrustSettingsResultUnspecified:
				// Certificates with unspecified trust should be added to a pool
				// of intermediates for chain building, but we don't support it
				// at the moment. This is Issue 35631.

			default:
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: unknown trust setting for %v: %d\n", cert.Subject, result)
				}
			}
		}
	}

	pool := NewCertPool()
	for _, cert := range trustedRoots {
		if !untrustedRoots[string(cert.Raw)] {
			pool.AddCert(cert)
		}
	}
	return pool, nil
}

// exportCertificate returns a *Certificate for a SecCertificateRef.
func exportCertificate(cert macOS.CFRef) (*Certificate, error) {
	data, err := macOS.SecItemExport(cert)
	if err != nil {
		return nil, err
	}
	defer macOS.CFRelease(data)
	der := macOS.CFDataToSlice(data)

	return ParseCertificate(der)
}

// isRootCertificate reports whether Subject and Issuer match.
func isRootCertificate(cert *Certificate) bool {
	return bytes.Equal(cert.RawSubject, cert.RawIssuer)
}

// sslTrustSettingsResult obtains the final kSecTrustSettingsResult value for a
// certificate in the user or admin domain, combining usage constraints for the
// SSL SecTrustSettingsPolicy,
//
// It ignores SecTrustSettingsKeyUsage and kSecTrustSettingsAllowedError, and
// doesn't support kSecTrustSettingsDefaultRootCertSetting.
//
// https://developer.apple.com/documentation/security/1400261-sectrustsettingscopytrustsetting
func sslTrustSettingsResult(cert macOS.CFRef) (macOS.SecTrustSettingsResult, error) {
	trustSettings, err := macOS.SecTrustSettingsCopyTrustSettings(cert, macOS.SecTrustSettingsDomainUser)
	// According to Apple's SecTrustServer.c, "user trust settings overrule
	// admin trust settings", but the rules of the override are unclear. Let's
	// assume admin trust settings are applicable if and only if there are no
	// user trust settings.
	if err == macOS.ErrNoTrustSettings {
		trustSettings, err = macOS.SecTrustSettingsCopyTrustSettings(cert, macOS.SecTrustSettingsDomainAdmin)
		// "no trust settings [...] means 'this certificate must be verified to a known trusted certificate'"
		if err == macOS.ErrNoTrustSettings {
			return macOS.SecTrustSettingsResultUnspecified, nil
		}
	}
	if err != nil {
		return 0, err
	}
	defer macOS.CFRelease(trustSettings)

	// "An empty trust settings array means 'always trust this certificate' with an
	// overall trust setting for the certificate of kSecTrustSettingsResultTrustRoot."
	if macOS.CFArrayGetCount(trustSettings) == 0 {
		return macOS.SecTrustSettingsResultTrustRoot, nil
	}

	isSSLPolicy := func(policyRef macOS.CFRef) bool {
		properties := macOS.SecPolicyCopyProperties(policyRef)
		defer macOS.CFRelease(properties)
		if v, ok := macOS.CFDictionaryGetValueIfPresent(properties, macOS.SecPolicyOid); ok {
			return macOS.CFEqual(v, macOS.CFRef(macOS.SecPolicyAppleSSL))
		}
		return false
	}

	for i := 0; i < macOS.CFArrayGetCount(trustSettings); i++ {
		tSetting := macOS.CFArrayGetValueAtIndex(trustSettings, i)

		// First, check if this trust setting is constrained to a non-SSL policy.
		if policyRef, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsPolicy); ok {
			if !isSSLPolicy(policyRef) {
				continue
			}
		}

		// Then check if it is restricted to a hostname, so not a root.
		if _, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsPolicyString); ok {
			continue
		}

		cfNum, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsResultKey)
		// "If this key is not present, a default value of kSecTrustSettingsResultTrustRoot is assumed."
		if !ok {
			return macOS.SecTrustSettingsResultTrustRoot, nil
		}
		result, err := macOS.CFNumberGetValue(cfNum)
		if err != nil {
			return 0, err
		}

		// If multiple dictionaries match, we are supposed to "OR" them,
		// the semantics of which are not clear. Since TrustRoot and TrustAsRoot
		// are mutually exclusive, Deny should probably override, and Invalid and
		// Unspecified be overridden, approximate this by stopping at the first
		// TrustRoot, TrustAsRoot or Deny.
		switch r := macOS.SecTrustSettingsResult(result); r {
		case macOS.SecTrustSettingsResultTrustRoot,
			macOS.SecTrustSettingsResultTrustAsRoot,
			macOS.SecTrustSettingsResultDeny:
			return r, nil
		}
	}

	// If trust settings are present, but none of them match the policy...
	// the docs don't tell us what to do.
	//
	// "Trust settings for a given use apply if any of the dictionaries in the
	// certificate’s trust settings array satisfies the specified use." suggests
	// that it's as if there were no trust settings at all, so we should maybe
	// fallback to the admin trust settings? TODO(golang.org/issue/38888).

	return macOS.SecTrustSettingsResultUnspecified, nil
}
<commit_msg>crypto/x509: match cgo and Apple behavior in domain fallback of macOS roots<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"bytes"
	"crypto/x509/internal/macOS"
	"fmt"
	"os"
	"strings"
)

var debugDarwinRoots = strings.Contains(os.Getenv("GODEBUG"), "x509roots=1")

func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
	return nil, nil
}

// loadSystemRootsWithCgo is set in root_cgo_darwin_amd64.go when cgo is
// available, and is only used for testing.
var loadSystemRootsWithCgo func() (*CertPool, error)

func loadSystemRoots() (*CertPool, error) {
	var trustedRoots []*Certificate
	untrustedRoots := make(map[string]bool)

	// macOS has three trust domains: one for CAs added by users to their
	// "login" keychain, one for CAs added by Admins to the "System" keychain,
	// and one for the CAs that ship with the OS.
	for _, domain := range []macOS.SecTrustSettingsDomain{
		macOS.SecTrustSettingsDomainUser,
		macOS.SecTrustSettingsDomainAdmin,
		macOS.SecTrustSettingsDomainSystem,
	} {
		certs, err := macOS.SecTrustSettingsCopyCertificates(domain)
		if err == macOS.ErrNoTrustSettings {
			continue
		} else if err != nil {
			return nil, err
		}
		defer macOS.CFRelease(certs)

		for i := 0; i < macOS.CFArrayGetCount(certs); i++ {
			c := macOS.CFArrayGetValueAtIndex(certs, i)
			cert, err := exportCertificate(c)
			if err != nil {
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: domain %d, certificate #%d: %v\n", domain, i, err)
				}
				continue
			}

			var result macOS.SecTrustSettingsResult
			if domain == macOS.SecTrustSettingsDomainSystem {
				// Certs found in the system domain are always trusted. If the user
				// configures "Never Trust" on such a cert, it will also be found in the
				// admin or user domain, causing it to be added to untrustedRoots.
				result = macOS.SecTrustSettingsResultTrustRoot
			} else {
				result, err = sslTrustSettingsResult(c)
				if err != nil {
					if debugDarwinRoots {
						fmt.Fprintf(os.Stderr, "crypto/x509: trust settings for %v: %v\n", cert.Subject, err)
					}
					continue
				}
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: trust settings for %v: %d\n", cert.Subject, result)
				}
			}

			switch result {
			// "Note the distinction between the results kSecTrustSettingsResultTrustRoot
			// and kSecTrustSettingsResultTrustAsRoot: The former can only be applied to
			// root (self-signed) certificates; the latter can only be applied to
			// non-root certificates."
			case macOS.SecTrustSettingsResultTrustRoot:
				if isRootCertificate(cert) {
					trustedRoots = append(trustedRoots, cert)
				}
			case macOS.SecTrustSettingsResultTrustAsRoot:
				if !isRootCertificate(cert) {
					trustedRoots = append(trustedRoots, cert)
				}

			case macOS.SecTrustSettingsResultDeny:
				// Add this certificate to untrustedRoots, which are subtracted
				// from trustedRoots, so that we don't have to evaluate policies
				// for every root in the system domain, but still apply user and
				// admin policies that override system roots.
				untrustedRoots[string(cert.Raw)] = true

			case macOS.SecTrustSettingsResultUnspecified:
				// Certificates with unspecified trust should be added to a pool
				// of intermediates for chain building, but we don't support it
				// at the moment. This is Issue 35631.

			default:
				if debugDarwinRoots {
					fmt.Fprintf(os.Stderr, "crypto/x509: unknown trust setting for %v: %d\n", cert.Subject, result)
				}
			}
		}
	}

	pool := NewCertPool()
	for _, cert := range trustedRoots {
		if !untrustedRoots[string(cert.Raw)] {
			pool.AddCert(cert)
		}
	}
	return pool, nil
}

// exportCertificate returns a *Certificate for a SecCertificateRef.
func exportCertificate(cert macOS.CFRef) (*Certificate, error) {
	data, err := macOS.SecItemExport(cert)
	if err != nil {
		return nil, err
	}
	defer macOS.CFRelease(data)
	der := macOS.CFDataToSlice(data)

	return ParseCertificate(der)
}

// isRootCertificate reports whether Subject and Issuer match.
func isRootCertificate(cert *Certificate) bool {
	return bytes.Equal(cert.RawSubject, cert.RawIssuer)
}

// sslTrustSettingsResult obtains the final kSecTrustSettingsResult value for a
// certificate in the user or admin domain, combining usage constraints for the
// SSL SecTrustSettingsPolicy,
//
// It ignores SecTrustSettingsKeyUsage and kSecTrustSettingsAllowedError, and
// doesn't support kSecTrustSettingsDefaultRootCertSetting.
//
// https://developer.apple.com/documentation/security/1400261-sectrustsettingscopytrustsetting
func sslTrustSettingsResult(cert macOS.CFRef) (macOS.SecTrustSettingsResult, error) {
	// In Apple's implementation user trust settings override admin trust settings
	// (which themselves override system trust settings). If SecTrustSettingsCopyTrustSettings
	// fails, or returns a NULL trust settings, when looking for the user trust
	// settings then fallback to checking the admin trust settings.
	//
	// See Security-59306.41.2/trust/headers/SecTrustSettings.h for a description of
	// the trust settings overrides, and SecLegacyAnchorSourceCopyUsageConstraints in
	// Security-59306.41.2/trust/trustd/SecCertificateSource.c for a concrete example
	// of how Apple applies the override in the case of NULL trust settings, or non
	// success errors.
	trustSettings, err := macOS.SecTrustSettingsCopyTrustSettings(cert, macOS.SecTrustSettingsDomainUser)
	if err != nil || trustSettings == 0 {
		if debugDarwinRoots && err != macOS.ErrNoTrustSettings {
			fmt.Fprintf(os.Stderr, "crypto/x509: SecTrustSettingsCopyTrustSettings for SecTrustSettingsDomainUser failed: %s\n", err)
		}
		trustSettings, err = macOS.SecTrustSettingsCopyTrustSettings(cert, macOS.SecTrustSettingsDomainAdmin)
	}
	if err != nil || trustSettings == 0 {
		// If there are neither user nor admin trust settings for a certificate returned
		// from SecTrustSettingsCopyCertificates Apple returns kSecTrustSettingsResultInvalid,
		// as this method is intended to return certificates _which have trust settings_.
		// The most likely case for this being triggered is that the existing trust settings
		// are invalid and cannot be properly parsed. In this case SecTrustSettingsCopyTrustSettings
		// returns errSecInvalidTrustSettings. The existing cgo implementation returns
		// kSecTrustSettingsResultUnspecified in this case, which mostly matches the Apple
		// implementation because we don't do anything with certificates marked with this
		// result.
		//
		// See SecPVCGetTrustSettingsResult in Security-59306.41.2/trust/trustd/SecPolicyServer.c
		if debugDarwinRoots && err != macOS.ErrNoTrustSettings {
			fmt.Fprintf(os.Stderr, "crypto/x509: SecTrustSettingsCopyTrustSettings for SecTrustSettingsDomainAdmin failed: %s\n", err)
		}
		return macOS.SecTrustSettingsResultUnspecified, nil
	}
	defer macOS.CFRelease(trustSettings)

	// "An empty trust settings array means 'always trust this certificate' with an
	// overall trust setting for the certificate of kSecTrustSettingsResultTrustRoot."
	if macOS.CFArrayGetCount(trustSettings) == 0 {
		return macOS.SecTrustSettingsResultTrustRoot, nil
	}

	isSSLPolicy := func(policyRef macOS.CFRef) bool {
		properties := macOS.SecPolicyCopyProperties(policyRef)
		defer macOS.CFRelease(properties)
		if v, ok := macOS.CFDictionaryGetValueIfPresent(properties, macOS.SecPolicyOid); ok {
			return macOS.CFEqual(v, macOS.CFRef(macOS.SecPolicyAppleSSL))
		}
		return false
	}

	for i := 0; i < macOS.CFArrayGetCount(trustSettings); i++ {
		tSetting := macOS.CFArrayGetValueAtIndex(trustSettings, i)

		// First, check if this trust setting is constrained to a non-SSL policy.
		if policyRef, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsPolicy); ok {
			if !isSSLPolicy(policyRef) {
				continue
			}
		}

		// Then check if it is restricted to a hostname, so not a root.
		if _, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsPolicyString); ok {
			continue
		}

		cfNum, ok := macOS.CFDictionaryGetValueIfPresent(tSetting, macOS.SecTrustSettingsResultKey)
		// "If this key is not present, a default value of kSecTrustSettingsResultTrustRoot is assumed."
		if !ok {
			return macOS.SecTrustSettingsResultTrustRoot, nil
		}
		result, err := macOS.CFNumberGetValue(cfNum)
		if err != nil {
			return 0, err
		}

		// If multiple dictionaries match, we are supposed to "OR" them,
		// the semantics of which are not clear. Since TrustRoot and TrustAsRoot
		// are mutually exclusive, Deny should probably override, and Invalid and
		// Unspecified be overridden, approximate this by stopping at the first
		// TrustRoot, TrustAsRoot or Deny.
		switch r := macOS.SecTrustSettingsResult(result); r {
		case macOS.SecTrustSettingsResultTrustRoot,
			macOS.SecTrustSettingsResultTrustAsRoot,
			macOS.SecTrustSettingsResultDeny:
			return r, nil
		}
	}

	// If trust settings are present, but none of them match the policy...
	// the docs don't tell us what to do.
	//
	// "Trust settings for a given use apply if any of the dictionaries in the
	// certificate’s trust settings array satisfies the specified use." suggests
	// that it's as if there were no trust settings at all, so we should maybe
	// fallback to the admin trust settings? TODO(golang.org/issue/38888).

	return macOS.SecTrustSettingsResultUnspecified, nil
}
<|endoftext|>


<commit_before>package mailgun

import (
	"errors"
	"github.com/mbanzon/simplehttp"
)

type Message struct {
	from     string
	to       []string
	cc       []string
	bcc      []string
	subject  string
	text     string
	html     string
	tags     []string
	campaigns []string
}

type sendMessageResponse struct {
	Message string `json:"message"`
	Id      string `json:"id"`
}

func NewMessage(from string, subject string, text string, to ...string) *Message {
	return &Message{from: from, subject: subject, text: text, to: to}
}

func (m *Message) AddRecipient(recipient string) {
	m.to = append(m.to, recipient)
}

func (m *Message) AddCC(recipient string) {
	m.cc = append(m.cc, recipient)
}

func (m *Message) AddBCC(recipient string) {
	m.bcc = append(m.bcc, recipient)
}

func (m *Message) SetHtml(html string) {
	m.html = html
}
func (m *Message) AddTag(tag string) {
	m.tags = append(m.tags, tag)
}

func (m *Message) AddCampaign(campain string) {
	m.campaigns = append(m.campaigns, campain)
}

func (m *mailgunImpl) Send(message *Message) (mes string, id string, err error) {
	if !message.validateMessage() {
		err = errors.New("Message not valid")
	} else {
		r := simplehttp.NewPostRequest(generateApiUrl(m, messagesEndpoint))
		r.AddFormValue("from", message.from)
		r.AddFormValue("subject", message.subject)
		r.AddFormValue("text", message.text)
		for _, to := range message.to {
			r.AddFormValue("to", to)
		}
		for _, cc := range message.cc {
			r.AddFormValue("cc", cc)
		}
		for _, bcc := range message.bcc {
			r.AddFormValue("bcc", bcc)
		}
		for _, tag := range message.tags {
			r.AddFormValue("o:tag", tag)
		}
		for _, campain := range message.campaigns {
			r.AddFormValue("o:campain", campain)
		}
		if message.html != "" {
			r.AddFormValue("html", message.html)
		}
		r.SetBasicAuth(basicAuthUser, m.ApiKey())

		var response sendMessageResponse
		err = r.MakeJSONRequest(&response)
		if err == nil {
			mes = response.Message
			id = response.Id
		}
	}

	return
}

func (m *Message) validateMessage() bool {
	if m == nil {
		return false
	}

	if m.from == "" {
		return false
	}

	if !validateStringList(m.to, true) {
		return false
	}

	if !validateStringList(m.cc, false) {
		return false
	}

	if !validateStringList(m.bcc, false) {
		return false
	}

	if !validateStringList(m.tags, false) {
		return false
	}

	if !validateStringList(m.campaigns, false) || len(m.campaigns) > 3 {
		return false
	}

	if m.text == "" {
		return false
	}

	return true
}

func validateStringList(list []string, requireOne bool) bool {
	hasOne := false

	if list == nil {
		return !requireOne
	} else {
		for _, a := range list {
			if a == "" {
				return false
			} else {
				hasOne = hasOne || true
			}
		}
	}

	return hasOne
}
<commit_msg>Gofmt.<commit_after>package mailgun

import (
	"errors"
	"github.com/mbanzon/simplehttp"
)

type Message struct {
	from      string
	to        []string
	cc        []string
	bcc       []string
	subject   string
	text      string
	html      string
	tags      []string
	campaigns []string
}

type sendMessageResponse struct {
	Message string `json:"message"`
	Id      string `json:"id"`
}

func NewMessage(from string, subject string, text string, to ...string) *Message {
	return &Message{from: from, subject: subject, text: text, to: to}
}

func (m *Message) AddRecipient(recipient string) {
	m.to = append(m.to, recipient)
}

func (m *Message) AddCC(recipient string) {
	m.cc = append(m.cc, recipient)
}

func (m *Message) AddBCC(recipient string) {
	m.bcc = append(m.bcc, recipient)
}

func (m *Message) SetHtml(html string) {
	m.html = html
}
func (m *Message) AddTag(tag string) {
	m.tags = append(m.tags, tag)
}

func (m *Message) AddCampaign(campain string) {
	m.campaigns = append(m.campaigns, campain)
}

func (m *mailgunImpl) Send(message *Message) (mes string, id string, err error) {
	if !message.validateMessage() {
		err = errors.New("Message not valid")
	} else {
		r := simplehttp.NewPostRequest(generateApiUrl(m, messagesEndpoint))
		r.AddFormValue("from", message.from)
		r.AddFormValue("subject", message.subject)
		r.AddFormValue("text", message.text)
		for _, to := range message.to {
			r.AddFormValue("to", to)
		}
		for _, cc := range message.cc {
			r.AddFormValue("cc", cc)
		}
		for _, bcc := range message.bcc {
			r.AddFormValue("bcc", bcc)
		}
		for _, tag := range message.tags {
			r.AddFormValue("o:tag", tag)
		}
		for _, campain := range message.campaigns {
			r.AddFormValue("o:campain", campain)
		}
		if message.html != "" {
			r.AddFormValue("html", message.html)
		}
		r.SetBasicAuth(basicAuthUser, m.ApiKey())

		var response sendMessageResponse
		err = r.MakeJSONRequest(&response)
		if err == nil {
			mes = response.Message
			id = response.Id
		}
	}

	return
}

func (m *Message) validateMessage() bool {
	if m == nil {
		return false
	}

	if m.from == "" {
		return false
	}

	if !validateStringList(m.to, true) {
		return false
	}

	if !validateStringList(m.cc, false) {
		return false
	}

	if !validateStringList(m.bcc, false) {
		return false
	}

	if !validateStringList(m.tags, false) {
		return false
	}

	if !validateStringList(m.campaigns, false) || len(m.campaigns) > 3 {
		return false
	}

	if m.text == "" {
		return false
	}

	return true
}

func validateStringList(list []string, requireOne bool) bool {
	hasOne := false

	if list == nil {
		return !requireOne
	} else {
		for _, a := range list {
			if a == "" {
				return false
			} else {
				hasOne = hasOne || true
			}
		}
	}

	return hasOne
}
<|endoftext|>


<commit_before>package dht

import (
	"bytes"
	"context"
	"errors"
	"fmt"

	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/routing"

	record "github.com/libp2p/go-libp2p-record"
	recpb "github.com/libp2p/go-libp2p-record/pb"
	"github.com/multiformats/go-multihash"

	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
)

type ProtocolMessenger struct {
	m         *messageManager
	validator record.Validator
}

func NewProtocolMessenger(host host.Host, protocols []protocol.ID, validator record.Validator) *ProtocolMessenger {
	return &ProtocolMessenger{
		m: &messageManager{
			host:      host,
			strmap:    make(map[peer.ID]*messageSender),
			protocols: protocols,
		},
		validator: validator,
	}
}

// putValueToPeer stores the given key/value pair at the peer 'p'
func (pm *ProtocolMessenger) PutValue(ctx context.Context, p peer.ID, rec *recpb.Record) error {
	pmes := pb.NewMessage(pb.Message_PUT_VALUE, rec.Key, 0)
	pmes.Record = rec
	rpmes, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		logger.Debugw("failed to put value to peer", "to", p, "key", loggableRecordKeyBytes(rec.Key), "error", err)
		return err
	}

	if !bytes.Equal(rpmes.GetRecord().Value, pmes.GetRecord().Value) {
		logger.Infow("value not put correctly", "put-message", pmes, "get-message", rpmes)
		return errors.New("value not put correctly")
	}

	return nil
}

// GetValue queries a particular peer p for the value for
// key. It returns the value and a list of closer peers.
func (pm *ProtocolMessenger) GetValue(ctx context.Context, p peer.ID, key string) (*recpb.Record, []*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_GET_VALUE, []byte(key), 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, nil, err
	}

	// Perhaps we were given closer peers
	peers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())

	if rec := respMsg.GetRecord(); rec != nil {
		// Success! We were given the value
		logger.Debug("got value")

		// make sure record is valid.
		err = pm.validator.Validate(string(rec.GetKey()), rec.GetValue())
		if err != nil {
			logger.Debug("received invalid record (discarded)")
			// return a sentinal to signify an invalid record was received
			err = errInvalidRecord
			rec = new(recpb.Record)
		}
		return rec, peers, err
	}

	if len(peers) > 0 {
		return nil, peers, nil
	}

	return nil, nil, routing.ErrNotFound
}

// findPeerSingle asks peer 'p' if they know where the peer with id 'id' is
func (pm *ProtocolMessenger) GetClosestPeers(ctx context.Context, p peer.ID, id peer.ID) ([]*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_FIND_NODE, []byte(id), 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, err
	}
	peers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())
	return peers, nil
}

func (pm *ProtocolMessenger) PutProvider(ctx context.Context, p peer.ID, key multihash.Multihash, host host.Host) error {
	pi := peer.AddrInfo{
		ID:    host.ID(),
		Addrs: host.Addrs(),
	}

	// // only share WAN-friendly addresses ??
	// pi.Addrs = addrutil.WANShareableAddrs(pi.Addrs)
	if len(pi.Addrs) < 1 {
		return fmt.Errorf("no known addresses for self, cannot put provider")
	}

	pmes := pb.NewMessage(pb.Message_ADD_PROVIDER, key, 0)
	pmes.ProviderPeers = pb.RawPeerInfosToPBPeers([]peer.AddrInfo{pi})

	return pm.m.sendMessage(ctx, p, pmes)
}

func (pm *ProtocolMessenger) GetProviders(ctx context.Context, p peer.ID, key multihash.Multihash) ([]*peer.AddrInfo, []*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_GET_PROVIDERS, key, 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, nil, err
	}
	provs := pb.PBPeersToPeerInfos(respMsg.GetProviderPeers())
	closerPeers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())
	return provs, closerPeers, nil
}

// Ping sends a ping message to the passed peer and waits for a response.
func (pm *ProtocolMessenger) Ping(ctx context.Context, p peer.ID) error {
	req := pb.NewMessage(pb.Message_PING, nil, 0)
	resp, err := pm.m.sendRequest(ctx, p, req)
	if err != nil {
		return fmt.Errorf("sending request: %w", err)
	}
	if resp.Type != pb.Message_PING {
		return fmt.Errorf("got unexpected response type: %v", resp.Type)
	}
	return nil
}
<commit_msg>docs: cleaned up comments<commit_after>package dht

import (
	"bytes"
	"context"
	"errors"
	"fmt"

	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/routing"

	record "github.com/libp2p/go-libp2p-record"
	recpb "github.com/libp2p/go-libp2p-record/pb"
	"github.com/multiformats/go-multihash"

	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
)

// ProtocolMessenger can be used for sending DHT messages to peers and processing their responses.
// This decouples the wire protocol format from both the DHT protocol implementation and from the implementation of the
// routing.Routing interface.
//
// TODO: This is still strongly coupled with the existing implementation of what happens when a peer actually sends a
// message on the wire (e.g. reusing streams, reusing connections, metrics tracking, etc.).
type ProtocolMessenger struct {
	m         *messageManager
	validator record.Validator
}

// NewProtocolMessenger creates a new ProtocolMessenger that is used for sending DHT messages to peers and processing
// their responses.
func NewProtocolMessenger(host host.Host, protocols []protocol.ID, validator record.Validator) *ProtocolMessenger {
	return &ProtocolMessenger{
		m: &messageManager{
			host:      host,
			strmap:    make(map[peer.ID]*messageSender),
			protocols: protocols,
		},
		validator: validator,
	}
}

// PutValue asks a peer to store the given key/value pair.
func (pm *ProtocolMessenger) PutValue(ctx context.Context, p peer.ID, rec *recpb.Record) error {
	pmes := pb.NewMessage(pb.Message_PUT_VALUE, rec.Key, 0)
	pmes.Record = rec
	rpmes, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		logger.Debugw("failed to put value to peer", "to", p, "key", loggableRecordKeyBytes(rec.Key), "error", err)
		return err
	}

	if !bytes.Equal(rpmes.GetRecord().Value, pmes.GetRecord().Value) {
		logger.Infow("value not put correctly", "put-message", pmes, "get-message", rpmes)
		return errors.New("value not put correctly")
	}

	return nil
}

// GetValue asks a peer for the value corresponding to the given key. Also returns the K closest peers to the key
// as described in GetClosestPeers.
func (pm *ProtocolMessenger) GetValue(ctx context.Context, p peer.ID, key string) (*recpb.Record, []*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_GET_VALUE, []byte(key), 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, nil, err
	}

	// Perhaps we were given closer peers
	peers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())

	if rec := respMsg.GetRecord(); rec != nil {
		// Success! We were given the value
		logger.Debug("got value")

		// make sure record is valid.
		err = pm.validator.Validate(string(rec.GetKey()), rec.GetValue())
		if err != nil {
			logger.Debug("received invalid record (discarded)")
			// return a sentinel to signify an invalid record was received
			err = errInvalidRecord
			rec = new(recpb.Record)
		}
		return rec, peers, err
	}

	if len(peers) > 0 {
		return nil, peers, nil
	}

	return nil, nil, routing.ErrNotFound
}

// GetClosestPeers asks a peer to return the K (a DHT-wide parameter) DHT server peers closest in XOR space to the id
// Note: If the peer happens to know another peer whose peerID exactly matches the given id it will return that peer
// even if that peer is not a DHT server node.
func (pm *ProtocolMessenger) GetClosestPeers(ctx context.Context, p peer.ID, id peer.ID) ([]*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_FIND_NODE, []byte(id), 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, err
	}
	peers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())
	return peers, nil
}

// PutProvider asks a peer to store that we are a provider for the given key.
func (pm *ProtocolMessenger) PutProvider(ctx context.Context, p peer.ID, key multihash.Multihash, host host.Host) error {
	pi := peer.AddrInfo{
		ID:    host.ID(),
		Addrs: host.Addrs(),
	}

	// TODO: We may want to limit the type of addresses in our provider records
	// For example, in a WAN-only DHT prohibit sharing non-WAN addresses (e.g. 192.168.0.100)
	if len(pi.Addrs) < 1 {
		return fmt.Errorf("no known addresses for self, cannot put provider")
	}

	pmes := pb.NewMessage(pb.Message_ADD_PROVIDER, key, 0)
	pmes.ProviderPeers = pb.RawPeerInfosToPBPeers([]peer.AddrInfo{pi})

	return pm.m.sendMessage(ctx, p, pmes)
}

// GetProviders asks a peer for the providers it knows of for a given key. Also returns the K closest peers to the key
// as described in GetClosestPeers.
func (pm *ProtocolMessenger) GetProviders(ctx context.Context, p peer.ID, key multihash.Multihash) ([]*peer.AddrInfo, []*peer.AddrInfo, error) {
	pmes := pb.NewMessage(pb.Message_GET_PROVIDERS, key, 0)
	respMsg, err := pm.m.sendRequest(ctx, p, pmes)
	if err != nil {
		return nil, nil, err
	}
	provs := pb.PBPeersToPeerInfos(respMsg.GetProviderPeers())
	closerPeers := pb.PBPeersToPeerInfos(respMsg.GetCloserPeers())
	return provs, closerPeers, nil
}

// Ping sends a ping message to the passed peer and waits for a response.
func (pm *ProtocolMessenger) Ping(ctx context.Context, p peer.ID) error {
	req := pb.NewMessage(pb.Message_PING, nil, 0)
	resp, err := pm.m.sendRequest(ctx, p, req)
	if err != nil {
		return fmt.Errorf("sending request: %w", err)
	}
	if resp.Type != pb.Message_PING {
		return fmt.Errorf("got unexpected response type: %v", resp.Type)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"container/heap"
	"fmt"
	"rand"
	"time"
)

const nRequester = 100
const nWorker = 10

// Simulation of some work: just sleep for a while and report how long.
func op() int {
	n := rand.Int63n(1e9)
	time.Sleep(nWorker * n)
	return int(n)
}

type Request struct {
	fn func() int
	c  chan int
}

func requester(work chan Request) {
	c := make(chan int)
	for {
		time.Sleep(rand.Int63n(nWorker * 2e9))
		work <- Request{op, c}
		<-c
	}
}

type Worker struct {
	i        int
	requests chan Request
	pending  int
}

func (w *Worker) work(done chan *Worker) {
	for {
		req := <-w.requests
		req.c <- req.fn()
		done <- w
	}
}

type Pool []*Worker

func (p Pool) Len() int { return len(p) }

func (p Pool) Less(i, j int) bool {
	return p[i].pending < p[j].pending
}

func (p *Pool) Swap(i, j int) {
	a := *p
	a[i], a[j] = a[j], a[i]
	a[i].i = i
	a[j].i = j
}

func (p *Pool) Push(x interface{}) {
	a := *p
	n := len(a)
	a = a[0 : n+1]
	w := x.(*Worker)
	a[n] = w
	w.i = n
	*p = a
}

func (p *Pool) Pop() interface{} {
	a := *p
	*p = a[0 : len(a)-1]
	w := a[len(a)-1]
	w.i = -1 // for safety
	return w
}

type Balancer struct {
	pool Pool
	done chan *Worker
	i    int
}

func NewBalancer() *Balancer {
	done := make(chan *Worker, nWorker)
	b := &Balancer{make(Pool, 0, nWorker), done, 0}
	for i := 0; i < nWorker; i++ {
		w := &Worker{requests: make(chan Request, nRequester)}
		heap.Push(&b.pool, w)
		go w.work(b.done)
	}
	return b
}

func (b *Balancer) balance(work chan Request) {
	for {
		select {
		case req := <-work:
			b.dispatch(req)
		case w := <-b.done:
			b.completed(w)
		}
		b.print()
	}
}

func (b *Balancer) print() {
	sum := 0
	sumsq := 0
	for _, w := range b.pool {
		fmt.Printf("%d ", w.pending)
		sum += w.pending
		sumsq += w.pending * w.pending
	}
	avg := float64(sum) / float64(len(b.pool))
	variance := float64(sumsq)/float64(len(b.pool)) - avg*avg
	fmt.Printf(" %.2f %.2f\n", avg, variance)
}

func (b *Balancer) dispatch(req Request) {
	if false {
		w := b.pool[b.i]
		w.requests <- req
		w.pending++
		b.i++
		if b.i >= len(b.pool) {
			b.i = 0
		}
		return
	}

	w := heap.Pop(&b.pool).(*Worker)
	w.requests <- req
	w.pending++
	//	fmt.Printf("started %p; now %d\n", w, w.pending)
	heap.Push(&b.pool, w)
}

func (b *Balancer) completed(w *Worker) {
	if false {
		w.pending--
		return
	}

	w.pending--
	//	fmt.Printf("finished %p; now %d\n", w, w.pending)
	heap.Remove(&b.pool, w.i)
	heap.Push(&b.pool, w)
}

func main() {
	work := make(chan Request)
	for i := 0; i < nRequester; i++ {
		go requester(work)
	}
	NewBalancer().balance(work)
}
<commit_msg>doc: add round-robin flag to io2010 balance example.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"container/heap"
	"flag"
	"fmt"
	"rand"
	"time"
)

const nRequester = 100
const nWorker = 10

var roundRobin = flag.Bool("r", false, "use round-robin scheduling")

// Simulation of some work: just sleep for a while and report how long.
func op() int {
	n := rand.Int63n(1e9)
	time.Sleep(nWorker * n)
	return int(n)
}

type Request struct {
	fn func() int
	c  chan int
}

func requester(work chan Request) {
	c := make(chan int)
	for {
		time.Sleep(rand.Int63n(nWorker * 2e9))
		work <- Request{op, c}
		<-c
	}
}

type Worker struct {
	i        int
	requests chan Request
	pending  int
}

func (w *Worker) work(done chan *Worker) {
	for {
		req := <-w.requests
		req.c <- req.fn()
		done <- w
	}
}

type Pool []*Worker

func (p Pool) Len() int { return len(p) }

func (p Pool) Less(i, j int) bool {
	return p[i].pending < p[j].pending
}

func (p *Pool) Swap(i, j int) {
	a := *p
	a[i], a[j] = a[j], a[i]
	a[i].i = i
	a[j].i = j
}

func (p *Pool) Push(x interface{}) {
	a := *p
	n := len(a)
	a = a[0 : n+1]
	w := x.(*Worker)
	a[n] = w
	w.i = n
	*p = a
}

func (p *Pool) Pop() interface{} {
	a := *p
	*p = a[0 : len(a)-1]
	w := a[len(a)-1]
	w.i = -1 // for safety
	return w
}

type Balancer struct {
	pool Pool
	done chan *Worker
	i    int
}

func NewBalancer() *Balancer {
	done := make(chan *Worker, nWorker)
	b := &Balancer{make(Pool, 0, nWorker), done, 0}
	for i := 0; i < nWorker; i++ {
		w := &Worker{requests: make(chan Request, nRequester)}
		heap.Push(&b.pool, w)
		go w.work(b.done)
	}
	return b
}

func (b *Balancer) balance(work chan Request) {
	for {
		select {
		case req := <-work:
			b.dispatch(req)
		case w := <-b.done:
			b.completed(w)
		}
		b.print()
	}
}

func (b *Balancer) print() {
	sum := 0
	sumsq := 0
	for _, w := range b.pool {
		fmt.Printf("%d ", w.pending)
		sum += w.pending
		sumsq += w.pending * w.pending
	}
	avg := float64(sum) / float64(len(b.pool))
	variance := float64(sumsq)/float64(len(b.pool)) - avg*avg
	fmt.Printf(" %.2f %.2f\n", avg, variance)
}

func (b *Balancer) dispatch(req Request) {
	if *roundRobin {
		w := b.pool[b.i]
		w.requests <- req
		w.pending++
		b.i++
		if b.i >= len(b.pool) {
			b.i = 0
		}
		return
	}

	w := heap.Pop(&b.pool).(*Worker)
	w.requests <- req
	w.pending++
	//	fmt.Printf("started %p; now %d\n", w, w.pending)
	heap.Push(&b.pool, w)
}

func (b *Balancer) completed(w *Worker) {
	if *roundRobin {
		w.pending--
		return
	}

	w.pending--
	//	fmt.Printf("finished %p; now %d\n", w, w.pending)
	heap.Remove(&b.pool, w.i)
	heap.Push(&b.pool, w)
}

func main() {
	flag.Parse()
	work := make(chan Request)
	for i := 0; i < nRequester; i++ {
		go requester(work)
	}
	NewBalancer().balance(work)
}
<|endoftext|>


<commit_before>// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chunkenc

import (
	"math"

	"github.com/prometheus/prometheus/model/histogram"
)

func writeHistogramChunkLayout(b *bstream, schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span) {
	putVarbitInt(b, int64(schema))
	putZeroThreshold(b, zeroThreshold)
	putHistogramChunkLayoutSpans(b, positiveSpans)
	putHistogramChunkLayoutSpans(b, negativeSpans)
}

func readHistogramChunkLayout(b *bstreamReader) (
	schema int32, zeroThreshold float64,
	positiveSpans, negativeSpans []histogram.Span,
	err error,
) {
	v, err := readVarbitInt(b)
	if err != nil {
		return
	}
	schema = int32(v)

	zeroThreshold, err = readZeroThreshold(b)
	if err != nil {
		return
	}

	positiveSpans, err = readHistogramChunkLayoutSpans(b)
	if err != nil {
		return
	}

	negativeSpans, err = readHistogramChunkLayoutSpans(b)
	if err != nil {
		return
	}

	return
}

func putHistogramChunkLayoutSpans(b *bstream, spans []histogram.Span) {
	putVarbitInt(b, int64(len(spans)))
	for _, s := range spans {
		putVarbitUint(b, uint64(s.Length))
		putVarbitInt(b, int64(s.Offset))
	}
}

func readHistogramChunkLayoutSpans(b *bstreamReader) ([]histogram.Span, error) {
	var spans []histogram.Span
	num, err := readVarbitInt(b)
	if err != nil {
		return nil, err
	}
	for i := 0; i < int(num); i++ {

		length, err := readVarbitUint(b)
		if err != nil {
			return nil, err
		}

		offset, err := readVarbitInt(b)
		if err != nil {
			return nil, err
		}

		spans = append(spans, histogram.Span{
			Length: uint32(length),
			Offset: int32(offset),
		})
	}
	return spans, nil
}

// putZeroThreshold writes the zero threshold to the bstream. It stores typical
// values in just one byte, but needs 9 bytes for other values. In detail:
//
// * If the threshold is 0, store a single zero byte.
//
// * If the threshold is a power of 2 between (and including) 2^-243 and 2^10,
//   take the exponent from the IEEE 754 representation of the threshold, which
//   covers a range between (and including) -242 and 11. (2^-243 is 0.5*2^-242
//   in IEEE 754 representation, and 2^10 is 0.5*2^11.) Add 243 to the exponent
//   and store the result (which will be between 1 and 254) as a single
//   byte. Note that small powers of two are preferred values for the zero
//   threshould. The default value for the zero threshold is 2^-128 (or
//   0.5*2^-127 in IEEE 754 representation) and will therefore be encoded as a
//   single byte (with value 116).
//
// * In all other cases, store 255 as a single byte, followed by the 8 bytes of
//   the threshold as a float64, i.e. taking 9 bytes in total.
func putZeroThreshold(b *bstream, threshold float64) {
	if threshold == 0 {
		b.writeByte(0)
		return
	}
	frac, exp := math.Frexp(threshold)
	if frac != 0.5 || exp < -242 || exp > 11 {
		b.writeByte(255)
		b.writeBits(math.Float64bits(threshold), 64)
		return
	}
	b.writeByte(byte(exp + 243))
}

// readZeroThreshold reads the zero threshold written with putZeroThreshold.
func readZeroThreshold(br *bstreamReader) (float64, error) {
	b, err := br.ReadByte()
	if err != nil {
		return 0, err
	}
	switch b {
	case 0:
		return 0, nil
	case 255:
		v, err := br.readBits(64)
		if err != nil {
			return 0, err
		}
		return math.Float64frombits(v), nil
	default:
		return math.Ldexp(0.5, int(b-243)), nil
	}
}

type bucketIterator struct {
	spans  []histogram.Span
	span   int // Span position of last yielded bucket.
	bucket int // Bucket position within span of last yielded bucket.
	idx    int // Bucket index (globally across all spans) of last yielded bucket.
}

func newBucketIterator(spans []histogram.Span) *bucketIterator {
	b := bucketIterator{
		spans:  spans,
		span:   0,
		bucket: -1,
		idx:    -1,
	}
	if len(spans) > 0 {
		b.idx += int(spans[0].Offset)
	}
	return &b
}

func (b *bucketIterator) Next() (int, bool) {
	// We're already out of bounds.
	if b.span >= len(b.spans) {
		return 0, false
	}
try:
	if b.bucket < int(b.spans[b.span].Length-1) { // Try to move within same span.
		b.bucket++
		b.idx++
		return b.idx, true
	} else if b.span < len(b.spans)-1 { // Try to move from one span to the next.
		b.span++
		b.idx += int(b.spans[b.span].Offset + 1)
		b.bucket = 0
		if b.spans[b.span].Length == 0 {
			// Pathological case that should never happen. We can't use this span, let's try again.
			goto try
		}
		return b.idx, true
	}
	// We're out of options.
	return 0, false
}

// An Interjection describes how many new buckets have to be introduced before
// processing the pos'th delta from the original slice.
type Interjection struct {
	pos int
	num int
}

// compareSpans returns the interjections to convert a slice of deltas to a new
// slice representing an expanded set of buckets, or false if incompatible
// (e.g. if buckets were removed).
//
// Example:
//
// Let's say the old buckets look like this:
//
//   span syntax: [offset, length]
//   spans      : [ 0 , 2 ]               [2,1]                   [ 3 , 2 ]                     [3,1]       [1,1]
//   bucket idx : [0]   [1]    2     3    [4]    5     6     7    [8]   [9]    10    11    12   [13]   14   [15]
//   raw values    6     3                 3                       2     4                       5           1
//   deltas        6    -3                 0                      -1     2                       1          -4
//
// But now we introduce a new bucket layout. (Carefully chosen example where we
// have a span appended, one unchanged[*], one prepended, and two merge - in
// that order.)
//
// [*] unchanged in terms of which bucket indices they represent. but to achieve
// that, their offset needs to change if "disrupted" by spans changing ahead of
// them
//
//                                         \/ this one is "unchanged"
//   spans      : [  0  ,  3    ]         [1,1]       [    1    ,   4     ]                     [  3  ,   3    ]
//   bucket idx : [0]   [1]   [2]    3    [4]    5    [6]   [7]   [8]   [9]    10    11    12   [13]  [14]  [15]
//   raw values    6     3     0           3           0     0     2     4                       5     0     1
//   deltas        6    -3    -3           3          -3     0     2     2                       1    -5     1
//   delta mods:                          / \                     / \                                       / \
//
// Note that whenever any new buckets are introduced, the subsequent "old"
// bucket needs to readjust its delta to the new base of 0. Thus, for the caller
// who wants to transform the set of original deltas to a new set of deltas to
// match a new span layout that adds buckets, we simply need to generate a list
// of interjections.
//
// Note: Within compareSpans we don't have to worry about the changes to the
// spans themselves, thanks to the iterators we get to work with the more useful
// bucket indices (which of course directly correspond to the buckets we have to
// adjust).
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
	ai := newBucketIterator(a)
	bi := newBucketIterator(b)

	var interjections []Interjection

	// When inter.num becomes > 0, this becomes a valid interjection that
	// should be yielded when we finish a streak of new buckets.
	var inter Interjection

	av, aOK := ai.Next()
	bv, bOK := bi.Next()
loop:
	for {
		switch {
		case aOK && bOK:
			switch {
			case av == bv: // Both have an identical value. move on!
				// Finish WIP interjection and reset.
				if inter.num > 0 {
					interjections = append(interjections, inter)
				}
				inter.num = 0
				av, aOK = ai.Next()
				bv, bOK = bi.Next()
				inter.pos++
			case av < bv: // b misses a value that is in a.
				return interjections, false
			case av > bv: // a misses a value that is in b. Forward b and recompare.
				inter.num++
				bv, bOK = bi.Next()
			}
		case aOK && !bOK: // b misses a value that is in a.
			return interjections, false
		case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
			inter.num++
			bv, bOK = bi.Next()
		default: // Both iterators ran out. We're done.
			if inter.num > 0 {
				interjections = append(interjections, inter)
			}
			break loop
		}
	}

	return interjections, true
}

// interject merges 'in' with the provided interjections and writes them into
// 'out', which must already have the appropriate length.
func interject(in, out []int64, interjections []Interjection) []int64 {
	var (
		j      int   // Position in out.
		v      int64 // The last value seen.
		interj int   // The next interjection to process.
	)
	for i, d := range in {
		if interj < len(interjections) && i == interjections[interj].pos {

			// We have an interjection!
			// Add interjection.num new delta values such that their
			// bucket values equate 0.
			out[j] = int64(-v)
			j++
			for x := 1; x < interjections[interj].num; x++ {
				out[j] = 0
				j++
			}
			interj++

			// Now save the value from the input. The delta value we
			// should save is the original delta value + the last
			// value of the point before the interjection (to undo
			// the delta that was introduced by the interjection).
			out[j] = d + v
			j++
			v = d + v
			continue
		}

		// If there was no interjection, the original delta is still
		// valid.
		out[j] = d
		j++
		v += d
	}
	switch interj {
	case len(interjections):
		// All interjections processed. Nothing more to do.
	case len(interjections) - 1:
		// One more interjection to process at the end.
		out[j] = int64(-v)
		j++
		for x := 1; x < interjections[interj].num; x++ {
			out[j] = 0
			j++
		}
	default:
		panic("unprocessed interjections left")
	}
	return out
}
<commit_msg>Encode zero threshold first<commit_after>// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chunkenc

import (
	"math"

	"github.com/prometheus/prometheus/model/histogram"
)

func writeHistogramChunkLayout(b *bstream, schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span) {
	putZeroThreshold(b, zeroThreshold)
	putVarbitInt(b, int64(schema))
	putHistogramChunkLayoutSpans(b, positiveSpans)
	putHistogramChunkLayoutSpans(b, negativeSpans)
}

func readHistogramChunkLayout(b *bstreamReader) (
	schema int32, zeroThreshold float64,
	positiveSpans, negativeSpans []histogram.Span,
	err error,
) {
	zeroThreshold, err = readZeroThreshold(b)
	if err != nil {
		return
	}

	v, err := readVarbitInt(b)
	if err != nil {
		return
	}
	schema = int32(v)

	positiveSpans, err = readHistogramChunkLayoutSpans(b)
	if err != nil {
		return
	}

	negativeSpans, err = readHistogramChunkLayoutSpans(b)
	if err != nil {
		return
	}

	return
}

func putHistogramChunkLayoutSpans(b *bstream, spans []histogram.Span) {
	putVarbitInt(b, int64(len(spans)))
	for _, s := range spans {
		putVarbitUint(b, uint64(s.Length))
		putVarbitInt(b, int64(s.Offset))
	}
}

func readHistogramChunkLayoutSpans(b *bstreamReader) ([]histogram.Span, error) {
	var spans []histogram.Span
	num, err := readVarbitInt(b)
	if err != nil {
		return nil, err
	}
	for i := 0; i < int(num); i++ {

		length, err := readVarbitUint(b)
		if err != nil {
			return nil, err
		}

		offset, err := readVarbitInt(b)
		if err != nil {
			return nil, err
		}

		spans = append(spans, histogram.Span{
			Length: uint32(length),
			Offset: int32(offset),
		})
	}
	return spans, nil
}

// putZeroThreshold writes the zero threshold to the bstream. It stores typical
// values in just one byte, but needs 9 bytes for other values. In detail:
//
// * If the threshold is 0, store a single zero byte.
//
// * If the threshold is a power of 2 between (and including) 2^-243 and 2^10,
//   take the exponent from the IEEE 754 representation of the threshold, which
//   covers a range between (and including) -242 and 11. (2^-243 is 0.5*2^-242
//   in IEEE 754 representation, and 2^10 is 0.5*2^11.) Add 243 to the exponent
//   and store the result (which will be between 1 and 254) as a single
//   byte. Note that small powers of two are preferred values for the zero
//   threshould. The default value for the zero threshold is 2^-128 (or
//   0.5*2^-127 in IEEE 754 representation) and will therefore be encoded as a
//   single byte (with value 116).
//
// * In all other cases, store 255 as a single byte, followed by the 8 bytes of
//   the threshold as a float64, i.e. taking 9 bytes in total.
func putZeroThreshold(b *bstream, threshold float64) {
	if threshold == 0 {
		b.writeByte(0)
		return
	}
	frac, exp := math.Frexp(threshold)
	if frac != 0.5 || exp < -242 || exp > 11 {
		b.writeByte(255)
		b.writeBits(math.Float64bits(threshold), 64)
		return
	}
	b.writeByte(byte(exp + 243))
}

// readZeroThreshold reads the zero threshold written with putZeroThreshold.
func readZeroThreshold(br *bstreamReader) (float64, error) {
	b, err := br.ReadByte()
	if err != nil {
		return 0, err
	}
	switch b {
	case 0:
		return 0, nil
	case 255:
		v, err := br.readBits(64)
		if err != nil {
			return 0, err
		}
		return math.Float64frombits(v), nil
	default:
		return math.Ldexp(0.5, int(b-243)), nil
	}
}

type bucketIterator struct {
	spans  []histogram.Span
	span   int // Span position of last yielded bucket.
	bucket int // Bucket position within span of last yielded bucket.
	idx    int // Bucket index (globally across all spans) of last yielded bucket.
}

func newBucketIterator(spans []histogram.Span) *bucketIterator {
	b := bucketIterator{
		spans:  spans,
		span:   0,
		bucket: -1,
		idx:    -1,
	}
	if len(spans) > 0 {
		b.idx += int(spans[0].Offset)
	}
	return &b
}

func (b *bucketIterator) Next() (int, bool) {
	// We're already out of bounds.
	if b.span >= len(b.spans) {
		return 0, false
	}
try:
	if b.bucket < int(b.spans[b.span].Length-1) { // Try to move within same span.
		b.bucket++
		b.idx++
		return b.idx, true
	} else if b.span < len(b.spans)-1 { // Try to move from one span to the next.
		b.span++
		b.idx += int(b.spans[b.span].Offset + 1)
		b.bucket = 0
		if b.spans[b.span].Length == 0 {
			// Pathological case that should never happen. We can't use this span, let's try again.
			goto try
		}
		return b.idx, true
	}
	// We're out of options.
	return 0, false
}

// An Interjection describes how many new buckets have to be introduced before
// processing the pos'th delta from the original slice.
type Interjection struct {
	pos int
	num int
}

// compareSpans returns the interjections to convert a slice of deltas to a new
// slice representing an expanded set of buckets, or false if incompatible
// (e.g. if buckets were removed).
//
// Example:
//
// Let's say the old buckets look like this:
//
//   span syntax: [offset, length]
//   spans      : [ 0 , 2 ]               [2,1]                   [ 3 , 2 ]                     [3,1]       [1,1]
//   bucket idx : [0]   [1]    2     3    [4]    5     6     7    [8]   [9]    10    11    12   [13]   14   [15]
//   raw values    6     3                 3                       2     4                       5           1
//   deltas        6    -3                 0                      -1     2                       1          -4
//
// But now we introduce a new bucket layout. (Carefully chosen example where we
// have a span appended, one unchanged[*], one prepended, and two merge - in
// that order.)
//
// [*] unchanged in terms of which bucket indices they represent. but to achieve
// that, their offset needs to change if "disrupted" by spans changing ahead of
// them
//
//                                         \/ this one is "unchanged"
//   spans      : [  0  ,  3    ]         [1,1]       [    1    ,   4     ]                     [  3  ,   3    ]
//   bucket idx : [0]   [1]   [2]    3    [4]    5    [6]   [7]   [8]   [9]    10    11    12   [13]  [14]  [15]
//   raw values    6     3     0           3           0     0     2     4                       5     0     1
//   deltas        6    -3    -3           3          -3     0     2     2                       1    -5     1
//   delta mods:                          / \                     / \                                       / \
//
// Note that whenever any new buckets are introduced, the subsequent "old"
// bucket needs to readjust its delta to the new base of 0. Thus, for the caller
// who wants to transform the set of original deltas to a new set of deltas to
// match a new span layout that adds buckets, we simply need to generate a list
// of interjections.
//
// Note: Within compareSpans we don't have to worry about the changes to the
// spans themselves, thanks to the iterators we get to work with the more useful
// bucket indices (which of course directly correspond to the buckets we have to
// adjust).
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
	ai := newBucketIterator(a)
	bi := newBucketIterator(b)

	var interjections []Interjection

	// When inter.num becomes > 0, this becomes a valid interjection that
	// should be yielded when we finish a streak of new buckets.
	var inter Interjection

	av, aOK := ai.Next()
	bv, bOK := bi.Next()
loop:
	for {
		switch {
		case aOK && bOK:
			switch {
			case av == bv: // Both have an identical value. move on!
				// Finish WIP interjection and reset.
				if inter.num > 0 {
					interjections = append(interjections, inter)
				}
				inter.num = 0
				av, aOK = ai.Next()
				bv, bOK = bi.Next()
				inter.pos++
			case av < bv: // b misses a value that is in a.
				return interjections, false
			case av > bv: // a misses a value that is in b. Forward b and recompare.
				inter.num++
				bv, bOK = bi.Next()
			}
		case aOK && !bOK: // b misses a value that is in a.
			return interjections, false
		case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
			inter.num++
			bv, bOK = bi.Next()
		default: // Both iterators ran out. We're done.
			if inter.num > 0 {
				interjections = append(interjections, inter)
			}
			break loop
		}
	}

	return interjections, true
}

// interject merges 'in' with the provided interjections and writes them into
// 'out', which must already have the appropriate length.
func interject(in, out []int64, interjections []Interjection) []int64 {
	var (
		j      int   // Position in out.
		v      int64 // The last value seen.
		interj int   // The next interjection to process.
	)
	for i, d := range in {
		if interj < len(interjections) && i == interjections[interj].pos {

			// We have an interjection!
			// Add interjection.num new delta values such that their
			// bucket values equate 0.
			out[j] = int64(-v)
			j++
			for x := 1; x < interjections[interj].num; x++ {
				out[j] = 0
				j++
			}
			interj++

			// Now save the value from the input. The delta value we
			// should save is the original delta value + the last
			// value of the point before the interjection (to undo
			// the delta that was introduced by the interjection).
			out[j] = d + v
			j++
			v = d + v
			continue
		}

		// If there was no interjection, the original delta is still
		// valid.
		out[j] = d
		j++
		v += d
	}
	switch interj {
	case len(interjections):
		// All interjections processed. Nothing more to do.
	case len(interjections) - 1:
		// One more interjection to process at the end.
		out[j] = int64(-v)
		j++
		for x := 1; x < interjections[interj].num; x++ {
			out[j] = 0
			j++
		}
	default:
		panic("unprocessed interjections left")
	}
	return out
}
<|endoftext|>


<commit_before>package perl

/*
#cgo CFLAGS: -D_THREAD_SAFE -pthread -I../vendor/perl-5.20.1
#cgo LDFLAGS: -fstack-protector -L/usr/local/lib -L$GOPATH/github.com/ian-kent/purl/vendor/perl-5.20.1 -lperl -ldl -lm -lutil -lc -fno-common -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include -I/usr/local/lib/perl5/5.20.1/darwin-2level/CORE
#include "c/purl.h"
*/
import "C"
import (
	"sync"
	"unsafe"
)

var perlMutex sync.Mutex
var xsMap = make(map[string]*func(...string) string)

type Purl struct {
	init    bool
	destroy bool
}

func (p *Purl) Init() {
	perlMutex.Lock()
	if !p.init {
		C.PurlInit()
		p.init = true
		p.destroy = false
	}
	perlMutex.Unlock()
}

func (p *Purl) Destroy() {
	perlMutex.Lock()
	if !p.destroy {
		C.PurlDestroy()
		p.destroy = true
		p.init = false
	}
	perlMutex.Unlock()
}

func (p *Purl) RegisterXS(name string, f func(...string) string) {
	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	xsMap[name] = &f
	p.Eval(`
package main {
	*{"` + name + `"} = sub { Purl::XS->Invoke("` + name + `") };
}
`)
}

func (p *Purl) Eval(src string) string {
	csrc := C.CString(src)
	defer C.free(unsafe.Pointer(csrc))

	cres := C.EvalPerl(csrc)
	if cres == nil {
		return ""
	}

	// Not sure if this is required? causes memory issues
	//defer C.free(unsafe.Pointer(cres))
	return C.GoString(cres)
}

func newString(s string) *C.SV {
	cs := C.CString(s)
	defer C.free(unsafe.Pointer(cs))
	str := C.Perl_newSVpvn(cs, C.STRLEN(len(s)))
	return str
}
<commit_msg>Documentation<commit_after>package perl

/*
#cgo CFLAGS: -D_THREAD_SAFE -pthread -I../vendor/perl-5.20.1
#cgo LDFLAGS: -fstack-protector -L/usr/local/lib -L$GOPATH/github.com/ian-kent/purl/vendor/perl-5.20.1 -lperl -ldl -lm -lutil -lc -fno-common -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include -I/usr/local/lib/perl5/5.20.1/darwin-2level/CORE
#include "c/purl.h"
*/
import "C"
import (
	"sync"
	"unsafe"
)

var perlMutex sync.Mutex
var xsMap = make(map[string]*func(...string) string)

// Purl is a Perl interpreter instance.
//
// You must call Init before calling any other functions.
// You must call Destroy when finished with the instance.
type Purl struct {
	init    bool
	destroy bool
}

// Init initialises the Perl interpreter instance
func (p *Purl) Init() {
	perlMutex.Lock()
	if !p.init {
		C.PurlInit()
		p.init = true
		p.destroy = false
	}
	perlMutex.Unlock()
}

// Destroy destroys the Perl interpreter instance
func (p *Purl) Destroy() {
	perlMutex.Lock()
	if !p.destroy {
		C.PurlDestroy()
		p.destroy = true
		p.init = false
	}
	perlMutex.Unlock()
}

// RegisterXS makes a Go function available in Perl
// via an XS callback.
//
// It currently only supports a variadic string input
// and a single scalar string output.
func (p *Purl) RegisterXS(name string, f func(...string) string) {
	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	xsMap[name] = &f
	p.Eval(`
package main {
	*{"` + name + `"} = sub { Purl::XS->Invoke("` + name + `") };
}
`)
}

// Eval evaluates Perl code.
//
// Interpreter state is persisted until Destroy is called.
func (p *Purl) Eval(src string) string {
	csrc := C.CString(src)
	defer C.free(unsafe.Pointer(csrc))

	cres := C.EvalPerl(csrc)
	if cres == nil {
		return ""
	}

	// Not sure if this is required? causes memory issues
	//defer C.free(unsafe.Pointer(cres))
	return C.GoString(cres)
}

func newString(s string) *C.SV {
	cs := C.CString(s)
	defer C.free(unsafe.Pointer(cs))
	str := C.Perl_newSVpvn(cs, C.STRLEN(len(s)))
	return str
}
<|endoftext|>