<commit_before>package zog

import (
	"fmt"
	"strings"
	"testing"
)

func TestAssembleAll(t *testing.T) {
	testUtilRunAll(t, func(t *testing.T, byteForm []byte, stringForm string) {
		testAssembleOne(t, stringForm)
	})
}

func TestAssembleRich(t *testing.T) {
	testCases := []struct {
		prog        string
		byteFormStr string
	}{
		{`ld a, (foo) : foo: defb abh`, "3a 03 00 ab"},
		{`defw 1234h`, "3412"},
		{`org 0100h : start: jp start`, "c3 00 01"},
		{`defb 10h`, "10"},
		{`defs 03h`, "00 00 00"},
		{"LD HL, 0x1000", "21 00 10"},
		{"LD HL, 0x1000 : LD A, B : PUSH HL", "21 00 10 78 e5"},
		{"LD HL, 0x1000 ; LD A, B : PUSH HL", "21 00 10"},
		{"  LD HL, 0x1000 ; LD A, B : PUSH HL", "21 00 10"},
	}
	for _, tc := range testCases {
		fmt.Printf("Assemble: %s\n", tc.prog)
		assembly, err := Assemble(tc.prog)
		if err != nil {
			t.Fatalf("Failed to assemble [%s]: %s", tc.prog, err)
		}
		buf := Encode(assembly.Instructions())
		byteFormStr := strings.ToLower(tc.byteFormStr)
		byteFormStr = strings.Replace(byteFormStr, " ", "", -1)

		hexBufStr := strings.ToLower(bufToHex(buf))

		if hexBufStr != byteFormStr {
			t.Fatalf("Encoded instructions doesn't match got [%s] expected [%s]", hexBufStr, byteFormStr)
		} else {
			fmt.Printf("Matched OK\n")
		}
	}
}

func TestAssembleBasic(t *testing.T) {
	testCases := []string{
		"RLC (IX+1), B",
		"RLC (IX+1)",

		"SRL (IX+1), B",
		"SRL (IX+1)",

		"SET 7, (IX+1), B",
		"SET 7, (IX+1)",

		"RES 7, (IX+1), B",
		"RES 7, (IX+1)",

		"BIT 7, (IX+1)",

		"LD A, (IX+10)",
		"LD A, (IX-10)",

		// TODO: test hex parses
		//		"LD A, (IX+0x0a)",
		//		"LD A, (IX-0x0a)",
		//		"LD A, (IX+0ah)",
		//		"LD A, (IX-0ah)",

		"OUT (0xff), A",
		"IN A, (0xff)",
		"OUT (c), A",
		"IN A, (c)",

		"EX (SP), HL",

		"LD (0x1234), A",

		"inc iy",
		"inc iyh",

		"add iy, bc",

		"INC B",
		"DEC B",

		"LD A, B",
		"LD A, 0x10",
		"LD A, 0x10",

		"INC DE",
		"ADD DE, HL",
		"EX AF,AF'",
		"RET C",
		"CALL DE",

		"RET C",
		"RST 8",
		"RST 16",
		"DJNZ -10",
		"CALL Z, DE",

		"RL A",
		"SET 4, A",
		"SLA F",

		"LD DE, 0x1234",
		"LD DE, (0x1234)",
		"LD (0x1234), HL",

		"LD (0x1234), H",

		"LD A, (HL)",
		"LD (HL), A",
	}

	for _, s := range testCases {
		testAssembleOne(t, s)
	}
}

func testAssembleOne(t *testing.T, s string) {
	assembly, err := Assemble(s)
	if err != nil {
		t.Fatalf("Failed to assemble [%s]: %s", s, err)
	}

	assembledStr := ""
	for _, linst := range assembly.Linsts {
		assembledStr += linst.Inst.String() + "\n"
	}
	if !compareAssembly(assembledStr, s) {
		t.Fatalf("Assembled str not equal [%s] != [%s]", assembledStr, s)
	}
}
<commit_msg>remove bogus tests from assemble_test (they used to pass?)<commit_after>package zog

import (
	"fmt"
	"strings"
	"testing"
)

func TestAssembleAll(t *testing.T) {
	testUtilRunAll(t, func(t *testing.T, byteForm []byte, stringForm string) {
		testAssembleOne(t, stringForm)
	})
}

func TestAssembleRich(t *testing.T) {
	testCases := []struct {
		prog        string
		byteFormStr string
	}{
		{`ld a, (foo) : foo: defb abh`, "3a 03 00 ab"},
		{`defw 1234h`, "3412"},
		{`org 0100h : start: jp start`, "c3 00 01"},
		{`defb 10h`, "10"},
		{`defs 03h`, "00 00 00"},
		{"LD HL, 0x1000", "21 00 10"},
		{"LD HL, 0x1000 : LD A, B : PUSH HL", "21 00 10 78 e5"},
		{"LD HL, 0x1000 ; LD A, B : PUSH HL", "21 00 10"},
		{"  LD HL, 0x1000 ; LD A, B : PUSH HL", "21 00 10"},
	}
	for _, tc := range testCases {
		fmt.Printf("Assemble: %s\n", tc.prog)
		assembly, err := Assemble(tc.prog)
		if err != nil {
			t.Fatalf("Failed to assemble [%s]: %s", tc.prog, err)
		}
		buf := Encode(assembly.Instructions())
		byteFormStr := strings.ToLower(tc.byteFormStr)
		byteFormStr = strings.Replace(byteFormStr, " ", "", -1)

		hexBufStr := strings.ToLower(bufToHex(buf))

		if hexBufStr != byteFormStr {
			t.Fatalf("Encoded instructions doesn't match got [%s] expected [%s]", hexBufStr, byteFormStr)
		} else {
			fmt.Printf("Matched OK\n")
		}
	}
}

func TestAssembleBasic(t *testing.T) {
	testCases := []string{
		"RLC (IX+1), B",
		"RLC (IX+1)",

		"SRL (IX+1), B",
		"SRL (IX+1)",

		"SET 7, (IX+1), B",
		"SET 7, (IX+1)",

		"RES 7, (IX+1), B",
		"RES 7, (IX+1)",

		"BIT 7, (IX+1)",

		"LD A, (IX+10)",
		"LD A, (IX-10)",

		// TODO: test hex parses
		//		"LD A, (IX+0x0a)",
		//		"LD A, (IX-0x0a)",
		//		"LD A, (IX+0ah)",
		//		"LD A, (IX-0ah)",

		"OUT (0xff), A",
		"IN A, (0xff)",
		"OUT (c), A",
		"IN A, (c)",

		"EX (SP), HL",

		"LD (0x1234), A",

		"inc iy",
		"inc iyh",

		"add iy, bc",

		"INC B",
		"DEC B",

		"LD A, B",
		"LD A, 0x10",
		"LD A, 0x10",

		"INC DE",
		//		"ADD DE, HL",
		"EX AF,AF'",
		"RET C",
		"CALL DE",

		"RET C",
		"RST 8",
		"RST 16",
		"DJNZ -10",
		"CALL Z, DE",

		"RL A",
		"SET 4, A",
		//		"SLA F",
		"SLA (HL)",

		"LD DE, 0x1234",
		"LD DE, (0x1234)",
		"LD (0x1234), HL",

		"LD (0x1234), H",

		"LD A, (HL)",
		"LD (HL), A",
	}

	for _, s := range testCases {
		testAssembleOne(t, s)
	}
}

func testAssembleOne(t *testing.T, s string) {
	assembly, err := Assemble(s)
	if err != nil {
		t.Fatalf("Failed to assemble [%s]: %s", s, err)
	}

	assembledStr := ""
	for _, linst := range assembly.Linsts {
		assembledStr += linst.Inst.String() + "\n"
	}
	if !compareAssembly(assembledStr, s) {
		t.Fatalf("Assembled str not equal [%s] != [%s]", assembledStr, s)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"math"
	"sort"
	"strings"
	"time"
)

var percentiles = []float64{0.5, 0.75, 0.95, 0.99}

var last = time.Now()

func log(content string) {
	now := time.Now()
	logWithTimestamp := func(t time.Time, c string) {
		fmt.Printf("%02d:%02d:%02d:%02d\t%s\n", t.Day(), t.Hour(), t.Minute(), t.Second(), content)
	}
	logWithTimestamp(last, content)
	logWithTimestamp(now, content)
	last = now
}

// LogTitle prints an empty line and the title of the benchmark
func LogTitle(title string) {
	fmt.Println()
	fmt.Println(title)
}

// LogEVar prints all the environemnt variables
func LogEVar(vars map[string]interface{}) {
	for k, v := range vars {
		fmt.Printf("%s=%v ", k, v)
	}
	fmt.Println()
}

// LogLabels prints the labels of the result table
func LogLabels(labels ...string) {
	content := "time\t"
	for _, percentile := range percentiles {
		content += fmt.Sprintf("%%%02d\t", int(percentile*100))
	}
	content += strings.Join(labels, "\t")
	fmt.Println(content)
}

// LogResult prints the item of the result table
func LogResult(latencies []int, variables ...string) {
	sort.Ints(latencies)
	results := []float64{}
	for _, percentile := range percentiles {
		n := int(math.Ceil((1 - percentile) * float64(len(latencies))))
		result := float64(latencies[len(latencies)-n]) / 1000000
		results = append(results, result)
	}
	var str string
	for _, result := range results {
		str += fmt.Sprintf("%.2f\t", result)
	}
	log(str + strings.Join(variables, "\t"))
}

// Itoas converts int numbers to a slice of string
func Itoas(nums ...int) []string {
	r := []string{}
	for _, n := range nums {
		r = append(r, fmt.Sprintf("%d", n))
	}
	return r
}

// Ftoas converts float64 numbers to a slice of string
func Ftoas(nums ...float64) []string {
	r := []string{}
	for _, n := range nums {
		r = append(r, fmt.Sprintf("%0.4f", n))
	}
	return r
}
<commit_msg>fix misspell "environment" in helpers.go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"math"
	"sort"
	"strings"
	"time"
)

var percentiles = []float64{0.5, 0.75, 0.95, 0.99}

var last = time.Now()

func log(content string) {
	now := time.Now()
	logWithTimestamp := func(t time.Time, c string) {
		fmt.Printf("%02d:%02d:%02d:%02d\t%s\n", t.Day(), t.Hour(), t.Minute(), t.Second(), content)
	}
	logWithTimestamp(last, content)
	logWithTimestamp(now, content)
	last = now
}

// LogTitle prints an empty line and the title of the benchmark
func LogTitle(title string) {
	fmt.Println()
	fmt.Println(title)
}

// LogEVar prints all the environment variables
func LogEVar(vars map[string]interface{}) {
	for k, v := range vars {
		fmt.Printf("%s=%v ", k, v)
	}
	fmt.Println()
}

// LogLabels prints the labels of the result table
func LogLabels(labels ...string) {
	content := "time\t"
	for _, percentile := range percentiles {
		content += fmt.Sprintf("%%%02d\t", int(percentile*100))
	}
	content += strings.Join(labels, "\t")
	fmt.Println(content)
}

// LogResult prints the item of the result table
func LogResult(latencies []int, variables ...string) {
	sort.Ints(latencies)
	results := []float64{}
	for _, percentile := range percentiles {
		n := int(math.Ceil((1 - percentile) * float64(len(latencies))))
		result := float64(latencies[len(latencies)-n]) / 1000000
		results = append(results, result)
	}
	var str string
	for _, result := range results {
		str += fmt.Sprintf("%.2f\t", result)
	}
	log(str + strings.Join(variables, "\t"))
}

// Itoas converts int numbers to a slice of string
func Itoas(nums ...int) []string {
	r := []string{}
	for _, n := range nums {
		r = append(r, fmt.Sprintf("%d", n))
	}
	return r
}

// Ftoas converts float64 numbers to a slice of string
func Ftoas(nums ...float64) []string {
	r := []string{}
	for _, n := range nums {
		r = append(r, fmt.Sprintf("%0.4f", n))
	}
	return r
}
<|endoftext|>


<commit_before>package voxter

import (
	"fmt"
	"time"
	"strings"

	"github.com/gosimple/slug"
	. "github.com/intelsdi-x/snap-plugin-utilities/logger"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
)

const (
	// Name of plugin
	Name = "voxter"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
	// stat url
	statsURL = "https://vortex2.voxter.com/api/"
)

var (
	statusMap = map[string]int{"up": 0, "down": 1}
)

func init() {
	slug.CustomSub = map[string]string{".": "_"}
}

// make sure that we actually satisify required interface
var _ plugin.CollectorPlugin = (*Voxter)(nil)

type Voxter struct {
}

// CollectMetrics collects metrics for testing
func (v *Voxter) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error
	metrics := make([]plugin.MetricType, 0)
	conf := mts[0].Config().Table()
	apiKey, ok := conf["voxter_key"]
	if !ok || apiKey.(ctypes.ConfigValueStr).Value == "" {
		LogError("voxter_key missing from config.")
		return nil, fmt.Errorf("voxter_key missing from config, %v", conf)
	}
	client, err := NewClient(statsURL, apiKey.(ctypes.ConfigValueStr).Value, false)
	if err != nil {
		LogError("failed to create voxter api client.", "error", err)
		return nil, err
	}
	LogDebug("request to collect metrics", "metric_count", len(mts))

	resp, err := v.EndpointMetrics(client, mts)
	if err != nil {
		LogError("failed to collect metrics.", "error", err)
		return nil, err
	}
	metrics = resp

	LogDebug("collecting metrics completed", "metric_count", len(metrics))
	return metrics, nil
}

//GetMetricTypes returns metric types for testing
func (v *Voxter) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := []plugin.MetricType{}

	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "*", "endpoints", "*", "registrations"),
		Config_: cfg.ConfigDataNode,
	})
	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "*", "endpoints", "*", "channels", "inbound"),
		Config_: cfg.ConfigDataNode,
	})
	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "*", "endpoints", "*", "channels", "outbound"),
		Config_: cfg.ConfigDataNode,
	})

	return mts, nil
}

func (v *Voxter) EndpointMetrics(client *Client, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var metrics []plugin.MetricType
	cSlug := slug.Make("piston")
	endpoints, err := client.EndpointStats()
	if err != nil {
		return nil, err
	}
	metrics = make([]plugin.MetricType, len(endpoints) * 3)
	for _, e := range endpoints {
		marr := strings.Split(e.Name, ".")
		for i, v := range marr {
			marr[i] = slug.Make(v)
		}
		for i, j := 0, len(marr) - 1; i < j; i, j = i+1, j-1 {
			marr[i], marr[j] = marr[j], marr[i]
		}
		mSlug := strings.Join(marr, "/")
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Registrations,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", cSlug, "endpoints", mSlug, "registrations"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Channels.Inbound,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", mSlug, "channels", "inbound"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Channels.Outbound,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", mSlug, "channels", "outbound"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
	}

	return metrics, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (v *Voxter) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("voxter_key", true)
	p := cpolicy.NewPolicyNode()
	p.Add(rule)

	c.Add([]string{"raintank", "apps", "voxter"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<commit_msg>reorder metric definition<commit_after>package voxter

import (
	"fmt"
	"time"
	"strings"

	"github.com/gosimple/slug"
	. "github.com/intelsdi-x/snap-plugin-utilities/logger"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
)

const (
	// Name of plugin
	Name = "voxter"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
	// stat url
	statsURL = "https://vortex2.voxter.com/api/"
)

var (
	statusMap = map[string]int{"up": 0, "down": 1}
)

func init() {
	slug.CustomSub = map[string]string{".": "_"}
}

// make sure that we actually satisify required interface
var _ plugin.CollectorPlugin = (*Voxter)(nil)

type Voxter struct {
}

// CollectMetrics collects metrics for testing
func (v *Voxter) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error
	metrics := make([]plugin.MetricType, 0)
	conf := mts[0].Config().Table()
	apiKey, ok := conf["voxter_key"]
	if !ok || apiKey.(ctypes.ConfigValueStr).Value == "" {
		LogError("voxter_key missing from config.")
		return nil, fmt.Errorf("voxter_key missing from config, %v", conf)
	}
	client, err := NewClient(statsURL, apiKey.(ctypes.ConfigValueStr).Value, false)
	if err != nil {
		LogError("failed to create voxter api client.", "error", err)
		return nil, err
	}
	LogDebug("request to collect metrics", "metric_count", len(mts))

	resp, err := v.EndpointMetrics(client, mts)
	if err != nil {
		LogError("failed to collect metrics.", "error", err)
		return nil, err
	}
	metrics = resp

	LogDebug("collecting metrics completed", "metric_count", len(metrics))
	return metrics, nil
}

//GetMetricTypes returns metric types for testing
func (v *Voxter) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := []plugin.MetricType{}

	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", "*", "*", "registrations"),
		Config_: cfg.ConfigDataNode,
	})
	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", "*", "*", "channels", "inbound"),
		Config_: cfg.ConfigDataNode,
	})
	mts = append(mts, plugin.MetricType{
		Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", "*", "*", "channels", "outbound"),
		Config_: cfg.ConfigDataNode,
	})

	return mts, nil
}

func (v *Voxter) EndpointMetrics(client *Client, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var metrics []plugin.MetricType
	cSlug := slug.Make("piston")
	endpoints, err := client.EndpointStats()
	if err != nil {
		return nil, err
	}
	metrics = make([]plugin.MetricType, len(endpoints) * 3)
	for _, e := range endpoints {
		marr := strings.Split(e.Name, ".")
		for i, v := range marr {
			marr[i] = slug.Make(v)
		}
		for i, j := 0, len(marr) - 1; i < j; i, j = i+1, j-1 {
			marr[i], marr[j] = marr[j], marr[i]
		}
		mSlug := strings.Join(marr, "_")
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Registrations,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", cSlug, mSlug, "registrations"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Channels.Inbound,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", cSlug, mSlug, "channels", "inbound"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
		metrics = append(metrics, plugin.MetricType{
			Data_: e.Channels.Outbound,
			Namespace_: core.NewNamespace("raintank", "apps", "voxter", "endpoints", cSlug, mSlug, "channels", "outbound"),
			Timestamp_: time.Now(),
			Version_: mts[0].Version(),
		})
	}

	return metrics, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (v *Voxter) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("voxter_key", true)
	p := cpolicy.NewPolicyNode()
	p.Add(rule)

	c.Add([]string{"raintank", "apps", "voxter"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<|endoftext|>


<commit_before>// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package redis

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

func cannotConvert(d reflect.Value, s interface{}) error {
	return fmt.Errorf("redigo: Scan cannot convert from %s to %s",
		reflect.TypeOf(s), d.Type())
}

func convertAssignBytes(d reflect.Value, s []byte) (err error) {
	switch d.Type().Kind() {
	case reflect.Float32, reflect.Float64:
		var x float64
		x, err = strconv.ParseFloat(string(s), d.Type().Bits())
		d.SetFloat(x)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		var x int64
		x, err = strconv.ParseInt(string(s), 10, d.Type().Bits())
		d.SetInt(x)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		var x uint64
		x, err = strconv.ParseUint(string(s), 10, d.Type().Bits())
		d.SetUint(x)
	case reflect.Bool:
		var x bool
		x, err = strconv.ParseBool(string(s))
		d.SetBool(x)
	case reflect.String:
		d.SetString(string(s))
	case reflect.Slice:
		if d.Type().Elem().Kind() != reflect.Uint8 {
			err = cannotConvert(d, s)
		} else {
			d.SetBytes(s)
		}
	default:
		err = cannotConvert(d, s)
	}
	return
}

func convertAssignInt(d reflect.Value, s int64) (err error) {
	switch d.Type().Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		d.SetInt(s)
		if d.Int() != s {
			err = strconv.ErrRange
			d.SetInt(0)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		if s < 0 {
			err = strconv.ErrRange
		} else {
			x := uint64(s)
			d.SetUint(x)
			if d.Uint() != x {
				err = strconv.ErrRange
				d.SetUint(0)
			}
		}
	case reflect.Bool:
		d.SetBool(s != 0)
	default:
		err = cannotConvert(d, s)
	}
	return
}

func convertAssignValues(d reflect.Value, s []interface{}) (err error) {
	if d.Type().Kind() != reflect.Slice {
		return cannotConvert(d, s)
	}
	if len(s) > d.Cap() {
		d.Set(reflect.MakeSlice(d.Type(), len(s), len(s)))
	} else {
		d.SetLen(len(s))
	}
	for i := 0; i < len(s); i++ {
		switch s := s[i].(type) {
		case []byte:
			err = convertAssignBytes(d.Index(i), s)
		case int64:
			err = convertAssignInt(d.Index(i), s)
		default:
			err = cannotConvert(d, s)
		}
		if err != nil {
			break
		}
	}
	return
}

func convertAssign(d interface{}, s interface{}) (err error) {
	// Handle the most common destination types using type switches and
	// fall back to reflection for all other types.
	switch s := s.(type) {
	case nil:
		// ingore
	case []byte:
		switch d := d.(type) {
		case *string:
			*d = string(s)
		case *int:
			*d, err = strconv.Atoi(string(s))
		case *bool:
			*d, err = strconv.ParseBool(string(s))
		case *[]byte:
			*d = s
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignBytes(d.Elem(), s)
			}
		}
	case int64:
		switch d := d.(type) {
		case *int:
			x := int(s)
			if int64(x) != s {
				err = strconv.ErrRange
				x = 0
			}
			*d = x
		case *bool:
			*d = s != 0
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignInt(d.Elem(), s)
			}
		}
	case []interface{}:
		switch d := d.(type) {
		case *[]interface{}:
			*d = s
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignValues(d.Elem(), s)
			}
		}
	case Error:
		err = s
	default:
		err = cannotConvert(reflect.ValueOf(d), s)
	}
	return
}

// Scan copies from the multi-bulk src to the values pointed at by dest.
//
// The values pointed at by test must be a numeric type, boolean, string,
// []byte, interface{} or a slice of these types. Scan uses the standard
// strconv package to convert bulk values to numeric and boolean types.
//
// If a dest value is nil, then the corresponding src value is skipped.
//
// If the multi-bulk value is nil, then the corresponding dest value is not
// modified.
//
// To enable easy use of Scan in a loop, Scan returns the slice of src
// following the copied values.
func Scan(src []interface{}, dest ...interface{}) ([]interface{}, error) {
	if len(src) < len(dest) {
		return nil, errors.New("redigo: Scan multibulk short")
	}
	var err error
	for i, d := range dest {
		err = convertAssign(d, src[i])
		if err != nil {
			break
		}
	}
	return src[len(dest):], err
}

type fieldSpec struct {
	name  string
	index []int
	//omitEmpty bool
}

type structSpec struct {
	m map[string]*fieldSpec
	l []*fieldSpec
}

func (ss *structSpec) fieldSpec(name []byte) *fieldSpec {
	return ss.m[string(name)]
}

func compileStructSpec(t reflect.Type, depth map[string]int, index []int, ss *structSpec) {
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		switch {
		case f.PkgPath != "":
			// Ignore unexported fields.
		case f.Anonymous:
			// TODO: Handle pointers. Requires change to decoder and
			// protection against infinite recursion.
			if f.Type.Kind() == reflect.Struct {
				compileStructSpec(f.Type, depth, append(index, i), ss)
			}
		default:
			fs := &fieldSpec{name: f.Name}
			tag := f.Tag.Get("redis")
			p := strings.Split(tag, ",")
			if len(p) > 0 && p[0] != "-" {
				if len(p[0]) > 0 {
					fs.name = p[0]
				}
				for _, s := range p[1:] {
					switch s {
					//case "omitempty":
					//  fs.omitempty = true
					default:
						panic(errors.New("redigo: unknown field flag " + s + " for type " + t.Name()))
					}
				}
			}
			d, found := depth[fs.name]
			if !found {
				d = 1 << 30
			}
			switch {
			case len(index) == d:
				// At same depth, remove from result.
				delete(ss.m, fs.name)
				j := 0
				for i := 0; i < len(ss.l); i++ {
					if fs.name != ss.l[i].name {
						ss.l[j] = ss.l[i]
						j += 1
					}
				}
				ss.l = ss.l[:j]
			case len(index) < d:
				fs.index = make([]int, len(index)+1)
				copy(fs.index, index)
				fs.index[len(index)] = i
				depth[fs.name] = len(index)
				ss.m[fs.name] = fs
				ss.l = append(ss.l, fs)
			}
		}
	}
}

var (
	structSpecMutex  sync.RWMutex
	structSpecCache  = make(map[reflect.Type]*structSpec)
	defaultFieldSpec = &fieldSpec{}
)

func structSpecForType(t reflect.Type) *structSpec {

	structSpecMutex.RLock()
	ss, found := structSpecCache[t]
	structSpecMutex.RUnlock()
	if found {
		return ss
	}

	structSpecMutex.Lock()
	defer structSpecMutex.Unlock()
	ss, found = structSpecCache[t]
	if found {
		return ss
	}

	ss = &structSpec{m: make(map[string]*fieldSpec)}
	compileStructSpec(t, make(map[string]int), nil, ss)
	structSpecCache[t] = ss
	return ss
}

// ScanStruct scans a multi-bulk src containing alternating names and values to
// a struct. The HGETALL and CONFIG GET commands return replies in this format.
//
// ScanStruct uses the struct field name to match values in the response. Use
// 'redis' field tag to override the name:
//
//      Field int `redis:"myName"`
//
// Fields with the tag redis:"-" are ignored.
func ScanStruct(src []interface{}, dest interface{}) error {
	d := reflect.ValueOf(dest)
	if d.Kind() != reflect.Ptr || d.IsNil() {
		return errors.New("redigo: ScanStruct value must be non-nil pointer")
	}
	d = d.Elem()
	ss := structSpecForType(d.Type())

	if len(src)%2 != 0 {
		return errors.New("redigo: ScanStruct expects even number of values in values")
	}

	for i := 0; i < len(src); i += 2 {
		name, ok := src[i].([]byte)
		if !ok {
			return errors.New("redigo: ScanStruct key not a bulk value")
		}
		fs := ss.fieldSpec(name)
		if fs == nil {
			continue
		}
		f := d.FieldByIndex(fs.index)
		var err error
		switch s := src[i+1].(type) {
		case nil:
			// ignore
		case []byte:
			err = convertAssignBytes(f, s)
		case int64:
			err = convertAssignInt(f, s)
		default:
			err = cannotConvert(f, s)
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Fix typo in comment.<commit_after>// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package redis

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

func cannotConvert(d reflect.Value, s interface{}) error {
	return fmt.Errorf("redigo: Scan cannot convert from %s to %s",
		reflect.TypeOf(s), d.Type())
}

func convertAssignBytes(d reflect.Value, s []byte) (err error) {
	switch d.Type().Kind() {
	case reflect.Float32, reflect.Float64:
		var x float64
		x, err = strconv.ParseFloat(string(s), d.Type().Bits())
		d.SetFloat(x)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		var x int64
		x, err = strconv.ParseInt(string(s), 10, d.Type().Bits())
		d.SetInt(x)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		var x uint64
		x, err = strconv.ParseUint(string(s), 10, d.Type().Bits())
		d.SetUint(x)
	case reflect.Bool:
		var x bool
		x, err = strconv.ParseBool(string(s))
		d.SetBool(x)
	case reflect.String:
		d.SetString(string(s))
	case reflect.Slice:
		if d.Type().Elem().Kind() != reflect.Uint8 {
			err = cannotConvert(d, s)
		} else {
			d.SetBytes(s)
		}
	default:
		err = cannotConvert(d, s)
	}
	return
}

func convertAssignInt(d reflect.Value, s int64) (err error) {
	switch d.Type().Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		d.SetInt(s)
		if d.Int() != s {
			err = strconv.ErrRange
			d.SetInt(0)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		if s < 0 {
			err = strconv.ErrRange
		} else {
			x := uint64(s)
			d.SetUint(x)
			if d.Uint() != x {
				err = strconv.ErrRange
				d.SetUint(0)
			}
		}
	case reflect.Bool:
		d.SetBool(s != 0)
	default:
		err = cannotConvert(d, s)
	}
	return
}

func convertAssignValues(d reflect.Value, s []interface{}) (err error) {
	if d.Type().Kind() != reflect.Slice {
		return cannotConvert(d, s)
	}
	if len(s) > d.Cap() {
		d.Set(reflect.MakeSlice(d.Type(), len(s), len(s)))
	} else {
		d.SetLen(len(s))
	}
	for i := 0; i < len(s); i++ {
		switch s := s[i].(type) {
		case []byte:
			err = convertAssignBytes(d.Index(i), s)
		case int64:
			err = convertAssignInt(d.Index(i), s)
		default:
			err = cannotConvert(d, s)
		}
		if err != nil {
			break
		}
	}
	return
}

func convertAssign(d interface{}, s interface{}) (err error) {
	// Handle the most common destination types using type switches and
	// fall back to reflection for all other types.
	switch s := s.(type) {
	case nil:
		// ingore
	case []byte:
		switch d := d.(type) {
		case *string:
			*d = string(s)
		case *int:
			*d, err = strconv.Atoi(string(s))
		case *bool:
			*d, err = strconv.ParseBool(string(s))
		case *[]byte:
			*d = s
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignBytes(d.Elem(), s)
			}
		}
	case int64:
		switch d := d.(type) {
		case *int:
			x := int(s)
			if int64(x) != s {
				err = strconv.ErrRange
				x = 0
			}
			*d = x
		case *bool:
			*d = s != 0
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignInt(d.Elem(), s)
			}
		}
	case []interface{}:
		switch d := d.(type) {
		case *[]interface{}:
			*d = s
		case *interface{}:
			*d = s
		case nil:
			// skip value
		default:
			if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr {
				err = cannotConvert(d, s)
			} else {
				err = convertAssignValues(d.Elem(), s)
			}
		}
	case Error:
		err = s
	default:
		err = cannotConvert(reflect.ValueOf(d), s)
	}
	return
}

// Scan copies from the multi-bulk src to the values pointed at by dest.
//
// The values pointed at by dest must be a numeric type, boolean, string,
// []byte, interface{} or a slice of these types. Scan uses the standard
// strconv package to convert bulk values to numeric and boolean types.
//
// If a dest value is nil, then the corresponding src value is skipped.
//
// If the multi-bulk value is nil, then the corresponding dest value is not
// modified.
//
// To enable easy use of Scan in a loop, Scan returns the slice of src
// following the copied values.
func Scan(src []interface{}, dest ...interface{}) ([]interface{}, error) {
	if len(src) < len(dest) {
		return nil, errors.New("redigo: Scan multibulk short")
	}
	var err error
	for i, d := range dest {
		err = convertAssign(d, src[i])
		if err != nil {
			break
		}
	}
	return src[len(dest):], err
}

type fieldSpec struct {
	name  string
	index []int
	//omitEmpty bool
}

type structSpec struct {
	m map[string]*fieldSpec
	l []*fieldSpec
}

func (ss *structSpec) fieldSpec(name []byte) *fieldSpec {
	return ss.m[string(name)]
}

func compileStructSpec(t reflect.Type, depth map[string]int, index []int, ss *structSpec) {
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		switch {
		case f.PkgPath != "":
			// Ignore unexported fields.
		case f.Anonymous:
			// TODO: Handle pointers. Requires change to decoder and
			// protection against infinite recursion.
			if f.Type.Kind() == reflect.Struct {
				compileStructSpec(f.Type, depth, append(index, i), ss)
			}
		default:
			fs := &fieldSpec{name: f.Name}
			tag := f.Tag.Get("redis")
			p := strings.Split(tag, ",")
			if len(p) > 0 && p[0] != "-" {
				if len(p[0]) > 0 {
					fs.name = p[0]
				}
				for _, s := range p[1:] {
					switch s {
					//case "omitempty":
					//  fs.omitempty = true
					default:
						panic(errors.New("redigo: unknown field flag " + s + " for type " + t.Name()))
					}
				}
			}
			d, found := depth[fs.name]
			if !found {
				d = 1 << 30
			}
			switch {
			case len(index) == d:
				// At same depth, remove from result.
				delete(ss.m, fs.name)
				j := 0
				for i := 0; i < len(ss.l); i++ {
					if fs.name != ss.l[i].name {
						ss.l[j] = ss.l[i]
						j += 1
					}
				}
				ss.l = ss.l[:j]
			case len(index) < d:
				fs.index = make([]int, len(index)+1)
				copy(fs.index, index)
				fs.index[len(index)] = i
				depth[fs.name] = len(index)
				ss.m[fs.name] = fs
				ss.l = append(ss.l, fs)
			}
		}
	}
}

var (
	structSpecMutex  sync.RWMutex
	structSpecCache  = make(map[reflect.Type]*structSpec)
	defaultFieldSpec = &fieldSpec{}
)

func structSpecForType(t reflect.Type) *structSpec {

	structSpecMutex.RLock()
	ss, found := structSpecCache[t]
	structSpecMutex.RUnlock()
	if found {
		return ss
	}

	structSpecMutex.Lock()
	defer structSpecMutex.Unlock()
	ss, found = structSpecCache[t]
	if found {
		return ss
	}

	ss = &structSpec{m: make(map[string]*fieldSpec)}
	compileStructSpec(t, make(map[string]int), nil, ss)
	structSpecCache[t] = ss
	return ss
}

// ScanStruct scans a multi-bulk src containing alternating names and values to
// a struct. The HGETALL and CONFIG GET commands return replies in this format.
//
// ScanStruct uses the struct field name to match values in the response. Use
// 'redis' field tag to override the name:
//
//      Field int `redis:"myName"`
//
// Fields with the tag redis:"-" are ignored.
func ScanStruct(src []interface{}, dest interface{}) error {
	d := reflect.ValueOf(dest)
	if d.Kind() != reflect.Ptr || d.IsNil() {
		return errors.New("redigo: ScanStruct value must be non-nil pointer")
	}
	d = d.Elem()
	ss := structSpecForType(d.Type())

	if len(src)%2 != 0 {
		return errors.New("redigo: ScanStruct expects even number of values in values")
	}

	for i := 0; i < len(src); i += 2 {
		name, ok := src[i].([]byte)
		if !ok {
			return errors.New("redigo: ScanStruct key not a bulk value")
		}
		fs := ss.fieldSpec(name)
		if fs == nil {
			continue
		}
		f := d.FieldByIndex(fs.index)
		var err error
		switch s := src[i+1].(type) {
		case nil:
			// ignore
		case []byte:
			err = convertAssignBytes(f, s)
		case int64:
			err = convertAssignInt(f, s)
		default:
			err = cannotConvert(f, s)
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package presence

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"
)

var nextId chan string
var testTimeoutDuration = time.Second * 1

func init() {
	nextId = make(chan string)
	go func() {
		id := 0
		for {
			id++
			nextId <- "id" + strconv.Itoa(id)
		}
	}()
}

func initPresence() (*Session, error) {
	connStr := os.Getenv("REDIS_URI")
	if connStr == "" {
		connStr = "localhost:6379"
	}

	backend, err := NewRedis(connStr, 10, testTimeoutDuration)
	if err != nil {
		return nil, err
	}

	// adjust config for redis instance
	c := backend.(*Redis).redis.Pool().Get()
	if _, err := c.Do("CONFIG", "SET", "notify-keyspace-events", "Ex$"); err != nil {
		return nil, err
	}

	ses, err := New(backend)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func withConn(f func(s *Session)) error {
	s, err := initPresence()
	if err != nil {
		return err
	}

	f(s)

	return s.Close()
}

func TestOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatalf("non existing id can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(id); err != nil {
			t.Fatalf("existing id can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOnlineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Online(ids...); err != nil {
			t.Fatalf("non existing ids can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(ids...); err != nil {
			t.Fatalf("existing ids can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatalf("non existing id can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(id); err != nil {
			t.Fatalf("existing id can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOfflineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Offline(ids...); err != nil {
			t.Fatalf("non existing ids can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(ids...); err != nil {
			t.Fatalf("existing ids can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]

		if res.Status != Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOffline(t *testing.T) {
	err := withConn(func(s *Session) {

		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		// mark all of them as online first
		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Online {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		// mark all of them as offline first
		if err := s.Offline(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Offline {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiMixed(t *testing.T) {
	err := withConn(func(s *Session) {
		onlineId := <-nextId
		offlineId := <-nextId

		ids := []string{onlineId, offlineId}

		if err := s.Online(onlineId); err != nil {
			t.Fatal(err)
		}

		if err := s.Offline(offlineId); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if status[0].Status != Online {
			t.Fatalf("%s should be %s, but it is %s", status[0].ID, Online, status[0].Status)
		}
		if status[1].Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", status[1].ID, Offline, status[0].Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOrder(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for i := range status {
			if status[i].ID != ids[i] {
				t.Fatalf("%dth status should be %s, but it is %s", i, ids[i], status[i].ID)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusLen(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if len(status) != len(ids) {
			t.Fatalf("Status response len should be: %d, but got: %d", len(ids), len(status))
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusWithTimeout(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration)

		// get the status of the id
		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status == Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestSubscriptions(t *testing.T) {
	err := withConn(func(s *Session) {

		ids := []string{<-nextId, <-nextId, <-nextId}

		// sleep until expiration
		time.Sleep(testTimeoutDuration)

		onlineCount := 0
		offlineCount := 0
		go func() {
			for event := range s.ListenStatusChanges() {
				switch event.Status {
				case Online:
					onlineCount++
				case Offline:
					offlineCount++
				}
			}
		}()

		err := s.Online(ids...)
		if err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration * 2)

		if onlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("online count should be: %d, but it is: %d", len(ids), onlineCount),
			)
		}

		if offlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("offline count should be: %d, but it is: %d", len(ids), offlineCount),
			)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Redis: sleep more than required<commit_after>package presence

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"
)

var nextId chan string
var testTimeoutDuration = time.Second * 1

func init() {
	nextId = make(chan string)
	go func() {
		id := 0
		for {
			id++
			nextId <- "id" + strconv.Itoa(id)
		}
	}()
}

func initPresence() (*Session, error) {
	connStr := os.Getenv("REDIS_URI")
	if connStr == "" {
		connStr = "localhost:6379"
	}

	backend, err := NewRedis(connStr, 10, testTimeoutDuration)
	if err != nil {
		return nil, err
	}

	// adjust config for redis instance
	c := backend.(*Redis).redis.Pool().Get()
	if _, err := c.Do("CONFIG", "SET", "notify-keyspace-events", "Ex$"); err != nil {
		return nil, err
	}

	ses, err := New(backend)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func withConn(f func(s *Session)) error {
	s, err := initPresence()
	if err != nil {
		return err
	}

	f(s)

	return s.Close()
}

func TestOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatalf("non existing id can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(id); err != nil {
			t.Fatalf("existing id can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOnlineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Online(ids...); err != nil {
			t.Fatalf("non existing ids can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(ids...); err != nil {
			t.Fatalf("existing ids can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatalf("non existing id can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(id); err != nil {
			t.Fatalf("existing id can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOfflineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Offline(ids...); err != nil {
			t.Fatalf("non existing ids can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(ids...); err != nil {
			t.Fatalf("existing ids can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]

		if res.Status != Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOffline(t *testing.T) {
	err := withConn(func(s *Session) {

		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		// mark all of them as online first
		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Online {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		// mark all of them as offline first
		if err := s.Offline(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Offline {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiMixed(t *testing.T) {
	err := withConn(func(s *Session) {
		onlineId := <-nextId
		offlineId := <-nextId

		ids := []string{onlineId, offlineId}

		if err := s.Online(onlineId); err != nil {
			t.Fatal(err)
		}

		if err := s.Offline(offlineId); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if status[0].Status != Online {
			t.Fatalf("%s should be %s, but it is %s", status[0].ID, Online, status[0].Status)
		}
		if status[1].Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", status[1].ID, Offline, status[0].Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOrder(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for i := range status {
			if status[i].ID != ids[i] {
				t.Fatalf("%dth status should be %s, but it is %s", i, ids[i], status[i].ID)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusLen(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if len(status) != len(ids) {
			t.Fatalf("Status response len should be: %d, but got: %d", len(ids), len(status))
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusWithTimeout(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration * 2)

		// get the status of the id
		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status == Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestSubscriptions(t *testing.T) {
	err := withConn(func(s *Session) {

		ids := []string{<-nextId, <-nextId, <-nextId}

		// sleep until expiration
		time.Sleep(testTimeoutDuration * 2)

		onlineCount := 0
		offlineCount := 0
		go func() {
			for event := range s.ListenStatusChanges() {
				switch event.Status {
				case Online:
					onlineCount++
				case Offline:
					offlineCount++
				}
			}
		}()

		err := s.Online(ids...)
		if err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration * 2)

		if onlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("online count should be: %d, but it is: %d", len(ids), onlineCount),
			)
		}

		if offlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("offline count should be: %d, but it is: %d", len(ids), offlineCount),
			)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gcf runs a GCF function that triggers in 2 scenarios:
//
// 1) completion of prophet-flume job in borg. On triggering it sets up new
//    cloud BT table, scales up BT cluster (if needed) and starts a dataflow job.
// 2) completion of BT cache ingestion dataflow job. It scales BT cluster down
//    (if needed).
// The trigger function BTImportController requires a set of environment
// variables to be set. They are stored in prod/*.yaml files for prod.
package gcf

import (
	"context"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigtable"
	"cloud.google.com/go/storage"
	"github.com/pkg/errors"
	dataflow "google.golang.org/api/dataflow/v1b3"
)

const (
	dataFilePattern    = "cache.csv*"
	createTableRetries = 3
	columnFamily       = "csv"

	// NOTE: The following three files represents the state of a BT import. They
	// get written under:
	//
	//		<controlPath>/<TableID>/
	//
	// Init: written by borg to start BT import.
	initFile = "init.txt"
	// Launched: written by this cloud function to mark launching of BT import job.
	launchedFile = "launched.txt"
	// Completed: written by dataflow to mark completion of BT import.
	completedFile = "completed.txt"
	// Default region
	region = "us-central1"
)

// GCSEvent is the payload of a GCS event.
type GCSEvent struct {
	Name   string `json:"name"`
	Bucket string `json:"bucket"`
}

// joinURL joins url components.
// path.Join does work well for url, for example gs:// is changaed to gs:/
func joinURL(base string, paths ...string) string {
	p := path.Join(paths...)
	return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/"))
}

// Return the GCS bucket and object from path in the form of gs://<bucket>/<object>
func parsePath(path string) (string, string, error) {
	parts := strings.Split(path, "/")
	if parts[0] != "gs:" || parts[1] != "" || len(parts) < 3 {
		return "", "", errors.Errorf("Unexpected path: %s", path)
	}
	return parts[2], strings.Join(parts[3:], "/"), nil
}

func doesObjectExist(ctx context.Context, path string) (bool, error) {
	bucket, object, err := parsePath(path)
	if err != nil {
		return false, err
	}
	gcsClient, err := storage.NewClient(ctx)
	if err != nil {
		return false, errors.Wrap(err, "Failed to create gcsClient")
	}
	_, err = gcsClient.Bucket(bucket).Object(object).Attrs(ctx)
	if err == storage.ErrObjectNotExist {
		return false, nil
	}
	return true, nil
}

func writeToGCS(ctx context.Context, path, data string) error {
	bucket, object, err := parsePath(path)
	if err != nil {
		return err
	}
	gcsClient, err := storage.NewClient(ctx)
	if err != nil {
		return errors.Wrap(err, "Failed to create gcsClient")
	}
	w := gcsClient.Bucket(bucket).Object(object).NewWriter(ctx)
	defer w.Close()
	_, err = fmt.Fprint(w, data)
	return errors.WithMessagef(err, "Failed to write data to %s/%s", bucket, object)
}

func launchDataflowJob(
	ctx context.Context,
	projectID string,
	instance string,
	tableID string,
	dataPath string,
	controlPath string,
	dataflowTemplate string,
) error {
	dataflowService, err := dataflow.NewService(ctx)
	if err != nil {
		return errors.Wrap(err, "Unable to create dataflow service")
	}
	dataFile := joinURL(dataPath, tableID, dataFilePattern)
	launchedPath := joinURL(controlPath, tableID, launchedFile)
	completedPath := joinURL(controlPath, tableID, completedFile)
	params := &dataflow.LaunchTemplateParameters{
		JobName: tableID,
		Parameters: map[string]string{
			"inputFile":          dataFile,
			"completionFile":     completedPath,
			"bigtableInstanceId": instance,
			"bigtableTableId":    tableID,
			"bigtableProjectId":  projectID,
			"region":             region,
		},
	}
	log.Printf("[%s/%s] Launching dataflow job: %s -> %s\n", instance, tableID, dataFile, launchedPath)
	launchCall := dataflow.NewProjectsTemplatesService(dataflowService).Launch(projectID, params)
	_, err = launchCall.GcsPath(dataflowTemplate).Do()
	if err != nil {
		return errors.WithMessagef(err, "Unable to launch dataflow job (%s, %s): %v\n", dataFile, launchedPath)
	}
	return nil
}

func setupBT(ctx context.Context, btProjectID, btInstance, tableID string) error {
	adminClient, err := bigtable.NewAdminClient(ctx, btProjectID, btInstance)
	if err != nil {
		return errors.Wrap(err, "Unable to create a table admin client")
	}
	// Create table. We retry 3 times in 1 minute intervals.
	dctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
	defer cancel()
	var ok bool
	for ii := 0; ii < createTableRetries; ii++ {
		log.Printf("Creating new bigtable table (%d): %s/%s", ii, btInstance, tableID)
		err = adminClient.CreateTable(dctx, tableID)
		if err != nil {
			log.Printf("Error creating table %s, retry...", err)
		} else {
			ok = true
			break
		}
		time.Sleep(1 * time.Minute)
	}
	if !ok {
		return errors.Errorf("Unable to create table: %s, got error: %v", tableID, err)
	}
	// Create table columnFamily.
	log.Printf("Creating column family %s in table %s/%s", columnFamily, btInstance, tableID)
	if err := adminClient.CreateColumnFamily(dctx, tableID, columnFamily); err != nil {
		return errors.WithMessagef(err, "Unable to create column family: csv for table: %s, got error: %v", tableID)
	}
	return nil
}

func scaleBT(ctx context.Context, projectID, instance, cluster string, numNodes int32) error {
	// Scale up bigtable cluster. This helps speed up the dataflow job.
	// We scale down again once dataflow job completes.
	instanceAdminClient, err := bigtable.NewInstanceAdminClient(ctx, projectID)
	dctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
	defer cancel()
	if err != nil {
		return errors.Wrap(err, "Unable to create a table instance admin client")
	}
	log.Printf("Scaling BT %s cluster %s to %d nodes", instance, cluster, numNodes)
	if err := instanceAdminClient.UpdateCluster(dctx, instance, cluster, numNodes); err != nil {
		return errors.WithMessagef(err, "Unable to resize bigtable cluster %s to %d: %v", cluster, numNodes)
	}
	return nil
}

func getBTNodes(ctx context.Context, projectID, instance, cluster string) (int, error) {
	instanceAdminClient, err := bigtable.NewInstanceAdminClient(ctx, projectID)
	if err != nil {
		return 0, errors.Wrap(err, "Unable to create a table instance admin client")
	}
	clusterInfo, err := instanceAdminClient.GetCluster(ctx, instance, cluster)
	if err != nil {
		return 0, errors.Wrap(err, "Unable to get cluster information")
	}
	return clusterInfo.ServeNodes, nil
}

func btImportControllerInternal(ctx context.Context, e GCSEvent) error {
	projectID := os.Getenv("projectID")
	instance := os.Getenv("instance")
	cluster := os.Getenv("cluster")
	nodesHigh := os.Getenv("nodesHigh")
	nodesLow := os.Getenv("nodesLow")
	dataflowTemplate := os.Getenv("dataflowTemplate")
	dataPath := os.Getenv("dataPath")
	controlPath := os.Getenv("controlPath")
	if projectID == "" {
		return errors.New("projectID is not set in environment")
	}
	if instance == "" {
		return errors.New("instance is not set in environment")
	}
	if cluster == "" {
		return errors.New("cluster is not set in environment")
	}
	if dataflowTemplate == "" {
		return errors.New("dataflowTemplate is not set in environment")
	}
	if dataPath == "" {
		return errors.New("dataPath is not set in environment")
	}
	if controlPath == "" {
		return errors.New("controlPath is not set in environment")
	}
	// Get low and high nodes number
	nodesH, err := strconv.Atoi(nodesHigh)
	if err != nil {
		return errors.Wrap(err, "Unable to parse 'nodesHigh' as an integer")
	}
	nodesL, err := strconv.Atoi(nodesLow)
	if err != nil {
		return errors.Wrap(err, "Unable to parse 'nodesLow' as an integer")
	}
	// Get control bucket and object
	controlBucket, controlFolder, err := parsePath(controlPath)
	if err != nil {
		return err
	}
	if e.Bucket != controlBucket {
		log.Printf("Trigger bucket '%s' != '%s', skip processing", e.Bucket, controlBucket)
		return nil
	}
	// Get table ID.
	parts := strings.Split(e.Name, "/")
	tableID := parts[len(parts)-2]
	triggerFolder := strings.Join(parts[0:len(parts)-2], "/")
	if triggerFolder != controlFolder {
		log.Printf("Control folder '%s' != '%s', skip processing", triggerFolder, controlFolder)
		return nil
	}

	numNodes, err := getBTNodes(ctx, projectID, instance, cluster)
	if err != nil {
		return err
	}

	if strings.HasSuffix(e.Name, initFile) {
		log.Printf("[%s] State Init", e.Name)
		// Called when the state-machine is at Init. Logic below moves it to Launched state.
		launchedPath := joinURL(controlPath, tableID, launchedFile)
		exist, err := doesObjectExist(ctx, launchedPath)
		if err != nil {
			return errors.WithMessagef(err, "Failed to check %s", launchedFile)
		}
		if exist {
			return errors.WithMessagef(err, "Cache was already built for %s", tableID)
		}
		if err := setupBT(ctx, projectID, instance, tableID); err != nil {
			return err
		}
		if numNodes < nodesH {
			if err := scaleBT(ctx, projectID, instance, cluster, int32(nodesH)); err != nil {
				return err
			}
		}
		err = launchDataflowJob(ctx, projectID, instance, tableID, dataPath, controlPath, dataflowTemplate)
		if err != nil {
			return err
		}
		// Save the fact that we've launched the dataflow job.
		err = writeToGCS(ctx, launchedPath, "")
		if err != nil {
			return err
		}
		log.Printf("[%s] State Launched", e.Name)
	} else if strings.HasSuffix(e.Name, completedFile) {
		log.Printf("[%s] State Completed", e.Name)
		// Called when the state-machine moves to Completed state from Launched.
		if numNodes == nodesH {
			// Only scale down BT nodes when the current high node is set up this config.
			// This requires different high nodes in different config.
			if err := scaleBT(ctx, projectID, instance, cluster, int32(nodesL)); err != nil {
				return err
			}
		}
		// TODO: else, notify Mixer to load the BT table.
		log.Printf("[%s] Completed work", e.Name)
	}
	return nil
}

// BTImportController consumes a GCS event and runs an import state machine.
func BTImportController(ctx context.Context, e GCSEvent) error {
	err := btImportControllerInternal(ctx, e)
	if err != nil {
		// Panic gets reported to Cloud Logging Error Reporting that we can then
		// alert on
		// (https://cloud.google.com/functions/docs/monitoring/error-reporting#functions-errors-log-go)
		panic(errors.Wrap(err, "panic"))
	}
	return nil
}
<commit_msg>Check valid trigger file (#147)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gcf runs a GCF function that triggers in 2 scenarios:
//
// 1) completion of prophet-flume job in borg. On triggering it sets up new
//    cloud BT table, scales up BT cluster (if needed) and starts a dataflow job.
// 2) completion of BT cache ingestion dataflow job. It scales BT cluster down
//    (if needed).
// The trigger function BTImportController requires a set of environment
// variables to be set. They are stored in prod/*.yaml files for prod.
package gcf

import (
	"context"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigtable"
	"cloud.google.com/go/storage"
	"github.com/pkg/errors"
	dataflow "google.golang.org/api/dataflow/v1b3"
)

const (
	dataFilePattern    = "cache.csv*"
	createTableRetries = 3
	columnFamily       = "csv"

	// NOTE: The following three files represents the state of a BT import. They
	// get written under:
	//
	//		<controlPath>/<TableID>/
	//
	// Init: written by borg to start BT import.
	initFile = "init.txt"
	// Launched: written by this cloud function to mark launching of BT import job.
	launchedFile = "launched.txt"
	// Completed: written by dataflow to mark completion of BT import.
	completedFile = "completed.txt"
	// Default region
	region = "us-central1"
)

// GCSEvent is the payload of a GCS event.
type GCSEvent struct {
	Name   string `json:"name"`
	Bucket string `json:"bucket"`
}

// joinURL joins url components.
// path.Join does work well for url, for example gs:// is changaed to gs:/
func joinURL(base string, paths ...string) string {
	p := path.Join(paths...)
	return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/"))
}

// Return the GCS bucket and object from path in the form of gs://<bucket>/<object>
func parsePath(path string) (string, string, error) {
	parts := strings.Split(path, "/")
	if parts[0] != "gs:" || parts[1] != "" || len(parts) < 3 {
		return "", "", errors.Errorf("Unexpected path: %s", path)
	}
	return parts[2], strings.Join(parts[3:], "/"), nil
}

func doesObjectExist(ctx context.Context, path string) (bool, error) {
	bucket, object, err := parsePath(path)
	if err != nil {
		return false, err
	}
	gcsClient, err := storage.NewClient(ctx)
	if err != nil {
		return false, errors.Wrap(err, "Failed to create gcsClient")
	}
	_, err = gcsClient.Bucket(bucket).Object(object).Attrs(ctx)
	if err == storage.ErrObjectNotExist {
		return false, nil
	}
	return true, nil
}

func writeToGCS(ctx context.Context, path, data string) error {
	bucket, object, err := parsePath(path)
	if err != nil {
		return err
	}
	gcsClient, err := storage.NewClient(ctx)
	if err != nil {
		return errors.Wrap(err, "Failed to create gcsClient")
	}
	w := gcsClient.Bucket(bucket).Object(object).NewWriter(ctx)
	defer w.Close()
	_, err = fmt.Fprint(w, data)
	return errors.WithMessagef(err, "Failed to write data to %s/%s", bucket, object)
}

func launchDataflowJob(
	ctx context.Context,
	projectID string,
	instance string,
	tableID string,
	dataPath string,
	controlPath string,
	dataflowTemplate string,
) error {
	dataflowService, err := dataflow.NewService(ctx)
	if err != nil {
		return errors.Wrap(err, "Unable to create dataflow service")
	}
	dataFile := joinURL(dataPath, tableID, dataFilePattern)
	launchedPath := joinURL(controlPath, tableID, launchedFile)
	completedPath := joinURL(controlPath, tableID, completedFile)
	params := &dataflow.LaunchTemplateParameters{
		JobName: tableID,
		Parameters: map[string]string{
			"inputFile":          dataFile,
			"completionFile":     completedPath,
			"bigtableInstanceId": instance,
			"bigtableTableId":    tableID,
			"bigtableProjectId":  projectID,
			"region":             region,
		},
	}
	log.Printf("[%s/%s] Launching dataflow job: %s -> %s\n", instance, tableID, dataFile, launchedPath)
	launchCall := dataflow.NewProjectsTemplatesService(dataflowService).Launch(projectID, params)
	_, err = launchCall.GcsPath(dataflowTemplate).Do()
	if err != nil {
		return errors.WithMessagef(err, "Unable to launch dataflow job (%s, %s): %v\n", dataFile, launchedPath)
	}
	return nil
}

func setupBT(ctx context.Context, btProjectID, btInstance, tableID string) error {
	adminClient, err := bigtable.NewAdminClient(ctx, btProjectID, btInstance)
	if err != nil {
		return errors.Wrap(err, "Unable to create a table admin client")
	}
	// Create table. We retry 3 times in 1 minute intervals.
	dctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
	defer cancel()
	var ok bool
	for ii := 0; ii < createTableRetries; ii++ {
		log.Printf("Creating new bigtable table (%d): %s/%s", ii, btInstance, tableID)
		err = adminClient.CreateTable(dctx, tableID)
		if err != nil {
			log.Printf("Error creating table %s, retry...", err)
		} else {
			ok = true
			break
		}
		time.Sleep(1 * time.Minute)
	}
	if !ok {
		return errors.Errorf("Unable to create table: %s, got error: %v", tableID, err)
	}
	// Create table columnFamily.
	log.Printf("Creating column family %s in table %s/%s", columnFamily, btInstance, tableID)
	if err := adminClient.CreateColumnFamily(dctx, tableID, columnFamily); err != nil {
		return errors.WithMessagef(err, "Unable to create column family: csv for table: %s, got error: %v", tableID)
	}
	return nil
}

func scaleBT(ctx context.Context, projectID, instance, cluster string, numNodes int32) error {
	// Scale up bigtable cluster. This helps speed up the dataflow job.
	// We scale down again once dataflow job completes.
	instanceAdminClient, err := bigtable.NewInstanceAdminClient(ctx, projectID)
	dctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
	defer cancel()
	if err != nil {
		return errors.Wrap(err, "Unable to create a table instance admin client")
	}
	log.Printf("Scaling BT %s cluster %s to %d nodes", instance, cluster, numNodes)
	if err := instanceAdminClient.UpdateCluster(dctx, instance, cluster, numNodes); err != nil {
		return errors.WithMessagef(err, "Unable to resize bigtable cluster %s to %d: %v", cluster, numNodes)
	}
	return nil
}

func getBTNodes(ctx context.Context, projectID, instance, cluster string) (int, error) {
	instanceAdminClient, err := bigtable.NewInstanceAdminClient(ctx, projectID)
	if err != nil {
		return 0, errors.Wrap(err, "Unable to create a table instance admin client")
	}
	clusterInfo, err := instanceAdminClient.GetCluster(ctx, instance, cluster)
	if err != nil {
		return 0, errors.Wrap(err, "Unable to get cluster information")
	}
	return clusterInfo.ServeNodes, nil
}

func btImportControllerInternal(ctx context.Context, e GCSEvent) error {
	projectID := os.Getenv("projectID")
	instance := os.Getenv("instance")
	cluster := os.Getenv("cluster")
	nodesHigh := os.Getenv("nodesHigh")
	nodesLow := os.Getenv("nodesLow")
	dataflowTemplate := os.Getenv("dataflowTemplate")
	dataPath := os.Getenv("dataPath")
	controlPath := os.Getenv("controlPath")
	if projectID == "" {
		return errors.New("projectID is not set in environment")
	}
	if instance == "" {
		return errors.New("instance is not set in environment")
	}
	if cluster == "" {
		return errors.New("cluster is not set in environment")
	}
	if dataflowTemplate == "" {
		return errors.New("dataflowTemplate is not set in environment")
	}
	if dataPath == "" {
		return errors.New("dataPath is not set in environment")
	}
	if controlPath == "" {
		return errors.New("controlPath is not set in environment")
	}
	// Get low and high nodes number
	nodesH, err := strconv.Atoi(nodesHigh)
	if err != nil {
		return errors.Wrap(err, "Unable to parse 'nodesHigh' as an integer")
	}
	nodesL, err := strconv.Atoi(nodesLow)
	if err != nil {
		return errors.Wrap(err, "Unable to parse 'nodesLow' as an integer")
	}
	// Get control bucket and object
	controlBucket, controlFolder, err := parsePath(controlPath)
	if err != nil {
		return err
	}
	if e.Bucket != controlBucket {
		log.Printf("Trigger bucket '%s' != '%s', skip processing", e.Bucket, controlBucket)
		return nil
	}
	// Get table ID.
	// e.Name should is like "**/*/branch_2021_01_01_01_01/launched.txt"
	parts := strings.Split(e.Name, "/")
	if len(parts) < 3 {
		log.Printf("Ignore irrelevant trigger from file %s", e.Name)
		return nil
	}
	tableID := parts[len(parts)-2]
	triggerFolder := strings.Join(parts[0:len(parts)-2], "/")
	if triggerFolder != controlFolder {
		log.Printf("Control folder '%s' != '%s', skip processing", triggerFolder, controlFolder)
		return nil
	}

	numNodes, err := getBTNodes(ctx, projectID, instance, cluster)
	if err != nil {
		return err
	}

	if strings.HasSuffix(e.Name, initFile) {
		log.Printf("[%s] State Init", e.Name)
		// Called when the state-machine is at Init. Logic below moves it to Launched state.
		launchedPath := joinURL(controlPath, tableID, launchedFile)
		exist, err := doesObjectExist(ctx, launchedPath)
		if err != nil {
			return errors.WithMessagef(err, "Failed to check %s", launchedFile)
		}
		if exist {
			return errors.WithMessagef(err, "Cache was already built for %s", tableID)
		}
		if err := setupBT(ctx, projectID, instance, tableID); err != nil {
			return err
		}
		if numNodes < nodesH {
			if err := scaleBT(ctx, projectID, instance, cluster, int32(nodesH)); err != nil {
				return err
			}
		}
		err = launchDataflowJob(ctx, projectID, instance, tableID, dataPath, controlPath, dataflowTemplate)
		if err != nil {
			return err
		}
		// Save the fact that we've launched the dataflow job.
		err = writeToGCS(ctx, launchedPath, "")
		if err != nil {
			return err
		}
		log.Printf("[%s] State Launched", e.Name)
	} else if strings.HasSuffix(e.Name, completedFile) {
		log.Printf("[%s] State Completed", e.Name)
		// Called when the state-machine moves to Completed state from Launched.
		if numNodes == nodesH {
			// Only scale down BT nodes when the current high node is set up this config.
			// This requires different high nodes in different config.
			if err := scaleBT(ctx, projectID, instance, cluster, int32(nodesL)); err != nil {
				return err
			}
		}
		// TODO: else, notify Mixer to load the BT table.
		log.Printf("[%s] Completed work", e.Name)
	}
	return nil
}

// BTImportController consumes a GCS event and runs an import state machine.
func BTImportController(ctx context.Context, e GCSEvent) error {
	err := btImportControllerInternal(ctx, e)
	if err != nil {
		// Panic gets reported to Cloud Logging Error Reporting that we can then
		// alert on
		// (https://cloud.google.com/functions/docs/monitoring/error-reporting#functions-errors-log-go)
		panic(errors.Wrap(err, "panic"))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/gyepisam/fileutils"
	"github.com/gyepisam/multiflag"
	"github.com/gyepisam/redux"
)

const (
	DEFAULT_TARGET = string(redux.TASK_PREFIX) + "all"
	DEFAULT_DO     = DEFAULT_TARGET + ".do"
)

var cmdRedo = &Command{
	UsageLine: "redux redo [OPTION]... [TARGET]...",
	Short:     "Builds files atomically.",
	LinkName:  "redo",
}

func init() {
	// break loop
	cmdRedo.Run = runRedo

	text := `
The redo command builds files atomically by running a do script asssociated with the target.

redo normally requires one or more target arguments.
If no target arguments are provided, redo runs the default target %s in the current directory
if its do script %s exists.
`
	cmdRedo.Long = fmt.Sprintf(text, DEFAULT_TARGET, DEFAULT_DO)
}

var (
	verbosity *multiflag.Value
	debug     *multiflag.Value
	isTask    bool
	shArgs    string
)

func init() {
	flg := flag.NewFlagSet("redo", flag.ContinueOnError)

	verbosity = multiflag.BoolSet(flg, "verbose", "false", "Be verbose. Repeat for intensity.", "v")

	debug = multiflag.BoolSet(flg, "debug", "false", "Print debugging output.", "d")

	flg.BoolVar(&isTask, "task", false, "Run .do script for side effects and ignore output.")

	flg.StringVar(&shArgs, "sh", "", "Extra arguments for /bin/sh.")

	cmdRedo.Flag = flg
}

func runRedo(targets []string) error {

	// set options from environment if not provided.
	if verbosity.NArg() == 0 {
		for i := len(os.Getenv("REDO_VERBOSE")); i > 0; i-- {
			verbosity.Set("true")
		}
	}

	if debug.NArg() == 0 {
		if len(os.Getenv("REDO_DEBUG")) > 0 {
			debug.Set("true")
		}
	}

	if s := shArgs; s != "" {
		os.Setenv("REDO_SHELL_ARGS", s)
		redux.ShellArgs = s
	}

	// if shell args are set, ensure that at least minimal verbosity is also set.
	if redux.ShellArgs != "" && (verbosity.NArg() == 0) {
		verbosity.Set("true")
	}

	// Set explicit options to avoid clobbering environment inherited options.
	if n := verbosity.NArg(); n > 0 {
		os.Setenv("REDO_VERBOSE", strings.Repeat("x", n))
		redux.Verbosity = n
	}

	if n := debug.NArg(); n > 0 {
		os.Setenv("REDO_DEBUG", "true")
		redux.Debug = true
	}

	// If no arguments are specified, use run default target if its .do file exists.
	// Otherwise, print usage and exit.
	if len(targets) == 0 {
		if found, err := fileutils.FileExists(DEFAULT_DO); err != nil {
			return err
		} else if found {
			targets = append(targets, DEFAULT_TARGET)
		} else {
			cmdRedo.Flag.Usage()
			os.Exit(1)
			return nil
		}
	}

	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	// It *is* slower to reinitialize for each target, but doing
	// so guarantees that a single redo call with multiple targets that
	// potentially have differing roots will work correctly.
	for _, path := range targets {
		if file, err := redux.NewFile(wd, path); err != nil {
			return err
		} else {
			file.IsTaskFlag = isTask
			if err := file.Redo(); err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>Default to @all only in toplevel redo invocation<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/gyepisam/fileutils"
	"github.com/gyepisam/multiflag"
	"github.com/gyepisam/redux"
)

const (
	DEFAULT_TARGET = string(redux.TASK_PREFIX) + "all"
	DEFAULT_DO     = DEFAULT_TARGET + ".do"
)

var cmdRedo = &Command{
	UsageLine: "redux redo [OPTION]... [TARGET]...",
	Short:     "Builds files atomically.",
	LinkName:  "redo",
}

func init() {
	// break loop
	cmdRedo.Run = runRedo

	text := `
The redo command builds files atomically by running a do script asssociated with the target.

redo normally requires one or more target arguments.
If no target arguments are provided, redo runs the default target %s in the current directory
if its do script %s exists.
`
	cmdRedo.Long = fmt.Sprintf(text, DEFAULT_TARGET, DEFAULT_DO)
}

var (
	verbosity *multiflag.Value
	debug     *multiflag.Value
	isTask    bool
	shArgs    string
)

func init() {
	flg := flag.NewFlagSet("redo", flag.ContinueOnError)

	verbosity = multiflag.BoolSet(flg, "verbose", "false", "Be verbose. Repeat for intensity.", "v")

	debug = multiflag.BoolSet(flg, "debug", "false", "Print debugging output.", "d")

	flg.BoolVar(&isTask, "task", false, "Run .do script for side effects and ignore output.")

	flg.StringVar(&shArgs, "sh", "", "Extra arguments for /bin/sh.")

	cmdRedo.Flag = flg
}

func runRedo(targets []string) error {

	// set options from environment if not provided.
	if verbosity.NArg() == 0 {
		for i := len(os.Getenv("REDO_VERBOSE")); i > 0; i-- {
			verbosity.Set("true")
		}
	}

	if debug.NArg() == 0 {
		if len(os.Getenv("REDO_DEBUG")) > 0 {
			debug.Set("true")
		}
	}

	if s := shArgs; s != "" {
		os.Setenv("REDO_SHELL_ARGS", s)
		redux.ShellArgs = s
	}

	// if shell args are set, ensure that at least minimal verbosity is also set.
	if redux.ShellArgs != "" && (verbosity.NArg() == 0) {
		verbosity.Set("true")
	}

	// Set explicit options to avoid clobbering environment inherited options.
	if n := verbosity.NArg(); n > 0 {
		os.Setenv("REDO_VERBOSE", strings.Repeat("x", n))
		redux.Verbosity = n
	}

	if n := debug.NArg(); n > 0 {
		os.Setenv("REDO_DEBUG", "true")
		redux.Debug = true
	}

	// If no arguments are specified, use run default target if its .do file exists.
	// Otherwise, print usage and exit.
	if len(targets) == 0 && os.Getenv("REDO_DEPTH") == "" {
		if found, err := fileutils.FileExists(DEFAULT_DO); err != nil {
			return err
		} else if found {
			targets = append(targets, DEFAULT_TARGET)
		} else {
			cmdRedo.Flag.Usage()
			os.Exit(1)
			return nil
		}
	}

	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	// It *is* slower to reinitialize for each target, but doing
	// so guarantees that a single redo call with multiple targets that
	// potentially have differing roots will work correctly.
	for _, path := range targets {
		if file, err := redux.NewFile(wd, path); err != nil {
			return err
		} else {
			file.IsTaskFlag = isTask
			if err := file.Redo(); err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package builder

import (
	"github.com/blablacar/cnt/dist"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/utils"
	"io/ioutil"
	"os"
	"strings"
)

func (cnt *Img) Build() error {
	log.Get().Info("Building Image : ", cnt.manifest.NameAndVersion)

	os.MkdirAll(cnt.rootfs, 0777)

	cnt.processFrom()
	cnt.copyInternals()
	cnt.copyRunlevelsScripts()

	cnt.runLevelBuildSetup()

	cnt.writeImgManifest()
	cnt.writeCntManifest() // TODO move that, here because we update the version number to generated version

	cnt.runBuild()
	cnt.copyAttributes()
	cnt.copyConfd()
	cnt.copyFiles()
	cnt.runBuildLate()

	cnt.tarAci(false)
	//	ExecCmd("chown " + os.Getenv("SUDO_USER") + ": " + target + "/*") //TODO chown
	return nil
}

func (i *Img) CheckBuilt() {
	if _, err := os.Stat(i.target + PATH_IMAGE_ACI); os.IsNotExist(err) {
		if err := i.Build(); err != nil {
			log.Get().Panic("Cannot continue since build failed")
		}
	}
}

///////////////////////////////////////////////////////

func (cnt *Img) writeCntManifest() {
	utils.CopyFile(cnt.path+PATH_CNT_MANIFEST, cnt.target+PATH_CNT_MANIFEST)
}

func (cnt *Img) runBuildLate() {
	res, err := utils.IsDirEmpty(cnt.target + PATH_RUNLEVELS + PATH_BUILD_LATE)
	res2, err2 := utils.IsDirEmpty(cnt.rootfs + PATH_CNT + PATH_RUNLEVELS + PATH_INHERIT_BUILD_LATE)
	if (res && res2) || (err != nil && err2 != nil) {
		return
	}

	{
		rootfs := "${TARGET}/rootfs"
		if cnt.manifest.Build.NoBuildImage() {
			rootfs = ""
		}
		build := strings.Replace(BUILD_SCRIPT_LATE, "%%ROOTFS%%", rootfs, 1)
		ioutil.WriteFile(cnt.target+"/build-late.sh", []byte(build), 0777)
	}

	if err := utils.ExecCmd("systemd-nspawn", "--version"); err == nil {
		log.Get().Info("Run with systemd-nspawn")
		if err := utils.ExecCmd("systemd-nspawn", "--directory="+cnt.rootfs, "--capability=all",
			"--bind="+cnt.target+"/:/target", "target/build-late.sh"); err != nil {
			log.Get().Panic("Build step did not succeed", err)
		}
	} else {
		log.Get().Panic("systemd-nspawn is required")
	}
}

func (cnt *Img) runBuild() {
	if res, err := utils.IsDirEmpty(cnt.target + PATH_RUNLEVELS + PATH_BUILD); res || err != nil {
		return
	}
	if err := utils.ExecCmd("systemd-nspawn", "--version"); err != nil {
		log.Get().Panic("systemd-nspawn is required")
	}

	rootfs := "${TARGET}/rootfs"
	if cnt.manifest.Build.NoBuildImage() {
		rootfs = ""
	}
	build := strings.Replace(BUILD_SCRIPT, "%%ROOTFS%%", rootfs, 1)
	ioutil.WriteFile(cnt.target+"/build.sh", []byte(build), 0777)

	if err := utils.ExecCmd("systemd-nspawn", "--directory="+cnt.rootfs, "--capability=all",
		"--bind="+cnt.target+"/:/target", "--share-system", "target/build.sh"); err != nil {
		log.Get().Panic("Build step did not succeed", err)
	}
}

func (cnt *Img) processFrom() {
	if cnt.manifest.From == "" {
		return
	}
	if err := utils.ExecCmd("bash", "-c", "rkt image list --fields name --no-legend | grep -q "+cnt.manifest.From.String()); err != nil {
		utils.ExecCmd("rkt", "--insecure-skip-verify=true", "fetch", cnt.manifest.From.String())
	}
	if err := utils.ExecCmd("rkt", "image", "render", "--overwrite", cnt.manifest.From.String(), cnt.target); err != nil {
		log.Get().Panic("Cannot render from image"+cnt.manifest.From.String(), err)
	}
}

func (cnt *Img) copyInternals() {
	log.Get().Info("Copy internals")
	os.MkdirAll(cnt.rootfs+PATH_CNT+PATH_BIN, 0755)
	os.MkdirAll(cnt.rootfs+"/bin", 0755)     // this is required or systemd-nspawn will create symlink on it
	os.MkdirAll(cnt.rootfs+"/usr/bin", 0755) // this is required by systemd-nspawn

	busybox, _ := dist.Asset("dist/bindata/busybox")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/busybox", busybox, 0777); err != nil {
		log.Get().Panic(err)
	}

	confd, _ := dist.Asset("dist/bindata/confd")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/confd", confd, 0777); err != nil {
		log.Get().Panic(err)
	}

	attributeMerger, _ := dist.Asset("dist/bindata/attributes-merger")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/attributes-merger", attributeMerger, 0777); err != nil {
		log.Get().Panic(err)
	}

	confdFile := `backend = "env"
confdir = "/cnt"
prefix = "/confd"
log-level = "debug"
`
	os.MkdirAll(cnt.rootfs+PATH_CNT+"/prestart", 0755)
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+"/prestart/confd.toml", []byte(confdFile), 0777); err != nil {
		log.Get().Panic(err)
	}

	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/prestart", []byte(PRESTART), 0777); err != nil {
		log.Get().Panic(err)
	}
}

func (cnt *Img) copyRunlevelsScripts() {
	log.Get().Info("Copy Runlevels scripts")
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD, cnt.target+PATH_RUNLEVELS+PATH_BUILD)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD_LATE, cnt.target+PATH_RUNLEVELS+PATH_BUILD_LATE)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD_SETUP, cnt.target+PATH_RUNLEVELS+PATH_BUILD_SETUP)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_EARLY, cnt.target+PATH_RUNLEVELS+PATH_PRESTART_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_LATE, cnt.target+PATH_RUNLEVELS+PATH_PRESTART_LATE)

	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_EARLY, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_PRESTART_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_LATE, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_PRESTART_LATE)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_INHERIT_BUILD_EARLY, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_INHERIT_BUILD_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_INHERIT_BUILD_LATE, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_INHERIT_BUILD_LATE)
}

func (cnt *Img) runLevelBuildSetup() {
	files, err := ioutil.ReadDir(cnt.path + PATH_RUNLEVELS + PATH_BUILD_SETUP)
	if err != nil {
		return
	}

	os.Setenv("BASEDIR", cnt.path)
	os.Setenv("TARGET", cnt.target)
	for _, f := range files {
		if !f.IsDir() {
			log.Get().Info("Running Build setup level : ", f.Name())
			if err := utils.ExecCmd(cnt.path + PATH_RUNLEVELS + PATH_BUILD_SETUP + "/" + f.Name()); err != nil {
				log.Get().Panic(err)
			}
		}
	}
}

func (cnt *Img) copyConfd() {
	utils.CopyDir(cnt.path+PATH_CONFD+PATH_CONFDOTD, cnt.rootfs+PATH_CNT+PATH_CONFDOTD)
	utils.CopyDir(cnt.path+PATH_CONFD+PATH_TEMPLATES, cnt.rootfs+PATH_CNT+PATH_TEMPLATES)
}

func (cnt *Img) copyFiles() {
	utils.CopyDir(cnt.path+PATH_FILES, cnt.rootfs)
}

func (cnt *Img) copyAttributes() {
	utils.CopyDir(cnt.path+PATH_ATTRIBUTES, cnt.rootfs+PATH_CNT+PATH_ATTRIBUTES+"/"+cnt.manifest.NameAndVersion.ShortName())
}

func (cnt *Img) writeImgManifest() {
	log.Get().Debug("Writing aci manifest")
	utils.WriteImageManifest(&cnt.manifest, cnt.target+PATH_MANIFEST, cnt.manifest.NameAndVersion.Name())
}
<commit_msg>remove --share-system from run build<commit_after>package builder

import (
	"github.com/blablacar/cnt/dist"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/utils"
	"io/ioutil"
	"os"
	"strings"
)

func (cnt *Img) Build() error {
	log.Get().Info("Building Image : ", cnt.manifest.NameAndVersion)

	os.MkdirAll(cnt.rootfs, 0777)

	cnt.processFrom()
	cnt.copyInternals()
	cnt.copyRunlevelsScripts()

	cnt.runLevelBuildSetup()

	cnt.writeImgManifest()
	cnt.writeCntManifest() // TODO move that, here because we update the version number to generated version

	cnt.runBuild()
	cnt.copyAttributes()
	cnt.copyConfd()
	cnt.copyFiles()
	cnt.runBuildLate()

	cnt.tarAci(false)
	//	ExecCmd("chown " + os.Getenv("SUDO_USER") + ": " + target + "/*") //TODO chown
	return nil
}

func (i *Img) CheckBuilt() {
	if _, err := os.Stat(i.target + PATH_IMAGE_ACI); os.IsNotExist(err) {
		if err := i.Build(); err != nil {
			log.Get().Panic("Cannot continue since build failed")
		}
	}
}

///////////////////////////////////////////////////////

func (cnt *Img) writeCntManifest() {
	utils.CopyFile(cnt.path+PATH_CNT_MANIFEST, cnt.target+PATH_CNT_MANIFEST)
}

func (cnt *Img) runBuildLate() {
	res, err := utils.IsDirEmpty(cnt.target + PATH_RUNLEVELS + PATH_BUILD_LATE)
	res2, err2 := utils.IsDirEmpty(cnt.rootfs + PATH_CNT + PATH_RUNLEVELS + PATH_INHERIT_BUILD_LATE)
	if (res && res2) || (err != nil && err2 != nil) {
		return
	}

	{
		rootfs := "${TARGET}/rootfs"
		if cnt.manifest.Build.NoBuildImage() {
			rootfs = ""
		}
		build := strings.Replace(BUILD_SCRIPT_LATE, "%%ROOTFS%%", rootfs, 1)
		ioutil.WriteFile(cnt.target+"/build-late.sh", []byte(build), 0777)
	}

	if err := utils.ExecCmd("systemd-nspawn", "--version"); err == nil {
		log.Get().Info("Run with systemd-nspawn")
		if err := utils.ExecCmd("systemd-nspawn", "--directory="+cnt.rootfs, "--capability=all",
			"--bind="+cnt.target+"/:/target", "target/build-late.sh"); err != nil {
			log.Get().Panic("Build step did not succeed", err)
		}
	} else {
		log.Get().Panic("systemd-nspawn is required")
	}
}

func (cnt *Img) runBuild() {
	if res, err := utils.IsDirEmpty(cnt.target + PATH_RUNLEVELS + PATH_BUILD); res || err != nil {
		return
	}
	if err := utils.ExecCmd("systemd-nspawn", "--version"); err != nil {
		log.Get().Panic("systemd-nspawn is required")
	}

	rootfs := "${TARGET}/rootfs"
	if cnt.manifest.Build.NoBuildImage() {
		rootfs = ""
	}
	build := strings.Replace(BUILD_SCRIPT, "%%ROOTFS%%", rootfs, 1)
	ioutil.WriteFile(cnt.target+"/build.sh", []byte(build), 0777)

	if err := utils.ExecCmd("systemd-nspawn", "--directory="+cnt.rootfs, "--capability=all",
		"--bind="+cnt.target+"/:/target", "target/build.sh"); err != nil {
		log.Get().Panic("Build step did not succeed", err)
	}
}

func (cnt *Img) processFrom() {
	if cnt.manifest.From == "" {
		return
	}
	if err := utils.ExecCmd("bash", "-c", "rkt image list --fields name --no-legend | grep -q "+cnt.manifest.From.String()); err != nil {
		utils.ExecCmd("rkt", "--insecure-skip-verify=true", "fetch", cnt.manifest.From.String())
	}
	if err := utils.ExecCmd("rkt", "image", "render", "--overwrite", cnt.manifest.From.String(), cnt.target); err != nil {
		log.Get().Panic("Cannot render from image"+cnt.manifest.From.String(), err)
	}
}

func (cnt *Img) copyInternals() {
	log.Get().Info("Copy internals")
	os.MkdirAll(cnt.rootfs+PATH_CNT+PATH_BIN, 0755)
	os.MkdirAll(cnt.rootfs+"/bin", 0755)     // this is required or systemd-nspawn will create symlink on it
	os.MkdirAll(cnt.rootfs+"/usr/bin", 0755) // this is required by systemd-nspawn

	busybox, _ := dist.Asset("dist/bindata/busybox")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/busybox", busybox, 0777); err != nil {
		log.Get().Panic(err)
	}

	confd, _ := dist.Asset("dist/bindata/confd")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/confd", confd, 0777); err != nil {
		log.Get().Panic(err)
	}

	attributeMerger, _ := dist.Asset("dist/bindata/attributes-merger")
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/attributes-merger", attributeMerger, 0777); err != nil {
		log.Get().Panic(err)
	}

	confdFile := `backend = "env"
confdir = "/cnt"
prefix = "/confd"
log-level = "debug"
`
	os.MkdirAll(cnt.rootfs+PATH_CNT+"/prestart", 0755)
	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+"/prestart/confd.toml", []byte(confdFile), 0777); err != nil {
		log.Get().Panic(err)
	}

	if err := ioutil.WriteFile(cnt.rootfs+PATH_CNT+PATH_BIN+"/prestart", []byte(PRESTART), 0777); err != nil {
		log.Get().Panic(err)
	}
}

func (cnt *Img) copyRunlevelsScripts() {
	log.Get().Info("Copy Runlevels scripts")
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD, cnt.target+PATH_RUNLEVELS+PATH_BUILD)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD_LATE, cnt.target+PATH_RUNLEVELS+PATH_BUILD_LATE)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_BUILD_SETUP, cnt.target+PATH_RUNLEVELS+PATH_BUILD_SETUP)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_EARLY, cnt.target+PATH_RUNLEVELS+PATH_PRESTART_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_LATE, cnt.target+PATH_RUNLEVELS+PATH_PRESTART_LATE)

	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_EARLY, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_PRESTART_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_PRESTART_LATE, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_PRESTART_LATE)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_INHERIT_BUILD_EARLY, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_INHERIT_BUILD_EARLY)
	utils.CopyDir(cnt.path+PATH_RUNLEVELS+PATH_INHERIT_BUILD_LATE, cnt.target+PATH_ROOTFS+PATH_CNT+PATH_RUNLEVELS+PATH_INHERIT_BUILD_LATE)
}

func (cnt *Img) runLevelBuildSetup() {
	files, err := ioutil.ReadDir(cnt.path + PATH_RUNLEVELS + PATH_BUILD_SETUP)
	if err != nil {
		return
	}

	os.Setenv("BASEDIR", cnt.path)
	os.Setenv("TARGET", cnt.target)
	for _, f := range files {
		if !f.IsDir() {
			log.Get().Info("Running Build setup level : ", f.Name())
			if err := utils.ExecCmd(cnt.path + PATH_RUNLEVELS + PATH_BUILD_SETUP + "/" + f.Name()); err != nil {
				log.Get().Panic(err)
			}
		}
	}
}

func (cnt *Img) copyConfd() {
	utils.CopyDir(cnt.path+PATH_CONFD+PATH_CONFDOTD, cnt.rootfs+PATH_CNT+PATH_CONFDOTD)
	utils.CopyDir(cnt.path+PATH_CONFD+PATH_TEMPLATES, cnt.rootfs+PATH_CNT+PATH_TEMPLATES)
}

func (cnt *Img) copyFiles() {
	utils.CopyDir(cnt.path+PATH_FILES, cnt.rootfs)
}

func (cnt *Img) copyAttributes() {
	utils.CopyDir(cnt.path+PATH_ATTRIBUTES, cnt.rootfs+PATH_CNT+PATH_ATTRIBUTES+"/"+cnt.manifest.NameAndVersion.ShortName())
}

func (cnt *Img) writeImgManifest() {
	log.Get().Debug("Writing aci manifest")
	utils.WriteImageManifest(&cnt.manifest, cnt.target+PATH_MANIFEST, cnt.manifest.NameAndVersion.Name())
}
<|endoftext|>


<commit_before>// Package calendar generates and has utlilty functions to generate an HTML calendar for use in nlgids.
package calendar

import (
	"fmt"
	"sort"
	"time"
)

// TODO: next button should have a ref to point to something.

var avail = [...]string{"past", "busy", "free"}

type Available int

const (
	past Available = iota
	busy
	free
)

func (a Available) String() string { return avail[a] }

// Calendar holds the HTML that makes up the calendar. Each
// day is indexed by the 12 o' clock night time time.Time.
// All date are in the UTC timezone.
type Calendar struct {
	days  map[time.Time]Available
	begin time.Time
	end   time.Time
	// month we're in (mostly)?
}

type times []time.Time

func (t times) Len() int           { return len(t) }
func (t times) Less(i, j int) bool { return t[i].Before(t[j]) }
func (t times) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }

func (c *Calendar) heading() string {
	// lang!
	s := `

`
	s += "
Sun	Mon	Tue	Wed	Thu	Fri	Sat
\n"
	return s
}

func (c *Calendar) Header() string {
	// Template on Calendar that gets the month from c (or we set it).
	s := `

bla

`
	return s
}

func (c *Calendar) Footer() string {
	return `

`
}

func (c *Calendar) openTR() string  { return "
\n" }
func (c *Calendar) closeTR() string { return "
\n" }

func (c *Calendar) entry(t time.Time) string {
	d := c.days[t]
	day := fmt.Sprintf("%02d", t.Day())
	class := fmt.Sprintf("\t", d)
	close := "	\n"
	href := ""
	switch d {
	case free:
		date := fmt.Sprintf("%4d-%02d-%02d", t.Year(), t.Month(), t.Day())
		href = fmt.Sprintf("%s", date, d) // SetDate is defined on the page/form itself
		class = fmt.Sprintf("\t", d)
	case busy:
		href = day
	case past:
		href = day
	}
	s := class + href + close
	return s
}

func (c *Calendar) HTML() string {
	s := c.Header()
	s += "
\n"
	s += c.html()
	s += "
"
	s += c.Footer()
	return s
}

func (c *Calendar) sort() times {
	keys := times{}
	for k := range c.days {
		keys = append(keys, k)
	}

	sort.Sort(keys)
	return keys
}

func (c *Calendar) html() string {
	keys := c.sort()

	s := c.heading()
	i := 0
	for _, k := range keys {
		if i%7 == 0 {
			if i > 0 {
				s += c.closeTR()
			}
			s += c.closeTR()
		}
		s += c.entry(k)
		i++
	}
	s += c.closeTR()
	return s
}

// New creates a new month calendar based on d, d must be in the form: YYYY-MM-DD.
func New(d string) (*Calendar, error) {
	date, err := time.Parse("2006-01-02", d)
	if err != nil {
		return nil, err
	}
	cal := &Calendar{days: make(map[time.Time]Available)}

	now := time.Now()
	/// If we see now we set the class now (or something)

	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
	first := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC)
	last := time.Date(date.Year(), date.Month()+1, 1, 0, 0, 0, 0, time.UTC)
	last = last.Add(-24 * time.Hour)

	// Add the remaining days of the previous month.
	for i := 0; i < int(first.Weekday()); i++ {
		lastMonthDay := first.AddDate(0, 0, -1*(i+1))
		cal.days[lastMonthDay] = free

		if lastMonthDay.Before(today) {
			cal.days[lastMonthDay] = past
		}
	}

	// Loop from i to lastDay and add the entire month.
	for i := 1; i <= last.Day(); i++ {
		day := time.Date(date.Year(), date.Month(), i, 0, 0, 0, 0, time.UTC)

		cal.days[day] = free

		if day.Before(today) {
			cal.days[day] = past
		}
	}

	// These are dates in the new month.
	j := 1
	for i := int(last.Weekday()) + 1; i < 7; i++ {
		nextMonthDay := last.AddDate(0, 0, j)
		cal.days[nextMonthDay] = free

		if nextMonthDay.Before(today) {
			cal.days[nextMonthDay] = past
		}

		j++
	}
	times := cal.sort()
	if len(times) > 0 {
		cal.begin = times[0]
		cal.end = times[len(times)-1]
	}

	return cal, nil
}
<commit_msg>Empty string defaults to now<commit_after>// Package calendar generates and has utlilty functions to generate an HTML calendar for use in nlgids.
package calendar

import (
	"fmt"
	"sort"
	"time"
)

// TODO: next button should have a ref to point to something.

var avail = [...]string{"past", "busy", "free"}

type Available int

const (
	past Available = iota
	busy
	free
)

func (a Available) String() string { return avail[a] }

// Calendar holds the HTML that makes up the calendar. Each
// day is indexed by the 12 o' clock night time time.Time.
// All date are in the UTC timezone.
type Calendar struct {
	days  map[time.Time]Available
	begin time.Time
	end   time.Time
	// month we're in (mostly)?
}

type times []time.Time

func (t times) Len() int           { return len(t) }
func (t times) Less(i, j int) bool { return t[i].Before(t[j]) }
func (t times) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }

func (c *Calendar) heading() string {
	// lang!
	s := `

`
	s += "
Sun	Mon	Tue	Wed	Thu	Fri	Sat
\n"
	return s
}

func (c *Calendar) Header() string {
	// Template on Calendar that gets the month from c (or we set it).
	s := `

bla

`
	return s
}

func (c *Calendar) Footer() string {
	return `

`
}

func (c *Calendar) openTR() string  { return "
\n" }
func (c *Calendar) closeTR() string { return "
\n" }

func (c *Calendar) entry(t time.Time) string {
	d := c.days[t]
	day := fmt.Sprintf("%02d", t.Day())
	class := fmt.Sprintf("\t", d)
	close := "	\n"
	href := ""
	switch d {
	case free:
		date := fmt.Sprintf("%4d-%02d-%02d", t.Year(), t.Month(), t.Day())
		href = fmt.Sprintf("%s", date, d) // SetDate is defined on the page/form itself
		class = fmt.Sprintf("\t", d)
	case busy:
		href = day
	case past:
		href = day
	}
	s := class + href + close
	return s
}

func (c *Calendar) HTML() string {
	s := c.Header()
	s += "
\n"
	s += c.html()
	s += "
"
	s += c.Footer()
	return s
}

func (c *Calendar) sort() times {
	keys := times{}
	for k := range c.days {
		keys = append(keys, k)
	}

	sort.Sort(keys)
	return keys
}

func (c *Calendar) html() string {
	keys := c.sort()

	s := c.heading()
	i := 0
	for _, k := range keys {
		if i%7 == 0 {
			if i > 0 {
				s += c.closeTR()
			}
			s += c.closeTR()
		}
		s += c.entry(k)
		i++
	}
	s += c.closeTR()
	return s
}

// New creates a new month calendar based on d, d must be in the form: YYYY-MM-DD.
// D can also be the empty string, then the current date is assumed.
func New(d string) (*Calendar, error) {
	date, now := time.Now(), time.Now()
	if d != "" {
		var err error
		date, err = time.Parse("2006-01-02", d)
		if err != nil {
			return nil, err
		}
	}

	cal := &Calendar{days: make(map[time.Time]Available)}

	/// If we see now we set the class now (or something)

	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
	first := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.UTC)
	last := time.Date(date.Year(), date.Month()+1, 1, 0, 0, 0, 0, time.UTC)
	last = last.Add(-24 * time.Hour)

	// Add the remaining days of the previous month.
	for i := 0; i < int(first.Weekday()); i++ {
		lastMonthDay := first.AddDate(0, 0, -1*(i+1))
		cal.days[lastMonthDay] = free

		if lastMonthDay.Before(today) {
			cal.days[lastMonthDay] = past
		}
	}

	// Loop from i to lastDay and add the entire month.
	for i := 1; i <= last.Day(); i++ {
		day := time.Date(date.Year(), date.Month(), i, 0, 0, 0, 0, time.UTC)

		cal.days[day] = free

		if day.Before(today) {
			cal.days[day] = past
		}
	}

	// These are dates in the new month.
	j := 1
	for i := int(last.Weekday()) + 1; i < 7; i++ {
		nextMonthDay := last.AddDate(0, 0, j)
		cal.days[nextMonthDay] = free

		if nextMonthDay.Before(today) {
			cal.days[nextMonthDay] = past
		}

		j++
	}
	times := cal.sort()
	if len(times) > 0 {
		cal.begin = times[0]
		cal.end = times[len(times)-1]
	}

	return cal, nil
}
<|endoftext|>