<commit_before>package duck

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestIntOK(t *testing.T) {
	i, ok := Int("2")
	require.True(t, ok)
	require.Equal(t, int64(2), i)

	i, ok = Int("2.00")
	require.True(t, ok)
	require.Equal(t, int64(2), i)

	i, ok = Int(int8(12))
	require.True(t, ok)
	require.Equal(t, int64(12), i)

	i, ok = Int(int16(12))
	require.True(t, ok)
	require.Equal(t, int64(12), i)

	i, ok = Int(int32(12))
	require.True(t, ok)
	require.Equal(t, int64(12), i)

	i, ok = Int(int64(12))
	require.True(t, ok)
	require.Equal(t, int64(12), i)

	i, ok = Int(uint64(12))
	require.True(t, ok)
	require.Equal(t, int64(12), i)

	i, ok = Int(float32(3.0))
	require.True(t, ok)
	require.Equal(t, int64(3), i)

	i, ok = Int(float64(3.0))
	require.True(t, ok)
	require.Equal(t, int64(3), i)

	i, ok = Int(false)
	require.True(t, ok)
	require.Equal(t, int64(0), i)

	i, ok = Int(true)
	require.True(t, ok)
	require.Equal(t, int64(1), i)

	ptrtst := int64(5)
	i, ok = Int(&ptrtst)
	require.True(t, ok)
	require.Equal(t, int64(5), i)
}

func TestIntNOK(t *testing.T) {
	_, ok := Int(" 2 ")
	require.False(t, ok)

	_, ok = Int(nil)
	require.False(t, ok)

	_, ok = Int("1.1")
	require.False(t, ok)

	_, ok = Int(1.1)
	require.False(t, ok)
}

func TestFloatOK(t *testing.T) {
	i, ok := Float("2")
	require.True(t, ok)
	require.Equal(t, float64(2), i)

	i, ok = Float("2.345")
	require.True(t, ok)
	require.Equal(t, float64(2.345), i)

	i, ok = Float(int8(12))
	require.True(t, ok)
	require.Equal(t, float64(12), i)

	i, ok = Float(int16(12))
	require.True(t, ok)
	require.Equal(t, float64(12), i)

	i, ok = Float(int32(12))
	require.True(t, ok)
	require.Equal(t, float64(12), i)

	i, ok = Float(int64(12))
	require.True(t, ok)
	require.Equal(t, float64(12), i)

	i, ok = Float(uint64(12))
	require.True(t, ok)
	require.Equal(t, float64(12), i)

	i, ok = Float(float32(3.0))
	require.True(t, ok)
	require.Equal(t, float64(3), i)

	i, ok = Float(float64(3.0))
	require.True(t, ok)
	require.Equal(t, float64(3), i)

	i, ok = Float(false)
	require.True(t, ok)
	require.Equal(t, float64(0), i)

	i, ok = Float(true)
	require.True(t, ok)
	require.Equal(t, float64(1), i)

	ptrtst := int64(5)
	i, ok = Float(&ptrtst)
	require.True(t, ok)
	require.Equal(t, float64(5), i)
}

func TestFloatNOK(t *testing.T) {
	_, ok := Float(" 2 ")
	require.False(t, ok)

	_, ok = Float(nil)
	require.False(t, ok)

}

func TestBoolOK(t *testing.T) {
	i, ok := Bool("2")
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool("0")
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool("2.0")
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool("0.0")
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool("true")
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool("false")
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool(2)
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool(0)
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool(uint(2))
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool(uint(0))
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool(1.0)
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool(0.0)
	require.True(t, ok)
	require.False(t, i)

	i, ok = Bool(true)
	require.True(t, ok)
	require.True(t, i)

	i, ok = Bool(false)
	require.True(t, ok)
	require.False(t, i)
}

func TestBoolNOK(t *testing.T) {
	_, ok := Bool(" 2 ")
	require.False(t, ok)

	_, ok = Bool(nil)
	require.False(t, ok)

}

func TestStringOK(t *testing.T) {
	i, ok := String("hello")
	require.True(t, ok)
	require.Equal(t, "hello", i)

	i, ok = String(int8(12))
	require.True(t, ok)
	require.Equal(t, "12", i)

	i, ok = String(int16(12))
	require.True(t, ok)
	require.Equal(t, "12", i)

	i, ok = String(int32(12))
	require.True(t, ok)
	require.Equal(t, "12", i)

	i, ok = String(int64(12))
	require.True(t, ok)
	require.Equal(t, "12", i)

	i, ok = String(uint64(12))
	require.True(t, ok)
	require.Equal(t, "12", i)

	i, ok = String(float32(3.0))
	require.True(t, ok)
	require.Equal(t, "3", i)

	i, ok = String(float64(3.0))
	require.True(t, ok)
	require.Equal(t, "3", i)

	i, ok = String(false)
	require.True(t, ok)
	require.Equal(t, "false", i)

	i, ok = String(true)
	require.True(t, ok)
	require.Equal(t, "true", i)

	ptrtst := int64(5)
	i, ok = String(&ptrtst)
	require.True(t, ok)
	require.Equal(t, "5", i)
}

func TestStringNOK(t *testing.T) {
	_, ok := String(nil)
	require.False(t, ok)

}

func TestJSONString(t *testing.T) {
	i, ok := JSONString(float32(3.0))
	require.True(t, ok)
	require.Equal(t, "3", i)

	d := map[string]interface{}{"hi": 3}
	i, ok = JSONString(d)
	require.True(t, ok)
	require.Equal(t, `{"hi":3}`, i)
}
<commit_msg>Changed tests to testcases<commit_after>package duck

import (
	"fmt"
	"math"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestInt(t *testing.T) {
	ptrtst := int64(5)
	cases := []struct {
		in  interface{}
		ok  bool
		out int64
	}{
		{"2", true, 2},
		{"2.00", true, 2},
		{"1.1", false, 0},
		{" 2 ", false, 0},
		{"Inf", false, 0},
		{int(12), true, 12},
		{int8(12), true, 12},
		{int16(12), true, 12},
		{int32(12), true, 12},
		{int64(12), true, 12},
		{uint(12), true, 12},
		{uint8(12), true, 12},
		{uint16(12), true, 12},
		{uint32(12), true, 12},
		{uint64(12), true, 12},
		{float32(3.0), true, 3},
		{float64(3.0), true, 3},
		{3.3, false, 0},
		{math.Inf(1), false, 0},
		{math.NaN(), false, 0},
		{true, true, 1},
		{false, true, 0},
		{&ptrtst, true, 5},
		{nil, false, 0},
	}

	for _, c := range cases {
		out, ok := Int(c.in)
		require.Equal(t, c.ok, ok, fmt.Sprintf("%v", c))
		if c.ok {
			require.Equal(t, c.out, out, fmt.Sprintf("%v", c))
		}
	}

}

func TestFloat(t *testing.T) {
	ptrtst := int64(5)
	cases := []struct {
		in  interface{}
		ok  bool
		out float64
	}{
		{"2", true, 2},
		{"2.345", true, 2.345},
		{" 2 ", false, 0},
		{"Inf", true, math.Inf(1)},
		{int(12), true, 12},
		{int8(12), true, 12},
		{int16(12), true, 12},
		{int32(12), true, 12},
		{int64(12), true, 12},
		{uint(12), true, 12},
		{uint8(12), true, 12},
		{uint16(12), true, 12},
		{uint32(12), true, 12},
		{uint64(12), true, 12},
		{float32(3.0), true, 3},
		{float64(3.0), true, 3},
		{3.3, true, 3.3},
		{math.Inf(1), true, math.Inf(1)},
		{true, true, 1},
		{false, true, 0},
		{&ptrtst, true, 5},
		{nil, false, 0},
	}

	for _, c := range cases {
		out, ok := Float(c.in)
		require.Equal(t, c.ok, ok, fmt.Sprintf("%v", c))
		if c.ok {
			require.Equal(t, c.out, out, fmt.Sprintf("%v", c))
		}
	}

	//Testing NaN needs its own little thing
	out, ok := Float("NaN")
	require.True(t, ok)
	require.True(t, math.IsNaN(out))
	out, ok = Float(math.NaN())
	require.True(t, ok)
	require.True(t, math.IsNaN(out))

}

func TestBool(t *testing.T) {
	ptrtst := int64(5)
	cases := []struct {
		in  interface{}
		ok  bool
		out bool
	}{
		{"2", true, true},
		{"0", true, false},
		{"2.0", true, true},
		{"0.0", true, false},
		{" 2 ", false, false},
		{"Inf", true, true},
		{int(12), true, true},
		{int(0), true, false},
		{uint(12), true, true},
		{uint(0), true, false},
		{3.45, true, true},
		{0.0, true, false},
		{"true", true, true},
		{"false", true, false},
		{true, true, true},
		{false, true, false},
		{nil, false, false},
		{&ptrtst, true, true},
	}

	for _, c := range cases {
		out, ok := Bool(c.in)
		require.Equal(t, c.ok, ok, fmt.Sprintf("%v", c))
		if c.ok {
			require.Equal(t, c.out, out, fmt.Sprintf("%v", c))
		}
	}
}

func TestString(t *testing.T) {
	ptrtst := int64(5)
	cases := []struct {
		in  interface{}
		ok  bool
		out string
	}{
		{"hello", true, "hello"},
		{int(12), true, "12"},
		{int(0), true, "0"},
		{uint(12), true, "12"},
		{uint(0), true, "0"},
		{3.45, true, "3.45"},
		{0.0, true, "0"},
		{3.0, true, "3"},
		{true, true, "true"},
		{false, true, "false"},
		{nil, false, ""},
		{&ptrtst, true, "5"},
	}

	for _, c := range cases {
		out, ok := String(c.in)
		require.Equal(t, c.ok, ok, fmt.Sprintf("%v", c))
		if c.ok {
			require.Equal(t, c.out, out, fmt.Sprintf("%v", c))
		}
	}
}

func TestJSONString(t *testing.T) {
	cases := []struct {
		in  interface{}
		ok  bool
		out string
	}{
		{3.0, true, "3"},
		{map[string]interface{}{"hi": 3}, true, `{"hi":3}`},
	}

	for _, c := range cases {
		out, ok := JSONString(c.in)
		require.Equal(t, c.ok, ok, fmt.Sprintf("%v", c))
		if c.ok {
			require.Equal(t, c.out, out, fmt.Sprintf("%v", c))
		}
	}
}
<|endoftext|>


<commit_before>package contacts

import (
	"os"
	"testing"

	"github.com/ticketmatic/tm-go/ticketmatic"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/contactaddresstypes"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/contacttitles"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/phonenumbertypes"
)

func TestGet(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	list, err := Getlist(c, &ticketmatic.ContactQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(list.Data) <= 0 {
		t.Errorf("Unexpected list.Data length, got %#v, expected greater than %#v", len(list.Data), 0)
	}

	contact, err := Get(c, list.Data[0].Id)
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id != list.Data[0].Id {
		t.Errorf("Unexpected contact.Id, got %#v, expected %#v", contact.Id, list.Data[0].Id)
	}

}

func TestCreate(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "John",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	updated, err := Update(c, contact.Id, &ticketmatic.Contact{
		Lastname: "Doe",
	})
	if err != nil {
		t.Fatal(err)
	}

	if updated.Id != contact.Id {
		t.Errorf("Unexpected updated.Id, got %#v, expected %#v", updated.Id, contact.Id)
	}

	if updated.Firstname != "John" {
		t.Errorf("Unexpected updated.Firstname, got %#v, expected %#v", updated.Firstname, "John")
	}

	if updated.Lastname != "Doe" {
		t.Errorf("Unexpected updated.Lastname, got %#v, expected %#v", updated.Lastname, "Doe")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreatecustom(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	titles, err := contacttitles.Getlist(c, &ticketmatic.ContactTitleQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(titles.Data) <= 0 {
		t.Errorf("Unexpected titles.Data length, got %#v, expected greater than %#v", len(titles.Data), 0)
	}

	addrtypes, err := contactaddresstypes.Getlist(c, &ticketmatic.ContactAddressTypeQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(addrtypes.Data) <= 0 {
		t.Errorf("Unexpected addrtypes.Data length, got %#v, expected greater than %#v", len(addrtypes.Data), 0)
	}

	ptypes, err := phonenumbertypes.Getlist(c, &ticketmatic.PhoneNumberTypeQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(ptypes.Data) <= 1 {
		t.Errorf("Unexpected ptypes.Data length, got %#v, expected greater than %#v", len(ptypes.Data), 1)
	}

	contact, err := Create(c, &ticketmatic.Contact{
		Addresses: []*ticketmatic.Address{
			&ticketmatic.Address{
				City:        "Nieuwerkerk Aan Den Ijssel",
				Countrycode: "NL",
				Street1:     "Kerkstraat",
				Street2:     "1",
				Typeid:      addrtypes.Data[0].Id,
				Zip:         "2914 AH",
			},
		},
		Birthdate:       ticketmatic.NewTime(ticketmatic.MustParseTime("1959-09-21")),
		Customertitleid: titles.Data[0].Id,
		Email:           "john@worldonline.nl",
		Firstname:       "John",
		Lastname:        "Johns",
		Middlename:      "J",
		Phonenumbers: []*ticketmatic.Phonenumber{
			&ticketmatic.Phonenumber{
				Number: "+31222222222",
				Typeid: ptypes.Data[0].Id,
			},
			&ticketmatic.Phonenumber{
				Number: "+31222222222",
				Typeid: ptypes.Data[1].Id,
			},
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	if contact.Addresses[0].Countrycode != "NL" {
		t.Errorf("Unexpected contact.Addresses[0].Countrycode, got %#v, expected %#v", contact.Addresses[0].Countrycode, "NL")
	}

	if contact.Addresses[0].Country != "Netherlands" {
		t.Errorf("Unexpected contact.Addresses[0].Country, got %#v, expected %#v", contact.Addresses[0].Country, "Netherlands")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreateunicode(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "JØhñ",
		Lastname:  "ポテト 👌 ไก่",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "JØhñ" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "JØhñ")
	}

	if contact.Lastname != "ポテト 👌 ไก่" {
		t.Errorf("Unexpected contact.Lastname, got %#v, expected %#v", contact.Lastname, "ポテト 👌 ไก่")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestArchived(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "John",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	req, err := Getlist(c, &ticketmatic.ContactQuery{
		Includearchived: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) <= 0 {
		t.Errorf("Unexpected req.Data length, got %#v, expected greater than %#v", len(req.Data), 0)
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

	req2, err := Getlist(c, &ticketmatic.ContactQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) <= len(req2.Data) {
		t.Errorf("Unexpected req.Data length, got %#v, expected greater than %#v", len(req.Data), len(req2.Data))
	}

	req3, err := Getlist(c, &ticketmatic.ContactQuery{
		Includearchived: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) != len(req3.Data) {
		t.Errorf("Unexpected req.Data length, got %#v, expected %#v", len(req.Data), len(req3.Data))
	}

}
<commit_msg>Extend contact test<commit_after>package contacts

import (
	"os"
	"testing"

	"github.com/ticketmatic/tm-go/ticketmatic"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/contactaddresstypes"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/contacttitles"
	"github.com/ticketmatic/tm-go/ticketmatic/settings/system/phonenumbertypes"
)

func TestGet(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	list, err := Getlist(c, &ticketmatic.ContactQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(list.Data) <= 0 {
		t.Errorf("Unexpected list.Data length, got %#v, expected greater than %#v", len(list.Data), 0)
	}

	contact, err := Get(c, list.Data[0].Id)
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id != list.Data[0].Id {
		t.Errorf("Unexpected contact.Id, got %#v, expected %#v", contact.Id, list.Data[0].Id)
	}

}

func TestCreate(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "John",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	updated, err := Update(c, contact.Id, &ticketmatic.Contact{
		Lastname: "Doe",
	})
	if err != nil {
		t.Fatal(err)
	}

	if updated.Id != contact.Id {
		t.Errorf("Unexpected updated.Id, got %#v, expected %#v", updated.Id, contact.Id)
	}

	if updated.Firstname != "John" {
		t.Errorf("Unexpected updated.Firstname, got %#v, expected %#v", updated.Firstname, "John")
	}

	if updated.Lastname != "Doe" {
		t.Errorf("Unexpected updated.Lastname, got %#v, expected %#v", updated.Lastname, "Doe")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreatecustom(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	titles, err := contacttitles.Getlist(c, &ticketmatic.ContactTitleQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(titles.Data) <= 0 {
		t.Errorf("Unexpected titles.Data length, got %#v, expected greater than %#v", len(titles.Data), 0)
	}

	addrtypes, err := contactaddresstypes.Getlist(c, &ticketmatic.ContactAddressTypeQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(addrtypes.Data) <= 0 {
		t.Errorf("Unexpected addrtypes.Data length, got %#v, expected greater than %#v", len(addrtypes.Data), 0)
	}

	ptypes, err := phonenumbertypes.Getlist(c, &ticketmatic.PhoneNumberTypeQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(ptypes.Data) <= 1 {
		t.Errorf("Unexpected ptypes.Data length, got %#v, expected greater than %#v", len(ptypes.Data), 1)
	}

	contact, err := Create(c, &ticketmatic.Contact{
		Addresses: []*ticketmatic.Address{
			&ticketmatic.Address{
				City:        "Nieuwerkerk Aan Den Ijssel",
				Countrycode: "NL",
				Street1:     "Kerkstraat",
				Street2:     "1",
				Typeid:      addrtypes.Data[0].Id,
				Zip:         "2914 AH",
			},
		},
		Birthdate:       ticketmatic.NewTime(ticketmatic.MustParseTime("1959-09-21")),
		Customertitleid: titles.Data[0].Id,
		Email:           "john@worldonline.nl",
		Firstname:       "John",
		Lastname:        "Johns",
		Middlename:      "J",
		Phonenumbers: []*ticketmatic.Phonenumber{
			&ticketmatic.Phonenumber{
				Number: "+31222222222",
				Typeid: ptypes.Data[0].Id,
			},
			&ticketmatic.Phonenumber{
				Number: "+31222222222",
				Typeid: ptypes.Data[1].Id,
			},
		},
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	if contact.Addresses[0].Countrycode != "NL" {
		t.Errorf("Unexpected contact.Addresses[0].Countrycode, got %#v, expected %#v", contact.Addresses[0].Countrycode, "NL")
	}

	if contact.Addresses[0].Country != "Netherlands" {
		t.Errorf("Unexpected contact.Addresses[0].Country, got %#v, expected %#v", contact.Addresses[0].Country, "Netherlands")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreateunicode(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "JØhñ",
		Lastname:  "ポテト 👌 ไก่",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "JØhñ" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "JØhñ")
	}

	if contact.Lastname != "ポテト 👌 ไก่" {
		t.Errorf("Unexpected contact.Lastname, got %#v, expected %#v", contact.Lastname, "ポテト 👌 ไก่")
	}

	contact2, err := Get(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

	if contact2.Id == 0 {
		t.Errorf("Unexpected contact2.Id, got %#v, expected different value", contact2.Id)
	}

	if contact2.Firstname != "JØhñ" {
		t.Errorf("Unexpected contact2.Firstname, got %#v, expected %#v", contact2.Firstname, "JØhñ")
	}

	if contact2.Lastname != "ポテト 👌 ไก่" {
		t.Errorf("Unexpected contact2.Lastname, got %#v, expected %#v", contact2.Lastname, "ポテト 👌 ไก่")
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

}

func TestArchived(t *testing.T) {
	var err error

	accountcode := os.Getenv("TM_TEST_ACCOUNTCODE")
	accesskey := os.Getenv("TM_TEST_ACCESSKEY")
	secretkey := os.Getenv("TM_TEST_SECRETKEY")
	c := ticketmatic.NewClient(accountcode, accesskey, secretkey)

	contact, err := Create(c, &ticketmatic.Contact{
		Firstname: "John",
	})
	if err != nil {
		t.Fatal(err)
	}

	if contact.Id == 0 {
		t.Errorf("Unexpected contact.Id, got %#v, expected different value", contact.Id)
	}

	if contact.Firstname != "John" {
		t.Errorf("Unexpected contact.Firstname, got %#v, expected %#v", contact.Firstname, "John")
	}

	req, err := Getlist(c, &ticketmatic.ContactQuery{
		Includearchived: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) <= 0 {
		t.Errorf("Unexpected req.Data length, got %#v, expected greater than %#v", len(req.Data), 0)
	}

	err = Delete(c, contact.Id)
	if err != nil {
		t.Fatal(err)
	}

	req2, err := Getlist(c, &ticketmatic.ContactQuery{})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) <= len(req2.Data) {
		t.Errorf("Unexpected req.Data length, got %#v, expected greater than %#v", len(req.Data), len(req2.Data))
	}

	req3, err := Getlist(c, &ticketmatic.ContactQuery{
		Includearchived: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(req.Data) != len(req3.Data) {
		t.Errorf("Unexpected req.Data length, got %#v, expected %#v", len(req.Data), len(req3.Data))
	}

}
<|endoftext|>


<commit_before>// Package storages allows multiple implementation on how to store short URLs.
package storages

type Storage interface {
	// Hash(string) returns the short URL that would be generated from a long URL
	Code(string) string
	// Save(string) takes a full URL and returns the short URL after saving it to storage
	Save(string) (string, error)
	// Load(string) takes a short URL and returns the original full URL by retrieving it from storage
	Load(string) (string, error)
}
<commit_msg>Storge naming doesn't have to be deterministic<commit_after>// Package storages allows multiple implementation on how to store short URLs.
package storages

type Storage interface {
	// Save(string) takes a full URL and returns the short URL after saving it to storage
	Save(string) (string, error)
	// Load(string) takes a short URL and returns the original full URL by retrieving it from storage
	Load(string) (string, error)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"

	"github.com/Shopify/sarama"
	"github.com/golang/glog"
)

const HTTP_FORM_ENCODING = "application/x-www-form-urlencoded"

type HTTPTransporter struct {
	*ConcurrencyStartStopControl
	Callback    *WorkerCallback
	Serializer  string
	ContentType string

	// config
	config            *CallbackItemConfig
	transporterConfig TransporterConfig

	// http context
	discardBuffer []byte
	httpClient    *http.Client

	// parent
	manager *PartitionManager

	// stat variables
	consumed       uint64 // consumed message count
	delivered      uint64 // success delivered message count
	skipped        uint64 // skipped message count
	requests       uint64 // total issued http requests count
	netFailures    uint64 // net failed http requests
	serverFailures uint64 // server failed http requests
	startTime      time.Time

	// stat tracker variables, previously /stat/tracker interface required variable
	lastRecordTimestamp    int64
	currentRecordTimestamp int64
	lastLogId              string
	lastOffset             int64
}

type MessageBody struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

func NewHTTPTransporter() Transporter {
	return &HTTPTransporter{
		ConcurrencyStartStopControl: NewConcurrencyStartStopControl(),
	}
}

func (ht *HTTPTransporter) Init(config *CallbackItemConfig, transporterConfig TransporterConfig, manager *PartitionManager) error {
	ht.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	ht.Serializer = config.Serializer
	ht.ContentType = config.ContentType
	ht.transporterConfig = transporterConfig
	ht.discardBuffer = make([]byte, 4096)
	ht.manager = manager

	// build http client
	ht.httpClient = &http.Client{Transport: GetServer().GetHttpTransport()}
	ht.httpClient.Timeout = ht.Callback.Timeout

	return nil
}

func (ht *HTTPTransporter) ResetStat() {
	// stat variable
	atomic.StoreUint64(&ht.consumed, 0)
	atomic.StoreUint64(&ht.delivered, 0)
	atomic.StoreUint64(&ht.skipped, 0)
	atomic.StoreUint64(&ht.requests, 0)
	atomic.StoreUint64(&ht.netFailures, 0)
	atomic.StoreUint64(&ht.serverFailures, 0)
	ht.startTime = time.Now().Local()

	// stat tracker variable
	ht.lastRecordTimestamp = -1
	ht.currentRecordTimestamp = -1
	ht.lastLogId = ""
	ht.lastOffset = -1
}

func (ht *HTTPTransporter) Run() error {
	ht.markStart()
	defer ht.markStop()

	arbiter := ht.manager.GetArbiter()
	messages := arbiter.MessageChannel()
	offsets := arbiter.OffsetChannel()

transporterLoop:
	for {
		select {
		case message := <-messages:
			glog.V(1).Infof("Recevied message [topic:%s][partition:%d][url:%s][offset:%d]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset)
			atomic.AddUint64(&ht.consumed, 1)
			ht.processMessage(message, offsets)
		case <-ht.WaitForCloseChannel():
			break transporterLoop
		}

	}

	glog.V(1).Infof("HTTPTransporter exited [topic:%s][partition:%d][url:%s]", ht.manager.Topic, ht.manager.Partition, ht.Callback.Url)

	return nil
}

func (ht *HTTPTransporter) processMessage(message *sarama.ConsumerMessage, offsets chan<- int64) {
	var messageData MessageBody

	// deserialize message
	switch ht.Serializer {
	case "", "raw":
		messageData.Data = string(message.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(message.Value, &messageData)
		// ignore message json decode failure
	}

	// delivery Content-Type
	if "" != ht.ContentType {
		messageData.ContentType = ht.ContentType
	} else if "" == ht.ContentType {
		ht.ContentType = HTTP_FORM_ENCODING
	}

	rpcStartTime := time.Now()
	retried := 0
	deliverySuccess := false

	for {
		for i := 0; i <= ht.Callback.RetryTimes; i++ {
			atomic.AddUint64(&ht.requests, 1)
			deliverySuccess = ht.delivery(&messageData, message, retried)

			if deliverySuccess {
				// success
				break
			}

			retried++
		}

		if deliverySuccess {
			// success
			atomic.AddUint64(&ht.delivered, 1)
			break
		} else if ht.Callback.BypassFailed {
			// failed
			atomic.AddUint64(&ht.skipped, 1)
			glog.Errorf(
				"Message skipped due to delivery retryTimes exceeded [topic:%s][partition:%d][url:%s][offset:%d][retryTimes:%d][bypassFailed:%t]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset, ht.Callback.RetryTimes, ht.Callback.BypassFailed)
			break
		}

		glog.Errorf(
			"Retry delivery after %s due to delivery retryTime exceeded [topic:%s][partition:%d][url:%s][offset:%d][retryTimes:%d][bypassFailed:%t][failedSleep:%.2fms]",
			ht.Callback.FailedSleep.String(), message.Topic, message.Partition, ht.Callback.Url, message.Offset, ht.Callback.RetryTimes, ht.Callback.BypassFailed,
			ht.Callback.FailedSleep.Seconds()*1000)

		// wait for FailedSleep times for another retry round
		time.Sleep(ht.Callback.FailedSleep)
	}

	rpcStopTime := time.Now()

	// total time from proxy to pusher complete sending
	totalTime := float64(-1)
	if ht.Serializer == "json" {
		totalTime = float64(rpcStopTime.UnixNano()/1000000 - messageData.TimeStamp)

		// record tracker status if success
		if deliverySuccess && messageData.TimeStamp > ht.currentRecordTimestamp {
			ht.lastRecordTimestamp, ht.currentRecordTimestamp = ht.currentRecordTimestamp, messageData.TimeStamp
			ht.lastLogId = messageData.LogId
			ht.lastOffset = message.Offset
		}
	}

	glog.Infof("Committed message [topic:%s][partition:%d][url:%s][offset:%d][skipped:%t][cost:%.2fms][totalCost:%.2fms][retried:%d][payloadLength:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset, !deliverySuccess,
		rpcStopTime.Sub(rpcStartTime).Seconds()*1000,
		totalTime, retried, len(messageData.Data))

	glog.V(1).Infof("HTTP Transporter commit message to arbiter [topic:%s][partition:%d][url:%s][offset:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset)

	offsets <- message.Offset

	glog.V(1).Infof("HTTP Transporter processed message [topic:%s][partition:%d][url:%s][offset:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset)
}

func (ht *HTTPTransporter) delivery(messageData *MessageBody, message *sarama.ConsumerMessage, retryTime int) bool {
	req, _ := http.NewRequest("POST", ht.Callback.Url, strings.NewReader(messageData.Data))
	req.Header.Set("Content-Type", messageData.ContentType)
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retryTime))
	req.Header.Set("X-Kmq-Topic", message.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", message.Partition))
	req.Header.Set("X-Kmq-Partition-Key", messageData.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", message.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", messageData.LogId))
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", messageData.TimeStamp))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")

	rpcStartTime := time.Now()
	res, err := ht.httpClient.Do(req)
	rpcStopTime := time.Now()

	rpcTime := rpcStopTime.Sub(rpcStartTime).Seconds() * 1000

	success := false

	if err == nil {
		defer res.Body.Close()

		if 200 == res.StatusCode {
			// success
			success = true
			// discard body
			// better use io.Copy(ioutil.Discard, res.Body), but io.Copy/ioutil.Discard is too slow
			for {
				_, e := res.Body.Read(ht.discardBuffer)
				if e != nil {
					break
				}
			}
		} else {
			// error response code, read body
			responseBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				responseBody = []byte{}
			}
			atomic.AddUint64(&ht.serverFailures, 1)
			glog.Errorf(
				"Delivery failed [topic:%s][partition:%d][url:%s][offset:%d][retryTime:%d][responseCode:%d][cost:%.2fms][responseBody:%s]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset, retryTime, res.StatusCode, rpcTime,
				url.QueryEscape(string(responseBody)))
		}
	} else {
		atomic.AddUint64(&ht.netFailures, 1)
		glog.Errorf(
			"Delivery failed [topic:%s][partition:%d][url:%s][offset:%d][retryTime:%d][cost:%.2fms][err:%s]",
			message.Topic, message.Partition, ht.Callback.Url, message.Offset, retryTime, rpcTime, err.Error())
	}

	return success
}

func (ht *HTTPTransporter) GetStat() interface{} {
	result := make(map[string]interface{})

	// stat
	result["consumed"] = atomic.LoadUint64(&ht.consumed)
	result["delivered"] = atomic.LoadUint64(&ht.delivered)
	result["skipped"] = atomic.LoadUint64(&ht.skipped)
	result["requests"] = atomic.LoadUint64(&ht.requests)
	result["net_failure_requests"] = atomic.LoadUint64(&ht.netFailures)
	result["server_failure_requests"] = atomic.LoadUint64(&ht.serverFailures)
	result["start_time"] = ht.startTime

	// stat tracker
	result["last_record_op_time"] = ht.lastRecordTimestamp
	result["current_record_op_time"] = ht.currentRecordTimestamp
	result["last_logid"] = ht.lastLogId
	result["last_offset"] = ht.lastOffset

	return result
}

func init() {
	RegisterTransporter("HTTP", NewHTTPTransporter)
}
<commit_msg>Fix content-type assignment typo in http_transporter.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"

	"github.com/Shopify/sarama"
	"github.com/golang/glog"
)

const HTTP_FORM_ENCODING = "application/x-www-form-urlencoded"

type HTTPTransporter struct {
	*ConcurrencyStartStopControl
	Callback    *WorkerCallback
	Serializer  string
	ContentType string

	// config
	config            *CallbackItemConfig
	transporterConfig TransporterConfig

	// http context
	discardBuffer []byte
	httpClient    *http.Client

	// parent
	manager *PartitionManager

	// stat variables
	consumed       uint64 // consumed message count
	delivered      uint64 // success delivered message count
	skipped        uint64 // skipped message count
	requests       uint64 // total issued http requests count
	netFailures    uint64 // net failed http requests
	serverFailures uint64 // server failed http requests
	startTime      time.Time

	// stat tracker variables, previously /stat/tracker interface required variable
	lastRecordTimestamp    int64
	currentRecordTimestamp int64
	lastLogId              string
	lastOffset             int64
}

type MessageBody struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

func NewHTTPTransporter() Transporter {
	return &HTTPTransporter{
		ConcurrencyStartStopControl: NewConcurrencyStartStopControl(),
	}
}

func (ht *HTTPTransporter) Init(config *CallbackItemConfig, transporterConfig TransporterConfig, manager *PartitionManager) error {
	ht.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	ht.Serializer = config.Serializer
	ht.ContentType = config.ContentType
	ht.transporterConfig = transporterConfig
	ht.discardBuffer = make([]byte, 4096)
	ht.manager = manager

	// build http client
	ht.httpClient = &http.Client{Transport: GetServer().GetHttpTransport()}
	ht.httpClient.Timeout = ht.Callback.Timeout

	return nil
}

func (ht *HTTPTransporter) ResetStat() {
	// stat variable
	atomic.StoreUint64(&ht.consumed, 0)
	atomic.StoreUint64(&ht.delivered, 0)
	atomic.StoreUint64(&ht.skipped, 0)
	atomic.StoreUint64(&ht.requests, 0)
	atomic.StoreUint64(&ht.netFailures, 0)
	atomic.StoreUint64(&ht.serverFailures, 0)
	ht.startTime = time.Now().Local()

	// stat tracker variable
	ht.lastRecordTimestamp = -1
	ht.currentRecordTimestamp = -1
	ht.lastLogId = ""
	ht.lastOffset = -1
}

func (ht *HTTPTransporter) Run() error {
	ht.markStart()
	defer ht.markStop()

	arbiter := ht.manager.GetArbiter()
	messages := arbiter.MessageChannel()
	offsets := arbiter.OffsetChannel()

transporterLoop:
	for {
		select {
		case message := <-messages:
			glog.V(1).Infof("Recevied message [topic:%s][partition:%d][url:%s][offset:%d]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset)
			atomic.AddUint64(&ht.consumed, 1)
			ht.processMessage(message, offsets)
		case <-ht.WaitForCloseChannel():
			break transporterLoop
		}

	}

	glog.V(1).Infof("HTTPTransporter exited [topic:%s][partition:%d][url:%s]", ht.manager.Topic, ht.manager.Partition, ht.Callback.Url)

	return nil
}

func (ht *HTTPTransporter) processMessage(message *sarama.ConsumerMessage, offsets chan<- int64) {
	var messageData MessageBody

	// deserialize message
	switch ht.Serializer {
	case "", "raw":
		messageData.Data = string(message.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(message.Value, &messageData)
		// ignore message json decode failure
	}

	// delivery Content-Type
	if ht.ContentType != "" {
		messageData.ContentType = ht.ContentType
	} else if messageData.ContentType == "" {
		messageData.ContentType = HTTP_FORM_ENCODING
	}

	rpcStartTime := time.Now()
	retried := 0
	deliverySuccess := false

	for {
		for i := 0; i <= ht.Callback.RetryTimes; i++ {
			atomic.AddUint64(&ht.requests, 1)
			deliverySuccess = ht.delivery(&messageData, message, retried)

			if deliverySuccess {
				// success
				break
			}

			retried++
		}

		if deliverySuccess {
			// success
			atomic.AddUint64(&ht.delivered, 1)
			break
		} else if ht.Callback.BypassFailed {
			// failed
			atomic.AddUint64(&ht.skipped, 1)
			glog.Errorf(
				"Message skipped due to delivery retryTimes exceeded [topic:%s][partition:%d][url:%s][offset:%d][retryTimes:%d][bypassFailed:%t]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset, ht.Callback.RetryTimes, ht.Callback.BypassFailed)
			break
		}

		glog.Errorf(
			"Retry delivery after %s due to delivery retryTime exceeded [topic:%s][partition:%d][url:%s][offset:%d][retryTimes:%d][bypassFailed:%t][failedSleep:%.2fms]",
			ht.Callback.FailedSleep.String(), message.Topic, message.Partition, ht.Callback.Url, message.Offset, ht.Callback.RetryTimes, ht.Callback.BypassFailed,
			ht.Callback.FailedSleep.Seconds()*1000)

		// wait for FailedSleep times for another retry round
		time.Sleep(ht.Callback.FailedSleep)
	}

	rpcStopTime := time.Now()

	// total time from proxy to pusher complete sending
	totalTime := float64(-1)
	if ht.Serializer == "json" {
		totalTime = float64(rpcStopTime.UnixNano()/1000000 - messageData.TimeStamp)

		// record tracker status if success
		if deliverySuccess && messageData.TimeStamp > ht.currentRecordTimestamp {
			ht.lastRecordTimestamp, ht.currentRecordTimestamp = ht.currentRecordTimestamp, messageData.TimeStamp
			ht.lastLogId = messageData.LogId
			ht.lastOffset = message.Offset
		}
	}

	glog.Infof("Committed message [topic:%s][partition:%d][url:%s][offset:%d][skipped:%t][cost:%.2fms][totalCost:%.2fms][retried:%d][payloadLength:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset, !deliverySuccess,
		rpcStopTime.Sub(rpcStartTime).Seconds()*1000,
		totalTime, retried, len(messageData.Data))

	glog.V(1).Infof("HTTP Transporter commit message to arbiter [topic:%s][partition:%d][url:%s][offset:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset)

	offsets <- message.Offset

	glog.V(1).Infof("HTTP Transporter processed message [topic:%s][partition:%d][url:%s][offset:%d]",
		message.Topic, message.Partition, ht.Callback.Url, message.Offset)
}

func (ht *HTTPTransporter) delivery(messageData *MessageBody, message *sarama.ConsumerMessage, retryTime int) bool {
	req, _ := http.NewRequest("POST", ht.Callback.Url, strings.NewReader(messageData.Data))
	req.Header.Set("Content-Type", messageData.ContentType)
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retryTime))
	req.Header.Set("X-Kmq-Topic", message.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", message.Partition))
	req.Header.Set("X-Kmq-Partition-Key", messageData.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", message.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", messageData.LogId))
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", messageData.TimeStamp))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")

	rpcStartTime := time.Now()
	res, err := ht.httpClient.Do(req)
	rpcStopTime := time.Now()

	rpcTime := rpcStopTime.Sub(rpcStartTime).Seconds() * 1000

	success := false

	if err == nil {
		defer res.Body.Close()

		if 200 == res.StatusCode {
			// success
			success = true
			// discard body
			// better use io.Copy(ioutil.Discard, res.Body), but io.Copy/ioutil.Discard is too slow
			for {
				_, e := res.Body.Read(ht.discardBuffer)
				if e != nil {
					break
				}
			}
		} else {
			// error response code, read body
			responseBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				responseBody = []byte{}
			}
			atomic.AddUint64(&ht.serverFailures, 1)
			glog.Errorf(
				"Delivery failed [topic:%s][partition:%d][url:%s][offset:%d][retryTime:%d][responseCode:%d][cost:%.2fms][responseBody:%s]",
				message.Topic, message.Partition, ht.Callback.Url, message.Offset, retryTime, res.StatusCode, rpcTime,
				url.QueryEscape(string(responseBody)))
		}
	} else {
		atomic.AddUint64(&ht.netFailures, 1)
		glog.Errorf(
			"Delivery failed [topic:%s][partition:%d][url:%s][offset:%d][retryTime:%d][cost:%.2fms][err:%s]",
			message.Topic, message.Partition, ht.Callback.Url, message.Offset, retryTime, rpcTime, err.Error())
	}

	return success
}

func (ht *HTTPTransporter) GetStat() interface{} {
	result := make(map[string]interface{})

	// stat
	result["consumed"] = atomic.LoadUint64(&ht.consumed)
	result["delivered"] = atomic.LoadUint64(&ht.delivered)
	result["skipped"] = atomic.LoadUint64(&ht.skipped)
	result["requests"] = atomic.LoadUint64(&ht.requests)
	result["net_failure_requests"] = atomic.LoadUint64(&ht.netFailures)
	result["server_failure_requests"] = atomic.LoadUint64(&ht.serverFailures)
	result["start_time"] = ht.startTime

	// stat tracker
	result["last_record_op_time"] = ht.lastRecordTimestamp
	result["current_record_op_time"] = ht.currentRecordTimestamp
	result["last_logid"] = ht.lastLogId
	result["last_offset"] = ht.lastOffset

	return result
}

func init() {
	RegisterTransporter("HTTP", NewHTTPTransporter)
}
<|endoftext|>


<commit_before>package handlers

import (
	"net/http"
	"os"
)

func contact(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		templates["contact"].ExecuteTemplate(w, "layout", reCaptchaSiteKey)

	case "POST":
		senderName := r.PostFormValue("name")
		senderEmail := r.PostFormValue("email")
		message := r.PostFormValue("message")
		var response string

			err := contactHackSoc(senderName, senderEmail, message)

			if err == nil {
				response = "Your message has been received."
			} else {
				response = err.Error()
			}
		} else {
			response = "Turing test failed. Please try again."
		}

		templates["message"].ExecuteTemplate(w, "layout", messageModel{"Contact", response})

	default:
		errorHandler(w, r, http.StatusBadRequest)
	}
}

func contactHackSoc(senderName, senderEmail, message string) error {
	if !validator.IsValidName(senderName) {
		senderName = "Anonymous"
	}

	if senderEmail == "" {
		senderEmail = os.Getenv("NOREPLY_EMAIL")
	}

	receiverName := "HackSoc"
	receiverEmail := os.Getenv("CONTACT_EMAIL")
	subject := "Contact Form Message"

	return emailService.Send(senderName, senderEmail, receiverName, receiverEmail, subject, message)
}
<commit_msg>Updated contact.go<commit_after>package handlers

import (
	"github.com/hacksoc-manchester/www/helpers/validator"
	"github.com/hacksoc-manchester/www/services/emailService"
	"net/http"
	"os"
)

func contact(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		templates["contact"].ExecuteTemplate(w, "layout", reCaptchaSiteKey)

	case "POST":
		senderName := r.PostFormValue("name")
		senderEmail := r.PostFormValue("email")
		message := r.PostFormValue("message")
		var response string

		if reCaptcha.Verify(*r) {
			err := contactHackSoc(senderName, senderEmail, message)

			if err == nil {
				response = "Your message has been received."
			} else {
				response = err.Error()
			}
		} else {
			response = "Turing test failed. Please try again."
		}

		templates["message"].ExecuteTemplate(w, "layout", messageModel{"Contact", response})

	default:
		errorHandler(w, r, http.StatusBadRequest)
	}
}

func contactHackSoc(senderName, senderEmail, message string) error {
	if !validator.IsValidName(senderName) {
		senderName = "Anonymous"
	}

	if senderEmail == "" {
		senderEmail = os.Getenv("NOREPLY_EMAIL")
	}

	receiverName := "HackSoc"
	receiverEmail := os.Getenv("CONTACT_EMAIL")
	subject := "Contact Form Message"

	return emailService.Send(senderName, senderEmail, receiverName, receiverEmail, subject, message)
}
<|endoftext|>


<commit_before>package image

import (
	"bytes"
	"image"
	"sort"
	"sync"

	"github.com/jonathanpittman/tiff"
)

type Handler interface {
	Process(*tiff.TIFF) (image.Image, error)
	GetConfig(*tiff.TIFF) (image.Config, error)
	CanHandle(*tiff.TIFF) bool
}

var (
	// Handlers based on the value in the Make tag (tag id 271)
	altHandlersFromMakeTagValue = struct {
		mu            sync.RWMutex
		makeToHandler map[string]Handler
	}{
		makeToHandler: make(map[string]Handler, 1),
	}

	// Handlers based on the presence of a tag id in IFD0.
	altHandlersFromTagPresence = struct {
		mu           sync.RWMutex
		tagToHandler map[uint16]Handler
	}{
		tagToHandler: make(map[uint16]Handler, 1),
	}
)

// For now, findAlternates will only check against two concepts.  One, is the
// "Make" tag and the other checks the presence of tags.
func findAlternates(t *tiff.TIFF) Handler {
	ifd0 := t.IFDs[0]
	// Do tag presence check first.  This is useful for identifying
	// tiff files that conform to a certain specification that uses
	// a tag to identify that specification.  For example, tiff/ep
	// uses the "TIFF/EPStandardID" tag (id 37398) to indicate the
	// version number of tiff/ep in use for this tiff file.  DNG has
	// the "DNGVersion" tag (id 50706) for the same.  And Leaf .MOS
	// files do not indicate make/model, but they do have a custom
	// private tag that they use for certain data (tag id 34310).
	// The idea here is that a tiff/ep package or dng package or
	// leaf mos package would be able to better handle processing
	// the tiff than the generic package.
	for _, tagID := range ListRegisteredTagPresenceIDs() {
		if ifd0.HasField(tagID) {
			hndlr := GetHandlerByTagPresence(tagID)
			if hndlr != nil && hndlr.CanHandle(t) {
				return hndlr
			}
		}
	}

	// Check for a specific make next
	// Tag ID 271 is a baseline tag for the "Make" or maker or
	// manufacturer of a device that created this tiff.  The idea
	// here is that a separate manufacturer package may be more
	// directly aware of how to process a specific tiff as opposed
	// to playing guessing games in a generic package.
	if ifd0.HasField(271) {
		f := ifd0.GetField(271)
		maker := string(bytes.TrimRight(f.Value().Bytes(), " \x00"))
		hndlr := GetHandlerByMake(maker)
		if hndlr != nil && hndlr.CanHandle(t) {
			return hndlr
		}
	}
	return nil
}

func RegisterHandlerByMake(m string, h Handler) {
	altHandlersFromMakeTagValue.mu.Lock()
	defer altHandlersFromMakeTagValue.mu.Unlock()
	altHandlersFromMakeTagValue.makeToHandler[m] = h
}

func GetHandlerByMake(m string) Handler {
	altHandlersFromMakeTagValue.mu.RLock()
	defer altHandlersFromMakeTagValue.mu.RUnlock()
	return altHandlersFromMakeTagValue.makeToHandler[m]
}

func RegisterHandlerByTagPresence(t uint16, h Handler) {
	altHandlersFromTagPresence.mu.Lock()
	defer altHandlersFromTagPresence.mu.Unlock()
	altHandlersFromTagPresence.tagToHandler[t] = h
}

func GetHandlerByTagPresence(t uint16) Handler {
	altHandlersFromTagPresence.mu.RLock()
	defer altHandlersFromTagPresence.mu.RUnlock()
	return altHandlersFromTagPresence.tagToHandler[t]
}

func ListRegisteredTagPresenceIDs() []uint16 {
	altHandlersFromTagPresence.mu.RLock()
	defer altHandlersFromTagPresence.mu.RUnlock()
	ids := make([]uint16, 0, len(altHandlersFromTagPresence.tagToHandler))
	for k := range altHandlersFromTagPresence.tagToHandler {
		ids = append(ids, k)
	}
	sort.Sort(uint16Slice(ids))
	return ids
}
<commit_msg>Change to alternate handling to specify what it is handling<commit_after>package image

import (
	"bytes"
	"sort"
	"sync"

	"github.com/jonathanpittman/tiff"
)

var (
	// Handlers based on the value in the Make tag (tag id 271)
	// If a manufacturer specific package exists and is imported, one would
	// hope it would know best how to handle decoding the tiff.
	altTIFFHandlersFromMakeTagValue = struct {
		mu            sync.RWMutex
		makeToHandler map[string]TIFFHandler
	}{
		makeToHandler: make(map[string]TIFFHandler, 1),
	}

	// Handlers based on the presence of a tag id in IFD0.
	altTIFFHandlersFromTagPresence = struct {
		mu           sync.RWMutex
		tagToHandler map[uint16]TIFFHandler
	}{
		tagToHandler: make(map[uint16]TIFFHandler, 1),
	}
)

func findAlternateIFDHandler(ifd tiff.IFD) IFDHandler {
	return nil
}

// For now, findAlternates will only check against two concepts.  One, is the
// "Make" tag and the other checks the presence of tags.
func findAlternateTIFFHandler(t tiff.TIFF) TIFFHandler {
	ifd0 := t.IFDs()[0]
	// Do tag presence check first.  This is useful for identifying
	// tiff files that conform to a certain specification that uses
	// a tag to identify that specification.  For example, tiff/ep
	// uses the "TIFF/EPStandardID" tag (id 37398) to indicate the
	// version number of tiff/ep in use for this tiff file.  DNG has
	// the "DNGVersion" tag (id 50706) for the same.  And Leaf .MOS
	// files do not indicate make/model, but they do have a custom
	// private tag that they use for certain data (tag id 34310).
	// The idea here is that a tiff/ep package or dng package or
	// leaf mos package would be able to better handle processing
	// the tiff than the generic package.
	for _, tagID := range ListRegisteredTagPresenceIDs() {
		if ifd0.HasField(tagID) {
			hndlr := GetHandlerByTagPresence(tagID)
			if hndlr != nil && hndlr.CanHandle(t) {
				return hndlr
			}
		}
	}

	// Check for a specific make next
	// Tag ID 271 is a baseline tag for the "Make" or maker or
	// manufacturer of a device that created this tiff.  The idea
	// here is that a separate manufacturer package may be more
	// directly aware of how to process a specific tiff as opposed
	// to playing guessing games in a generic package.
	if ifd0.HasField(271) {
		f := ifd0.GetField(271)
		maker := string(bytes.TrimRight(f.Value().Bytes(), " \x00"))
		hndlr := GetHandlerByMake(maker)
		if hndlr != nil && hndlr.CanHandle(t) {
			return hndlr
		}
	}
	return nil
}

func RegisterHandlerByMake(m string, h TIFFHandler) {
	altTIFFHandlersFromMakeTagValue.mu.Lock()
	defer altTIFFHandlersFromMakeTagValue.mu.Unlock()
	altTIFFHandlersFromMakeTagValue.makeToHandler[m] = h
}

func GetHandlerByMake(m string) TIFFHandler {
	altTIFFHandlersFromMakeTagValue.mu.RLock()
	defer altTIFFHandlersFromMakeTagValue.mu.RUnlock()
	return altTIFFHandlersFromMakeTagValue.makeToHandler[m]
}

func RegisterHandlerByTagPresence(t uint16, h TIFFHandler) {
	altTIFFHandlersFromTagPresence.mu.Lock()
	defer altTIFFHandlersFromTagPresence.mu.Unlock()
	altTIFFHandlersFromTagPresence.tagToHandler[t] = h
}

func GetHandlerByTagPresence(t uint16) TIFFHandler {
	altTIFFHandlersFromTagPresence.mu.RLock()
	defer altTIFFHandlersFromTagPresence.mu.RUnlock()
	return altTIFFHandlersFromTagPresence.tagToHandler[t]
}

func ListRegisteredTagPresenceIDs() []uint16 {
	altTIFFHandlersFromTagPresence.mu.RLock()
	defer altTIFFHandlersFromTagPresence.mu.RUnlock()
	ids := make([]uint16, 0, len(altTIFFHandlersFromTagPresence.tagToHandler))
	for k := range altTIFFHandlersFromTagPresence.tagToHandler {
		ids = append(ids, k)
	}
	sort.Sort(uint16Slice(ids))
	return ids
}
<|endoftext|>


<commit_before>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	readOnly = flag.Bool("readOnly", false,
		"If true, refuse all Fetch and Update requests. For debugging only")
	disableUpdates = flag.Bool("disableUpdates", false,
		"If true, refuse all Update requests. For debugging only")
	disableTriggers = flag.Bool("disableTriggers", false,
		"If true, do not run any triggers. For debugging only")
)

var lastUpdateHadTriggerFailures bool

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	if *readOnly || *disableUpdates {
		txt := "Update() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	disableScannerFunc(true)
	defer disableScannerFunc(false)
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	copyFilesToCache(request.FilesToCopyToCache, rootDirectoryName)
	makeObjectCopies(request.MultiplyUsedObjects)
	lastUpdateHadTriggerFailures = false
	if len(oldTriggers.Triggers) > 0 {
		makeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		makeHardlinks(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		doDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers, false)
		makeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		changeInodes(request.InodesToChange, rootDirectoryName, &oldTriggers,
			false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		if runTriggers(matchedOldTriggers, "stop") {
			lastUpdateHadTriggerFailures = true
		}
	}
	makeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	makeHardlinks(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	doDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers, true)
	makeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	changeInodes(request.InodesToChange, rootDirectoryName, request.Triggers,
		true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	if runTriggers(matchedNewTriggers, "start") {
		lastUpdateHadTriggerFailures = true
	}
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Post-Update() debugging sleep complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		if copyFile(destPathname, sourcePathname) {
			logger.Printf("Copied: %s to cache\n", sourcePathname)
		}
	}
}

func copyFile(destPathname, sourcePathname string) bool {
	sourceFile, err := os.Open(sourcePathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer sourceFile.Close()
	dirname := path.Dir(destPathname)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return false
	}
	destFile, err := os.Create(destPathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer destFile.Close()
	_, err = io.Copy(destFile, sourceFile)
	if err != nil {
		return false
	}
	return true
}

func makeObjectCopies(multiplyUsedObjects map[hash.Hash]uint64) {
	for hash, numCopies := range multiplyUsedObjects {
		if numCopies < 2 {
			continue
		}
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(hash))
		for numCopies--; numCopies > 0; numCopies-- {
			ext := strings.Repeat("~", int(numCopies))
			if copyFile(objectPathname+ext, objectPathname) {
				logger.Printf("Copied object: %x%s\n", hash, ext)
			}
		}
	}
}

func makeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			switch inode := inode.GenericInode.(type) {
			case *filesystem.RegularInode:
				makeRegularInode(fullPathname, inode, multiplyUsedObjects)
			case *filesystem.SymlinkInode:
				makeSymlinkInode(fullPathname, inode)
			case *filesystem.SpecialInode:
				makeSpecialInode(fullPathname, inode)
			}
		}
	}
}

func makeRegularInode(fullPathname string, inode *filesystem.RegularInode,
	multiplyUsedObjects map[hash.Hash]uint64) {
	var err error
	if inode.Size > 0 {
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(inode.Hash))
		numCopies := multiplyUsedObjects[inode.Hash]
		if numCopies > 1 {
			numCopies--
			objectPathname += strings.Repeat("~", int(numCopies))
			if numCopies < 2 {
				delete(multiplyUsedObjects, inode.Hash)
			}
		}
		err = os.Rename(objectPathname, fullPathname)
	} else {
		_, err = os.Create(fullPathname)
	}
	if err != nil {
		logger.Println(err)
		return
	}
	if err := inode.WriteMetadata(fullPathname); err != nil {
		logger.Println(err)
	} else {
		if inode.Size > 0 {
			logger.Printf("Made inode: %s from: %x\n", fullPathname, inode.Hash)
		} else {
			logger.Printf("Made empty inode: %s\n", fullPathname)
		}
	}
}

func makeSymlinkInode(fullPathname string, inode *filesystem.SymlinkInode) {
	if err := inode.Write(fullPathname); err != nil {
		logger.Println(err)
	} else {
		logger.Printf("Made symlink inode: %s -> %s\n",
			fullPathname, inode.Symlink)
	}
}

func makeSpecialInode(fullPathname string, inode *filesystem.SpecialInode) {
	if err := inode.Write(fullPathname); err != nil {
		logger.Println(err)
	} else {
		logger.Printf("Made special inode: %s\n", fullPathname)
	}
}

func makeHardlinks(hardlinksToMake []sub.Hardlink, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		triggers.Match(hardlink.NewLink)
		if takeAction {
			targetPathname := path.Join(rootDirectoryName, hardlink.Target)
			linkPathname := path.Join(rootDirectoryName, hardlink.NewLink)
			if err := os.Link(targetPathname, linkPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Linked: %s => %s\n",
					linkPathname, targetPathname)
			}
		}
	}
}

func doDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			if err := os.RemoveAll(fullPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Deleted: %s\n", fullPathname)
			}
		}
	}
}

func makeDirectories(directoriesToMake []sub.Inode, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if skipPath(newdir.Name) {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			inode, ok := newdir.GenericInode.(*filesystem.DirectoryInode)
			if !ok {
				logger.Println("%s is not a directory!\n", newdir.Name)
				continue
			}
			if err := inode.Write(fullPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Made directory: %s\n", fullPathname)
			}
		}
	}
}

func changeInodes(inodesToChange []sub.Inode, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			if err := inode.WriteMetadata(fullPathname); err != nil {
				logger.Println(err)
				continue
			}
			logger.Printf("Changed inode: %s\n", fullPathname)
		}
	}
}

func skipPath(pathname string) bool {
	if scannerConfiguration.ScanFilter.Match(pathname) {
		return true
	}
	if pathname == "/.subd" {
		return true
	}
	if strings.HasPrefix(pathname, "/.subd/") {
		return true
	}
	return false
}

func runTriggers(triggers []*triggers.Trigger, action string) bool {
	hadFailures := false
	logPrefix := ""
	if *disableTriggers {
		logPrefix = "Disabled: "
	}
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print(logPrefix, "Rebooting")
				if *disableTriggers {
					return hadFailures
				}
				cmd := exec.Command("reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					hadFailures = true
					logger.Print(err)
				}
				return hadFailures
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("%sAction: service %s %s\n",
			logPrefix, trigger.Service, action)
		if *disableTriggers {
			continue
		}
		cmd := exec.Command("run-in-mntns", ppid, "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			hadFailures = true
			logger.Print(err)
		}
	}
	return hadFailures
}
<commit_msg>Remove 15 second debugging delay after Update() completed.<commit_after>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	readOnly = flag.Bool("readOnly", false,
		"If true, refuse all Fetch and Update requests. For debugging only")
	disableUpdates = flag.Bool("disableUpdates", false,
		"If true, refuse all Update requests. For debugging only")
	disableTriggers = flag.Bool("disableTriggers", false,
		"If true, do not run any triggers. For debugging only")
)

var lastUpdateHadTriggerFailures bool

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	if *readOnly || *disableUpdates {
		txt := "Update() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	disableScannerFunc(true)
	defer disableScannerFunc(false)
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	copyFilesToCache(request.FilesToCopyToCache, rootDirectoryName)
	makeObjectCopies(request.MultiplyUsedObjects)
	lastUpdateHadTriggerFailures = false
	if len(oldTriggers.Triggers) > 0 {
		makeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		makeHardlinks(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		doDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers, false)
		makeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		changeInodes(request.InodesToChange, rootDirectoryName, &oldTriggers,
			false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		if runTriggers(matchedOldTriggers, "stop") {
			lastUpdateHadTriggerFailures = true
		}
	}
	makeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	makeHardlinks(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	doDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers, true)
	makeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	changeInodes(request.InodesToChange, rootDirectoryName, request.Triggers,
		true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	if runTriggers(matchedNewTriggers, "start") {
		lastUpdateHadTriggerFailures = true
	}
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		if copyFile(destPathname, sourcePathname) {
			logger.Printf("Copied: %s to cache\n", sourcePathname)
		}
	}
}

func copyFile(destPathname, sourcePathname string) bool {
	sourceFile, err := os.Open(sourcePathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer sourceFile.Close()
	dirname := path.Dir(destPathname)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return false
	}
	destFile, err := os.Create(destPathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer destFile.Close()
	_, err = io.Copy(destFile, sourceFile)
	if err != nil {
		return false
	}
	return true
}

func makeObjectCopies(multiplyUsedObjects map[hash.Hash]uint64) {
	for hash, numCopies := range multiplyUsedObjects {
		if numCopies < 2 {
			continue
		}
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(hash))
		for numCopies--; numCopies > 0; numCopies-- {
			ext := strings.Repeat("~", int(numCopies))
			if copyFile(objectPathname+ext, objectPathname) {
				logger.Printf("Copied object: %x%s\n", hash, ext)
			}
		}
	}
}

func makeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			switch inode := inode.GenericInode.(type) {
			case *filesystem.RegularInode:
				makeRegularInode(fullPathname, inode, multiplyUsedObjects)
			case *filesystem.SymlinkInode:
				makeSymlinkInode(fullPathname, inode)
			case *filesystem.SpecialInode:
				makeSpecialInode(fullPathname, inode)
			}
		}
	}
}

func makeRegularInode(fullPathname string, inode *filesystem.RegularInode,
	multiplyUsedObjects map[hash.Hash]uint64) {
	var err error
	if inode.Size > 0 {
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(inode.Hash))
		numCopies := multiplyUsedObjects[inode.Hash]
		if numCopies > 1 {
			numCopies--
			objectPathname += strings.Repeat("~", int(numCopies))
			if numCopies < 2 {
				delete(multiplyUsedObjects, inode.Hash)
			}
		}
		err = os.Rename(objectPathname, fullPathname)
	} else {
		_, err = os.Create(fullPathname)
	}
	if err != nil {
		logger.Println(err)
		return
	}
	if err := inode.WriteMetadata(fullPathname); err != nil {
		logger.Println(err)
	} else {
		if inode.Size > 0 {
			logger.Printf("Made inode: %s from: %x\n", fullPathname, inode.Hash)
		} else {
			logger.Printf("Made empty inode: %s\n", fullPathname)
		}
	}
}

func makeSymlinkInode(fullPathname string, inode *filesystem.SymlinkInode) {
	if err := inode.Write(fullPathname); err != nil {
		logger.Println(err)
	} else {
		logger.Printf("Made symlink inode: %s -> %s\n",
			fullPathname, inode.Symlink)
	}
}

func makeSpecialInode(fullPathname string, inode *filesystem.SpecialInode) {
	if err := inode.Write(fullPathname); err != nil {
		logger.Println(err)
	} else {
		logger.Printf("Made special inode: %s\n", fullPathname)
	}
}

func makeHardlinks(hardlinksToMake []sub.Hardlink, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		triggers.Match(hardlink.NewLink)
		if takeAction {
			targetPathname := path.Join(rootDirectoryName, hardlink.Target)
			linkPathname := path.Join(rootDirectoryName, hardlink.NewLink)
			if err := os.Link(targetPathname, linkPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Linked: %s => %s\n",
					linkPathname, targetPathname)
			}
		}
	}
}

func doDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			if err := os.RemoveAll(fullPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Deleted: %s\n", fullPathname)
			}
		}
	}
}

func makeDirectories(directoriesToMake []sub.Inode, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if skipPath(newdir.Name) {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			inode, ok := newdir.GenericInode.(*filesystem.DirectoryInode)
			if !ok {
				logger.Println("%s is not a directory!\n", newdir.Name)
				continue
			}
			if err := inode.Write(fullPathname); err != nil {
				logger.Println(err)
			} else {
				logger.Printf("Made directory: %s\n", fullPathname)
			}
		}
	}
}

func changeInodes(inodesToChange []sub.Inode, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			if err := inode.WriteMetadata(fullPathname); err != nil {
				logger.Println(err)
				continue
			}
			logger.Printf("Changed inode: %s\n", fullPathname)
		}
	}
}

func skipPath(pathname string) bool {
	if scannerConfiguration.ScanFilter.Match(pathname) {
		return true
	}
	if pathname == "/.subd" {
		return true
	}
	if strings.HasPrefix(pathname, "/.subd/") {
		return true
	}
	return false
}

func runTriggers(triggers []*triggers.Trigger, action string) bool {
	hadFailures := false
	logPrefix := ""
	if *disableTriggers {
		logPrefix = "Disabled: "
	}
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print(logPrefix, "Rebooting")
				if *disableTriggers {
					return hadFailures
				}
				cmd := exec.Command("reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					hadFailures = true
					logger.Print(err)
				}
				return hadFailures
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("%sAction: service %s %s\n",
			logPrefix, trigger.Service, action)
		if *disableTriggers {
			continue
		}
		cmd := exec.Command("run-in-mntns", ppid, "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			hadFailures = true
			logger.Print(err)
		}
	}
	return hadFailures
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package subnet

import (
	"fmt"
	"path"
	"sync"
	"time"

	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
)

type subnetRegistry interface {
	getConfig() (*etcd.Response, error)
	getSubnets() (*etcd.Response, error)
	createSubnet(sn, data string, ttl uint64) (*etcd.Response, error)
	updateSubnet(sn, data string, ttl uint64) (*etcd.Response, error)
	watchSubnets(since uint64, stop chan bool) (*etcd.Response, error)
}

type EtcdConfig struct {
	Endpoints []string
	Keyfile   string
	Certfile  string
	CAFile    string
	Prefix    string
}

type etcdSubnetRegistry struct {
	mux     sync.Mutex
	cli     *etcd.Client
	etcdCfg *EtcdConfig
}

func newEtcdClient(c *EtcdConfig) (*etcd.Client, error) {
	if c.Keyfile != "" || c.Certfile != "" || c.CAFile != "" {
		return etcd.NewTLSClient(c.Endpoints, c.Certfile, c.Keyfile, c.CAFile)
	} else {
		return etcd.NewClient(c.Endpoints), nil
	}
}

func newEtcdSubnetRegistry(config *EtcdConfig) (subnetRegistry, error) {
	r := &etcdSubnetRegistry{
		etcdCfg: config,
	}

	var err error
	r.cli, err = newEtcdClient(config)
	if err != nil {
		return nil, err
	}

	return r, nil
}

func (esr *etcdSubnetRegistry) getConfig() (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "config")
	resp, err := esr.client().Get(key, false, false)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func (esr *etcdSubnetRegistry) getSubnets() (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets")
	return esr.client().Get(key, false, true)
}

func (esr *etcdSubnetRegistry) createSubnet(sn, data string, ttl uint64) (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets", sn)
	resp, err := esr.client().Create(key, data, ttl)
	if err != nil {
		return nil, err
	}

	ensureExpiration(resp, ttl)
	return resp, nil
}

func (esr *etcdSubnetRegistry) updateSubnet(sn, data string, ttl uint64) (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets", sn)
	resp, err := esr.client().Set(key, data, ttl)
	if err != nil {
		return nil, err
	}

	ensureExpiration(resp, ttl)
	return resp, nil
}

func (esr *etcdSubnetRegistry) watchSubnets(since uint64, stop chan bool) (*etcd.Response, error) {
	for {
		key := path.Join(esr.etcdCfg.Prefix, "subnets")
		resp, err := esr.client().RawWatch(key, since, true, nil, stop)

		if err != nil {
			if err == etcd.ErrWatchStoppedByUser {
				return nil, nil
			} else {
				return nil, err
			}
		}

		if len(resp.Body) == 0 {
			// etcd timed out, go back but recreate the client as the underlying
			// http transport gets hosed (http://code.google.com/p/go/issues/detail?id=8648)
			esr.resetClient()
			continue
		}

		return resp.Unmarshal()
	}
}

func (esr *etcdSubnetRegistry) client() *etcd.Client {
	esr.mux.Lock()
	defer esr.mux.Unlock()
	return esr.cli
}

func (esr *etcdSubnetRegistry) resetClient() {
	esr.mux.Lock()
	defer esr.mux.Unlock()

	var err error
	esr.cli.Close()
	esr.cli, err = newEtcdClient(esr.etcdCfg)
	if err != nil {
		panic(fmt.Errorf("resetClient: error recreating etcd client: %v", err))
	}
}

func ensureExpiration(resp *etcd.Response, ttl uint64) {
	if resp.Node.Expiration == nil {
		// should not be but calc it ourselves in this case
		log.Info("Expiration field missing on etcd response, calculating locally")
		exp := time.Now().Add(time.Duration(ttl) * time.Second)
		resp.Node.Expiration = &exp
	}
}
<commit_msg>turn on go-etcd logging<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package subnet

import (
	"fmt"
	golog "log"
	"os"
	"path"
	"sync"
	"time"

	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
)

type subnetRegistry interface {
	getConfig() (*etcd.Response, error)
	getSubnets() (*etcd.Response, error)
	createSubnet(sn, data string, ttl uint64) (*etcd.Response, error)
	updateSubnet(sn, data string, ttl uint64) (*etcd.Response, error)
	watchSubnets(since uint64, stop chan bool) (*etcd.Response, error)
}

type EtcdConfig struct {
	Endpoints []string
	Keyfile   string
	Certfile  string
	CAFile    string
	Prefix    string
}

type etcdSubnetRegistry struct {
	mux     sync.Mutex
	cli     *etcd.Client
	etcdCfg *EtcdConfig
}

func init() {
	etcd.SetLogger(golog.New(os.Stderr, "go-etcd", golog.LstdFlags))
}

func newEtcdClient(c *EtcdConfig) (*etcd.Client, error) {
	if c.Keyfile != "" || c.Certfile != "" || c.CAFile != "" {
		return etcd.NewTLSClient(c.Endpoints, c.Certfile, c.Keyfile, c.CAFile)
	} else {
		return etcd.NewClient(c.Endpoints), nil
	}
}

func newEtcdSubnetRegistry(config *EtcdConfig) (subnetRegistry, error) {
	r := &etcdSubnetRegistry{
		etcdCfg: config,
	}

	var err error
	r.cli, err = newEtcdClient(config)
	if err != nil {
		return nil, err
	}

	return r, nil
}

func (esr *etcdSubnetRegistry) getConfig() (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "config")
	resp, err := esr.client().Get(key, false, false)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func (esr *etcdSubnetRegistry) getSubnets() (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets")
	return esr.client().Get(key, false, true)
}

func (esr *etcdSubnetRegistry) createSubnet(sn, data string, ttl uint64) (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets", sn)
	resp, err := esr.client().Create(key, data, ttl)
	if err != nil {
		return nil, err
	}

	ensureExpiration(resp, ttl)
	return resp, nil
}

func (esr *etcdSubnetRegistry) updateSubnet(sn, data string, ttl uint64) (*etcd.Response, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets", sn)
	resp, err := esr.client().Set(key, data, ttl)
	if err != nil {
		return nil, err
	}

	ensureExpiration(resp, ttl)
	return resp, nil
}

func (esr *etcdSubnetRegistry) watchSubnets(since uint64, stop chan bool) (*etcd.Response, error) {
	for {
		key := path.Join(esr.etcdCfg.Prefix, "subnets")
		resp, err := esr.client().RawWatch(key, since, true, nil, stop)

		if err != nil {
			if err == etcd.ErrWatchStoppedByUser {
				return nil, nil
			} else {
				return nil, err
			}
		}

		if len(resp.Body) == 0 {
			// etcd timed out, go back but recreate the client as the underlying
			// http transport gets hosed (http://code.google.com/p/go/issues/detail?id=8648)
			esr.resetClient()
			continue
		}

		return resp.Unmarshal()
	}
}

func (esr *etcdSubnetRegistry) client() *etcd.Client {
	esr.mux.Lock()
	defer esr.mux.Unlock()
	return esr.cli
}

func (esr *etcdSubnetRegistry) resetClient() {
	esr.mux.Lock()
	defer esr.mux.Unlock()

	var err error
	esr.cli.Close()
	esr.cli, err = newEtcdClient(esr.etcdCfg)
	if err != nil {
		panic(fmt.Errorf("resetClient: error recreating etcd client: %v", err))
	}
}

func ensureExpiration(resp *etcd.Response, ttl uint64) {
	if resp.Node.Expiration == nil {
		// should not be but calc it ourselves in this case
		log.Info("Expiration field missing on etcd response, calculating locally")
		exp := time.Now().Add(time.Duration(ttl) * time.Second)
		resp.Node.Expiration = &exp
	}
}
<|endoftext|>