<commit_before>// Copyright 2015 Thomas Burke <tburke@tb99.com>. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package iso8211 implements ISO 8211 parsing.
// It is targeted to NOAA IHO S-57 format vector chart files.
//
// ISO 8211 is one of those baroque 1990's era binary file formats.
//   file: LeadRecord, DataRecord...
//   Record : Header, data
//   LeadRecord : Header, FieldType...
//   DataRecord : Header, Field...
//   FieldType : FieldHeader, SubField tags and formats
//   Field : SubFields
//
// References:
//   http://www.iho.int/iho_pubs/standard/S-57Ed3.1/31Main.pdf
//   http://sourceforge.net/projects/py-iso8211/
//   https://www.iso.org/obp/ui/#iso:std:iso-iec:8211:ed-2:v1:en
//   http://mcmcweb.er.usgs.gov/sdts/SDTS_standard_nov97/p3body.html
//   http://www.charts.noaa.gov/ENCs/ENCs.shtml
package iso8211

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
)

type RawHeader struct {
	Record_length                    [5]byte
	Interchange_level                byte
	Leader_id                        byte
	InLineCode                       byte
	Version                          byte
	Application_indicator            byte
	Field_control_length             [2]byte
	Base_address                     [5]byte
	Extended_character_set_indicator [3]byte
	Size_of_field_length             byte
	Size_of_field_position           byte
	Reserved                         byte
	Size_of_field_tag                byte
}

type DirEntry struct {
	Tag      []byte
	Length   int
	Position int
}

type Header struct {
	Record_length                        uint64
	Interchange_level                    byte
	Leader_id                            byte
	InLineCode                           byte
	Version                              byte
	Application_indicator                byte
	Field_control_length                 uint64
	Base_address                         uint64
	Extended_character_set_indicator     []byte
	Length_size, Position_size, Tag_size int8
	Entries                              []DirEntry
}

type LeadRecord struct {
	Header     Header
	FieldTypes map[string]FieldType
}

type Field struct {
	Tag       string
	Length    int
	Position  int
	FieldType FieldType
	SubFields []interface{}
}

type DataRecord struct {
	Header Header
	Lead   *LeadRecord
	Fields []Field
}

type RawFieldHeader struct {
	Data_structure     byte
	Data_type          byte
	Auxiliary_controls [2]byte
	Printable_ft       byte
	Printable_ut       byte
	Escape_seq         [3]byte
}

type SubFieldType struct {
	Kind reflect.Kind
	Size int
	Tag  []byte
}

type FieldType struct {
	Tag                string
	Length             int
	Position           int
	Data_structure     byte
	Data_type          byte
	Auxiliary_controls []byte
	Printable_ft       byte
	Printable_ut       byte
	Escape_seq         []byte
	Name               []byte
	Array_descriptor   []byte
	Format_controls    []byte
	SubFields          []SubFieldType
}

func (header *Header) Read(file io.Reader) error {
	var err error
	var ddr RawHeader
	ddrSize := uint64(binary.Size(ddr))
	// Read the header
	err = binary.Read(file, binary.LittleEndian, &ddr)
	if err != nil {
		return err
	}
	header.Record_length, _ = strconv.ParseUint(string(ddr.Record_length[:]), 10, 64)
	header.Interchange_level = ddr.Interchange_level
	header.Leader_id = ddr.Leader_id
	header.InLineCode = ddr.InLineCode
	header.Version = ddr.Version
	header.Application_indicator = ddr.Application_indicator
	header.Field_control_length, _ = strconv.ParseUint(string(ddr.Field_control_length[:]), 10, 64)
	header.Base_address, _ = strconv.ParseUint(string(ddr.Base_address[:]), 10, 64)
	header.Extended_character_set_indicator = ddr.Extended_character_set_indicator[:]
	header.Length_size = int8(ddr.Size_of_field_length - '0')
	header.Position_size = int8(ddr.Size_of_field_position - '0')
	header.Tag_size = int8(ddr.Size_of_field_tag - '0')
	// Read the directory
	entries := (header.Base_address - 1 - ddrSize) / uint64(header.Length_size+header.Position_size+header.Tag_size)
	header.Entries = make([]DirEntry, entries)
	dir := make([]byte, header.Base_address-ddrSize)
	file.Read(dir)
	buf := bytes.NewBuffer(dir)
	for idx := uint64(0); idx < entries; idx++ {
		header.Entries[idx].Tag = buf.Next(int(header.Tag_size))
		header.Entries[idx].Length, _ = strconv.Atoi(string(buf.Next(int(header.Length_size))[:]))
		header.Entries[idx].Position, _ = strconv.Atoi(string(buf.Next(int(header.Position_size))[:]))
	}
	return err
}

func (lead *LeadRecord) Read(file io.Reader) error {
	var err error
	err = lead.Header.Read(file)
	if err != nil {
		return err
	}
	if lead.Header.Leader_id != 'L' {
		return errors.New("Record is not a Lead record")
	}
	err = lead.ReadFields(file)
	return err
}

func (lead *LeadRecord) ReadFields(file io.Reader) error {
	var err error
	lead.FieldTypes = make(map[string]FieldType, len(lead.Header.Entries))
	for _, d := range lead.Header.Entries {
		field := FieldType{Tag: string(d.Tag), Length: d.Length, Position: d.Position}
		field.Read(file)
		lead.FieldTypes[field.Tag] = field
	}
	return err
}

func (field *Field) Read(file io.Reader) error {
	var err error
	data := make([]byte, field.Length)
	file.Read(data)
	if field.FieldType.Tag != "" {
		field.SubFields = field.FieldType.Decode(data[:field.Length-1])
	}
	return err
}

func (data *DataRecord) Read(file io.Reader) error {
	var err error
	err = data.Header.Read(file)
	if err != nil {
		return err
	}
	if data.Header.Leader_id != 'D' {
		return errors.New("Record is not a Data record")
	}
	err = data.ReadFields(file)
	return err
}

func (data *DataRecord) ReadFields(file io.Reader) error {
	var err error
	data.Fields = make([]Field, len(data.Header.Entries))
	for i, d := range data.Header.Entries {
		field := Field{Tag: string(d.Tag), Length: d.Length, Position: d.Position}
		if data.Lead != nil {
			field.FieldType = data.Lead.FieldTypes[field.Tag]
		}
		err = field.Read(file)
		data.Fields[i] = field
	}
	return err
}

func (dir *FieldType) Read(file io.Reader) error {
	var field RawFieldHeader
	err := binary.Read(file, binary.LittleEndian, &field)
	dir.Data_structure = field.Data_structure
	dir.Data_type = field.Data_type
	dir.Auxiliary_controls = field.Auxiliary_controls[:]
	dir.Printable_ft = field.Printable_ft
	dir.Printable_ut = field.Printable_ut
	dir.Escape_seq = field.Escape_seq[:]
	fdata := make([]byte, dir.Length-9)
	file.Read(fdata)
	desc := bytes.Split(fdata[:dir.Length-10], []byte{'\x1f'})
	dir.Name = desc[0]
	dir.Array_descriptor = desc[1]
	if len(desc) > 2 {
		dir.Format_controls = desc[2]
	}
	return err
}

/*
Format parses the ISO-8211 format controls and array descriptors.

Section 7.2.2.1 of the IHO S-57 Publication.
http://www.iho.int/iho_pubs/standard/S-57Ed3.1/31Main.pdf

Array Descriptor and Format Controls. The array descriptor is a ! separated
list of tags describing the data field. If it begins with a * the tag list
is repeated. The format controls decribe the format of the data for each tag.

eg: Descriptor AGEN!FIDN!FIDS , Format (b12,b14,b12) is three binary encoded
integers. AGEN is an int16, FIDN an int32 and FIDS an int16. The 'b' indicates
binary int, '1' indicates unsigned, the second digit indicates the number of
bytes.
Decriptor *YCOO!XCOO, Format (2b24) is two binary encoded integers. Both are
int32s, the '2' after the 'b' indicates signed. The * in the descriptor
indicates that pair is repeated to fill the data field.
*/
func (dir *FieldType) Format() []SubFieldType {
	if dir.SubFields != nil {
		return dir.SubFields
	}
	var re = regexp.MustCompile(`(\d*)(\w+)\(*(\d*)\)*`)

	if len(dir.Format_controls) > 2 {
		Tags := bytes.Split(dir.Array_descriptor, []byte{'!'})
		Tagidx := 0
		types := make([]SubFieldType, len(Tags))
		for _, a := range re.FindAllSubmatch(dir.Format_controls, -1) {
			i := 1
			if len(a[1]) > 0 {
				i, _ = strconv.Atoi(string(a[1]))
			}
			var size int
			if len(a[3]) > 0 {
				size, _ = strconv.Atoi(string(a[3]))
			}
			for ; i > 0; i-- {
				switch a[2][0] {
				case 'A':
					types[Tagidx] = SubFieldType{reflect.String, size, Tags[Tagidx]}
				case 'I':
				case 'R':
					types[Tagidx] = SubFieldType{reflect.String, size, Tags[Tagidx]}
				case 'B':
					types[Tagidx] = SubFieldType{reflect.Array, size / 8, Tags[Tagidx]}
				case 'b':
					switch string(a[2][1:]) {
					case "11":
						types[Tagidx] = SubFieldType{reflect.Uint8, 1, Tags[Tagidx]}
					case "12":
						types[Tagidx] = SubFieldType{reflect.Uint16, 2, Tags[Tagidx]}
					case "14":
						types[Tagidx] = SubFieldType{reflect.Uint32, 4, Tags[Tagidx]}
					case "21":
						types[Tagidx] = SubFieldType{reflect.Int8, 1, Tags[Tagidx]}
					case "22":
						types[Tagidx] = SubFieldType{reflect.Int16, 2, Tags[Tagidx]}
					case "24":
						types[Tagidx] = SubFieldType{reflect.Int32, 4, Tags[Tagidx]}
					}
				}
				Tagidx++
			}
		}
		dir.SubFields = types
	}
	return dir.SubFields
}

func (dir FieldType) Decode(buffer []byte) []interface{} {
	buf := bytes.NewBuffer(buffer)
	var values []interface{}
	for buf.Len() > 0 {
		for _, ftype := range dir.Format() {
			switch ftype.Kind {
			case reflect.Uint8:
				{
					var v uint8
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Uint16:
				{
					var v uint16
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Uint32:
				{
					var v uint32
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int8:
				{
					var v int8
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int16:
				{
					var v int16
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int32:
				{
					var v int32
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			default:
				{
					if ftype.Size == 0 {
						i, _ := buf.ReadString('\x1f')
						if len(i) > 0 {
							values = append(values, i[:len(i)-1])
						} else {
							values = append(values, "")
						}
					} else {
						i := buf.Next(ftype.Size)
						values = append(values, string(i))
					}
				}
			}
		}
	}
	return values
}
<commit_msg>More documentation.<commit_after>// Copyright 2015 Thomas Burke <tburke@tb99.com>. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package iso8211 implements ISO 8211 parsing.
// It is targeted to NOAA IHO S-57 format vector chart files.
//
// ISO 8211 is one of those baroque 1990's era binary file formats.
//   file: LeadRecord, DataRecord...
//   Record : Header, data
//   LeadRecord : Header, FieldType...
//   DataRecord : Header, Field...
//   FieldType : FieldHeader, SubField tags and formats
//   Field : SubFields
//
// References:
//   http://www.iho.int/iho_pubs/standard/S-57Ed3.1/31Main.pdf
//   http://sourceforge.net/projects/py-iso8211/
//   https://www.iso.org/obp/ui/#iso:std:iso-iec:8211:ed-2:v1:en
//   http://mcmcweb.er.usgs.gov/sdts/SDTS_standard_nov97/p3body.html
//   http://www.charts.noaa.gov/ENCs/ENCs.shtml
package iso8211

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
)

// RawHeader is a convenience for directly loading the on-disk
// binary Header format.
type RawHeader struct {
	Record_length                    [5]byte
	Interchange_level                byte
	Leader_id                        byte
	InLineCode                       byte
	Version                          byte
	Application_indicator            byte
	Field_control_length             [2]byte
	Base_address                     [5]byte
	Extended_character_set_indicator [3]byte
	Size_of_field_length             byte
	Size_of_field_position           byte
	Reserved                         byte
	Size_of_field_tag                byte
}

// DirEntry describes each following Field
type DirEntry struct {
	Tag      []byte
	Length   int
	Position int
}

// Header holds the overall layout for a Record.
type Header struct {
	Record_length                        uint64
	Interchange_level                    byte
	Leader_id                            byte
	InLineCode                           byte
	Version                              byte
	Application_indicator                byte
	Field_control_length                 uint64
	Base_address                         uint64
	Extended_character_set_indicator     []byte
	Length_size, Position_size, Tag_size int8
	Entries                              []DirEntry
}

// LeadRecord is the first Record in a file. It has metadata for each
// Field in the file.
type LeadRecord struct {
	Header     Header
	FieldTypes map[string]FieldType
}

type Field struct {
	Tag       string
	Length    int
	Position  int
	FieldType FieldType
	SubFields []interface{}
}

// DataRecord contains data for a set of Fields and their SubFields.
type DataRecord struct {
	Header Header
	Lead   *LeadRecord
	Fields []Field
}

type RawFieldHeader struct {
	Data_structure     byte
	Data_type          byte
	Auxiliary_controls [2]byte
	Printable_ft       byte
	Printable_ut       byte
	Escape_seq         [3]byte
}

type SubFieldType struct {
	Kind reflect.Kind
	Size int
	Tag  []byte
}

type FieldType struct {
	Tag                string
	Length             int
	Position           int
	Data_structure     byte
	Data_type          byte
	Auxiliary_controls []byte
	Printable_ft       byte
	Printable_ut       byte
	Escape_seq         []byte
	Name               []byte
	Array_descriptor   []byte
	Format_controls    []byte
	SubFields          []SubFieldType
}

// Read loads a binary format RawHeader and its DirEntries into
// the Header model.
func (header *Header) Read(file io.Reader) error {
	var err error
	var ddr RawHeader
	ddrSize := uint64(binary.Size(ddr))
	// Read the header
	err = binary.Read(file, binary.LittleEndian, &ddr)
	if err != nil {
		return err
	}
	header.Record_length, _ = strconv.ParseUint(string(ddr.Record_length[:]), 10, 64)
	header.Interchange_level = ddr.Interchange_level
	header.Leader_id = ddr.Leader_id
	header.InLineCode = ddr.InLineCode
	header.Version = ddr.Version
	header.Application_indicator = ddr.Application_indicator
	header.Field_control_length, _ = strconv.ParseUint(string(ddr.Field_control_length[:]), 10, 64)
	header.Base_address, _ = strconv.ParseUint(string(ddr.Base_address[:]), 10, 64)
	header.Extended_character_set_indicator = ddr.Extended_character_set_indicator[:]
	header.Length_size = int8(ddr.Size_of_field_length - '0')
	header.Position_size = int8(ddr.Size_of_field_position - '0')
	header.Tag_size = int8(ddr.Size_of_field_tag - '0')
	// Read the directory
	entries := (header.Base_address - 1 - ddrSize) / uint64(header.Length_size+header.Position_size+header.Tag_size)
	header.Entries = make([]DirEntry, entries)
	dir := make([]byte, header.Base_address-ddrSize)
	file.Read(dir)
	buf := bytes.NewBuffer(dir)
	for idx := uint64(0); idx < entries; idx++ {
		header.Entries[idx].Tag = buf.Next(int(header.Tag_size))
		header.Entries[idx].Length, _ = strconv.Atoi(string(buf.Next(int(header.Length_size))[:]))
		header.Entries[idx].Position, _ = strconv.Atoi(string(buf.Next(int(header.Position_size))[:]))
	}
	return err
}

// Read loads the LeadRecord Header and the FieldTypes
func (lead *LeadRecord) Read(file io.Reader) error {
	var err error
	err = lead.Header.Read(file)
	if err != nil {
		return err
	}
	if lead.Header.Leader_id != 'L' {
		return errors.New("Record is not a Lead record")
	}
	err = lead.ReadFields(file)
	return err
}

func (lead *LeadRecord) ReadFields(file io.Reader) error {
	var err error
	lead.FieldTypes = make(map[string]FieldType, len(lead.Header.Entries))
	for _, d := range lead.Header.Entries {
		field := FieldType{Tag: string(d.Tag), Length: d.Length, Position: d.Position}
		field.Read(file)
		lead.FieldTypes[field.Tag] = field
	}
	return err
}

func (field *Field) Read(file io.Reader) error {
	var err error
	data := make([]byte, field.Length)
	file.Read(data)
	if field.FieldType.Tag != "" {
		field.SubFields = field.FieldType.Decode(data[:field.Length-1])
	}
	return err
}

func (data *DataRecord) Read(file io.Reader) error {
	var err error
	err = data.Header.Read(file)
	if err != nil {
		return err
	}
	if data.Header.Leader_id != 'D' {
		return errors.New("Record is not a Data record")
	}
	err = data.ReadFields(file)
	return err
}

func (data *DataRecord) ReadFields(file io.Reader) error {
	var err error
	data.Fields = make([]Field, len(data.Header.Entries))
	for i, d := range data.Header.Entries {
		field := Field{Tag: string(d.Tag), Length: d.Length, Position: d.Position}
		if data.Lead != nil {
			field.FieldType = data.Lead.FieldTypes[field.Tag]
		}
		err = field.Read(file)
		data.Fields[i] = field
	}
	return err
}

func (dir *FieldType) Read(file io.Reader) error {
	var field RawFieldHeader
	err := binary.Read(file, binary.LittleEndian, &field)
	dir.Data_structure = field.Data_structure
	dir.Data_type = field.Data_type
	dir.Auxiliary_controls = field.Auxiliary_controls[:]
	dir.Printable_ft = field.Printable_ft
	dir.Printable_ut = field.Printable_ut
	dir.Escape_seq = field.Escape_seq[:]
	fdata := make([]byte, dir.Length-9)
	file.Read(fdata)
	desc := bytes.Split(fdata[:dir.Length-10], []byte{'\x1f'})
	dir.Name = desc[0]
	dir.Array_descriptor = desc[1]
	if len(desc) > 2 {
		dir.Format_controls = desc[2]
	}
	return err
}

/*
Format parses the ISO-8211 format controls and array descriptors.

Based on Section 7.2.2.1 of the IHO S-57 Publication.
http://www.iho.int/iho_pubs/standard/S-57Ed3.1/31Main.pdf

Array Descriptor and Format Controls. The array descriptor is a ! separated
list of tags describing the data field. If it begins with a * the tag list
is repeated. The format controls decribe the format of the data for each tag.

eg: Descriptor AGEN!FIDN!FIDS , Format (b12,b14,b12) is three binary encoded
integers. AGEN is an int16, FIDN an int32 and FIDS an int16. The 'b' indicates
binary int, '1' indicates unsigned, the second digit indicates the number of
bytes.
Decriptor *YCOO!XCOO, Format (2b24) is two binary encoded integers. Both are
int32s, the '2' after the 'b' indicates signed. The * in the descriptor
indicates that pair is repeated to fill the data field.
*/
func (dir *FieldType) Format() []SubFieldType {
	if dir.SubFields != nil {
		return dir.SubFields
	}
	var re = regexp.MustCompile(`(\d*)(\w+)\(*(\d*)\)*`)

	if len(dir.Format_controls) > 2 {
		Tags := bytes.Split(dir.Array_descriptor, []byte{'!'})
		Tagidx := 0
		types := make([]SubFieldType, len(Tags))
		for _, a := range re.FindAllSubmatch(dir.Format_controls, -1) {
			i := 1
			if len(a[1]) > 0 {
				i, _ = strconv.Atoi(string(a[1]))
			}
			var size int
			if len(a[3]) > 0 {
				size, _ = strconv.Atoi(string(a[3]))
			}
			for ; i > 0; i-- {
				switch a[2][0] {
				case 'A':
					types[Tagidx] = SubFieldType{reflect.String, size, Tags[Tagidx]}
				case 'I':
				case 'R':
					types[Tagidx] = SubFieldType{reflect.String, size, Tags[Tagidx]}
				case 'B':
					types[Tagidx] = SubFieldType{reflect.Array, size / 8, Tags[Tagidx]}
				case 'b':
					switch string(a[2][1:]) {
					case "11":
						types[Tagidx] = SubFieldType{reflect.Uint8, 1, Tags[Tagidx]}
					case "12":
						types[Tagidx] = SubFieldType{reflect.Uint16, 2, Tags[Tagidx]}
					case "14":
						types[Tagidx] = SubFieldType{reflect.Uint32, 4, Tags[Tagidx]}
					case "21":
						types[Tagidx] = SubFieldType{reflect.Int8, 1, Tags[Tagidx]}
					case "22":
						types[Tagidx] = SubFieldType{reflect.Int16, 2, Tags[Tagidx]}
					case "24":
						types[Tagidx] = SubFieldType{reflect.Int32, 4, Tags[Tagidx]}
					}
				}
				Tagidx++
			}
		}
		dir.SubFields = types
	}
	return dir.SubFields
}

// Decode uses the FieldType Format to convert the binary file format
// SubFields into an array of Go data types.
func (dir FieldType) Decode(buffer []byte) []interface{} {
	buf := bytes.NewBuffer(buffer)
	var values []interface{}
	for buf.Len() > 0 {
		for _, ftype := range dir.Format() {
			switch ftype.Kind {
			case reflect.Uint8:
				{
					var v uint8
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Uint16:
				{
					var v uint16
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Uint32:
				{
					var v uint32
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int8:
				{
					var v int8
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int16:
				{
					var v int16
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			case reflect.Int32:
				{
					var v int32
					binary.Read(buf, binary.LittleEndian, &v)
					values = append(values, v)
				}
			default:
				{
					if ftype.Size == 0 {
						i, _ := buf.ReadString('\x1f')
						if len(i) > 0 {
							values = append(values, i[:len(i)-1])
						} else {
							values = append(values, "")
						}
					} else {
						i := buf.Next(ftype.Size)
						values = append(values, string(i))
					}
				}
			}
		}
	}
	return values
}
<|endoftext|>


<commit_before>package server

import (
	"../hashtree"
	"../network"
	"bytes"
	"io"
	"os"
	"testing"
)

const (
	testDatabase = ".testDatabase"
	testLevelLow = 0
)

func TestFileIO(t *testing.T) {
	err := os.RemoveAll(testDatabase)
	if err != nil {
		t.Fatal(err)
	}
	testFileSize(0, t)
	testFileSize(1, t)
	testFileSize(1024, t)
	testFileSize(1025, t)
	testFileSize(2345, t)
	testFileSize(12345, t)
}

func testFileSize(size hashtree.Bytes, t *testing.T) {
	t.Log("testing size:", size)
	d := OpenSimpleDatabase(testDatabase, testLevelLow)

	id := d.ImportFromReader(&testFile{length: size})
	if hashtree.Bytes(id.GetLength()) != size {
		t.Fatalf("Length is %x, should be %x", id.GetLength(), size)
	}

	buf := make([]byte, 1024)
	n := 0
	for i := 0; i < int(size); i += n {
		n, _ = d.GetAt(buf, id, hashtree.Bytes(i))
		for j := 0; j < n; j++ {
			if buf[j] != testFileG(i+j) {
				t.Fatalf("at:%d, got:%x, expected:%x, for file:%s", i+j, buf[j], testFileG(i+j), id.CompactId())
			}
		}
	}

	hash := hashtree.NewTree()
	leafs := refHash.Nodes(size)
	t.Log("leafs:", leafs)
	levels := hash.Levels(leafs)
	for i := hashtree.Level(testLevelLow); i < levels-1; i++ {
		req := network.InnerHashes{
			Height: int32p(int(i)),
			From:   int32p(0),
			Length: int32p(int(hash.LevelWidth(leafs, i))),
		}
		got, _ := d.GetInnerHashes(id, req)
		list := got.GetHashes()
		hash.Write(list)
		listSum := hash.Sum(nil)
		hash.Reset()
		if !bytes.Equal(listSum, id.Hash) {
			t.Fatalf("Req:%s , got hashes:%x, len:%d, sums to:%x, expected:%x", req.String(), list, len(list), listSum, id.Hash)
		}
	}
}

type testFile struct {
	index  hashtree.Bytes
	length hashtree.Bytes
}

func (f *testFile) Read(b []byte) (int, error) {
	if f.index == f.length {
		return 0, io.EOF
	}
	b[0] = testFileG(int(f.index))
	f.index++
	return 1, nil
}

func testFileG(index int) byte {
	return byte(index)
}

func int32p(n int) *int32 {
	m := int32(n)
	return &m
}
<commit_msg>generates files that doesn't repeat so shortly<commit_after>package server

import (
	"../hashtree"
	"../network"
	"bytes"
	"io"
	"os"
	"testing"
)

const (
	testDatabase = ".testDatabase"
	testLevelLow = 0
)

func TestFileIO(t *testing.T) {
	err := os.RemoveAll(testDatabase)
	if err != nil {
		t.Fatal(err)
	}
	testFileSize(0, t)
	testFileSize(1, t)
	testFileSize(1024, t)
	testFileSize(1025, t)
	testFileSize(2345, t)
	testFileSize(12345, t)
}

func testFileSize(size hashtree.Bytes, t *testing.T) {
	t.Log("testing size:", size)
	d := OpenSimpleDatabase(testDatabase, testLevelLow)

	id := d.ImportFromReader(&testFile{length: size})
	if hashtree.Bytes(id.GetLength()) != size {
		t.Fatalf("Length is %x, should be %x", id.GetLength(), size)
	}

	buf := make([]byte, 1024)
	n := 0
	for i := 0; i < int(size); i += n {
		n, _ = d.GetAt(buf, id, hashtree.Bytes(i))
		for j := 0; j < n; j++ {
			if buf[j] != testFileG(i+j) {
				t.Fatalf("at:%d, got:%x, expected:%x, for file:%s", i+j, buf[j], testFileG(i+j), id.CompactId())
			}
		}
	}

	hash := hashtree.NewTree()
	leafs := refHash.Nodes(size)
	t.Log("leafs:", leafs)
	levels := hash.Levels(leafs)
	for i := hashtree.Level(testLevelLow); i < levels-1; i++ {
		req := network.InnerHashes{
			Height: int32p(int(i)),
			From:   int32p(0),
			Length: int32p(int(hash.LevelWidth(leafs, i))),
		}
		got, _ := d.GetInnerHashes(id, req)
		list := got.GetHashes()
		hash.Write(list)
		listSum := hash.Sum(nil)
		hash.Reset()
		if !bytes.Equal(listSum, id.Hash) {
			t.Fatalf("Req:%s , got hashes:%x, len:%d, sums to:%x, expected:%x", req.String(), list, len(list), listSum, id.Hash)
		}
	}
}

type testFile struct {
	index  hashtree.Bytes
	length hashtree.Bytes
}

func (f *testFile) Read(b []byte) (int, error) {
	if f.index == f.length {
		return 0, io.EOF
	}
	b[0] = testFileG(int(f.index))
	f.index++
	return 1, nil
}

func testFileG(index int) byte {
	return byte(index * index / 10007)
}

func int32p(n int) *int32 {
	m := int32(n)
	return &m
}
<|endoftext|>


<commit_before>package image

import (
	stdimg "image"
	"image/png"
	"log"
	"os"
)

// Image defines an RGB image
type Image struct {
	stdimg.NRGBA
}

// New returns a new Image with the pixel slice initialized
func New(width int, height int) *Image {
	return &Image{*stdimg.NewNRGBA(stdimg.Rect(0, 0, width, height))}
}

// Save saves the image as a ppm image.
func (img *Image) Save(filename string) {
	file, err := os.Create(filename)
	if err != nil {
		panic(err)
	}

	defer file.Close()
	f, err := os.Create(filename + ".png")
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Small comment fix<commit_after>package image

import (
	stdimg "image"
	"image/png"
	"log"
	"os"
)

// Image defines an RGB image
type Image struct {
	stdimg.NRGBA
}

// New returns a new Image with the pixel slice initialized
func New(width int, height int) *Image {
	return &Image{*stdimg.NewNRGBA(stdimg.Rect(0, 0, width, height))}
}

// Save saves the image as a png file.
func (img *Image) Save(filename string) {
	file, err := os.Create(filename)
	if err != nil {
		panic(err)
	}

	defer file.Close()
	f, err := os.Create(filename + ".png")
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package secrets

import (
	"net/http"

	"github.com/docker/go-plugins-helpers/sdk"
)

const (
	manifest = `{"Implements": ["secretprovider"]}`
	getPath  = "/SecretProvider.GetSecret"
)

// Request is the plugin secret request
type Request struct {
	SecretName          string            `json:",omitempty"` // SecretName is the name of the secret to request from the plugin
	SecretLabels        map[string]string `json:",omitempty"` // SecretLabels capture environment names and other metadata pertaining to the secret
	ServiceHostname     string            `json:",omitempty"` // ServiceHostname is the hostname of the service, can be used for x509 certificate
	ServiceName         string            `json:",omitempty"` // ServiceName is the name of the service that requested the secret
	ServiceID           string            `json:",omitempty"` // ServiceID is the name of the service that requested the secret
	ServiceLabels       map[string]string `json:",omitempty"` // ServiceLabels capture environment names and other metadata pertaining to the service
	TaskID              string            `json:",omitempty"` // TaskID is the ID of the task that the secret is assigned to
	TaskName            string            `json:",omitempty"` // TaskName is the name of the task that the secret is assigned to
	ServiceEndpointSpec *EndpointSpec     `json:",omitempty"` // ServiceEndpointSpec holds the specification for endpoints
}

// Response contains the plugin secret value
type Response struct {
	Value []byte `json:",omitempty"` // Value is the value of the secret
	Err   string `json:",omitempty"` // Err is the error response of the plugin
}

// EndpointSpec represents the spec of an endpoint.
type EndpointSpec struct {
	Mode  int32        `json:",omitempty"`
	Ports []PortConfig `json:",omitempty"`
}

// PortConfig represents the config of a port.
type PortConfig struct {
	Name     string `json:",omitempty"`
	Protocol int32  `json:",omitempty"`
	// TargetPort is the port inside the container
	TargetPort uint32 `json:",omitempty"`
	// PublishedPort is the port on the swarm hosts
	PublishedPort uint32 `json:",omitempty"`
	// PublishMode is the mode in which port is published
	PublishMode int32 `json:",omitempty"`
}

// Driver represent the interface a driver must fulfill.
type Driver interface {
	// Get gets a secret from a remote secret store
	Get(Request) Response
}

// Handler forwards requests and responses between the docker daemon and the plugin.
type Handler struct {
	driver Driver
	sdk.Handler
}

// NewHandler initializes the request handler with a driver implementation.
func NewHandler(driver Driver) *Handler {
	h := &Handler{driver, sdk.NewHandler(manifest)}
	h.initMux()
	return h
}

func (h *Handler) initMux() {
	h.HandleFunc(getPath, func(w http.ResponseWriter, r *http.Request) {
		var req Request
		if err := sdk.DecodeRequest(w, r, &req); err != nil {
			return
		}
		res := h.driver.Get(req)
		sdk.EncodeResponse(w, res, res.Err != "")
	})
}
<commit_msg>Add TaskImage while we're at it<commit_after>package secrets

import (
	"net/http"

	"github.com/docker/go-plugins-helpers/sdk"
)

const (
	manifest = `{"Implements": ["secretprovider"]}`
	getPath  = "/SecretProvider.GetSecret"
)

// Request is the plugin secret request
type Request struct {
	SecretName          string            `json:",omitempty"` // SecretName is the name of the secret to request from the plugin
	SecretLabels        map[string]string `json:",omitempty"` // SecretLabels capture environment names and other metadata pertaining to the secret
	ServiceHostname     string            `json:",omitempty"` // ServiceHostname is the hostname of the service, can be used for x509 certificate
	ServiceName         string            `json:",omitempty"` // ServiceName is the name of the service that requested the secret
	ServiceID           string            `json:",omitempty"` // ServiceID is the name of the service that requested the secret
	ServiceLabels       map[string]string `json:",omitempty"` // ServiceLabels capture environment names and other metadata pertaining to the service
	TaskID              string            `json:",omitempty"` // TaskID is the ID of the task that the secret is assigned to
	TaskName            string            `json:",omitempty"` // TaskName is the name of the task that the secret is assigned to
	TaskImage           string            `json:",omitempty"` // TaskName is the image of the task that the secret is assigned to
	ServiceEndpointSpec *EndpointSpec     `json:",omitempty"` // ServiceEndpointSpec holds the specification for endpoints
}

// Response contains the plugin secret value
type Response struct {
	Value []byte `json:",omitempty"` // Value is the value of the secret
	Err   string `json:",omitempty"` // Err is the error response of the plugin
}

// EndpointSpec represents the spec of an endpoint.
type EndpointSpec struct {
	Mode  int32        `json:",omitempty"`
	Ports []PortConfig `json:",omitempty"`
}

// PortConfig represents the config of a port.
type PortConfig struct {
	Name     string `json:",omitempty"`
	Protocol int32  `json:",omitempty"`
	// TargetPort is the port inside the container
	TargetPort uint32 `json:",omitempty"`
	// PublishedPort is the port on the swarm hosts
	PublishedPort uint32 `json:",omitempty"`
	// PublishMode is the mode in which port is published
	PublishMode int32 `json:",omitempty"`
}

// Driver represent the interface a driver must fulfill.
type Driver interface {
	// Get gets a secret from a remote secret store
	Get(Request) Response
}

// Handler forwards requests and responses between the docker daemon and the plugin.
type Handler struct {
	driver Driver
	sdk.Handler
}

// NewHandler initializes the request handler with a driver implementation.
func NewHandler(driver Driver) *Handler {
	h := &Handler{driver, sdk.NewHandler(manifest)}
	h.initMux()
	return h
}

func (h *Handler) initMux() {
	h.HandleFunc(getPath, func(w http.ResponseWriter, r *http.Request) {
		var req Request
		if err := sdk.DecodeRequest(w, r, &req); err != nil {
			return
		}
		res := h.driver.Get(req)
		sdk.EncodeResponse(w, res, res.Err != "")
	})
}
<|endoftext|>


<commit_before>package dbr

import (
	"bytes"
	"database/sql"
	"fmt"
	"reflect"
	"time"
)

// InsertBuilder contains the clauses for an INSERT statement
type InsertBuilder struct {
	*Session
	runner

	Into string
	Cols []string
	Vals [][]interface{}
	Recs []interface{}
}

// InsertInto instantiates a InsertBuilder for the given table
func (sess *Session) InsertInto(into string) *InsertBuilder {
	return &InsertBuilder{
		Session: sess,
		runner:  sess.cxn.Db,
		Into:    into,
	}
}

// InsertInto instantiates a InsertBuilder for the given table bound to a transaction
func (tx *Tx) InsertInto(into string) *InsertBuilder {
	return &InsertBuilder{
		Session: tx.Session,
		runner:  tx.Tx,
		Into:    into,
	}
}

// Columns appends columns to insert in the statement
func (b *InsertBuilder) Columns(columns ...string) *InsertBuilder {
	b.Cols = columns
	return b
}

// Values appends a set of values to the statement
func (b *InsertBuilder) Values(vals ...interface{}) *InsertBuilder {
	b.Vals = append(b.Vals, vals)
	return b
}

// Record pulls in values to match Columns from the record
func (b *InsertBuilder) Record(record interface{}) *InsertBuilder {
	b.Recs = append(b.Recs, record)
	return b
}

// Pair adds a key/value pair to the statement
func (b *InsertBuilder) Pair(column string, value interface{}) *InsertBuilder {
	b.Cols = append(b.Cols, column)
	lenVals := len(b.Vals)
	if lenVals == 0 {
		args := []interface{}{value}
		b.Vals = [][]interface{}{args}
	} else if lenVals == 1 {
		b.Vals[0] = append(b.Vals[0], value)
	} else {
		panic("pair only allows you to specify 1 record to insret")
	}
	return b
}

// ToSql serialized the InsertBuilder to a SQL string
// It returns the string with placeholders and a slice of query arguments
func (b *InsertBuilder) ToSql() (string, []interface{}) {
	if len(b.Into) == 0 {
		panic("no table specified")
	}
	if len(b.Cols) == 0 {
		panic("no columns specified")
	}
	if len(b.Vals) == 0 && len(b.Recs) == 0 {
		panic("no values or records specified")
	}

	var sql bytes.Buffer
	var placeholder bytes.Buffer // Build the placeholder like "(?,?,?)"
	var args []interface{}

	sql.WriteString("INSERT INTO ")
	sql.WriteString(b.Into)
	sql.WriteString(" (")

	// Simulataneously write the cols to the sql buffer, and build a placeholder
	placeholder.WriteRune('(')
	for i, c := range b.Cols {
		if i > 0 {
			sql.WriteRune(',')
			placeholder.WriteRune(',')
		}
		Quoter.writeQuotedColumn(c, &sql)
		placeholder.WriteRune('?')
	}
	sql.WriteString(") VALUES ")
	placeholder.WriteRune(')')
	placeholderStr := placeholder.String()

	// Go thru each value we want to insert. Write the placeholders, and collect args
	for i, row := range b.Vals {
		if i > 0 {
			sql.WriteRune(',')
		}
		sql.WriteString(placeholderStr)

		for _, v := range row {
			args = append(args, v)
		}
	}
	anyVals := len(b.Vals) > 0

	// Go thru the records. Write the placeholders, and do reflection on the records to extract args
	for i, rec := range b.Recs {
		if i > 0 || anyVals {
			sql.WriteRune(',')
		}
		sql.WriteString(placeholderStr)

		ind := reflect.Indirect(reflect.ValueOf(rec))
		vals, err := b.valuesFor(ind.Type(), ind, b.Cols)
		if err != nil {
			panic(err.Error())
		}
		for _, v := range vals {
			args = append(args, v)
		}
	}

	return sql.String(), args
}

// Exec executes the statement represented by the InsertBuilder
// It returns the raw database/sql Result and an error if there was one
func (b *InsertBuilder) Exec() (sql.Result, error) {
	sql, args := b.ToSql()

	fullSql, err := Interpolate(sql, args)
	if err != nil {
		return nil, b.EventErrKv("dbr.insert.exec.interpolate", err, kvs{"sql": sql, "args": fmt.Sprint(args)})
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.insert", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	result, err := b.runner.Exec(fullSql)
	if err != nil {
		return result, b.EventErrKv("dbr.insert.exec.exec", err, kvs{"sql": fullSql})
	}

	// If the structure has an "Id" field which is an int64, set it from the LastInsertId(). Otherwise, don't bother.
	if len(b.Recs) == 1 {
		rec := b.Recs[0]
		val := reflect.Indirect(reflect.ValueOf(rec))
		if val.Kind() == reflect.Struct && val.CanSet() {
			if idField := val.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.Int64 {
				if lastID, err := result.LastInsertId(); err == nil {
					idField.Set(reflect.ValueOf(lastID))
				} else {
					b.EventErrKv("dbr.insert.exec.last_inserted_id", err, kvs{"sql": fullSql})
				}
			}
		}
	}

	return result, nil
}
<commit_msg>Add Maps prop & InsertMap() func to support insert a record directly from a map<commit_after>package dbr

import (
	"bytes"
	"database/sql"
	"fmt"
	"reflect"
	"time"
)

// InsertBuilder contains the clauses for an INSERT statement
type InsertBuilder struct {
	*Session
	runner

	Into string
	Cols []string
	Vals [][]interface{}
	Recs []interface{}
	Maps map[string]interface{}
}

// InsertInto instantiates a InsertBuilder for the given table
func (sess *Session) InsertInto(into string) *InsertBuilder {
	return &InsertBuilder{
		Session: sess,
		runner:  sess.cxn.Db,
		Into:    into,
	}
}

// InsertInto instantiates a InsertBuilder for the given table bound to a transaction
func (tx *Tx) InsertInto(into string) *InsertBuilder {
	return &InsertBuilder{
		Session: tx.Session,
		runner:  tx.Tx,
		Into:    into,
	}
}

// Columns appends columns to insert in the statement
func (b *InsertBuilder) Columns(columns ...string) *InsertBuilder {
	b.Cols = columns
	return b
}

// Values appends a set of values to the statement
func (b *InsertBuilder) Values(vals ...interface{}) *InsertBuilder {
	b.Vals = append(b.Vals, vals)
	return b
}

// Record pulls in values to match Columns from the record
func (b *InsertBuilder) Record(record interface{}) *InsertBuilder {
	b.Recs = append(b.Recs, record)
	return b
}

// Record pulls in values to match Columns from the record
func (b *InsertBuilder) Map(m map[string]interface{}) *InsertBuilder {
	b.Maps = m
	return b
}

// Pair adds a key/value pair to the statement
func (b *InsertBuilder) Pair(column string, value interface{}) *InsertBuilder {
	b.Cols = append(b.Cols, column)
	lenVals := len(b.Vals)
	if lenVals == 0 {
		args := []interface{}{value}
		b.Vals = [][]interface{}{args}
	} else if lenVals == 1 {
		b.Vals[0] = append(b.Vals[0], value)
	} else {
		panic("pair only allows you to specify 1 record to insret")
	}
	return b
}

// ToSql serialized the InsertBuilder to a SQL string
// It returns the string with placeholders and a slice of query arguments
func (b *InsertBuilder) ToSql() (string, []interface{}) {
	if len(b.Into) == 0 {
		panic("no table specified")
	}
	if len(b.Cols) == 0 && len(b.Maps) == 0 {
		panic("no columns or map specified")
	} else if len(b.Maps) == 0 {
		if len(b.Vals) == 0 && len(b.Recs) == 0 {
			panic("no values or records specified")
		}
		if len(b.Cols) == 0 && (len(b.Vals) > 0 || len(b.Recs) > 0) {
			panic("no columns specified")
		}
	}

	var sql bytes.Buffer

	sql.WriteString("INSERT INTO ")
	sql.WriteString(b.Into)
	sql.WriteString(" (")

	if len(b.Maps) != 0 {
		return b.MapToSql(sql)
	}

	var args []interface{}
	var placeholder bytes.Buffer // Build the placeholder like "(?,?,?)"
	// Simulataneously write the cols to the sql buffer, and build a placeholder
	placeholder.WriteRune('(')
	for i, c := range b.Cols {
		if i > 0 {
			sql.WriteRune(',')
			placeholder.WriteRune(',')
		}
		Quoter.writeQuotedColumn(c, &sql)
		placeholder.WriteRune('?')
	}
	sql.WriteString(") VALUES ")
	placeholder.WriteRune(')')
	placeholderStr := placeholder.String()

	// Go thru each value we want to insert. Write the placeholders, and collect args
	for i, row := range b.Vals {
		if i > 0 {
			sql.WriteRune(',')
		}
		sql.WriteString(placeholderStr)

		for _, v := range row {
			args = append(args, v)
		}
	}
	anyVals := len(b.Vals) > 0

	// Go thru the records. Write the placeholders, and do reflection on the records to extract args
	for i, rec := range b.Recs {
		if i > 0 || anyVals {
			sql.WriteRune(',')
		}
		sql.WriteString(placeholderStr)

		ind := reflect.Indirect(reflect.ValueOf(rec))
		vals, err := b.valuesFor(ind.Type(), ind, b.Cols)
		if err != nil {
			panic(err.Error())
		}
		for _, v := range vals {
			args = append(args, v)
		}
	}

	return sql.String(), args
}

// MapToSql serialized the InsertBuilder to a SQL string
// It goes through the Maps param and combined its keys/values into the SQL query string
// It returns the string with placeholders and a slice of query arguments
func (b *InsertBuilder) MapToSql(sql bytes.Buffer) (string, []interface{}) {
	keys := make([]string, len(b.Maps))
	vals := make([]interface{}, len(b.Maps))
	i := 0
	for k, v := range b.Maps {
		keys[i] = k
		vals[i] = v
		i++
	}
	var args []interface{}
	var placeholder bytes.Buffer // Build the placeholder like "(?,?,?)"

	placeholder.WriteRune('(')
	for i, c := range keys {
		if i > 0 {
			sql.WriteRune(',')
			placeholder.WriteRune(',')
		}
		Quoter.writeQuotedColumn(c, &sql)
		placeholder.WriteRune('?')
	}
	sql.WriteString(") VALUES ")
	placeholder.WriteRune(')')
	sql.WriteString(placeholder.String())

	for _, row := range vals {
		args = append(args, row)
	}

	return sql.String(), args
}

// Exec executes the statement represented by the InsertBuilder
// It returns the raw database/sql Result and an error if there was one
func (b *InsertBuilder) Exec() (sql.Result, error) {
	sql, args := b.ToSql()

	fullSql, err := Interpolate(sql, args)
	if err != nil {
		return nil, b.EventErrKv("dbr.insert.exec.interpolate", err, kvs{"sql": sql, "args": fmt.Sprint(args)})
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.insert", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	result, err := b.runner.Exec(fullSql)
	if err != nil {
		return result, b.EventErrKv("dbr.insert.exec.exec", err, kvs{"sql": fullSql})
	}

	// If the structure has an "Id" field which is an int64, set it from the LastInsertId(). Otherwise, don't bother.
	if len(b.Recs) == 1 {
		rec := b.Recs[0]
		val := reflect.Indirect(reflect.ValueOf(rec))
		if val.Kind() == reflect.Struct && val.CanSet() {
			if idField := val.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.Int64 {
				if lastID, err := result.LastInsertId(); err == nil {
					idField.Set(reflect.ValueOf(lastID))
				} else {
					b.EventErrKv("dbr.insert.exec.last_inserted_id", err, kvs{"sql": fullSql})
				}
			}
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>package fleet

import (
	"crypto/rand"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	fleetClient "github.com/coreos/fleet/client"
	"github.com/monder/kaylee/engine"
	"github.com/monder/kaylee/spec"
	"strings"
	"time"
)

type Fleet struct {
	API    fleetClient.API
	Prefix string
}

func (fleet *Fleet) ScheduleUnit(unit *spec.Spec, force bool) {
	specData, _ := json.Marshal(unit)
	specHash := sha1.Sum(specData)

	replicaUnit := true
	if unit.Replicas == 0 {
		unit.Replicas = 1
		replicaUnit = false
	}
	if unit.MaxReplicasPerHost == 0 {
		unit.MaxReplicasPerHost = unit.Replicas
	}
	// Make a list of units we should replace
	var unitsToRemove []string
	existingUnits, _ := fleet.API.Units()
	for _, u := range existingUnits {
		if strings.HasPrefix(u.Name, fmt.Sprintf("%s:%s:", fleet.Prefix, unit.Name)) {
			if !force && strings.HasPrefix(u.Name, fmt.Sprintf("%s:%s:%x:", fleet.Prefix, unit.Name, specHash[:3])) {
				// If the unit is already somewhere in the cluster
				return
			}
			unitsToRemove = append(unitsToRemove, u.Name)
		}
	}

	// Generate unique ids based on replica count and max replicas
	conflictIds := make([]string, unit.MaxReplicasPerHost)
	for i := 0; i < unit.MaxReplicasPerHost; i++ {
		r := make([]byte, 3)
		rand.Read(r) //TODO err
		conflictIds[i] = fmt.Sprintf("%x", r)
	}

	// Schedule replicas
	for i := 1; i <= unit.Replicas; i++ {
		unitName := fmt.Sprintf("%s:%s:%x:%s:%d.service",
			fleet.Prefix, unit.Name, specHash[:3], conflictIds[i%len(conflictIds)], i)
		var conflictStrings []string
		if replicaUnit {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:%x:%s:*.service",
				fleet.Prefix, unit.Name, specHash[:3], conflictIds[i%len(conflictIds)]))
		} else {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:*.service", fleet.Prefix, unit.Name))
		}
		for _, c := range unit.Conflicts {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:*.service", fleet.Prefix, c))
		}
		fleetUnit, err := engine.GetFleetUnit(unit, unitName, conflictStrings)
		if err != nil {
			fmt.Println("Unable to create unit:", err)
		}
		err = fleet.API.CreateUnit(fleetUnit)
		if err != nil {
			fmt.Println("Unable to create unit:", err)
		}
		fleet.waitForUnitStart(unitName)
		if len(unitsToRemove) > 0 {
			fmt.Println("Deleting unit:", unitsToRemove[0])
			fleet.API.DestroyUnit(unitsToRemove[0])
			unitsToRemove = unitsToRemove[1:]
		} else {
			fmt.Println("No more units to remove")
		}
	}
	for _, unit := range unitsToRemove {
		fmt.Println("Deleting unit:", unit)
		fleet.API.DestroyUnit(unit)
	}
}

func (fleet *Fleet) waitForUnitStart(name string) {
	fmt.Println("Waiting for unit to start:", name)
	prevState := "undefined"
	for i := 0; i < 60; i++ {
		currentState := "unknown"
		states, err := fleet.API.UnitStates()
		if err != nil {
			fmt.Println("Unable to retrieve unit state")
			continue
		}
		for _, state := range states {
			if state.Name == name {
				currentState = state.SystemdSubState
				break
			}
		}
		if currentState != prevState {
			if prevState != "undefined" {
				fmt.Print("\n")
			}
			fmt.Printf("%s: ", currentState)
			prevState = currentState
		} else {
			fmt.Print(".")
		}
		if currentState == "running" {
			fmt.Print("\n")
			fmt.Println("Unit started:", name)
			return
		}
		time.Sleep(time.Second)
	}
	fmt.Println("Unable to schedule unit:", name)
}
<commit_msg>Destroy first for non replica units.<commit_after>package fleet

import (
	"crypto/rand"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	fleetClient "github.com/coreos/fleet/client"
	"github.com/monder/kaylee/engine"
	"github.com/monder/kaylee/spec"
	"strings"
	"time"
)

type Fleet struct {
	API    fleetClient.API
	Prefix string
}

func (fleet *Fleet) ScheduleUnit(unit *spec.Spec, force bool) {
	specData, _ := json.Marshal(unit)
	specHash := sha1.Sum(specData)

	replicaUnit := true
	if unit.Replicas == 0 {
		unit.Replicas = 1
		replicaUnit = false
	}

	if unit.MaxReplicasPerHost == 0 {
		unit.MaxReplicasPerHost = unit.Replicas
	}
	// Make a list of units we should replace
	var unitsToRemove []string
	existingUnits, _ := fleet.API.Units()
	for _, u := range existingUnits {
		if strings.HasPrefix(u.Name, fmt.Sprintf("%s:%s:", fleet.Prefix, unit.Name)) {
			if !force && strings.HasPrefix(u.Name, fmt.Sprintf("%s:%s:%x:", fleet.Prefix, unit.Name, specHash[:3])) {
				// If the unit is already somewhere in the cluster
				return
			}
			unitsToRemove = append(unitsToRemove, u.Name)
		}
	}

	// Generate unique ids based on replica count and max replicas
	conflictIds := make([]string, unit.MaxReplicasPerHost)
	for i := 0; i < unit.MaxReplicasPerHost; i++ {
		r := make([]byte, 3)
		rand.Read(r) //TODO err
		conflictIds[i] = fmt.Sprintf("%x", r)
	}

	// Schedule replicas
	for i := 1; i <= unit.Replicas; i++ {
		unitName := fmt.Sprintf("%s:%s:%x:%s:%d.service",
			fleet.Prefix, unit.Name, specHash[:3], conflictIds[i%len(conflictIds)], i)
		var conflictStrings []string
		if replicaUnit {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:%x:%s:*.service",
				fleet.Prefix, unit.Name, specHash[:3], conflictIds[i%len(conflictIds)]))
		} else {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:*.service", fleet.Prefix, unit.Name))
		}
		for _, c := range unit.Conflicts {
			conflictStrings = append(conflictStrings, fmt.Sprintf("%s:%s:*.service", fleet.Prefix, c))
		}
		fleetUnit, err := engine.GetFleetUnit(unit, unitName, conflictStrings)
		if err != nil {
			fmt.Println("Unable to create unit:", err)
		}

		if !replicaUnit { // If its not replica unit - first delete
			if len(unitsToRemove) > 0 {
				fmt.Println("Deleting unit:", unitsToRemove[0])
				fleet.destroyFleetUnit(unitsToRemove[0])
				unitsToRemove = unitsToRemove[1:]
			}
		}

		err = fleet.API.CreateUnit(fleetUnit)
		if err != nil {
			fmt.Println("Unable to create unit:", err)
		}
		err = fleet.waitForUnitStart(unitName)
		if err != nil {
			fmt.Println("Unable to schedule unit:", err)
			return //TODO
		}
		if replicaUnit { // If it is a replica unit - destroy after successfull start
			if len(unitsToRemove) > 0 {
				fmt.Println("Deleting unit:", unitsToRemove[0])
				fleet.destroyFleetUnit(unitsToRemove[0])
				unitsToRemove = unitsToRemove[1:]
			} else {
				fmt.Println("No more units to remove")
			}
		}
	}
	for _, unit := range unitsToRemove {
		fmt.Println("Deleting unit:", unit)
		fleet.destroyFleetUnit(unit)
	}
}

func (fleet *Fleet) destroyFleetUnit(name string) {
	//fmt.Println("Stopping:", name)
	//fleet.API.SetUnitTargetState(name, "loaded")
	//time.Sleep(30 * time.Second) // TODO wait for stop

	// https://github.com/coreos/fleet/issues/1000
	fmt.Println("Destroying:", name)
	fleet.API.DestroyUnit(name)
}

func (fleet *Fleet) waitForUnitStart(name string) error {
	fmt.Println("Waiting for unit to start:", name)
	prevState := "undefined"
	for i := 0; i < 300; i++ {
		currentState := "unknown"
		states, err := fleet.API.UnitStates()
		if err != nil {
			fmt.Println("Unable to retrieve unit state")
			continue
		}
		for _, state := range states {
			if state.Name == name {
				currentState = state.SystemdSubState
				break
			}
		}
		if currentState != prevState {
			if prevState != "undefined" {
				fmt.Print("\n")
			}
			fmt.Printf("%s: ", currentState)
			prevState = currentState
		} else {
			fmt.Print(".")
		}
		if currentState == "running" {
			fmt.Print("\n")
			fmt.Println("Unit started:", name)
			return nil
		}
		time.Sleep(time.Second)
	}
	return fmt.Errorf("Unit %s failed to start after 5 minutes", name)
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strings"
	"unicode/utf16"
)

func readTFrame(b []byte) (string, error) {
	txt, err := parseText(b)
	if err != nil {
		return "", err
	}
	return strings.Join(strings.Split(txt, string([]byte{0})), ""), nil
}

func parseText(b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}
	return decodeText(b[0], b[1:])
}

func decodeText(enc byte, b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}

	switch enc {
	case 0: // ISO-8859-1
		return decodeISO8859(b), nil

	case 1: // UTF-16 with byte order marker
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16WithBOM(b)

	case 2: // UTF-16 without byte order (assuming BigEndian)
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16(b, binary.BigEndian), nil

	case 3: // UTF-8
		return string(b), nil

	default:
		return "", fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func encodingDelim(enc byte) ([]byte, error) {
	switch enc {
	case 0, 3: // see decodeText above
		return []byte{0}, nil
	case 1, 2: // see decodeText above
		return []byte{0, 0}, nil
	default:
		return nil, fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func decodeISO8859(b []byte) string {
	r := make([]rune, len(b))
	for i, x := range b {
		r[i] = rune(x)
	}
	return string(r)
}

func decodeUTF16WithBOM(b []byte) (string, error) {
	var bo binary.ByteOrder
	switch {
	case b[0] == 0xFE && b[1] == 0xFF:
		bo = binary.BigEndian

	case b[0] == 0xFF && b[1] == 0xFE:
		bo = binary.LittleEndian

	default:
		return "", fmt.Errorf("invalid byte order marker %x %x", b[0], b[1])
	}
	return decodeUTF16(b[2:], bo), nil
}

func decodeUTF16(b []byte, bo binary.ByteOrder) string {
	s := make([]uint16, 0, len(b)/2)
	for i := 0; i < len(b); i += 2 {
		s = append(s, bo.Uint16(b[i:i+2]))
	}
	return string(utf16.Decode(s))
}

// Comm is a type used in COMM and USLT tag. It's a text with a description and
// a specified language
type Comm struct {
	Language    string
	Description string
	Text        string
}

// String returns a string representation of the underlying Comm instance.
func (t Comm) String() string {
	return fmt.Sprintf("Text{Lang: '%v', Description: '%v', %v lines}",
		t.Language, t.Description, strings.Count(t.Text, "\n"))
}

// IDv2.{3,4}
// -- Header
// <Header for 'Unsynchronised lyrics/text transcription', ID: "USLT">
// <Header for 'Comment', ID: "COMM">
// -- readTextWithDescrFrame
// Text encoding       $xx
// Language            $xx xx xx
// Content descriptor  <text string according to encoding> $00 (00)
// Lyrics/text         <full text string according to encoding>
func readTextWithDescrFrame(b []byte) (*Comm, error) {
	enc := b[0]
	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}

	descTextSplit := bytes.SplitN(b[4:], delim, 2)
	desc, err := decodeText(enc, descTextSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag description text: %v", err)
	}

	text, err := decodeText(enc, descTextSplit[1])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag text: %v", err)
	}

	return &Comm{
		Language:    string(b[1:4]),
		Description: desc,
		Text:        text,
	}, nil
}

var pictureTypes = map[byte]string{
	0x00: "Other",
	0x01: "32x32 pixels 'file icon' (PNG only)",
	0x02: "Other file icon",
	0x03: "Cover (front)",
	0x04: "Cover (back)",
	0x05: "Leaflet page",
	0x06: "Media (e.g. lable side of CD)",
	0x07: "Lead artist/lead performer/soloist",
	0x08: "Artist/performer",
	0x09: "Conductor",
	0x0A: "Band/Orchestra",
	0x0B: "Composer",
	0x0C: "Lyricist/text writer",
	0x0D: "Recording Location",
	0x0E: "During recording",
	0x0F: "During performance",
	0x10: "Movie/video screen capture",
	0x11: "A bright coloured fish",
	0x12: "Illustration",
	0x13: "Band/artist logotype",
	0x14: "Publisher/Studio logotype",
}

// Picture is a type which represents an attached picture extracted from metadata.
type Picture struct {
	Ext         string // Extension of the picture file.
	MIMEType    string // MIMEType of the picture.
	Type        string // Type of the picture (see pictureTypes).
	Description string // Description.
	Data        []byte // Raw picture data.
}

// String returns a string representation of the underlying Picture instance.
func (p Picture) String() string {
	return fmt.Sprintf("Picture{Ext: %v, MIMEType: %v, Type: %v, Description: %v, Data.Size: %v}",
		p.Ext, p.MIMEType, p.Type, p.Description, len(p.Data))
}

// IDv2.2
// -- Header
// Attached picture   "PIC"
// Frame size         $xx xx xx
// -- readPICFrame
// Text encoding      $xx
// Image format       $xx xx xx
// Picture type       $xx
// Description        <textstring> $00 (00)
// Picture data       <binary data>
func readPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	ext := string(b[1:4])
	picType := b[4]

	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}

	descDataSplit := bytes.SplitN(b[5:], delim, 2)
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding PIC description text: %v", err)
	}

	var mimeType string
	switch ext {
	case "jpeg", "jpg":
		mimeType = "image/jpeg"
	case "png":
		mimeType = "image/png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}

// IDv2.{3,4}
// -- Header
// <Header for 'Attached picture', ID: "APIC">
// -- readAPICFrame
// Text encoding   $xx
// MIME type       <text string> $00
// Picture type    $xx
// Description     <text string according to encoding> $00 (00)
// Picture data    <binary data>
func readAPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	mimeDataSplit := bytes.SplitN(b[1:], []byte{0}, 2)
	mimeType := string(mimeDataSplit[0])

	b = mimeDataSplit[1]
	picType := b[0]

	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}

	descDataSplit := bytes.SplitN(b[1:], delim, 2)
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding APIC description text: %v", err)
	}

	var ext string
	switch mimeType {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}
<commit_msg>Fix bug in the split method<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strings"
	"unicode/utf16"
)

func readTFrame(b []byte) (string, error) {
	txt, err := parseText(b)
	if err != nil {
		return "", err
	}
	return strings.Join(strings.Split(txt, string([]byte{0})), ""), nil
}

func parseText(b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}
	return decodeText(b[0], b[1:])
}

func decodeText(enc byte, b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}

	switch enc {
	case 0: // ISO-8859-1
		return decodeISO8859(b), nil

	case 1: // UTF-16 with byte order marker
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16WithBOM(b)

	case 2: // UTF-16 without byte order (assuming BigEndian)
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16(b, binary.BigEndian), nil

	case 3: // UTF-8
		return string(b), nil

	default:
		return "", fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func encodingDelim(enc byte) ([]byte, error) {
	switch enc {
	case 0, 3: // see decodeText above
		return []byte{0}, nil
	case 1, 2: // see decodeText above
		return []byte{0, 0}, nil
	default:
		return nil, fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func dataSplit(b []byte, enc byte) ([][]byte, error) {
	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}
	result := bytes.SplitN(b, delim, 2)

	if len(result) <= 1 {
		return result, nil
	}

	if result[1][0] == 0 {
		// there was a double (or triple) 0 and we cut too early
		result[0] = append(result[0], make([]byte, 0)...)
		result[1] = result[1][1:]
	}

	return result, nil
}

func decodeISO8859(b []byte) string {
	r := make([]rune, len(b))
	for i, x := range b {
		r[i] = rune(x)
	}
	return string(r)
}

func decodeUTF16WithBOM(b []byte) (string, error) {
	var bo binary.ByteOrder
	switch {
	case b[0] == 0xFE && b[1] == 0xFF:
		bo = binary.BigEndian

	case b[0] == 0xFF && b[1] == 0xFE:
		bo = binary.LittleEndian

	default:
		return "", fmt.Errorf("invalid byte order marker %x %x", b[0], b[1])
	}
	return decodeUTF16(b[2:], bo), nil
}

func decodeUTF16(b []byte, bo binary.ByteOrder) string {
	s := make([]uint16, 0, len(b)/2)
	for i := 0; i < len(b); i += 2 {
		s = append(s, bo.Uint16(b[i:i+2]))
	}
	return string(utf16.Decode(s))
}

// Comm is a type used in COMM and USLT tag. It's a text with a description and
// a specified language
type Comm struct {
	Language    string
	Description string
	Text        string
}

// String returns a string representation of the underlying Comm instance.
func (t Comm) String() string {
	return fmt.Sprintf("Text{Lang: '%v', Description: '%v', %v lines}",
		t.Language, t.Description, strings.Count(t.Text, "\n"))
}

// IDv2.{3,4}
// -- Header
// <Header for 'Unsynchronised lyrics/text transcription', ID: "USLT">
// <Header for 'Comment', ID: "COMM">
// -- readTextWithDescrFrame
// Text encoding       $xx
// Language            $xx xx xx
// Content descriptor  <text string according to encoding> $00 (00)
// Lyrics/text         <full text string according to encoding>
func readTextWithDescrFrame(b []byte) (*Comm, error) {
	enc := b[0]

	descTextSplit, err := dataSplit(b[4:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descTextSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag description text: %v", err)
	}

	text, err := decodeText(enc, descTextSplit[1])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag text: %v", err)
	}

	return &Comm{
		Language:    string(b[1:4]),
		Description: desc,
		Text:        text,
	}, nil
}

var pictureTypes = map[byte]string{
	0x00: "Other",
	0x01: "32x32 pixels 'file icon' (PNG only)",
	0x02: "Other file icon",
	0x03: "Cover (front)",
	0x04: "Cover (back)",
	0x05: "Leaflet page",
	0x06: "Media (e.g. lable side of CD)",
	0x07: "Lead artist/lead performer/soloist",
	0x08: "Artist/performer",
	0x09: "Conductor",
	0x0A: "Band/Orchestra",
	0x0B: "Composer",
	0x0C: "Lyricist/text writer",
	0x0D: "Recording Location",
	0x0E: "During recording",
	0x0F: "During performance",
	0x10: "Movie/video screen capture",
	0x11: "A bright coloured fish",
	0x12: "Illustration",
	0x13: "Band/artist logotype",
	0x14: "Publisher/Studio logotype",
}

// Picture is a type which represents an attached picture extracted from metadata.
type Picture struct {
	Ext         string // Extension of the picture file.
	MIMEType    string // MIMEType of the picture.
	Type        string // Type of the picture (see pictureTypes).
	Description string // Description.
	Data        []byte // Raw picture data.
}

// String returns a string representation of the underlying Picture instance.
func (p Picture) String() string {
	return fmt.Sprintf("Picture{Ext: %v, MIMEType: %v, Type: %v, Description: %v, Data.Size: %v}",
		p.Ext, p.MIMEType, p.Type, p.Description, len(p.Data))
}

// IDv2.2
// -- Header
// Attached picture   "PIC"
// Frame size         $xx xx xx
// -- readPICFrame
// Text encoding      $xx
// Image format       $xx xx xx
// Picture type       $xx
// Description        <textstring> $00 (00)
// Picture data       <binary data>
func readPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	ext := string(b[1:4])
	picType := b[4]

	descDataSplit, err := dataSplit(b[5:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding PIC description text: %v", err)
	}

	var mimeType string
	switch ext {
	case "jpeg", "jpg":
		mimeType = "image/jpeg"
	case "png":
		mimeType = "image/png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}

// IDv2.{3,4}
// -- Header
// <Header for 'Attached picture', ID: "APIC">
// -- readAPICFrame
// Text encoding   $xx
// MIME type       <text string> $00
// Picture type    $xx
// Description     <text string according to encoding> $00 (00)
// Picture data    <binary data>
func readAPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	mimeDataSplit := bytes.SplitN(b[1:], []byte{0}, 2)
	mimeType := string(mimeDataSplit[0])

	b = mimeDataSplit[1]
	picType := b[0]

	descDataSplit, err := dataSplit(b[1:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding APIC description text: %v", err)
	}

	var ext string
	switch mimeType {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"reflect"
	"testing"
)

type unOneTestCase struct {
	ok   bool
	in   []byte
	want interface{}
}

func unOneTestCases() map[string]unOneTestCase {
	c := map[string]unOneTestCase{
		"ok": {
			ok:   true,
			in:   []byte(`{"i":123, "f":123.456, "b":true,"s":"abc"}`),
			want: One{I: int64(123), F: float64(123.456), B: true, S: "abc"},
		},
	}
	return c
}

func TestUnmarshal(t *testing.T) {
	for name, tc := range unOneTestCases() {
		t.Run(name, func(t *testing.T) {
			testUnmarshal(t, tc)
		})
	}
}

func testUnmarshal(t *testing.T, tc unOneTestCase) {
	var got One
	err := json.Unmarshal(tc.in, &got)
	if !tc.ok {
		if !reflect.DeepEqual(got, tc.want) || err == nil {
			t.Errorf("got %+v, %v; want %+v, <error>", got, err, tc.want)
		}
		return
	}
	if !reflect.DeepEqual(got, tc.want) || err != nil {
		t.Errorf("got %+v, %+v; want %+v, <nil>", got, err, tc.want)
	}
}
<commit_msg>jj: fix tests<commit_after>package main

import (
	"encoding/json"
	"reflect"
	"testing"
)

type unOneTestCase struct {
	ok   bool
	in   []byte
	want interface{}
}

func unOneTestCases() map[string]unOneTestCase {
	c := map[string]unOneTestCase{
		"ok": {
			ok:   true,
			in:   []byte(`{"i":123, "f":123.456, "b":true,"s":"abc"}`),
			want: One{I: 123, F: 123.456, B: true, S: "abc"},
		},
	}
	return c
}

func TestUnmarshal(t *testing.T) {
	for name, tc := range unOneTestCases() {
		t.Run(name, func(t *testing.T) {
			testUnmarshal(t, tc)
		})
	}
}

func testUnmarshal(t *testing.T, tc unOneTestCase) {
	var got One
	//in := bytes.Replace(tc.in, []byte(":NaN"), []byte(":null"), -1)
	err := json.Unmarshal(tc.in, &got)
	if !tc.ok {
		if !reflect.DeepEqual(got, tc.want) || err == nil {
			t.Errorf("got %+v, %v; want %+v, <error>", got, err, tc.want)
		}
		return
	}
	if !reflect.DeepEqual(got, tc.want) || err != nil {
		t.Errorf("got %+v, %+v; want %+v, <nil>", got, err, tc.want)
	}
}
<|endoftext|>


<commit_before>package imux

import (
	log "github.com/Sirupsen/logrus"
	"github.com/hkparker/TLJ"
	"net"
	"reflect"
	"sync"
	"time"
)

// A function that can be called by an IMUXSocket to reconnect after an error
type Redialer func() (net.Conn, error)

// A function that generates Redialers for specific bind addresses
type RedialerGenerator func(string) Redialer

// A map of all TLJ servers used to read chunks back from sessions
var sessionResponsesTLJServers = make(map[string]tlj.Server)
var srtsMux sync.Mutex

// A client socket that transports data in an imux session, autoreconnecting
type IMUXSocket struct {
	IMUXer   DataIMUX
	Redialer Redialer
}

// Dial a new connection in an imux session, creating a TLJ server for
// responses if needed.  Read data from the sockets IMUXer and write it up.
func (imux_socket *IMUXSocket) init(session_id string) {
	log.WithFields(log.Fields{
		"at": "IMUXSocket.init",
	}).Debug("starting imux socket")
	tlj_server := imuxClientSocketTLJServer(session_id)
	cooldown := 10 * time.Second
	for {
		log.WithFields(log.Fields{
			"at": "IMUXSocket.init",
		}).Debug("dialing imux socket")
		socket, err := imux_socket.Redialer()
		if err != nil {
			log.WithFields(log.Fields{
				"at":    "IMUXSocket.init",
				"error": err.Error(),
			}).Error("error dialing imux socket, entering cooldown")
			time.Sleep(cooldown)
			continue
		}
		tlj_server.Insert(socket)
		writer, err := tlj.NewStreamWriter(socket, type_store(), reflect.TypeOf(Chunk{}))
		if err != nil {
			log.WithFields(log.Fields{
				"at":    "IMUXSocket.init",
				"error": err.Error(),
			}).Error("error creating stream writer, entering cooldown")
			time.Sleep(cooldown)
			continue
		}

		for {
			chunk := <-imux_socket.IMUXer.Chunks
			log.WithFields(log.Fields{
				"at":          "IMUXSocket.init",
				"sequence_id": chunk.SequenceID,
				"socket_id":   chunk.SocketID,
				"session_id":  chunk.SessionID,
			}).Debug("writing chunk up transport socket")
			err := writer.Write(chunk)
			if err != nil {
				imux_socket.IMUXer.Stale <- chunk
				log.WithFields(log.Fields{
					"at":          "IMUXSocket.init",
					"error":       err.Error(),
					"sequence_id": chunk.SequenceID,
					"socket_id":   chunk.SocketID,
					"session_id":  chunk.SessionID,
				}).Error("error writing chunk up transport socket")
				break
			}
		}
		log.WithFields(log.Fields{
			"at": "IMUXSocket.init",
		}).Debug("transport socket dies, redailing after cooldown")
		time.Sleep(cooldown)
	}
}

// Create a TLJ server for a session if needed, or return the already existing server
func imuxClientSocketTLJServer(session_id string) tlj.Server {
	srtsMux.Lock()
	if server, exists := sessionResponsesTLJServers[session_id]; exists {
		return server
	}
	tlj_server := tlj.Server{
		TypeStore:       type_store(),
		Tag:             tag_socket,
		Tags:            make(map[net.Conn][]string),
		Sockets:         make(map[string][]net.Conn),
		Events:          make(map[string]map[uint16][]func(interface{}, tlj.TLJContext)),
		Requests:        make(map[string]map[uint16][]func(interface{}, tlj.TLJContext)),
		FailedServer:    make(chan error, 1),
		FailedSockets:   make(chan net.Conn, 200),
		TagManipulation: &sync.Mutex{},
		InsertRequests:  &sync.Mutex{},
		InsertEvents:    &sync.Mutex{},
	}
	go func(server tlj.Server) {
		for {
			<-server.FailedSockets
		}
	}(tlj_server)
	tlj_server.Accept("all", reflect.TypeOf(Chunk{}), func(iface interface{}, context tlj.TLJContext) {
		if chunk, ok := iface.(*Chunk); ok {
			cwqMux.Lock()
			if writer, ok := client_write_queues[chunk.SocketID]; ok {
				log.WithFields(log.Fields{
					"at":         "imuxClientSocketTLJServer",
					"session_id": session_id,
				}).Debug("accepting response chunk in transport socket TLJ server")
				writer.Chunks <- chunk
			} else {
				log.WithFields(log.Fields{
					"at":         "imuxClientSocketTLJServer",
					"session_id": session_id,
					"socket_id":  chunk.SocketID,
				}).Error("could not find write queue for response chunk")
			}
			cwqMux.Unlock()
		}
	})
	sessionResponsesTLJServers[session_id] = tlj_server
	srtsMux.Unlock()
	log.WithFields(log.Fields{
		"at":         "imuxClientSocketTLJServer",
		"session_id": session_id,
	}).Debug("created new TLJ server for session")
	return tlj_server
}
<commit_msg>Unlock srtsMux in imuxClientSocketTLJServer<commit_after>package imux

import (
	log "github.com/Sirupsen/logrus"
	"github.com/hkparker/TLJ"
	"net"
	"reflect"
	"sync"
	"time"
)

// A function that can be called by an IMUXSocket to reconnect after an error
type Redialer func() (net.Conn, error)

// A function that generates Redialers for specific bind addresses
type RedialerGenerator func(string) Redialer

// A map of all TLJ servers used to read chunks back from sessions
var sessionResponsesTLJServers = make(map[string]tlj.Server)
var srtsMux sync.Mutex

// A client socket that transports data in an imux session, autoreconnecting
type IMUXSocket struct {
	IMUXer   DataIMUX
	Redialer Redialer
}

// Dial a new connection in an imux session, creating a TLJ server for
// responses if needed.  Read data from the sockets IMUXer and write it up.
func (imux_socket *IMUXSocket) init(session_id string) {
	log.WithFields(log.Fields{
		"at": "IMUXSocket.init",
	}).Debug("starting imux socket")
	tlj_server := imuxClientSocketTLJServer(session_id)
	cooldown := 10 * time.Second
	for {
		log.WithFields(log.Fields{
			"at": "IMUXSocket.init",
		}).Debug("dialing imux socket")
		socket, err := imux_socket.Redialer()
		if err != nil {
			log.WithFields(log.Fields{
				"at":    "IMUXSocket.init",
				"error": err.Error(),
			}).Error("error dialing imux socket, entering cooldown")
			time.Sleep(cooldown)
			continue
		}
		tlj_server.Insert(socket)
		writer, err := tlj.NewStreamWriter(socket, type_store(), reflect.TypeOf(Chunk{}))
		if err != nil {
			log.WithFields(log.Fields{
				"at":    "IMUXSocket.init",
				"error": err.Error(),
			}).Error("error creating stream writer, entering cooldown")
			time.Sleep(cooldown)
			continue
		}

		for {
			chunk := <-imux_socket.IMUXer.Chunks
			log.WithFields(log.Fields{
				"at":          "IMUXSocket.init",
				"sequence_id": chunk.SequenceID,
				"socket_id":   chunk.SocketID,
				"session_id":  chunk.SessionID,
			}).Debug("writing chunk up transport socket")
			err := writer.Write(chunk)
			if err != nil {
				imux_socket.IMUXer.Stale <- chunk
				log.WithFields(log.Fields{
					"at":          "IMUXSocket.init",
					"error":       err.Error(),
					"sequence_id": chunk.SequenceID,
					"socket_id":   chunk.SocketID,
					"session_id":  chunk.SessionID,
				}).Error("error writing chunk up transport socket")
				break
			}
		}
		log.WithFields(log.Fields{
			"at": "IMUXSocket.init",
		}).Debug("transport socket dies, redailing after cooldown")
		time.Sleep(cooldown)
	}
}

// Create a TLJ server for a session if needed, or return the already existing server
func imuxClientSocketTLJServer(session_id string) tlj.Server {
	srtsMux.Lock()
	defer srtsMux.Unlock()
	if server, exists := sessionResponsesTLJServers[session_id]; exists {
		return server
	}
	tlj_server := tlj.Server{
		TypeStore:       type_store(),
		Tag:             tag_socket,
		Tags:            make(map[net.Conn][]string),
		Sockets:         make(map[string][]net.Conn),
		Events:          make(map[string]map[uint16][]func(interface{}, tlj.TLJContext)),
		Requests:        make(map[string]map[uint16][]func(interface{}, tlj.TLJContext)),
		FailedServer:    make(chan error, 1),
		FailedSockets:   make(chan net.Conn, 200),
		TagManipulation: &sync.Mutex{},
		InsertRequests:  &sync.Mutex{},
		InsertEvents:    &sync.Mutex{},
	}
	go func(server tlj.Server) {
		for {
			<-server.FailedSockets
		}
	}(tlj_server)
	tlj_server.Accept("all", reflect.TypeOf(Chunk{}), func(iface interface{}, context tlj.TLJContext) {
		if chunk, ok := iface.(*Chunk); ok {
			cwqMux.Lock()
			if writer, ok := client_write_queues[chunk.SocketID]; ok {
				log.WithFields(log.Fields{
					"at":         "imuxClientSocketTLJServer",
					"session_id": session_id,
				}).Debug("accepting response chunk in transport socket TLJ server")
				writer.Chunks <- chunk
			} else {
				log.WithFields(log.Fields{
					"at":         "imuxClientSocketTLJServer",
					"session_id": session_id,
					"socket_id":  chunk.SocketID,
				}).Error("could not find write queue for response chunk")
			}
			cwqMux.Unlock()
		}
	})
	sessionResponsesTLJServers[session_id] = tlj_server
	log.WithFields(log.Fields{
		"at":         "imuxClientSocketTLJServer",
		"session_id": session_id,
	}).Debug("created new TLJ server for session")
	return tlj_server
}
<|endoftext|>


<commit_before>package main

import (
	"koding/db/mongodb/modelhelper"
	"log"
	"socialapi/config"
	"socialapi/workers/email/emailsender"

	"github.com/koding/eventexporter"
	"github.com/koding/runner"
)

var (
	Name        = "MailSender"
	QueueLength = 1
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		log.Fatal(err)
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	segmentExporter := eventexporter.NewSegmentIOExporter(appConfig.Segment, QueueLength)
	datadogExporter := eventexporter.NewDatadogExporter(r.DogStatsD)
	exporter := eventexporter.NewMultiExporter(segmentExporter, datadogExporter)

	constructor := emailsender.New(exporter, r.Log, appConfig)
	r.ShutdownHandler = constructor.Close

	r.SetContext(constructor)

	r.Register(emailsender.Mail{}).On(emailsender.SendEmailEventName).Handle((*emailsender.Controller).Process)

	r.Listen()
	r.Wait()
}
<commit_msg>go/druid: add druidExporter into multiExporter<commit_after>package main

import (
	"koding/db/mongodb/modelhelper"
	"log"
	"socialapi/config"
	"socialapi/workers/email/emailsender"

	"github.com/koding/eventexporter"
	"github.com/koding/runner"
)

var (
	Name        = "MailSender"
	QueueLength = 1
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		log.Fatal(err)
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	segmentExporter := eventexporter.NewSegmentIOExporter(appConfig.Segment, QueueLength)
	datadogExporter := eventexporter.NewDatadogExporter(r.DogStatsD)
	druidExporter := eventexporter.NewDruidExporter("address")

	exporter := eventexporter.NewMultiExporter(segmentExporter, datadogExporter, druidExporter)

	constructor := emailsender.New(exporter, r.Log, appConfig)
	r.ShutdownHandler = constructor.Close

	r.SetContext(constructor)

	r.Register(emailsender.Mail{}).On(emailsender.SendEmailEventName).Handle((*emailsender.Controller).Process)

	r.Listen()
	r.Wait()
}
<|endoftext|>


<commit_before>// inspired by a solution in https://stackoverflow.com/questions/47134293/compare-structs-except-one-field-golang/47134781

// need to extend this to cover
// 	nested structs
// 	other data types like slice and maps

// see go-cmp-except.go for better solution

package main

import (
	"fmt"
	"reflect"
	"time"

	mapset "github.com/deckarep/golang-set"
)

type Parent struct {
	Name string
	// NameP *string
	Date1 time.Time
	Date2 time.Time
	Child child
}

type child struct {
	Address string
}

func EqualExcept(f *Parent, other *Parent, ExceptField mapset.Set) bool {
	val := reflect.ValueOf(f).Elem()
	otherFields := reflect.Indirect(reflect.ValueOf(other))

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		if ExceptField.Contains(typeField.Name) {
			continue
		}

		value := val.Field(i)
		otherValue := otherFields.FieldByName(typeField.Name)

		if value.Interface() != otherValue.Interface() {
			return false
		}
	}
	return true
}

func main() {
	n1 := "NNN"
	n2 := "NNN"
	// p1 := "PPP"
	// p2 := "PPP"

	f1 := &Parent{
		Name: n1,
		// &p1,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	f2 := &Parent{
		Name: n2,
		// &p2,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Date1", "Date2")))
	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Name")))

}<commit_msg>using go-cmp to compare struct fields while ignoring some fields<commit_after>// inspired by a solution in https://stackoverflow.com/questions/47134293/compare-structs-except-one-field-golang/47134781

// need to extend this to cover
// 	nested structs
// 	other data types like slice and maps

// see go-cmp-except.go for better solution

package main

import (
	"fmt"
	"reflect"
	"time"

	mapset "github.com/deckarep/golang-set"
)

type Parent struct {
	Name string
	// NameP *string
	Date1 time.Time
	Date2 time.Time
	Child child
}

type child struct {
	Address string
}

func EqualExcept(f *Parent, other *Parent, ExceptField mapset.Set) bool {
	val := reflect.ValueOf(f).Elem()
	otherFields := reflect.Indirect(reflect.ValueOf(other))

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		if ExceptField.Contains(typeField.Name) {
			continue
		}

		value := val.Field(i)
		otherValue := otherFields.FieldByName(typeField.Name)

		if value.Interface() != otherValue.Interface() {
			return false
		}
	}
	return true
}

func main() {
	n1 := "NNN"
	n2 := "NNN"
	// p1 := "PPP"
	// p2 := "PPP"

	f1 := &Parent{
		Name: n1,
		// &p1,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	f2 := &Parent{
		Name: n2,
		// &p2,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Date1", "Date2")))
	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Name")))

}

<|endoftext|>