<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"math"
	"reflect"
	"testing"

	pb "go.etcd.io/etcd/raft/raftpb"
)

func TestStorageTerm(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		i uint64

		werr   error
		wterm  uint64
		wpanic bool
	}{
		{2, ErrCompacted, 0, false},
		{3, nil, 3, false},
		{4, nil, 4, false},
		{5, nil, 5, false},
		{6, ErrUnavailable, 0, false},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}

		func() {
			defer func() {
				if r := recover(); r != nil {
					if !tt.wpanic {
						t.Errorf("%d: panic = %v, want %v", i, true, tt.wpanic)
					}
				}
			}()

			term, err := s.Term(tt.i)
			if err != tt.werr {
				t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
			}
			if term != tt.wterm {
				t.Errorf("#%d: term = %d, want %d", i, term, tt.wterm)
			}
		}()
	}
}

func TestStorageEntries(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}
	tests := []struct {
		lo, hi, maxsize uint64

		werr     error
		wentries []pb.Entry
	}{
		{2, 6, math.MaxUint64, ErrCompacted, nil},
		{3, 4, math.MaxUint64, ErrCompacted, nil},
		{4, 5, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}}},
		{4, 6, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		{4, 7, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}},
		// even if maxsize is zero, the first entry should be returned
		{4, 7, 0, nil, []pb.Entry{{Index: 4, Term: 4}}},
		// limit to 2
		{4, 7, uint64(ents[1].Size() + ents[2].Size()), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		// limit to 2
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size()/2), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size() - 1), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		// all
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size()), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		entries, err := s.Entries(tt.lo, tt.hi, tt.maxsize)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(entries, tt.wentries) {
			t.Errorf("#%d: entries = %v, want %v", i, entries, tt.wentries)
		}
	}
}

func TestStorageLastIndex(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	s := &MemoryStorage{ents: ents}

	last, err := s.LastIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if last != 5 {
		t.Errorf("term = %d, want %d", last, 5)
	}

	s.Append([]pb.Entry{{Index: 6, Term: 5}})
	last, err = s.LastIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if last != 6 {
		t.Errorf("last = %d, want %d", last, 5)
	}
}

func TestStorageFirstIndex(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	s := &MemoryStorage{ents: ents}

	first, err := s.FirstIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if first != 4 {
		t.Errorf("first = %d, want %d", first, 4)
	}

	s.Compact(4)
	first, err = s.FirstIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if first != 5 {
		t.Errorf("first = %d, want %d", first, 5)
	}
}

func TestStorageCompact(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		i uint64

		werr   error
		windex uint64
		wterm  uint64
		wlen   int
	}{
		{2, ErrCompacted, 3, 3, 3},
		{3, ErrCompacted, 3, 3, 3},
		{4, nil, 4, 4, 2},
		{5, nil, 5, 5, 1},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		err := s.Compact(tt.i)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if s.ents[0].Index != tt.windex {
			t.Errorf("#%d: index = %d, want %d", i, s.ents[0].Index, tt.windex)
		}
		if s.ents[0].Term != tt.wterm {
			t.Errorf("#%d: term = %d, want %d", i, s.ents[0].Term, tt.wterm)
		}
		if len(s.ents) != tt.wlen {
			t.Errorf("#%d: len = %d, want %d", i, len(s.ents), tt.wlen)
		}
	}
}

func TestStorageCreateSnapshot(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	cs := &pb.ConfState{Nodes: []uint64{1, 2, 3}}
	data := []byte("data")

	tests := []struct {
		i uint64

		werr  error
		wsnap pb.Snapshot
	}{
		{4, nil, pb.Snapshot{Data: data, Metadata: pb.SnapshotMetadata{Index: 4, Term: 4, ConfState: *cs}}},
		{5, nil, pb.Snapshot{Data: data, Metadata: pb.SnapshotMetadata{Index: 5, Term: 5, ConfState: *cs}}},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		snap, err := s.CreateSnapshot(tt.i, cs, data)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(snap, tt.wsnap) {
			t.Errorf("#%d: snap = %+v, want %+v", i, snap, tt.wsnap)
		}
	}
}

func TestStorageAppend(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		entries []pb.Entry

		werr     error
		wentries []pb.Entry
	}{
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}},
		},
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 6}, {Index: 5, Term: 6}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 6}, {Index: 5, Term: 6}},
		},
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
		},
		// truncate incoming entries, truncate the existing entries and append
		{
			[]pb.Entry{{Index: 2, Term: 3}, {Index: 3, Term: 3}, {Index: 4, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 5}},
		},
		// truncate the existing entries and append
		{
			[]pb.Entry{{Index: 4, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 5}},
		},
		// direct append
		{
			[]pb.Entry{{Index: 6, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
		},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		err := s.Append(tt.entries)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(s.ents, tt.wentries) {
			t.Errorf("#%d: entries = %v, want %v", i, s.ents, tt.wentries)
		}
	}
}

func TestStorageApplySnapshot(t *testing.T) {
	cs := &pb.ConfState{Nodes: []uint64{1, 2, 3}}
	data := []byte("data")

	tests := []pb.Snapshot{{Data: data, Metadata: pb.SnapshotMetadata{Index: 4, Term: 4, ConfState: *cs}},
		{Data: data, Metadata: pb.SnapshotMetadata{Index: 3, Term: 3, ConfState: *cs}},
	}

	s := NewMemoryStorage()

	//Apply Snapshot successful
	i := 0
	tt := tests[i]
	err := s.ApplySnapshot(tt)
	if err != nil {
		t.Errorf("#%d: err = %v, want %v", i, err, nil)
	}

	//Apply Snapshot fails due to ErrSnapOutOfDate
	i = 1
	tt = tests[i]
	err = s.ApplySnapshot(tt)
	if err != ErrSnapOutOfDate {
		t.Errorf("#%d: err = %v, want %v", i, err, ErrSnapOutOfDate)
	}
}
<commit_msg>raft:  add a test case in TestStorageAppend<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"math"
	"reflect"
	"testing"

	pb "go.etcd.io/etcd/raft/raftpb"
)

func TestStorageTerm(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		i uint64

		werr   error
		wterm  uint64
		wpanic bool
	}{
		{2, ErrCompacted, 0, false},
		{3, nil, 3, false},
		{4, nil, 4, false},
		{5, nil, 5, false},
		{6, ErrUnavailable, 0, false},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}

		func() {
			defer func() {
				if r := recover(); r != nil {
					if !tt.wpanic {
						t.Errorf("%d: panic = %v, want %v", i, true, tt.wpanic)
					}
				}
			}()

			term, err := s.Term(tt.i)
			if err != tt.werr {
				t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
			}
			if term != tt.wterm {
				t.Errorf("#%d: term = %d, want %d", i, term, tt.wterm)
			}
		}()
	}
}

func TestStorageEntries(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}
	tests := []struct {
		lo, hi, maxsize uint64

		werr     error
		wentries []pb.Entry
	}{
		{2, 6, math.MaxUint64, ErrCompacted, nil},
		{3, 4, math.MaxUint64, ErrCompacted, nil},
		{4, 5, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}}},
		{4, 6, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		{4, 7, math.MaxUint64, nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}},
		// even if maxsize is zero, the first entry should be returned
		{4, 7, 0, nil, []pb.Entry{{Index: 4, Term: 4}}},
		// limit to 2
		{4, 7, uint64(ents[1].Size() + ents[2].Size()), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		// limit to 2
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size()/2), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size() - 1), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}}},
		// all
		{4, 7, uint64(ents[1].Size() + ents[2].Size() + ents[3].Size()), nil, []pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 6}}},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		entries, err := s.Entries(tt.lo, tt.hi, tt.maxsize)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(entries, tt.wentries) {
			t.Errorf("#%d: entries = %v, want %v", i, entries, tt.wentries)
		}
	}
}

func TestStorageLastIndex(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	s := &MemoryStorage{ents: ents}

	last, err := s.LastIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if last != 5 {
		t.Errorf("term = %d, want %d", last, 5)
	}

	s.Append([]pb.Entry{{Index: 6, Term: 5}})
	last, err = s.LastIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if last != 6 {
		t.Errorf("last = %d, want %d", last, 5)
	}
}

func TestStorageFirstIndex(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	s := &MemoryStorage{ents: ents}

	first, err := s.FirstIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if first != 4 {
		t.Errorf("first = %d, want %d", first, 4)
	}

	s.Compact(4)
	first, err = s.FirstIndex()
	if err != nil {
		t.Errorf("err = %v, want nil", err)
	}
	if first != 5 {
		t.Errorf("first = %d, want %d", first, 5)
	}
}

func TestStorageCompact(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		i uint64

		werr   error
		windex uint64
		wterm  uint64
		wlen   int
	}{
		{2, ErrCompacted, 3, 3, 3},
		{3, ErrCompacted, 3, 3, 3},
		{4, nil, 4, 4, 2},
		{5, nil, 5, 5, 1},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		err := s.Compact(tt.i)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if s.ents[0].Index != tt.windex {
			t.Errorf("#%d: index = %d, want %d", i, s.ents[0].Index, tt.windex)
		}
		if s.ents[0].Term != tt.wterm {
			t.Errorf("#%d: term = %d, want %d", i, s.ents[0].Term, tt.wterm)
		}
		if len(s.ents) != tt.wlen {
			t.Errorf("#%d: len = %d, want %d", i, len(s.ents), tt.wlen)
		}
	}
}

func TestStorageCreateSnapshot(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	cs := &pb.ConfState{Nodes: []uint64{1, 2, 3}}
	data := []byte("data")

	tests := []struct {
		i uint64

		werr  error
		wsnap pb.Snapshot
	}{
		{4, nil, pb.Snapshot{Data: data, Metadata: pb.SnapshotMetadata{Index: 4, Term: 4, ConfState: *cs}}},
		{5, nil, pb.Snapshot{Data: data, Metadata: pb.SnapshotMetadata{Index: 5, Term: 5, ConfState: *cs}}},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		snap, err := s.CreateSnapshot(tt.i, cs, data)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(snap, tt.wsnap) {
			t.Errorf("#%d: snap = %+v, want %+v", i, snap, tt.wsnap)
		}
	}
}

func TestStorageAppend(t *testing.T) {
	ents := []pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}}
	tests := []struct {
		entries []pb.Entry

		werr     error
		wentries []pb.Entry
	}{
		{
			[]pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}},
		},
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}},
		},
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 6}, {Index: 5, Term: 6}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 6}, {Index: 5, Term: 6}},
		},
		{
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
		},
		// truncate incoming entries, truncate the existing entries and append
		{
			[]pb.Entry{{Index: 2, Term: 3}, {Index: 3, Term: 3}, {Index: 4, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 5}},
		},
		// truncate the existing entries and append
		{
			[]pb.Entry{{Index: 4, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 5}},
		},
		// direct append
		{
			[]pb.Entry{{Index: 6, Term: 5}},
			nil,
			[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 5}, {Index: 6, Term: 5}},
		},
	}

	for i, tt := range tests {
		s := &MemoryStorage{ents: ents}
		err := s.Append(tt.entries)
		if err != tt.werr {
			t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
		}
		if !reflect.DeepEqual(s.ents, tt.wentries) {
			t.Errorf("#%d: entries = %v, want %v", i, s.ents, tt.wentries)
		}
	}
}

func TestStorageApplySnapshot(t *testing.T) {
	cs := &pb.ConfState{Nodes: []uint64{1, 2, 3}}
	data := []byte("data")

	tests := []pb.Snapshot{{Data: data, Metadata: pb.SnapshotMetadata{Index: 4, Term: 4, ConfState: *cs}},
		{Data: data, Metadata: pb.SnapshotMetadata{Index: 3, Term: 3, ConfState: *cs}},
	}

	s := NewMemoryStorage()

	//Apply Snapshot successful
	i := 0
	tt := tests[i]
	err := s.ApplySnapshot(tt)
	if err != nil {
		t.Errorf("#%d: err = %v, want %v", i, err, nil)
	}

	//Apply Snapshot fails due to ErrSnapOutOfDate
	i = 1
	tt = tests[i]
	err = s.ApplySnapshot(tt)
	if err != ErrSnapOutOfDate {
		t.Errorf("#%d: err = %v, want %v", i, err, ErrSnapOutOfDate)
	}
}
<|endoftext|>


<commit_before>package SchemaHandler

import (
	"errors"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
)

/*
PathMap Example
            root(a dummy root)  (Path: "root", Children: A)
             |
             A  (Path:"root/A", Childend: B,C)
        /           \
B(Path:"root/A/B")   C(Path:"root/A/C")
*/

// PathMapType records the path and its children; This is used in Marshal for improve performance.
type PathMapType struct {
	Path     string
	Children map[string]*PathMapType
}

func NewPathMap(path string) *PathMapType {
	pathMap := new(PathMapType)
	pathMap.Path = path
	pathMap.Children = make(map[string]*PathMapType)
	return pathMap
}

func (self *PathMapType) Add(path []string) {
	ln := len(path)
	if ln <= 1 {
		return
	}
	c := path[1]
	if _, ok := self.Children[c]; !ok {
		self.Children[c] = NewPathMap(self.Path + "." + c)
	}
	self.Children[c].Add(path[1:])
}

/////////////////pathMap///////////////////////////

// SchemaHandler stores the schema data
type SchemaHandler struct {
	SchemaElements []*parquet.SchemaElement
	MapIndex       map[string]int32
	IndexMap       map[int32]string
	PathMap        *PathMapType
	Infos          []map[string]interface{}

	InPathToExPath map[string]string
	ExPathToInPath map[string]string

	ValueColumns []string
}

// setValueColumns collects leaf nodes' full path in SchemaHandler.ValueColumns
func (self *SchemaHandler) setValueColumns() {
	for i := 0; i < len(self.SchemaElements); i++ {
		schema := self.SchemaElements[i]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			pathStr := self.IndexMap[int32(i)]
			self.ValueColumns = append(self.ValueColumns, pathStr)
		}
	}
}

func (self *SchemaHandler) GetColumnNum() int64 {
	return int64(len(self.ValueColumns))
}

// setPathMap builds the PathMap from leaf SchemaElement
func (self *SchemaHandler) setPathMap() {
	self.PathMap = NewPathMap(self.GetRootName())
	for i := 0; i < len(self.SchemaElements); i++ {
		schema := self.SchemaElements[i]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			pathStr := self.IndexMap[int32(i)]
			self.PathMap.Add(Common.StrToPath(pathStr))
		}
	}
}

// GetRepetitionType returns the repetition type of a column by it's schema path
func (self *SchemaHandler) GetRepetitionType(path []string) (parquet.FieldRepetitionType, error) {
	pathStr := Common.PathToStr(path)
	if index, ok := self.MapIndex[pathStr]; ok {
		return self.SchemaElements[index].GetRepetitionType(), nil
	} else {
		return 0, errors.New("Name Not In Schema")
	}
}

// MaxDefinitionLevel returns the max definition level type of a column by it's schema path
func (self *SchemaHandler) MaxDefinitionLevel(path []string) (int32, error) {
	var res int32 = 0
	ln := len(path)
	for i := 2; i <= ln; i++ {
		rt, err := self.GetRepetitionType(path[:i])
		if err != nil {
			return 0, err
		}
		if rt != parquet.FieldRepetitionType_REQUIRED {
			res++
		}
	}
	return res, nil
}

// MaxRepetitionLevel returns the max repetition level type of a column by it's schema path
func (self *SchemaHandler) MaxRepetitionLevel(path []string) (int32, error) {
	var res int32 = 0
	ln := len(path)
	for i := 2; i <= ln; i++ {
		rt, err := self.GetRepetitionType(path[:i])
		if err != nil {
			return 0, err
		}
		if rt == parquet.FieldRepetitionType_REPEATED {
			res++
		}
	}
	return res, nil
}

func (self *SchemaHandler) CreateInExMap() {
	//use DFS get path of schema
	schemas := self.SchemaElements
	ln := int32(len(schemas))
	var pos int32 = 0
	stack := make([][]int32, 0) //stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 {
			item := make([]int32, 2)
			item[0] = pos
			item[1] = int32(*schemas[pos].NumChildren)
			stack = append(stack, item)
			pos++
		} else {
			top := stack[len(stack)-1]
			if top[1] == 0 {
				inPath, exPath := make([]string, 0), make([]string, 0)
				for i := 0; i < len(stack); i++ {
					inPath = append(inPath, self.Infos[stack[i][0]]["inname"].(string))
					exPath = append(exPath, self.Infos[stack[i][0]]["exname"].(string))
				}
				inPathStr, exPathStr := Common.PathToStr(inPath), Common.PathToStr(exPath)
				self.ExPathToInPath[exPathStr] = inPathStr
				self.InPathToExPath[inPathStr] = exPathStr

				stack = stack[:len(stack)-1]
			} else {
				top[1]--
				item := make([]int32, 2)
				item[0] = pos
				item[1] = schemas[pos].GetNumChildren()
				stack = append(stack, item)
				pos++
			}
		}
	}
}

//Get root name from the schema handler
func (self *SchemaHandler) GetRootName() string {
	if len(self.SchemaElements) <= 0 {
		return ""
	}
	return self.SchemaElements[0].GetName()
}

type Item struct {
	GoType reflect.Type
	Info   map[string]interface{}
}

func NewItem() *Item {
	item := new(Item)
	item.Info = make(map[string]interface{})
	return item
}

//Create schema handler from a object
func NewSchemaHandlerFromStruct(obj interface{}) (sh *SchemaHandler, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	ot := reflect.TypeOf(obj).Elem()
	item := NewItem()
	item.GoType = ot
	item.Info["inname"] = "parquet_go_root"
	item.Info["exname"] = "parquet_go_root"
	item.Info["repetitiontype"] = parquet.FieldRepetitionType(-1)

	stack := make([]*Item, 1)
	stack[0] = item
	schemaElements := make([]*parquet.SchemaElement, 0)
	infos := make([]map[string]interface{}, 0)

	for len(stack) > 0 {
		ln := len(stack)
		item = stack[ln-1]
		stack = stack[:ln-1]

		if item.GoType.Kind() == reflect.Struct {
			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt
			numField := int32(item.GoType.NumField())
			schema.NumChildren = &numField
			schema.Type = nil
			schemaElements = append(schemaElements, schema)

			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			for i := int(numField - 1); i >= 0; i-- {
				f := item.GoType.Field(i)
				newItem := NewItem()
				newItem.Info = Common.TagToMap(f.Tag.Get("parquet"))
				newItem.Info["inname"] = f.Name
				newItem.GoType = f.Type
				if f.Type.Kind() == reflect.Ptr {
					newItem.GoType = f.Type.Elem()
					newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				}
				stack = append(stack, newItem)
			}
		} else if item.GoType.Kind() == reflect.Slice &&
			item.Info["repetitiontype"].(parquet.FieldRepetitionType) != parquet.FieldRepetitionType_REPEATED {

			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt1 := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt1
			var numField1 int32 = 1
			schema.NumChildren = &numField1
			schema.Type = nil
			ct1 := parquet.ConvertedType_LIST
			schema.ConvertedType = &ct1
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			schema = parquet.NewSchemaElement()
			schema.Name = "list"
			rt2 := parquet.FieldRepetitionType_REPEATED
			schema.RepetitionType = &rt2
			schema.Type = nil
			var numField2 int32 = 1
			schema.NumChildren = &numField2
			schemaElements = append(schemaElements, schema)
			info = Common.NewTagMapFromCopy(item.Info)
			info["inname"], info["exname"] = "list", "list"
			infos = append(infos, info)

			newItem := NewItem()
			newItem.Info = Common.GetValueTagMap(item.Info)
			newItem.Info["inname"] = "element"
			newItem.Info["exname"] = "element"
			newItem.GoType = item.GoType.Elem()
			if newItem.GoType.Kind() == reflect.Ptr {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				newItem.GoType = item.GoType.Elem().Elem()
			} else {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			}
			stack = append(stack, newItem)

		} else if item.GoType.Kind() == reflect.Slice &&
			item.Info["repetitiontype"].(parquet.FieldRepetitionType) == parquet.FieldRepetitionType_REPEATED {
			newItem := NewItem()
			newItem.Info = item.Info
			newItem.GoType = item.GoType.Elem()
			stack = append(stack, newItem)

		} else if item.GoType.Kind() == reflect.Map {
			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt1 := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt1
			var numField1 int32 = 1
			schema.NumChildren = &numField1
			schema.Type = nil
			ct1 := parquet.ConvertedType_MAP
			schema.ConvertedType = &ct1
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			schema = parquet.NewSchemaElement()
			schema.Name = "key_value"
			rt2 := parquet.FieldRepetitionType_REPEATED
			schema.RepetitionType = &rt2
			var numField2 int32 = 2
			schema.NumChildren = &numField2
			schema.Type = nil
			ct2 := parquet.ConvertedType_MAP_KEY_VALUE
			schema.ConvertedType = &ct2
			schemaElements = append(schemaElements, schema)
			info = Common.NewTagMapFromCopy(item.Info)
			info["inname"], info["exname"] = "key_value", "key_value"
			infos = append(infos, info)

			newItem := NewItem()
			newItem.Info = Common.GetValueTagMap(item.Info)
			newItem.GoType = item.GoType.Elem()
			if newItem.GoType.Kind() == reflect.Ptr {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				newItem.GoType = item.GoType.Elem().Elem()
			} else {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			}
			stack = append(stack, newItem)

			newItem = NewItem()
			newItem.Info = Common.GetKeyTagMap(item.Info)
			newItem.GoType = item.GoType.Key()
			newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			stack = append(stack, newItem)

		} else {
			schema := Common.NewSchemaElementFromTagMap(item.Info)
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)
		}
	}
	res := NewSchemaHandlerFromSchemaList(schemaElements)
	res.Infos = infos
	res.CreateInExMap()
	return res, nil
}

//Create schema handler from schema list
func NewSchemaHandlerFromSchemaList(schemas []*parquet.SchemaElement) *SchemaHandler {
	schemaHandler := new(SchemaHandler)
	schemaHandler.MapIndex = make(map[string]int32)
	schemaHandler.IndexMap = make(map[int32]string)
	schemaHandler.InPathToExPath = make(map[string]string)
	schemaHandler.ExPathToInPath = make(map[string]string)
	schemaHandler.SchemaElements = schemas

	//use DFS get path of schema
	ln := int32(len(schemas))
	var pos int32 = 0
	stack := make([][]int32, 0) //stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 {
			item := make([]int32, 2)
			item[0] = pos
			item[1] = int32(*schemas[pos].NumChildren)
			stack = append(stack, item)
			pos++
		} else {
			top := stack[len(stack)-1]
			if top[1] == 0 {
				path := make([]string, 0)
				for i := 0; i < len(stack); i++ {
					path = append(path, schemas[stack[i][0]].GetName())
				}
				schemaHandler.MapIndex[Common.PathToStr(path)] = top[0]
				schemaHandler.IndexMap[top[0]] = Common.PathToStr(path)
				stack = stack[:len(stack)-1]
			} else {
				top[1]--
				item := make([]int32, 2)
				item[0] = pos
				item[1] = schemas[pos].GetNumChildren()
				stack = append(stack, item)
				pos++
			}
		}
	}
	//	log.Println("NewSchemaHandlerFromSchemaList Finished")
	schemaHandler.setPathMap()
	schemaHandler.setValueColumns()
	return schemaHandler
}
<commit_msg>simply schemaHandler dfs<commit_after>package SchemaHandler

import (
	"errors"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
)

/*
PathMap Example
            root(a dummy root)  (Path: "root", Children: A)
             |
             A  (Path:"root/A", Childend: B,C)
        /           \
B(Path:"root/A/B")   C(Path:"root/A/C")
*/

// PathMapType records the path and its children; This is used in Marshal for improve performance.
type PathMapType struct {
	Path     string
	Children map[string]*PathMapType
}

func NewPathMap(path string) *PathMapType {
	pathMap := new(PathMapType)
	pathMap.Path = path
	pathMap.Children = make(map[string]*PathMapType)
	return pathMap
}

func (self *PathMapType) Add(path []string) {
	ln := len(path)
	if ln <= 1 {
		return
	}
	c := path[1]
	if _, ok := self.Children[c]; !ok {
		self.Children[c] = NewPathMap(self.Path + "." + c)
	}
	self.Children[c].Add(path[1:])
}

/////////////////pathMap///////////////////////////

// SchemaHandler stores the schema data
type SchemaHandler struct {
	SchemaElements []*parquet.SchemaElement
	MapIndex       map[string]int32
	IndexMap       map[int32]string
	PathMap        *PathMapType
	Infos          []map[string]interface{}

	InPathToExPath map[string]string
	ExPathToInPath map[string]string

	ValueColumns []string
}

// setValueColumns collects leaf nodes' full path in SchemaHandler.ValueColumns
func (self *SchemaHandler) setValueColumns() {
	for i := 0; i < len(self.SchemaElements); i++ {
		schema := self.SchemaElements[i]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			pathStr := self.IndexMap[int32(i)]
			self.ValueColumns = append(self.ValueColumns, pathStr)
		}
	}
}

func (self *SchemaHandler) GetColumnNum() int64 {
	return int64(len(self.ValueColumns))
}

// setPathMap builds the PathMap from leaf SchemaElement
func (self *SchemaHandler) setPathMap() {
	self.PathMap = NewPathMap(self.GetRootName())
	for i := 0; i < len(self.SchemaElements); i++ {
		schema := self.SchemaElements[i]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			pathStr := self.IndexMap[int32(i)]
			self.PathMap.Add(Common.StrToPath(pathStr))
		}
	}
}

// GetRepetitionType returns the repetition type of a column by it's schema path
func (self *SchemaHandler) GetRepetitionType(path []string) (parquet.FieldRepetitionType, error) {
	pathStr := Common.PathToStr(path)
	if index, ok := self.MapIndex[pathStr]; ok {
		return self.SchemaElements[index].GetRepetitionType(), nil
	} else {
		return 0, errors.New("Name Not In Schema")
	}
}

// MaxDefinitionLevel returns the max definition level type of a column by it's schema path
func (self *SchemaHandler) MaxDefinitionLevel(path []string) (int32, error) {
	var res int32 = 0
	ln := len(path)
	for i := 2; i <= ln; i++ {
		rt, err := self.GetRepetitionType(path[:i])
		if err != nil {
			return 0, err
		}
		if rt != parquet.FieldRepetitionType_REQUIRED {
			res++
		}
	}
	return res, nil
}

// MaxRepetitionLevel returns the max repetition level type of a column by it's schema path
func (self *SchemaHandler) MaxRepetitionLevel(path []string) (int32, error) {
	var res int32 = 0
	ln := len(path)
	for i := 2; i <= ln; i++ {
		rt, err := self.GetRepetitionType(path[:i])
		if err != nil {
			return 0, err
		}
		if rt == parquet.FieldRepetitionType_REPEATED {
			res++
		}
	}
	return res, nil
}

func (self *SchemaHandler) CreateInExMap() {
	//use DFS get path of schema
	schemas := self.SchemaElements
	ln := int32(len(schemas))
	var pos int32 = 0
	stack := make([][2]int32, 0) // stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 || stack[len(stack)-1][1] > 0 {
			if len(stack) > 0 {
				stack[len(stack)-1][1]--
			}
			item := [2]int32{pos, schemas[pos].GetNumChildren()}
			stack = append(stack, item)
			pos++
		} else { // leaf node
			inPath, exPath := make([]string, 0), make([]string, 0)
			for i := 0; i < len(stack); i++ {
				inPath = append(inPath, self.Infos[stack[i][0]]["inname"].(string))
				exPath = append(exPath, self.Infos[stack[i][0]]["exname"].(string))
			}
			inPathStr, exPathStr := Common.PathToStr(inPath), Common.PathToStr(exPath)
			self.ExPathToInPath[exPathStr] = inPathStr
			self.InPathToExPath[inPathStr] = exPathStr
			stack = stack[:len(stack)-1]
		}
	}
}

//Get root name from the schema handler
func (self *SchemaHandler) GetRootName() string {
	if len(self.SchemaElements) <= 0 {
		return ""
	}
	return self.SchemaElements[0].GetName()
}

type Item struct {
	GoType reflect.Type
	Info   map[string]interface{}
}

func NewItem() *Item {
	item := new(Item)
	item.Info = make(map[string]interface{})
	return item
}

//Create schema handler from a object
func NewSchemaHandlerFromStruct(obj interface{}) (sh *SchemaHandler, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	ot := reflect.TypeOf(obj).Elem()
	item := NewItem()
	item.GoType = ot
	item.Info["inname"] = "parquet_go_root"
	item.Info["exname"] = "parquet_go_root"
	item.Info["repetitiontype"] = parquet.FieldRepetitionType(-1)

	stack := make([]*Item, 1)
	stack[0] = item
	schemaElements := make([]*parquet.SchemaElement, 0)
	infos := make([]map[string]interface{}, 0)

	for len(stack) > 0 {
		ln := len(stack)
		item = stack[ln-1]
		stack = stack[:ln-1]

		if item.GoType.Kind() == reflect.Struct {
			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt
			numField := int32(item.GoType.NumField())
			schema.NumChildren = &numField
			schema.Type = nil
			schemaElements = append(schemaElements, schema)

			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			for i := int(numField - 1); i >= 0; i-- {
				f := item.GoType.Field(i)
				newItem := NewItem()
				newItem.Info = Common.TagToMap(f.Tag.Get("parquet"))
				newItem.Info["inname"] = f.Name
				newItem.GoType = f.Type
				if f.Type.Kind() == reflect.Ptr {
					newItem.GoType = f.Type.Elem()
					newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				}
				stack = append(stack, newItem)
			}
		} else if item.GoType.Kind() == reflect.Slice &&
			item.Info["repetitiontype"].(parquet.FieldRepetitionType) != parquet.FieldRepetitionType_REPEATED {

			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt1 := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt1
			var numField1 int32 = 1
			schema.NumChildren = &numField1
			schema.Type = nil
			ct1 := parquet.ConvertedType_LIST
			schema.ConvertedType = &ct1
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			schema = parquet.NewSchemaElement()
			schema.Name = "list"
			rt2 := parquet.FieldRepetitionType_REPEATED
			schema.RepetitionType = &rt2
			schema.Type = nil
			var numField2 int32 = 1
			schema.NumChildren = &numField2
			schemaElements = append(schemaElements, schema)
			info = Common.NewTagMapFromCopy(item.Info)
			info["inname"], info["exname"] = "list", "list"
			infos = append(infos, info)

			newItem := NewItem()
			newItem.Info = Common.GetValueTagMap(item.Info)
			newItem.Info["inname"] = "element"
			newItem.Info["exname"] = "element"
			newItem.GoType = item.GoType.Elem()
			if newItem.GoType.Kind() == reflect.Ptr {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				newItem.GoType = item.GoType.Elem().Elem()
			} else {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			}
			stack = append(stack, newItem)

		} else if item.GoType.Kind() == reflect.Slice &&
			item.Info["repetitiontype"].(parquet.FieldRepetitionType) == parquet.FieldRepetitionType_REPEATED {
			newItem := NewItem()
			newItem.Info = item.Info
			newItem.GoType = item.GoType.Elem()
			stack = append(stack, newItem)

		} else if item.GoType.Kind() == reflect.Map {
			schema := parquet.NewSchemaElement()
			schema.Name = item.Info["inname"].(string)
			rt1 := item.Info["repetitiontype"].(parquet.FieldRepetitionType)
			schema.RepetitionType = &rt1
			var numField1 int32 = 1
			schema.NumChildren = &numField1
			schema.Type = nil
			ct1 := parquet.ConvertedType_MAP
			schema.ConvertedType = &ct1
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)

			schema = parquet.NewSchemaElement()
			schema.Name = "key_value"
			rt2 := parquet.FieldRepetitionType_REPEATED
			schema.RepetitionType = &rt2
			var numField2 int32 = 2
			schema.NumChildren = &numField2
			schema.Type = nil
			ct2 := parquet.ConvertedType_MAP_KEY_VALUE
			schema.ConvertedType = &ct2
			schemaElements = append(schemaElements, schema)
			info = Common.NewTagMapFromCopy(item.Info)
			info["inname"], info["exname"] = "key_value", "key_value"
			infos = append(infos, info)

			newItem := NewItem()
			newItem.Info = Common.GetValueTagMap(item.Info)
			newItem.GoType = item.GoType.Elem()
			if newItem.GoType.Kind() == reflect.Ptr {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_OPTIONAL
				newItem.GoType = item.GoType.Elem().Elem()
			} else {
				newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			}
			stack = append(stack, newItem)

			newItem = NewItem()
			newItem.Info = Common.GetKeyTagMap(item.Info)
			newItem.GoType = item.GoType.Key()
			newItem.Info["repetitiontype"] = parquet.FieldRepetitionType_REQUIRED
			stack = append(stack, newItem)

		} else {
			schema := Common.NewSchemaElementFromTagMap(item.Info)
			schemaElements = append(schemaElements, schema)
			info := Common.NewTagMapFromCopy(item.Info)
			infos = append(infos, info)
		}
	}
	res := NewSchemaHandlerFromSchemaList(schemaElements)
	res.Infos = infos
	res.CreateInExMap()
	return res, nil
}

// NewSchemaHandlerFromSchemaList creates schema handler from schema list
func NewSchemaHandlerFromSchemaList(schemas []*parquet.SchemaElement) *SchemaHandler {
	schemaHandler := new(SchemaHandler)
	schemaHandler.MapIndex = make(map[string]int32)
	schemaHandler.IndexMap = make(map[int32]string)
	schemaHandler.InPathToExPath = make(map[string]string)
	schemaHandler.ExPathToInPath = make(map[string]string)
	schemaHandler.SchemaElements = schemas

	//use DFS get path of schema
	ln := int32(len(schemas))
	var pos int32 = 0
	stack := make([][2]int32, 0) //stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 || stack[len(stack)-1][1] > 0 {
			if len(stack) > 0 {
				stack[len(stack)-1][1]--
			}
			item := [2]int32{pos, schemas[pos].GetNumChildren()}
			stack = append(stack, item)
			pos++
		} else {
			path := make([]string, 0)
			for i := 0; i < len(stack); i++ {
				path = append(path, schemas[stack[i][0]].GetName())
			}
			topPos := stack[len(stack)-1][0]
			schemaHandler.MapIndex[Common.PathToStr(path)] = topPos
			schemaHandler.IndexMap[topPos] = Common.PathToStr(path)
			stack = stack[:len(stack)-1]
		}
	}
	schemaHandler.setPathMap()
	schemaHandler.setValueColumns()
	return schemaHandler
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
	"time"

	"github.com/tendermint/tendermint/types"
)

//-----------------------------------------------------------------------------
// RoundStepType enum type

// RoundStepType enumerates the state of the consensus state machine
type RoundStepType uint8 // These must be numeric, ordered.

const (
	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
)

// String returns a string
func (rs RoundStepType) String() string {
	switch rs {
	case RoundStepNewHeight:
		return "RoundStepNewHeight"
	case RoundStepNewRound:
		return "RoundStepNewRound"
	case RoundStepPropose:
		return "RoundStepPropose"
	case RoundStepPrevote:
		return "RoundStepPrevote"
	case RoundStepPrevoteWait:
		return "RoundStepPrevoteWait"
	case RoundStepPrecommit:
		return "RoundStepPrecommit"
	case RoundStepPrecommitWait:
		return "RoundStepPrecommitWait"
	case RoundStepCommit:
		return "RoundStepCommit"
	default:
		return "RoundStepUnknown" // Cannot panic.
	}
}

//-----------------------------------------------------------------------------

// RoundState defines the internal consensus state.
// NOTE: Not thread safe. Should only be manipulated by functions downstream
// of the cs.receiveRoutine
type RoundState struct {
	Height             int64 // Height we are working on
	Round              int
	Step               RoundStepType
	StartTime          time.Time
	CommitTime         time.Time // Subjective time when +2/3 precommits for Block at Round were found
	Validators         *types.ValidatorSet
	Proposal           *types.Proposal
	ProposalBlock      *types.Block
	ProposalBlockParts *types.PartSet
	LockedRound        int
	LockedBlock        *types.Block
	LockedBlockParts   *types.PartSet
	ValidRound         int
	ValidBlock         *types.Block
	ValidBlockParts    *types.PartSet
	Votes              *HeightVoteSet
	CommitRound        int            //
	LastCommit         *types.VoteSet // Last precommits at Height-1
	LastValidators     *types.ValidatorSet
}

// RoundStateEvent returns the H/R/S of the RoundState as an event.
func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
	// XXX: copy the RoundState
	// if we want to avoid this, we may need synchronous events after all
	rs_ := *rs
	edrs := types.EventDataRoundState{
		Height:     rs.Height,
		Round:      rs.Round,
		Step:       rs.Step.String(),
		RoundState: &rs_,
	}
	return edrs
}

// String returns a string
func (rs *RoundState) String() string {
	return rs.StringIndented("")
}

// StringIndented returns a string
func (rs *RoundState) StringIndented(indent string) string {
	return fmt.Sprintf(`RoundState{
%s  H:%v R:%v S:%v
%s  StartTime:     %v
%s  CommitTime:    %v
%s  Validators:    %v
%s  Proposal:      %v
%s  ProposalBlock: %v %v
%s  LockedRound:   %v
%s  LockedBlock:   %v %v
%s  ValidRound:   %v
%s  ValidBlock:   %v %v
%s  Votes:         %v
%s  LastCommit:    %v
%s  LastValidators:%v
%s}`,
		indent, rs.Height, rs.Round, rs.Step,
		indent, rs.StartTime,
		indent, rs.CommitTime,
		indent, rs.Validators.StringIndented(indent+"    "),
		indent, rs.Proposal,
		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
		indent, rs.LockedRound,
		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
		indent, rs.ValidRound,
		indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(),
		indent, rs.Votes.StringIndented(indent+"    "),
		indent, rs.LastCommit.StringShort(),
		indent, rs.LastValidators.StringIndented(indent+"    "),
		indent)
}

// StringShort returns a string
func (rs *RoundState) StringShort() string {
	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
		rs.Height, rs.Round, rs.Step, rs.StartTime)
}
<commit_msg>lower_case json field names<commit_after>package types

import (
	"fmt"
	"time"

	"github.com/tendermint/tendermint/types"
)

//-----------------------------------------------------------------------------
// RoundStepType enum type

// RoundStepType enumerates the state of the consensus state machine
type RoundStepType uint8 // These must be numeric, ordered.

const (
	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
)

// String returns a string
func (rs RoundStepType) String() string {
	switch rs {
	case RoundStepNewHeight:
		return "RoundStepNewHeight"
	case RoundStepNewRound:
		return "RoundStepNewRound"
	case RoundStepPropose:
		return "RoundStepPropose"
	case RoundStepPrevote:
		return "RoundStepPrevote"
	case RoundStepPrevoteWait:
		return "RoundStepPrevoteWait"
	case RoundStepPrecommit:
		return "RoundStepPrecommit"
	case RoundStepPrecommitWait:
		return "RoundStepPrecommitWait"
	case RoundStepCommit:
		return "RoundStepCommit"
	default:
		return "RoundStepUnknown" // Cannot panic.
	}
}

//-----------------------------------------------------------------------------

// RoundState defines the internal consensus state.
// NOTE: Not thread safe. Should only be manipulated by functions downstream
// of the cs.receiveRoutine
type RoundState struct {
	Height             int64               `json:"height"` // Height we are working on
	Round              int                 `json:"round"`
	Step               RoundStepType       `json:"step"`
	StartTime          time.Time           `json:"start_time"`
	CommitTime         time.Time           `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found
	Validators         *types.ValidatorSet `json:"validators"`
	Proposal           *types.Proposal     `json:"proposal"`
	ProposalBlock      *types.Block        `json:"proposal_block"`
	ProposalBlockParts *types.PartSet      `json:"proposal_block_parts"`
	LockedRound        int                 `json:"locked_round"`
	LockedBlock        *types.Block        `json:"locked_block"`
	LockedBlockParts   *types.PartSet      `json:"locked_block_parts"`
	ValidRound         int                 `json:"valid_round"`
	ValidBlock         *types.Block        `json:"valid_block"`
	ValidBlockParts    *types.PartSet      `json:"valid_block_parts"`
	Votes              *HeightVoteSet      `json:"votes"`
	CommitRound        int                 `json:"commit_round"` //
	LastCommit         *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
	LastValidators     *types.ValidatorSet `json:"last_validators"`
}

// RoundStateEvent returns the H/R/S of the RoundState as an event.
func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
	// XXX: copy the RoundState
	// if we want to avoid this, we may need synchronous events after all
	rs_ := *rs
	edrs := types.EventDataRoundState{
		Height:     rs.Height,
		Round:      rs.Round,
		Step:       rs.Step.String(),
		RoundState: &rs_,
	}
	return edrs
}

// String returns a string
func (rs *RoundState) String() string {
	return rs.StringIndented("")
}

// StringIndented returns a string
func (rs *RoundState) StringIndented(indent string) string {
	return fmt.Sprintf(`RoundState{
%s  H:%v R:%v S:%v
%s  StartTime:     %v
%s  CommitTime:    %v
%s  Validators:    %v
%s  Proposal:      %v
%s  ProposalBlock: %v %v
%s  LockedRound:   %v
%s  LockedBlock:   %v %v
%s  ValidRound:   %v
%s  ValidBlock:   %v %v
%s  Votes:         %v
%s  LastCommit:    %v
%s  LastValidators:%v
%s}`,
		indent, rs.Height, rs.Round, rs.Step,
		indent, rs.StartTime,
		indent, rs.CommitTime,
		indent, rs.Validators.StringIndented(indent+"    "),
		indent, rs.Proposal,
		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
		indent, rs.LockedRound,
		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
		indent, rs.ValidRound,
		indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(),
		indent, rs.Votes.StringIndented(indent+"    "),
		indent, rs.LastCommit.StringShort(),
		indent, rs.LastValidators.StringIndented(indent+"    "),
		indent)
}

// StringShort returns a string
func (rs *RoundState) StringShort() string {
	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
		rs.Height, rs.Round, rs.Step, rs.StartTime)
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"

	"github.com/giantswarm/inago/task"
)

func (c controller) executeTaskAction(f func(ctx context.Context, req Request) (*task.Task, error), ctx context.Context, req Request) error {
	taskObject, err := f(ctx, req)
	if err != nil {
		return maskAny(err)
	}
	closer := make(<-chan struct{})
	taskObject, err = c.WaitForTask(ctx, taskObject.ID, closer)
	if err != nil {
		return maskAny(err)
	}
	if task.HasFailedStatus(taskObject) {
		return maskAny(fmt.Errorf(taskObject.Error))
	}
	return nil
}

func (c controller) getNumRunningSlices(req Request) (int, error) {
	groupStatus, err := c.groupStatus(req)
	if IsUnitNotFound(err) {
		return 0, nil
	} else if err != nil {
		return 0, maskAny(err)
	}

	grouped, err := UnitStatusList(groupStatus).Group()
	if err != nil {
		return 0, maskAny(err)
	}

	var sliceIDs []string
	for _, us := range grouped {
		groupedStatuses := grouped.unitStatusesBySliceID(us.SliceID)
		if len(groupedStatuses) > 1 {
			// This group has an inconsistent state. Thus we do not consider it
			// running.
			continue
		}

		aggregator := Aggregator{
			Logger: c.Config.Logger,
		}
		ok, err := aggregator.unitHasStatus(groupedStatuses[0], StatusRunning)
		if err != nil {
			return 0, maskAny(err)
		}
		if !ok {
			continue
		}

		sliceIDs = append(sliceIDs, us.SliceID)
	}

	return len(sliceIDs), nil
}

func (c controller) isGroupRemovalAllowed(req Request, minAlive int) (bool, error) {
	numRunning, err := c.getNumRunningSlices(req)
	if err != nil {
		return false, maskAny(err)
	}

	if numRunning > minAlive {
		return true, nil
	}

	return false, nil
}

func (c controller) isGroupAdditionAllowed(req Request, maxGrowth int) (bool, error) {
	numRunning, err := c.getNumRunningSlices(req)
	if err != nil {
		return false, maskAny(err)
	}

	if numRunning < maxGrowth {
		return true, nil
	}

	return false, nil
}

func (c controller) addFirst(ctx context.Context, req Request, opts UpdateOptions) error {
	newReq, err := c.runAddWorker(ctx, req, opts)
	if err != nil {
		return maskAny(err)
	}
	n, err := c.getNumRunningSlices(newReq)
	if err != nil {
		return maskAny(err)
	}
	if n != len(newReq.SliceIDs) {
		return maskAnyf(updateFailedError, "slice not running: %v", newReq.SliceIDs)
	}
	err = c.runRemoveWorker(ctx, req)
	if err != nil {
		return maskAny(err)
	}

	return nil
}

func (c controller) runAddWorker(ctx context.Context, req Request, opts UpdateOptions) (Request, error) {
	// Create new random IDs.
	req.DesiredSlices = 1
	req.SliceIDs = nil
	newReq, err := c.ExtendWithRandomSliceIDs(req)
	if err != nil {
	       return Request{}, maskAny(err)
	}

	// Submit.
	if err := c.executeTaskAction(c.Submit, ctx, newReq); err != nil {
		return Request{}, maskAny(err)
	}

	// Start.
	if err := c.executeTaskAction(c.Start, ctx, newReq); err != nil {
		return Request{}, maskAny(err)
	}

	time.Sleep(time.Duration(opts.ReadySecs) * time.Second)

	return req, nil
}

func (c controller) removeFirst(ctx context.Context, req Request, opts UpdateOptions) error {
	err := c.runRemoveWorker(ctx, req)
	if err != nil {
		return maskAny(err)
	}
	newReq, err := c.runAddWorker(ctx, req, opts)
	if err != nil {
		return maskAny(err)
	}
	n, err := c.getNumRunningSlices(newReq)
	if err != nil {
		return maskAny(err)
	}
	if n != len(newReq.SliceIDs) {
		return maskAnyf(updateFailedError, "slice not running: %v", newReq.SliceIDs)
	}

	return nil
}

func (c controller) runRemoveWorker(ctx context.Context, req Request) error {
	// Stop.
	if err := c.executeTaskAction(c.Stop, ctx, req); err != nil {
		return maskAny(err)
	}

	// Destroy.
	if err := c.executeTaskAction(c.Destroy, ctx, req); err != nil {
		return maskAny(err)
	}
	return nil
}

// UpdateOptions represents the options defining the strategy of an update
// process. Lets have a look at how the update process of 3 group slices would
// look like using the given options.
//
//     MaxGrowth    1
//     MinAlive     2
//     ReadySecs    30
//
//     @1 (running)  ->  @1 (stopped/destroyed)
//     @2 (running)  ->  @2 (running)            ->  @2 (stopped/destroyed)
//     @3 (running)  ->  @3 (running)            ->  @3 (running)            ->  @3 (stopped/destroyed)
//                   ->  @1 (submitted/running)  ->  @1 (running)            ->  @1 (running)            ->  @1 (running)
//                                               ->  @2 (submitted/running)  ->  @2 (running)            ->  @2 (running)
//                                                                           ->  @3 (submitted/running)  ->  @3 (running)
//
type UpdateOptions struct {
	// MaxGrowth represents the number of groups allowed to be added at a given
	// time. No more than MaxGrowth groups will be added at the same time during
	// the update process.
	MaxGrowth int

	// MinAlive represents the number of groups required to stay healthy during
	// the update process. No more than MinAlive groups will be removed at the
	// same time during the update process.
	MinAlive int

	// ReadySecs represents the number of seconds required to wait before ending
	// the update process of one group and starting the update process of another
	// group. This is basically a cool down where the update process sleeps
	// before updating the next group.
	ReadySecs int
}

func (c controller) UpdateWithStrategy(ctx context.Context, req Request, opts UpdateOptions) error {
	fail := make(chan error, 1)
	numTotal := len(req.SliceIDs)
	done := make(chan struct{}, numTotal)
	var addInProgress int64
	var removeInProgress int64

	if !req.isSliceable() {
		return maskAnyf(updateNotAllowedError, "cannot update unsliceable group")
	}

	for _, sliceID := range req.SliceIDs {
		if sliceID == "" {
			return maskAnyf(updateNotAllowedError, "group misses slice ID")
		}
	}

	if numTotal < opts.MinAlive {
		return maskAnyf(updateNotAllowedError, "invalid min alive option")
	}

	for _, sliceID := range req.SliceIDs {
		newReq := req
		newReq.SliceIDs = []string{sliceID}

		for {
			// add
			maxGrowth := opts.MaxGrowth + numTotal - opts.MinAlive - int(addInProgress)
			ok, err := c.isGroupAdditionAllowed(req, maxGrowth)
			if err != nil {
				return maskAny(err)
			}
			if ok {
				go func() {
					atomic.AddInt64(&addInProgress, 1)
					err := c.addFirst(ctx, newReq, opts)
					if err != nil {
						fail <- maskAny(err)
						return
					}
					atomic.AddInt64(&addInProgress, -1)
					done <- struct{}{}
				}()

				break
			}

			// remove
			minAlive := opts.MinAlive + int(removeInProgress)
			ok, err = c.isGroupRemovalAllowed(req, minAlive)
			if err != nil {
				return maskAny(err)
			}
			if ok {
				go func() {
					atomic.AddInt64(&removeInProgress, 1)
					err := c.removeFirst(ctx, newReq, opts)
					if err != nil {
						fail <- maskAny(err)
						return
					}
					atomic.AddInt64(&removeInProgress, -1)
					done <- struct{}{}
				}()

				break
			}

			time.Sleep(c.WaitSleep)
		}
	}

	tc := 0
	for {
		select {
		case err := <-fail:
			return maskAny(err)
		case <-done:
			tc++
			if tc == numTotal {
				return nil
			}
		case <-time.After(c.WaitTimeout):
			return maskAny(waitTimeoutReachedError)
		}
	}
}
<commit_msg>Update: We generate new sliceIDs in newReq, not Req. So we must return the right thing<commit_after>package controller

import (
	"fmt"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"

	"github.com/giantswarm/inago/task"
)

// executeTaskAction executes the given function f to create a new task and block until
// it finishes. If the task fails, an error is raised.
func (c controller) executeTaskAction(f func(ctx context.Context, req Request) (*task.Task, error), ctx context.Context, req Request) error {
	taskObject, err := f(ctx, req)
	if err != nil {
		return maskAny(err)
	}
	closer := make(<-chan struct{})
	taskObject, err = c.WaitForTask(ctx, taskObject.ID, closer)
	if err != nil {
		return maskAny(err)
	}
	if task.HasFailedStatus(taskObject) {
		return maskAny(fmt.Errorf(taskObject.Error))
	}
	return nil
}

func (c controller) getNumRunningSlices(req Request) (int, error) {
	groupStatus, err := c.groupStatus(req)
	if IsUnitNotFound(err) {
		return 0, nil
	} else if err != nil {
		return 0, maskAny(err)
	}

	grouped, err := UnitStatusList(groupStatus).Group()
	if err != nil {
		return 0, maskAny(err)
	}

	var sliceIDs []string
	for _, us := range grouped {
		groupedStatuses := grouped.unitStatusesBySliceID(us.SliceID)
		if len(groupedStatuses) > 1 {
			// This group has an inconsistent state. Thus we do not consider it
			// running.
			continue
		}

		aggregator := Aggregator{
			Logger: c.Config.Logger,
		}
		ok, err := aggregator.unitHasStatus(groupedStatuses[0], StatusRunning)
		if err != nil {
			return 0, maskAny(err)
		}
		if !ok {
			continue
		}

		sliceIDs = append(sliceIDs, us.SliceID)
	}

	return len(sliceIDs), nil
}

func (c controller) isGroupRemovalAllowed(req Request, minAlive int) (bool, error) {
	numRunning, err := c.getNumRunningSlices(req)
	if err != nil {
		return false, maskAny(err)
	}

	if numRunning > minAlive {
		return true, nil
	}

	return false, nil
}

func (c controller) isGroupAdditionAllowed(req Request, maxGrowth int) (bool, error) {
	numRunning, err := c.getNumRunningSlices(req)
	if err != nil {
		return false, maskAny(err)
	}

	if numRunning < maxGrowth {
		return true, nil
	}

	return false, nil
}

func (c controller) addFirst(ctx context.Context, req Request, opts UpdateOptions) error {
	newReq, err := c.runAddWorker(ctx, req, opts)
	if err != nil {
		return maskAny(err)
	}
	n, err := c.getNumRunningSlices(newReq)
	if err != nil {
		return maskAny(err)
	}
	if n != len(newReq.SliceIDs) {
		return maskAnyf(updateFailedError, "addFirst: slice not running: %d != %v", n, newReq.SliceIDs)
	}
	err = c.runRemoveWorker(ctx, req)
	if err != nil {
		return maskAny(err)
	}

	return nil
}

func (c controller) runAddWorker(ctx context.Context, req Request, opts UpdateOptions) (Request, error) {
	// Create new random IDs.
	req.DesiredSlices = 1
	req.SliceIDs = nil
	newReq, err := c.ExtendWithRandomSliceIDs(req)
	if err != nil {
		return Request{}, maskAny(err)
	}

	// Submit.
	if err := c.executeTaskAction(c.Submit, ctx, newReq); err != nil {
		return Request{}, maskAny(err)
	}

	// Start.
	if err := c.executeTaskAction(c.Start, ctx, newReq); err != nil {
		return Request{}, maskAny(err)
	}

	time.Sleep(time.Duration(opts.ReadySecs) * time.Second)

	return newReq, nil
}

func (c controller) removeFirst(ctx context.Context, req Request, opts UpdateOptions) error {
	err := c.runRemoveWorker(ctx, req)
	if err != nil {
		return maskAny(err)
	}

	newReq, err := c.runAddWorker(ctx, req, opts)
	if err != nil {
		return maskAny(err)
	}
	n, err := c.getNumRunningSlices(newReq)
	if err != nil {
		return maskAny(err)
	}
	if n != len(newReq.SliceIDs) {
		return maskAnyf(updateFailedError, "removeFirst: slice not running: %d != %v", n, newReq.SliceIDs)
	}

	return nil
}

func (c controller) runRemoveWorker(ctx context.Context, req Request) error {
	// Stop.
	if err := c.executeTaskAction(c.Stop, ctx, req); err != nil {
		return maskAny(err)
	}

	// Destroy.
	if err := c.executeTaskAction(c.Destroy, ctx, req); err != nil {
		return maskAny(err)
	}
	return nil
}

// UpdateOptions represents the options defining the strategy of an update
// process. Lets have a look at how the update process of 3 group slices would
// look like using the given options.
//
//     MaxGrowth    1
//     MinAlive     2
//     ReadySecs    30
//
//     @1 (running)  ->  @1 (stopped/destroyed)
//     @2 (running)  ->  @2 (running)            ->  @2 (stopped/destroyed)
//     @3 (running)  ->  @3 (running)            ->  @3 (running)            ->  @3 (stopped/destroyed)
//                   ->  @1 (submitted/running)  ->  @1 (running)            ->  @1 (running)            ->  @1 (running)
//                                               ->  @2 (submitted/running)  ->  @2 (running)            ->  @2 (running)
//                                                                           ->  @3 (submitted/running)  ->  @3 (running)
//
type UpdateOptions struct {
	// MaxGrowth represents the number of groups allowed to be added at a given
	// time. No more than MaxGrowth groups will be added at the same time during
	// the update process.
	MaxGrowth int

	// MinAlive represents the number of groups required to stay healthy during
	// the update process. No more than MinAlive groups will be removed at the
	// same time during the update process.
	MinAlive int

	// ReadySecs represents the number of seconds required to wait before ending
	// the update process of one group and starting the update process of another
	// group. This is basically a cool down where the update process sleeps
	// before updating the next group.
	ReadySecs int
}

func (c controller) UpdateWithStrategy(ctx context.Context, req Request, opts UpdateOptions) error {
	fail := make(chan error, 1)
	numTotal := len(req.SliceIDs)
	done := make(chan struct{}, numTotal)
	var addInProgress int64
	var removeInProgress int64

	if !req.isSliceable() {
		return maskAnyf(updateNotAllowedError, "cannot update unsliceable group")
	}

	for _, sliceID := range req.SliceIDs {
		if sliceID == "" {
			return maskAnyf(updateNotAllowedError, "group misses slice ID")
		}
	}

	if numTotal < opts.MinAlive {
		return maskAnyf(updateNotAllowedError, "invalid min alive option")
	}

	for _, sliceID := range req.SliceIDs {
		newReq := req
		newReq.SliceIDs = []string{sliceID}

		for {
			// add
			maxGrowth := opts.MaxGrowth + numTotal - opts.MinAlive - int(addInProgress)
			ok, err := c.isGroupAdditionAllowed(req, maxGrowth)
			if err != nil {
				return maskAny(err)
			}
			if ok {
				go func() {
					atomic.AddInt64(&addInProgress, 1)
					err := c.addFirst(ctx, newReq, opts)
					if err != nil {
						fail <- maskAny(err)
						return
					}
					atomic.AddInt64(&addInProgress, -1)
					done <- struct{}{}
				}()

				break
			}

			// remove
			minAlive := opts.MinAlive + int(removeInProgress)
			ok, err = c.isGroupRemovalAllowed(req, minAlive)
			if err != nil {
				return maskAny(err)
			}
			if ok {
				go func() {
					atomic.AddInt64(&removeInProgress, 1)
					err := c.removeFirst(ctx, newReq, opts)
					if err != nil {
						fail <- maskAny(err)
						return
					}
					atomic.AddInt64(&removeInProgress, -1)
					done <- struct{}{}
				}()

				break
			}

			time.Sleep(c.WaitSleep)
		}
	}

	tc := 0
	for {
		select {
		case err := <-fail:
			return maskAny(err)
		case <-done:
			tc++
			if tc == numTotal {
				return nil
			}
		case <-time.After(c.WaitTimeout):
			return maskAny(waitTimeoutReachedError)
		}
	}
}
<|endoftext|>


<commit_before>/* https://leetcode.com/problems/longest-substring-without-repeating-characters/#/description

Given a string, find the length of the longest substring without repeating characters.

Examples:

Given "abcabcbb", the answer is "abc", which the length is 3.

Given "bbbbb", the answer is "b", with the length of 1.

Given "pwwkew", the answer is "wke", with the length of 3. Note that the answer must be a substring, "pwke" is a subsequence and not a substring.

*/

package main

import "fmt"

func lengthOfLongestSubstring(s string) int {
	var (
		max_length int
		total      = len(s)
	)

	for i := 0; i < total; i++ {
		subString := make(map[byte]int)
		subString[s[i]] = i
		for j := i + 1; j < total; j++ {
			if _, ok := subString[s[j]]; ok {
				break
			} else {
				subString[s[j]] = j
			}
		}
		subLen := len(subString)
		if subLen > max_length {
			max_length = subLen
		}
	}
	return max_length
}

func main() {
	str := []string{"a", "au", "abcabcbb", "bbbbb", "pwwkew"}
	for _, s := range str {
		fmt.Println(s)
		fmt.Println(lengthOfLongestSubstring(s))
	}
}
<commit_msg>i -j instead of len(map)<commit_after>/* https://leetcode.com/problems/longest-substring-without-repeating-characters/#/description

Given a string, find the length of the longest substring without repeating characters.

Examples:

Given "abcabcbb", the answer is "abc", which the length is 3.

Given "bbbbb", the answer is "b", with the length of 1.

Given "pwwkew", the answer is "wke", with the length of 3. Note that the answer must be a substring, "pwke" is a subsequence and not a substring.

*/

package main

import "fmt"

func lengthOfLongestSubstring(s string) int {
	var (
		max_length       int
		i, j, cur, total = 0, 0, 0, len(s)
	)

	for ; i < total; i++ {
		subString := make(map[byte]int)
		subString[s[i]] = i
		for j = i + 1; j < total; j++ {
			if _, ok := subString[s[j]]; ok {
				break
			} else {
				subString[s[j]] = j
			}
		}
		cur = j - i
		if cur > max_length {
			max_length = cur
		}
	}
	return max_length
}

func main() {
	str := []string{"a", "au", "abcabcbb", "bbbbb", "pwwkew"}
	for _, s := range str {
		fmt.Println(s)
		fmt.Println(lengthOfLongestSubstring(s))
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package keyserver

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strings"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/agl/ed25519"
	"github.com/andres-erbsen/clock"
	"github.com/syndtr/goleveldb/leveldb"

	"github.com/yahoo/coname/keyserver/kv/leveldbkv"
	"github.com/yahoo/coname/keyserver/replication/raftlog"
	raftproto "github.com/yahoo/coname/keyserver/replication/raftlog/proto"
	"github.com/yahoo/coname/proto"
)

func parsePrivateKey(der []byte) (crypto.PrivateKey, error) {
	// Copied from the parsePrivateKey function in crypto/tls
	if key, err := x509.ParsePKCS1PrivateKey(der); err == nil {
		return key, nil
	}
	if key, err := x509.ParsePKCS8PrivateKey(der); err == nil {
		switch key := key.(type) {
		case *rsa.PrivateKey, *ecdsa.PrivateKey:
			return key, nil
		default:
			return nil, fmt.Errorf("found unknown private key type in PKCS#8 wrapping")
		}
	}
	if key, err := x509.ParseECPrivateKey(der); err == nil {
		return key, nil
	}

	return nil, fmt.Errorf("failed to parse private key")
}

// This getKey interprets key IDs as paths, and loads private keys from the
// specified file
func getKey(keyid string) (crypto.PrivateKey, error) {
	fileContents, err := ioutil.ReadFile(keyid)
	if err != nil {
		return nil, err
	}
	if strings.HasSuffix(keyid, ".ed25519secret") {
		if got, want := len(fileContents), ed25519.PrivateKeySize; got != want {
			return nil, fmt.Errorf("ed25519 private key %s has wrong size %d (want %d)", keyid, got, want)
		}
		return fileContents, nil
	} else {
		keyPEM := fileContents
		var keyDER *pem.Block
		for {
			keyDER, keyPEM = pem.Decode(keyPEM)
			if keyDER == nil {
				return nil, fmt.Errorf("failed to parse key PEM in %s", keyid)
			}
			if keyDER.Type == "PRIVATE KEY" || strings.HasSuffix(keyDER.Type, " PRIVATE KEY") {
				break
			}
		}
		return parsePrivateKey(keyDER.Bytes)
	}
}

func majority(nReplicas int) int {
	return nReplicas/2 + 1
}

func RunWithConfig(cfg *proto.ReplicaConfig) {
	// TODO: since we only want to support precisely this ratification policy,
	// this should be moved into server.go
	ratificationPolicy := &proto.AuthorizationPolicy{
		PublicKeys: make(map[uint64]*proto.PublicKey),
		Quorum: &proto.QuorumExpr{
			Threshold: uint32(majority(len(cfg.KeyserverConfig.InitialReplicas))),
		},
	}
	replicaIDs := []uint64{}
	for _, replica := range cfg.KeyserverConfig.InitialReplicas {
		replicaIDs = append(replicaIDs, replica.ID)
		replicaExpr := &proto.QuorumExpr{
			Threshold: 1,
		}
		for _, pk := range replica.PublicKeys {
			pkid := proto.KeyID(pk)
			ratificationPolicy.PublicKeys[pkid] = pk
			replicaExpr.Candidates = append(replicaExpr.Candidates, pkid)
		}
		ratificationPolicy.Quorum.Subexpressions = append(ratificationPolicy.Quorum.Subexpressions, replicaExpr)
	}

	leveldb, err := leveldb.OpenFile(cfg.LevelDBPath, nil)
	if err != nil {
		log.Fatalf("Couldn't open DB in directory %s: %s", cfg.LevelDBPath, err)
	}
	db := leveldbkv.Wrap(leveldb)

	clk := clock.New()

	raftListener, err := net.Listen("tcp", cfg.RaftAddr)
	if err != nil {
		log.Fatalf("Couldn't bind to Raft node address %s: %s", cfg.RaftAddr, err)
	}
	defer raftListener.Close()
	raftTLS, err := cfg.RaftTLS.Config(getKey)
	if err != nil {
		log.Fatalf("Bad Raft TLS configuration: %s", err)
	}
	raftCreds := credentials.NewTLS(raftTLS)
	raftServer := grpc.NewServer(grpc.Creds(raftCreds))
	go raftServer.Serve(raftListener)
	defer raftServer.Stop()

	dialRaftPeer := func(id uint64) raftproto.RaftClient {
		// TODO use current, not initial, config
		for _, replica := range cfg.KeyserverConfig.InitialReplicas {
			if replica.ID == id {
				conn, err := grpc.Dial(replica.RaftAddr, grpc.WithTransportCredentials(raftCreds))
				if err != nil {
					log.Panicf("Raft GRPC dial failed: %s", err)
				}
				return raftproto.NewRaftClient(conn)
			}
		}
		log.Panicf("No raft peer %x in configuration", id)
		return nil
	}

	raft := raftlog.New(
		cfg.ReplicaID, replicaIDs, db, []byte{tableReplicationLogPrefix},
		clk, cfg.RaftHeartbeat.Duration(), raftServer, dialRaftPeer,
	)
	defer raft.Stop()

	server, err := Open(cfg, db, raft, ratificationPolicy, clk, getKey, net.LookupTXT)
	if err != nil {
		log.Fatalf("Failed to initialize keyserver: %s", err)
	}
	server.Start()
	defer server.Stop()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	<-ch
}
<commit_msg>VRF secret<commit_after>// Copyright 2014-2015 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package keyserver

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strings"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/agl/ed25519"
	"github.com/andres-erbsen/clock"
	"github.com/syndtr/goleveldb/leveldb"

	"github.com/yahoo/coname/keyserver/kv/leveldbkv"
	"github.com/yahoo/coname/keyserver/replication/raftlog"
	raftproto "github.com/yahoo/coname/keyserver/replication/raftlog/proto"
	"github.com/yahoo/coname/proto"
	"github.com/yahoo/coname/vrf"
)

func parsePrivateKey(der []byte) (crypto.PrivateKey, error) {
	// Copied from the parsePrivateKey function in crypto/tls
	if key, err := x509.ParsePKCS1PrivateKey(der); err == nil {
		return key, nil
	}
	if key, err := x509.ParsePKCS8PrivateKey(der); err == nil {
		switch key := key.(type) {
		case *rsa.PrivateKey, *ecdsa.PrivateKey:
			return key, nil
		default:
			return nil, fmt.Errorf("found unknown private key type in PKCS#8 wrapping")
		}
	}
	if key, err := x509.ParseECPrivateKey(der); err == nil {
		return key, nil
	}

	return nil, fmt.Errorf("failed to parse private key")
}

// This getKey interprets key IDs as paths, and loads private keys from the
// specified file
func getKey(keyid string) (crypto.PrivateKey, error) {
	fileContents, err := ioutil.ReadFile(keyid)
	if err != nil {
		return nil, err
	}
	if strings.HasSuffix(keyid, ".ed25519secret") {
		if got, want := len(fileContents), ed25519.PrivateKeySize; got != want {
			return nil, fmt.Errorf("ed25519 private key %s has wrong size %d (want %d)", keyid, got, want)
		}
		return fileContents, nil
	} else if strings.HasSuffix(keyid, ".vrfsecret") {
		if got, want := len(fileContents), vrf.SecretKeySize; got != want {
			return nil, fmt.Errorf("VRF private key %s has wrong size %d (want %d)", keyid, got, want)
		}
		return fileContents, nil
	} else {
		keyPEM := fileContents
		var keyDER *pem.Block
		for {
			keyDER, keyPEM = pem.Decode(keyPEM)
			if keyDER == nil {
				return nil, fmt.Errorf("failed to parse key PEM in %s", keyid)
			}
			if keyDER.Type == "PRIVATE KEY" || strings.HasSuffix(keyDER.Type, " PRIVATE KEY") {
				break
			}
		}
		return parsePrivateKey(keyDER.Bytes)
	}
}

func majority(nReplicas int) int {
	return nReplicas/2 + 1
}

func RunWithConfig(cfg *proto.ReplicaConfig) {
	// TODO: since we only want to support precisely this ratification policy,
	// this should be moved into server.go
	ratificationPolicy := &proto.AuthorizationPolicy{
		PublicKeys: make(map[uint64]*proto.PublicKey),
		Quorum: &proto.QuorumExpr{
			Threshold: uint32(majority(len(cfg.KeyserverConfig.InitialReplicas))),
		},
	}
	replicaIDs := []uint64{}
	for _, replica := range cfg.KeyserverConfig.InitialReplicas {
		replicaIDs = append(replicaIDs, replica.ID)
		replicaExpr := &proto.QuorumExpr{
			Threshold: 1,
		}
		for _, pk := range replica.PublicKeys {
			pkid := proto.KeyID(pk)
			ratificationPolicy.PublicKeys[pkid] = pk
			replicaExpr.Candidates = append(replicaExpr.Candidates, pkid)
		}
		ratificationPolicy.Quorum.Subexpressions = append(ratificationPolicy.Quorum.Subexpressions, replicaExpr)
	}

	leveldb, err := leveldb.OpenFile(cfg.LevelDBPath, nil)
	if err != nil {
		log.Fatalf("Couldn't open DB in directory %s: %s", cfg.LevelDBPath, err)
	}
	db := leveldbkv.Wrap(leveldb)

	clk := clock.New()

	raftListener, err := net.Listen("tcp", cfg.RaftAddr)
	if err != nil {
		log.Fatalf("Couldn't bind to Raft node address %s: %s", cfg.RaftAddr, err)
	}
	defer raftListener.Close()
	raftTLS, err := cfg.RaftTLS.Config(getKey)
	if err != nil {
		log.Fatalf("Bad Raft TLS configuration: %s", err)
	}
	raftCreds := credentials.NewTLS(raftTLS)
	raftServer := grpc.NewServer(grpc.Creds(raftCreds))
	go raftServer.Serve(raftListener)
	defer raftServer.Stop()

	dialRaftPeer := func(id uint64) raftproto.RaftClient {
		// TODO use current, not initial, config
		for _, replica := range cfg.KeyserverConfig.InitialReplicas {
			if replica.ID == id {
				conn, err := grpc.Dial(replica.RaftAddr, grpc.WithTransportCredentials(raftCreds))
				if err != nil {
					log.Panicf("Raft GRPC dial failed: %s", err)
				}
				return raftproto.NewRaftClient(conn)
			}
		}
		log.Panicf("No raft peer %x in configuration", id)
		return nil
	}

	raft := raftlog.New(
		cfg.ReplicaID, replicaIDs, db, []byte{tableReplicationLogPrefix},
		clk, cfg.RaftHeartbeat.Duration(), raftServer, dialRaftPeer,
	)
	defer raft.Stop()

	server, err := Open(cfg, db, raft, ratificationPolicy, clk, getKey, net.LookupTXT)
	if err != nil {
		log.Fatalf("Failed to initialize keyserver: %s", err)
	}
	server.Start()
	defer server.Stop()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	<-ch
}
<|endoftext|>


<commit_before>package mcstore

import (
	"net/http"

	"sync"

	"github.com/emicklei/go-restful"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// apikeyFilter implements a filter for checking the apikey
// passed in with a request.
type apikeyFilter struct {
	users   dai.Users
	apikeys map[string]*schema.User
	mutex   sync.RWMutex
}

// newAPIKeyFilter creates a new apikeyFilter instance.
func newAPIKeyFilter(users dai.Users) *apikeyFilter {
	return &apikeyFilter{
		users:   users,
		apikeys: make(map[string]*schema.User),
	}
}

// Filter implements the Filter interface for apikey lookup. It checks if an apikey is
// valid. If the apikey is found it sets the "user" attribute to the user structure. If
// the apikey is invalid then the filter doesn't pass the request on, and instead returns
// an http.StatusUnauthorized.
func (f *apikeyFilter) Filter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
	apikey := req.Request.URL.Query().Get("apikey")
	user, found := f.getUser(apikey)
	if !found {
		resp.WriteErrorString(http.StatusUnauthorized, "Not authorized")
		return
	}

	req.SetAttribute("user", *user)
	chain.ProcessFilter(req, resp)
}

// getUser matches the user with the apikey. If it cannot find
// a match then it returns false. getUser caches the key/user
// pair in f.apikeys.
func (f *apikeyFilter) getUser(apikey string) (*schema.User, bool) {
	if apikey == "" {
		// No key was passed.
		return nil, false
	}

	user, found := f.getUserWithReadLock(apikey)
	if !found {
		user, err := f.users.ByAPIKey(apikey)
		if err != nil {
			return nil, false
		}
		f.setUserWithWriteLock(apikey, user)
		return user, true
	}

	return user, true
}

// getUserWithReadLock will acquire a read lock and look the user up in the
// hash table cache.
func (f *apikeyFilter) getUserWithReadLock(apikey string) (*schema.User, bool) {
	defer f.mutex.RUnlock()
	f.mutex.RLock()

	user, found := f.apikeys[apikey]
	return user, found
}

// setUserWithWriteLock will acquire a write lock and add the user to the
// hash table cache.
func (f *apikeyFilter) setUserWithWriteLock(apikey string, user *schema.User) {
	defer f.mutex.Unlock()
	f.mutex.Lock()

	f.apikeys[apikey] = user
}
<commit_msg>Switch filter to getting the session from request.Attribute("session"). Create a new dai.Users for each call, since it requires a session to be used.<commit_after>package mcstore

import (
	"net/http"

	"sync"

	r "github.com/dancannon/gorethink"
	"github.com/emicklei/go-restful"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// apikeyFilter implements a filter for checking the apikey
// passed in with a request.
type apikeyFilter struct {
	apikeys map[string]*schema.User
	mutex   sync.RWMutex
}

// newAPIKeyFilter creates a new apikeyFilter instance.
func newAPIKeyFilter() *apikeyFilter {
	return &apikeyFilter{
		apikeys: make(map[string]*schema.User),
	}
}

// Filter implements the Filter interface for apikey lookup. It checks if an apikey is
// valid. If the apikey is found it sets the "user" attribute to the user structure. If
// the apikey is invalid then the filter doesn't pass the request on, and instead returns
// an http.StatusUnauthorized.
func (f *apikeyFilter) Filter(request *restful.Request, response *restful.Response, chain *restful.FilterChain) {
	apikey := request.Request.URL.Query().Get("apikey")
	session := request.Attribute("session").(*r.Session)
	rusers := dai.NewRUsers(session)
	user, found := f.getUser(apikey, rusers)
	if !found {
		response.WriteErrorString(http.StatusUnauthorized, "Not authorized")
	} else {
		request.SetAttribute("user", *user)
		chain.ProcessFilter(request, response)
	}
}

// getUser matches the user with the apikey. If it cannot find a match then it returns false.
// getUser caches the key/user pair in f.apikeys.
func (f *apikeyFilter) getUser(apikey string, users dai.Users) (*schema.User, bool) {
	if apikey == "" {
		// No key was passed.
		return nil, false
	}

	user, found := f.getUserWithReadLock(apikey)
	if !found {
		user, err := users.ByAPIKey(apikey)
		if err != nil {
			return nil, false
		}
		f.setUserWithWriteLock(apikey, user)
		return user, true
	}

	return user, true
}

// getUserWithReadLock will acquire a read lock and look the user up in the
// hash table cache.
func (f *apikeyFilter) getUserWithReadLock(apikey string) (*schema.User, bool) {
	defer f.mutex.RUnlock()
	f.mutex.RLock()

	user, found := f.apikeys[apikey]
	return user, found
}

// setUserWithWriteLock will acquire a write lock and add the user to the
// hash table cache.
func (f *apikeyFilter) setUserWithWriteLock(apikey string, user *schema.User) {
	defer f.mutex.Unlock()
	f.mutex.Lock()

	f.apikeys[apikey] = user
}
<|endoftext|>


<commit_before>package pod

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/golang/glog"
	"github.com/hyperhq/hyper/server/httputils"
	"golang.org/x/net/context"
)

func (p *podRouter) getPodInfo(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	data, err := p.backend.CmdGetPodInfo(r.Form.Get("podName"))
	if err != nil {
		return err
	}

	return httputils.WriteJSON(w, http.StatusOK, data)
}

func (p *podRouter) getPodStats(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	data, err := p.backend.CmdGetPodStats(r.Form.Get("podId"))
	if err != nil {
		return err
	}

	return httputils.WriteJSON(w, http.StatusOK, data)
}

func (p *podRouter) getList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	item := r.Form.Get("item")
	auxiliary := httputils.BoolValue(r, "auxiliary")
	pod := r.Form.Get("pod")
	vm := r.Form.Get("vm")

	glog.V(1).Infof("List type is %s, specified pod: [%s], specified vm: [%s], list auxiliary pod: %v", item, pod, vm, auxiliary)

	env, err := p.backend.CmdList(item, pod, vm, auxiliary)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	podArgs, _ := ioutil.ReadAll(r.Body)
	autoRemove := false
	if r.Form.Get("remove") == "yes" || r.Form.Get("remove") == "true" {
		autoRemove = true
	}
	glog.V(1).Infof("Args string is %s, autoremove %v", string(podArgs), autoRemove)

	env, err := p.backend.CmdCreatePod(string(podArgs), autoRemove)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodLabels(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	labels := make(map[string]string)

	if err := json.Unmarshal([]byte(r.Form.Get("labels")), &labels); err != nil {
		return err
	}

	override := false
	if r.Form.Get("override") == "true" || r.Form.Get("override") == "yes" {
		override = true
	}

	env, err := p.backend.CmdSetPodLabels(podId, override, labels)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodStart(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	vmId := r.Form.Get("vmId")
	tag := r.Form.Get("tag")

	if tag == "" {
		env, err := p.backend.CmdStartPod(nil, nil, podId, vmId, tag)
		if err != nil {
			return err
		}

		return env.WriteJSON(w, http.StatusOK)
	} else {
		// Setting up the streaming http interface.
		inStream, outStream, err := httputils.HijackConnection(w)
		if err != nil {
			return err
		}
		defer httputils.CloseStreams(inStream, outStream)

		fmt.Fprintf(outStream, "HTTP/1.1 101 UPGRADED\r\nContent-Type: application/vnd.docker.raw-stream\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n")

		if _, err = p.backend.CmdStartPod(inStream, outStream.(io.WriteCloser), podId, vmId, tag); err != nil {
			return err
		}
		w.WriteHeader(http.StatusNoContent)
		return nil
	}
}

func (p *podRouter) postPodStop(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	stopVm := r.Form.Get("stopVm")

	env, err := p.backend.CmdStopPod(podId, stopVm)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) postPodPause(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")

	if err := p.backend.CmdPausePod(podId); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (p *podRouter) postPodUnpause(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	if err := p.backend.CmdUnpausePod(podId); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (p *podRouter) postVmCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	cpu, err := strconv.Atoi(r.Form.Get("cpu"))
	if err != nil {
		return err
	}
	mem, err := strconv.Atoi(r.Form.Get("mem"))
	if err != nil {
		return err
	}
	async := false
	if r.Form.Get("async") == "yes" || r.Form.Get("async") == "true" {
		async = true
	}

	env, err := p.backend.CmdCreateVm(cpu, mem, async)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) deletePod(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	env, err := p.backend.CmdCleanPod(podId)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) deleteVm(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	vmId := r.Form.Get("vm")
	env, err := p.backend.CmdKillVm(vmId)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}
<commit_msg>setup default memory and cpu for hyper vm<commit_after>package pod

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/golang/glog"
	"github.com/hyperhq/hyper/server/httputils"
	"golang.org/x/net/context"
)

func (p *podRouter) getPodInfo(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	data, err := p.backend.CmdGetPodInfo(r.Form.Get("podName"))
	if err != nil {
		return err
	}

	return httputils.WriteJSON(w, http.StatusOK, data)
}

func (p *podRouter) getPodStats(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	data, err := p.backend.CmdGetPodStats(r.Form.Get("podId"))
	if err != nil {
		return err
	}

	return httputils.WriteJSON(w, http.StatusOK, data)
}

func (p *podRouter) getList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	item := r.Form.Get("item")
	auxiliary := httputils.BoolValue(r, "auxiliary")
	pod := r.Form.Get("pod")
	vm := r.Form.Get("vm")

	glog.V(1).Infof("List type is %s, specified pod: [%s], specified vm: [%s], list auxiliary pod: %v", item, pod, vm, auxiliary)

	env, err := p.backend.CmdList(item, pod, vm, auxiliary)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	podArgs, _ := ioutil.ReadAll(r.Body)
	autoRemove := false
	if r.Form.Get("remove") == "yes" || r.Form.Get("remove") == "true" {
		autoRemove = true
	}
	glog.V(1).Infof("Args string is %s, autoremove %v", string(podArgs), autoRemove)

	env, err := p.backend.CmdCreatePod(string(podArgs), autoRemove)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodLabels(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	labels := make(map[string]string)

	if err := json.Unmarshal([]byte(r.Form.Get("labels")), &labels); err != nil {
		return err
	}

	override := false
	if r.Form.Get("override") == "true" || r.Form.Get("override") == "yes" {
		override = true
	}

	env, err := p.backend.CmdSetPodLabels(podId, override, labels)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusCreated)
}

func (p *podRouter) postPodStart(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	vmId := r.Form.Get("vmId")
	tag := r.Form.Get("tag")

	if tag == "" {
		env, err := p.backend.CmdStartPod(nil, nil, podId, vmId, tag)
		if err != nil {
			return err
		}

		return env.WriteJSON(w, http.StatusOK)
	} else {
		// Setting up the streaming http interface.
		inStream, outStream, err := httputils.HijackConnection(w)
		if err != nil {
			return err
		}
		defer httputils.CloseStreams(inStream, outStream)

		fmt.Fprintf(outStream, "HTTP/1.1 101 UPGRADED\r\nContent-Type: application/vnd.docker.raw-stream\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n")

		if _, err = p.backend.CmdStartPod(inStream, outStream.(io.WriteCloser), podId, vmId, tag); err != nil {
			return err
		}
		w.WriteHeader(http.StatusNoContent)
		return nil
	}
}

func (p *podRouter) postPodStop(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	stopVm := r.Form.Get("stopVm")

	env, err := p.backend.CmdStopPod(podId, stopVm)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) postPodPause(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")

	if err := p.backend.CmdPausePod(podId); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (p *podRouter) postPodUnpause(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	if err := p.backend.CmdUnpausePod(podId); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (p *podRouter) postVmCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	var (
		cpu   = 1
		mem   = 128
		async = false
		err   error
	)
	if value := r.Form.Get("cpu"); value != "" {
		cpu, err = strconv.Atoi(value)
		if err != nil {
			return err
		}
	}
	if value := r.Form.Get("mem"); value != "" {
		mem, err = strconv.Atoi(value)
		if err != nil {
			return err
		}
	}
	if r.Form.Get("async") == "yes" || r.Form.Get("async") == "true" {
		async = true
	}

	env, err := p.backend.CmdCreateVm(cpu, mem, async)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) deletePod(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	podId := r.Form.Get("podId")
	env, err := p.backend.CmdCleanPod(podId)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}

func (p *podRouter) deleteVm(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	vmId := r.Form.Get("vm")
	env, err := p.backend.CmdKillVm(vmId)
	if err != nil {
		return err
	}

	return env.WriteJSON(w, http.StatusOK)
}
<|endoftext|>


<commit_before>package starchat_test

import (
	"."
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"io/ioutil"
	"testing"
)

func TestHandler(t *testing.T) {
	handler := starchat.NewHandler(nil)
	server := httptest.NewServer(handler)
	defer server.Close()

	client := &http.Client{}
	{
		req, err := http.NewRequest("GET", server.URL + "/", nil)
		if err != nil {
			t.Fatal(err)
		}
		resp, err := client.Do(req)
		if err != nil {
			t.Fatal(err)
		}
		contentType := resp.Header.Get("Content-Type")
		wantContentType := "text/html; charset=utf-8"
		if contentType != wantContentType {
			t.Errorf("Content-Type = %q; want %q", contentType, wantContentType)
		}
	}
	{
		req, err := http.NewRequest("GET", server.URL + "/users/user1/ping", nil)
		if err != nil {
			t.Fatal(err)
		}
		resp, err := client.Do(req)
		if err != nil {
			t.Fatal(err)
		}
		contentType := resp.Header.Get("Content-Type")
		wantContentType := "application/json; charset=utf-8"
		if contentType != wantContentType {
			t.Errorf("Content-Type = %q; want %q", contentType, wantContentType)
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		v := map[string]string{}
		if err := json.Unmarshal(body, &v); err != nil {
			t.Fatal(err)
		}
		if v["result"] != "pong" {
			t.Errorf(`v["result"] = %q; want %q`, v["result"], "pong")
		}
	}
}
<commit_msg>Modified tests<commit_after>package starchat_test

import (
	"."
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"io/ioutil"
	"testing"
)

type testCase struct {
	RequestMethod       string
	RequestPath         string
	RequestBody         []byte // []string?
	ResponseStatusCode  int
	ResponseContentType string
	ResponseJSONBody    map[string]string // ?
}

var testCases = []testCase{
	{
		RequestMethod:       "GET",
		RequestPath:         "/",
		ResponseStatusCode:  http.StatusOK,
		ResponseContentType: "text/html; charset=utf-8",
	},
	{
		RequestMethod:      "GET",
		RequestPath:        "/users/foo/ping",
		ResponseStatusCode: http.StatusOK,
	ResponseJSONBody: map[string]string{
			"result": "pong",
		},
	},
}

func checkTestCase(t *testing.T, rootURL string, tc *testCase) {
	// TODO: Use RequestBody
	url := rootURL + tc.RequestPath
	req, err := http.NewRequest(tc.RequestMethod, url, nil)
	if err != nil {
		t.Fatal(err)
	}
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	{
		statusCode     := resp.StatusCode
		wantStatusCode := tc.ResponseStatusCode
		if statusCode != wantStatusCode {
			t.Errorf("Status Code = %q; want %q", statusCode, wantStatusCode)
		}
	}
	{
		contentType     := resp.Header.Get("Content-Type")
		wantContentType := "application/json; charset=utf-8"
		if tc.ResponseContentType != "" {
			wantContentType = tc.ResponseContentType
		}
		if contentType != wantContentType {
			t.Errorf("Content-Type = %q; want %q", contentType, wantContentType)
		}
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if tc.ResponseJSONBody != nil {
		v := map[string]interface{}{}
		if err := json.Unmarshal(body, &v); err != nil {
			t.Fatal(err)
		}
		//t.Log(v)
		for key, value := range v {
			wantValue := tc.ResponseJSONBody[key]
			if value != wantValue {
				t.Errorf("v[%q] = %q; want %q", key, value, wantValue)
			}
		}
	}
}

func TestHandler(t *testing.T) {
	handler := starchat.NewHandler(nil)
	server := httptest.NewServer(handler)
	defer server.Close()

	for _, testCase := range testCases {
		checkTestCase(t, server.URL, &testCase)
	}
}
<|endoftext|>