<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 * Runtime type representation.
 *
 * The following files know the exact layout of these
 * data structures and must be kept in sync with this file:
 *
 *	../../cmd/gc/reflect.c
 *	../reflect/type.go
 *	type.h
 */

package runtime

import "unsafe"

// The compiler can only construct empty interface values at
// compile time; non-empty interface values get created
// during initialization.  Type is an empty interface
// so that the compiler can lay out references as data.
type Type interface{}

// All types begin with a few common fields needed for
// the interface runtime.
type commonType struct {
	size          uintptr // size in bytes
	hash          uint32  // hash of type; avoids computation in hash tables
	alg           uint8   // algorithm for copy+hash+cmp (../runtime/runtime.h:/AMEM)
	align         uint8   // alignment of variable with this type
	fieldAlign    uint8   // alignment of struct field with this type
	kind          uint8   // enumeration for C
	string        *string // string form; unnecessary  but undeniably useful
	*uncommonType         // (relatively) uncommon fields
}

// Values for commonType.kind.
const (
	kindBool = 1 + iota
	kindInt
	kindInt8
	kindInt16
	kindInt32
	kindInt64
	kindUint
	kindUint8
	kindUint16
	kindUint32
	kindUint64
	kindUintptr
	kindFloat
	kindFloat32
	kindFloat64
	kindArray
	kindChan
	kindFunc
	kindInterface
	kindMap
	kindPtr
	kindSlice
	kindString
	kindStruct
	kindUnsafePointer

	kindNoPointers = 1 << 7 // OR'ed into kind
)

// Method on non-interface type
type method struct {
	name    *string        // name of method
	pkgPath *string        // nil for exported Names; otherwise import path
	mtyp    *Type          // method type (without receiver)
	typ     *Type          // .(*FuncType) underneath (with receiver)
	ifn     unsafe.Pointer // fn used in interface call (one-word receiver)
	tfn     unsafe.Pointer // fn used for normal method call
}

// uncommonType is present only for types with names or methods
// (if T is a named type, the uncommonTypes for T and *T have methods).
// Using a pointer to this struct reduces the overall size required
// to describe an unnamed type with no methods.
type uncommonType struct {
	name    *string  // name of type
	pkgPath *string  // import path; nil for built-in types like int, string
	methods []method // methods associated with type
}

// BoolType represents a boolean type.
type BoolType commonType

// Float32Type represents a float32 type.
type Float32Type commonType

// Float64Type represents a float64 type.
type Float64Type commonType

// FloatType represents a float type.
type FloatType commonType

// Complex64Type represents a complex64 type.
type Complex64Type commonType

// Complex128Type represents a complex128 type.
type Complex128Type commonType

// ComplexType represents a complex type.
type ComplexType commonType

// Int16Type represents an int16 type.
type Int16Type commonType

// Int32Type represents an int32 type.
type Int32Type commonType

// Int64Type represents an int64 type.
type Int64Type commonType

// Int8Type represents an int8 type.
type Int8Type commonType

// IntType represents an int type.
type IntType commonType

// Uint16Type represents a uint16 type.
type Uint16Type commonType

// Uint32Type represents a uint32 type.
type Uint32Type commonType

// Uint64Type represents a uint64 type.
type Uint64Type commonType

// Uint8Type represents a uint8 type.
type Uint8Type commonType

// UintType represents a uint type.
type UintType commonType

// StringType represents a string type.
type StringType commonType

// UintptrType represents a uintptr type.
type UintptrType commonType

// UnsafePointerType represents an unsafe.Pointer type.
type UnsafePointerType commonType

// ArrayType represents a fixed array type.
type ArrayType struct {
	commonType
	elem *Type // array element type
	len  uintptr
}

// SliceType represents a slice type.
type SliceType struct {
	commonType
	elem *Type // slice element type
}

// ChanDir represents a channel type's direction.
type ChanDir int

const (
	RecvDir ChanDir             = 1 << iota // <-chan
	SendDir                                 // chan<-
	BothDir = RecvDir | SendDir             // chan
)

// ChanType represents a channel type.
type ChanType struct {
	commonType
	elem *Type   // channel element type
	dir  uintptr // channel direction (ChanDir)
}

// FuncType represents a function type.
type FuncType struct {
	commonType
	dotdotdot bool    // last input parameter is ...
	in        []*Type // input parameter types
	out       []*Type // output parameter types
}

// Method on interface type
type imethod struct {
	name    *string // name of method
	pkgPath *string // nil for exported Names; otherwise import path
	typ     *Type   // .(*FuncType) underneath
}

// InterfaceType represents an interface type.
type InterfaceType struct {
	commonType
	methods []imethod // sorted by hash
}

// MapType represents a map type.
type MapType struct {
	commonType
	key  *Type // map key type
	elem *Type // map element (value) type
}

// PtrType represents a pointer type.
type PtrType struct {
	commonType
	elem *Type // pointer element (pointed at) type
}

// Struct field
type structField struct {
	name    *string // nil for embedded fields
	pkgPath *string // nil for exported Names; otherwise import path
	typ     *Type   // type of field
	tag     *string // nil if no tag
	offset  uintptr // byte offset of field within struct
}

// StructType represents a struct type.
type StructType struct {
	commonType
	fields []structField // sorted by offset
}

/*
 * Must match iface.c:/Itab and compilers.
 */
type Itable struct {
	Itype  *Type // (*tab.inter).(*InterfaceType) is the interface type
	Type   *Type
	link   *Itable
	bad    int32
	unused int32
	Fn     [100000]uintptr // bigger than we'll ever see
}
<commit_msg>runtime: delete old types<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 * Runtime type representation.
 *
 * The following files know the exact layout of these
 * data structures and must be kept in sync with this file:
 *
 *	../../cmd/gc/reflect.c
 *	../reflect/type.go
 *	type.h
 */

package runtime

import "unsafe"

// The compiler can only construct empty interface values at
// compile time; non-empty interface values get created
// during initialization.  Type is an empty interface
// so that the compiler can lay out references as data.
type Type interface{}

// All types begin with a few common fields needed for
// the interface runtime.
type commonType struct {
	size          uintptr // size in bytes
	hash          uint32  // hash of type; avoids computation in hash tables
	alg           uint8   // algorithm for copy+hash+cmp (../runtime/runtime.h:/AMEM)
	align         uint8   // alignment of variable with this type
	fieldAlign    uint8   // alignment of struct field with this type
	kind          uint8   // enumeration for C
	string        *string // string form; unnecessary  but undeniably useful
	*uncommonType         // (relatively) uncommon fields
}

// Values for commonType.kind.
const (
	kindBool = 1 + iota
	kindInt
	kindInt8
	kindInt16
	kindInt32
	kindInt64
	kindUint
	kindUint8
	kindUint16
	kindUint32
	kindUint64
	kindUintptr
	kindFloat
	kindFloat32
	kindFloat64
	kindArray
	kindChan
	kindFunc
	kindInterface
	kindMap
	kindPtr
	kindSlice
	kindString
	kindStruct
	kindUnsafePointer

	kindNoPointers = 1 << 7 // OR'ed into kind
)

// Method on non-interface type
type method struct {
	name    *string        // name of method
	pkgPath *string        // nil for exported Names; otherwise import path
	mtyp    *Type          // method type (without receiver)
	typ     *Type          // .(*FuncType) underneath (with receiver)
	ifn     unsafe.Pointer // fn used in interface call (one-word receiver)
	tfn     unsafe.Pointer // fn used for normal method call
}

// uncommonType is present only for types with names or methods
// (if T is a named type, the uncommonTypes for T and *T have methods).
// Using a pointer to this struct reduces the overall size required
// to describe an unnamed type with no methods.
type uncommonType struct {
	name    *string  // name of type
	pkgPath *string  // import path; nil for built-in types like int, string
	methods []method // methods associated with type
}

// BoolType represents a boolean type.
type BoolType commonType

// FloatType represents a float type.
type FloatType commonType

// ComplexType represents a complex type.
type ComplexType commonType

// IntType represents an int type.
type IntType commonType

// UintType represents a uint type.
type UintType commonType

// StringType represents a string type.
type StringType commonType

// UintptrType represents a uintptr type.
type UintptrType commonType

// UnsafePointerType represents an unsafe.Pointer type.
type UnsafePointerType commonType

// ArrayType represents a fixed array type.
type ArrayType struct {
	commonType
	elem *Type // array element type
	len  uintptr
}

// SliceType represents a slice type.
type SliceType struct {
	commonType
	elem *Type // slice element type
}

// ChanDir represents a channel type's direction.
type ChanDir int

const (
	RecvDir ChanDir             = 1 << iota // <-chan
	SendDir                                 // chan<-
	BothDir = RecvDir | SendDir             // chan
)

// ChanType represents a channel type.
type ChanType struct {
	commonType
	elem *Type   // channel element type
	dir  uintptr // channel direction (ChanDir)
}

// FuncType represents a function type.
type FuncType struct {
	commonType
	dotdotdot bool    // last input parameter is ...
	in        []*Type // input parameter types
	out       []*Type // output parameter types
}

// Method on interface type
type imethod struct {
	name    *string // name of method
	pkgPath *string // nil for exported Names; otherwise import path
	typ     *Type   // .(*FuncType) underneath
}

// InterfaceType represents an interface type.
type InterfaceType struct {
	commonType
	methods []imethod // sorted by hash
}

// MapType represents a map type.
type MapType struct {
	commonType
	key  *Type // map key type
	elem *Type // map element (value) type
}

// PtrType represents a pointer type.
type PtrType struct {
	commonType
	elem *Type // pointer element (pointed at) type
}

// Struct field
type structField struct {
	name    *string // nil for embedded fields
	pkgPath *string // nil for exported Names; otherwise import path
	typ     *Type   // type of field
	tag     *string // nil if no tag
	offset  uintptr // byte offset of field within struct
}

// StructType represents a struct type.
type StructType struct {
	commonType
	fields []structField // sorted by offset
}

/*
 * Must match iface.c:/Itab and compilers.
 */
type Itable struct {
	Itype  *Type // (*tab.inter).(*InterfaceType) is the interface type
	Type   *Type
	link   *Itable
	bad    int32
	unused int32
	Fn     [100000]uintptr // bigger than we'll ever see
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	// Uncomment when you want to rollback to 2.2.1 version.
	oldwal "k8s.io/kubernetes/third_party/forked/etcd221/wal"
	// Uncomment when you want to rollback to 2.3.7 version.
	// oldwal "k8s.io/kubernetes/third_party/forked/etcd237/wal"

	"github.com/coreos/etcd/etcdserver"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/etcdserver/membership"
	"github.com/coreos/etcd/mvcc/backend"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	"github.com/coreos/go-semver/semver"
	"k8s.io/klog"
)

const rollbackVersion = "2.2.0"

// RollbackV3ToV2 rolls back an etcd 3.0.x data directory to the 2.x.x version specified by rollbackVersion.
func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error {
	dbpath := path.Join(migrateDatadir, "member", "snap", "db")
	klog.Infof("Rolling db file %s back to etcd 2.x", dbpath)

	// etcd3 store backend. We will use it to parse v3 data files and extract information.
	be := backend.NewDefaultBackend(dbpath)
	tx := be.BatchTx()

	// etcd2 store backend. We will use v3 data to update this and then save snapshot to disk.
	st := store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
	expireTime := time.Now().Add(ttl)

	tx.Lock()
	err := tx.UnsafeForEach([]byte("key"), func(k, v []byte) error {
		kv := &mvccpb.KeyValue{}
		kv.Unmarshal(v)

		// This is compact key.
		if !strings.HasPrefix(string(kv.Key), "/") {
			return nil
		}

		ttlOpt := store.TTLOptionSet{}
		if kv.Lease != 0 {
			ttlOpt = store.TTLOptionSet{ExpireTime: expireTime}
		}

		if !isTombstone(k) {
			sk := path.Join(strings.Trim(etcdserver.StoreKeysPrefix, "/"), string(kv.Key))
			_, err := st.Set(sk, false, string(kv.Value), ttlOpt)
			if err != nil {
				return err
			}
		} else {
			st.Delete(string(kv.Key), false, false)
		}

		return nil
	})
	if err != nil {
		return err
	}
	tx.Unlock()

	if err := traverseAndDeleteEmptyDir(st, "/"); err != nil {
		return err
	}

	// rebuild cluster state.
	metadata, hardstate, oldSt, err := rebuild(migrateDatadir)
	if err != nil {
		return err
	}

	// In the following, it's low level logic that saves metadata and data into v2 snapshot.
	backupPath := migrateDatadir + ".rollback.backup"
	if err := os.Rename(migrateDatadir, backupPath); err != nil {
		return err
	}
	if err := os.MkdirAll(path.Join(migrateDatadir, "member", "snap"), 0777); err != nil {
		return err
	}
	walDir := path.Join(migrateDatadir, "member", "wal")

	w, err := oldwal.Create(walDir, metadata)
	if err != nil {
		return err
	}
	err = w.SaveSnapshot(walpb.Snapshot{Index: hardstate.Commit, Term: hardstate.Term})
	w.Close()
	if err != nil {
		return err
	}

	event, err := oldSt.Get(etcdserver.StoreClusterPrefix, true, false)
	if err != nil {
		return err
	}
	// nodes (members info) for ConfState
	nodes := []uint64{}
	traverseMetadata(event.Node, func(n *store.NodeExtern) {
		if n.Key != etcdserver.StoreClusterPrefix {
			// update store metadata
			v := ""
			if !n.Dir {
				v = *n.Value
			}
			if n.Key == path.Join(etcdserver.StoreClusterPrefix, "version") {
				v = rollbackVersion
			}
			if _, err := st.Set(n.Key, n.Dir, v, store.TTLOptionSet{}); err != nil {
				klog.Error(err)
			}

			// update nodes
			fields := strings.Split(n.Key, "/")
			if len(fields) == 4 && fields[2] == "members" {
				nodeID, err := strconv.ParseUint(fields[3], 16, 64)
				if err != nil {
					klog.Fatalf("failed to parse member ID (%s): %v", fields[3], err)
				}
				nodes = append(nodes, nodeID)
			}
		}
	})

	data, err := st.Save()
	if err != nil {
		return err
	}
	raftSnap := raftpb.Snapshot{
		Data: data,
		Metadata: raftpb.SnapshotMetadata{
			Index: hardstate.Commit,
			Term:  hardstate.Term,
			ConfState: raftpb.ConfState{
				Nodes: nodes,
			},
		},
	}
	snapshotter := snap.New(path.Join(migrateDatadir, "member", "snap"))
	if err := snapshotter.SaveSnap(raftSnap); err != nil {
		return err
	}
	klog.Infof("Finished successfully")
	return nil
}

func traverseMetadata(head *store.NodeExtern, handleFunc func(*store.NodeExtern)) {
	q := []*store.NodeExtern{head}

	for len(q) > 0 {
		n := q[0]
		q = q[1:]

		handleFunc(n)

		for _, next := range n.Nodes {
			q = append(q, next)
		}
	}
}

const (
	revBytesLen       = 8 + 1 + 8
	markedRevBytesLen = revBytesLen + 1
	markBytePosition  = markedRevBytesLen - 1

	markTombstone byte = 't'
)

func isTombstone(b []byte) bool {
	return len(b) == markedRevBytesLen && b[markBytePosition] == markTombstone
}

func traverseAndDeleteEmptyDir(st store.Store, dir string) error {
	e, err := st.Get(dir, true, false)
	if err != nil {
		return err
	}
	if len(e.Node.Nodes) == 0 {
		st.Delete(dir, true, true)
		return nil
	}
	for _, node := range e.Node.Nodes {
		if !node.Dir {
			klog.V(2).Infof("key: %s", node.Key[len(etcdserver.StoreKeysPrefix):])
		} else {
			err := traverseAndDeleteEmptyDir(st, node.Key)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func rebuild(datadir string) ([]byte, *raftpb.HardState, store.Store, error) {
	waldir := path.Join(datadir, "member", "wal")
	snapdir := path.Join(datadir, "member", "snap")

	ss := snap.New(snapdir)
	snapshot, err := ss.Load()
	if err != nil && err != snap.ErrNoSnapshot {
		return nil, nil, nil, err
	}

	var walsnap walpb.Snapshot
	if snapshot != nil {
		walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
	}

	w, err := wal.OpenForRead(waldir, walsnap)
	if err != nil {
		return nil, nil, nil, err
	}
	defer w.Close()

	meta, hardstate, ents, err := w.ReadAll()
	if err != nil {
		return nil, nil, nil, err
	}

	st := store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
	if snapshot != nil {
		err := st.Recovery(snapshot.Data)
		if err != nil {
			return nil, nil, nil, err
		}
	}

	cluster := membership.NewCluster("")
	cluster.SetStore(st)
	cluster.Recover(func(*semver.Version) {})

	applier := etcdserver.NewApplierV2(st, cluster)
	for _, ent := range ents {
		if ent.Type == raftpb.EntryConfChange {
			var cc raftpb.ConfChange
			pbutil.MustUnmarshal(&cc, ent.Data)
			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				m := new(membership.Member)
				if err := json.Unmarshal(cc.Context, m); err != nil {
					return nil, nil, nil, err
				}
				cluster.AddMember(m)
			case raftpb.ConfChangeRemoveNode:
				id := types.ID(cc.NodeID)
				cluster.RemoveMember(id)
			case raftpb.ConfChangeUpdateNode:
				m := new(membership.Member)
				if err := json.Unmarshal(cc.Context, m); err != nil {
					return nil, nil, nil, err
				}
				cluster.UpdateRaftAttributes(m.ID, m.RaftAttributes)
			}
			continue
		}

		var raftReq pb.InternalRaftRequest
		if !pbutil.MaybeUnmarshal(&raftReq, ent.Data) { // backward compatible
			var r pb.Request
			pbutil.MustUnmarshal(&r, ent.Data)
			applyRequest(&r, applier)
		} else {
			if raftReq.V2 != nil {
				req := raftReq.V2
				applyRequest(req, applier)
			}
		}
	}

	return meta, &hardstate, st, nil
}

func toTTLOptions(r *pb.Request) store.TTLOptionSet {
	refresh, _ := pbutil.GetBool(r.Refresh)
	ttlOptions := store.TTLOptionSet{Refresh: refresh}
	if r.Expiration != 0 {
		ttlOptions.ExpireTime = time.Unix(0, r.Expiration)
	}
	return ttlOptions
}

func applyRequest(r *pb.Request, applyV2 etcdserver.ApplierV2) {
	// TODO: find a sane way to perform this cast or avoid it in the first place
	reqV2 := &etcdserver.RequestV2{
		ID:               r.ID,
		Method:           r.Method,
		Path:             r.Path,
		Val:              r.Val,
		Dir:              r.Dir,
		PrevValue:        r.PrevValue,
		PrevIndex:        r.PrevIndex,
		PrevExist:        r.PrevExist,
		Expiration:       r.Expiration,
		Wait:             r.Wait,
		Since:            r.Since,
		Recursive:        r.Recursive,
		Sorted:           r.Sorted,
		Quorum:           r.Quorum,
		Time:             r.Time,
		Stream:           r.Stream,
		Refresh:          r.Refresh,
		XXX_unrecognized: r.XXX_unrecognized,
	}
	toTTLOptions(r)
	switch r.Method {
	case "PUT":
		applyV2.Put(reqV2)
	case "DELETE":
		applyV2.Delete(reqV2)
	case "POST", "QGET", "SYNC":
		return
	default:
		klog.Fatal("unknown command")
	}
}
<commit_msg>Use go idiomatic to replace loop<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	// Uncomment when you want to rollback to 2.2.1 version.
	oldwal "k8s.io/kubernetes/third_party/forked/etcd221/wal"
	// Uncomment when you want to rollback to 2.3.7 version.
	// oldwal "k8s.io/kubernetes/third_party/forked/etcd237/wal"

	"github.com/coreos/etcd/etcdserver"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/etcdserver/membership"
	"github.com/coreos/etcd/mvcc/backend"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	"github.com/coreos/go-semver/semver"
	"k8s.io/klog"
)

const rollbackVersion = "2.2.0"

// RollbackV3ToV2 rolls back an etcd 3.0.x data directory to the 2.x.x version specified by rollbackVersion.
func RollbackV3ToV2(migrateDatadir string, ttl time.Duration) error {
	dbpath := path.Join(migrateDatadir, "member", "snap", "db")
	klog.Infof("Rolling db file %s back to etcd 2.x", dbpath)

	// etcd3 store backend. We will use it to parse v3 data files and extract information.
	be := backend.NewDefaultBackend(dbpath)
	tx := be.BatchTx()

	// etcd2 store backend. We will use v3 data to update this and then save snapshot to disk.
	st := store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
	expireTime := time.Now().Add(ttl)

	tx.Lock()
	err := tx.UnsafeForEach([]byte("key"), func(k, v []byte) error {
		kv := &mvccpb.KeyValue{}
		kv.Unmarshal(v)

		// This is compact key.
		if !strings.HasPrefix(string(kv.Key), "/") {
			return nil
		}

		ttlOpt := store.TTLOptionSet{}
		if kv.Lease != 0 {
			ttlOpt = store.TTLOptionSet{ExpireTime: expireTime}
		}

		if !isTombstone(k) {
			sk := path.Join(strings.Trim(etcdserver.StoreKeysPrefix, "/"), string(kv.Key))
			_, err := st.Set(sk, false, string(kv.Value), ttlOpt)
			if err != nil {
				return err
			}
		} else {
			st.Delete(string(kv.Key), false, false)
		}

		return nil
	})
	if err != nil {
		return err
	}
	tx.Unlock()

	if err := traverseAndDeleteEmptyDir(st, "/"); err != nil {
		return err
	}

	// rebuild cluster state.
	metadata, hardstate, oldSt, err := rebuild(migrateDatadir)
	if err != nil {
		return err
	}

	// In the following, it's low level logic that saves metadata and data into v2 snapshot.
	backupPath := migrateDatadir + ".rollback.backup"
	if err := os.Rename(migrateDatadir, backupPath); err != nil {
		return err
	}
	if err := os.MkdirAll(path.Join(migrateDatadir, "member", "snap"), 0777); err != nil {
		return err
	}
	walDir := path.Join(migrateDatadir, "member", "wal")

	w, err := oldwal.Create(walDir, metadata)
	if err != nil {
		return err
	}
	err = w.SaveSnapshot(walpb.Snapshot{Index: hardstate.Commit, Term: hardstate.Term})
	w.Close()
	if err != nil {
		return err
	}

	event, err := oldSt.Get(etcdserver.StoreClusterPrefix, true, false)
	if err != nil {
		return err
	}
	// nodes (members info) for ConfState
	nodes := []uint64{}
	traverseMetadata(event.Node, func(n *store.NodeExtern) {
		if n.Key != etcdserver.StoreClusterPrefix {
			// update store metadata
			v := ""
			if !n.Dir {
				v = *n.Value
			}
			if n.Key == path.Join(etcdserver.StoreClusterPrefix, "version") {
				v = rollbackVersion
			}
			if _, err := st.Set(n.Key, n.Dir, v, store.TTLOptionSet{}); err != nil {
				klog.Error(err)
			}

			// update nodes
			fields := strings.Split(n.Key, "/")
			if len(fields) == 4 && fields[2] == "members" {
				nodeID, err := strconv.ParseUint(fields[3], 16, 64)
				if err != nil {
					klog.Fatalf("failed to parse member ID (%s): %v", fields[3], err)
				}
				nodes = append(nodes, nodeID)
			}
		}
	})

	data, err := st.Save()
	if err != nil {
		return err
	}
	raftSnap := raftpb.Snapshot{
		Data: data,
		Metadata: raftpb.SnapshotMetadata{
			Index: hardstate.Commit,
			Term:  hardstate.Term,
			ConfState: raftpb.ConfState{
				Nodes: nodes,
			},
		},
	}
	snapshotter := snap.New(path.Join(migrateDatadir, "member", "snap"))
	if err := snapshotter.SaveSnap(raftSnap); err != nil {
		return err
	}
	klog.Infof("Finished successfully")
	return nil
}

func traverseMetadata(head *store.NodeExtern, handleFunc func(*store.NodeExtern)) {
	q := []*store.NodeExtern{head}

	for len(q) > 0 {
		n := q[0]
		q = q[1:]

		handleFunc(n)

		q = append(q, n.Nodes...)
	}
}

const (
	revBytesLen       = 8 + 1 + 8
	markedRevBytesLen = revBytesLen + 1
	markBytePosition  = markedRevBytesLen - 1

	markTombstone byte = 't'
)

func isTombstone(b []byte) bool {
	return len(b) == markedRevBytesLen && b[markBytePosition] == markTombstone
}

func traverseAndDeleteEmptyDir(st store.Store, dir string) error {
	e, err := st.Get(dir, true, false)
	if err != nil {
		return err
	}
	if len(e.Node.Nodes) == 0 {
		st.Delete(dir, true, true)
		return nil
	}
	for _, node := range e.Node.Nodes {
		if !node.Dir {
			klog.V(2).Infof("key: %s", node.Key[len(etcdserver.StoreKeysPrefix):])
		} else {
			err := traverseAndDeleteEmptyDir(st, node.Key)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func rebuild(datadir string) ([]byte, *raftpb.HardState, store.Store, error) {
	waldir := path.Join(datadir, "member", "wal")
	snapdir := path.Join(datadir, "member", "snap")

	ss := snap.New(snapdir)
	snapshot, err := ss.Load()
	if err != nil && err != snap.ErrNoSnapshot {
		return nil, nil, nil, err
	}

	var walsnap walpb.Snapshot
	if snapshot != nil {
		walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
	}

	w, err := wal.OpenForRead(waldir, walsnap)
	if err != nil {
		return nil, nil, nil, err
	}
	defer w.Close()

	meta, hardstate, ents, err := w.ReadAll()
	if err != nil {
		return nil, nil, nil, err
	}

	st := store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
	if snapshot != nil {
		err := st.Recovery(snapshot.Data)
		if err != nil {
			return nil, nil, nil, err
		}
	}

	cluster := membership.NewCluster("")
	cluster.SetStore(st)
	cluster.Recover(func(*semver.Version) {})

	applier := etcdserver.NewApplierV2(st, cluster)
	for _, ent := range ents {
		if ent.Type == raftpb.EntryConfChange {
			var cc raftpb.ConfChange
			pbutil.MustUnmarshal(&cc, ent.Data)
			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				m := new(membership.Member)
				if err := json.Unmarshal(cc.Context, m); err != nil {
					return nil, nil, nil, err
				}
				cluster.AddMember(m)
			case raftpb.ConfChangeRemoveNode:
				id := types.ID(cc.NodeID)
				cluster.RemoveMember(id)
			case raftpb.ConfChangeUpdateNode:
				m := new(membership.Member)
				if err := json.Unmarshal(cc.Context, m); err != nil {
					return nil, nil, nil, err
				}
				cluster.UpdateRaftAttributes(m.ID, m.RaftAttributes)
			}
			continue
		}

		var raftReq pb.InternalRaftRequest
		if !pbutil.MaybeUnmarshal(&raftReq, ent.Data) { // backward compatible
			var r pb.Request
			pbutil.MustUnmarshal(&r, ent.Data)
			applyRequest(&r, applier)
		} else {
			if raftReq.V2 != nil {
				req := raftReq.V2
				applyRequest(req, applier)
			}
		}
	}

	return meta, &hardstate, st, nil
}

func toTTLOptions(r *pb.Request) store.TTLOptionSet {
	refresh, _ := pbutil.GetBool(r.Refresh)
	ttlOptions := store.TTLOptionSet{Refresh: refresh}
	if r.Expiration != 0 {
		ttlOptions.ExpireTime = time.Unix(0, r.Expiration)
	}
	return ttlOptions
}

func applyRequest(r *pb.Request, applyV2 etcdserver.ApplierV2) {
	// TODO: find a sane way to perform this cast or avoid it in the first place
	reqV2 := &etcdserver.RequestV2{
		ID:               r.ID,
		Method:           r.Method,
		Path:             r.Path,
		Val:              r.Val,
		Dir:              r.Dir,
		PrevValue:        r.PrevValue,
		PrevIndex:        r.PrevIndex,
		PrevExist:        r.PrevExist,
		Expiration:       r.Expiration,
		Wait:             r.Wait,
		Since:            r.Since,
		Recursive:        r.Recursive,
		Sorted:           r.Sorted,
		Quorum:           r.Quorum,
		Time:             r.Time,
		Stream:           r.Stream,
		Refresh:          r.Refresh,
		XXX_unrecognized: r.XXX_unrecognized,
	}
	toTTLOptions(r)
	switch r.Method {
	case "PUT":
		applyV2.Put(reqV2)
	case "DELETE":
		applyV2.Delete(reqV2)
	case "POST", "QGET", "SYNC":
		return
	default:
		klog.Fatal("unknown command")
	}
}
<|endoftext|>


<commit_before>// Package ldpreload-label-injector contains tool for injecting ldpreload-specific labels into kubernetes yaml files
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var (
	// command line flags
	outputFile    = flag.String("o", "", "Output file override.")
	help          = flag.Bool("h", false, "Switch to show help")
	useDebugLabel = flag.Bool("d", false, "Switch to used debug ldpreload label")
	proxyName     = flag.String("p", "", "Name of proxy container that should be used.")
)

// injectParams pass important information from command line flags to injector
type injectParams struct {
	useDebugLabel bool
	proxyName     string
}

// main is the main method for ldpreload label injector
// This tool parses block structure of yaml file and currently support only these type of structures:
// 1. basic yaml blocks, i.e.
// ....root:
//....   block1:
//........block2:
// 2.the compact nested mapping, i.e.
//.... mapping:
//.... - item1attribute1:
//....   item1attribute2:
//.... - item2attribute1:
//....   item2attribute2:
func main() {
	// handle initial tasks and simple cases
	flag.Parse() //can't be in init()
	if *help || flag.NArg() == 0 {
		printHelp()
		return
	}

	// resolve input/output files arguments, info passing to injector and start file processing
	inputFile := flag.Arg(0)
	if len(*outputFile) == 0 {
		outputFile = &inputFile
	}
	injectParams := injectParams{
		useDebugLabel: *useDebugLabel,
		proxyName:     *proxyName,
	}
	if err := processFile(inputFile, *outputFile, injectParams); err != nil {
		panic(fmt.Errorf("Can't process file %v : %v ", inputFile, err))
	}
}

// printHelp prints properly structured help for command line environment
func printHelp() {
	fmt.Print(`ldpreload-label-injector injects ldpreload labels to kubernetes yaml files.
Usage:
  ldpreload-label-injector [input file]

Flags:
  -o [output file]  Sets output for modified kubernetes yaml file. This overrides default behaviour that takes input file as output file and modifies input file in-place.
  -d                Adds ldpreload debug label to yaml kubernetes files
  -p                Sets the name of container that should be used as proxy. If not set, proxy is not used.
  -h                Prints this help
`)
}

// processFile takes content of input file, injects that the ldpreload labels and outputs it as output file
func processFile(inputFile string, outputFile string, params injectParams) (err error) {
	fmt.Printf("Processing file %v (to output %v)... ", inputFile, outputFile)
	content, err := ioutil.ReadFile(inputFile)
	if err != nil {
		return
	}
	converted, err := inject(string(content), params)
	if err != nil {
		return
	}

	err = ioutil.WriteFile(outputFile, []byte(converted), fileMode(inputFile))
	fmt.Println("Done")
	return
}

// fileMode computes most appropriate file permissions for output file
func fileMode(inputFile string) os.FileMode {
	fileMode := os.FileMode(0644) //default permissions
	inputFileInfo, err := os.Stat(inputFile)
	if err == nil {
		fileMode = inputFileInfo.Mode()
	} else {
		fmt.Printf("\n  Can't detect file permissions for input file. Using permissions %v for output file. Error: %v \n", fileMode, err)
	}
	return fileMode
}
<commit_msg>added reading from standard input and writing to standard output changed default output and when help is showed<commit_after>// Package ldpreload-label-injector contains tool for injecting ldpreload-specific labels into kubernetes yaml files
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var (
	// command line flags
	inputFile     = flag.String("f", "", "Input file")
	outputFile    = flag.String("o", "", "Output file")
	help          = flag.Bool("h", false, "Switch to show help")
	useDebugLabel = flag.Bool("d", false, "Switch to used debug ldpreload label")
	proxyName     = flag.String("p", "", "Name of proxy container that should be used")
)

// injectParams pass important information from command line flags to injector
type injectParams struct {
	useDebugLabel bool
	proxyName     string
}

const helpContent = `ldpreload-label-injector injects ldpreload labels to kubernetes yaml files.
Usage:
  ldpreload-label-injector [input file]

Flags:
  -o [output file]  Sets output for modified kubernetes yaml file. This overrides default behaviour that takes input file as output file and modifies input file in-place.
  -d                Adds ldpreload debug label to yaml kubernetes files
  -p                Sets the name of container that should be used as proxy. If not set, proxy is not used.
  -h                Prints this help
`

// main is the main method for ldpreload label injector
// This tool parses block structure of yaml file and currently support only these type of structures:
// 1. basic yaml blocks, i.e.
// ....root:
//....   block1:
//........block2:
// 2.the compact nested mapping, i.e.
//.... mapping:
//.... - item1attribute1:
//....   item1attribute2:
//.... - item2attribute1:
//....   item2attribute2:
func main() {
	// handle initial tasks and simple cases
	flag.Parse() //can't be in init()
	if *help {
		fmt.Print(helpContent)
		return
	}
	injectParams := injectParams{
		useDebugLabel: *useDebugLabel,
		proxyName:     *proxyName,
	}

	// transform input to output
	content, err := readInput()
	if err != nil {
		panic(fmt.Errorf("Can't read input: %v ", err))
	}
	injectedContent, err := inject(string(content), injectParams)
	if err != nil {
		panic(fmt.Errorf("Can't inject ldpreload labels: %v ", err))
	}
	err = writeOutput(injectedContent)
	if err != nil {
		panic(fmt.Errorf("Can't write output: %v ", err))
	}
}

func readInput() (content string, err error) {
	if *inputFile == "" {
		err = fmt.Errorf("Input is not specified, please use -f parameter.\n" + helpContent)
		return
	}
	var contentBytes []byte
	if *inputFile == "-" {
		contentBytes, err = ioutil.ReadAll(os.Stdin)
	} else {
		contentBytes, err = ioutil.ReadFile(*inputFile)
	}
	if err != nil {
		return
	}
	content = string(contentBytes)
	return
}

func writeOutput(content string) error {
	if *outputFile == "" {
		fmt.Print(content)
		return nil
	}
	return ioutil.WriteFile(*outputFile, []byte(content), fileMode(*inputFile))
}

// fileMode computes most appropriate file permissions for output file
func fileMode(input string) os.FileMode {
	fileMode := os.FileMode(0644) //default permissions
	if input == "" || input == "-" {
		return fileMode //input is not file -> can't detect file permissions from input -> using default permissions
	}
	inputFileInfo, err := os.Stat(input)
	if err == nil {
		fileMode = inputFileInfo.Mode()
	} else {
		fmt.Printf("\n  Can't detect file permissions for input file. Using permissions %v for output file. Error: %v \n", fileMode, err)
	}
	return fileMode
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package sync

import (
	"github.com/unknwon/com"
)

// UniqueQueue is a queue which guarantees only one instance of same
// identity is in the line. Instances with same identity will be
// discarded if there is already one in the line.
//
// This queue is particularly useful for preventing duplicated task
// of same purpose.
type UniqueQueue struct {
	table  *StatusTable
	queue  chan string
	closed chan struct{}
}

// NewUniqueQueue initializes and returns a new UniqueQueue object.
func NewUniqueQueue(queueLength int) *UniqueQueue {
	if queueLength <= 0 {
		queueLength = 100
	}

	return &UniqueQueue{
		table:  NewStatusTable(),
		queue:  make(chan string, queueLength),
		closed: make(chan struct{}),
	}
}

// Close closes this queue
func (q *UniqueQueue) Close() {
	select {
	case <-q.closed:
	default:
		q.table.lock.Lock()
		select {
		case <-q.closed:
		default:
			close(q.closed)
		}
		q.table.lock.Unlock()
	}
}

// IsClosed returns a channel that is closed when this Queue is closed
func (q *UniqueQueue) IsClosed() <-chan struct{} {
	return q.closed
}

// IDs returns the current ids in the pool
func (q *UniqueQueue) IDs() []interface{} {
	q.table.lock.Lock()
	defer q.table.lock.Unlock()
	ids := make([]interface{}, 0, len(q.table.pool))
	for id := range q.table.pool {
		ids = append(ids, id)
	}
	return ids
}

// Queue returns channel of queue for retrieving instances.
func (q *UniqueQueue) Queue() <-chan string {
	return q.queue
}

// Exist returns true if there is an instance with given identity
// exists in the queue.
func (q *UniqueQueue) Exist(id interface{}) bool {
	return q.table.IsRunning(com.ToStr(id))
}

// AddFunc adds new instance to the queue with a custom runnable function,
// the queue is blocked until the function exits.
func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
	idStr := com.ToStr(id)
	q.table.lock.Lock()
	if _, ok := q.table.pool[idStr]; ok {
		return
	}
	q.table.pool[idStr] = struct{}{}
	if fn != nil {
		fn()
	}
	q.table.lock.Unlock()
	select {
	case <-q.closed:
		return
	case q.queue <- idStr:
		return
	}
}

// Add adds new instance to the queue.
func (q *UniqueQueue) Add(id interface{}) {
	q.AddFunc(id, nil)
}

// Remove removes instance from the queue.
func (q *UniqueQueue) Remove(id interface{}) {
	q.table.Stop(com.ToStr(id))
}
<commit_msg>Fix missing unlock in uniquequeue (#9790)<commit_after>// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package sync

import (
	"github.com/unknwon/com"
)

// UniqueQueue is a queue which guarantees only one instance of same
// identity is in the line. Instances with same identity will be
// discarded if there is already one in the line.
//
// This queue is particularly useful for preventing duplicated task
// of same purpose.
type UniqueQueue struct {
	table  *StatusTable
	queue  chan string
	closed chan struct{}
}

// NewUniqueQueue initializes and returns a new UniqueQueue object.
func NewUniqueQueue(queueLength int) *UniqueQueue {
	if queueLength <= 0 {
		queueLength = 100
	}

	return &UniqueQueue{
		table:  NewStatusTable(),
		queue:  make(chan string, queueLength),
		closed: make(chan struct{}),
	}
}

// Close closes this queue
func (q *UniqueQueue) Close() {
	select {
	case <-q.closed:
	default:
		q.table.lock.Lock()
		select {
		case <-q.closed:
		default:
			close(q.closed)
		}
		q.table.lock.Unlock()
	}
}

// IsClosed returns a channel that is closed when this Queue is closed
func (q *UniqueQueue) IsClosed() <-chan struct{} {
	return q.closed
}

// IDs returns the current ids in the pool
func (q *UniqueQueue) IDs() []interface{} {
	q.table.lock.Lock()
	defer q.table.lock.Unlock()
	ids := make([]interface{}, 0, len(q.table.pool))
	for id := range q.table.pool {
		ids = append(ids, id)
	}
	return ids
}

// Queue returns channel of queue for retrieving instances.
func (q *UniqueQueue) Queue() <-chan string {
	return q.queue
}

// Exist returns true if there is an instance with given identity
// exists in the queue.
func (q *UniqueQueue) Exist(id interface{}) bool {
	return q.table.IsRunning(com.ToStr(id))
}

// AddFunc adds new instance to the queue with a custom runnable function,
// the queue is blocked until the function exits.
func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
	idStr := com.ToStr(id)
	q.table.lock.Lock()
	if _, ok := q.table.pool[idStr]; ok {
		q.table.lock.Unlock()
		return
	}
	q.table.pool[idStr] = struct{}{}
	if fn != nil {
		fn()
	}
	q.table.lock.Unlock()
	select {
	case <-q.closed:
		return
	case q.queue <- idStr:
		return
	}
}

// Add adds new instance to the queue.
func (q *UniqueQueue) Add(id interface{}) {
	q.AddFunc(id, nil)
}

// Remove removes instance from the queue.
func (q *UniqueQueue) Remove(id interface{}) {
	q.table.Stop(com.ToStr(id))
}
<|endoftext|>


<commit_before>package engines

import (
	"fmt"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Google search engine
type Google struct {
	Host, URI string
	SSL       bool
}

// CreateGoogle is the func to create the search engine with default values
func CreateGoogle(ssl bool) Google {
	var schema string
	if ssl {
		schema = "https"
	} else {
		schema = "http"
	}

	return Google{
		schema + "://www.google.com",
		"search?q=site:%s+%s",
		ssl,
	}
}

// GetLinks for a `query` and `site`.
// `site`: What site you want to get links for
// `query`: Search query you want to search on `site` for.
func (google Google) GetLinks(query, site string) (links []string, err error) {
	doc, err := goquery.NewDocument(google.createSearchURL(query, site))
	if err != nil {
		return
	}

	filterLink := func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			links = append(links, google.Host+href)
		}
	}

	doc.Find("a.l").Each(filterLink)
	if len(links) == 0 {
		doc.Find(".r a").Each(filterLink)
	}

	return
}

// createSearchURL is used to create the google search URL.
// Example: `https://www.google.de/search?q=site:stackoverflow.com+windows+get+date+command+line`
func (google Google) createSearchURL(query, site string) string {
	uri := fmt.Sprintf(google.URI, site, url.QueryEscape(query))
	return google.Host + "/" + uri
}
<commit_msg>The ssl flag is not used<commit_after>package engines

import (
	"fmt"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Google search engine
type Google struct {
	Host, URI string
}

// CreateGoogle is the func to create the search engine with default values
func CreateGoogle(ssl bool) Google {
	var schema string
	if ssl {
		schema = "https"
	} else {
		schema = "http"
	}

	return Google{
		schema + "://www.google.com",
		"search?q=site:%s+%s",
	}
}

// GetLinks for a `query` and `site`.
// `site`: What site you want to get links for
// `query`: Search query you want to search on `site` for.
func (google Google) GetLinks(query, site string) (links []string, err error) {
	doc, err := goquery.NewDocument(google.createSearchURL(query, site))
	if err != nil {
		return
	}

	filterLink := func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			links = append(links, google.Host+href)
		}
	}

	doc.Find("a.l").Each(filterLink)
	if len(links) == 0 {
		doc.Find(".r a").Each(filterLink)
	}

	return
}

// createSearchURL is used to create the google search URL.
// Example: `https://www.google.com/search?q=site:stackoverflow.com+windows+get+date+command+line`
func (google Google) createSearchURL(query, site string) string {
	uri := fmt.Sprintf(google.URI, site, url.QueryEscape(query))
	return google.Host + "/" + uri
}
<|endoftext|>


<commit_before>package carbonitexplugin

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/iopred/bruxism"
)

type carbonitexPlugin struct {
	bruxism.SimplePlugin
	key string
}

func (p *carbonitexPlugin) carbonitexPluginLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	go p.Run(bot, service)
	return nil
}

func (p *carbonitexPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	for {
		<-time.After(5 * time.Minute)

		http.PostForm("https://www.carbonitex.net/discord/data/botdata.php", url.Values{"key": {p.key}, "servercount": {fmt.Sprintf("%d", service.ChannelCount())}})

		<-time.After(55 * time.Minute)
	}

}

// New will create a new carbonitex plugin.
// This plugin reports the server count to the carbonitex service.
func New(key string) bruxism.Plugin {
	p := &carbonitexPlugin{
		SimplePlugin: *bruxism.NewSimplePlugin("Carbonitex"),
		key:          key,
	}
	p.LoadFunc = p.carbonitexPluginLoadFunc
	return p
}
<commit_msg>Update Carbonitex plugin.<commit_after>package carbonitexplugin

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/iopred/bruxism"
)

type carbonitexPlugin struct {
	bruxism.SimplePlugin
	key string
}

func (p *carbonitexPlugin) carbonitexPluginLoadFunc(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	go p.Run(bot, service)
	return nil
}

func (p *carbonitexPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	for {
		<-time.After(5 * time.Minute)

		resp, err := http.PostForm("https://www.carbonitex.net/discord/data/botdata.php", url.Values{"key": {p.key}, "servercount": {fmt.Sprintf("%d", service.ChannelCount())}})

		if err == nil {
			htmlData, err := ioutil.ReadAll(resp.Body)
			if err == nil {
				resp.Body.Close()
			}
		}

		<-time.After(55 * time.Minute)
	}

}

// New will create a new carbonitex plugin.
// This plugin reports the server count to the carbonitex service.
func New(key string) bruxism.Plugin {
	p := &carbonitexPlugin{
		SimplePlugin: *bruxism.NewSimplePlugin("Carbonitex"),
		key:          key,
	}
	p.LoadFunc = p.carbonitexPluginLoadFunc
	return p
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	"log"

	"github.com/grokify/gotilla/image/imageutil"
)

func main() {
	urls := []string{
		"https://example.com/img1.jpg",
		"https://example.com/img2.jpg",
		"https://example.com/img3.jpg",
		"https://example.com/img4.jpg"}

	writeManual := true
	writeManualIntermediate := true

	img12, err := imageutil.MergeXSameYRead([]string{urls[0], urls[1]}, true)
	if err != nil {
		log.Fatal(err)
	}
	if writeManual && writeManualIntermediate {
		outfile12 := "_img12.jpg"
		err = imageutil.WriteFileJPEG(outfile12, img12, imageutil.JPEGQualityMax)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("WROTE [%v]\n", outfile12)
	}
	img34, err := imageutil.MergeXSameYRead([]string{urls[2], urls[3]}, true)
	if err != nil {
		log.Fatal(err)
	}

	if writeManual && writeManualIntermediate {
		outfile34 := "_img34.jpg"
		err = imageutil.WriteFileJPEG(outfile34, img34, imageutil.JPEGQualityMax)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("WROTE [%v]\n", outfile34)
	}
	if writeManual {
		img4 := imageutil.MergeYSameX([]image.Image{img12, img34}, true)
		outfile := "_img4_manual.jpg"
		err = imageutil.WriteFileJPEG(outfile, img4, imageutil.JPEGQualityMax)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("WROTE [%v]\n", outfile)
	}

	outfile4 := "_img4_auto.jpg"

	img4a, err := imageutil.MatrixMergeRead(
		[][]string{
			[]string{urls[0]},
			[]string{urls[0], urls[1]},
			[]string{urls[2], urls[3], urls[2]},
		},
		true, true)

	err = imageutil.WriteFileJPEG(outfile4, img4a, imageutil.JPEGQualityMax)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("WROTE [%v]\n", outfile4)

	fmt.Println("DONE")
}
<commit_msg>fix: imageutil: update example<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/grokify/gotilla/image/imageutil"
)

func main() {
	urlsmatrix := [][]string{
		[]string{"https://raw.githubusercontent.com/grokify/gotilla/master/image/imageutil/read_testdata/gopher_appengine_color.jpg"},
		[]string{
			"https://raw.githubusercontent.com/grokify/gotilla/master/image/imageutil/read_testdata/gopher_color.jpg",
			"https://raw.githubusercontent.com/grokify/gotilla/master/image/imageutil/read_testdata/gopher_color.jpg"},
	}

	outfile := "_merged.jpg"

	matrix, err := imageutil.MatrixRead(urlsmatrix)
	if err != nil {
		log.Fatal(err)
	}
	merged := matrix.Merge(true, true)

	err = imageutil.WriteFileJPEG(outfile, merged, imageutil.JPEGQualityMax)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("WROTE [%v]\n", outfile)

	fmt.Println("DONE")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vincent-petithory/kraken/admin"
	"github.com/vincent-petithory/kraken/admin/client"
	"github.com/vincent-petithory/kraken/fileserver"
)

// Environnement var for the url on which the admin service is accessible.
const envKrakenURL = "KRAKEN_URL"

func loadKrakenURL() (*url.URL, error) {
	rawurl := os.Getenv(envKrakenURL)
	if rawurl == "" {
		rawurl = "http://localhost:4214"
	}
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if !u.IsAbs() {
		return nil, fmt.Errorf("%v is not an absolute URL", u)
	}
	if u.Path != "" {
		return nil, fmt.Errorf("%v has a path, which is not allowed", u)
	}
	return u, nil
}

type flagSet struct {
	ServerAddBind    string
	MountTarget      string
	FileServerType   string
	FileServerParams string
}

func clientCmd(client *client.Client, flags *flagSet, runFn func(*client.Client, *flagSet, *cobra.Command, []string)) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		runFn(client, flags, cmd, args)
	}
}

func main() {
	log.SetFlags(0)
	krakenURL, err := loadKrakenURL()
	if err != nil {
		log.Fatal(err)
	}
	c := client.New(krakenURL)

	flags := &flagSet{}

	serversGetCmd := &cobra.Command{
		Use:   "ls",
		Short: "List the available servers",
		Long:  "List the available servers",
		Run:   clientCmd(c, flags, serverList),
	}

	serverAddCmd := &cobra.Command{
		Use:   "add [PORT]",
		Short: "Add a new server",
		Long:  "Add a new server listening on PORT, or a random port if not provided",
		Run:   clientCmd(c, flags, serverAdd),
	}
	serverAddCmd.Flags().StringVarP(&flags.ServerAddBind, "bind", "b", "", "Address to bind to, defaults to not bind")

	serverRmCmd := &cobra.Command{
		Use:   "rm PORT",
		Short: "Remove a server",
		Long:  "Remove a server listening on PORT",
		Run:   clientCmd(c, flags, serverRm),
	}

	serverClearCmd := &cobra.Command{
		Use:   "clear",
		Short: "Remove all servers",
		Long:  "Remove all available servers",
		Run:   clientCmd(c, flags, serverRmAll),
	}

	mountsGetCmd := &cobra.Command{
		Use:   "lsmount PORT",
		Short: "List the mounts of a server",
		Long:  "List the mounts of a the server listening on PORT",
		Run:   clientCmd(c, flags, mountList),
	}

	mountAddCmd := &cobra.Command{
		Use:   "mount PORT SOURCE",
		Short: "Mount a directory on a server",
		Long: `Mount the SOURCE directory on the server listening on PORT.
By default, SOURCE is mounted on /$(basename SOURCE)`,
		Run: clientCmd(c, flags, mountAdd),
	}
	mountAddCmd.Flags().StringVarP(&flags.MountTarget, "target", "t", "", "Alternate mount target; it must start with / and not end with /")
	mountAddCmd.Flags().StringVarP(&flags.FileServerType, "fs", "f", "default", "File server type to use for this mount point")
	mountAddCmd.Flags().StringVarP(&flags.FileServerParams, "fsp", "p", "{}", "File server params; they must be specified as a valid JSON object.")

	mountRmCmd := &cobra.Command{
		Use:   "umount PORT MOUNT_ID",
		Short: "Unmount a directory on a server",
		Long:  "Removes the mount point MOUNT_ID, on the server listening on PORT",
		Run:   clientCmd(c, flags, mountRm),
	}

	fileServersGetCmd := &cobra.Command{
		Use:   "fileservers",
		Short: "Lists the available file servers",
		Long:  "Lists the available file servers",
		Run:   clientCmd(c, flags, fileServerList),
	}

	rootCmd := &cobra.Command{
		Use: "krakenctl",
	}
	rootCmd.AddCommand(
		// server commands
		serversGetCmd,
		serverAddCmd,
		serverRmCmd,
		serverClearCmd,
		// mount commands
		mountsGetCmd,
		mountAddCmd,
		mountRmCmd,
		// fileserver commands
		fileServersGetCmd,
	)
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func serverList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	srvs, err := client.GetServers()
	if err != nil {
		log.Fatal(err)
	}

	for _, srv := range srvs {
		addr := net.JoinHostPort(srv.BindAddress, strconv.Itoa(int(srv.Port)))
		fmt.Print(addr)
		if len(srv.Mounts) == 0 {
			fmt.Println(" (no mounts)")
			continue
		}
		fmt.Println()
		for _, mount := range srv.Mounts {
			fmt.Printf("  * %s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
		}
		fmt.Println()
	}
}

func serverAdd(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 1 {
		cmd.Usage()
		return
	}
	var (
		srv *admin.Server
		err error
	)
	if len(args) == 0 {
		srv, err = client.AddServerWithRandomPort(admin.CreateServerRequest{BindAddress: flags.ServerAddBind})
	} else {
		var port int
		port, err = strconv.Atoi(args[0])
		if err != nil {
			log.Fatalf("error parsing port: %v", err)
		}
		srv, err = client.AddServer(uint16(port), admin.CreateServerRequest{BindAddress: flags.ServerAddBind})
	}
	if err != nil {
		log.Fatal(err)
	}
	addr := net.JoinHostPort(srv.BindAddress, strconv.Itoa(int(srv.Port)))
	fmt.Printf("server available on %s\n", addr)
}

func serverRm(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) == 0 || len(args) > 1 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	if err := client.RemoveServer(uint16(port)); err != nil {
		log.Fatal(err)
	}
}

func serverRmAll(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	if err := client.RemoveAllServers(); err != nil {
		log.Fatal(err)
	}
}

func fileServerList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	fsrvs, err := client.GetFileServers()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(strings.Join(fsrvs, ", "))
}

func mountList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) == 0 || len(args) > 1 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	mounts, err := client.GetMounts(uint16(port))
	if err != nil {
		log.Fatal(err)
	}

	for _, mount := range mounts {
		fmt.Printf("%s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
	}
}

func mountAdd(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) != 2 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	source, err := filepath.Abs(args[1])
	if err != nil {
		log.Fatal(err)
	}

	target := "/" + filepath.Base(source)
	if flags.MountTarget != "" {
		target = flags.MountTarget
	}
	var fsParams fileserver.Params
	if err := json.Unmarshal([]byte(flags.FileServerParams), &fsParams); err != nil {
		log.Fatal(err)
	}

	mount, err := client.AddMount(uint16(port), admin.CreateServerMountRequest{
		target,
		source,
		flags.FileServerType,
		fsParams,
	})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
}

func mountRm(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) == 0 || len(args) > 2 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	mountID := args[1]
	if err := client.RemoveMount(uint16(port), mountID); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>krakenctl: fix args len checks<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vincent-petithory/kraken/admin"
	"github.com/vincent-petithory/kraken/admin/client"
	"github.com/vincent-petithory/kraken/fileserver"
)

// Environnement var for the url on which the admin service is accessible.
const envKrakenURL = "KRAKEN_URL"

func loadKrakenURL() (*url.URL, error) {
	rawurl := os.Getenv(envKrakenURL)
	if rawurl == "" {
		rawurl = "http://localhost:4214"
	}
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if !u.IsAbs() {
		return nil, fmt.Errorf("%v is not an absolute URL", u)
	}
	if u.Path != "" {
		return nil, fmt.Errorf("%v has a path, which is not allowed", u)
	}
	return u, nil
}

type flagSet struct {
	ServerAddBind    string
	MountTarget      string
	FileServerType   string
	FileServerParams string
}

func clientCmd(client *client.Client, flags *flagSet, runFn func(*client.Client, *flagSet, *cobra.Command, []string)) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		runFn(client, flags, cmd, args)
	}
}

func main() {
	log.SetFlags(0)
	krakenURL, err := loadKrakenURL()
	if err != nil {
		log.Fatal(err)
	}
	c := client.New(krakenURL)

	flags := &flagSet{}

	serversGetCmd := &cobra.Command{
		Use:   "ls",
		Short: "List the available servers",
		Long:  "List the available servers",
		Run:   clientCmd(c, flags, serverList),
	}

	serverAddCmd := &cobra.Command{
		Use:   "add [PORT]",
		Short: "Add a new server",
		Long:  "Add a new server listening on PORT, or a random port if not provided",
		Run:   clientCmd(c, flags, serverAdd),
	}
	serverAddCmd.Flags().StringVarP(&flags.ServerAddBind, "bind", "b", "", "Address to bind to, defaults to not bind")

	serverRmCmd := &cobra.Command{
		Use:   "rm PORT",
		Short: "Remove a server",
		Long:  "Remove a server listening on PORT",
		Run:   clientCmd(c, flags, serverRm),
	}

	serverClearCmd := &cobra.Command{
		Use:   "clear",
		Short: "Remove all servers",
		Long:  "Remove all available servers",
		Run:   clientCmd(c, flags, serverRmAll),
	}

	mountsGetCmd := &cobra.Command{
		Use:   "lsmount PORT",
		Short: "List the mounts of a server",
		Long:  "List the mounts of a the server listening on PORT",
		Run:   clientCmd(c, flags, mountList),
	}

	mountAddCmd := &cobra.Command{
		Use:   "mount PORT SOURCE",
		Short: "Mount a directory on a server",
		Long: `Mount the SOURCE directory on the server listening on PORT.
By default, SOURCE is mounted on /$(basename SOURCE)`,
		Run: clientCmd(c, flags, mountAdd),
	}
	mountAddCmd.Flags().StringVarP(&flags.MountTarget, "target", "t", "", "Alternate mount target; it must start with / and not end with /")
	mountAddCmd.Flags().StringVarP(&flags.FileServerType, "fs", "f", "default", "File server type to use for this mount point")
	mountAddCmd.Flags().StringVarP(&flags.FileServerParams, "fsp", "p", "{}", "File server params; they must be specified as a valid JSON object.")

	mountRmCmd := &cobra.Command{
		Use:   "umount PORT MOUNT_ID",
		Short: "Unmount a directory on a server",
		Long:  "Removes the mount point MOUNT_ID, on the server listening on PORT",
		Run:   clientCmd(c, flags, mountRm),
	}

	fileServersGetCmd := &cobra.Command{
		Use:   "fileservers",
		Short: "Lists the available file servers",
		Long:  "Lists the available file servers",
		Run:   clientCmd(c, flags, fileServerList),
	}

	rootCmd := &cobra.Command{
		Use: "krakenctl",
	}
	rootCmd.AddCommand(
		// server commands
		serversGetCmd,
		serverAddCmd,
		serverRmCmd,
		serverClearCmd,
		// mount commands
		mountsGetCmd,
		mountAddCmd,
		mountRmCmd,
		// fileserver commands
		fileServersGetCmd,
	)
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func serverList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	srvs, err := client.GetServers()
	if err != nil {
		log.Fatal(err)
	}

	for _, srv := range srvs {
		addr := net.JoinHostPort(srv.BindAddress, strconv.Itoa(int(srv.Port)))
		fmt.Print(addr)
		if len(srv.Mounts) == 0 {
			fmt.Println(" (no mounts)")
			continue
		}
		fmt.Println()
		for _, mount := range srv.Mounts {
			fmt.Printf("  * %s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
		}
		fmt.Println()
	}
}

func serverAdd(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 1 {
		cmd.Usage()
		return
	}
	var (
		srv *admin.Server
		err error
	)
	if len(args) == 0 {
		srv, err = client.AddServerWithRandomPort(admin.CreateServerRequest{BindAddress: flags.ServerAddBind})
	} else {
		var port int
		port, err = strconv.Atoi(args[0])
		if err != nil {
			log.Fatalf("error parsing port: %v", err)
		}
		srv, err = client.AddServer(uint16(port), admin.CreateServerRequest{BindAddress: flags.ServerAddBind})
	}
	if err != nil {
		log.Fatal(err)
	}
	addr := net.JoinHostPort(srv.BindAddress, strconv.Itoa(int(srv.Port)))
	fmt.Printf("server available on %s\n", addr)
}

func serverRm(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	if err := client.RemoveServer(uint16(port)); err != nil {
		log.Fatal(err)
	}
}

func serverRmAll(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	if err := client.RemoveAllServers(); err != nil {
		log.Fatal(err)
	}
}

func fileServerList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) > 0 {
		cmd.Usage()
		return
	}
	fsrvs, err := client.GetFileServers()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(strings.Join(fsrvs, ", "))
}

func mountList(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	mounts, err := client.GetMounts(uint16(port))
	if err != nil {
		log.Fatal(err)
	}

	for _, mount := range mounts {
		fmt.Printf("%s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
	}
}

func mountAdd(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) != 2 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	source, err := filepath.Abs(args[1])
	if err != nil {
		log.Fatal(err)
	}

	target := "/" + filepath.Base(source)
	if flags.MountTarget != "" {
		target = flags.MountTarget
	}
	var fsParams fileserver.Params
	if err := json.Unmarshal([]byte(flags.FileServerParams), &fsParams); err != nil {
		log.Fatal(err)
	}

	mount, err := client.AddMount(uint16(port), admin.CreateServerMountRequest{
		target,
		source,
		flags.FileServerType,
		fsParams,
	})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s: %s -> %s\n", mount.ID, mount.Source, mount.Target)
}

func mountRm(client *client.Client, flags *flagSet, cmd *cobra.Command, args []string) {
	if len(args) != 2 {
		cmd.Usage()
		return
	}
	port, err := strconv.Atoi(args[0])
	if err != nil {
		log.Fatalf("error parsing port: %v", err)
	}
	mountID := args[1]
	if err := client.RemoveMount(uint16(port), mountID); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"sync"
	"time"
)

type TurbulenceIntensitySummary struct {
	*BaseController
}

type TurbulenceIntensity struct {
	ID                string ` bson:"_id" , json:"_id" `
	Projectname       string
	Turbine           string
	Timestamp         time.Time
	WindspeedBin      float64
	WindSpeedTotal    float64
	WindSpeedStdTotal float64
	WindSpeedCount    float64
	WindSpeedStdCount float64
	Type              string
}

func (m *TurbulenceIntensity) TableName() string {
	return "rpt_turbulenceintensity"
}

type LatestTurbulence struct {
	ID          string ` bson:"_id" , json:"_id" `
	Projectname string
	LastUpdate  time.Time
	Type        string
}

func (m *LatestTurbulence) TableName() string {
	return "log_latestturbulence"
}

func (ev *TurbulenceIntensitySummary) CreateTurbulenceIntensitySummary(base *BaseController) {
	ev.BaseController = base

	ev.Log.AddLog("===================== Start processing Turbulence Intensity Summary...", sInfo)

	var wg sync.WaitGroup
	wg.Add(2)

	go ev.processDataScada(&wg)
	go ev.processDataMet(&wg)

	wg.Wait()

	ev.Log.AddLog("===================== End processing Turbulence Intensity Summary...", sInfo)
}

func (ev *TurbulenceIntensitySummary) getProjectList() (result []string) {
	ev.Log.AddLog("Get Project List", sInfo)

	projectData := []tk.M{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From("ref_project").Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getProjectList due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&projectData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getProjectList due to : %s", e.Error()), sError)
		return
	}
	result = []string{}
	for _, val := range projectData {
		result = append(result, val.GetString("projectid"))
	}
	ev.Log.AddLog("Finish getting Project List", sInfo)

	return
}

func (ev *TurbulenceIntensitySummary) getLatestData(tipe string) (result map[string]time.Time) {
	ev.Log.AddLog("Get latest data for each turbine", sInfo)

	latestData := []LatestTurbulence{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From(new(LatestTurbulence).TableName()).
		Where(dbox.Eq("type", tipe)).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getLatestData due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&latestData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getLatestData due to : %s", e.Error()), sError)
		return
	}
	result = map[string]time.Time{}
	for _, val := range latestData {
		result[val.Projectname] = val.LastUpdate
	}
	ev.Log.AddLog("Finish getting latest data for each turbine", sInfo)

	return
}

func (ev *TurbulenceIntensitySummary) updateLastData(projectname, tipe string, maxTimeStamp time.Time) {
	if !maxTimeStamp.IsZero() {
		data := LatestTurbulence{}
		data.Projectname = projectname
		data.ID = tk.Sprintf("%s_%s", data.Projectname, tipe)
		data.LastUpdate = maxTimeStamp
		data.Type = tipe

		e := ev.Ctx.Connection.NewQuery().SetConfig("multiexec", true).
			From(new(LatestTurbulence).TableName()).Save().Exec(tk.M{"data": data})

		if e != nil {
			ev.Log.AddLog(tk.Sprintf("Error on Save at updateLastData due to : %s", e.Error()), sError)
		}
	}
	ev.Log.AddLog(tk.Sprintf("Finish updating last data for %s on %s at %s", projectname, tipe, maxTimeStamp.String()), sInfo)
}

func (ev *TurbulenceIntensitySummary) processDataScada(wgScada *sync.WaitGroup) {
	defer wgScada.Done()

	t0 := time.Now()
	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("SCADA")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorker(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration processing scada data %f minutes", time.Since(t0).Minutes()), sInfo)
}

func (ev *TurbulenceIntensitySummary) projectWorker(projectname string, lastUpdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	countWS := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeed", false}},
				tk.M{"$gte": []interface{}{"$windspeed", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	countWSStd := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeedstddev", false}},
				tk.M{"$gte": []interface{}{"$windspeedstddev", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}

	ev.Log.AddLog(tk.Sprintf("Update data %s from %s", projectname, lastUpdate.String()), sInfo)
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastUpdate}},
				tk.M{"projectname": projectname},
				tk.M{"type": "SCADA"},
			},
		}},
		tk.M{"$group": tk.M{
			"_id": tk.M{
				"projectname":  "$projectname",
				"turbine":      "$turbine",
				"windspeedbin": "$windspeedbin",
				"timestamp":    "$dateinfo.dateid",
			},
			"windspeedtotal":    tk.M{"$sum": "$windspeed"},
			"windspeedstdtotal": tk.M{"$sum": "$windspeedstddev"},
			"windspeedcount":    tk.M{"$sum": countWS},
			"windspeedstdcount": tk.M{"$sum": countWSStd},
		}},
	}

	csr, e := ev.Ctx.Connection.NewQuery().
		From("TurbulenceIntensity10Min").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	turbulenceData := []tk.M{}
	e = csr.Fetch(&turbulenceData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(turbulenceData)
	totalWorker := 4
	dataChan := make(chan TurbulenceIntensity, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From(new(TurbulenceIntensity).TableName()).Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := TurbulenceIntensity{}
	maxTimeStamp := time.Time{}

	for _, _data := range turbulenceData {
		data = TurbulenceIntensity{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		data.Projectname = ids.GetString("projectname")
		data.Turbine = ids.GetString("turbine")
		data.Timestamp = ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.WindspeedBin = ids.GetFloat64("windspeedbin")
		data.ID = tk.Sprintf("%s_%s_%s_%s", data.Projectname, data.Turbine, tk.Sprintf("%.1f", data.WindspeedBin), data.Timestamp.Format("20060102"))

		data.WindSpeedTotal = _data.GetFloat64("windspeedtotal")
		data.WindSpeedStdTotal = _data.GetFloat64("windspeedstdtotal")
		data.WindSpeedCount = _data.GetFloat64("windspeedcount")
		data.WindSpeedStdCount = _data.GetFloat64("windspeedstdcount")
		data.Type = "SCADA"

		if data.Timestamp.After(maxTimeStamp) {
			maxTimeStamp = data.Timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "SCADA", maxTimeStamp)
}

func (ev *TurbulenceIntensitySummary) projectWorkerMet(projectname string, lastupdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	countWS := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeed", false}},
				tk.M{"$gte": []interface{}{"$windspeed", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	countWSStd := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeedstddev", false}},
				tk.M{"$gte": []interface{}{"$windspeedstddev", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastupdate}},
				tk.M{"projectname": projectname},
				tk.M{"type": "MET"},
			},
		}},
		tk.M{"$group": tk.M{
			"_id": tk.M{
				"projectname":  "$projectname",
				"windspeedbin": "$windspeedbin",
				"timestamp":    "$dateinfo.dateid",
			},
			"windspeedtotal":    tk.M{"$sum": "$windspeed"},
			"windspeedstdtotal": tk.M{"$sum": "$windspeedstddev"},
			"windspeedcount":    tk.M{"$sum": countWS},
			"windspeedstdcount": tk.M{"$sum": countWSStd},
		}},
	}

	turbulenceData := []tk.M{}
	csr, e := ev.Ctx.Connection.NewQuery().
		From(new(MetTower).TableName()).
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	e = csr.Fetch(&turbulenceData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on Fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(turbulenceData)
	totalWorker := 4
	dataChan := make(chan TurbulenceIntensity, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From(new(TurbulenceIntensity).TableName()).Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := TurbulenceIntensity{}
	maxTimeStamp := time.Time{}

	for _, _data := range turbulenceData {
		data = TurbulenceIntensity{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		data.Projectname = ids.GetString("projectname")
		data.Timestamp = ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.WindspeedBin = ids.GetFloat64("windspeedbin")
		data.ID = tk.Sprintf("%s_%s_%s", data.Projectname, tk.Sprintf("%.1f", data.WindspeedBin), data.Timestamp.Format("20060102"))

		data.WindSpeedTotal = _data.GetFloat64("windspeedtotal")
		data.WindSpeedStdTotal = _data.GetFloat64("windspeedstdtotal")
		data.WindSpeedCount = _data.GetFloat64("windspeedcount")
		data.WindSpeedStdCount = _data.GetFloat64("windspeedstdcount")
		data.Type = "SCADA"

		if data.Timestamp.After(maxTimeStamp) {
			maxTimeStamp = data.Timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "MET", maxTimeStamp)
}

func (ev *TurbulenceIntensitySummary) processDataMet(wgMet *sync.WaitGroup) {
	defer wgMet.Done()

	t0 := time.Now()

	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("MET")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorkerMet(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration process met tower data %f minutes", time.Since(t0).Minutes()), sInfo)
}
<commit_msg>fixing tablename of met tower<commit_after>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"sync"
	"time"
)

type TurbulenceIntensitySummary struct {
	*BaseController
}

type TurbulenceIntensity struct {
	ID                string ` bson:"_id" , json:"_id" `
	Projectname       string
	Turbine           string
	Timestamp         time.Time
	WindspeedBin      float64
	WindSpeedTotal    float64
	WindSpeedStdTotal float64
	WindSpeedCount    float64
	WindSpeedStdCount float64
	Type              string
}

func (m *TurbulenceIntensity) TableName() string {
	return "rpt_turbulenceintensity"
}

type LatestTurbulence struct {
	ID          string ` bson:"_id" , json:"_id" `
	Projectname string
	LastUpdate  time.Time
	Type        string
}

func (m *LatestTurbulence) TableName() string {
	return "log_latestturbulence"
}

func (ev *TurbulenceIntensitySummary) CreateTurbulenceIntensitySummary(base *BaseController) {
	ev.BaseController = base

	ev.Log.AddLog("===================== Start processing Turbulence Intensity Summary...", sInfo)

	var wg sync.WaitGroup
	wg.Add(2)

	go ev.processDataScada(&wg)
	go ev.processDataMet(&wg)

	wg.Wait()

	ev.Log.AddLog("===================== End processing Turbulence Intensity Summary...", sInfo)
}

func (ev *TurbulenceIntensitySummary) getProjectList() (result []string) {
	ev.Log.AddLog("Get Project List", sInfo)

	projectData := []tk.M{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From("ref_project").Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getProjectList due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&projectData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getProjectList due to : %s", e.Error()), sError)
		return
	}
	result = []string{}
	for _, val := range projectData {
		result = append(result, val.GetString("projectid"))
	}
	ev.Log.AddLog("Finish getting Project List", sInfo)

	return
}

func (ev *TurbulenceIntensitySummary) getLatestData(tipe string) (result map[string]time.Time) {
	ev.Log.AddLog("Get latest data for each turbine", sInfo)

	latestData := []LatestTurbulence{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From(new(LatestTurbulence).TableName()).
		Where(dbox.Eq("type", tipe)).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getLatestData due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&latestData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getLatestData due to : %s", e.Error()), sError)
		return
	}
	result = map[string]time.Time{}
	for _, val := range latestData {
		result[val.Projectname] = val.LastUpdate
	}
	ev.Log.AddLog("Finish getting latest data for each turbine", sInfo)

	return
}

func (ev *TurbulenceIntensitySummary) updateLastData(projectname, tipe string, maxTimeStamp time.Time) {
	if !maxTimeStamp.IsZero() {
		data := LatestTurbulence{}
		data.Projectname = projectname
		data.ID = tk.Sprintf("%s_%s", data.Projectname, tipe)
		data.LastUpdate = maxTimeStamp
		data.Type = tipe

		e := ev.Ctx.Connection.NewQuery().SetConfig("multiexec", true).
			From(new(LatestTurbulence).TableName()).Save().Exec(tk.M{"data": data})

		if e != nil {
			ev.Log.AddLog(tk.Sprintf("Error on Save at updateLastData due to : %s", e.Error()), sError)
		}
	}
	ev.Log.AddLog(tk.Sprintf("Finish updating last data for %s on %s at %s", projectname, tipe, maxTimeStamp.String()), sInfo)
}

func (ev *TurbulenceIntensitySummary) processDataScada(wgScada *sync.WaitGroup) {
	defer wgScada.Done()

	t0 := time.Now()
	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("SCADA")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorker(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration processing scada data %f minutes", time.Since(t0).Minutes()), sInfo)
}

func (ev *TurbulenceIntensitySummary) projectWorker(projectname string, lastUpdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	countWS := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeed", false}},
				tk.M{"$gte": []interface{}{"$windspeed", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	countWSStd := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeedstddev", false}},
				tk.M{"$gte": []interface{}{"$windspeedstddev", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}

	ev.Log.AddLog(tk.Sprintf("Update data %s from %s", projectname, lastUpdate.String()), sInfo)
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastUpdate}},
				tk.M{"projectname": projectname},
				tk.M{"type": "SCADA"},
			},
		}},
		tk.M{"$group": tk.M{
			"_id": tk.M{
				"projectname":  "$projectname",
				"turbine":      "$turbine",
				"windspeedbin": "$windspeedbin",
				"timestamp":    "$dateinfo.dateid",
			},
			"windspeedtotal":    tk.M{"$sum": "$windspeed"},
			"windspeedstdtotal": tk.M{"$sum": "$windspeedstddev"},
			"windspeedcount":    tk.M{"$sum": countWS},
			"windspeedstdcount": tk.M{"$sum": countWSStd},
		}},
	}

	csr, e := ev.Ctx.Connection.NewQuery().
		From("TurbulenceIntensity10Min").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	turbulenceData := []tk.M{}
	e = csr.Fetch(&turbulenceData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(turbulenceData)
	totalWorker := 4
	dataChan := make(chan TurbulenceIntensity, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From(new(TurbulenceIntensity).TableName()).Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := TurbulenceIntensity{}
	maxTimeStamp := time.Time{}

	for _, _data := range turbulenceData {
		data = TurbulenceIntensity{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		data.Projectname = ids.GetString("projectname")
		data.Turbine = ids.GetString("turbine")
		data.Timestamp = ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.WindspeedBin = ids.GetFloat64("windspeedbin")
		data.ID = tk.Sprintf("%s_%s_%s_%s", data.Projectname, data.Turbine, tk.Sprintf("%.1f", data.WindspeedBin), data.Timestamp.Format("20060102"))

		data.WindSpeedTotal = _data.GetFloat64("windspeedtotal")
		data.WindSpeedStdTotal = _data.GetFloat64("windspeedstdtotal")
		data.WindSpeedCount = _data.GetFloat64("windspeedcount")
		data.WindSpeedStdCount = _data.GetFloat64("windspeedstdcount")
		data.Type = "SCADA"

		if data.Timestamp.After(maxTimeStamp) {
			maxTimeStamp = data.Timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "SCADA", maxTimeStamp)
}

func (ev *TurbulenceIntensitySummary) projectWorkerMet(projectname string, lastupdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	countWS := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeed", false}},
				tk.M{"$gte": []interface{}{"$windspeed", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	countWSStd := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{
			"$and": []tk.M{
				tk.M{"$ifNull": []interface{}{"$windspeedstddev", false}},
				tk.M{"$gte": []interface{}{"$windspeedstddev", -200}},
			},
		}).
		Set("then", 1).
		Set("else", 0)}
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastupdate}},
				tk.M{"projectname": projectname},
				tk.M{"type": "MET"},
			},
		}},
		tk.M{"$group": tk.M{
			"_id": tk.M{
				"projectname":  "$projectname",
				"windspeedbin": "$windspeedbin",
				"timestamp":    "$dateinfo.dateid",
			},
			"windspeedtotal":    tk.M{"$sum": "$windspeed"},
			"windspeedstdtotal": tk.M{"$sum": "$windspeedstddev"},
			"windspeedcount":    tk.M{"$sum": countWS},
			"windspeedstdcount": tk.M{"$sum": countWSStd},
		}},
	}

	turbulenceData := []tk.M{}
	csr, e := ev.Ctx.Connection.NewQuery().
		From("TurbulenceIntensity10Min").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	e = csr.Fetch(&turbulenceData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on Fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(turbulenceData)
	totalWorker := 4
	dataChan := make(chan TurbulenceIntensity, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From(new(TurbulenceIntensity).TableName()).Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := TurbulenceIntensity{}
	maxTimeStamp := time.Time{}

	for _, _data := range turbulenceData {
		data = TurbulenceIntensity{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		data.Projectname = ids.GetString("projectname")
		data.Timestamp = ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.WindspeedBin = ids.GetFloat64("windspeedbin")
		data.ID = tk.Sprintf("%s_%s_%s", data.Projectname, tk.Sprintf("%.1f", data.WindspeedBin), data.Timestamp.Format("20060102"))

		data.WindSpeedTotal = _data.GetFloat64("windspeedtotal")
		data.WindSpeedStdTotal = _data.GetFloat64("windspeedstdtotal")
		data.WindSpeedCount = _data.GetFloat64("windspeedcount")
		data.WindSpeedStdCount = _data.GetFloat64("windspeedstdcount")
		data.Type = "SCADA"

		if data.Timestamp.After(maxTimeStamp) {
			maxTimeStamp = data.Timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "MET", maxTimeStamp)
}

func (ev *TurbulenceIntensitySummary) processDataMet(wgMet *sync.WaitGroup) {
	defer wgMet.Done()

	t0 := time.Now()

	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("MET")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorkerMet(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration process met tower data %f minutes", time.Since(t0).Minutes()), sInfo)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/joho/godotenv"
	"github.com/robfig/cron"
	"github.com/smeriwether/honeybadger-go"
	kube "github.com/wearemolecule/kubeclient"
	"golang.org/x/build/kubernetes/api"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
)

var configDir string
var kubeClient *kube.Client
var jobLock map[string]string

func init() {
	flag.StringVar(&configDir, "dir", ".", "Directory where config is located")
}

func main() {
	configureHoneybadger()
	defer honeybadger.Monitor()

	jobLock = make(map[string]string)
	flag.Parse()
	err := godotenv.Load()
	if err != nil {
		log.Println(err)
	}

	kubeClient, err = kube.GetKubeClientFromEnv()
	if err != nil {
		nErr := fmt.Errorf("Failed to connect to kubernetes. Error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		panic(nErr)
	}

	b, err := ioutil.ReadFile(filePath("schedule.yml"))
	if err != nil {
		nErr := fmt.Errorf("Unable to read schedule yaml, error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Fatal(nErr)
	}

	var config JobList
	err = yaml.Unmarshal(b, &config)
	if err != nil {
		nErr := fmt.Errorf("Unable to unmarshal schedule yaml, error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Fatal(nErr)
	}

	log.Println(config)
	c := cron.New()
	for _, job := range config {
		c.AddJob(job.Cron, job)
	}

	c.Start()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill)

	s := <-sigs
	log.Println("Got signal: ", s)
}

type JobList map[string]Job

type Job struct {
	Cron        string
	Template    string
	Description string
	Args        []string
	Namespace   string
}

func (j Job) Run() {
	if _, ok := jobLock[j.Template]; ok {
		nErr := fmt.Errorf("Unable to start new job (%s) because it is already running", j.Description)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Println(nErr)
		return
	}
	log.Println("Running", j.Description)

	//TODO not thread safe
	jobLock[j.Template] = "started"
	defer delete(jobLock, j.Template)

	if err := createTaskPod(j); err != nil {
		nErr := fmt.Errorf("Failed to create task pod for job %s, error: %v", j.Description, err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Println(nErr)
		return
	}

	log.Println("Done", j.Description)
}

func filePath(filename string) string {
	return fmt.Sprintf("%s/%s", configDir, filename)
}

func createTaskPod(j Job) error {
	ctx := context.TODO()

	podData, err := ioutil.ReadFile(filePath(j.Template))
	pod := api.Pod{}
	if err != nil {
		return errors.New(fmt.Sprintf("Error reading task pod.\n%v", err))
	}

	err = json.Unmarshal(podData, &pod)
	if err != nil {
		return errors.New(fmt.Sprintf("Error parsing task pod.\n%v", err))
	}

	pod.Spec.Containers[0].Args = j.Args
	pod.ObjectMeta.Namespace = j.Namespace

	newPod, err := kubeClient.CreatePod(ctx, &pod)
	if err != nil {
		return errors.New(fmt.Sprintf("Error creating task pod.\n%v", err))
	}

	statuses, err := kubeClient.WatchPod(ctx, newPod.Namespace, newPod.Name, newPod.ResourceVersion)
	if err != nil {
		return errors.New(fmt.Sprintf("Error watching task pod.\n%v", err))
	}

	for status := range statuses {
		podStatus := status.Pod.Status
		if podStatus.Phase == "Failed" {
			_ = kubeClient.DeletePod(ctx, newPod.Namespace, newPod.Name)
			return errors.New(fmt.Sprintf("Task pod failed.\n%v", err))
		}
		if podStatus.Phase == "Succeeded" {
			if logs, err := kubeClient.PodLog(ctx, newPod.Namespace, newPod.Name); err != nil {
				log.Println("Failed to get logs for pod %s in namespace %s\n", newPod.Name, newPod.Namespace)
			} else {
				log.Println(logs)
			}
			if err = kubeClient.DeletePod(ctx, newPod.Namespace, newPod.Name); err != nil {
				nErr := fmt.Errorf("Failed to delete task pod for job %s, error: %v", j.Description, err)
				honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
				log.Println(nErr)
			}
			break
		}
	}

	return nil
}

func configureHoneybadger() {
	honeybadger.Configure(honeybadger.Configuration{APIKey: os.Getenv("HONEYBADGER_API_KEY")})
}
<commit_msg>Edit honeybadger config and failed task pod error msg<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"time"

	honeybadger "github.com/honeybadger-io/honeybadger-go"
	"github.com/joho/godotenv"
	"github.com/robfig/cron"
	kube "github.com/wearemolecule/kubeclient"
	"golang.org/x/build/kubernetes/api"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
)

var configDir string
var kubeClient *kube.Client
var jobLock map[string]string

func init() {
	flag.StringVar(&configDir, "dir", ".", "Directory where config is located")
}

func main() {
	configureHoneybadger()
	defer honeybadger.Monitor()

	jobLock = make(map[string]string)
	flag.Parse()
	err := godotenv.Load()
	if err != nil {
		log.Println(err)
	}

	kubeClient, err = kube.GetKubeClientFromEnv()
	if err != nil {
		nErr := fmt.Errorf("Failed to connect to kubernetes. Error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		panic(nErr)
	}

	b, err := ioutil.ReadFile(filePath("schedule.yml"))
	if err != nil {
		nErr := fmt.Errorf("Unable to read schedule yaml, error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Fatal(nErr)
	}

	var config JobList
	err = yaml.Unmarshal(b, &config)
	if err != nil {
		nErr := fmt.Errorf("Unable to unmarshal schedule yaml, error: %v", err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Fatal(nErr)
	}

	log.Println(config)
	c := cron.New()
	for _, job := range config {
		c.AddJob(job.Cron, job)
	}

	c.Start()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill)

	s := <-sigs
	log.Println("Got signal: ", s)
}

type JobList map[string]Job

type Job struct {
	Cron        string
	Template    string
	Description string
	Args        []string
	Namespace   string
}

func (j Job) Run() {
	if _, ok := jobLock[j.Template]; ok {
		nErr := fmt.Errorf("Unable to start new job (%s) because it is already running", j.Description)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Println(nErr)
		return
	}
	log.Println("Running", j.Description)

	//TODO not thread safe
	jobLock[j.Template] = "started"
	defer delete(jobLock, j.Template)

	if err := createTaskPod(j); err != nil {
		nErr := fmt.Errorf("Failed to create task pod for job %s, error: %v", j.Description, err)
		honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
		log.Println(nErr)
		return
	}

	log.Println("Done", j.Description)
}

func filePath(filename string) string {
	return fmt.Sprintf("%s/%s", configDir, filename)
}

func createTaskPod(j Job) error {
	ctx := context.TODO()

	podData, err := ioutil.ReadFile(filePath(j.Template))
	pod := api.Pod{}
	if err != nil {
		return errors.New(fmt.Sprintf("Error reading task pod.\n%v", err))
	}

	err = json.Unmarshal(podData, &pod)
	if err != nil {
		return errors.New(fmt.Sprintf("Error parsing task pod.\n%v", err))
	}

	pod.Spec.Containers[0].Args = j.Args
	pod.ObjectMeta.Namespace = j.Namespace

	newPod, err := kubeClient.CreatePod(ctx, &pod)
	if err != nil {
		return errors.New(fmt.Sprintf("Error creating task pod.\n%v", err))
	}

	statuses, err := kubeClient.WatchPod(ctx, newPod.Namespace, newPod.Name, newPod.ResourceVersion)
	if err != nil {
		return errors.New(fmt.Sprintf("Error watching task pod.\n%v", err))
	}

	for status := range statuses {
		podStatus := status.Pod.Status
		if podStatus.Phase == "Failed" {
			_ = kubeClient.DeletePod(ctx, newPod.Namespace, newPod.Name)
			return errors.New(fmt.Sprintf("Task pod %s in namespace %s failed.", newPod.Name, newPod.Namespace))
		}
		if podStatus.Phase == "Succeeded" {
			if logs, err := kubeClient.PodLog(ctx, newPod.Namespace, newPod.Name); err != nil {
				log.Println("Failed to get logs for pod %s in namespace %s\n", newPod.Name, newPod.Namespace)
			} else {
				log.Println(logs)
			}
			if err = kubeClient.DeletePod(ctx, newPod.Namespace, newPod.Name); err != nil {
				nErr := fmt.Errorf("Failed to delete task pod for job %s, error: %v", j.Description, err)
				honeybadger.Notify(nErr, honeybadger.Fingerprint{fmt.Sprintf("%d", time.Now().Unix())})
				log.Println(nErr)
			}
			break
		}
	}

	return nil
}

func configureHoneybadger() {
	honeybadger.Configure(honeybadger.Configuration{APIKey: os.Getenv("HONEYBADGER_API_KEY")})
}
<|endoftext|>