<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"io"
	"log"
	"os"
	"path"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/migrate"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/version"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
)

type Storage interface {
	// Save function saves ents and state to the underlying stable storage.
	// Save MUST block until st and ents are on stable storage.
	Save(st raftpb.HardState, ents []raftpb.Entry) error
	// SaveSnap function saves snapshot to the underlying stable storage.
	SaveSnap(snap raftpb.Snapshot) error
	// Close closes the Storage and performs finalization.
	Close() error
}

type storage struct {
	*wal.WAL
	*snap.Snapshotter
}

func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
	return &storage{w, s}
}

// SaveSnap saves the snapshot to disk and release the locked
// wal files since they will not be used.
func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
	walsnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	err := st.WAL.SaveSnapshot(walsnap)
	if err != nil {
		return err
	}
	err = st.Snapshotter.SaveSnap(snap)
	if err != nil {
		return err
	}
	err = st.WAL.ReleaseLockTo(snap.Metadata.Index)
	if err != nil {
		return err
	}
	return nil
}

func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
	var (
		err       error
		wmetadata []byte
	)

	for i := 0; i < 2; i++ {
		if w, err = wal.Open(waldir, snap); err != nil {
			log.Fatalf("etcdserver: open wal error: %v", err)
		}
		if wmetadata, st, ents, err = w.ReadAll(); err != nil {
			w.Close()
			if i != 0 || err != io.ErrUnexpectedEOF {
				log.Fatalf("etcdserver: read wal error: %v", err)
			}
			if !wal.Repair(waldir) {
				log.Fatalf("etcdserver: WAL error (%v) cannot be repaired", err)
			} else {
				log.Printf("etcdserver: repaired WAL error (%v)", err)
			}
			continue
		}
		break
	}
	var metadata pb.Metadata
	pbutil.MustUnmarshal(&metadata, wmetadata)
	id = types.ID(metadata.NodeID)
	cid = types.ID(metadata.ClusterID)
	return
}

// upgradeWAL converts an older version of the etcdServer data to the newest version.
// It must ensure that, after upgrading, the most recent version is present.
func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
	switch ver {
	case version.DataDir0_4:
		log.Print("etcdserver: converting v0.4 log to v2.0")
		err := migrate.Migrate4To2(baseDataDir, name)
		if err != nil {
			log.Fatalf("etcdserver: failed migrating data-dir: %v", err)
			return err
		}
		fallthrough
	case version.DataDir2_0:
		err := makeMemberDir(baseDataDir)
		if err != nil {
			return err
		}
		fallthrough
	case version.DataDir2_0_1:
		fallthrough
	default:
		log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
	}
	return nil
}

func makeMemberDir(dir string) error {
	membdir := path.Join(dir, "member")
	_, err := os.Stat(membdir)
	switch {
	case err == nil:
		return nil
	case !os.IsNotExist(err):
		return err
	}
	if err := os.MkdirAll(membdir, 0700); err != nil {
		return err
	}
	names := []string{"snap", "wal"}
	for _, name := range names {
		if err := os.Rename(path.Join(dir, name), path.Join(membdir, name)); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>etcdserver: make the wal repairing logic clear<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"io"
	"log"
	"os"
	"path"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/migrate"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/version"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
)

type Storage interface {
	// Save function saves ents and state to the underlying stable storage.
	// Save MUST block until st and ents are on stable storage.
	Save(st raftpb.HardState, ents []raftpb.Entry) error
	// SaveSnap function saves snapshot to the underlying stable storage.
	SaveSnap(snap raftpb.Snapshot) error
	// Close closes the Storage and performs finalization.
	Close() error
}

type storage struct {
	*wal.WAL
	*snap.Snapshotter
}

func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
	return &storage{w, s}
}

// SaveSnap saves the snapshot to disk and release the locked
// wal files since they will not be used.
func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
	walsnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	err := st.WAL.SaveSnapshot(walsnap)
	if err != nil {
		return err
	}
	err = st.Snapshotter.SaveSnap(snap)
	if err != nil {
		return err
	}
	err = st.WAL.ReleaseLockTo(snap.Metadata.Index)
	if err != nil {
		return err
	}
	return nil
}

func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
	var (
		err       error
		wmetadata []byte
	)

	repaired := false
	for {
		if w, err = wal.Open(waldir, snap); err != nil {
			log.Fatalf("etcdserver: open wal error: %v", err)
		}
		if wmetadata, st, ents, err = w.ReadAll(); err != nil {
			w.Close()
			// we can only repair ErrUnexpectedEOF and we never repair twice.
			if repaired || err != io.ErrUnexpectedEOF {
				log.Fatalf("etcdserver: read wal error (%v) and cannot be repaired", err)
			}
			if !wal.Repair(waldir) {
				log.Fatalf("etcdserver: WAL error (%v) cannot be repaired", err)
			} else {
				log.Printf("etcdserver: repaired WAL error (%v)", err)
				repaired = true
			}
			continue
		}
		break
	}
	var metadata pb.Metadata
	pbutil.MustUnmarshal(&metadata, wmetadata)
	id = types.ID(metadata.NodeID)
	cid = types.ID(metadata.ClusterID)
	return
}

// upgradeWAL converts an older version of the etcdServer data to the newest version.
// It must ensure that, after upgrading, the most recent version is present.
func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
	switch ver {
	case version.DataDir0_4:
		log.Print("etcdserver: converting v0.4 log to v2.0")
		err := migrate.Migrate4To2(baseDataDir, name)
		if err != nil {
			log.Fatalf("etcdserver: failed migrating data-dir: %v", err)
			return err
		}
		fallthrough
	case version.DataDir2_0:
		err := makeMemberDir(baseDataDir)
		if err != nil {
			return err
		}
		fallthrough
	case version.DataDir2_0_1:
		fallthrough
	default:
		log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
	}
	return nil
}

func makeMemberDir(dir string) error {
	membdir := path.Join(dir, "member")
	_, err := os.Stat(membdir)
	switch {
	case err == nil:
		return nil
	case !os.IsNotExist(err):
		return err
	}
	if err := os.MkdirAll(membdir, 0700); err != nil {
		return err
	}
	names := []string{"snap", "wal"}
	for _, name := range names {
		if err := os.Rename(path.Join(dir, name), path.Join(membdir, name)); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"fmt"
	"github.com/FactomProject/FactomCode/common"
	"github.com/FactomProject/FactomCode/factomlog"
	"github.com/FactomProject/FactomCode/util"
	"github.com/FactomProject/factoid/block"
	"github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"os"
	"sort"
)

func GetEntryCreditBalance(pubKey *[32]byte) (int32, error) {

	return eCreditMap[string(pubKey[:])], nil
}

func exportDChain(chain *common.DChain) {
	if len(chain.Blocks) == 0 || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	// get all ecBlocks from db
	dBlocks, _ := db.FetchAllDBlocks()
	sort.Sort(util.ByDBlockIDAccending(dBlocks))

	for _, block := range dBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportEChain(chain *common.EChain) {
	if procLog.Level() < factomlog.Info {
		return
	}

	eBlocks, _ := db.FetchAllEBlocksByChain(chain.ChainID)
	sort.Sort(util.ByEBlockIDAccending(*eBlocks))

	for _, block := range *eBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}

		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.EBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportECChain(chain *common.ECChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all ecBlocks from db
	ecBlocks, _ := db.FetchAllECBlocks()
	sort.Sort(util.ByECBlockIDAccending(ecBlocks))

	for _, block := range ecBlocks {
		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportAChain(chain *common.AdminChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	aBlocks, _ := db.FetchAllABlocks()
	sort.Sort(util.ByABlockIDAccending(aBlocks))

	for _, block := range aBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportFctChain(chain *common.FctChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	FBlocks, _ := db.FetchAllFBlocks()
	sort.Sort(util.ByFBlockIDAccending(FBlocks))

	for _, block := range FBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

// to export individual block once at a time - for debugging ------------------------
func exportDBlock(block *common.DirectoryBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := dchain.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportEBlock(block *common.EBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}

	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportECBlock(block *common.ECBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ECChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportABlock(block *common.AdminBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.AdminChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportFctBlock(block block.IFBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.GetChainID().String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
	if err != nil {
		panic(err)
	}

}

//--------------------------------------

func getPrePaidChainKey(entryHash *common.Hash, chainIDHash *common.Hash) string {
	return chainIDHash.String() + entryHash.String()
}

func copyCreditMap(
	originalMap map[string]int32,
	newMap map[string]int32) {
	newMap = make(map[string]int32)

	// copy every element from the original map
	for k, v := range originalMap {
		newMap[k] = v
	}

}

func printCreditMap() {
	procLog.Debug("eCreditMap:")
	for key := range eCreditMap {
		procLog.Debugf("Key: %x Value %d\n", key, eCreditMap[key])
	}
}

func fileNotExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return true
	}
	return err != nil
}

// HaveBlockInDB returns whether or not the chain instance has the block represented
// by the passed hash.  This includes checking the various places a block can
// be like part of the main chain, on a side chain, or in the orphan pool.
//
// This function is NOT safe for concurrent access.
func HaveBlockInDB(hash *common.Hash) (bool, error) {
	util.Trace(spew.Sdump(hash))

	if hash == nil || dchain.Blocks == nil || len(dchain.Blocks) == 0 {
		return false, nil
	}

	// double check the block ids
	for i := 0; i < len(dchain.Blocks); i = i + 1 {
		if dchain.Blocks[i] == nil {
			continue
		}
		if dchain.Blocks[i].DBHash == nil {
			dchain.Blocks[i].DBHash, _ = common.CreateHash(dchain.Blocks[i])
		}
		if dchain.Blocks[i].DBHash.IsSameAs(hash) {
			return true, nil
		}
	}

	return false, nil
}
<commit_msg>Changed EBock num in export for debugging<commit_after>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"fmt"
	"github.com/FactomProject/FactomCode/common"
	"github.com/FactomProject/FactomCode/factomlog"
	"github.com/FactomProject/FactomCode/util"
	"github.com/FactomProject/factoid/block"
	"github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"os"
	"sort"
)

func GetEntryCreditBalance(pubKey *[32]byte) (int32, error) {

	return eCreditMap[string(pubKey[:])], nil
}

func exportDChain(chain *common.DChain) {
	if len(chain.Blocks) == 0 || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	// get all ecBlocks from db
	dBlocks, _ := db.FetchAllDBlocks()
	sort.Sort(util.ByDBlockIDAccending(dBlocks))

	for _, block := range dBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportEChain(chain *common.EChain) {
	if procLog.Level() < factomlog.Info {
		return
	}

	eBlocks, _ := db.FetchAllEBlocksByChain(chain.ChainID)
	sort.Sort(util.ByEBlockIDAccending(*eBlocks))

	for _, block := range *eBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}

		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportECChain(chain *common.ECChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all ecBlocks from db
	ecBlocks, _ := db.FetchAllECBlocks()
	sort.Sort(util.ByECBlockIDAccending(ecBlocks))

	for _, block := range ecBlocks {
		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportAChain(chain *common.AdminChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	aBlocks, _ := db.FetchAllABlocks()
	sort.Sort(util.ByABlockIDAccending(aBlocks))

	for _, block := range aBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportFctChain(chain *common.FctChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	FBlocks, _ := db.FetchAllFBlocks()
	sort.Sort(util.ByFBlockIDAccending(FBlocks))

	for _, block := range FBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

// to export individual block once at a time - for debugging ------------------------
func exportDBlock(block *common.DirectoryBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := dchain.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportEBlock(block *common.EBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}

	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportECBlock(block *common.ECBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ECChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportABlock(block *common.AdminBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.AdminChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportFctBlock(block block.IFBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.GetChainID().String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
	if err != nil {
		panic(err)
	}

}

//--------------------------------------

func getPrePaidChainKey(entryHash *common.Hash, chainIDHash *common.Hash) string {
	return chainIDHash.String() + entryHash.String()
}

func copyCreditMap(
	originalMap map[string]int32,
	newMap map[string]int32) {
	newMap = make(map[string]int32)

	// copy every element from the original map
	for k, v := range originalMap {
		newMap[k] = v
	}

}

func printCreditMap() {
	procLog.Debug("eCreditMap:")
	for key := range eCreditMap {
		procLog.Debugf("Key: %x Value %d\n", key, eCreditMap[key])
	}
}

func fileNotExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return true
	}
	return err != nil
}

// HaveBlockInDB returns whether or not the chain instance has the block represented
// by the passed hash.  This includes checking the various places a block can
// be like part of the main chain, on a side chain, or in the orphan pool.
//
// This function is NOT safe for concurrent access.
func HaveBlockInDB(hash *common.Hash) (bool, error) {
	util.Trace(spew.Sdump(hash))

	if hash == nil || dchain.Blocks == nil || len(dchain.Blocks) == 0 {
		return false, nil
	}

	// double check the block ids
	for i := 0; i < len(dchain.Blocks); i = i + 1 {
		if dchain.Blocks[i] == nil {
			continue
		}
		if dchain.Blocks[i].DBHash == nil {
			dchain.Blocks[i].DBHash, _ = common.CreateHash(dchain.Blocks[i])
		}
		if dchain.Blocks[i].DBHash.IsSameAs(hash) {
			return true, nil
		}
	}

	return false, nil
}
<|endoftext|>


<commit_before>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"text/template"
	"time"
)

// dirMetadata is the location of the x-common repository
// on the filesystem.
// We're making the assumption that the x-common repository
// has been cloned to the same parent directory as the xgo
// repository. E.g.
//
//     $ tree -L 1 .
//     .
//     ├── x-common
//     └── xgo
var dirMetadata string

// dirExercise is the location that the test cases should be generated to.
// This assumes that the generator script lives in the .meta directory within
// the exercise directory. Falls back to the present working directory.
var dirExercise string

// genClient creates an http client with a 10 second timeout so we don't get
// stuck waiting for a response.
var genClient = &http.Client{Timeout: 10 * time.Second}

const (
	// canonicalDataURL is the URL for the raw canonical-data.json data,
	// requires exercise name.
	canonicalDataURL = "https://raw.githubusercontent.com/exercism/x-common/master/exercises/%s/canonical-data.json"
	// commitsURL is the GitHub api endpoint for the canonical-data.json
	// file commit history, requires exercise name.
	commitsURL = "https://api.github.com/repos/exercism/x-common/commits?path=exercises/%s/canonical-data.json"
)

// Header tells how the test data was generated, for display in the header of cases_test.go
type Header struct {
	// Ori is a deprecated short name for Origin.
	// TODO: Remove Ori once everything switches to Origin.
	Ori     string
	Origin  string
	Commit  string
	Version string
}

func (h Header) String() string {
	s := fmt.Sprintf("// Source: %s\n", h.Origin)
	if h.Commit != "" {
		s += fmt.Sprintf("// Commit: %s\n", h.Commit)
	}
	if h.Version != "" {
		s += fmt.Sprintf("// x-common version: %s\n", h.Version)
	}
	return s
}

func init() {
	if _, path, _, ok := runtime.Caller(0); ok {
		dirMetadata = filepath.Join(path, "..", "..", "..", "x-common")
	}
	if _, path, _, ok := runtime.Caller(2); ok {
		dirExercise = filepath.Join(path, "..", "..")
	}
	if dirExercise == "" {
		dirExercise = "."
	}
}

// Gen generates the exercise cases_test.go file from the relevant canonical-data.json
func Gen(exercise string, j interface{}, t *template.Template) error {
	if dirMetadata == "" {
		return errors.New("unable to determine current path")
	}
	jFile := filepath.Join("exercises", exercise, "canonical-data.json")
	// try to find and read the local json source file
	log.Printf("[LOCAL] fetching %s test data\n", exercise)
	jPath, jOrigin, jCommit := getLocal(jFile)
	if jPath != "" {
		log.Printf("[LOCAL] source: %s\n", jPath)
	}
	jSrc, err := ioutil.ReadFile(filepath.Join(jPath, jFile))
	if err != nil {
		// fetch json data remotely if there's no local file
		log.Println("[LOCAL] No test data found")
		log.Printf("[REMOTE] fetching %s test data\n", exercise)
		jSrc, jOrigin, jCommit, err = getRemote(exercise)
		if err != nil {
			return err
		}
	}

	// unmarshal the json source to a Go structure
	if err = json.Unmarshal(jSrc, j); err != nil {
		// This error message is usually enough if the problem is a wrong
		// data structure defined here. Sadly it doesn't locate the error well
		// in the case of invalid JSON.  Use a real validator tool if you can't
		// spot the problem right away.
		return fmt.Errorf(`unexpected data structure: %v`, err)
	}

	// These fields are guaranteed to be in every problem
	var commonMetadata struct {
		Version string
	}
	if err := json.Unmarshal(jSrc, &commonMetadata); err != nil {
		return fmt.Errorf(`Didn't contain version: %v`, err)
	}

	// package up a little meta data
	d := struct {
		Header
		J interface{}
	}{Header{
		Ori:     jOrigin,
		Origin:  jOrigin,
		Commit:  jCommit,
		Version: commonMetadata.Version,
	}, j}

	// render the Go test cases
	var b bytes.Buffer
	if err = t.Execute(&b, &d); err != nil {
		return err
	}
	// clean it up
	src, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}
	// write output file for the Go test cases.
	return ioutil.WriteFile(filepath.Join(dirExercise, "cases_test.go"), src, 0666)
}

func getLocal(jFile string) (jPath, jOrigin, jCommit string) {
	// Ideally draw from a .json which is pulled from the official x-common
	// repository.  For development however, accept a file in current directory
	// if there is no .json in source control.  Also allow an override in any
	// case by environment variable.
	if jPath = os.Getenv("EXTEST"); jPath > "" {
		return jPath, "local file", "" // override
	}
	c := exec.Command("git", "log", "-1", "--oneline", jFile)
	c.Dir = dirMetadata
	ori, err := c.Output()
	if err != nil {
		return "", "local file", "" // no source control
	}
	if _, err = os.Stat(filepath.Join(c.Dir, jFile)); err != nil {
		return "", "local file", "" // not in source control
	}
	// good.  return source control dir and commit.
	return c.Dir, "exercism/x-common", string(bytes.TrimSpace(ori))
}

func getRemote(exercise string) (body []byte, jOrigin string, jCommit string, err error) {
	url := fmt.Sprintf(canonicalDataURL, exercise)
	resp, err := genClient.Get(url)
	if err != nil {
		return []byte{}, "", "", err
	}
	if resp.StatusCode != http.StatusOK {
		return []byte{}, "", "", fmt.Errorf("error fetching remote data: %s", resp.Status)
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, "", "", err
	}
	c, err := getRemoteCommit(exercise)
	if err != nil {
		// we always expect to have the commit in the cases_test.go
		// file, so return the error if we can't fetch it
		return []byte{}, "", "", err
	}
	log.Printf("[REMOTE] source: %s\n", url)
	return body, "exercism/x-common", c, nil
}

func getRemoteCommit(exercise string) (string, error) {
	type Commits struct {
		Sha    string
		Commit struct {
			Message string
		}
	}
	resp, err := genClient.Get(fmt.Sprintf(commitsURL, exercise))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	var c []Commits
	err = json.NewDecoder(resp.Body).Decode(&c)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s %s", c[0].Sha[0:7], c[0].Commit.Message), nil
}
<commit_msg>gen.go: minor formatting fixes<commit_after>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"text/template"
	"time"
)

// dirMetadata is the location of the x-common repository on the filesystem.
// We're making the assumption that the x-common repository has been cloned to
// the same parent directory as the xgo repository.
// E.g.
//
//     $ tree -L 1 .
//     .
//     ├── x-common
//     └── xgo
var dirMetadata string

// dirExercise is the location that the test cases should be generated to.
// This assumes that the generator script lives in the .meta directory within
// the exercise directory. Falls back to the present working directory.
var dirExercise string

// genClient creates an http client with a 10 second timeout so we don't get
// stuck waiting for a response.
var genClient = &http.Client{Timeout: 10 * time.Second}

const (
	// canonicalDataURL is the URL for the raw canonical-data.json data,
	// requires exercise name.
	canonicalDataURL = "https://raw.githubusercontent.com/exercism/x-common/master/exercises/%s/canonical-data.json"
	// commitsURL is the GitHub api endpoint for the canonical-data.json
	// file commit history, requires exercise name.
	commitsURL = "https://api.github.com/repos/exercism/x-common/commits?path=exercises/%s/canonical-data.json"
)

// Header tells how the test data was generated, for display in the header of
// cases_test.go
type Header struct {
	// Ori is a deprecated short name for Origin.
	// TODO: Remove Ori once everything switches to Origin.
	Ori     string
	Origin  string
	Commit  string
	Version string
}

func (h Header) String() string {
	s := fmt.Sprintf("// Source: %s\n", h.Origin)
	if h.Commit != "" {
		s += fmt.Sprintf("// Commit: %s\n", h.Commit)
	}
	if h.Version != "" {
		s += fmt.Sprintf("// x-common version: %s\n", h.Version)
	}
	return s
}

func init() {
	if _, path, _, ok := runtime.Caller(0); ok {
		dirMetadata = filepath.Join(path, "..", "..", "..", "x-common")
	}
	if _, path, _, ok := runtime.Caller(2); ok {
		dirExercise = filepath.Join(path, "..", "..")
	}
	if dirExercise == "" {
		dirExercise = "."
	}
}

// Gen generates the exercise cases_test.go file from the relevant canonical-data.json
func Gen(exercise string, j interface{}, t *template.Template) error {
	if dirMetadata == "" {
		return errors.New("unable to determine current path")
	}
	jFile := filepath.Join("exercises", exercise, "canonical-data.json")
	// try to find and read the local json source file
	log.Printf("[LOCAL] fetching %s test data\n", exercise)
	jPath, jOrigin, jCommit := getLocal(jFile)
	if jPath != "" {
		log.Printf("[LOCAL] source: %s\n", jPath)
	}
	jSrc, err := ioutil.ReadFile(filepath.Join(jPath, jFile))
	if err != nil {
		// fetch json data remotely if there's no local file
		log.Println("[LOCAL] No test data found")
		log.Printf("[REMOTE] fetching %s test data\n", exercise)
		jSrc, jOrigin, jCommit, err = getRemote(exercise)
		if err != nil {
			return err
		}
	}

	// unmarshal the json source to a Go structure
	if err = json.Unmarshal(jSrc, j); err != nil {
		// This error message is usually enough if the problem is a wrong
		// data structure defined here. Sadly it doesn't locate the error well
		// in the case of invalid JSON.  Use a real validator tool if you can't
		// spot the problem right away.
		return fmt.Errorf(`unexpected data structure: %v`, err)
	}

	// These fields are guaranteed to be in every problem
	var commonMetadata struct {
		Version string
	}
	if err := json.Unmarshal(jSrc, &commonMetadata); err != nil {
		return fmt.Errorf(`didn't contain version: %v`, err)
	}

	// package up a little meta data
	d := struct {
		Header
		J interface{}
	}{Header{
		Ori:     jOrigin,
		Origin:  jOrigin,
		Commit:  jCommit,
		Version: commonMetadata.Version,
	}, j}

	// render the Go test cases
	var b bytes.Buffer
	if err := t.Execute(&b, &d); err != nil {
		return err
	}
	// clean it up
	src, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}
	// write output file for the Go test cases.
	return ioutil.WriteFile(filepath.Join(dirExercise, "cases_test.go"), src, 0666)
}

func getLocal(jFile string) (jPath, jOrigin, jCommit string) {
	// Ideally draw from a .json which is pulled from the official x-common
	// repository.  For development however, accept a file in current directory
	// if there is no .json in source control.  Also allow an override in any
	// case by environment variable.
	if jPath := os.Getenv("EXTEST"); jPath > "" {
		return jPath, "local file", "" // override
	}
	c := exec.Command("git", "log", "-1", "--oneline", jFile)
	c.Dir = dirMetadata
	origin, err := c.Output()
	if err != nil {
		return "", "local file", "" // no source control
	}
	if _, err = os.Stat(filepath.Join(c.Dir, jFile)); err != nil {
		return "", "local file", "" // not in source control
	}
	// good.  return source control dir and commit.
	return c.Dir, "exercism/x-common", string(bytes.TrimSpace(origin))
}

func getRemote(exercise string) (body []byte, jOrigin string, jCommit string, err error) {
	url := fmt.Sprintf(canonicalDataURL, exercise)
	resp, err := genClient.Get(url)
	if err != nil {
		return []byte{}, "", "", err
	}
	if resp.StatusCode != http.StatusOK {
		return []byte{}, "", "", fmt.Errorf("error fetching remote data: %s", resp.Status)
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, "", "", err
	}
	c, err := getRemoteCommit(exercise)
	if err != nil {
		// we always expect to have the commit in the cases_test.go
		// file, so return the error if we can't fetch it
		return []byte{}, "", "", err
	}
	log.Printf("[REMOTE] source: %s\n", url)
	return body, "exercism/x-common", c, nil
}

func getRemoteCommit(exercise string) (string, error) {
	type Commits struct {
		Sha    string
		Commit struct {
			Message string
		}
	}
	resp, err := genClient.Get(fmt.Sprintf(commitsURL, exercise))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	var c []Commits
	err = json.NewDecoder(resp.Body).Decode(&c)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s %s", c[0].Sha[0:7], c[0].Commit.Message), nil
}
<|endoftext|>


<commit_before>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"text/template"
)

// dirMetadata is the location of the x-common repository
// on the filesystem.
// We're making the assumption that the x-common repository
// has been cloned to the same parent directory as the xgo
// repository. E.g.
//
//     $ tree -L 1 .
//     .
//     ├── x-common
//     └── xgo
var dirMetadata string

// dirProblem is the location that the test cases should be generated to.
// This assumes that the generator script lives in the same directory as
// the problem.
// Falls back to the present working directory.
var dirProblem string

func init() {
	if _, path, _, ok := runtime.Caller(0); ok {
		dirMetadata = filepath.Join(path, "..", "..", "..", "x-common")
	}
	if _, path, _, ok := runtime.Caller(2); ok {
		dirProblem = filepath.Join(path, "..")
	}
	if dirProblem == "" {
		dirProblem = "."
	}
}

func Gen(exercise string, j interface{}, t *template.Template) error {
	if dirMetadata == "" {
		return errors.New("unable to determine current path")
	}
	jFile := filepath.Join("exercises", exercise, "canonical-data.json")
	// find and read the json source file
	jPath, jOri, jCommit := getPath(jFile)
	jSrc, err := ioutil.ReadFile(filepath.Join(jPath, jFile))
	if err != nil {
		return err
	}

	// unmarshal the json source to a Go structure
	if err = json.Unmarshal(jSrc, j); err != nil {
		// This error message is usually enough if the problem is a wrong
		// data structure defined here. Sadly it doesn't locate the error well
		// in the case of invalid JSON.  Use a real validator tool if you can't
		// spot the problem right away.
		return fmt.Errorf(`unexpected data structure: %v`, err)
	}

	// package up a little meta data
	d := struct {
		Ori    string
		Commit string
		J      interface{}
	}{jOri, jCommit, j}

	// render the Go test cases
	var b bytes.Buffer
	if err = t.Execute(&b, &d); err != nil {
		return err
	}
	// clean it up
	src, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}
	// write output file for the Go test cases.
	return ioutil.WriteFile(filepath.Join(dirProblem, "cases_test.go"), src, 0777)
}

func getPath(jFile string) (jPath, jOri, jCommit string) {
	// Ideally draw from a .json which is pulled from the official x-common
	// repository.  For development however, accept a file in current directory
	// if there is no .json in source control.  Also allow an override in any
	// case by environment variable.
	if jPath = os.Getenv("EXTEST"); jPath > "" {
		return jPath, "local file", "" // override
	}
	c := exec.Command("git", "log", "-1", "--oneline")
	c.Dir = dirMetadata
	ori, err := c.Output()
	if err != nil {
		return "", "local file", "" // no source control
	}
	if _, err = os.Stat(filepath.Join(c.Dir, jFile)); err != nil {
		return "", "local file", "" // not in source control
	}
	// good.  return source control dir and commit.
	return c.Dir, "exercism/x-common", string(bytes.TrimSpace(ori))
}
<commit_msg>gen.go: add filename to git log<commit_after>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"text/template"
)

// dirMetadata is the location of the x-common repository
// on the filesystem.
// We're making the assumption that the x-common repository
// has been cloned to the same parent directory as the xgo
// repository. E.g.
//
//     $ tree -L 1 .
//     .
//     ├── x-common
//     └── xgo
var dirMetadata string

// dirProblem is the location that the test cases should be generated to.
// This assumes that the generator script lives in the same directory as
// the problem.
// Falls back to the present working directory.
var dirProblem string

func init() {
	if _, path, _, ok := runtime.Caller(0); ok {
		dirMetadata = filepath.Join(path, "..", "..", "..", "x-common")
	}
	if _, path, _, ok := runtime.Caller(2); ok {
		dirProblem = filepath.Join(path, "..")
	}
	if dirProblem == "" {
		dirProblem = "."
	}
}

func Gen(exercise string, j interface{}, t *template.Template) error {
	if dirMetadata == "" {
		return errors.New("unable to determine current path")
	}
	jFile := filepath.Join("exercises", exercise, "canonical-data.json")
	// find and read the json source file
	jPath, jOri, jCommit := getPath(jFile)
	jSrc, err := ioutil.ReadFile(filepath.Join(jPath, jFile))
	if err != nil {
		return err
	}

	// unmarshal the json source to a Go structure
	if err = json.Unmarshal(jSrc, j); err != nil {
		// This error message is usually enough if the problem is a wrong
		// data structure defined here. Sadly it doesn't locate the error well
		// in the case of invalid JSON.  Use a real validator tool if you can't
		// spot the problem right away.
		return fmt.Errorf(`unexpected data structure: %v`, err)
	}

	// package up a little meta data
	d := struct {
		Ori    string
		Commit string
		J      interface{}
	}{jOri, jCommit, j}

	// render the Go test cases
	var b bytes.Buffer
	if err = t.Execute(&b, &d); err != nil {
		return err
	}
	// clean it up
	src, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}
	// write output file for the Go test cases.
	return ioutil.WriteFile(filepath.Join(dirProblem, "cases_test.go"), src, 0777)
}

func getPath(jFile string) (jPath, jOri, jCommit string) {
	// Ideally draw from a .json which is pulled from the official x-common
	// repository.  For development however, accept a file in current directory
	// if there is no .json in source control.  Also allow an override in any
	// case by environment variable.
	if jPath = os.Getenv("EXTEST"); jPath > "" {
		return jPath, "local file", "" // override
	}
	c := exec.Command("git", "log", "-1", "--oneline", jFile)
	c.Dir = dirMetadata
	ori, err := c.Output()
	if err != nil {
		return "", "local file", "" // no source control
	}
	if _, err = os.Stat(filepath.Join(c.Dir, jFile)); err != nil {
		return "", "local file", "" // not in source control
	}
	// good.  return source control dir and commit.
	return c.Dir, "exercism/x-common", string(bytes.TrimSpace(ori))
}
<|endoftext|>


<commit_before>// Code generated by protoc-gen-gogo.
// source: envelope.proto
// DO NOT EDIT!

/*
Package events is a generated protocol buffer package.

It is generated from these files:
	envelope.proto
	error.proto
	heartbeat.proto
	http.proto
	log.proto
	metric.proto
	uuid.proto

It has these top-level messages:
	Envelope
*/
package events

import proto "github.com/gogo/protobuf/proto"
import math "math"

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf

// / Type of the wrapped event.
type Envelope_EventType int32

const (
	Envelope_Heartbeat       Envelope_EventType = 1
	Envelope_HttpStart       Envelope_EventType = 2
	Envelope_HttpStop        Envelope_EventType = 3
	Envelope_HttpStartStop   Envelope_EventType = 4
	Envelope_LogMessage      Envelope_EventType = 5
	Envelope_ValueMetric     Envelope_EventType = 6
	Envelope_CounterEvent    Envelope_EventType = 7
	Envelope_Error           Envelope_EventType = 8
	Envelope_ContainerMetric Envelope_EventType = 9
)

var Envelope_EventType_name = map[int32]string{
	1: "Heartbeat",
	2: "HttpStart",
	3: "HttpStop",
	4: "HttpStartStop",
	5: "LogMessage",
	6: "ValueMetric",
	7: "CounterEvent",
	8: "Error",
	9: "ContainerMetric",
}
var Envelope_EventType_value = map[string]int32{
	"Heartbeat":       1,
	"HttpStart":       2,
	"HttpStop":        3,
	"HttpStartStop":   4,
	"LogMessage":      5,
	"ValueMetric":     6,
	"CounterEvent":    7,
	"Error":           8,
	"ContainerMetric": 9,
}

func (x Envelope_EventType) Enum() *Envelope_EventType {
	p := new(Envelope_EventType)
	*p = x
	return p
}
func (x Envelope_EventType) String() string {
	return proto.EnumName(Envelope_EventType_name, int32(x))
}
func (x *Envelope_EventType) UnmarshalJSON(data []byte) error {
	value, err := proto.UnmarshalJSONEnum(Envelope_EventType_value, data, "Envelope_EventType")
	if err != nil {
		return err
	}
	*x = Envelope_EventType(value)
	return nil
}

// / Envelope wraps an Event and adds metadata.
type Envelope struct {
	Origin           *string             `protobuf:"bytes,1,req,name=origin" json:"origin,omitempty"`
	EventType        *Envelope_EventType `protobuf:"varint,2,req,name=eventType,enum=events.Envelope_EventType" json:"eventType,omitempty"`
	Timestamp        *int64              `protobuf:"varint,6,opt,name=timestamp" json:"timestamp,omitempty"`
	Heartbeat        *Heartbeat          `protobuf:"bytes,3,opt,name=heartbeat" json:"heartbeat,omitempty"`
	HttpStart        *HttpStart          `protobuf:"bytes,4,opt,name=httpStart" json:"httpStart,omitempty"`
	HttpStop         *HttpStop           `protobuf:"bytes,5,opt,name=httpStop" json:"httpStop,omitempty"`
	HttpStartStop    *HttpStartStop      `protobuf:"bytes,7,opt,name=httpStartStop" json:"httpStartStop,omitempty"`
	LogMessage       *LogMessage         `protobuf:"bytes,8,opt,name=logMessage" json:"logMessage,omitempty"`
	ValueMetric      *ValueMetric        `protobuf:"bytes,9,opt,name=valueMetric" json:"valueMetric,omitempty"`
	CounterEvent     *CounterEvent       `protobuf:"bytes,10,opt,name=counterEvent" json:"counterEvent,omitempty"`
	Error            *Error              `protobuf:"bytes,11,opt,name=error" json:"error,omitempty"`
	ContainerMetric  *ContainerMetric    `protobuf:"bytes,12,opt,name=containerMetric" json:"containerMetric,omitempty"`
	XXX_unrecognized []byte              `json:"-"`
}

func (m *Envelope) Reset()         { *m = Envelope{} }
func (m *Envelope) String() string { return proto.CompactTextString(m) }
func (*Envelope) ProtoMessage()    {}

func (m *Envelope) GetOrigin() string {
	if m != nil && m.Origin != nil {
		return *m.Origin
	}
	return ""
}

func (m *Envelope) GetEventType() Envelope_EventType {
	if m != nil && m.EventType != nil {
		return *m.EventType
	}
	return Envelope_Heartbeat
}

func (m *Envelope) GetTimestamp() int64 {
	if m != nil && m.Timestamp != nil {
		return *m.Timestamp
	}
	return 0
}

func (m *Envelope) GetHeartbeat() *Heartbeat {
	if m != nil {
		return m.Heartbeat
	}
	return nil
}

func (m *Envelope) GetHttpStart() *HttpStart {
	if m != nil {
		return m.HttpStart
	}
	return nil
}

func (m *Envelope) GetHttpStop() *HttpStop {
	if m != nil {
		return m.HttpStop
	}
	return nil
}

func (m *Envelope) GetHttpStartStop() *HttpStartStop {
	if m != nil {
		return m.HttpStartStop
	}
	return nil
}

func (m *Envelope) GetLogMessage() *LogMessage {
	if m != nil {
		return m.LogMessage
	}
	return nil
}

func (m *Envelope) GetValueMetric() *ValueMetric {
	if m != nil {
		return m.ValueMetric
	}
	return nil
}

func (m *Envelope) GetCounterEvent() *CounterEvent {
	if m != nil {
		return m.CounterEvent
	}
	return nil
}

func (m *Envelope) GetError() *Error {
	if m != nil {
		return m.Error
	}
	return nil
}

func (m *Envelope) GetContainerMetric() *ContainerMetric {
	if m != nil {
		return m.ContainerMetric
	}
	return nil
}

func init() {
	proto.RegisterEnum("events.Envelope_EventType", Envelope_EventType_name, Envelope_EventType_value)
}
<commit_msg>Update to latest version of dropsonde-protocol<commit_after>// Code generated by protoc-gen-gogo.
// source: envelope.proto
// DO NOT EDIT!

/*
Package events is a generated protocol buffer package.

It is generated from these files:
	envelope.proto
	error.proto
	heartbeat.proto
	http.proto
	log.proto
	metric.proto
	uuid.proto

It has these top-level messages:
	Envelope
	Tag
*/
package events

import proto "github.com/gogo/protobuf/proto"
import math "math"

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf

// / Type of the wrapped event.
type Envelope_EventType int32

const (
	Envelope_Heartbeat       Envelope_EventType = 1
	Envelope_HttpStart       Envelope_EventType = 2
	Envelope_HttpStop        Envelope_EventType = 3
	Envelope_HttpStartStop   Envelope_EventType = 4
	Envelope_LogMessage      Envelope_EventType = 5
	Envelope_ValueMetric     Envelope_EventType = 6
	Envelope_CounterEvent    Envelope_EventType = 7
	Envelope_Error           Envelope_EventType = 8
	Envelope_ContainerMetric Envelope_EventType = 9
)

var Envelope_EventType_name = map[int32]string{
	1: "Heartbeat",
	2: "HttpStart",
	3: "HttpStop",
	4: "HttpStartStop",
	5: "LogMessage",
	6: "ValueMetric",
	7: "CounterEvent",
	8: "Error",
	9: "ContainerMetric",
}
var Envelope_EventType_value = map[string]int32{
	"Heartbeat":       1,
	"HttpStart":       2,
	"HttpStop":        3,
	"HttpStartStop":   4,
	"LogMessage":      5,
	"ValueMetric":     6,
	"CounterEvent":    7,
	"Error":           8,
	"ContainerMetric": 9,
}

func (x Envelope_EventType) Enum() *Envelope_EventType {
	p := new(Envelope_EventType)
	*p = x
	return p
}
func (x Envelope_EventType) String() string {
	return proto.EnumName(Envelope_EventType_name, int32(x))
}
func (x *Envelope_EventType) UnmarshalJSON(data []byte) error {
	value, err := proto.UnmarshalJSONEnum(Envelope_EventType_value, data, "Envelope_EventType")
	if err != nil {
		return err
	}
	*x = Envelope_EventType(value)
	return nil
}

// / Envelope wraps an Event and adds metadata.
type Envelope struct {
	Origin           *string             `protobuf:"bytes,1,req,name=origin" json:"origin,omitempty"`
	EventType        *Envelope_EventType `protobuf:"varint,2,req,name=eventType,enum=events.Envelope_EventType" json:"eventType,omitempty"`
	Timestamp        *int64              `protobuf:"varint,6,opt,name=timestamp" json:"timestamp,omitempty"`
	Heartbeat        *Heartbeat          `protobuf:"bytes,3,opt,name=heartbeat" json:"heartbeat,omitempty"`
	HttpStart        *HttpStart          `protobuf:"bytes,4,opt,name=httpStart" json:"httpStart,omitempty"`
	HttpStop         *HttpStop           `protobuf:"bytes,5,opt,name=httpStop" json:"httpStop,omitempty"`
	HttpStartStop    *HttpStartStop      `protobuf:"bytes,7,opt,name=httpStartStop" json:"httpStartStop,omitempty"`
	LogMessage       *LogMessage         `protobuf:"bytes,8,opt,name=logMessage" json:"logMessage,omitempty"`
	ValueMetric      *ValueMetric        `protobuf:"bytes,9,opt,name=valueMetric" json:"valueMetric,omitempty"`
	CounterEvent     *CounterEvent       `protobuf:"bytes,10,opt,name=counterEvent" json:"counterEvent,omitempty"`
	Error            *Error              `protobuf:"bytes,11,opt,name=error" json:"error,omitempty"`
	ContainerMetric  *ContainerMetric    `protobuf:"bytes,12,opt,name=containerMetric" json:"containerMetric,omitempty"`
	Tags             []*Tag              `protobuf:"bytes,13,rep,name=tags" json:"tags,omitempty"`
	XXX_unrecognized []byte              `json:"-"`
}

func (m *Envelope) Reset()         { *m = Envelope{} }
func (m *Envelope) String() string { return proto.CompactTextString(m) }
func (*Envelope) ProtoMessage()    {}

func (m *Envelope) GetOrigin() string {
	if m != nil && m.Origin != nil {
		return *m.Origin
	}
	return ""
}

func (m *Envelope) GetEventType() Envelope_EventType {
	if m != nil && m.EventType != nil {
		return *m.EventType
	}
	return Envelope_Heartbeat
}

func (m *Envelope) GetTimestamp() int64 {
	if m != nil && m.Timestamp != nil {
		return *m.Timestamp
	}
	return 0
}

func (m *Envelope) GetHeartbeat() *Heartbeat {
	if m != nil {
		return m.Heartbeat
	}
	return nil
}

func (m *Envelope) GetHttpStart() *HttpStart {
	if m != nil {
		return m.HttpStart
	}
	return nil
}

func (m *Envelope) GetHttpStop() *HttpStop {
	if m != nil {
		return m.HttpStop
	}
	return nil
}

func (m *Envelope) GetHttpStartStop() *HttpStartStop {
	if m != nil {
		return m.HttpStartStop
	}
	return nil
}

func (m *Envelope) GetLogMessage() *LogMessage {
	if m != nil {
		return m.LogMessage
	}
	return nil
}

func (m *Envelope) GetValueMetric() *ValueMetric {
	if m != nil {
		return m.ValueMetric
	}
	return nil
}

func (m *Envelope) GetCounterEvent() *CounterEvent {
	if m != nil {
		return m.CounterEvent
	}
	return nil
}

func (m *Envelope) GetError() *Error {
	if m != nil {
		return m.Error
	}
	return nil
}

func (m *Envelope) GetContainerMetric() *ContainerMetric {
	if m != nil {
		return m.ContainerMetric
	}
	return nil
}

func (m *Envelope) GetTags() []*Tag {
	if m != nil {
		return m.Tags
	}
	return nil
}

// / Tag holds a key-value pair
type Tag struct {
	Key              *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"`
	Value            *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *Tag) Reset()         { *m = Tag{} }
func (m *Tag) String() string { return proto.CompactTextString(m) }
func (*Tag) ProtoMessage()    {}

func (m *Tag) GetKey() string {
	if m != nil && m.Key != nil {
		return *m.Key
	}
	return ""
}

func (m *Tag) GetValue() string {
	if m != nil && m.Value != nil {
		return *m.Value
	}
	return ""
}

func init() {
	proto.RegisterEnum("events.Envelope_EventType", Envelope_EventType_name, Envelope_EventType_value)
}
<|endoftext|>


<commit_before>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferpanel/v2"
	"github.com/pufferpanel/pufferpanel/v2/panel/models"
	"github.com/pufferpanel/pufferpanel/v2/panel/services"
	"github.com/pufferpanel/pufferpanel/v2/panel/web/handlers"
	"github.com/pufferpanel/pufferpanel/v2/response"
	"github.com/pufferpanel/pufferpanel/v2/scope"
	uuid "github.com/satori/go.uuid"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

func registerNodes(g *gin.RouterGroup) {
	g.Handle("GET", "", handlers.OAuth2Handler(scope.NodesView, false), getAllNodes)
	g.Handle("OPTIONS", "", response.CreateOptions("GET"))
	g.Handle("POST", "", handlers.OAuth2Handler(scope.NodesEdit, false), createNode)

	g.Handle("GET", "/:id", handlers.OAuth2Handler(scope.NodesView, false), getNode)
	g.Handle("PUT", "/:id", handlers.OAuth2Handler(scope.NodesEdit, false), updateNode)
	g.Handle("DELETE", "/:id", handlers.OAuth2Handler(scope.NodesEdit, false), deleteNode)
	g.Handle("OPTIONS", "/:id", response.CreateOptions("PUT", "GET", "POST", "DELETE"))

	g.Handle("GET", "/:id/deployment", handlers.OAuth2Handler(scope.NodesDeploy, false), deployNode)
	g.Handle("OPTIONS", "/:id/deployment", response.CreateOptions("GET"))

	//g.Handle("POST", "/:id/reset", handlers.OAuth2(scope.NodesDeploy, false), response.NotImplemented)
	//g.Handle("OPTIONS", "/:id/reset", response.CreateOptions("POST"))
}

// @Summary Get nodes
// @Description Gets all nodes registered to the panel
// @Accept json
// @Produce json
// @Success 200 {object} models.NodesView "Nodes"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Router /nodes [get]
func getAllNodes(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	var nodes *models.Nodes
	if nodes, err = ns.GetAll(); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := models.FromNodes(nodes)

	c.JSON(http.StatusOK, data)
}

// @Summary Get node
// @Description Gets information about a single node
// @Accept json
// @Produce json
// @Success 200 {object} models.NodeView "Nodes"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Router /nodes/{id} [get]
func getNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := models.FromNode(node)

	c.JSON(http.StatusOK, data)
}

// @Summary Create node
// @Description Creates a node
// @Accept json
// @Produce json
// @Success 200 {object} models.NodeView "Node created"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Identifier"
// @Router /nodes [post]
func createNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	model := &models.NodeView{}
	if err = c.BindJSON(model); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if err = model.Valid(false); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	create := &models.Node{}
	model.CopyToModel(create)
	create.Secret = strings.Replace(uuid.NewV4().String(), "-", "", -1)
	if err = ns.Create(create); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, create)
}

// @Summary Update node
// @Description Updates a node with given information
// @Accept json
// @Produce json
// @Success 204 {object} response.Empty
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Id"
// @Router /nodes/{id} [put]
func updateNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	viewModel := &models.NodeView{}
	if err = c.BindJSON(viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	id, ok := validateId(c)
	if !ok {
		return
	}

	if err = viewModel.Valid(true); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	viewModel.CopyToModel(node)
	if err = ns.Update(node); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

// @Summary Deletes a node
// @Description Deletes the node
// @Accept json
// @Produce json
// @Success 204 {object} response.Empty
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Id"
// @Router /nodes/{id} [delete]
func deleteNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	err = ns.Delete(node.ID)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func deployNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	services.ValidateTokenLoaded()
	file, err := ioutil.ReadFile(viper.GetString("panel.token.public"))
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := &models.Deployment{
		ClientId:     fmt.Sprintf(".node_%d", node.ID),
		ClientSecret: node.Secret,
		PublicKey:    string(file),
	}

	c.JSON(http.StatusOK, data)
}

func validateId(c *gin.Context) (uint, bool) {
	param := c.Param("id")

	id, err := strconv.Atoi(param)

	if response.HandleError(c, err, http.StatusBadRequest) || id <= 0 {
		response.HandleError(c, pufferpanel.ErrFieldTooSmall("id", 0), http.StatusBadRequest)
		return 0, false
	}

	return uint(id), true
}
<commit_msg>Add localhost workaround for displaying SFTP info [skip]<commit_after>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferpanel/v2"
	"github.com/pufferpanel/pufferpanel/v2/panel/models"
	"github.com/pufferpanel/pufferpanel/v2/panel/services"
	"github.com/pufferpanel/pufferpanel/v2/panel/web/handlers"
	"github.com/pufferpanel/pufferpanel/v2/response"
	"github.com/pufferpanel/pufferpanel/v2/scope"
	uuid "github.com/satori/go.uuid"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

func registerNodes(g *gin.RouterGroup) {
	g.Handle("GET", "", handlers.OAuth2Handler(scope.NodesView, false), getAllNodes)
	g.Handle("OPTIONS", "", response.CreateOptions("GET"))
	g.Handle("POST", "", handlers.OAuth2Handler(scope.NodesEdit, false), createNode)

	g.Handle("GET", "/:id", handlers.OAuth2Handler(scope.NodesView, false), getNode)
	g.Handle("PUT", "/:id", handlers.OAuth2Handler(scope.NodesEdit, false), updateNode)
	g.Handle("DELETE", "/:id", handlers.OAuth2Handler(scope.NodesEdit, false), deleteNode)
	g.Handle("OPTIONS", "/:id", response.CreateOptions("PUT", "GET", "POST", "DELETE"))

	g.Handle("GET", "/:id/deployment", handlers.OAuth2Handler(scope.NodesDeploy, false), deployNode)
	g.Handle("OPTIONS", "/:id/deployment", response.CreateOptions("GET"))

	//g.Handle("POST", "/:id/reset", handlers.OAuth2(scope.NodesDeploy, false), response.NotImplemented)
	//g.Handle("OPTIONS", "/:id/reset", response.CreateOptions("POST"))
}

// @Summary Get nodes
// @Description Gets all nodes registered to the panel
// @Accept json
// @Produce json
// @Success 200 {object} models.NodesView "Nodes"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Router /nodes [get]
func getAllNodes(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	var nodes *models.Nodes
	if nodes, err = ns.GetAll(); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := models.FromNodes(nodes)

	//HACK: For our local node, we actually need to override the public IP
	for _, d := range *data {
		if d.PrivateHost == "127.0.0.1" && d.PublicHost == "127.0.0.1" {
			d.PublicHost = c.Request.Host
			break
		}
	}

	c.JSON(http.StatusOK, data)
}

// @Summary Get node
// @Description Gets information about a single node
// @Accept json
// @Produce json
// @Success 200 {object} models.NodeView "Nodes"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Router /nodes/{id} [get]
func getNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := models.FromNode(node)

	c.JSON(http.StatusOK, data)
}

// @Summary Create node
// @Description Creates a node
// @Accept json
// @Produce json
// @Success 200 {object} models.NodeView "Node created"
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Identifier"
// @Router /nodes [post]
func createNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	model := &models.NodeView{}
	if err = c.BindJSON(model); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if err = model.Valid(false); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	create := &models.Node{}
	model.CopyToModel(create)
	create.Secret = strings.Replace(uuid.NewV4().String(), "-", "", -1)
	if err = ns.Create(create); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, create)
}

// @Summary Update node
// @Description Updates a node with given information
// @Accept json
// @Produce json
// @Success 204 {object} response.Empty
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Id"
// @Router /nodes/{id} [put]
func updateNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	viewModel := &models.NodeView{}
	if err = c.BindJSON(viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	id, ok := validateId(c)
	if !ok {
		return
	}

	if err = viewModel.Valid(true); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	viewModel.CopyToModel(node)
	if err = ns.Update(node); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

// @Summary Deletes a node
// @Description Deletes the node
// @Accept json
// @Produce json
// @Success 204 {object} response.Empty
// @Failure 400 {object} response.Error
// @Failure 403 {object} response.Error
// @Failure 404 {object} response.Error
// @Failure 500 {object} response.Error
// @Param id path string true "Node Id"
// @Router /nodes/{id} [delete]
func deleteNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	err = ns.Delete(node.ID)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func deployNode(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	ns := &services.Node{DB: db}

	id, ok := validateId(c)
	if !ok {
		return
	}

	node, err := ns.Get(id)
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	services.ValidateTokenLoaded()
	file, err := ioutil.ReadFile(viper.GetString("panel.token.public"))
	if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	data := &models.Deployment{
		ClientId:     fmt.Sprintf(".node_%d", node.ID),
		ClientSecret: node.Secret,
		PublicKey:    string(file),
	}

	c.JSON(http.StatusOK, data)
}

func validateId(c *gin.Context) (uint, bool) {
	param := c.Param("id")

	id, err := strconv.Atoi(param)

	if response.HandleError(c, err, http.StatusBadRequest) || id <= 0 {
		response.HandleError(c, pufferpanel.ErrFieldTooSmall("id", 0), http.StatusBadRequest)
		return 0, false
	}

	return uint(id), true
}
<|endoftext|>


<commit_before>package vfsswift

import (
	"errors"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

// maxNbFilesToDelete is the maximal number of files that we will try to delete
// in a single call to swift.
const maxNbFilesToDelete = 8000

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift to
// delete files in the same container.
const maxSimultaneousCalls = 8

var errFailFast = errors.New("fail fast")

// DeleteContainer removes all the files inside the given container, and then
// deletes it.
func DeleteContainer(c *swift.Connection, container string) error {
	_, _, err := c.Container(container)
	if err == swift.ContainerNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	objectNames, err := c.ObjectNamesAll(container, nil)
	if err != nil {
		return err
	}
	if len(objectNames) > 0 {
		if err = deleteContainerFiles(c, container, objectNames); err != nil {
			return err
		}
	}
	return c.ContainerDelete(container)
}

func deleteContainerFiles(c *swift.Connection, container string, objectNames []string) error {
	nb := 1 + (len(objectNames)-1)/maxNbFilesToDelete
	ch := make(chan error)

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	for i := 0; i < nb; i++ {
		begin := i * maxNbFilesToDelete
		end := (i + 1) * maxNbFilesToDelete
		if end > len(objectNames) {
			end = len(objectNames)
		}
		objectToDelete := objectNames[begin:end]
		go func() {
			k := <-tokens
			_, err := c.BulkDelete(container, objectToDelete)
			ch <- err
			tokens <- k
		}()
	}

	var errm error
	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}
<commit_msg>Improve the Swift container deletion (#2143)<commit_after>package vfsswift

import (
	"errors"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

// maxNbFilesToDelete is the maximal number of files that we will try to delete
// in a single call to swift.
const maxNbFilesToDelete = 8000

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift to
// delete files in the same container.
const maxSimultaneousCalls = 8

var errFailFast = errors.New("fail fast")

// DeleteContainer removes all the files inside the given container, and then
// deletes it.
func DeleteContainer(c *swift.Connection, container string) error {
	_, _, err := c.Container(container)
	if err == swift.ContainerNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	objectNames, err := c.ObjectNamesAll(container, nil)
	if err != nil {
		return err
	}
	if len(objectNames) > 0 {
		if err = deleteContainerFiles(c, container, objectNames); err != nil {
			return err
		}
	}

	// XXX Swift has told us that all the files have been deleted on the bulk
	// delete, but it only means that they have been deleted on one object
	// server (at least). And, when we try to delete the container, Swift can
	// send an error as some container servers will still have objects
	// registered for this container. We will try several times to delete the
	// container to work-around this limitation.
	for i := 0; i < 5; i++ {
		err = c.ContainerDelete(container)
		if err == nil {
			return nil
		}
		time.Sleep(5 * time.Second)
	}
	return err
}

func deleteContainerFiles(c *swift.Connection, container string, objectNames []string) error {
	nb := 1 + (len(objectNames)-1)/maxNbFilesToDelete
	ch := make(chan error)

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	for i := 0; i < nb; i++ {
		begin := i * maxNbFilesToDelete
		end := (i + 1) * maxNbFilesToDelete
		if end > len(objectNames) {
			end = len(objectNames)
		}
		objectToDelete := objectNames[begin:end]
		go func() {
			k := <-tokens
			_, err := c.BulkDelete(container, objectToDelete)
			ch <- err
			tokens <- k
		}()
	}

	var errm error
	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}
<|endoftext|>


<commit_before><commit_msg>Only use ca cert if readable<commit_after><|endoftext|>


<commit_before><commit_msg>first commit<commit_after><|endoftext|>


<commit_before><commit_msg>Clean after your self, remove container at the end of the run<commit_after><|endoftext|>


<commit_before><commit_msg>All major bugs fixed. Still needs tweaks<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strings"
)

type line struct {
	ip, hostname string
}

const filename = "/etc/hosts"

type hostlist struct {
	lines []string
}

func (hl *hostlist) Read(fn string) error {
	b, err := ioutil.ReadFile(fn)
	if err != nil {
		return err
	}

	hl.Parse(b)
	return nil
}

func (hl *hostlist) Parse(b []byte) {
	hl.lines = strings.Split(string(b), "\n")
}

func (hl *hostlist) Write(fn string) error {
	return ioutil.WriteFile(fn, hl.Bytes(), 0644)
}

func (hl *hostlist) Bytes() []byte {
	return []byte(strings.Join(hl.lines, "\n"))

func (hl *hostlist) Contains(a, b string) (bool, error) {
	var ip, hostname string

	if net.ParseIP(a) == nil && net.ParseIP(b) == nil {
		return false, fmt.Errorf("neither %s or %s is a valid IP address", a, b)
	}

	if net.ParseIP(a) == nil {
		hostname = a
		ip = b
	}

	for _, line := range hl.lines {
		if line == fmt.Sprintf("%s\t%s", ip, hostname) {
			return true, nil
		}
	}

	return false, nil
}

func (hl *hostlist) Add(a, b string) error {
	var ip, hostname string

	if net.ParseIP(a) == nil && net.ParseIP(b) == nil {
		return fmt.Errorf("neither %s or %s is a valid IP address", a, b)
	}

	if net.ParseIP(a) == nil {
		hostname = a
		ip = b
	}

	hl.lines = append(hl.lines, fmt.Sprintf("%s\t%s", ip, hostname))
	return nil
}

func containsPart(haystack, needle string) bool {
	return strings.Contains(haystack, "\t"+needle) || strings.Contains(haystack, needle+"\t")
}

func reverse(a []int) []int {
	for i := len(a)/2 - 1; i >= 0; i-- {
		opp := len(a) - 1 - i
		a[i], a[opp] = a[opp], a[i]
	}

	return a
}

func (hl *hostlist) Remove(thing string) error {
	deletes := []int{}
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			deletes = append(deletes, i)
		}
	}

	for _, i := range reverse(deletes) {
		hl.lines = append(hl.lines[:i], hl.lines[i+1:]...)
	}

	return nil
}

func (hl *hostlist) Comment(thing string) error {
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			hl.lines[i] = "#" + line
		}
	}

	return nil
}

func (hl *hostlist) Uncomment(thing string) error {
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			hl.lines[i] = strings.TrimLeft(line, "#")
		}
	}

	return nil
}

func iAmRoot() bool {
	cmd := exec.Command("whoami")
	user, err := cmd.Output()
	if err != nil {
		// couldn't determine root due to error, run anyway - the user won't be able
		// to mod anything without root rights anyway
		return true
	}

	return strings.TrimSpace(string(user)) == "root"
}

func init() {

}

func main() {
	log.SetFlags(0)

	if !iAmRoot() {
		log.Fatal("Please run this program as Root!")
	}

	if len(os.Args) <= 1 {
		log.Fatal("Nothing to do, please specify command")
	}

	hosts := hostlist{}
	hosts.Read(filename)

	command := string(os.Args[1])

	// fmt.Println(command)
	switch command {
	default:
		log.Fatalf("Unknown command: %s", command)

	case "list", "ls":
		fmt.Println(string(hosts.Bytes()))

	case "del", "rm", "-":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to delete")
		}
		hosts.Remove(os.Args[2])

	case "ucom":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to uncomment")
		}
		hosts.Uncomment(os.Args[2])

	case "com":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to comment out")
		}
		hosts.Comment(os.Args[2])

	case "add", "+":
		if len(os.Args) != 4 {
			log.Fatal("Please give arguments in the form ip, hostname")
		}
		err := hosts.Add(os.Args[2], os.Args[3])
		if err != nil {
			log.Fatal(err)
		}

	case "has", "?", "contains":
		if len(os.Args) != 4 {
			log.Fatal("Please give arguments in the form ip, hostname")
		}

		yes, err := hosts.Contains(os.Args[2], os.Args[3])
		if err != nil {
			log.Fatal(err)
		}

		if yes {
			os.Exit(0) // exit code 0 means it was contained within
		}

		os.Exit(1) // exit code 1 means not contained within

	}

	hosts.Write(filename)
}
<commit_msg>move util functions to the top<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strings"
)

type line struct {
	ip, hostname string
}

const filename = "/etc/hosts"

func containsPart(haystack, needle string) bool {
	return strings.Contains(haystack, "\t"+needle) || strings.Contains(haystack, needle+"\t")
}

func reverse(a []int) []int {
	for i := len(a)/2 - 1; i >= 0; i-- {
		opp := len(a) - 1 - i
		a[i], a[opp] = a[opp], a[i]
	}

	return a
}

func amIRoot() bool {
	cmd := exec.Command("whoami")
	user, err := cmd.Output()
	if err != nil {
		// couldn't determine root due to error, run anyway - the user won't be able
		// to mod anything without root rights anyway
		return true
	}

	return strings.TrimSpace(string(user)) == "root"
}

type hostlist struct {
	lines []string
}

func (hl *hostlist) Read(fn string) error {
	b, err := ioutil.ReadFile(fn)
	if err != nil {
		return err
	}

	hl.Parse(b)
	return nil
}

func (hl *hostlist) Parse(b []byte) {
	hl.lines = strings.Split(string(b), "\n")
}

func (hl *hostlist) Write(fn string) error {
	return ioutil.WriteFile(fn, hl.Bytes(), 0644)
}

func (hl *hostlist) Bytes() []byte {
	return []byte(strings.Join(hl.lines, "\n"))

func (hl *hostlist) Contains(a, b string) (bool, error) {
	var ip, hostname string

	if net.ParseIP(a) == nil && net.ParseIP(b) == nil {
		return false, fmt.Errorf("neither %s or %s is a valid IP address", a, b)
	}

	if net.ParseIP(a) == nil {
		hostname = a
		ip = b
	}

	for _, line := range hl.lines {
		if line == fmt.Sprintf("%s\t%s", ip, hostname) {
			return true, nil
		}
	}

	return false, nil
}

func (hl *hostlist) Add(a, b string) error {
	var ip, hostname string

	if net.ParseIP(a) == nil && net.ParseIP(b) == nil {
		return fmt.Errorf("neither %s or %s is a valid IP address", a, b)
	}

	if net.ParseIP(a) == nil {
		hostname = a
		ip = b
	}

	hl.lines = append(hl.lines, fmt.Sprintf("%s\t%s", ip, hostname))
	return nil
}

func (hl *hostlist) Remove(thing string) error {
	deletes := []int{}
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			deletes = append(deletes, i)
		}
	}

	for _, i := range reverse(deletes) {
		hl.lines = append(hl.lines[:i], hl.lines[i+1:]...)
	}

	return nil
}

func (hl *hostlist) Comment(thing string) error {
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			hl.lines[i] = "#" + line
		}
	}

	return nil
}

func (hl *hostlist) Uncomment(thing string) error {
	for i, line := range hl.lines {
		if containsPart(line, thing) {
			hl.lines[i] = strings.TrimLeft(line, "#")
		}
	}

	return nil
}

func main() {
	log.SetFlags(0)

	if !amIRoot() {
		log.Fatal("Please run this program as Root!")
	}

	if len(os.Args) <= 1 {
		log.Fatal("Nothing to do, please specify command")
	}

	hosts := hostlist{}
	hosts.Read(filename)

	command := string(os.Args[1])

	// fmt.Println(command)
	switch command {
	default:
		log.Fatalf("Unknown command: %s", command)

	case "list", "ls":
		fmt.Println(string(hosts.Bytes()))

	case "del", "rm", "-":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to delete")
		}
		hosts.Remove(os.Args[2])

	case "ucom":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to uncomment")
		}
		hosts.Uncomment(os.Args[2])

	case "com":
		if len(os.Args) != 3 {
			log.Fatal("Please give an IP or hostname to comment out")
		}
		hosts.Comment(os.Args[2])

	case "add", "+":
		if len(os.Args) != 4 {
			log.Fatal("Please give arguments in the form ip, hostname")
		}
		err := hosts.Add(os.Args[2], os.Args[3])
		if err != nil {
			log.Fatal(err)
		}

	case "has", "?", "contains":
		if len(os.Args) != 4 {
			log.Fatal("Please give arguments in the form ip, hostname")
		}

		yes, err := hosts.Contains(os.Args[2], os.Args[3])
		if err != nil {
			log.Fatal(err)
		}

		if yes {
			os.Exit(0) // exit code 0 means it was contained within
		}

		os.Exit(1) // exit code 1 means not contained within

	}

	hosts.Write(filename)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	cfgPath = kingpin.Flag("config", `path to JSON configuration, defaults to "config.json"`).Short('c').String()
	outFile = kingpin.Flag("out", `path to output file, defaults to "config.go"`).Short('o').String()
	pack    = kingpin.Flag("package", `name of go package containing config code`).Short('p').Default("main").String()

	// Replace the variables below for testing
	fatalf = kingpin.Fatalf
)

func main() {
	kingpin.Parse()
	run()
}

// Make it easy for test to setup configuration values.
func run() {
	wd, err := os.Getwd()
	if err != nil {
		fatalf("failed to retrieve current directory: %s", err)
	}
	if *cfgPath == "" {
		*cfgPath = path.Join(wd, "config.json")
	}
	if *cfgPath, err = filepath.Abs(*cfgPath); err != nil {
		fatalf("failed to compute absolute file path for config: %s", err)
	}
	if *outFile == "" {
		*outFile = path.Join(wd, "config.go")
	}
	if _, err := os.Stat(*cfgPath); os.IsNotExist(err) {
		fatalf("no configuration file at %s", *cfgPath)
	}
	outDir := filepath.Dir(*outFile)
	s, err := os.Stat(outDir)
	if os.IsNotExist(err) {
		kingpin.FatalIfError(os.MkdirAll(outDir, 0777), "output directory")
	} else if err != nil || !s.IsDir() {
		fatalf("not a valid output directory: %s", outDir)
	}
	input, err := os.Open(*cfgPath)
	if err != nil {
		fatalf("failed to open JSON file: %s", err)
	}
	defer input.Close()
	decoder := json.NewDecoder(input)
	decoder.UseNumber()
	var data interface{}
	err = decoder.Decode(&data)
	if err != nil {
		fatalf("failed to unmarshal JSON: %s", err)
	}
	var tree Tree
	tree.Populate(data)
	if tree.Type != Struct {
		fatalf("invalid configuration file content, JSON must define an object")
	}
	tree.Normalize()
	tree.Name = "cfg"
	configs := make([]*Tree, len(tree.Children))
	for i, child := range tree.Children {
		configs[i] = &Tree{
			Name:     child.Name + "Cfg",
			Children: child.Children,
			Type:     child.Type,
			List:     child.List,
		}
	}
	vars := Variables{
		CmdLine: fmt.Sprintf("$ %s %s", os.Args[0], strings.Join(os.Args[1:], " ")),
		Pack:    *pack,
		Tree:    &tree,
		Configs: configs,
	}
	tmpl, err := template.New("gonfig").Funcs(template.FuncMap{"NewVar": NewVar}).Parse(configTemplate)
	if err != nil {
		fatalf("failed to compile template: %s", err)
	}
	outF, err := os.Create(*outFile)
	if err != nil {
		fatalf("failed to create file: %s", err)
	}
	defer outF.Close()
	err = tmpl.Execute(outF, &vars)

	// Print something
	fmt.Println(*outFile)
}

// Variables defines the data structure fed to the template to generate the final code.
type Variables struct {
	CmdLine string  // Command line used to invoke gonfig
	Pack    string  // Name of target package
	Tree    *Tree   // Top level data structure
	Configs []*Tree // Configuration entries
}

// Internal count used to generate unique variable names
var varCount int

// Generate unique Go variable name
func NewVar() string {
	varCount += 1
	return fmt.Sprintf("v%d", varCount)
}

const configTemplate string = `//************************************************************************//
//                     Configuration
//
// Generated with:
// {{.CmdLine}}
//
// The content of this file is auto-generated, DO NOT MODIFY
//************************************************************************//

package {{.Pack}}

import (
	"encoding/json"
        "os"
)

var ({{range .Tree.Children}}
	{{.Name}} {{.TypeName}}{{end}}
)
{{range .Configs}}{{if eq .Type.String "struct"}}
{{.FormatRaw}}
{{end}}{{end}}
{{.Tree.FormatRaw}}

// Load reads the JSON at the given path and initializes the package variables with the
// corresponding values.
func Load(path string) error {
        input, err := os.Open(path)
        if err != nil {
                return err
        }
	decoder := json.NewDecoder(input)
        var c Cfg
        err = decoder.Decode(&c)
        if err != nil {
                return err
	}{{range .Tree.Children}}{{$v := NewVar}}{{if eq .Type.String "struct"}}
	{{$v}} := {{.Name}}Cfg(c.{{.Name}}){{end}}
	{{.Name}} = {{if eq .Type.String "struct"}}&{{$v}}{{else}}c.{{.Name}}{{end}}{{end}}

        return nil
}
`

// Wrapper around tree.Format that exits the process in case of error.
func (t *Tree) FormatRaw() string {
	b, err := t.Format()
	if err != nil {
		fatalf("failed to format data structure: %s", err)
	}
	return string(b)
}

// Produce type name for tree node
func (t *Tree) TypeName() string {
	if t.Type == Struct {
		return "*" + t.Name.String() + "Cfg"
	}
	return t.Type.String()
}
<commit_msg>Remove extraneous blank lines in generated file.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	cfgPath = kingpin.Flag("config", `path to JSON configuration, defaults to "config.json"`).Short('c').String()
	outFile = kingpin.Flag("out", `path to output file, defaults to "config.go"`).Short('o').String()
	pack    = kingpin.Flag("package", `name of go package containing config code`).Short('p').Default("main").String()

	// Replace the variables below for testing
	fatalf = kingpin.Fatalf
)

func main() {
	kingpin.Parse()
	run()
}

// Make it easy for test to setup configuration values.
func run() {
	wd, err := os.Getwd()
	if err != nil {
		fatalf("failed to retrieve current directory: %s", err)
	}
	if *cfgPath == "" {
		*cfgPath = path.Join(wd, "config.json")
	}
	if *cfgPath, err = filepath.Abs(*cfgPath); err != nil {
		fatalf("failed to compute absolute file path for config: %s", err)
	}
	if *outFile == "" {
		*outFile = path.Join(wd, "config.go")
	}
	if _, err := os.Stat(*cfgPath); os.IsNotExist(err) {
		fatalf("no configuration file at %s", *cfgPath)
	}
	outDir := filepath.Dir(*outFile)
	s, err := os.Stat(outDir)
	if os.IsNotExist(err) {
		kingpin.FatalIfError(os.MkdirAll(outDir, 0777), "output directory")
	} else if err != nil || !s.IsDir() {
		fatalf("not a valid output directory: %s", outDir)
	}
	input, err := os.Open(*cfgPath)
	if err != nil {
		fatalf("failed to open JSON file: %s", err)
	}
	defer input.Close()
	decoder := json.NewDecoder(input)
	decoder.UseNumber()
	var data interface{}
	err = decoder.Decode(&data)
	if err != nil {
		fatalf("failed to unmarshal JSON: %s", err)
	}
	var tree Tree
	tree.Populate(data)
	if tree.Type != Struct {
		fatalf("invalid configuration file content, JSON must define an object")
	}
	tree.Normalize()
	tree.Name = "cfg"
	configs := make([]*Tree, len(tree.Children))
	for i, child := range tree.Children {
		configs[i] = &Tree{
			Name:     child.Name + "Cfg",
			Children: child.Children,
			Type:     child.Type,
			List:     child.List,
		}
	}
	vars := Variables{
		CmdLine: fmt.Sprintf("$ %s %s", os.Args[0], strings.Join(os.Args[1:], " ")),
		Pack:    *pack,
		Tree:    &tree,
		Configs: configs,
	}
	tmpl, err := template.New("gonfig").Funcs(template.FuncMap{"NewVar": NewVar}).Parse(configTemplate)
	if err != nil {
		fatalf("failed to compile template: %s", err)
	}
	outF, err := os.Create(*outFile)
	if err != nil {
		fatalf("failed to create file: %s", err)
	}
	defer outF.Close()
	err = tmpl.Execute(outF, &vars)

	// Print something
	fmt.Println(*outFile)
}

// Variables defines the data structure fed to the template to generate the final code.
type Variables struct {
	CmdLine string  // Command line used to invoke gonfig
	Pack    string  // Name of target package
	Tree    *Tree   // Top level data structure
	Configs []*Tree // Configuration entries
}

// Internal count used to generate unique variable names
var varCount int

// Generate unique Go variable name
func NewVar() string {
	varCount += 1
	return fmt.Sprintf("v%d", varCount)
}

const configTemplate string = `//************************************************************************//
//                     Configuration
//
// Generated with:
// {{.CmdLine}}
//
// The content of this file is auto-generated, DO NOT MODIFY
//************************************************************************//

package {{.Pack}}

import (
	"encoding/json"
        "os"
)

var ({{range .Tree.Children}}
	{{.Name}} {{.TypeName}}{{end}}
)
{{range .Configs}}{{if eq .Type.String "struct"}}
{{.FormatRaw}}{{end}}{{end}}
{{.Tree.FormatRaw}}
// Load reads the JSON at the given path and initializes the package variables with the
// corresponding values.
func Load(path string) error {
        input, err := os.Open(path)
        if err != nil {
                return err
        }
	decoder := json.NewDecoder(input)
        var c Cfg
        err = decoder.Decode(&c)
        if err != nil {
                return err
	}{{range .Tree.Children}}{{$v := NewVar}}{{if eq .Type.String "struct"}}
	{{$v}} := {{.Name}}Cfg(c.{{.Name}}){{end}}
	{{.Name}} = {{if eq .Type.String "struct"}}&{{$v}}{{else}}c.{{.Name}}{{end}}{{end}}

        return nil
}
`

// Wrapper around tree.Format that exits the process in case of error.
func (t *Tree) FormatRaw() string {
	b, err := t.Format()
	if err != nil {
		fatalf("failed to format data structure: %s", err)
	}
	return string(b)
}

// Produce type name for tree node
func (t *Tree) TypeName() string {
	if t.Type == Struct {
		return "*" + t.Name.String() + "Cfg"
	}
	return t.Type.String()
}
<|endoftext|>


<commit_before>package parsers

var assetListTestCases = []Case{
	{
		"Simple",
		`Hurricane	1	Combat Battlecruiser`,
		&AssetList{
			Items: []AssetItem{{Name: "Hurricane", Group: "Combat Battlecruiser", Quantity: 1}},
			lines: []int{0},
		},
		Input{},
		false, // This clashes with the simple contract format
	}, {
		"Typical",
		`720mm Gallium Cannon	1	Projectile Weapon	Medium	High	10 m3
Damage Control II	1	Damage Control		Low	5 m3
Experimental 10MN Microwarpdrive I	1	Propulsion Module		Medium	10 m3`,
		&AssetList{
			Items: []AssetItem{
				{Name: "720mm Gallium Cannon", Quantity: 1, Group: "Projectile Weapon", Category: "Medium", Slot: "High", Volume: 10},
				{Name: "Damage Control II", Quantity: 1, Group: "Damage Control", Slot: "Low", Volume: 5},
				{Name: "Experimental 10MN Microwarpdrive I", Quantity: 1, Group: "Propulsion Module", Size: "Medium", Volume: 10},
			},
			lines: []int{0, 1, 2}},
		Input{},
		true,
	}, {
		"Full",
		`200mm AutoCannon I	1	Projectile Weapon	Module	Small	High	5 m3	1
10MN Afterburner II	1	Propulsion Module	Module	Medium	5 m3	5	2
Warrior II	9`,
		&AssetList{
			Items: []AssetItem{
				{Name: "10MN Afterburner II", Quantity: 1, Group: "Propulsion Module", Category: "Module", Size: "Medium", MetaLevel: "5", TechLevel: "2", Volume: 5},
				{Name: "200mm AutoCannon I", Quantity: 1, Group: "Projectile Weapon", Category: "Module", Size: "Small", Slot: "High", MetaLevel: "1", Volume: 5},
				{Name: "Warrior II", Quantity: 9},
			},
			lines: []int{0, 1, 2},
		},
		Input{},
		true,
	}, {
		"With Volumes",
		`Sleeper Data Library	1080	Sleeper Components			10.82 m3`,
		&AssetList{
			Items: []AssetItem{{Name: "Sleeper Data Library", Quantity: 1080, Group: "Sleeper Components", Volume: 10.82}},
			lines: []int{0},
		},
		Input{},
		true,
	}, {
		"With thousands separators",
		`Sleeper Data Library	1,080
Sleeper Data Library	1'080
Sleeper Data Library	1.080`,
		&AssetList{
			Items: []AssetItem{
				{Name: "Sleeper Data Library", Quantity: 1080},
				{Name: "Sleeper Data Library", Quantity: 1080},
				{Name: "Sleeper Data Library", Quantity: 1080},
			},
			lines: []int{0, 1, 2},
		},
		Input{},
		false,
	}, {
		"With empty quantity",
		`Sleeper Data Library	`,
		&AssetList{
			Items: []AssetItem{
				{Name: "Sleeper Data Library", Quantity: 1},
			},
			lines: []int{0},
		},
		Input{},
		false,
	},
}
<commit_msg>Add test for asset parsing improvement<commit_after>package parsers

var assetListTestCases = []Case{
	{
		"Simple",
		`Hurricane	1	Combat Battlecruiser`,
		&AssetList{
			Items: []AssetItem{{Name: "Hurricane", Group: "Combat Battlecruiser", Quantity: 1}},
			lines: []int{0},
		},
		Input{},
		false, // This clashes with the simple contract format
	}, {
		"Typical",
		`720mm Gallium Cannon	1	Projectile Weapon	Medium	High	10 m3
Damage Control II	1	Damage Control		Low	5 m3
Experimental 10MN Microwarpdrive I	1	Propulsion Module		Medium	10 m3`,
		&AssetList{
			Items: []AssetItem{
				{Name: "720mm Gallium Cannon", Quantity: 1, Group: "Projectile Weapon", Category: "Medium", Slot: "High", Volume: 10},
				{Name: "Damage Control II", Quantity: 1, Group: "Damage Control", Slot: "Low", Volume: 5},
				{Name: "Experimental 10MN Microwarpdrive I", Quantity: 1, Group: "Propulsion Module", Size: "Medium", Volume: 10},
			},
			lines: []int{0, 1, 2}},
		Input{},
		true,
	}, {
		"Full",
		`200mm AutoCannon I	1	Projectile Weapon	Module	Small	High	5 m3	1
10MN Afterburner II	1	Propulsion Module	Module	Medium	5 m3	5	2
Warrior II	9`,
		&AssetList{
			Items: []AssetItem{
				{Name: "10MN Afterburner II", Quantity: 1, Group: "Propulsion Module", Category: "Module", Size: "Medium", MetaLevel: "5", TechLevel: "2", Volume: 5},
				{Name: "200mm AutoCannon I", Quantity: 1, Group: "Projectile Weapon", Category: "Module", Size: "Small", Slot: "High", MetaLevel: "1", Volume: 5},
				{Name: "Warrior II", Quantity: 9},
			},
			lines: []int{0, 1, 2},
		},
		Input{},
		true,
	}, {
		"With Volumes",
		`Sleeper Data Library	1080	Sleeper Components			10.82 m3`,
		&AssetList{
			Items: []AssetItem{{Name: "Sleeper Data Library", Quantity: 1080, Group: "Sleeper Components", Volume: 10.82}},
			lines: []int{0},
		},
		Input{},
		true,
	}, {
		"With thousands separators",
		`Sleeper Data Library	1,080
Sleeper Data Library	1'080
Sleeper Data Library	1.080`,
		&AssetList{
			Items: []AssetItem{
				{Name: "Sleeper Data Library", Quantity: 1080},
				{Name: "Sleeper Data Library", Quantity: 1080},
				{Name: "Sleeper Data Library", Quantity: 1080},
			},
			lines: []int{0, 1, 2},
		},
		Input{},
		false,
	}, {
		"With empty quantity",
		`Sleeper Data Library	`,
		&AssetList{
			Items: []AssetItem{
				{Name: "Sleeper Data Library", Quantity: 1},
			},
			lines: []int{0},
		},
		Input{},
		false,
	}, {
		"With asterisk",
		`Armor Plates*	477	Geborgene Materialien*`,
		&AssetList{
			Items: []AssetItem{
				{Name: "Armor Plates", Quantity: 477, Group: "Geborgene Materialien*"},
			},
			lines: []int{0},
		},
		Input{},
		false,
	},
}
<|endoftext|>