<commit_before>package server

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"log"
	"net/rpc"
	"sync"

	"mateusbraga/freestore/view"
)

var (
	consensusTable   map[int]consensusInstance
	consensusTableMu sync.RWMutex

	oldProposalNumberErr OldProposalNumberError
)

type consensusInstance struct {
	id                int
	taskChan          chan consensusTask
	callbackLearnChan chan interface{}
}

//func (ci *consensusInstance) Nuke() {
//close(ci.taskChan)
//}

type consensusTask interface{}

func init() {
	consensusTable = make(map[int]consensusInstance)
}

func getConsensus(id int) consensusInstance {
	consensusTableMu.Lock()
	defer consensusTableMu.Unlock()

	if ci, ok := consensusTable[id]; ok {
		return ci
	} else {
		ci := consensusInstance{id: id, taskChan: make(chan consensusTask, CHANNEL_DEFAULT_SIZE), callbackLearnChan: make(chan interface{}, 1)}
		go consensusWorker(ci)

		consensusTable[ci.id] = ci
		log.Println("Created consensusInstance:", ci)
		return ci
	}
}

func consensusWorker(ci consensusInstance) {
	var acceptedProposal Proposal     // highest numbered accepted proposal
	var lastPromiseProposalNumber int // highest numbered prepare request
	var learnCounter int              // number of learn requests received

	for {
		taskInterface, ok := <-ci.taskChan
		if !ok {
			log.Printf("consensusInstance %v done\n", ci)
			return
		}

		switch task := taskInterface.(type) {
		case *Prepare:
			log.Println("Processing prepare request")
			if task.N > lastPromiseProposalNumber {
				//setLastPromiseProposalNumber
				savePrepareRequest(ci.id, task)
				lastPromiseProposalNumber = task.N

				task.reply.N = acceptedProposal.N
				task.reply.Value = acceptedProposal.Value
			} else {
				task.reply.Err = oldProposalNumberErr
			}
			task.returnChan <- true
		case *Accept:
			log.Println("Processing accept request")
			if task.N >= lastPromiseProposalNumber {
				//setAcceptedProposal
				saveAcceptedProposal(ci.id, task.Proposal)
				acceptedProposal = *task.Proposal

				go spreadAcceptance(*task.Proposal)
			} else {
				task.reply.Err = oldProposalNumberErr
			}
			task.returnChan <- true
		case *Learn:
			log.Println("Processing learn request")
			learnCounter++
			if learnCounter == currentView.QuorumSize() {
				ci.callbackLearnChan <- task.Value
			}
		default:
			log.Fatalf("BUG in the ConsensusWorker switch, got %T %v\n", task, task)
		}
	}
}

// Propose proposes the value to be agreed upon on this consensus instance. It should be run only by the leader process to guarantee termination.
func Propose(ci consensusInstance, defaultValue interface{}) {
	log.Println("Got in propose")
	proposalNumber := getNextProposalNumber(ci.id)

	proposal := Proposal{N: proposalNumber, ConsensusId: ci.id}
	value, err := prepare(proposal)
	if err != nil {
		// Could not get quorum or old proposal number
		log.Println("WARN: Failed to propose. Could not pass prepare phase: ", err)
		return
	}

	if value == nil {
		value = defaultValue
	}

	proposal.Value = value
	if err := accept(proposal); err != nil {
		// Could not get quorum or old proposal number
		log.Println("WARN: Failed to propose. Could not pass accept phase: ", err)
		return
	}
}

// prepare is a stage of the Propose funcion
func prepare(proposal Proposal) (interface{}, error) {
	resultChan := make(chan Proposal, currentView.N())
	errChan := make(chan error, currentView.N())
	stopChan := make(chan bool, currentView.N())
	defer fillStopChan(stopChan, currentView.N())

	// Send read request to all
	for _, process := range currentView.GetMembers() {
		go prepareProcess(process, proposal, resultChan, errChan, stopChan)
	}

	// Get quorum
	var failed int
	var success int
	var highestNumberedAcceptedProposal Proposal
	for {
		select {
		case receivedProposal := <-resultChan:
			if receivedProposal.Err != nil {
				errChan <- receivedProposal.Err
				break
			}

			success++
			if highestNumberedAcceptedProposal.N < receivedProposal.N {
				highestNumberedAcceptedProposal = receivedProposal
			}

			if success == currentView.QuorumSize() {
				return highestNumberedAcceptedProposal.Value, nil
			}
		case err := <-errChan:
			log.Println("+1 failure to prepare:", err)
			failed++

			if failed > currentView.F() {
				return Value{}, errors.New("Failed to get prepare quorun")
			}
		}
	}
}

// accept is a stage of the Propose funcion.
func accept(proposal Proposal) error {
	resultChan := make(chan Proposal, currentView.N())
	errChan := make(chan error, currentView.N())
	stopChan := make(chan bool, currentView.N())
	defer fillStopChan(stopChan, currentView.N())

	// Send accept request to all
	for _, process := range currentView.GetMembers() {
		go acceptProcess(process, proposal, resultChan, errChan, stopChan)
	}

	// Get quorum
	var failed int
	var success int
	for {
		select {
		case receivedProposal := <-resultChan:
			if receivedProposal.Err != nil {
				errChan <- receivedProposal.Err
				break
			}

			success++

			if success == currentView.QuorumSize() {
				return nil
			}
		case err := <-errChan:
			log.Println("+1 failure to accept:", err)
			failed++

			if failed > currentView.F() {
				return errors.New("Failed to get accept quorun")
			}
		}
	}
}

// CheckForChosenValue checks to see if any value has already been agreed upon on this consensus instance.
func CheckForChosenValue(ci consensusInstance) (interface{}, error) {
	proposalNumber := getNextProposalNumber(ci.id)

	proposal := Proposal{N: proposalNumber, ConsensusId: ci.id}
	value, err := prepare(proposal)
	if err != nil {
		return nil, errors.New("Could not read prepare consensus")
	}
	if value == nil {
		return nil, errors.New("No value has been chosen")
	}

	return value, nil
}

// getNextProposalNumber to be used by this process. This function is a stage of the Propose funcion.
func getNextProposalNumber(consensusId int) (proposalNumber int) {
	if currentView.N() == 0 {
		log.Fatalln("currentView is empty")
	}

	thisProcessPosition := currentView.GetProcessPosition(thisProcess)

	lastProposalNumber, err := getLastProposalNumber(consensusId)
	if err != nil {
		proposalNumber = currentView.N() + thisProcessPosition
	} else {
		proposalNumber = (lastProposalNumber - (lastProposalNumber % currentView.N()) + currentView.N()) + thisProcessPosition
	}

	saveProposalNumber(consensusId, proposalNumber)
	return
}

// saveProposalNumber to permanent storage.
func saveProposalNumber(consensusId int, proposalNumber int) {
	proposalNumberBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalNumberBuffer)
	err := enc.Encode(proposalNumber)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("lastProposalNumber_%v", consensusId)), proposalNumberBuffer.Bytes())
	if err != nil {
		log.Fatalln("db.Set failed:", err)
	}
}

// getLastProposalNumber from permanent storage.
func getLastProposalNumber(consensusId int) (int, error) {
	lastProposalNumberBytes, err := db.Get(nil, []byte(fmt.Sprintf("lastProposalNumber_%v", consensusId)))
	if err != nil {
		log.Fatalln(err)
	} else if lastProposalNumberBytes == nil {
		return 0, errors.New("Last proposal number not found")
	} else {
		var lastProposalNumber int

		lastProposalNumberBuffer := bytes.NewBuffer(lastProposalNumberBytes)
		dec := gob.NewDecoder(lastProposalNumberBuffer)

		err := dec.Decode(&lastProposalNumber)
		if err != nil {
			log.Fatalln("dec.Decode failed:", err)
		}

		return lastProposalNumber, nil
	}

	log.Fatalln("BUG! Should never execute this command on getLastProposalNumber")
	return 0, nil
}

// saveAcceptedProposal to permanent storage.
// needs to be tested
func saveAcceptedProposal(consensusId int, proposal *Proposal) {
	proposalBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalBuffer)

	err := enc.Encode(proposal)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("acceptedProposal_%v", consensusId)), proposalBuffer.Bytes())
	if err != nil {
		log.Fatalln("ERROR to save acceptedProposal:", err)
	}
}

// savePrepareRequest to permanent storage
// needs to be tested
func savePrepareRequest(consensusId int, proposal *Prepare) {
	proposalBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalBuffer)
	err := enc.Encode(proposal)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("prepareRequest_%v", consensusId)), proposalBuffer.Bytes())
	if err != nil {
		log.Fatalln("ERROR to save prepareRequest:", err)
	}
}

// spreadAcceptance to all processes on the current view
func spreadAcceptance(proposal Proposal) {
	// Send acceptances to all
	// Can be improved: it is currently send too many messages at once
	for _, process := range currentView.GetMembers() {
		go spreadAcceptanceProcess(process, proposal)
	}
}

// prepareProcess sends a prepare proposal to process.
func prepareProcess(process view.Process, proposal Proposal, resultChan chan Proposal, errChan chan error, stopChan chan bool) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		select {
		case errChan <- err:
		case <-stopChan:
			log.Println("Failure masked:", err)
		}
		return
	}
	defer client.Close()

	var reply Proposal

	err = client.Call("ConsensusRequest.Prepare", proposal, &reply)
	if err != nil {
		select {
		case errChan <- err:
		case <-stopChan:
			log.Println("Failure masked:", err)
		}
		return
	}

	select {
	case resultChan <- reply:
	case <-stopChan:
	}
}

// acceptProcess sends a prepare proposal to process.
func acceptProcess(process view.Process, proposal Proposal, resultChan chan Proposal, errChan chan error, stopChan chan bool) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		select {
		case errChan <- err:
		case <-stopChan:
			log.Println("Failure masked:", err)
		}
		return
	}
	defer client.Close()

	var reply Proposal
	err = client.Call("ConsensusRequest.Accept", proposal, &reply)
	if err != nil {
		select {
		case errChan <- err:
		case <-stopChan:
			log.Println("Failure masked:", err)
		}
		return
	}

	select {
	case resultChan <- reply:
	case <-stopChan:
	}
}

// spreadAcceptance sends acceptance to process.
func spreadAcceptanceProcess(process view.Process, proposal Proposal) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		log.Println("WARN: spreadAcceptanceProcess:", err)
		return
	}
	defer client.Close()

	var reply Proposal
	err = client.Call("ConsensusRequest.Learn", proposal, &reply)
	if err != nil {
		log.Println("WARN: spreadAcceptanceProcess:", err)
		return
	}
}

// -------- REQUESTS -----------
type ConsensusRequest int

type Proposal struct {
	ConsensusId int //ConsensusId makes possible multiples consensus to run at the same time

	N int // N is the proposal number

	Value interface{} // Value proposed

	Err error // Err is used to return an error related to the proposal
}

type Prepare struct {
	*Proposal
	reply      *Proposal
	returnChan chan bool
}

type Accept struct {
	*Proposal
	reply      *Proposal
	returnChan chan bool
}

type Learn struct {
	*Proposal
}

// Prepare Request
func (r *ConsensusRequest) Prepare(arg Proposal, reply *Proposal) error {
	log.Println("New Prepare Request")
	ci := getConsensus(arg.ConsensusId)

	var prepare Prepare
	prepare.Proposal = &arg
	prepare.returnChan = make(chan bool)
	prepare.reply = reply

	ci.taskChan <- &prepare

	<-prepare.returnChan

	return nil
}

// Accept Request
func (r *ConsensusRequest) Accept(arg Proposal, reply *Proposal) error {
	log.Println("New Accept Request")
	ci := getConsensus(arg.ConsensusId)

	var accept Accept
	accept.Proposal = &arg
	accept.returnChan = make(chan bool)
	accept.reply = reply

	ci.taskChan <- &accept

	<-accept.returnChan

	return nil
}

// Learn Request
func (r *ConsensusRequest) Learn(arg Proposal, reply *Proposal) error {
	log.Println("New Learn Request")
	ci := getConsensus(arg.ConsensusId)

	var learn Learn
	learn.Proposal = &arg

	ci.taskChan <- &learn

	return nil
}

func init() {
	consensusRequest := new(ConsensusRequest)
	rpc.Register(consensusRequest)
}

// ------- ERRORS -----------
type OldProposalNumberError struct{}

func (e OldProposalNumberError) Error() string {
	return fmt.Sprint("Promised to a higher numbered prepare request")
}

func init() {
	gob.Register(new(OldProposalNumberError))
}

// ------- Auxiliary functions -----------
// fillStopChan send times * 'true' on stopChan. It is used to signal completion to readProcess and writeProcess.
func fillStopChan(stopChan chan bool, times int) {
	for i := 0; i < times; i++ {
		stopChan <- true
	}
}
<commit_msg>Fix server/consensus send functions<commit_after>package server

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"log"
	"net/rpc"
	"sync"

	"mateusbraga/freestore/view"
)

var (
	consensusTable   map[int]consensusInstance
	consensusTableMu sync.RWMutex

	oldProposalNumberErr OldProposalNumberError
)

type consensusInstance struct {
	id                int
	taskChan          chan consensusTask
	callbackLearnChan chan interface{}
}

type consensusTask interface{}

func init() {
	consensusTable = make(map[int]consensusInstance)
}

func getConsensus(id int) consensusInstance {
	consensusTableMu.Lock()
	defer consensusTableMu.Unlock()

	if ci, ok := consensusTable[id]; ok {
		return ci
	} else {
		ci := consensusInstance{id: id, taskChan: make(chan consensusTask, CHANNEL_DEFAULT_SIZE), callbackLearnChan: make(chan interface{}, 1)}
		go consensusWorker(ci)

		consensusTable[ci.id] = ci
		log.Println("Created consensusInstance:", ci)
		return ci
	}
}

func consensusWorker(ci consensusInstance) {
	var acceptedProposal Proposal     // highest numbered accepted proposal
	var lastPromiseProposalNumber int // highest numbered prepare request
	var learnCounter int              // number of learn requests received

	for {
		taskInterface, ok := <-ci.taskChan
		if !ok {
			log.Printf("consensusInstance %v done\n", ci)
			return
		}

		switch task := taskInterface.(type) {
		case *Prepare:
			log.Println("Processing prepare request")
			if task.N > lastPromiseProposalNumber {
				//setLastPromiseProposalNumber
				savePrepareRequest(ci.id, task)
				lastPromiseProposalNumber = task.N

				task.reply.N = acceptedProposal.N
				task.reply.Value = acceptedProposal.Value
			} else {
				task.reply.Err = oldProposalNumberErr
			}
			task.returnChan <- true
		case *Accept:
			log.Println("Processing accept request")
			if task.N >= lastPromiseProposalNumber {
				//setAcceptedProposal
				saveAcceptedProposal(ci.id, task.Proposal)
				acceptedProposal = *task.Proposal

				go spreadAcceptance(*task.Proposal)
			} else {
				task.reply.Err = oldProposalNumberErr
			}
			task.returnChan <- true
		case *Learn:
			log.Println("Processing learn request")
			learnCounter++
			if learnCounter == currentView.QuorumSize() {
				ci.callbackLearnChan <- task.Value
			}
		default:
			log.Fatalf("BUG in the ConsensusWorker switch, got %T %v\n", task, task)
		}
	}
}

// Propose proposes the value to be agreed upon on this consensus instance. It should be run only by the leader process to guarantee termination.
func Propose(ci consensusInstance, defaultValue interface{}) {
	log.Println("Got in propose")
	proposalNumber := getNextProposalNumber(ci.id)

	proposal := Proposal{N: proposalNumber, ConsensusId: ci.id}
	value, err := prepare(proposal)
	if err != nil {
		// Could not get quorum or old proposal number
		log.Println("WARN: Failed to propose. Could not pass prepare phase: ", err)
		return
	}

	if value == nil {
		value = defaultValue
	}

	proposal.Value = value
	if err := accept(proposal); err != nil {
		// Could not get quorum or old proposal number
		log.Println("WARN: Failed to propose. Could not pass accept phase: ", err)
		return
	}
}

// prepare is a stage of the Propose funcion
func prepare(proposal Proposal) (interface{}, error) {
	resultChan := make(chan Proposal, currentView.N())
	errChan := make(chan error, currentView.N())
	stopChan := make(chan bool, currentView.N())
	defer fillStopChan(stopChan, currentView.N())

	// Send read request to all
	for _, process := range currentView.GetMembers() {
		go prepareProcess(process, proposal, resultChan, errChan, stopChan)
	}

	// Get quorum
	var failed int
	var success int
	var highestNumberedAcceptedProposal Proposal
	for {
		select {
		case receivedProposal := <-resultChan:
			if receivedProposal.Err != nil {
				errChan <- receivedProposal.Err
				break
			}

			success++
			if highestNumberedAcceptedProposal.N < receivedProposal.N {
				highestNumberedAcceptedProposal = receivedProposal
			}

			if success == currentView.QuorumSize() {
				return highestNumberedAcceptedProposal.Value, nil
			}
		case err := <-errChan:
			log.Println("+1 failure to prepare:", err)
			failed++

			if failed > currentView.F() {
				return Value{}, errors.New("Failed to get prepare quorun")
			}
		}
	}
}

// accept is a stage of the Propose funcion.
func accept(proposal Proposal) error {
	resultChan := make(chan Proposal, currentView.N())
	errChan := make(chan error, currentView.N())
	stopChan := make(chan bool, currentView.N())
	defer fillStopChan(stopChan, currentView.N())

	// Send accept request to all
	for _, process := range currentView.GetMembers() {
		go acceptProcess(process, proposal, resultChan, errChan, stopChan)
	}

	// Get quorum
	var failed int
	var success int
	for {
		select {
		case receivedProposal := <-resultChan:
			if receivedProposal.Err != nil {
				errChan <- receivedProposal.Err
				break
			}

			success++

			if success == currentView.QuorumSize() {
				return nil
			}
		case err := <-errChan:
			log.Println("+1 failure to accept:", err)
			failed++

			if failed > currentView.F() {
				return errors.New("Failed to get accept quorun")
			}
		}
	}
}

// CheckForChosenValue checks to see if any value has already been agreed upon on this consensus instance.
func CheckForChosenValue(ci consensusInstance) (interface{}, error) {
	proposalNumber := getNextProposalNumber(ci.id)

	proposal := Proposal{N: proposalNumber, ConsensusId: ci.id}
	value, err := prepare(proposal)
	if err != nil {
		return nil, errors.New("Could not read prepare consensus")
	}
	if value == nil {
		return nil, errors.New("No value has been chosen")
	}

	return value, nil
}

// getNextProposalNumber to be used by this process. This function is a stage of the Propose funcion.
func getNextProposalNumber(consensusId int) (proposalNumber int) {
	if currentView.N() == 0 {
		log.Fatalln("currentView is empty")
	}

	thisProcessPosition := currentView.GetProcessPosition(thisProcess)

	lastProposalNumber, err := getLastProposalNumber(consensusId)
	if err != nil {
		proposalNumber = currentView.N() + thisProcessPosition
	} else {
		proposalNumber = (lastProposalNumber - (lastProposalNumber % currentView.N()) + currentView.N()) + thisProcessPosition
	}

	saveProposalNumber(consensusId, proposalNumber)
	return
}

// saveProposalNumber to permanent storage.
func saveProposalNumber(consensusId int, proposalNumber int) {
	proposalNumberBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalNumberBuffer)
	err := enc.Encode(proposalNumber)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("lastProposalNumber_%v", consensusId)), proposalNumberBuffer.Bytes())
	if err != nil {
		log.Fatalln("db.Set failed:", err)
	}
}

// getLastProposalNumber from permanent storage.
func getLastProposalNumber(consensusId int) (int, error) {
	lastProposalNumberBytes, err := db.Get(nil, []byte(fmt.Sprintf("lastProposalNumber_%v", consensusId)))
	if err != nil {
		log.Fatalln(err)
	} else if lastProposalNumberBytes == nil {
		return 0, errors.New("Last proposal number not found")
	} else {
		var lastProposalNumber int

		lastProposalNumberBuffer := bytes.NewBuffer(lastProposalNumberBytes)
		dec := gob.NewDecoder(lastProposalNumberBuffer)

		err := dec.Decode(&lastProposalNumber)
		if err != nil {
			log.Fatalln("dec.Decode failed:", err)
		}

		return lastProposalNumber, nil
	}

	log.Fatalln("BUG! Should never execute this command on getLastProposalNumber")
	return 0, nil
}

// saveAcceptedProposal to permanent storage.
// needs to be tested
func saveAcceptedProposal(consensusId int, proposal *Proposal) {
	proposalBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalBuffer)

	err := enc.Encode(proposal)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("acceptedProposal_%v", consensusId)), proposalBuffer.Bytes())
	if err != nil {
		log.Fatalln("ERROR to save acceptedProposal:", err)
	}
}

// savePrepareRequest to permanent storage
// needs to be tested
func savePrepareRequest(consensusId int, proposal *Prepare) {
	proposalBuffer := new(bytes.Buffer)
	enc := gob.NewEncoder(proposalBuffer)
	err := enc.Encode(proposal)
	if err != nil {
		log.Fatalln("enc.Encode failed:", err)
	}

	err = db.Set([]byte(fmt.Sprintf("prepareRequest_%v", consensusId)), proposalBuffer.Bytes())
	if err != nil {
		log.Fatalln("ERROR to save prepareRequest:", err)
	}
}

// spreadAcceptance to all processes on the current view
func spreadAcceptance(proposal Proposal) {
	// Send acceptances to all
	// Can be improved: it is currently send too many messages at once
	for _, process := range currentView.GetMembers() {
		go spreadAcceptanceProcess(process, proposal)
	}
}

// prepareProcess sends a prepare proposal to process.
func prepareProcess(process view.Process, proposal Proposal, resultChan chan Proposal, errChan chan error, stopChan chan bool) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		errChan <- err
		return
	}
	defer client.Close()

	var reply Proposal

	call := client.Go("ConsensusRequest.Prepare", proposal, &reply, nil)
	select {
	case <-call.Done:
		if call.Error != nil {
			errChan <- call.Error
		} else {
			resultChan <- reply
		}
	case <-stopChan:
	}
}

// acceptProcess sends a prepare proposal to process.
func acceptProcess(process view.Process, proposal Proposal, resultChan chan Proposal, errChan chan error, stopChan chan bool) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		errChan <- err
		return
	}
	defer client.Close()

	var reply Proposal
	call := client.Go("ConsensusRequest.Accept", proposal, &reply, nil)
	select {
	case <-call.Done:
		if call.Error != nil {
			errChan <- call.Error
		} else {
			resultChan <- reply
		}
	case <-stopChan:
	}
}

// spreadAcceptance sends acceptance to process.
func spreadAcceptanceProcess(process view.Process, proposal Proposal) {
	client, err := rpc.Dial("tcp", process.Addr)
	if err != nil {
		log.Println("WARN: spreadAcceptanceProcess:", err)
		return
	}
	defer client.Close()

	var reply Proposal
	err = client.Call("ConsensusRequest.Learn", proposal, &reply)
	if err != nil {
		log.Println("WARN: spreadAcceptanceProcess:", err)
		return
	}
}

// -------- REQUESTS -----------
type ConsensusRequest int

type Proposal struct {
	ConsensusId int //ConsensusId makes possible multiples consensus to run at the same time

	N int // N is the proposal number

	Value interface{} // Value proposed

	Err error // Err is used to return an error related to the proposal
}

type Prepare struct {
	*Proposal
	reply      *Proposal
	returnChan chan bool
}

type Accept struct {
	*Proposal
	reply      *Proposal
	returnChan chan bool
}

type Learn struct {
	*Proposal
}

// Prepare Request
func (r *ConsensusRequest) Prepare(arg Proposal, reply *Proposal) error {
	log.Println("New Prepare Request")
	ci := getConsensus(arg.ConsensusId)

	var prepare Prepare
	prepare.Proposal = &arg
	prepare.returnChan = make(chan bool)
	prepare.reply = reply

	ci.taskChan <- &prepare

	<-prepare.returnChan

	return nil
}

// Accept Request
func (r *ConsensusRequest) Accept(arg Proposal, reply *Proposal) error {
	log.Println("New Accept Request")
	ci := getConsensus(arg.ConsensusId)

	var accept Accept
	accept.Proposal = &arg
	accept.returnChan = make(chan bool)
	accept.reply = reply

	ci.taskChan <- &accept

	<-accept.returnChan

	return nil
}

// Learn Request
func (r *ConsensusRequest) Learn(arg Proposal, reply *Proposal) error {
	log.Println("New Learn Request")
	ci := getConsensus(arg.ConsensusId)

	var learn Learn
	learn.Proposal = &arg

	ci.taskChan <- &learn

	return nil
}

func init() {
	consensusRequest := new(ConsensusRequest)
	rpc.Register(consensusRequest)
}

// ------- ERRORS -----------
type OldProposalNumberError struct{}

func (e OldProposalNumberError) Error() string {
	return fmt.Sprint("Promised to a higher numbered prepare request")
}

func init() {
	gob.Register(new(OldProposalNumberError))
}

// ------- Auxiliary functions -----------
// fillStopChan send times * 'true' on stopChan. It is used to signal completion to readProcess and writeProcess.
func fillStopChan(stopChan chan bool, times int) {
	for i := 0; i < times; i++ {
		stopChan <- true
	}
}
<|endoftext|>


<commit_before>package fizz

import (
	"fmt"

	"github.com/mattn/anko/vm"

	core "github.com/mattn/anko/builtins"
)

type BubbleType int

const (
	E_CREATE_TABLE BubbleType = iota
	E_DROP_TABLE
	E_RENAME_TABLE
	E_RAW_SQL
	E_ADD_COLUMN
	E_DROP_COLUMN
	E_RENAME_COLUMN
	E_ADD_INDEX
	E_DROP_INDEX
	E_RENAME_INDEX
)

type Bubble struct {
	BubbleType BubbleType
	Data       interface{}
}

type Bubbler struct {
	Bubbles []Bubble
}

func NewBubbler() *Bubbler {
	return &Bubbler{
		Bubbles: []Bubble{},
	}
}

func (b *Bubbler) Bubble(s string) error {
	env := core.Import(vm.NewEnv())

	f := fizzer{b}

	// columns:
	env.Define("add_column", f.AddColumn())
	env.Define("drop_column", f.DropColumn())
	env.Define("rename_column", f.RenameColumn())

	env.Define("raw", f.RawSql())

	// indexes:
	env.Define("add_index", f.AddIndex())
	env.Define("drop_index", f.DropIndex())
	env.Define("rename_index", f.RenameIndex())

	// tables:
	env.Define("create_table", f.CreateTable())
	env.Define("drop_table", f.DropTable())
	env.Define("rename_table", f.RenameTable())

	// t := reflect.TypeOf(f)
	// v := reflect.ValueOf(f)
	// for i := 0; i < t.NumMethod(); i++ {
	// 	m := t.Method(i)
	// 	k := inflect.Underscore(m.Name)
	// 	fmt.Printf("### k -> %#v\n", k)
	// 	fmt.Printf("### m -> %#v\n", m)
	// 	mv := v.MethodByName(m.Name)
	// 	env.Define(k, mv.Call([]reflect.Value{}))
	// }

	_, err := env.Execute(s)
	fmt.Printf("### err -> %#v\n", err)
	return err
}
<commit_msg>removed some commented out code<commit_after>package fizz

import (
	"fmt"

	"github.com/mattn/anko/vm"

	core "github.com/mattn/anko/builtins"
)

type BubbleType int

const (
	E_CREATE_TABLE BubbleType = iota
	E_DROP_TABLE
	E_RENAME_TABLE
	E_RAW_SQL
	E_ADD_COLUMN
	E_DROP_COLUMN
	E_RENAME_COLUMN
	E_ADD_INDEX
	E_DROP_INDEX
	E_RENAME_INDEX
)

type Bubble struct {
	BubbleType BubbleType
	Data       interface{}
}

type Bubbler struct {
	Bubbles []Bubble
}

func NewBubbler() *Bubbler {
	return &Bubbler{
		Bubbles: []Bubble{},
	}
}

func (b *Bubbler) Bubble(s string) error {
	env := core.Import(vm.NewEnv())

	f := fizzer{b}

	// columns:
	env.Define("add_column", f.AddColumn())
	env.Define("drop_column", f.DropColumn())
	env.Define("rename_column", f.RenameColumn())

	env.Define("raw", f.RawSql())

	// indexes:
	env.Define("add_index", f.AddIndex())
	env.Define("drop_index", f.DropIndex())
	env.Define("rename_index", f.RenameIndex())

	// tables:
	env.Define("create_table", f.CreateTable())
	env.Define("drop_table", f.DropTable())
	env.Define("rename_table", f.RenameTable())

	_, err := env.Execute(s)
	fmt.Printf("### err -> %#v\n", err)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"

	pb "github.com/creiht/formic/proto"
	"github.com/pandemicsyn/ftls"

	"net"
)

var (
	usetls              = flag.Bool("tls", true, "Connection uses TLS if true, else plain TCP")
	certFile            = flag.String("cert_file", "/etc/oort/server.crt", "The TLS cert file")
	keyFile             = flag.String("key_file", "/etc/oort/server.key", "The TLS key file")
	port                = flag.Int("port", 9443, "The server port")
	oortValueHost       = flag.String("oortvaluehost", "127.0.0.1:6379", "host:port to use when connecting to oort value")
	oortGroupHost       = flag.String("oortgrouphost", "127.0.0.1:6380", "host:port to use when connecting to oort group")
	insecureSkipVerify  = flag.Bool("skipverify", true, "don't verify cert")
	oortClientMutualTLS = flag.Bool("mutualtls", false, "whether or not the server expects mutual tls auth")
	oortClientCert      = flag.String("oort-client-cert", "/etc/oort/client.crt", "cert file to use")
	oortClientKey       = flag.String("oort-client-key", "/etc/oort/client.key", "key file to use")
	oortClientCA        = flag.String("oort-client-ca", "/etc/oort/ca.pem", "ca file to use")
)

// FatalIf is just a lazy log/panic on error func
func FatalIf(err error, msg string) {
	if err != nil {
		grpclog.Fatalf("%s: %v", msg, err)
	}
}

func main() {
	flag.Parse()

	envtls := os.Getenv("FORMICD_TLS")
	if envtls == "true" {
		*usetls = true
	}

	envoortvhost := os.Getenv("FORMICD_OORT_VALUE_HOST")
	if envoortvhost != "" {
		*oortValueHost = envoortvhost
	}

	envoortghost := os.Getenv("FORMICD_OORT_GROUP_HOST")
	if envoortghost != "" {
		*oortGroupHost = envoortghost
	}

	envport := os.Getenv("FORMICD_PORT")
	if envport != "" {
		p, err := strconv.Atoi(envport)
		if err != nil {
			log.Println("Did not send valid port from env:", err)
		} else {
			*port = p
		}
	}

	envcert := os.Getenv("FORMICD_CERT_FILE")
	if envcert != "" {
		*certFile = envcert
	}

	envkey := os.Getenv("FORMICD_KEY_FILE")
	if envkey != "" {
		*keyFile = envkey
	}

	envSkipVerify := os.Getenv("FORMICD_INSECURE_SKIP_VERIFY")
	if envSkipVerify == "true" {
		*insecureSkipVerify = true
	}
	envMutualTLS := os.Getenv("FORMICD_MUTUAL_TLS")
	if envMutualTLS == "true" {
		*oortClientMutualTLS = true
	}
	envClientCA := os.Getenv("FORMICD_CLIENT_CA_FILE")
	if envClientCA != "" {
		*oortClientCA = envClientCA
	}
	envClientCert := os.Getenv("FORMICD_CLIENT_CERT_FILE")
	if envClientCert != "" {
		*oortClientCert = envClientCert
	}
	envClientKey := os.Getenv("FORMICD_CLIENT_KEY_FILE")
	if envClientKey != "" {
		*oortClientKey = envClientKey
	}

	var opts []grpc.ServerOption
	if *usetls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		FatalIf(err, "Couldn't load cert from file")
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	s := grpc.NewServer(opts...)
	copt, err := ftls.NewGRPCClientDialOpt(&ftls.Config{
		MutualTLS:          *oortClientMutualTLS,
		InsecureSkipVerify: *insecureSkipVerify,
		CertFile:           *oortClientCert,
		KeyFile:            *oortClientKey,
		CAFile:             *oortClientCA,
	})
	if err != nil {
		grpclog.Fatalln("Cannot setup tls config:", err)
	}
	fs, err := NewOortFS(*oortValueHost, *oortGroupHost, copt)
	if err != nil {
		grpclog.Fatalln(err)
	}
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	FatalIf(err, "Failed to bind to port")
	pb.RegisterApiServer(s, NewApiServer(fs))
	grpclog.Printf("Starting up on %d...\n", *port)
	s.Serve(l)
}
<commit_msg>Change SkipVerify default<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"

	pb "github.com/creiht/formic/proto"
	"github.com/pandemicsyn/ftls"

	"net"
)

var (
	usetls              = flag.Bool("tls", true, "Connection uses TLS if true, else plain TCP")
	certFile            = flag.String("cert_file", "/etc/oort/server.crt", "The TLS cert file")
	keyFile             = flag.String("key_file", "/etc/oort/server.key", "The TLS key file")
	port                = flag.Int("port", 9443, "The server port")
	oortValueHost       = flag.String("oortvaluehost", "127.0.0.1:6379", "host:port to use when connecting to oort value")
	oortGroupHost       = flag.String("oortgrouphost", "127.0.0.1:6380", "host:port to use when connecting to oort group")
	insecureSkipVerify  = flag.Bool("skipverify", false, "don't verify cert")
	oortClientMutualTLS = flag.Bool("mutualtls", false, "whether or not the server expects mutual tls auth")
	oortClientCert      = flag.String("oort-client-cert", "/etc/oort/client.crt", "cert file to use")
	oortClientKey       = flag.String("oort-client-key", "/etc/oort/client.key", "key file to use")
	oortClientCA        = flag.String("oort-client-ca", "/etc/oort/ca.pem", "ca file to use")
)

// FatalIf is just a lazy log/panic on error func
func FatalIf(err error, msg string) {
	if err != nil {
		grpclog.Fatalf("%s: %v", msg, err)
	}
}

func main() {
	flag.Parse()

	envtls := os.Getenv("FORMICD_TLS")
	if envtls == "true" {
		*usetls = true
	}

	envoortvhost := os.Getenv("FORMICD_OORT_VALUE_HOST")
	if envoortvhost != "" {
		*oortValueHost = envoortvhost
	}

	envoortghost := os.Getenv("FORMICD_OORT_GROUP_HOST")
	if envoortghost != "" {
		*oortGroupHost = envoortghost
	}

	envport := os.Getenv("FORMICD_PORT")
	if envport != "" {
		p, err := strconv.Atoi(envport)
		if err != nil {
			log.Println("Did not send valid port from env:", err)
		} else {
			*port = p
		}
	}

	envcert := os.Getenv("FORMICD_CERT_FILE")
	if envcert != "" {
		*certFile = envcert
	}

	envkey := os.Getenv("FORMICD_KEY_FILE")
	if envkey != "" {
		*keyFile = envkey
	}
	envSkipVerify := os.Getenv("FORMICD_INSECURE_SKIP_VERIFY")
	if envSkipVerify == "true" {
		*insecureSkipVerify = true
	}
	envMutualTLS := os.Getenv("FORMICD_MUTUAL_TLS")
	if envMutualTLS == "true" {
		*oortClientMutualTLS = true
	}
	envClientCA := os.Getenv("FORMICD_CLIENT_CA_FILE")
	if envClientCA != "" {
		*oortClientCA = envClientCA
	}
	envClientCert := os.Getenv("FORMICD_CLIENT_CERT_FILE")
	if envClientCert != "" {
		*oortClientCert = envClientCert
	}
	envClientKey := os.Getenv("FORMICD_CLIENT_KEY_FILE")
	if envClientKey != "" {
		*oortClientKey = envClientKey
	}

	var opts []grpc.ServerOption
	if *usetls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		FatalIf(err, "Couldn't load cert from file")
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	s := grpc.NewServer(opts...)
	copt, err := ftls.NewGRPCClientDialOpt(&ftls.Config{
		MutualTLS:          *oortClientMutualTLS,
		InsecureSkipVerify: *insecureSkipVerify,
		CertFile:           *oortClientCert,
		KeyFile:            *oortClientKey,
		CAFile:             *oortClientCA,
	})
	if err != nil {
		grpclog.Fatalln("Cannot setup tls config:", err)
	}
	fs, err := NewOortFS(*oortValueHost, *oortGroupHost, copt)
	if err != nil {
		grpclog.Fatalln(err)
	}
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	FatalIf(err, "Failed to bind to port")
	pb.RegisterApiServer(s, NewApiServer(fs))
	grpclog.Printf("Starting up on %d...\n", *port)
	s.Serve(l)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/zlib"
	"crypto/aes"
	"crypto/cipher"
	"encoding/base64"
	"fmt"
	"github.com/golang/snappy"
	goversion "github.com/hashicorp/go-version"
	flag "github.com/spf13/pflag"
	"io"
	"os"
)

var (
	version       string
	showVersion   bool
	inputFile     string
	outputFile    string
	encryptionKey string
	agentVersion  string
)

func init() {
	flag.BoolVarP(&showVersion, "version", "v", false, "Output version information and exit")
	flag.StringVarP(&inputFile, "backup-file", "i", "", "Binlogic CloudBackup file path")
	flag.StringVarP(&outputFile, "output-file", "o", "", "Outfile to save backup decrypted and uncompressed")
	flag.StringVarP(&encryptionKey, "encryption-key", "e", "", "Encryption key to decrypt backup file")
	flag.StringVarP(&agentVersion, "agent-version", "", "", "Agent version used to take the backup (if empty it's assumed <= 1.2.0)")
}

func showError(message error) {
	fmt.Fprintf(os.Stderr, "%s\n", message)
	os.Exit(0)
}

func parseArgs(args []string) error {
	flag.CommandLine.Parse(args[1:])

	if showVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	if len(inputFile) == 0 {
		return fmt.Errorf("Backup file is mandatory")
	}

	if len(outputFile) == 0 {
		return fmt.Errorf("Output file is mandatory")
	}

	if len(encryptionKey) == 0 {
		return fmt.Errorf("Encryption key is mandatory")
	}

	return nil
}

func main() {
	err := parseArgs(os.Args)
	if err != nil {
		showError(err)
	}

	err = prepareBackupFile(inputFile, encryptionKey, outputFile)
	if err != nil {
		showError(err)
	}

	fmt.Println("Process completed successfully")
	os.Exit(0)
}

func validateInputFile(filename string) error {
	finfo, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return fmt.Errorf("Backup file doesn't exists")
		}
	}

	if finfo.IsDir() {
		return fmt.Errorf("Expecting a file but %s is a directory", filename)
	}

	return nil
}

func validateOutputFile(filename string) error {
	_, err := os.Stat(filename)
	if err == nil {
		return fmt.Errorf("Output file exists %s. Please remove it", filename)
	}

	return nil
}

func getCipherStream(key string) (cipher.Stream, error) {
	keyBytes, err := base64.URLEncoding.DecodeString(key)
	if err != nil {
		return nil, err
	}

	block, err := aes.NewCipher(keyBytes)
	if err != nil {
		return nil, err
	}

	var iv [aes.BlockSize]byte

	return cipher.NewOFB(block, iv[:]), nil
}

func getCipherReader(key string, wrappedReader io.Reader) (io.Reader, error) {
	if key == "" {
		return wrappedReader, nil
	}

	stream, err := getCipherStream(key)
	if err != nil {
		return nil, err
	}

	return &cipher.StreamReader{S: stream, R: wrappedReader}, nil
}

func prepareBackupFile(filename, encryptionKey, output string) error {
	err := validateInputFile(filename)
	if err != nil {
		return fmt.Errorf("%s", err)
	}

	err = validateOutputFile(output)
	if err != nil {
		return fmt.Errorf("%s", err)
	}

	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Fail to open %s: %s", filename, err)
	}
	defer f.Close()

	is120OrLess, err := versionConstraint(agentVersion, "<= 1.2.0")

	if err != nil {
		return fmt.Errorf("%s creating version constraint", err)
	}

	var reader io.Reader

	if agentVersion == "" || is120OrLess {
		zipReader, err := zlib.NewReader(f)
		if err != nil {
			return fmt.Errorf("Fail opening zlib reader: %s. Check the agent version you took the backup with", err)
		}

		reader, err = getCipherReader(encryptionKey, zipReader)
		if err != nil {
			return fmt.Errorf("Fail to decrypt file %s using %s. Be sure encryption key is correct",
				filename, encryptionKey)
		}
	} else {
		cipherReader, err := getCipherReader(encryptionKey, f)

		if err != nil {
			return fmt.Errorf("%s while creating encrypted stream", err)
		}

		reader = snappy.NewReader(cipherReader)
	}

	of, err := os.Create(output)
	if err != nil {
		return fmt.Errorf("Fail to create %s: %s", output, err)
	}
	defer of.Close()

	_, err = io.Copy(of, reader)
	if err != nil {
		return fmt.Errorf("Fail while preparing backup file: %s", err)
	}

	return nil
}

func versionConstraint(ver, constraint string) (bool, error) {
	if ver == "0.0.0" || ver == "develop" || ver == "" {
		return false, nil
	}

	agentV, err := goversion.NewVersion(ver)

	if err != nil {
		return false, err
	}

	if c, err := goversion.NewConstraint(constraint); err != nil {
		return false, err
	} else {
		return c.Check(agentV), nil
	}
}
<commit_msg>CB-571 :: allow no compression-encryption for 1.11<commit_after>package main

import (
	"compress/zlib"
	"crypto/aes"
	"crypto/cipher"
	"encoding/base64"
	"fmt"
	"github.com/golang/snappy"
	goversion "github.com/hashicorp/go-version"
	flag "github.com/spf13/pflag"
	"io"
	"os"
)

var (
	version       string
	showVersion   bool
	inputFile     string
	outputFile    string
	encryptionKey string
	agentVersion  string
	decompress    bool
	decrypt       bool
)

func init() {
	flag.BoolVarP(&showVersion, "version", "v", false, "Output version information and exit")
	flag.StringVarP(&inputFile, "backup-file", "i", "", "Binlogic CloudBackup file path")
	flag.StringVarP(&outputFile, "output-file", "o", "", "Outfile to save backup decrypted and uncompressed")
	flag.StringVarP(&encryptionKey, "encryption-key", "e", "", "Encryption key to decrypt backup file")
	flag.StringVarP(&agentVersion, "agent-version", "", "", "Agent version used to take the backup (if empty it's assumed <= 1.2.0)")

	flag.BoolVarP(&decompress, "decompress", "z", false, "If agent version >= 1.10 then you need to specify "+
		"wether the backup needs to be decompressed or not")

	flag.BoolVarP(&decrypt, "decrypt", "y", false, "If agent version >= 1.10 then you need to specify "+
		"wether the backup needs to be decrypted or not")
}

func showError(message error) {
	fmt.Fprintf(os.Stderr, "%s\n", message)
	os.Exit(1)
}

func parseArgs(args []string) error {
	flag.CommandLine.Parse(args[1:])

	if showVersion {
		fmt.Fprintf(os.Stderr, "%s\n", version)
		os.Exit(0)
	}

	if len(inputFile) == 0 {
		return fmt.Errorf("Backup file is mandatory. Use -i option to pass it")
	}

	if len(outputFile) == 0 {
		return fmt.Errorf("Output file is mandatory. Use -o option to pass it")
	}

	if decrypt && len(encryptionKey) == 0 {
		return fmt.Errorf("Encryption key is mandatory. Use -e option to pass it")
	}

	return nil
}

func main() {
	err := parseArgs(os.Args)
	if err != nil {
		showError(err)
	}

	err = prepareBackupFile(inputFile, encryptionKey, outputFile)
	if err != nil {
		showError(err)
	}

	fmt.Fprintf(os.Stderr, "Process completed successfully\n")
	os.Exit(0)
}

func validateInputFile(filename string) error {
	finfo, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return fmt.Errorf("Backup file doesn't exists")
		}
	}

	if finfo.IsDir() {
		return fmt.Errorf("Expecting a file but %s is a directory", filename)
	}

	return nil
}

func validateOutputFile(filename string) error {
	_, err := os.Stat(filename)
	if err == nil {
		return fmt.Errorf("Output file exists %s. Please remove it", filename)
	}

	return nil
}

func getCipherStream(key string) (cipher.Stream, error) {
	keyBytes, err := base64.URLEncoding.DecodeString(key)
	if err != nil {
		return nil, err
	}

	block, err := aes.NewCipher(keyBytes)
	if err != nil {
		return nil, err
	}

	var iv [aes.BlockSize]byte

	return cipher.NewOFB(block, iv[:]), nil
}

func getCipherReader(key string, wrappedReader io.Reader) (io.Reader, error) {
	if key == "" {
		return wrappedReader, nil
	}

	stream, err := getCipherStream(key)
	if err != nil {
		return nil, err
	}

	return &cipher.StreamReader{S: stream, R: wrappedReader}, nil
}

func prepareBackupFile(filename, encryptionKey, output string) error {
	err := validateInputFile(filename)
	if err != nil {
		return fmt.Errorf("%s", err)
	}

	err = validateOutputFile(output)
	if err != nil {
		return fmt.Errorf("%s", err)
	}

	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Fail to open %s: %s", filename, err)
	}
	defer f.Close()

	is120OrLess, err := versionConstraint(agentVersion, "<= 1.2.0")

	if err != nil {
		return fmt.Errorf("%s creating version constraint", err)
	}

	is110OrLess, err := versionConstraint(agentVersion, "<= 1.10.0")

	if err != nil {
		return fmt.Errorf("%s creating version constraint", err)
	}

	var reader io.Reader

	if agentVersion == "" || is120OrLess {
		fmt.Fprint(os.Stderr, "Using old zlib reader\n")

		zipReader, err := zlib.NewReader(f)
		if err != nil {
			return fmt.Errorf("Fail opening zlib reader: %s. Check the agent version you took "+
				"the backup with and pass it with the --agent-version option", err)
		}

		reader, err = getCipherReader(encryptionKey, zipReader)
		if err != nil {
			return fmt.Errorf("Fail to decrypt file %s using %s. Be sure encryption key is correct",
				filename, encryptionKey)
		}
	} else if is110OrLess {
		fmt.Fprint(os.Stderr, "Using snappy reader\n")

		cipherReader, err := getCipherReader(encryptionKey, f)

		if err != nil {
			return fmt.Errorf("%s while creating encrypted stream", err)
		}

		reader = snappy.NewReader(cipherReader)
	} else {
		reader = f

		if decrypt {
			fmt.Fprint(os.Stderr, "Using cipher reader\n")

			cipherReader, err := getCipherReader(encryptionKey, f)

			if err != nil {
				return fmt.Errorf("%s while creating encrypted stream", err)
			}

			reader = cipherReader
		} else {
			fmt.Fprint(os.Stderr, "Not decrypting backup stream, if your file was "+
				"encrypted, the output file may be corrupt\n")
		}

		if decompress {
			fmt.Fprint(os.Stderr, "Using snappy reader\n")

			reader = snappy.NewReader(reader)
		} else {
			fmt.Fprint(os.Stderr, "Not decompressing backup stream, if your file was "+
				"compressed, the output file may be corrupt\n")
		}
	}

	of, err := os.Create(output)
	if err != nil {
		return fmt.Errorf("Failed to create %s: %s", output, err)
	}
	defer of.Close()

	_, err = io.Copy(of, reader)
	if err != nil {
		return fmt.Errorf("Fail while preparing backup file: %s", err)
	}

	return nil
}

func versionConstraint(ver, constraint string) (bool, error) {
	if ver == "0.0.0" || ver == "develop" || ver == "" {
		return false, nil
	}

	agentV, err := goversion.NewVersion(ver)

	if err != nil {
		return false, err
	}

	if c, err := goversion.NewConstraint(constraint); err != nil {
		return false, err
	} else {
		return c.Check(agentV), nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	_ "expvar"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	gocontext "golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/docker/containerd"
	api "github.com/docker/containerd/api/services/execution"
	"github.com/docker/containerd/content"
	"github.com/docker/containerd/log"
	"github.com/docker/containerd/utils"
	metrics "github.com/docker/go-metrics"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const usage = `
                    __        _                     __
  _________  ____  / /_____ _(_)___  ___  _________/ /
 / ___/ __ \/ __ \/ __/ __ ` + "`" + `/ / __ \/ _ \/ ___/ __  /
/ /__/ /_/ / / / / /_/ /_/ / / / / /  __/ /  / /_/ /
\___/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/   \__,_/

high performance container runtime
`

var (
	conf   = defaultConfig()
	global = log.WithModule(gocontext.Background(), "containerd")
)

func init() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Name, containerd.Package, c.App.Version)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "containerd"
	app.Version = containerd.Version
	app.Usage = usage
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config,c",
			Usage: "path to the configuration file",
			Value: "/etc/containerd/config.toml",
		},
		cli.StringFlag{
			Name:  "log-level,l",
			Usage: "set the logging level [debug, info, warn, error, fatal, panic]",
		},
		cli.StringFlag{
			Name:  "state",
			Usage: "containerd state directory",
		},
		cli.StringFlag{
			Name:  "socket,s",
			Usage: "socket path for containerd's GRPC server",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "containerd root directory",
		},
	}
	app.Before = before
	app.Action = func(context *cli.Context) error {
		start := time.Now()
		// start the signal handler as soon as we can to make sure that
		// we don't miss any signals during boot
		signals := make(chan os.Signal, 2048)
		signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
		log.G(global).Info("starting containerd boot...")

		// load all plugins into containerd
		if err := containerd.Load(filepath.Join(conf.Root, "plugins")); err != nil {
			return err
		}
		// start debug and metrics APIs
		if err := serveDebugAPI(); err != nil {
			return err
		}
		runtimes, err := loadRuntimes()
		if err != nil {
			return err
		}
		store, err := resolveContentStore()
		if err != nil {
			return err
		}
		services, err := loadServices(runtimes, store)
		if err != nil {
			return err
		}
		// start the GRPC api with the execution service registered
		server := newGRPCServer()
		for _, service := range services {
			if err := service.Register(server); err != nil {
				return err
			}
		}
		if err := serveGRPC(server); err != nil {
			return err
		}
		// start the prometheus metrics API for containerd
		serveMetricsAPI()

		log.G(global).Infof("containerd successfully booted in %fs", time.Now().Sub(start).Seconds())
		return handleSignals(signals, server)
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
		os.Exit(1)
	}
}

func before(context *cli.Context) error {
	if err := loadConfig(context.GlobalString("config")); err != nil &&
		!os.IsNotExist(err) {
		return err
	}
	// the order for config vs flag values is that flags will always override
	// the config values if they are set
	if err := setLevel(context); err != nil {
		return err
	}
	for _, v := range []struct {
		name string
		d    *string
	}{
		{
			name: "root",
			d:    &conf.Root,
		},
		{
			name: "state",
			d:    &conf.State,
		},
		{
			name: "socket",
			d:    &conf.GRPC.Socket,
		},
	} {
		if s := context.GlobalString(v.name); s != "" {
			*v.d = s
		}
	}
	return nil
}

func setLevel(context *cli.Context) error {
	l := context.GlobalString("log-level")
	if l == "" {
		l = conf.Debug.Level
	}
	if l != "" {
		lvl, err := logrus.ParseLevel(l)
		if err != nil {
			return err
		}
		logrus.SetLevel(lvl)
	}
	return nil
}

func serveMetricsAPI() {
	if conf.Metrics.Address != "" {
		log.G(global).WithField("metrics", conf.Metrics.Address).Info("starting metrics API...")
		h := newMetricsHandler()
		go func() {
			if err := http.ListenAndServe(conf.Metrics.Address, h); err != nil {
				log.G(global).WithError(err).Fatal("serve metrics API")
			}
		}()
	}
}

func newMetricsHandler() http.Handler {
	m := http.NewServeMux()
	m.Handle("/metrics", metrics.Handler())
	return m
}

func serveDebugAPI() error {
	path := conf.Debug.Socket
	if path == "" {
		return errors.New("debug socket path cannot be empty")
	}
	l, err := utils.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	log.G(global).WithField("debug", path).Info("starting debug API...")
	go func() {
		defer l.Close()
		// pprof and expvars are imported and automatically register their endpoints
		// under /debug
		if err := http.Serve(l, nil); err != nil {
			log.G(global).WithError(err).Fatal("serve debug API")
		}
	}()
	return nil
}

func resolveContentStore() (*content.Store, error) {
	cp := filepath.Join(conf.Root, "content")
	return content.NewStore(cp)
}

func loadRuntimes() (map[string]containerd.Runtime, error) {
	o := make(map[string]containerd.Runtime)
	for name, rr := range containerd.Registrations() {
		if rr.Type != containerd.RuntimePlugin {
			continue
		}
		log.G(global).Infof("loading runtime plugin %q...", name)
		ic := &containerd.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("runtime-%s", name)),
		}
		if rr.Config != nil {
			if err := conf.decodePlugin(name, rr.Config); err != nil {
				return nil, err
			}
			ic.Config = rr.Config
		}
		vr, err := rr.Init(ic)
		if err != nil {
			return nil, err
		}
		o[name] = vr.(containerd.Runtime)
	}
	return o, nil
}

func newGRPCServer() *grpc.Server {
	s := grpc.NewServer(grpc.UnaryInterceptor(interceptor))
	return s
}

func loadServices(runtimes map[string]containerd.Runtime, store *content.Store) ([]containerd.Service, error) {
	var o []containerd.Service
	for name, sr := range containerd.Registrations() {
		if sr.Type != containerd.GRPCPlugin {
			continue
		}
		log.G(global).Infof("loading grpc service plugin %q...", name)
		ic := &containerd.InitContext{
			Root:     conf.Root,
			State:    conf.State,
			Context:  log.WithModule(global, fmt.Sprintf("service-%s", name)),
			Runtimes: runtimes,
			Store:    store,
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		vs, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}
		o = append(o, vs.(containerd.Service))
	}
	return o, nil
}

func serveGRPC(server *grpc.Server) error {
	path := conf.GRPC.Socket
	if path == "" {
		return errors.New("--socket path cannot be empty")
	}
	l, err := utils.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	go func() {
		defer l.Close()
		if err := server.Serve(l); err != nil {
			log.G(global).WithError(err).Fatal("serve GRPC")
		}
	}()
	return nil
}

func interceptor(ctx gocontext.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler,
) (interface{}, error) {
	ctx = log.WithModule(ctx, "containerd")
	switch info.Server.(type) {
	case api.ContainerServiceServer:
		ctx = log.WithModule(global, "execution")
	default:
		fmt.Printf("unknown GRPC server type: %#v\n", info.Server)
	}
	return handler(ctx, req)
}

func handleSignals(signals chan os.Signal, server *grpc.Server) error {
	for s := range signals {
		log.G(global).WithField("signal", s).Debug("received signal")
		switch s {
		default:
			server.Stop()
			return nil
		}
	}
	return nil
}
<commit_msg>cmd/containerd: set correct module on interceptor for content<commit_after>package main

import (
	_ "expvar"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	gocontext "golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/docker/containerd"
	contentapi "github.com/docker/containerd/api/services/content"
	api "github.com/docker/containerd/api/services/execution"
	"github.com/docker/containerd/content"
	"github.com/docker/containerd/log"
	"github.com/docker/containerd/utils"
	metrics "github.com/docker/go-metrics"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const usage = `
                    __        _                     __
  _________  ____  / /_____ _(_)___  ___  _________/ /
 / ___/ __ \/ __ \/ __/ __ ` + "`" + `/ / __ \/ _ \/ ___/ __  /
/ /__/ /_/ / / / / /_/ /_/ / / / / /  __/ /  / /_/ /
\___/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/   \__,_/

high performance container runtime
`

var (
	conf   = defaultConfig()
	global = log.WithModule(gocontext.Background(), "containerd")
)

func init() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Name, containerd.Package, c.App.Version)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "containerd"
	app.Version = containerd.Version
	app.Usage = usage
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config,c",
			Usage: "path to the configuration file",
			Value: "/etc/containerd/config.toml",
		},
		cli.StringFlag{
			Name:  "log-level,l",
			Usage: "set the logging level [debug, info, warn, error, fatal, panic]",
		},
		cli.StringFlag{
			Name:  "state",
			Usage: "containerd state directory",
		},
		cli.StringFlag{
			Name:  "socket,s",
			Usage: "socket path for containerd's GRPC server",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "containerd root directory",
		},
	}
	app.Before = before
	app.Action = func(context *cli.Context) error {
		start := time.Now()
		// start the signal handler as soon as we can to make sure that
		// we don't miss any signals during boot
		signals := make(chan os.Signal, 2048)
		signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
		log.G(global).Info("starting containerd boot...")

		// load all plugins into containerd
		if err := containerd.Load(filepath.Join(conf.Root, "plugins")); err != nil {
			return err
		}
		// start debug and metrics APIs
		if err := serveDebugAPI(); err != nil {
			return err
		}
		runtimes, err := loadRuntimes()
		if err != nil {
			return err
		}
		store, err := resolveContentStore()
		if err != nil {
			return err
		}
		services, err := loadServices(runtimes, store)
		if err != nil {
			return err
		}
		// start the GRPC api with the execution service registered
		server := newGRPCServer()
		for _, service := range services {
			if err := service.Register(server); err != nil {
				return err
			}
		}
		if err := serveGRPC(server); err != nil {
			return err
		}
		// start the prometheus metrics API for containerd
		serveMetricsAPI()

		log.G(global).Infof("containerd successfully booted in %fs", time.Now().Sub(start).Seconds())
		return handleSignals(signals, server)
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
		os.Exit(1)
	}
}

func before(context *cli.Context) error {
	if err := loadConfig(context.GlobalString("config")); err != nil &&
		!os.IsNotExist(err) {
		return err
	}
	// the order for config vs flag values is that flags will always override
	// the config values if they are set
	if err := setLevel(context); err != nil {
		return err
	}
	for _, v := range []struct {
		name string
		d    *string
	}{
		{
			name: "root",
			d:    &conf.Root,
		},
		{
			name: "state",
			d:    &conf.State,
		},
		{
			name: "socket",
			d:    &conf.GRPC.Socket,
		},
	} {
		if s := context.GlobalString(v.name); s != "" {
			*v.d = s
		}
	}
	return nil
}

func setLevel(context *cli.Context) error {
	l := context.GlobalString("log-level")
	if l == "" {
		l = conf.Debug.Level
	}
	if l != "" {
		lvl, err := logrus.ParseLevel(l)
		if err != nil {
			return err
		}
		logrus.SetLevel(lvl)
	}
	return nil
}

func serveMetricsAPI() {
	if conf.Metrics.Address != "" {
		log.G(global).WithField("metrics", conf.Metrics.Address).Info("starting metrics API...")
		h := newMetricsHandler()
		go func() {
			if err := http.ListenAndServe(conf.Metrics.Address, h); err != nil {
				log.G(global).WithError(err).Fatal("serve metrics API")
			}
		}()
	}
}

func newMetricsHandler() http.Handler {
	m := http.NewServeMux()
	m.Handle("/metrics", metrics.Handler())
	return m
}

func serveDebugAPI() error {
	path := conf.Debug.Socket
	if path == "" {
		return errors.New("debug socket path cannot be empty")
	}
	l, err := utils.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	log.G(global).WithField("debug", path).Info("starting debug API...")
	go func() {
		defer l.Close()
		// pprof and expvars are imported and automatically register their endpoints
		// under /debug
		if err := http.Serve(l, nil); err != nil {
			log.G(global).WithError(err).Fatal("serve debug API")
		}
	}()
	return nil
}

func resolveContentStore() (*content.Store, error) {
	cp := filepath.Join(conf.Root, "content")
	return content.NewStore(cp)
}

func loadRuntimes() (map[string]containerd.Runtime, error) {
	o := make(map[string]containerd.Runtime)
	for name, rr := range containerd.Registrations() {
		if rr.Type != containerd.RuntimePlugin {
			continue
		}
		log.G(global).Infof("loading runtime plugin %q...", name)
		ic := &containerd.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("runtime-%s", name)),
		}
		if rr.Config != nil {
			if err := conf.decodePlugin(name, rr.Config); err != nil {
				return nil, err
			}
			ic.Config = rr.Config
		}
		vr, err := rr.Init(ic)
		if err != nil {
			return nil, err
		}
		o[name] = vr.(containerd.Runtime)
	}
	return o, nil
}

func newGRPCServer() *grpc.Server {
	s := grpc.NewServer(grpc.UnaryInterceptor(interceptor))
	return s
}

func loadServices(runtimes map[string]containerd.Runtime, store *content.Store) ([]containerd.Service, error) {
	var o []containerd.Service
	for name, sr := range containerd.Registrations() {
		if sr.Type != containerd.GRPCPlugin {
			continue
		}
		log.G(global).Infof("loading grpc service plugin %q...", name)
		ic := &containerd.InitContext{
			Root:     conf.Root,
			State:    conf.State,
			Context:  log.WithModule(global, fmt.Sprintf("service-%s", name)),
			Runtimes: runtimes,
			Store:    store,
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		vs, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}
		o = append(o, vs.(containerd.Service))
	}
	return o, nil
}

func serveGRPC(server *grpc.Server) error {
	path := conf.GRPC.Socket
	if path == "" {
		return errors.New("--socket path cannot be empty")
	}
	l, err := utils.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	go func() {
		defer l.Close()
		if err := server.Serve(l); err != nil {
			log.G(global).WithError(err).Fatal("serve GRPC")
		}
	}()
	return nil
}

func interceptor(ctx gocontext.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler,
) (interface{}, error) {
	ctx = log.WithModule(ctx, "containerd")
	switch info.Server.(type) {
	case api.ContainerServiceServer:
		ctx = log.WithModule(ctx, "execution")
	case contentapi.ContentServer:
		ctx = log.WithModule(ctx, "content")
	default:
		fmt.Printf("unknown GRPC server type: %#v\n", info.Server)
	}
	return handler(ctx, req)
}

func handleSignals(signals chan os.Signal, server *grpc.Server) error {
	for s := range signals {
		log.G(global).WithField("signal", s).Debug("received signal")
		switch s {
		default:
			server.Stop()
			return nil
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// fontgen is a commandline tool for generating pixel fonts supported by pixfont.
// First is to create an image of your pixel font in your favorite graphics
// program with your set of supported characters. Ex:
//
//      ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789
//
// Ensure that there is a solid color background, single-color font pixels (i.e.
// no anti-aliasing), and that a column of pixels separate each character to
// ensure best results. Then simply run:
//
//      ./fontgen -img mypixelfont.png -o myfont
//
// Add myfont.go to your project, then just use Font.DrawString(...) to add
// text to your image!
//
package main

import (
	"flag"
	"fmt"
	"go/format"
	"image"
	"image/color"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"unicode/utf8"

	"github.com/pbnjay/pixfont"
)

var (
	imageName = flag.String("img", "", "image file to extract pixel font from")
	startY    = flag.Int("y", 0, "starting Y position")
	height    = flag.Int("h", 0, "chop height")
	startX    = flag.Int("x", 0, "starting X position")
	width     = flag.Int("w", 0, "chop width")
	alphabet  = flag.String("a", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "alphabet to extract")
	varWidth  = flag.Bool("v", false, "produce variable width font")

	textName = flag.String("txt", "", "text file to extract pixel font from")
	outName  = flag.String("o", "", "package name to create (becomes <myfont>.go)")
)

func generatePixFont(name string, w, h int, v bool, d map[rune]map[int]string) {
	template := `
		package %s

		import "github.com/pbnjay/pixfont"

		var Font *pixfont.PixFont

		func init() {
			charMap := %#v
			data := %#v
			Font = pixfont.NewPixFont(%d, %d, charMap, data)
			Font.SetVariableWidth(%t)
		}
	`
	encoded := []uint32{}
	cm := make(map[rune]uint16)

	// convert from simple character encoding to packed bitfield
	// NB fonts should be at most 32 pixels wide to fit in the uint32
	//    (height is limited to uint8 255)
	idx := 0
	sub := -1 - (w / 8)
	for c, matrix := range d {
		sub += 1 + (w / 8)
		if sub > 3 {
			sub = 0
			idx = len(encoded)
		}

		cm[c] = uint16(sub) | (uint16(idx) << 2)
		for y := 0; y < h; y++ {
			var line uint32
			var b uint32 = 1
			if sub != 0 {
				line = encoded[idx+y]
				b <<= uint(8 * sub)
			}

			if ld, hasLine := matrix[y]; hasLine {
				for x := 0; x < w; x++ {
					if len(ld) > x && ld[x] == 'X' {
						line |= b
					}
					b <<= 1
				}
			}

			if sub == 0 {
				encoded = append(encoded, line)
			} else {
				encoded[idx+y] = line
			}
		}
	}

	fnt := pixfont.NewPixFont(uint8(w), uint8(h), cm, encoded)
	fnt.SetVariableWidth(v)

	f, err := os.OpenFile(name+".go", os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return
	}

	// draw a comment header using the new font
	sd := &pixfont.StringDrawable{}
	fnt.DrawString(sd, 0, 0, name, nil)
	fmt.Fprintln(f, sd.PrefixString("// "))

	// create the code from the template and go fmt it
	code := fmt.Sprintf(template, name, cm, encoded, w, h, v)
	bcode, _ := format.Source([]byte(code))
	fmt.Fprintln(f, string(bcode))

	f.Close()
}

func processImage(filename string) (allLetters map[rune]map[int]string, maxWidth int) {
	f, err := os.Open(filename)
	if err != nil {
		fmt.Fprint(os.Stderr, err.Error())
		return nil, 0
	}
	img, _, err := image.Decode(f)
	if err != nil {
		fmt.Fprint(os.Stderr, err.Error())
		return nil, 0
	}
	if *width == 0 {
		*width = img.Bounds().Dx() - *startX
	}
	if *height == 0 {
		*height = img.Bounds().Dy() - *startY
	}
	allLetters = make(map[rune]map[int]string)
	maxWidth = 0

	// generate a greyscale histogram of the image
	pxc := 0
	clrs := make(map[uint8]int)
	for y := 0; y < img.Bounds().Dy(); y++ {
		for x := 0; x < img.Bounds().Dx(); x++ {
			c := img.At(x, y)
			gc := color.GrayModel.Convert(c).(color.Gray)
			clrs[gc.Y]++
			pxc++
		}
	}
	// find a threshold pixel count for what colors to ignore as background
	// (ie assumes background image is fairly solid and colors occur much
	//  more often than font colors)
	pxt := pxc
	pxd := 0
	for pxd < (pxc/2) && pxt > 0 {
		pxt /= 2
		pxd = 0
		for _, n := range clrs {
			if n > pxt {
				pxd += n
			}
		}
	}

	// scan across the image in the crop region, saving pixels as you go.
	// if at any point we see an "empty" column of pixels, we assume it
	// is a character boundary and move to the next alphabet letter.
	curAlpha := 0
	curWidth := 0
	curLetter := make(map[int]string)
	for x := *startX; x < *startX+*width; x++ {
		curWidth++
		isEmpty := true
		ay := 0
		for y := *startY; y < *startY+*height; y++ {
			c := img.At(x, y)
			gc := color.GrayModel.Convert(c).(color.Gray)
			if clrs[gc.Y] <= pxt {
				if _, haveDots := curLetter[ay]; !haveDots {
					curLetter[ay] = strings.Repeat(" ", curWidth-1)
				}
				curLetter[ay] += "X"
				isEmpty = false
			} else {
				if _, haveDots := curLetter[ay]; haveDots {
					curLetter[ay] += " "
				}
			}
			ay++
		}

		if isEmpty {
			if len(curLetter) != 0 {
				if curAlpha < len(*alphabet) {
					curWidth-- // remove last blank column
					for yy, ln := range curLetter {
						if len(ln) >= curWidth {
							curLetter[yy] = ln[:curWidth]
						}
					}
					allLetters[rune((*alphabet)[curAlpha])] = curLetter
				}
				if curWidth > maxWidth {
					maxWidth = curWidth
				}
				curAlpha++
			}
			curWidth = 0
			curLetter = make(map[int]string)
		}
	}

	if *outName != "" {
		return
	}

	// output a simple text representation of the extracted characters
	for _, a := range *alphabet {
		if l, found := allLetters[a]; found {
			w := 0
			for yy := 0; yy < *height; yy++ {
				if len(l[yy]) > w {
					w = len(l[yy])
				}
			}

			leftPad := (maxWidth - w) / 2
			if *varWidth {
				leftPad = 0
			}
			for yy := 0; yy < *height; yy++ {
				l[yy] = strings.Repeat(" ", leftPad) + l[yy]
				fmt.Printf("%c  [%*s]\n", a, -maxWidth, l[yy])
			}
		}
	}
	return
}

func processText(filename string) (allLetters map[rune]map[int]string, maxWidth int) {
	newalpha := ""
	input, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return nil, 0
	}
	allLetters = make(map[rune]map[int]string)
	re := regexp.MustCompile(`[^\n]*\n`)
	count := 0
	hh := 0
	lastCh := rune(0)

	for _, bline := range re.FindAll(input, -1) {
		line := string(bline)
		c, pixoffs := utf8.DecodeRuneInString(line)
		pixoffs += 3
		if lastCh != c {
			count = 0
			hh = len(allLetters[lastCh])
			allLetters[c] = make(map[int]string)
			newalpha += string(c)
		}
		ww := len(line) - (pixoffs + 2)
		if ww > maxWidth {
			maxWidth = ww
		}
		allLetters[c][count] = line[pixoffs : pixoffs+ww]
		lastCh = c
		count++
	}

	*alphabet = newalpha
	*height = hh

	if *outName != "" {
		return
	}

	// output the same representation again, to allow user to verify it was parsed correctly
	for _, a := range *alphabet {
		if l, found := allLetters[a]; found {
			w := 0
			for yy := 0; yy < *height; yy++ {
				if len(l[yy]) > w {
					w = len(l[yy])
				}
			}

			leftPad := (maxWidth - w) / 2
			for yy := 0; yy < *height; yy++ {
				l[yy] = strings.Repeat(" ", leftPad) + l[yy]
				fmt.Printf("%c  [%*s]\n", a, -maxWidth, l[yy])
			}
		}
	}
	return
}

func main() {
	flag.Parse()

	allLetters := make(map[rune]map[int]string)
	maxWidth := 0

	if *imageName != "" {
		allLetters, maxWidth = processImage(*imageName)
	} else if *textName != "" {
		allLetters, maxWidth = processText(*textName)
	} else {
		fmt.Fprintln(os.Stderr, "-img or -txt should be provided")
		flag.Usage()
		return
	}

	if *outName != "" {
		generatePixFont(*outName, maxWidth, *height, *varWidth, allLetters)
		fmt.Fprintln(os.Stderr, "Created package file:", *outName+".go")
	}
}
<commit_msg>fixes #7 by ensuring the alphabet is processed using runes, not bytes<commit_after>// fontgen is a commandline tool for generating pixel fonts supported by pixfont.
// First is to create an image of your pixel font in your favorite graphics
// program with your set of supported characters. Ex:
//
//      ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789
//
// Ensure that there is a solid color background, single-color font pixels (i.e.
// no anti-aliasing), and that a column of pixels separate each character to
// ensure best results. Then simply run:
//
//      ./fontgen -img mypixelfont.png -o myfont
//
// Add myfont.go to your project, then just use Font.DrawString(...) to add
// text to your image!
//
package main

import (
	"flag"
	"fmt"
	"go/format"
	"image"
	"image/color"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"unicode/utf8"

	"github.com/pbnjay/pixfont"
)

var (
	imageName = flag.String("img", "", "image file to extract pixel font from")
	startY    = flag.Int("y", 0, "starting Y position")
	height    = flag.Int("h", 0, "chop height")
	startX    = flag.Int("x", 0, "starting X position")
	width     = flag.Int("w", 0, "chop width")
	alphabet  = flag.String("a", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "alphabet to extract")
	varWidth  = flag.Bool("v", false, "produce variable width font")

	textName = flag.String("txt", "", "text file to extract pixel font from")
	outName  = flag.String("o", "", "package name to create (becomes <myfont>.go)")
)

func generatePixFont(name string, w, h int, v bool, d map[rune]map[int]string) {
	template := `
		package %s

		import "github.com/pbnjay/pixfont"

		var Font *pixfont.PixFont

		func init() {
			charMap := %#v
			data := %#v
			Font = pixfont.NewPixFont(%d, %d, charMap, data)
			Font.SetVariableWidth(%t)
		}
	`
	encoded := []uint32{}
	cm := make(map[rune]uint16)

	// convert from simple character encoding to packed bitfield
	// NB fonts should be at most 32 pixels wide to fit in the uint32
	//    (height is limited to uint8 255)
	idx := 0
	sub := -1 - (w / 8)
	for c, matrix := range d {
		sub += 1 + (w / 8)
		if sub > 3 {
			sub = 0
			idx = len(encoded)
		}

		cm[c] = uint16(sub) | (uint16(idx) << 2)
		for y := 0; y < h; y++ {
			var line uint32
			var b uint32 = 1
			if sub != 0 {
				line = encoded[idx+y]
				b <<= uint(8 * sub)
			}

			if ld, hasLine := matrix[y]; hasLine {
				for x := 0; x < w; x++ {
					if len(ld) > x && ld[x] == 'X' {
						line |= b
					}
					b <<= 1
				}
			}

			if sub == 0 {
				encoded = append(encoded, line)
			} else {
				encoded[idx+y] = line
			}
		}
	}

	fnt := pixfont.NewPixFont(uint8(w), uint8(h), cm, encoded)
	fnt.SetVariableWidth(v)

	f, err := os.OpenFile(name+".go", os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return
	}

	// draw a comment header using the new font
	sd := &pixfont.StringDrawable{}
	fnt.DrawString(sd, 0, 0, name, nil)
	fmt.Fprintln(f, sd.PrefixString("// "))

	// create the code from the template and go fmt it
	code := fmt.Sprintf(template, name, cm, encoded, w, h, v)
	bcode, _ := format.Source([]byte(code))
	fmt.Fprintln(f, string(bcode))

	f.Close()
}

func processImage(filename string) (allLetters map[rune]map[int]string, maxWidth int) {
	f, err := os.Open(filename)
	if err != nil {
		fmt.Fprint(os.Stderr, err.Error())
		return nil, 0
	}
	img, _, err := image.Decode(f)
	if err != nil {
		fmt.Fprint(os.Stderr, err.Error())
		return nil, 0
	}
	if *width == 0 {
		*width = img.Bounds().Dx() - *startX
	}
	if *height == 0 {
		*height = img.Bounds().Dy() - *startY
	}
	allLetters = make(map[rune]map[int]string)
	maxWidth = 0

	// generate a greyscale histogram of the image
	pxc := 0
	clrs := make(map[uint8]int)
	for y := 0; y < img.Bounds().Dy(); y++ {
		for x := 0; x < img.Bounds().Dx(); x++ {
			c := img.At(x, y)
			gc := color.GrayModel.Convert(c).(color.Gray)
			clrs[gc.Y]++
			pxc++
		}
	}
	// find a threshold pixel count for what colors to ignore as background
	// (ie assumes background image is fairly solid and colors occur much
	//  more often than font colors)
	pxt := pxc
	pxd := 0
	for pxd < (pxc/2) && pxt > 0 {
		pxt /= 2
		pxd = 0
		for _, n := range clrs {
			if n > pxt {
				pxd += n
			}
		}
	}

	// scan across the image in the crop region, saving pixels as you go.
	// if at any point we see an "empty" column of pixels, we assume it
	// is a character boundary and move to the next alphabet letter.
	curAlpha := *alphabet
	curWidth := 0
	curLetter := make(map[int]string)
	for x := *startX; x < *startX+*width; x++ {
		curWidth++
		isEmpty := true
		ay := 0
		for y := *startY; y < *startY+*height; y++ {
			c := img.At(x, y)
			gc := color.GrayModel.Convert(c).(color.Gray)
			if clrs[gc.Y] <= pxt {
				if _, haveDots := curLetter[ay]; !haveDots {
					curLetter[ay] = strings.Repeat(" ", curWidth-1)
				}
				curLetter[ay] += "X"
				isEmpty = false
			} else {
				if _, haveDots := curLetter[ay]; haveDots {
					curLetter[ay] += " "
				}
			}
			ay++
		}

		if isEmpty {
			if len(curLetter) != 0 {
				if len(curAlpha) > 0 {
					curWidth-- // remove last blank column
					for yy, ln := range curLetter {
						if len(ln) >= curWidth {
							curLetter[yy] = ln[:curWidth]
						}
					}
					r, nbytes := utf8.DecodeRuneInString(curAlpha)
					allLetters[r] = curLetter
					curAlpha = curAlpha[nbytes:]
				}
				if curWidth > maxWidth {
					maxWidth = curWidth
				}
			}
			curWidth = 0
			curLetter = make(map[int]string)
		}
	}

	if *outName != "" {
		return
	}

	// output a simple text representation of the extracted characters
	for _, a := range *alphabet {
		if l, found := allLetters[a]; found {
			w := 0
			for yy := 0; yy < *height; yy++ {
				if len(l[yy]) > w {
					w = len(l[yy])
				}
			}

			leftPad := (maxWidth - w) / 2
			if *varWidth {
				leftPad = 0
			}
			for yy := 0; yy < *height; yy++ {
				l[yy] = strings.Repeat(" ", leftPad) + l[yy]
				fmt.Printf("%c  [%*s]\n", a, -maxWidth, l[yy])
			}
		}
	}
	return
}

func processText(filename string) (allLetters map[rune]map[int]string, maxWidth int) {
	newalpha := ""
	input, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return nil, 0
	}
	allLetters = make(map[rune]map[int]string)
	re := regexp.MustCompile(`[^\n]*\n`)
	count := 0
	hh := 0
	lastCh := rune(0)

	for _, bline := range re.FindAll(input, -1) {
		line := string(bline)
		c, pixoffs := utf8.DecodeRuneInString(line)
		pixoffs += 3
		if lastCh != c {
			count = 0
			hh = len(allLetters[lastCh])
			allLetters[c] = make(map[int]string)
			newalpha += string(c)
		}
		ww := len(line) - (pixoffs + 2)
		if ww > maxWidth {
			maxWidth = ww
		}
		allLetters[c][count] = line[pixoffs : pixoffs+ww]
		lastCh = c
		count++
	}

	*alphabet = newalpha
	*height = hh

	if *outName != "" {
		return
	}

	// output the same representation again, to allow user to verify it was parsed correctly
	for _, a := range *alphabet {
		if l, found := allLetters[a]; found {
			w := 0
			for yy := 0; yy < *height; yy++ {
				if len(l[yy]) > w {
					w = len(l[yy])
				}
			}

			leftPad := (maxWidth - w) / 2
			for yy := 0; yy < *height; yy++ {
				l[yy] = strings.Repeat(" ", leftPad) + l[yy]
				fmt.Printf("%c  [%*s]\n", a, -maxWidth, l[yy])
			}
		}
	}
	return
}

func main() {
	flag.Parse()

	allLetters := make(map[rune]map[int]string)
	maxWidth := 0

	if *imageName != "" {
		allLetters, maxWidth = processImage(*imageName)
	} else if *textName != "" {
		allLetters, maxWidth = processText(*textName)
	} else {
		fmt.Fprintln(os.Stderr, "-img or -txt should be provided")
		flag.Usage()
		return
	}

	if *outName != "" {
		generatePixFont(*outName, maxWidth, *height, *varWidth, allLetters)
		fmt.Fprintln(os.Stderr, "Created package file:", *outName+".go")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"io/ioutil"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/gh2gcs"
	"k8s.io/release/pkg/github"
	"k8s.io/release/pkg/log"
)

// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
	Use:               "gh2gcs --org kubernetes --repo release --bucket <bucket> --release-dir <release-dir> [--tags v0.0.0] [--include-prereleases] [--output-dir <temp-dir>]",
	Short:             "gh2gcs uploads GitHub releases to Google Cloud Storage",
	Example:           "gh2gcs --org kubernetes --repo release --bucket k8s-staging-release-test --release-dir release --tags v0.0.0,v0.0.1",
	SilenceUsage:      true,
	SilenceErrors:     true,
	PersistentPreRunE: initLogging,
	RunE: func(*cobra.Command, []string) error {
		return run(opts)
	},
}

type options struct {
	org                string
	repo               string
	tags               []string
	includePrereleases bool
	bucket             string
	releaseDir         string
	outputDir          string
	logLevel           string
}

var opts = &options{}

var (
	orgFlag                = "org"
	repoFlag               = "repo"
	tagsFlag               = "tags"
	includePrereleasesFlag = "include-prereleases"
	bucketFlag             = "bucket"
	releaseDirFlag         = "release-dir"
	outputDirFlag          = "output-dir"

	requiredFlags = []string{
		orgFlag,
		repoFlag,
		bucketFlag,
		releaseDirFlag,
	}
)

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		logrus.Fatal(err)
	}
}

func init() {
	rootCmd.PersistentFlags().StringVar(
		&opts.org,
		orgFlag,
		"",
		"GitHub org/user",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.repo,
		repoFlag,
		"",
		"GitHub repo",
	)

	rootCmd.PersistentFlags().StringSliceVar(
		&opts.tags,
		tagsFlag,
		[]string{},
		"release tags to upload to GCS",
	)

	rootCmd.PersistentFlags().BoolVar(
		&opts.includePrereleases,
		includePrereleasesFlag,
		false,
		"specifies whether prerelease assets should be uploaded to GCS",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.bucket,
		bucketFlag,
		"",
		"GCS bucket to upload to",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.releaseDir,
		releaseDirFlag,
		"",
		"directory to upload to within the specified GCS bucket",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.outputDir,
		outputDirFlag,
		"",
		"local directory for releases to be downloaded to",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.logLevel,
		"log-level",
		"info",
		"the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'",
	)

	for _, flag := range requiredFlags {
		if err := rootCmd.MarkPersistentFlagRequired(flag); err != nil {
			logrus.Fatal(err)
		}
	}
}

func initLogging(*cobra.Command, []string) error {
	return log.SetupGlobalLogger(opts.logLevel)
}

func run(opts *options) error {
	if err := opts.SetAndValidate(); err != nil {
		return errors.Wrap(err, "validating gh2gcs options")
	}

	if err := gcp.PreCheck(); err != nil {
		return errors.Wrap(err, "pre-checking for GCP package usage")
	}

	// Create a real GitHub API client
	gh := github.New()

	// TODO: Support downloading releases via yaml config
	uploadConfig := &gh2gcs.Config{}
	releaseConfig := &gh2gcs.ReleaseConfig{
		Org:                opts.org,
		Repo:               opts.repo,
		Tags:               []string{},
		IncludePrereleases: opts.includePrereleases,
		GCSBucket:          opts.bucket,
		ReleaseDir:         opts.releaseDir,
		GCSCopyOptions:     gh2gcs.DefaultGCSCopyOptions,
	}

	// TODO: Expose certain GCSCopyOptions for user configuration

	if len(opts.tags) > 0 {
		releaseConfig.Tags = opts.tags
	} else {
		releaseTags, err := gh.GetReleaseTags(opts.org, opts.repo, opts.includePrereleases)
		if err != nil {
			return errors.Wrap(err, "getting release tags")
		}

		releaseConfig.Tags = releaseTags
	}

	uploadConfig.ReleaseConfigs = append(uploadConfig.ReleaseConfigs, *releaseConfig)

	for _, rc := range uploadConfig.ReleaseConfigs {
		if err := gh2gcs.DownloadReleases(&rc, gh, opts.outputDir); err != nil {
			return errors.Wrap(err, "downloading release assets")
		}

		if err := gh2gcs.Upload(&rc, gh, opts.outputDir); err != nil {
			return errors.Wrap(err, "uploading release assets to GCS")
		}
	}

	return nil
}

// SetAndValidate sets some default options and verifies if options are valid
func (o *options) SetAndValidate() error {
	logrus.Info("Validating gh2gcs options...")

	// TODO: Temp dir should cleanup after itself
	if o.outputDir == "" {
		tmpDir, err := ioutil.TempDir("", "gh2gcs")
		if err != nil {
			return errors.Wrap(err, "creating temp directory")
		}

		o.outputDir = tmpDir
	}

	return nil
}
<commit_msg>gh2gcs: clean temporary folder if output-dir is not set<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"io/ioutil"
	"os"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/gcp"
	"k8s.io/release/pkg/gh2gcs"
	"k8s.io/release/pkg/github"
	"k8s.io/release/pkg/log"
)

// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
	Use:               "gh2gcs --org kubernetes --repo release --bucket <bucket> --release-dir <release-dir> [--tags v0.0.0] [--include-prereleases] [--output-dir <temp-dir>]",
	Short:             "gh2gcs uploads GitHub releases to Google Cloud Storage",
	Example:           "gh2gcs --org kubernetes --repo release --bucket k8s-staging-release-test --release-dir release --tags v0.0.0,v0.0.1",
	SilenceUsage:      true,
	SilenceErrors:     true,
	PersistentPreRunE: initLogging,
	RunE: func(*cobra.Command, []string) error {
		return run(opts)
	},
}

type options struct {
	org                string
	repo               string
	tags               []string
	includePrereleases bool
	bucket             string
	releaseDir         string
	outputDir          string
	logLevel           string
}

var opts = &options{}

var (
	orgFlag                = "org"
	repoFlag               = "repo"
	tagsFlag               = "tags"
	includePrereleasesFlag = "include-prereleases"
	bucketFlag             = "bucket"
	releaseDirFlag         = "release-dir"
	outputDirFlag          = "output-dir"

	requiredFlags = []string{
		orgFlag,
		repoFlag,
		bucketFlag,
		releaseDirFlag,
	}
)

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		logrus.Fatal(err)
	}

	if !rootCmd.Flags().Changed(outputDirFlag) {
		logrus.Infof("Cleaning temporary directory %s", opts.outputDir)
		os.RemoveAll(opts.outputDir)
	}
}

func init() {
	rootCmd.PersistentFlags().StringVar(
		&opts.org,
		orgFlag,
		"",
		"GitHub org/user",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.repo,
		repoFlag,
		"",
		"GitHub repo",
	)

	rootCmd.PersistentFlags().StringSliceVar(
		&opts.tags,
		tagsFlag,
		[]string{},
		"release tags to upload to GCS",
	)

	rootCmd.PersistentFlags().BoolVar(
		&opts.includePrereleases,
		includePrereleasesFlag,
		false,
		"specifies whether prerelease assets should be uploaded to GCS",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.bucket,
		bucketFlag,
		"",
		"GCS bucket to upload to",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.releaseDir,
		releaseDirFlag,
		"",
		"directory to upload to within the specified GCS bucket",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.outputDir,
		outputDirFlag,
		"",
		"local directory for releases to be downloaded to",
	)

	rootCmd.PersistentFlags().StringVar(
		&opts.logLevel,
		"log-level",
		"info",
		"the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'",
	)

	for _, flag := range requiredFlags {
		if err := rootCmd.MarkPersistentFlagRequired(flag); err != nil {
			logrus.Fatal(err)
		}
	}
}

func initLogging(*cobra.Command, []string) error {
	return log.SetupGlobalLogger(opts.logLevel)
}

func run(opts *options) error {
	if err := opts.SetAndValidate(); err != nil {
		return errors.Wrap(err, "validating gh2gcs options")
	}

	if err := gcp.PreCheck(); err != nil {
		return errors.Wrap(err, "pre-checking for GCP package usage")
	}

	// Create a real GitHub API client
	gh := github.New()

	// TODO: Support downloading releases via yaml config
	uploadConfig := &gh2gcs.Config{}
	releaseConfig := &gh2gcs.ReleaseConfig{
		Org:                opts.org,
		Repo:               opts.repo,
		Tags:               []string{},
		IncludePrereleases: opts.includePrereleases,
		GCSBucket:          opts.bucket,
		ReleaseDir:         opts.releaseDir,
		GCSCopyOptions:     gh2gcs.DefaultGCSCopyOptions,
	}

	// TODO: Expose certain GCSCopyOptions for user configuration

	if len(opts.tags) > 0 {
		releaseConfig.Tags = opts.tags
	} else {
		releaseTags, err := gh.GetReleaseTags(opts.org, opts.repo, opts.includePrereleases)
		if err != nil {
			return errors.Wrap(err, "getting release tags")
		}

		releaseConfig.Tags = releaseTags
	}

	uploadConfig.ReleaseConfigs = append(uploadConfig.ReleaseConfigs, *releaseConfig)

	for _, rc := range uploadConfig.ReleaseConfigs {
		if err := gh2gcs.DownloadReleases(&rc, gh, opts.outputDir); err != nil {
			return errors.Wrap(err, "downloading release assets")
		}

		if err := gh2gcs.Upload(&rc, gh, opts.outputDir); err != nil {
			return errors.Wrap(err, "uploading release assets to GCS")
		}
	}

	return nil
}

// SetAndValidate sets some default options and verifies if options are valid
func (o *options) SetAndValidate() error {
	logrus.Info("Validating gh2gcs options...")

	if o.outputDir == "" {
		tmpDir, err := ioutil.TempDir("", "gh2gcs")
		if err != nil {
			return errors.Wrap(err, "creating temp directory")
		}

		o.outputDir = tmpDir
	}

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	log "github.com/funkygao/log4go"
)

type Ping struct {
	Ui  cli.Ui
	Cmd string

	zone     string
	logfile  string
	interval time.Duration
}

// TODO run 3 nodes in a zone to monitor as daemon
// register the 3 nodes as host service tag.
func (this *Ping) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ping", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.DurationVar(&this.interval, "interval", time.Minute*5, "")
	cmdFlags.StringVar(&this.logfile, "logfile", "gk.ping.log", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z").
		invalid(args) {
		return 2
	}

	this.setupLog()

	for {
		this.diagnose()

		time.Sleep(this.interval)
	}

	return
}

func (this *Ping) setupLog() {
	log.DeleteFilter("stdout")

	filer := log.NewFileLogWriter(this.logfile, false)
	filer.SetFormat("[%d %T] [%L] (%S) %M")
	filer.SetRotate(true)
	filer.SetRotateSize(0)
	filer.SetRotateLines(0)
	filer.SetRotateDaily(true)
	log.AddFilter("file", log.INFO, filer)
}

func (this *Ping) diagnose() {
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		registeredBrokers := zkcluster.RegisteredInfo().Roster
		for _, broker := range registeredBrokers {
			kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
			if err != nil {
				log.Error("broker %s: %v", broker.Addr(), err)
				continue
			}

			_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
			if err != nil {
				log.Error("broker %s: %v", broker.Addr(), err)
			}
			kfk.Close()
		}
	})

}

func (*Ping) Synopsis() string {
	return "Ping liveness of all registered brokers in a zone"
}

func (this *Ping) Help() string {
	help := fmt.Sprintf(`
Usage: %s ping -z zone [options]

    Ping liveness of all registered brokers in a zone

Options:

    -interval duration
      Defaults 5m

    -logfile filename
      Defaults gk.ping.log in current directory

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>debug ping<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	log "github.com/funkygao/log4go"
)

type Ping struct {
	Ui  cli.Ui
	Cmd string

	zone     string
	logfile  string
	interval time.Duration
}

// TODO run 3 nodes in a zone to monitor as daemon
// register the 3 nodes as host service tag.
func (this *Ping) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ping", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.DurationVar(&this.interval, "interval", time.Minute*5, "")
	cmdFlags.StringVar(&this.logfile, "logfile", "gk.ping.log", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z").
		invalid(args) {
		return 2
	}

	this.setupLog()

	for {
		this.diagnose()

		time.Sleep(this.interval)
	}

	return
}

func (this *Ping) setupLog() {
	log.DeleteFilter("stdout")

	filer := log.NewFileLogWriter(this.logfile, false)
	filer.SetFormat("[%d %T] [%L] (%S) %M")
	filer.SetRotate(true)
	filer.SetRotateSize(0)
	filer.SetRotateLines(0)
	filer.SetRotateDaily(true)
	log.AddFilter("file", log.DEBUG, filer)
}

func (this *Ping) diagnose() {
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		registeredBrokers := zkcluster.RegisteredInfo().Roster
		for _, broker := range registeredBrokers {
			log.Debug("ping %s", broker.Addr())
			kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
			if err != nil {
				log.Error("broker %s: %v", broker.Addr(), err)
				continue
			}

			_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
			if err != nil {
				log.Error("broker %s: %v", broker.Addr(), err)
			}
			kfk.Close()
		}
	})

}

func (*Ping) Synopsis() string {
	return "Ping liveness of all registered brokers in a zone"
}

func (this *Ping) Help() string {
	help := fmt.Sprintf(`
Usage: %s ping -z zone [options]

    Ping liveness of all registered brokers in a zone

Options:

    -interval duration
      Defaults 5m

    -logfile filename
      Defaults gk.ping.log in current directory

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"

	"github.com/inconshreveable/log15"
	"github.com/mmcloughlin/pearl"
	"github.com/mmcloughlin/pearl/log"
	"github.com/mmcloughlin/pearl/meta"
	"github.com/mmcloughlin/pearl/torconfig"
	"github.com/spf13/cobra"
)

// serveCmd represents the serve command
var serveCmd = &cobra.Command{
	Use:   "serve",
	Short: "Start a relay server",
	RunE: func(cmd *cobra.Command, args []string) error {
		return serve()
	},
}
var (
	nickname string
	port     int
	logfile  string
)

func init() {
	serveCmd.Flags().StringVarP(&nickname, "nickname", "n", "pearl", "nickname")
	serveCmd.Flags().IntVarP(&port, "port", "p", 9111, "relay port")
	serveCmd.Flags().StringVarP(&logfile, "logfile", "l", "pearl.json", "log file")

	rootCmd.AddCommand(serveCmd)
}

func serve() error {
	config := &torconfig.Config{
		Nickname: nickname,
		ORPort:   uint16(port),
		Platform: meta.Platform.String(),
		Contact:  "https://github.com/mmcloughlin/pearl",
	}

	base := log15.New()
	fh, err := log15.FileHandler(logfile, log15.JsonFormat())
	if err != nil {
		return err
	}
	base.SetHandler(log15.MultiHandler(
		log15.LvlFilterHandler(log15.LvlInfo,
			log15.StreamHandler(os.Stdout, log15.TerminalFormat()),
		),
		fh,
	))
	logger := log.NewLog15(base)

	r, err := pearl.NewRouter(config, logger)
	if err != nil {
		return err
	}

	go func() {
		r.Serve()
	}()

	authority := "127.0.0.1:7000"
	desc := r.Descriptor()
	err = desc.PublishToAuthority(authority)
	if err != nil {
		return err
	}
	logger.With("authority", authority).Info("published descriptor")

	select {}
}
<commit_msg>fix build error<commit_after>package cmd

import (
	"os"

	"github.com/inconshreveable/log15"
	"github.com/mmcloughlin/pearl"
	"github.com/mmcloughlin/pearl/log"
	"github.com/mmcloughlin/pearl/meta"
	"github.com/mmcloughlin/pearl/torconfig"
	"github.com/spf13/cobra"
)

// serveCmd represents the serve command
var serveCmd = &cobra.Command{
	Use:   "serve",
	Short: "Start a relay server",
	RunE: func(cmd *cobra.Command, args []string) error {
		return serve()
	},
}
var (
	nickname string
	port     int
	logfile  string
)

func init() {
	serveCmd.Flags().StringVarP(&nickname, "nickname", "n", "pearl", "nickname")
	serveCmd.Flags().IntVarP(&port, "port", "p", 9111, "relay port")
	serveCmd.Flags().StringVarP(&logfile, "logfile", "l", "pearl.json", "log file")

	rootCmd.AddCommand(serveCmd)
}

func serve() error {
	config := &torconfig.Config{
		Nickname: nickname,
		ORPort:   uint16(port),
		Platform: meta.Platform.String(),
		Contact:  "https://github.com/mmcloughlin/pearl",
	}

	base := log15.New()
	fh, err := log15.FileHandler(logfile, log15.JsonFormat())
	if err != nil {
		return err
	}
	base.SetHandler(log15.MultiHandler(
		log15.LvlFilterHandler(log15.LvlInfo,
			log15.StreamHandler(os.Stdout, log15.TerminalFormat()),
		),
		fh,
	))
	logger := log.NewLog15(base)

	r, err := pearl.NewRouter(config, logger)
	if err != nil {
		return err
	}

	go func() {
		r.Serve()
	}()

	authority := "127.0.0.1:7000"
	desc, err := r.Descriptor()
	if err != nil {
		return err
	}
	err = desc.PublishToAuthority(authority)
	if err != nil {
		return err
	}
	logger.With("authority", authority).Info("published descriptor")

	select {}
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"crypto/elliptic"
	"fmt"
	"log"
	"net"
	"os"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/p2p"
)

func main() {
	logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel))
	key, _ := crypto.GenerateKey()
	marshaled := elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y)

	srv := p2p.Server{
		MaxPeers:   100,
		Identity:   p2p.NewSimpleClientIdentity("Ethereum(G)", "0.1", "Peer Server Two", marshaled),
		ListenAddr: ":30301",
		NAT:        p2p.UPNP(),
	}
	if err := srv.Start(); err != nil {
		fmt.Println("could not start server:", err)
		os.Exit(1)
	}

	// add seed peers
	seed, err := net.ResolveTCPAddr("tcp", "poc-8.ethdev.com:30303")
	if err != nil {
		fmt.Println("couldn't resolve:", err)
	} else {
		srv.SuggestPeer(seed.IP, seed.Port, nil)
	}

	select {}
}
<commit_msg>cmd/peerserver: use NoDial, don't use seed peers<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"crypto/elliptic"
	"log"
	"os"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/p2p"
)

func main() {
	logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel))
	key, _ := crypto.GenerateKey()
	marshaled := elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y)

	srv := p2p.Server{
		MaxPeers:   100,
		Identity:   p2p.NewSimpleClientIdentity("Ethereum(G)", "0.1", "Peer Server Two", marshaled),
		ListenAddr: ":30301",
		NAT:        p2p.UPNP(),
		NoDial:     true,
	}
	if err := srv.Start(); err != nil {
		log.Fatal("could not start server:", err)
	}
	select {}
}
<|endoftext|>