<commit_before>package sudoku

import (
	"fmt"
	"log"
	"math"
	"strings"
)

//The actual techniques are intialized in hs_techniques.go, and actually defined in hst_*.go files.
//Techniques is ALL technies. CheapTechniques is techniques that are reasonably cheap to compute.
//ExpensiveTechniques is techniques that should only be used if all else has failed.
var Techniques []SolveTechnique
var CheapTechniques []SolveTechnique
var ExpensiveTechniques []SolveTechnique

var GuessTechnique SolveTechnique

//EVERY technique, even the weird one like Guess
var AllTechniques []SolveTechnique

//Worst case scenario, how many times we'd call HumanSolve to get a difficulty.
const MAX_DIFFICULTY_ITERATIONS = 50

//This number is the 'Constant' term from the multiple linear regression to learn the weights.
var difficultyConstant float64

//How close we have to get to the average to feel comfortable our difficulty is converging.
const DIFFICULTY_CONVERGENCE = 0.0005

type SolveDirections []*SolveStep

type SolveStep struct {
	//The cells that will be affected by the techinque
	TargetCells CellList
	//The cells that together lead the techinque to being valid
	PointerCells CellList
	//The numbers we will remove (or, in the case of Fill, add)
	//TODO: shouldn't this be renamed TargetNums?
	TargetNums IntSlice
	//The numbers in pointerCells that lead us to remove TargetNums from TargetCells.
	//This is only very rarely needed (at this time only for hiddenSubset techniques)
	PointerNums IntSlice
	//The general technique that underlies this step.
	Technique SolveTechnique
}

func (self *SolveStep) IsUseful(grid *Grid) bool {
	//Returns true IFF calling Apply with this step and the given grid would result in some useful work. Does not modify the gri.d

	//All of this logic is substantially recreated in Apply.

	if self.Technique == nil {
		return false
	}

	//TODO: test this.
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.TargetNums) == 0 {
			return false
		}
		cell := self.TargetCells[0].InGrid(grid)
		return self.TargetNums[0] != cell.Number()
	} else {
		useful := false
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.TargetNums {
				//It's right to use Possible because it includes the logic of "it's not possible if there's a number in there already"
				//TODO: ensure the comment above is correct logically.
				if gridCell.Possible(exclude) {
					useful = true
				}
			}
		}
		return useful
	}
}

func (self *SolveStep) Apply(grid *Grid) {
	//All of this logic is substantially recreated in IsUseful.
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.TargetNums) == 0 {
			return
		}
		cell := self.TargetCells[0].InGrid(grid)
		cell.SetNumber(self.TargetNums[0])
	} else {
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.TargetNums {
				gridCell.setExcluded(exclude, true)
			}
		}
	}
}

func (self *SolveStep) Description() string {
	result := ""
	if self.Technique.IsFill() {
		result += fmt.Sprintf("We put %s in cell %s ", self.TargetNums.Description(), self.TargetCells.Description())
	} else {
		//TODO: pluralize based on length of lists.
		result += fmt.Sprintf("We remove the possibilities %s from cells %s ", self.TargetNums.Description(), self.TargetCells.Description())
	}
	result += "because " + self.Technique.Description(self) + "."
	return result
}

func (self *SolveStep) normalize() {
	//Puts the solve step in its normal status. In practice this means that the various slices are sorted, so that the Description of them is stable.
	self.PointerCells.Sort()
	self.TargetCells.Sort()
	self.TargetNums.Sort()
	self.PointerNums.Sort()
}

func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	for _, step := range self {
		techniqueCount[step.Technique.Name()] += 1
	}
	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self)))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

func (self SolveDirections) Description() []string {

	if len(self) == 0 {
		return []string{""}
	}

	descriptions := make([]string, len(self))

	for i, step := range self {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

func (self SolveDirections) Difficulty() float64 {
	//How difficult the solve directions described are. The measure of difficulty we use is
	//just summing up weights we see; this captures:
	//* Number of steps
	//* Average difficulty of steps
	//* Number of hard steps
	//* (kind of) the hardest step: because the difficulties go up expontentionally.

	//This method assumes the weights have been calibrated empirically to give scores between 0.0 and 1.0
	//without normalization here.

	if len(self) == 0 {
		//The puzzle was not able to be solved, apparently.
		return 1.0
	}

	accum := difficultyConstant
	for _, step := range self {
		accum += step.Technique.Difficulty()
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

func (self SolveDirections) Walkthrough(grid *Grid) string {

	//TODO: test this.

	clone := grid.Copy()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

func (self *Grid) HumanWalkthrough() string {
	steps := self.HumanSolution()
	return steps.Walkthrough(self)
}

func (self *Grid) HumanSolution() SolveDirections {
	clone := self.Copy()
	defer clone.Done()
	return clone.HumanSolve()
}

type branchPoint struct {
	//The point at which we branched for a guess. non-nil if we are in a branch.
	grid *Grid
	//The step we'll apply to get us into the branch point.
	step *SolveStep
	//Ther other numbers to try from the branch point.
	otherNums IntSlice
	//The steps we've taken since the branch point.
	branchSteps []*SolveStep
	//The earlier branch point
	previousBranchPoint *branchPoint
	nextBranchPoint     *branchPoint
}

func (self *Grid) HumanSolve() SolveDirections {

	var results []*SolveStep

	var branch *branchPoint

	numBranches := 0

	//Note: trying these all in parallel is much slower (~15x) than doing them in sequence.
	//The reason is that in sequence we bailed early as soon as we found one step; now we try them all.

	for !self.Solved() {

		var possibilities []*SolveStep

		if branch != nil && self.Invalid() {
			//We're in a branch, and got to a point where we found an invalidity.
			//We chose the wrong branch. We should unwind to the branch point and go down the other branch.

			//Unwind ourselves
			self.Load(branch.grid.DataString())

			//Throwout the steps down the wrong branch we took.
			branch.branchSteps = nil

			if len(branch.otherNums) > 0 {

				//Pop off the nextNum to do
				nextNum := branch.otherNums[0]
				branch.otherNums = branch.otherNums[1:]

				//Stuff it into the TargetNums for the branch step.
				branch.step.TargetNums = IntSlice{nextNum}

				//Stuff the possibility list with the mangled branch.step.
				possibilities = []*SolveStep{branch.step}
			} else {
				//Well, crap. We're out of luck, nothing more for us to do.
				//TODO: pick a DIFFERENT guess operation at this grid state.
				//TODO: we could also unravel this sub-branch and go up to a higher branching level.
			}

		} else {

			//Normal operation; get potential solve steps by running them all.
			possibilities = runTechniques(CheapTechniques, self)
		}

		//TODO: provide hints to the techniques of where to look based on the last filled cell

		//Now pick one to apply.
		if len(possibilities) == 0 {
			//Okay, let's try the ExpensiveTechniques, as a hail mary.
			possibilities = runTechniques(ExpensiveTechniques, self)
			if len(possibilities) == 0 {
				//Hmm, didn't find any possivbilities. We're getting to be out of options...

				//Try to guess as a hail mary
				possibilities = runTechniques([]SolveTechnique{GuessTechnique}, self)

				if len(possibilities) == 0 {
					//Okay, we're well and truly done--not even any guesses came up with something. Nothing we can do.
					break
				} else {
					//Yay, found something! remember the branch point, so we can jump back to it.

					//If we branch more than a few times, things are probably REALLY wrong.
					numBranches++
					if numBranches >= 10 {
						//F this.

						return nil
					}

					//Push new branch point onto the doubly-linked list of branch points
					newBranch := &branchPoint{
						previousBranchPoint: branch,
					}
					if branch != nil {
						branch.nextBranchPoint = newBranch
					}
					branch = newBranch

					//We're just going to choose the first one.
					possibilities = possibilities[0:1]

					//TODO: this doesn't hold our special excludes, which we might have worked quite a bit to set up.
					//Ideally we'd have a way to keep those overrides.
					//The worst case is that we have a few unnecessary Cull steps just before the branch point.
					branch.grid = self.Copy()
					branch.step = possibilities[0]
					branch.otherNums = branch.step.PointerNums

					//Null out the branchPointStep's pointerNums; their only point was to communicate out the other possibilities.
					//And from now on they'll just be confusing.
					branch.step.PointerNums = nil

				}
			}
		}

		//TODO: consider if we should stop picking techniques based on their weight here.
		//Now that Find returns a slice instead of a single, we're already much more likely to select an "easy" technique. ... Right?

		possibilitiesWeights := make([]float64, len(possibilities))
		for i, possibility := range possibilities {
			possibilitiesWeights[i] = possibility.Technique.Difficulty()
		}
		step := possibilities[randomIndexWithInvertedWeights(possibilitiesWeights)]

		if branch == nil {
			results = append(results, step)
		} else {
			//We're in a branch point; we don't know if it's the RIGHT branch
			//So keep the steps somewhere else so we can throw them out if we unwind.
			branch.branchSteps = append(branch.branchSteps, step)
		}
		step.Apply(self)

	}

	if !self.Solved() {
		//We couldn't solve the puzzle.
		return nil
	}

	if branch != nil {
		//Apparently we're in the branch where the solution acutally lay. commit those steps and return them.

		//Walk up to the first branch.
		currentBranch := branch
		for {
			if currentBranch.previousBranchPoint == nil {
				break
			}
			currentBranch = currentBranch.previousBranchPoint
		}

		//now currentBranch is the earliest branch point
		//Walk down the list and copy in all of those steps
		for currentBranch != nil {
			results = append(results, currentBranch.branchSteps...)
			currentBranch = currentBranch.nextBranchPoint
		}
	}

	return results
}

func runTechniques(techniques []SolveTechnique, grid *Grid) []*SolveStep {
	numTechniques := len(techniques)
	possibilitiesChan := make(chan []*SolveStep)

	var possibilities []*SolveStep

	for _, technique := range techniques {
		go func(theTechnique SolveTechnique) {
			possibilitiesChan <- theTechnique.Find(grid)
		}(technique)
	}

	//Collect all of the results
	for i := 0; i < numTechniques; i++ {
		for _, possibility := range <-possibilitiesChan {
			possibilities = append(possibilities, possibility)
		}
	}

	return possibilities
}

func (self *Grid) Difficulty() float64 {
	//This can be an extremely expensive method. Do not call repeatedly!
	//returns the difficulty of the grid, which is a number between 0.0 and 1.0.
	//This is a probabilistic measure; repeated calls may return different numbers, although generally we wait for the results to converge.

	//We solve the same puzzle N times, then ask each set of steps for their difficulty, and combine those to come up with the overall difficulty.

	accum := 0.0
	average := 0.0
	lastAverage := 0.0

	for i := 0; i < MAX_DIFFICULTY_ITERATIONS; i++ {
		grid := self.Copy()
		steps := grid.HumanSolve()
		difficulty := steps.Difficulty()

		accum += difficulty
		average = accum / (float64(i) + 1.0)

		if math.Abs(average-lastAverage) < DIFFICULTY_CONVERGENCE {
			//Okay, we've already converged. Just return early!
			return average
		}

		lastAverage = average
	}

	//We weren't converging... oh well!
	return average

}
<commit_msg>TESTS FAIL. Added a note.<commit_after>package sudoku

import (
	"fmt"
	"log"
	"math"
	"strings"
)

//The actual techniques are intialized in hs_techniques.go, and actually defined in hst_*.go files.
//Techniques is ALL technies. CheapTechniques is techniques that are reasonably cheap to compute.
//ExpensiveTechniques is techniques that should only be used if all else has failed.
var Techniques []SolveTechnique
var CheapTechniques []SolveTechnique
var ExpensiveTechniques []SolveTechnique

var GuessTechnique SolveTechnique

//EVERY technique, even the weird one like Guess
var AllTechniques []SolveTechnique

//Worst case scenario, how many times we'd call HumanSolve to get a difficulty.
const MAX_DIFFICULTY_ITERATIONS = 50

//This number is the 'Constant' term from the multiple linear regression to learn the weights.
var difficultyConstant float64

//How close we have to get to the average to feel comfortable our difficulty is converging.
const DIFFICULTY_CONVERGENCE = 0.0005

type SolveDirections []*SolveStep

type SolveStep struct {
	//The cells that will be affected by the techinque
	TargetCells CellList
	//The cells that together lead the techinque to being valid
	PointerCells CellList
	//The numbers we will remove (or, in the case of Fill, add)
	//TODO: shouldn't this be renamed TargetNums?
	TargetNums IntSlice
	//The numbers in pointerCells that lead us to remove TargetNums from TargetCells.
	//This is only very rarely needed (at this time only for hiddenSubset techniques)
	PointerNums IntSlice
	//The general technique that underlies this step.
	Technique SolveTechnique
}

func (self *SolveStep) IsUseful(grid *Grid) bool {
	//Returns true IFF calling Apply with this step and the given grid would result in some useful work. Does not modify the gri.d

	//All of this logic is substantially recreated in Apply.

	if self.Technique == nil {
		return false
	}

	//TODO: test this.
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.TargetNums) == 0 {
			return false
		}
		cell := self.TargetCells[0].InGrid(grid)
		return self.TargetNums[0] != cell.Number()
	} else {
		useful := false
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.TargetNums {
				//It's right to use Possible because it includes the logic of "it's not possible if there's a number in there already"
				//TODO: ensure the comment above is correct logically.
				if gridCell.Possible(exclude) {
					useful = true
				}
			}
		}
		return useful
	}
}

func (self *SolveStep) Apply(grid *Grid) {
	//All of this logic is substantially recreated in IsUseful.
	if self.Technique.IsFill() {
		if len(self.TargetCells) == 0 || len(self.TargetNums) == 0 {
			return
		}
		cell := self.TargetCells[0].InGrid(grid)
		cell.SetNumber(self.TargetNums[0])
	} else {
		for _, cell := range self.TargetCells {
			gridCell := cell.InGrid(grid)
			for _, exclude := range self.TargetNums {
				gridCell.setExcluded(exclude, true)
			}
		}
	}
}

func (self *SolveStep) Description() string {
	result := ""
	if self.Technique.IsFill() {
		result += fmt.Sprintf("We put %s in cell %s ", self.TargetNums.Description(), self.TargetCells.Description())
	} else {
		//TODO: pluralize based on length of lists.
		result += fmt.Sprintf("We remove the possibilities %s from cells %s ", self.TargetNums.Description(), self.TargetCells.Description())
	}
	result += "because " + self.Technique.Description(self) + "."
	return result
}

func (self *SolveStep) normalize() {
	//Puts the solve step in its normal status. In practice this means that the various slices are sorted, so that the Description of them is stable.
	self.PointerCells.Sort()
	self.TargetCells.Sort()
	self.TargetNums.Sort()
	self.PointerNums.Sort()
}

func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	for _, step := range self {
		techniqueCount[step.Technique.Name()] += 1
	}
	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self)))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

func (self SolveDirections) Description() []string {

	if len(self) == 0 {
		return []string{""}
	}

	descriptions := make([]string, len(self))

	for i, step := range self {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

func (self SolveDirections) Difficulty() float64 {
	//How difficult the solve directions described are. The measure of difficulty we use is
	//just summing up weights we see; this captures:
	//* Number of steps
	//* Average difficulty of steps
	//* Number of hard steps
	//* (kind of) the hardest step: because the difficulties go up expontentionally.

	//This method assumes the weights have been calibrated empirically to give scores between 0.0 and 1.0
	//without normalization here.

	if len(self) == 0 {
		//The puzzle was not able to be solved, apparently.
		return 1.0
	}

	accum := difficultyConstant
	for _, step := range self {
		accum += step.Technique.Difficulty()
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

func (self SolveDirections) Walkthrough(grid *Grid) string {

	//TODO: test this.

	clone := grid.Copy()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

func (self *Grid) HumanWalkthrough() string {
	steps := self.HumanSolution()
	return steps.Walkthrough(self)
}

func (self *Grid) HumanSolution() SolveDirections {
	clone := self.Copy()
	defer clone.Done()
	return clone.HumanSolve()
}

type branchPoint struct {
	//The point at which we branched for a guess. non-nil if we are in a branch.
	grid *Grid
	//The step we'll apply to get us into the branch point.
	step *SolveStep
	//Ther other numbers to try from the branch point.
	otherNums IntSlice
	//The steps we've taken since the branch point.
	branchSteps []*SolveStep
	//The earlier branch point
	previousBranchPoint *branchPoint
	nextBranchPoint     *branchPoint
}

func (self *Grid) HumanSolve() SolveDirections {

	var results []*SolveStep

	var branch *branchPoint

	numBranches := 0

	//Note: trying these all in parallel is much slower (~15x) than doing them in sequence.
	//The reason is that in sequence we bailed early as soon as we found one step; now we try them all.

	for !self.Solved() {

		var possibilities []*SolveStep

		if branch != nil && self.Invalid() {
			//We're in a branch, and got to a point where we found an invalidity.
			//We chose the wrong branch. We should unwind to the branch point and go down the other branch.

			//Unwind ourselves
			self.Load(branch.grid.DataString())

			//Throwout the steps down the wrong branch we took.
			branch.branchSteps = nil

			if len(branch.otherNums) > 0 {

				//Pop off the nextNum to do
				nextNum := branch.otherNums[0]
				branch.otherNums = branch.otherNums[1:]

				//Stuff it into the TargetNums for the branch step.
				branch.step.TargetNums = IntSlice{nextNum}

				//Stuff the possibility list with the mangled branch.step.
				possibilities = []*SolveStep{branch.step}
			} else {
				//Well, crap. We're out of luck, nothing more for us to do.
				//TODO: pick a DIFFERENT guess operation at this grid state.
				//TODO: we could also unravel this sub-branch and go up to a higher branching level.
			}

		} else {

			//Normal operation; get potential solve steps by running them all.
			possibilities = runTechniques(CheapTechniques, self)
		}

		//TODO: provide hints to the techniques of where to look based on the last filled cell

		//Now pick one to apply.
		if len(possibilities) == 0 {
			//Okay, let's try the ExpensiveTechniques, as a hail mary.
			possibilities = runTechniques(ExpensiveTechniques, self)
			if len(possibilities) == 0 {
				//Hmm, didn't find any possivbilities. We're getting to be out of options...

				//Try to guess as a hail mary
				possibilities = runTechniques([]SolveTechnique{GuessTechnique}, self)

				if len(possibilities) == 0 {
					//Okay, we're well and truly done--not even any guesses came up with something. Nothing we can do.
					break
				} else {
					//Yay, found something! remember the branch point, so we can jump back to it.

					//If we branch more than a few times, things are probably REALLY wrong.
					numBranches++

					//TODO: this is probably too low. Some puzzles really honestly do require tons of branching if we're totally stumped.
					if numBranches >= 10 {
						//F this.

						return nil
					}

					//Push new branch point onto the doubly-linked list of branch points
					newBranch := &branchPoint{
						previousBranchPoint: branch,
					}
					if branch != nil {
						branch.nextBranchPoint = newBranch
					}
					branch = newBranch

					//We're just going to choose the first one.
					possibilities = possibilities[0:1]

					//TODO: this doesn't hold our special excludes, which we might have worked quite a bit to set up.
					//Ideally we'd have a way to keep those overrides.
					//The worst case is that we have a few unnecessary Cull steps just before the branch point.
					branch.grid = self.Copy()
					branch.step = possibilities[0]
					branch.otherNums = branch.step.PointerNums

					//Null out the branchPointStep's pointerNums; their only point was to communicate out the other possibilities.
					//And from now on they'll just be confusing.
					branch.step.PointerNums = nil

				}
			}
		}

		//TODO: consider if we should stop picking techniques based on their weight here.
		//Now that Find returns a slice instead of a single, we're already much more likely to select an "easy" technique. ... Right?

		possibilitiesWeights := make([]float64, len(possibilities))
		for i, possibility := range possibilities {
			possibilitiesWeights[i] = possibility.Technique.Difficulty()
		}
		step := possibilities[randomIndexWithInvertedWeights(possibilitiesWeights)]

		if branch == nil {
			results = append(results, step)
		} else {
			//We're in a branch point; we don't know if it's the RIGHT branch
			//So keep the steps somewhere else so we can throw them out if we unwind.
			branch.branchSteps = append(branch.branchSteps, step)
		}
		step.Apply(self)

	}

	if !self.Solved() {
		//We couldn't solve the puzzle.
		return nil
	}

	if branch != nil {
		//Apparently we're in the branch where the solution acutally lay. commit those steps and return them.

		//Walk up to the first branch.
		currentBranch := branch
		for {
			if currentBranch.previousBranchPoint == nil {
				break
			}
			currentBranch = currentBranch.previousBranchPoint
		}

		//now currentBranch is the earliest branch point
		//Walk down the list and copy in all of those steps
		for currentBranch != nil {
			results = append(results, currentBranch.branchSteps...)
			currentBranch = currentBranch.nextBranchPoint
		}
	}

	return results
}

func runTechniques(techniques []SolveTechnique, grid *Grid) []*SolveStep {
	numTechniques := len(techniques)
	possibilitiesChan := make(chan []*SolveStep)

	var possibilities []*SolveStep

	for _, technique := range techniques {
		go func(theTechnique SolveTechnique) {
			possibilitiesChan <- theTechnique.Find(grid)
		}(technique)
	}

	//Collect all of the results
	for i := 0; i < numTechniques; i++ {
		for _, possibility := range <-possibilitiesChan {
			possibilities = append(possibilities, possibility)
		}
	}

	return possibilities
}

func (self *Grid) Difficulty() float64 {
	//This can be an extremely expensive method. Do not call repeatedly!
	//returns the difficulty of the grid, which is a number between 0.0 and 1.0.
	//This is a probabilistic measure; repeated calls may return different numbers, although generally we wait for the results to converge.

	//We solve the same puzzle N times, then ask each set of steps for their difficulty, and combine those to come up with the overall difficulty.

	accum := 0.0
	average := 0.0
	lastAverage := 0.0

	for i := 0; i < MAX_DIFFICULTY_ITERATIONS; i++ {
		grid := self.Copy()
		steps := grid.HumanSolve()
		difficulty := steps.Difficulty()

		accum += difficulty
		average = accum / (float64(i) + 1.0)

		if math.Abs(average-lastAverage) < DIFFICULTY_CONVERGENCE {
			//Okay, we've already converged. Just return early!
			return average
		}

		lastAverage = average
	}

	//We weren't converging... oh well!
	return average

}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bufio"
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonusllhist"
	"github.com/circonus-labs/go-apiclient"
	"github.com/pkg/errors"
)

func (m *CirconusMetrics) packageMetrics() (map[string]*apiclient.CheckBundleMetric, Metrics) {

	m.packagingmu.Lock()
	defer m.packagingmu.Unlock()

	if m.Debug {
		m.Log.Printf("packaging metrics\n")
	}

	ts := makeTimestamp(time.Now())

	counters, gauges, histograms, text := m.snapshot()
	newMetrics := make(map[string]*apiclient.CheckBundleMetric)
	output := make(Metrics, len(counters)+len(gauges)+len(histograms)+len(text))
	for name, value := range counters {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "numeric",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "L", Value: value, Timestamp: ts}
		}
	}

	for name, value := range gauges {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "numeric",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: m.getGaugeType(value), Value: value, Timestamp: ts}
		}
	}

	for name, value := range histograms {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "histogram",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "h", Value: value.DecStrings()} // histograms do NOT get timestamps
		}
	}

	for name, value := range text {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "text",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "s", Value: value, Timestamp: ts}
		}
	}

	m.lastMetrics.metricsmu.Lock()
	defer m.lastMetrics.metricsmu.Unlock()
	m.lastMetrics.metrics = &output
	m.lastMetrics.ts = time.Now()

	return newMetrics, output
}

// PromOutput returns lines of metrics in prom format
func (m *CirconusMetrics) PromOutput() (*bytes.Buffer, error) {
	m.lastMetrics.metricsmu.Lock()
	defer m.lastMetrics.metricsmu.Unlock()

	if m.lastMetrics.metrics == nil {
		return nil, errors.New("no metrics available")
	}

	var b bytes.Buffer
	w := bufio.NewWriter(&b)

	ts := m.lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	for name, metric := range *m.lastMetrics.metrics {
		switch metric.Type {
		case "n":
			if strings.HasPrefix(fmt.Sprintf("%v", metric.Value), "[H[") {
				continue // circonus histogram != prom "histogram" (aka percentile)
			}
		case "h":
			continue // circonus histogram != prom "histogram" (aka percentile)
		case "s":
			continue // text metrics unsupported
		}
		fmt.Fprintf(w, "%s %v %d\n", name, metric.Value, ts)
	}

	err := w.Flush()
	if err != nil {
		return nil, errors.Wrap(err, "flushing metric buffer")
	}

	return &b, err
}

// FlushMetricsNoReset flushes current metrics to a structure and returns it (does NOT send to Circonus).
func (m *CirconusMetrics) FlushMetricsNoReset() *Metrics {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return &Metrics{}
	}

	m.flushing = true
	m.flushmu.Unlock()

	// save values configured at startup
	resetC := m.resetCounters
	resetG := m.resetGauges
	resetH := m.resetHistograms
	resetT := m.resetText
	// override Reset* to false for this call
	m.resetCounters = false
	m.resetGauges = false
	m.resetHistograms = false
	m.resetText = false

	_, output := m.packageMetrics()

	// restore previous values
	m.resetCounters = resetC
	m.resetGauges = resetG
	m.resetHistograms = resetH
	m.resetText = resetT

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()

	return &output
}

// FlushMetrics flushes current metrics to a structure and returns it (does NOT send to Circonus)
func (m *CirconusMetrics) FlushMetrics() *Metrics {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return &Metrics{}
	}

	m.flushing = true
	m.flushmu.Unlock()

	_, output := m.packageMetrics()

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()

	return &output
}

// Flush metrics kicks off the process of sending metrics to Circonus
func (m *CirconusMetrics) Flush() {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return
	}

	m.flushing = true
	m.flushmu.Unlock()

	newMetrics, output := m.packageMetrics()

	if len(output) > 0 {
		m.submit(output, newMetrics)
	} else if m.Debug {
		m.Log.Printf("no metrics to send, skipping\n")
	}

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()
}

// Reset removes all existing counters and gauges.
func (m *CirconusMetrics) Reset() {
	m.cm.Lock()
	defer m.cm.Unlock()

	m.cfm.Lock()
	defer m.cfm.Unlock()

	m.gm.Lock()
	defer m.gm.Unlock()

	m.gfm.Lock()
	defer m.gfm.Unlock()

	m.hm.Lock()
	defer m.hm.Unlock()

	m.tm.Lock()
	defer m.tm.Unlock()

	m.tfm.Lock()
	defer m.tfm.Unlock()

	m.counters = make(map[string]uint64)
	m.counterFuncs = make(map[string]func() uint64)
	m.gauges = make(map[string]interface{})
	m.gaugeFuncs = make(map[string]func() int64)
	m.histograms = make(map[string]*Histogram)
	m.text = make(map[string]string)
	m.textFuncs = make(map[string]func() string)
}

// snapshot returns a copy of the values of all registered counters and gauges.
func (m *CirconusMetrics) snapshot() (
	map[string]uint64, // counters
	map[string]interface{}, // gauges
	map[string]*circonusllhist.Histogram, // histograms
	map[string]string) { // text

	m.hm.Lock()
	m.cm.Lock()
	m.cfm.Lock()
	m.gm.Lock()
	m.gfm.Lock()
	m.tm.Lock()
	m.tfm.Lock()

	var wg sync.WaitGroup

	h := make(map[string]*circonusllhist.Histogram, len(m.histograms))
	wg.Add(1)
	go func() {
		m.snapHistograms(h)
		m.hm.Unlock()
		wg.Done()
	}()

	c := make(map[string]uint64, len(m.counters)+len(m.counterFuncs))
	wg.Add(1)
	go func() {
		m.snapCounters(c)
		m.cm.Unlock()
		m.cfm.Unlock()
		wg.Done()
	}()

	g := make(map[string]interface{}, len(m.gauges)+len(m.gaugeFuncs))
	wg.Add(1)
	go func() {
		m.snapGauges(g)
		m.gm.Unlock()
		m.gfm.Unlock()
		wg.Done()
	}()

	t := make(map[string]string, len(m.text)+len(m.textFuncs))
	wg.Add(1)
	go func() {
		m.snapText(t)
		m.tm.Unlock()
		m.tfm.Unlock()
		wg.Done()
	}()

	wg.Wait()

	return c, g, h, t
}

func (m *CirconusMetrics) snapCounters(c map[string]uint64) {
	for n, v := range m.counters {
		c[n] = v
	}
	if m.resetCounters && len(c) > 0 {
		m.counters = make(map[string]uint64)
	}

	for n, f := range m.counterFuncs {
		c[n] = f()
	}
}

func (m *CirconusMetrics) snapGauges(g map[string]interface{}) {
	for n, v := range m.gauges {
		g[n] = v
	}
	if m.resetGauges && len(g) > 0 {
		m.gauges = make(map[string]interface{})
	}

	for n, f := range m.gaugeFuncs {
		g[n] = f()
	}
}

func (m *CirconusMetrics) snapHistograms(h map[string]*circonusllhist.Histogram) {
	for n, hist := range m.histograms {
		hist.rw.Lock()
		if m.resetHistograms {
			h[n] = hist.hist.CopyAndReset()
		} else {
			h[n] = hist.hist.Copy()
		}
		hist.rw.Unlock()
	}

	if m.resetHistograms && len(h) > 0 {
		m.histograms = make(map[string]*Histogram)
	}
}

func (m *CirconusMetrics) snapText(t map[string]string) {
	for n, v := range m.text {
		t[n] = v
	}
	if m.resetText && len(t) > 0 {
		m.text = make(map[string]string)
	}

	for n, f := range m.textFuncs {
		t[n] = f()
	}
}

// makeTimestamp returns timestamp in ms units for _ts metric value
func makeTimestamp(ts time.Time) uint64 {
	return uint64(ts.UTC().UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)))
}
<commit_msg>fix: put locking in snapshot handlers<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bufio"
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonusllhist"
	"github.com/circonus-labs/go-apiclient"
	"github.com/pkg/errors"
)

func (m *CirconusMetrics) packageMetrics() (map[string]*apiclient.CheckBundleMetric, Metrics) {

	m.packagingmu.Lock()
	defer m.packagingmu.Unlock()

	if m.Debug {
		m.Log.Printf("packaging metrics\n")
	}

	ts := makeTimestamp(time.Now())

	counters, gauges, histograms, text := m.snapshot()
	newMetrics := make(map[string]*apiclient.CheckBundleMetric)
	output := make(Metrics, len(counters)+len(gauges)+len(histograms)+len(text))
	for name, value := range counters {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "numeric",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "L", Value: value, Timestamp: ts}
		}
	}

	for name, value := range gauges {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "numeric",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: m.getGaugeType(value), Value: value, Timestamp: ts}
		}
	}

	for name, value := range histograms {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "histogram",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "h", Value: value.DecStrings()} // histograms do NOT get timestamps
		}
	}

	for name, value := range text {
		send := m.check.IsMetricActive(name)
		if !send && m.check.ActivateMetric(name) {
			send = true
			newMetrics[name] = &apiclient.CheckBundleMetric{
				Name:   name,
				Type:   "text",
				Status: "active",
			}
		}
		if send {
			output[name] = Metric{Type: "s", Value: value, Timestamp: ts}
		}
	}

	m.lastMetrics.metricsmu.Lock()
	defer m.lastMetrics.metricsmu.Unlock()
	m.lastMetrics.metrics = &output
	m.lastMetrics.ts = time.Now()

	return newMetrics, output
}

// PromOutput returns lines of metrics in prom format
func (m *CirconusMetrics) PromOutput() (*bytes.Buffer, error) {
	m.lastMetrics.metricsmu.Lock()
	defer m.lastMetrics.metricsmu.Unlock()

	if m.lastMetrics.metrics == nil {
		return nil, errors.New("no metrics available")
	}

	var b bytes.Buffer
	w := bufio.NewWriter(&b)

	ts := m.lastMetrics.ts.UnixNano() / int64(time.Millisecond)

	for name, metric := range *m.lastMetrics.metrics {
		switch metric.Type {
		case "n":
			if strings.HasPrefix(fmt.Sprintf("%v", metric.Value), "[H[") {
				continue // circonus histogram != prom "histogram" (aka percentile)
			}
		case "h":
			continue // circonus histogram != prom "histogram" (aka percentile)
		case "s":
			continue // text metrics unsupported
		}
		fmt.Fprintf(w, "%s %v %d\n", name, metric.Value, ts)
	}

	err := w.Flush()
	if err != nil {
		return nil, errors.Wrap(err, "flushing metric buffer")
	}

	return &b, err
}

// FlushMetricsNoReset flushes current metrics to a structure and returns it (does NOT send to Circonus).
func (m *CirconusMetrics) FlushMetricsNoReset() *Metrics {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return &Metrics{}
	}

	m.flushing = true
	m.flushmu.Unlock()

	// save values configured at startup
	resetC := m.resetCounters
	resetG := m.resetGauges
	resetH := m.resetHistograms
	resetT := m.resetText
	// override Reset* to false for this call
	m.resetCounters = false
	m.resetGauges = false
	m.resetHistograms = false
	m.resetText = false

	_, output := m.packageMetrics()

	// restore previous values
	m.resetCounters = resetC
	m.resetGauges = resetG
	m.resetHistograms = resetH
	m.resetText = resetT

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()

	return &output
}

// FlushMetrics flushes current metrics to a structure and returns it (does NOT send to Circonus)
func (m *CirconusMetrics) FlushMetrics() *Metrics {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return &Metrics{}
	}

	m.flushing = true
	m.flushmu.Unlock()

	_, output := m.packageMetrics()

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()

	return &output
}

// Flush metrics kicks off the process of sending metrics to Circonus
func (m *CirconusMetrics) Flush() {
	m.flushmu.Lock()
	if m.flushing {
		m.flushmu.Unlock()
		return
	}

	m.flushing = true
	m.flushmu.Unlock()

	newMetrics, output := m.packageMetrics()

	if len(output) > 0 {
		m.submit(output, newMetrics)
	} else if m.Debug {
		m.Log.Printf("no metrics to send, skipping\n")
	}

	m.flushmu.Lock()
	m.flushing = false
	m.flushmu.Unlock()
}

// Reset removes all existing counters and gauges.
func (m *CirconusMetrics) Reset() {
	m.cm.Lock()
	defer m.cm.Unlock()

	m.cfm.Lock()
	defer m.cfm.Unlock()

	m.gm.Lock()
	defer m.gm.Unlock()

	m.gfm.Lock()
	defer m.gfm.Unlock()

	m.hm.Lock()
	defer m.hm.Unlock()

	m.tm.Lock()
	defer m.tm.Unlock()

	m.tfm.Lock()
	defer m.tfm.Unlock()

	m.counters = make(map[string]uint64)
	m.counterFuncs = make(map[string]func() uint64)
	m.gauges = make(map[string]interface{})
	m.gaugeFuncs = make(map[string]func() int64)
	m.histograms = make(map[string]*Histogram)
	m.text = make(map[string]string)
	m.textFuncs = make(map[string]func() string)
}

// snapshot returns a copy of the values of all registered counters and gauges.
func (m *CirconusMetrics) snapshot() (
	map[string]uint64, // counters
	map[string]interface{}, // gauges
	map[string]*circonusllhist.Histogram, // histograms
	map[string]string) { // text

	var h map[string]*circonusllhist.Histogram
	var c map[string]uint64
	var g map[string]interface{}
	var t map[string]string

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		h = m.snapHistograms()
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		c = m.snapCounters()
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		g = m.snapGauges()
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		t = m.snapText()
		wg.Done()
	}()

	wg.Wait()

	return c, g, h, t
}

func (m *CirconusMetrics) snapCounters() map[string]uint64 {
	m.cm.Lock()
	m.cfm.Lock()

	c := make(map[string]uint64, len(m.counters)+len(m.counterFuncs))

	for n, v := range m.counters {
		c[n] = v
	}
	if m.resetCounters && len(c) > 0 {
		m.counters = make(map[string]uint64)
	}

	for n, f := range m.counterFuncs {
		c[n] = f()
	}

	m.cm.Unlock()
	m.cfm.Unlock()

	return c
}

func (m *CirconusMetrics) snapGauges() map[string]interface{} {
	m.gm.Lock()
	m.gfm.Lock()

	g := make(map[string]interface{}, len(m.gauges)+len(m.gaugeFuncs))

	for n, v := range m.gauges {
		g[n] = v
	}
	if m.resetGauges && len(g) > 0 {
		m.gauges = make(map[string]interface{})
	}

	for n, f := range m.gaugeFuncs {
		g[n] = f()
	}

	m.gm.Unlock()
	m.gfm.Unlock()

	return g
}

func (m *CirconusMetrics) snapHistograms() map[string]*circonusllhist.Histogram {
	m.hm.Lock()

	h := make(map[string]*circonusllhist.Histogram, len(m.histograms))

	for n, hist := range m.histograms {
		hist.rw.Lock()
		if m.resetHistograms {
			h[n] = hist.hist.CopyAndReset()
		} else {
			h[n] = hist.hist.Copy()
		}
		hist.rw.Unlock()
	}

	if m.resetHistograms && len(h) > 0 {
		m.histograms = make(map[string]*Histogram)
	}

	m.hm.Unlock()

	return h
}

func (m *CirconusMetrics) snapText() map[string]string {
	m.tm.Lock()
	m.tfm.Lock()

	t := make(map[string]string, len(m.text)+len(m.textFuncs))

	for n, v := range m.text {
		t[n] = v
	}
	if m.resetText && len(t) > 0 {
		m.text = make(map[string]string)
	}

	for n, f := range m.textFuncs {
		t[n] = f()
	}

	m.tm.Unlock()
	m.tfm.Unlock()

	return t
}

// makeTimestamp returns timestamp in ms units for _ts metric value
func makeTimestamp(ts time.Time) uint64 {
	return uint64(ts.UTC().UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)))
}
<|endoftext|>


<commit_before>package pps

import (
	"io"

	"go.pedge.io/proto/stream"
	"golang.org/x/net/context"
)

func NewJob(jobID string) *Job {
	return &Job{ID: jobID}
}

func NewPipeline(pipelineName string) *Pipeline {
	return &Pipeline{Name: pipelineName}
}

func CreateJob(
	client APIClient,
	image string,
	cmd []string,
	stdin []string,
	parallelism uint64,
	inputs []*JobInput,
	parentJobID string,
) (*Job, error) {
	var parentJob *Job
	if parentJobID != "" {
		parentJob = NewJob(parentJobID)
	}
	return client.CreateJob(
		context.Background(),
		&CreateJobRequest{
			Transform: &Transform{
				Image: image,
				Cmd:   cmd,
				Stdin: stdin,
			},
			Parallelism: parallelism,
			Inputs:      inputs,
			ParentJob:   parentJob,
		},
	)
}

func GetLogs(
	client APIClient,
	jobID string,
	writer io.Writer,
) error {
	getLogsClient, err := client.GetLogs(
		context.Background(),
		&GetLogsRequest{
			Job: NewJob(jobID),
		},
	)
	if err != nil {
		return err
	}
	return protostream.WriteFromStreamingBytesClient(getLogsClient, writer)
}

func CreatePipeline(
	client APIClient,
	name string,
	image string,
	cmd []string,
	stdin []string,
	parallelism uint64,
	inputs []*PipelineInput,
) error {
	_, err := client.CreatePipeline(
		context.Background(),
		&CreatePipelineRequest{
			Pipeline: NewPipeline(name),
			Transform: &Transform{
				Image: image,
				Cmd:   cmd,
				Stdin: stdin,
			},
			Parallelism: parallelism,
			Inputs:      inputs,
		},
	)
	return err
}

func DeletePipeline(
	client APIClient,
	name string,
) error {
	_, err := client.DeletePipeline(
		context.Background(),
		&DeletePipelineRequest{
			Pipeline: NewPipeline(name),
		},
	)
	return err
}
<commit_msg>Adds client docs for pps.<commit_after>package pps

import (
	"io"

	"go.pedge.io/proto/stream"
	"golang.org/x/net/context"
)

func NewJob(jobID string) *Job {
	return &Job{ID: jobID}
}

func NewPipeline(pipelineName string) *Pipeline {
	return &Pipeline{Name: pipelineName}
}

// CreateJob creates and runs a job in PPS.
// image is the Docker image to run the job in.
// cmd is the command passed to the Docker run invocation.
// NOTE as with Docker cmd is not run inside a shell that means that things
// like wildcard globbing (*), pipes (|) and file redirects (> and >>) will not
// work. To get that behavior you should have your command be a shell of your
// choice and pass a shell script to stdin.
// stdin is a slice of lines that are sent to your command on stdin. Lines need
// not end in newline characters.
// parallelism is how many copies of your container should run in parallel. You
// may pass 0 for parallelism in which case PPS will set the parallelism based
// on availabe resources.
// inputs specifies a set of Commits that will be visible to the job during runtime.
// parentJobID specifies the a job to use as a parent, it may be left empty in
// which case there is no parent job. If not left empty your job will use the
// parent Job's output commit as the parent of its output commit.
func CreateJob(
	client APIClient,
	image string,
	cmd []string,
	stdin []string,
	parallelism uint64,
	inputs []*JobInput,
	parentJobID string,
) (*Job, error) {
	var parentJob *Job
	if parentJobID != "" {
		parentJob = NewJob(parentJobID)
	}
	return client.CreateJob(
		context.Background(),
		&CreateJobRequest{
			Transform: &Transform{
				Image: image,
				Cmd:   cmd,
				Stdin: stdin,
			},
			Parallelism: parallelism,
			Inputs:      inputs,
			ParentJob:   parentJob,
		},
	)
}

// GetLogs gets logs from a job (logs includes stdout and stderr).
func GetLogs(
	client APIClient,
	jobID string,
	writer io.Writer,
) error {
	getLogsClient, err := client.GetLogs(
		context.Background(),
		&GetLogsRequest{
			Job: NewJob(jobID),
		},
	)
	if err != nil {
		return err
	}
	return protostream.WriteFromStreamingBytesClient(getLogsClient, writer)
}

// CreatePipeline creates a new pipeline, pipelines are the main computation
// object in PPS they create a flow of data from a set of input Repos to an
// output Repo (which has the same name as the pipeline). Whenever new data is
// committed to one of the input repos the pipelines will create jobs to bring
// the output Repo up to data.
// image is the Docker image to run the jobs in.
// cmd is the command passed to the Docker run invocation.
// NOTE as with Docker cmd is not run inside a shell that means that things
// like wildcard globbing (*), pipes (|) and file redirects (> and >>) will not
// work. To get that behavior you should have your command be a shell of your
// choice and pass a shell script to stdin.
// stdin is a slice of lines that are sent to your command on stdin. Lines need
// not end in newline characters.
// parallelism is how many copies of your container should run in parallel. You
// may pass 0 for parallelism in which case PPS will set the parallelism based
// on availabe resources.
// inputs specifies a set of Repos that will be visible to the jobs during runtime.
// commits to these repos will cause the pipeline to create new jobs to process them.
func CreatePipeline(
	client APIClient,
	name string,
	image string,
	cmd []string,
	stdin []string,
	parallelism uint64,
	inputs []*PipelineInput,
) error {
	_, err := client.CreatePipeline(
		context.Background(),
		&CreatePipelineRequest{
			Pipeline: NewPipeline(name),
			Transform: &Transform{
				Image: image,
				Cmd:   cmd,
				Stdin: stdin,
			},
			Parallelism: parallelism,
			Inputs:      inputs,
		},
	)
	return err
}

// DeletePipeline deletes a pipeline along with its output Repo.
func DeletePipeline(
	client APIClient,
	name string,
) error {
	_, err := client.DeletePipeline(
		context.Background(),
		&DeletePipelineRequest{
			Pipeline: NewPipeline(name),
		},
	)
	return err
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"log"
	"github.com/qadium/plumb/shell"
	// "gopkg.in/yaml.v2"
	// "io/ioutil"
)

func Bootstrap(commit string) error {
	log.Printf("==> Bootstraping plumb.")
	defer log.Printf("<== Bootstrap complete.")

	if err := shell.RunAndLog("docker",
		"run",
		"--rm",
		"--entrypoint=\"/bin/bash\"",
		"-v",
		"/var/run/docker.sock:/var/run/docker.sock",
		"centurylink/golang-builder",
		"-c",
		fmt.Sprintf(`git clone https://github.com/qadium/plumb /plumb \
&& cd /plumb \
&& git checkout -b build %s \
&& cp -r /plumb/manager/* /src \
&& cd /src \
&& /build.sh plumb/manager`, commit)); err != nil {
		return err
	}

	// use docker to compile the manager and copy the binary into
	// another docker container
	//
	// tag this new container

	return nil
}
<commit_msg>updated bootstrap<commit_after>package cli

import (
	"fmt"
	"github.com/qadium/plumb/bindata"
	"github.com/qadium/plumb/shell"
	"io/ioutil"
	"log"
	"os/user"
	"os"
	// "gopkg.in/yaml.v2"
)

func writeAsset(asset string, directory string) error {
	log.Printf(" |     Writing '%s'", asset)
	data, err := bindata.Asset(fmt.Sprintf("manager/%s", asset))
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(fmt.Sprintf("%s/%s", directory, asset), data, 0644); err != nil {
		return err
	}
	log.Printf("       Done.")
	return nil
}

func Bootstrap() error {
	// use docker to compile the manager and copy the binary into
	// another docker container
	//
	// tag this new container
	log.Printf("==> Bootstraping plumb.")
	defer log.Printf("<== Bootstrap complete.")

	log.Printf(" |  Creating temp directory.")
	usr, err := user.Current()
	if err != nil {
		return err
	}
	directory := fmt.Sprintf("%s/.plumb-bootstrap", usr.HomeDir)

	if err := os.MkdirAll(directory, 0755); err != nil {
		return err
	}
	defer func() {
		if err := os.RemoveAll(directory); err != nil {
			panic(err)
		}
	}()
	log.Printf("    Temp directory created at '%s'", directory)

	log.Printf(" |  Writing manager source files.")
	if err := writeAsset("manager.go", directory); err != nil {
		return err
	}
	if err := writeAsset("Dockerfile", directory); err != nil {
		return err
	}
	log.Printf("    Done")

	if err := shell.RunAndLog("docker",
		"run",
		"--rm",
		"-v",
		"/var/run/docker.sock:/var/run/docker.sock",
		"-v",
		fmt.Sprintf("%s:/src", directory),
		"centurylink/golang-builder",
		"plumb/manager"); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"strings"

	"github.com/spf13/cobra"

	fsPkg "github.com/giantswarm/yochu/fs"
	httpClientPkg "github.com/giantswarm/yochu/httpclient"
	s3Pkg "github.com/giantswarm/yochu/s3"
	distribution "github.com/giantswarm/yochu/steps/distribution"
	dockerPkg "github.com/giantswarm/yochu/steps/docker"
	etcdPkg "github.com/giantswarm/yochu/steps/etcd"
	fleetPkg "github.com/giantswarm/yochu/steps/fleet"
	ip6tablesPkg "github.com/giantswarm/yochu/steps/ip6tables"
	iptablesPkg "github.com/giantswarm/yochu/steps/iptables"
	k8sPkg "github.com/giantswarm/yochu/steps/k8s"
	overlay "github.com/giantswarm/yochu/steps/overlay"
	rktPkg "github.com/giantswarm/yochu/steps/rkt"
	systemdPkg "github.com/giantswarm/yochu/systemd"
)

var (
	globalFlags = struct {
		debug   bool
		verbose bool

		version string
		steps   string
	}{}

	yochuCmd = &cobra.Command{
		Use:   "yochu",
		Short: "provision swarm cluster",
		Long:  "provision swarm cluster",
		Run:   yochuRun,
	}
)

func init() {
	fsPkg.Configure(Verbosef)
	systemdPkg.Configure(Verbosef)

	httpClientPkg.Configure(Verbosef)
	s3Pkg.Configure(Verbosef)

	iptablesPkg.Configure(Verbosef)
	ip6tablesPkg.Configure(Verbosef)
	dockerPkg.Configure(Verbosef)
	fleetPkg.Configure(Verbosef)
	etcdPkg.Configure(Verbosef)
	distribution.Configure(Verbosef)
	overlay.Configure(Verbosef)
	rktPkg.Configure(Verbosef)
	k8sPkg.Configure(Verbosef)

	yochuCmd.PersistentFlags().BoolVarP(&globalFlags.debug, "debug", "d", false, "print debug output")
	yochuCmd.PersistentFlags().BoolVarP(&globalFlags.verbose, "verbose", "v", false, "print verbose output")

	yochuCmd.PersistentFlags().StringVar(&globalFlags.steps, "steps", "all", "comma separated steps to execute")
}

// NewYochuCmd returns a Cobra command to run Yochu.
func NewYochuCmd(version string) *cobra.Command {
	globalFlags.version = version

	yochuCmd.AddCommand(versionCmd)
	yochuCmd.AddCommand(setupCmd)
	yochuCmd.AddCommand(teardownCmd)
	yochuCmd.AddCommand(iptablesRulesCmd)
	yochuCmd.AddCommand(ip6tablesRulesCmd)
	yochuCmd.Execute()

	return yochuCmd
}

func yochuRun(cmd *cobra.Command, args []string) {
	cmd.Help()
}

// Check if the given step is listed in steps. steps is a comma separated list
// of steps. If there are no steps given by the cli, steps is all.
func execute(steps, step string) bool {
	if steps == "all" {
		return true
	}

	for _, s := range strings.Split(steps, ",") {
		if s == step {
			return true
		}
	}

	return false
}
<commit_msg>fixed spf13/cobra breaking changes<commit_after>package cli

import (
	"strings"

	"github.com/spf13/cobra"

	fsPkg "github.com/giantswarm/yochu/fs"
	httpClientPkg "github.com/giantswarm/yochu/httpclient"
	s3Pkg "github.com/giantswarm/yochu/s3"
	distribution "github.com/giantswarm/yochu/steps/distribution"
	dockerPkg "github.com/giantswarm/yochu/steps/docker"
	etcdPkg "github.com/giantswarm/yochu/steps/etcd"
	fleetPkg "github.com/giantswarm/yochu/steps/fleet"
	ip6tablesPkg "github.com/giantswarm/yochu/steps/ip6tables"
	iptablesPkg "github.com/giantswarm/yochu/steps/iptables"
	k8sPkg "github.com/giantswarm/yochu/steps/k8s"
	overlay "github.com/giantswarm/yochu/steps/overlay"
	rktPkg "github.com/giantswarm/yochu/steps/rkt"
	systemdPkg "github.com/giantswarm/yochu/systemd"
)

var (
	globalFlags = struct {
		debug   bool
		verbose bool

		version string
		steps   string
	}{}

	yochuCmd = &cobra.Command{
		Use:   "yochu",
		Short: "provision swarm cluster",
		Long:  "provision swarm cluster",
		Run:   yochuRun,
	}
)

func init() {
	fsPkg.Configure(Verbosef)
	systemdPkg.Configure(Verbosef)

	httpClientPkg.Configure(Verbosef)
	s3Pkg.Configure(Verbosef)

	iptablesPkg.Configure(Verbosef)
	ip6tablesPkg.Configure(Verbosef)
	dockerPkg.Configure(Verbosef)
	fleetPkg.Configure(Verbosef)
	etcdPkg.Configure(Verbosef)
	distribution.Configure(Verbosef)
	overlay.Configure(Verbosef)
	rktPkg.Configure(Verbosef)
	k8sPkg.Configure(Verbosef)

	yochuCmd.PersistentFlags().BoolVarP(&globalFlags.debug, "debug", "d", false, "print debug output")
	yochuCmd.PersistentFlags().BoolVarP(&globalFlags.verbose, "verbose", "v", false, "print verbose output")

	yochuCmd.PersistentFlags().StringVar(&globalFlags.steps, "steps", "all", "comma separated steps to execute")
}

// NewYochuCmd returns a Cobra command to run Yochu.
func NewYochuCmd(version string) *cobra.Command {
	globalFlags.version = version

	yochuCmd.AddCommand(versionCmd)
	yochuCmd.AddCommand(setupCmd)
	yochuCmd.AddCommand(teardownCmd)
	yochuCmd.AddCommand(iptablesRulesCmd)
	yochuCmd.AddCommand(ip6tablesRulesCmd)
	yochuCmd.Execute()

	return yochuCmd
}

func yochuRun(cmd *cobra.Command, args []string) {
	cmd.HelpFunc()(cmd, nil)
}

// Check if the given step is listed in steps. steps is a comma separated list
// of steps. If there are no steps given by the cli, steps is all.
func execute(steps, step string) bool {
	if steps == "all" {
		return true
	}

	for _, s := range strings.Split(steps, ",") {
		if s == step {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package client

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/jonaz/mdns"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/bus"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
)

/*

TO DO:

On startup, if we have internet, we need to get the lastest master node id and master update node id
from the cloud (a rest service).

The update id needs to be exported over mdns, and is used to see if we are up to date with the latest
master id. If there is a newer one, we need to shut homecloud and all the drivers down and restart
our process.

*/

var log = logger.GetLogger("Client")

type client struct {
	conn     *ninja.Connection
	master   bool
	masterID string
	bridged  bool
}

type bridgeStatus struct {
	Connected  bool `json:"connected"`
	Configured bool `json:"configured"`
}

func Start() {
	log.Infof("Starting client on Node: %s", config.Serial())

	conn, err := ninja.Connect("client")
	if err != nil {
		log.Fatalf("Failed to connect to sphere: %s", err)
	}

	client := &client{
		conn: conn,
	}

	if !client.isPaired() {
		err = UpdateSphereAvahiService(false, false)
		if err != nil {
			log.Fatalf("Failed to update avahi service: %s", err)
		}
	}

	client.updatePairingLight("black", false)

	conn.SubscribeRaw("$sphere/bridge/status", client.onBridgeStatus)

	client.start()

	err = UpdateSphereAvahiService(true, client.master)
	if err != nil {
		log.Fatalf("Failed to update avahi service: %s", err)
	}

}

func (c *client) start() {

	if !c.isPaired() {
		log.Infof("Client is unpaired. Attempting to pair.")
		if err := c.pair(); err != nil {
			log.Infof("An error occurred while pairing. Restarting. error: %s", err)
			os.Exit(1)
		}

		log.Infof("Pairing was successful.")
		// We reload the config so the creds can be picked up
		config.MustRefresh()

		if !c.isPaired() {
			log.Infof("Pairing appeared successful, but I did not get the credentials. Restarting.")
			os.Exit(1)
		}
	}

	log.Infof("Client is paired. Site: %s User: %s", config.MustString("siteId"), config.MustString("userId"))

	c.masterID = config.String(config.Serial(), "masterNodeId")

	if c.masterID == config.Serial() {
		log.Infof("I am the master, starting HomeCloud.")

		go func() {
			for {
				c.conn.SendNotification("$node/"+config.Serial()+"/module/start", "sphere-go-homecloud")

				time.Sleep(time.Second * 5)
			}
		}()

		c.master = true
	} else {
		log.Infof("I am a slave. The master is %s", c.masterID)
	}

	go func() {
		for {
			c.findPeers()
			time.Sleep(time.Second * 30)
		}
	}()
}

func (c *client) findPeers() {

	// Make a channel for results and start listening
	entriesCh := make(chan *mdns.ServiceEntry, 4)
	go func() {
		for entry := range entriesCh {
			info := parseMdnsInfo(entry.Info)

			id, ok := info["ninja.sphere.node_id"]

			if !ok {
				log.Warningf("Found a node, but couldn't get it's node id. %v", entry)
				continue
			}

			if id == config.Serial() {
				// It's me.
				continue
			}

			if id == c.masterID {
				log.Infof("Found the master node (%s) - %s", id, entry.Addr)

				if !c.bridged {
					c.bridgeToMaster(entry.Addr, entry.Port)
					c.bridged = true
				}
			} else {

				user, ok := info["ninja.sphere.user_id"]

				if !ok {
					log.Warningf("Found a node, but couldn't get it's user id. %v", entry)
					continue
				}

				if user == config.MustString("userId") {
					log.Infof("Found a sibling node (%s) - %s", id, entry.Addr)
				} else {
					log.Infof("Found a node owned by another user (%s) (%s) - %s", user, id, entry.Addr)
				}

			}
		}
	}()

	// Start the lookup
	mdns.Lookup("_ninja-homecloud-mqtt._tcp", entriesCh)
	close(entriesCh)
}
func (c *client) isPaired() bool {
	return config.HasString("siteId") && config.HasString("token") && config.HasString("userId") && config.HasString("nodeId")
}

func (c *client) bridgeToMaster(host net.IP, port int) {

	log.Debugf("Bridging to the master: %s:%d", host, port)

	mqttURL := fmt.Sprintf("%s:%d", host, port)

	clientID := "slave-" + config.Serial()

	log.Infof("Connecting to master %s using cid:%s", mqttURL, clientID)

	master := bus.MustConnect(mqttURL, clientID)

	local := c.conn.GetMqttClient()

	bridgeTopics := []string{"$node/#", "$device/#"}

	c.bridgeMqtt(master, local, true, bridgeTopics)
	c.bridgeMqtt(local, master, false, bridgeTopics)
}

// bridgeMqtt connects one mqtt broker to another. Shouldn't probably be doing this. But whatever.
func (c *client) bridgeMqtt(from, to bus.Bus, masterToSlave bool, topics []string) {

	var payload map[string]interface{}
	onMessage := func(topic string, payloadBytes []byte) {
		payload = map[string]interface{}{}
		json.Unmarshal(payloadBytes, &payload)

		source, hasSource := payload["$mesh-source"]

		interesting := false

		if masterToSlave {
			// Interesting if it's from the master or one of the other slaves
			interesting = !hasSource || (source.(string) != config.Serial())
		} else {
			// Interesting if it's from me
			interesting = !hasSource
		}

		if interesting {

			if !hasSource {
				if masterToSlave {
					payload["$mesh-source"] = "master"
				} else {
					payload["$mesh-source"] = config.Serial()
				}
			}

			jsonPayload, _ := json.Marshal(payload)
			to.Publish(topic, jsonPayload)
		}

	}

	for _, topic := range topics {
		_, err := from.Subscribe(topic, onMessage)
		if err != nil {
			log.Fatalf("Failed to subscribe to topic %s when bridging to master: %s", topic, err)
		}
	}

}

func (c *client) onBridgeStatus(status *bridgeStatus) bool {
	log.Debugf("Got bridge status. connected:%t configured:%t", status.Connected, status.Configured)

	if status.Connected {
		c.updatePairingLight("green", false)
	} else {
		c.updatePairingLight("red", true)
	}

	if !status.Configured && c.master {
		log.Infof("Configuring bridge")

		c.conn.PublishRawSingleValue("$sphere/bridge/connect", map[string]string{
			"url":   config.MustString("cloud", "url"),
			"token": config.MustString("token"),
		})
	}

	return true
}

func (c *client) updatePairingLight(color string, flash bool) {
	c.conn.PublishRaw("$hardware/status/pairing", map[string]interface{}{
		"color": color,
		"flash": flash,
	})
}

func getLocalIP() string {

	var localIP string
	var err error

	for {
		localIP, err = ninja.GetNetAddress()
		if err == nil {
			break
		}
		time.Sleep(time.Second)
	}
	return localIP
}

func (c *client) pair() error {

	var boardType string
	if config.HasString("boardType") {
		boardType = config.MustString("boardType")
	} else {
		boardType = fmt.Sprintf("custom-%s-%s", runtime.GOOS, runtime.GOARCH)
	}

	log.Debugf("Board type: %s", boardType)

	client := &http.Client{
		Timeout: time.Second * 60, // It's 20sec on the server so this *should* be ok
	}

	if config.Bool(false, "cloud", "allowSelfSigned") {
		log.Warningf("Allowing self-signed cerificate (should only be used to connect to development cloud)")
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	var creds *credentials

	for {
		url := fmt.Sprintf(config.MustString("cloud", "activation"), config.Serial(), getLocalIP(), boardType)

		log.Debugf("Activating at URL: %s", url)

		var err error
		creds, err = activate(client, url)

		if err != nil {
			log.Warningf("Activation error : %s", err)
			log.Warningf("Sleeping for 10sec")
			time.Sleep(time.Second * 10)
		} else if creds != nil {
			break
		}
	}

	log.Infof("Got credentials. Joining site: %s user: %s", creds.SiteID, creds.UserID)

	if creds.MasterNodeID == "" {
		log.Warningf("Cloud did not give us the master node id. So setting it to ourself (%s)", config.Serial())
		creds.MasterNodeID = config.Serial()
	}

	credsFile := config.String("/data/etc/opt/ninja/credentials.json", "credentialFile")

	log.Infof("Saving credentials to %s", credsFile)

	credsJSON, err := json.Marshal(creds)
	if err != nil {
		return fmt.Errorf("Failed to marshal credentials: %s", err)
	}

	// XXX: HACK: TODO: CHANGE ME BACK TO 600
	err = ioutil.WriteFile(credsFile, credsJSON, 0644)

	if err != nil {
		return fmt.Errorf("Failed to write credentials file: %s", err)
	}

	cmd := exec.Command("sync")
	out, err := cmd.Output()

	if err != nil {
		return fmt.Errorf("Failed to call sync after saving credentials: %s - %s", err, out)
	}

	return nil
}

type nodeClaimResponse struct {
	Type string              `json:"type"`
	Data responseCredentials `json:"data"`
}

type responseCredentials struct {
	UserID       string `json:"user_id"`
	SiteID       string `json:"site_id"`
	NodeID       string `json:"node_id"`
	Token        string `json:"token"`
	MasterNodeID string `json:"master_node_id"`
}

type credentials struct {
	UserID       string `json:"userId"`
	SiteID       string `json:"siteId"`
	NodeID       string `json:"nodeId"`
	Token        string `json:"token"`
	MasterNodeID string `json:"masterNodeId"`
}

func activate(client *http.Client, url string) (*credentials, error) {

	log.Debugf("Requesting url: %s", url)

	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == http.StatusRequestTimeout {
		return nil, nil
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Failed to activate: %s - %s", resp.Status, body)
	}

	log.Debugf("Got response: %s", body)

	var response nodeClaimResponse
	err = json.Unmarshal(body, &response)

	return &credentials{
		UserID:       response.Data.UserID,
		SiteID:       response.Data.SiteID,
		NodeID:       response.Data.NodeID,
		Token:        response.Data.Token,
		MasterNodeID: response.Data.MasterNodeID,
	}, err
}

func parseMdnsInfo(field string) map[string]string {
	vals := make(map[string]string)

	for _, part := range strings.Split(field, "|") {
		chunks := strings.Split(part, "=")
		if len(chunks) == 2 {
			vals[chunks[0]] = chunks[1]
		}
	}
	return vals
}
<commit_msg>Switch back to hishicorp's mdns<commit_after>package client

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/hashicorp/mdns"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/bus"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
)

/*

TO DO:

On startup, if we have internet, we need to get the lastest master node id and master update node id
from the cloud (a rest service).

The update id needs to be exported over mdns, and is used to see if we are up to date with the latest
master id. If there is a newer one, we need to shut homecloud and all the drivers down and restart
our process.

*/

var log = logger.GetLogger("Client")

type client struct {
	conn     *ninja.Connection
	master   bool
	masterID string
	bridged  bool
}

type bridgeStatus struct {
	Connected  bool `json:"connected"`
	Configured bool `json:"configured"`
}

func Start() {
	log.Infof("Starting client on Node: %s", config.Serial())

	conn, err := ninja.Connect("client")
	if err != nil {
		log.Fatalf("Failed to connect to sphere: %s", err)
	}

	client := &client{
		conn: conn,
	}

	if !client.isPaired() {
		err = UpdateSphereAvahiService(false, false)
		if err != nil {
			log.Fatalf("Failed to update avahi service: %s", err)
		}
	}

	client.updatePairingLight("black", false)

	conn.SubscribeRaw("$sphere/bridge/status", client.onBridgeStatus)

	client.start()

	err = UpdateSphereAvahiService(true, client.master)
	if err != nil {
		log.Fatalf("Failed to update avahi service: %s", err)
	}

}

func (c *client) start() {

	if !c.isPaired() {
		log.Infof("Client is unpaired. Attempting to pair.")
		if err := c.pair(); err != nil {
			log.Infof("An error occurred while pairing. Restarting. error: %s", err)
			os.Exit(1)
		}

		log.Infof("Pairing was successful.")
		// We reload the config so the creds can be picked up
		config.MustRefresh()

		if !c.isPaired() {
			log.Infof("Pairing appeared successful, but I did not get the credentials. Restarting.")
			os.Exit(1)
		}
	}

	log.Infof("Client is paired. Site: %s User: %s", config.MustString("siteId"), config.MustString("userId"))

	c.masterID = config.String(config.Serial(), "masterNodeId")

	if c.masterID == config.Serial() {
		log.Infof("I am the master, starting HomeCloud.")

		go func() {
			for {
				c.conn.SendNotification("$node/"+config.Serial()+"/module/start", "sphere-go-homecloud")

				time.Sleep(time.Second * 5)
			}
		}()

		c.master = true
	} else {
		log.Infof("I am a slave. The master is %s", c.masterID)
	}

	go func() {
		for {
			c.findPeers()
			time.Sleep(time.Second * 30)
		}
	}()
}

func (c *client) findPeers() {

	// Make a channel for results and start listening
	entriesCh := make(chan *mdns.ServiceEntry, 4)
	go func() {
		for entry := range entriesCh {
			info := parseMdnsInfo(entry.Info)

			id, ok := info["ninja.sphere.node_id"]

			if !ok {
				log.Warningf("Found a node, but couldn't get it's node id. %v", entry)
				continue
			}

			if id == config.Serial() {
				// It's me.
				continue
			}

			if id == c.masterID {
				log.Infof("Found the master node (%s) - %s", id, entry.Addr)

				if !c.bridged {
					c.bridgeToMaster(entry.Addr, entry.Port)
					c.bridged = true
				}
			} else {

				user, ok := info["ninja.sphere.user_id"]

				if !ok {
					log.Warningf("Found a node, but couldn't get it's user id. %v", entry)
					continue
				}

				if user == config.MustString("userId") {
					log.Infof("Found a sibling node (%s) - %s", id, entry.Addr)
				} else {
					log.Infof("Found a node owned by another user (%s) (%s) - %s", user, id, entry.Addr)
				}

			}
		}
	}()

	// Start the lookup
	mdns.Lookup("_ninja-homecloud-mqtt._tcp", entriesCh)
	close(entriesCh)
}
func (c *client) isPaired() bool {
	return config.HasString("siteId") && config.HasString("token") && config.HasString("userId") && config.HasString("nodeId")
}

func (c *client) bridgeToMaster(host net.IP, port int) {

	log.Debugf("Bridging to the master: %s:%d", host, port)

	mqttURL := fmt.Sprintf("%s:%d", host, port)

	clientID := "slave-" + config.Serial()

	log.Infof("Connecting to master %s using cid:%s", mqttURL, clientID)

	master := bus.MustConnect(mqttURL, clientID)

	local := c.conn.GetMqttClient()

	bridgeTopics := []string{"$node/#", "$device/#"}

	c.bridgeMqtt(master, local, true, bridgeTopics)
	c.bridgeMqtt(local, master, false, bridgeTopics)
}

// bridgeMqtt connects one mqtt broker to another. Shouldn't probably be doing this. But whatever.
func (c *client) bridgeMqtt(from, to bus.Bus, masterToSlave bool, topics []string) {

	var payload map[string]interface{}
	onMessage := func(topic string, payloadBytes []byte) {
		payload = map[string]interface{}{}
		json.Unmarshal(payloadBytes, &payload)

		source, hasSource := payload["$mesh-source"]

		interesting := false

		if masterToSlave {
			// Interesting if it's from the master or one of the other slaves
			interesting = !hasSource || (source.(string) != config.Serial())
		} else {
			// Interesting if it's from me
			interesting = !hasSource
		}

		if interesting {

			if !hasSource {
				if masterToSlave {
					payload["$mesh-source"] = "master"
				} else {
					payload["$mesh-source"] = config.Serial()
				}
			}

			jsonPayload, _ := json.Marshal(payload)
			to.Publish(topic, jsonPayload)
		}

	}

	for _, topic := range topics {
		_, err := from.Subscribe(topic, onMessage)
		if err != nil {
			log.Fatalf("Failed to subscribe to topic %s when bridging to master: %s", topic, err)
		}
	}

}

func (c *client) onBridgeStatus(status *bridgeStatus) bool {
	log.Debugf("Got bridge status. connected:%t configured:%t", status.Connected, status.Configured)

	if status.Connected {
		c.updatePairingLight("green", false)
	} else {
		c.updatePairingLight("red", true)
	}

	if !status.Configured && c.master {
		log.Infof("Configuring bridge")

		c.conn.PublishRawSingleValue("$sphere/bridge/connect", map[string]string{
			"url":   config.MustString("cloud", "url"),
			"token": config.MustString("token"),
		})
	}

	return true
}

func (c *client) updatePairingLight(color string, flash bool) {
	c.conn.PublishRaw("$hardware/status/pairing", map[string]interface{}{
		"color": color,
		"flash": flash,
	})
}

func getLocalIP() string {

	var localIP string
	var err error

	for {
		localIP, err = ninja.GetNetAddress()
		if err == nil {
			break
		}
		time.Sleep(time.Second)
	}
	return localIP
}

func (c *client) pair() error {

	var boardType string
	if config.HasString("boardType") {
		boardType = config.MustString("boardType")
	} else {
		boardType = fmt.Sprintf("custom-%s-%s", runtime.GOOS, runtime.GOARCH)
	}

	log.Debugf("Board type: %s", boardType)

	client := &http.Client{
		Timeout: time.Second * 60, // It's 20sec on the server so this *should* be ok
	}

	if config.Bool(false, "cloud", "allowSelfSigned") {
		log.Warningf("Allowing self-signed cerificate (should only be used to connect to development cloud)")
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	var creds *credentials

	for {
		url := fmt.Sprintf(config.MustString("cloud", "activation"), config.Serial(), getLocalIP(), boardType)

		log.Debugf("Activating at URL: %s", url)

		var err error
		creds, err = activate(client, url)

		if err != nil {
			log.Warningf("Activation error : %s", err)
			log.Warningf("Sleeping for 10sec")
			time.Sleep(time.Second * 10)
		} else if creds != nil {
			break
		}
	}

	log.Infof("Got credentials. Joining site: %s user: %s", creds.SiteID, creds.UserID)

	if creds.MasterNodeID == "" {
		log.Warningf("Cloud did not give us the master node id. So setting it to ourself (%s)", config.Serial())
		creds.MasterNodeID = config.Serial()
	}

	credsFile := config.String("/data/etc/opt/ninja/credentials.json", "credentialFile")

	log.Infof("Saving credentials to %s", credsFile)

	credsJSON, err := json.Marshal(creds)
	if err != nil {
		return fmt.Errorf("Failed to marshal credentials: %s", err)
	}

	// XXX: HACK: TODO: CHANGE ME BACK TO 600
	err = ioutil.WriteFile(credsFile, credsJSON, 0644)

	if err != nil {
		return fmt.Errorf("Failed to write credentials file: %s", err)
	}

	cmd := exec.Command("sync")
	out, err := cmd.Output()

	if err != nil {
		return fmt.Errorf("Failed to call sync after saving credentials: %s - %s", err, out)
	}

	return nil
}

type nodeClaimResponse struct {
	Type string              `json:"type"`
	Data responseCredentials `json:"data"`
}

type responseCredentials struct {
	UserID       string `json:"user_id"`
	SiteID       string `json:"site_id"`
	NodeID       string `json:"node_id"`
	Token        string `json:"token"`
	MasterNodeID string `json:"master_node_id"`
}

type credentials struct {
	UserID       string `json:"userId"`
	SiteID       string `json:"siteId"`
	NodeID       string `json:"nodeId"`
	Token        string `json:"token"`
	MasterNodeID string `json:"masterNodeId"`
}

func activate(client *http.Client, url string) (*credentials, error) {

	log.Debugf("Requesting url: %s", url)

	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == http.StatusRequestTimeout {
		return nil, nil
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Failed to activate: %s - %s", resp.Status, body)
	}

	log.Debugf("Got response: %s", body)

	var response nodeClaimResponse
	err = json.Unmarshal(body, &response)

	return &credentials{
		UserID:       response.Data.UserID,
		SiteID:       response.Data.SiteID,
		NodeID:       response.Data.NodeID,
		Token:        response.Data.Token,
		MasterNodeID: response.Data.MasterNodeID,
	}, err
}

func parseMdnsInfo(field string) map[string]string {
	vals := make(map[string]string)

	for _, part := range strings.Split(field, "|") {
		chunks := strings.Split(part, "=")
		if len(chunks) == 2 {
			vals[chunks[0]] = chunks[1]
		}
	}
	return vals
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/ajvb/kala/api"
	"github.com/ajvb/kala/job"

	"github.com/dghubble/sling"
)

// KalaClient is the base struct for this package.
type KalaClient struct {
	apiEndpoint string
	requester   *sling.Sling
}

// New is used to create a new KalaClient based off of the apiEndpoint
// Example:
// 		c := New("http://127.0.0.1:8000")
func New(apiEndpoint string) *KalaClient {
	if strings.HasPrefix(apiEndpoint, "/") {
		apiEndpoint = apiEndpoint[:len(apiEndpoint)-1]
	}

	return &KalaClient{
		apiEndpoint: apiEndpoint + api.ApiUrlPrefix,
		requester:   sling.New().Base(apiEndpoint + api.ApiUrlPrefix),
	}
}

// CreateJob is used for creating a new job within Kala. It uses a map of
// strings to strings.
// Example:
// 		c := New("http://127.0.0.1:8000")
// 		body := map[string]string{
//			"schedule": "R2/2015-06-04T19:25:16.828696-07:00/PT10S",
//			"name":		"test_job",
//			"command": 	"bash -c 'date'",
//		}
//		id, err := c.CreateJob(body)
func (kc *KalaClient) CreateJob(body map[string]string) (string, error) {
	id := &api.AddJobResponse{}
	_, err := kc.requester.New().Post(api.JobPath).JsonBody(body).Receive(id)
	if err != nil {
		return "", err
	}
	return id.Id, nil
}

// GetJob is used to retrieve a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		job, err := c.GetJob(id)
func (kc *KalaClient) GetJob(id string) (*job.Job, error) {
	j := &api.JobResponse{}
	_, err := kc.requester.New().Get(api.JobPath + id).Receive(j)
	if err != nil {
		return nil, err
	}
	return j.Job, nil
}

// GetAllJobs returns a map of string (ID's) to job.Job's which contains
// all Jobs currently within Kala.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		jobs, err := c.GetAllJobs()
func (kc *KalaClient) GetAllJobs() (map[string]*job.Job, error) {
	jobs := &api.ListJobsResponse{}
	_, err := kc.requester.New().Get(api.JobPath).Receive(jobs)
	if err != nil {
		return nil, err
	}
	return jobs.Jobs, nil
}

// DeleteJob is used to delete a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		ok, err := c.DeleteJob(id)
func (kc *KalaClient) DeleteJob(id string) (bool, error) {
	// nil is completely safe to use, as it is simply ignored in the sling library.
	resp, err := kc.requester.New().Delete(api.JobPath + id).Receive(nil)
	if err != nil {
		return false, err
	}
	if resp.StatusCode != http.StatusNoContent {
		return false, fmt.Errorf("Delete failed with a status code of %d", resp.StatusCode)
	}
	return true, nil
}

// GetJobStats is used to retrieve stats about a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		stats, err := c.GetJobStats(id)
func (kc *KalaClient) GetJobStats(id string) ([]*job.JobStat, error) {
	js := &api.ListJobStatsResponse{}
	_, err := kc.requester.New().Get(api.JobPath + "stats/" + id).Receive(js)
	if err != nil {
		return nil, err
	}
	return js.JobStats, nil
}

// StartJob is used to manually start a Job by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		ok, err := c.StartJob(id)
func (kc *KalaClient) StartJob(id string) (bool, error) {
	resp, err := kc.requester.New().Post(api.JobPath + "start/" + id).Receive(nil)
	if err != nil {
		return false, err
	}
	if resp.StatusCode != http.StatusNoContent {
		return false, nil
	}
	return true, nil
}

// GetKalaStats retrieves system-level metrics about Kala
// Example:
// 		c := New("http://127.0.0.1:8000")
//		stats, err := c.GetKalaStats()
func (kc *KalaClient) GetKalaStats() (*job.KalaStats, error) {
	ks := &api.KalaStatsResponse{}
	_, err := kc.requester.New().Get("stats").Receive(ks)
	if err != nil {
		return nil, err
	}
	return ks.Stats, nil
}
<commit_msg>Updated client for new sling methods<commit_after>package client

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/ajvb/kala/api"
	"github.com/ajvb/kala/job"

	"github.com/dghubble/sling"
)

// KalaClient is the base struct for this package.
type KalaClient struct {
	apiEndpoint string
	requester   *sling.Sling
}

// New is used to create a new KalaClient based off of the apiEndpoint
// Example:
// 		c := New("http://127.0.0.1:8000")
func New(apiEndpoint string) *KalaClient {
	if strings.HasPrefix(apiEndpoint, "/") {
		apiEndpoint = apiEndpoint[:len(apiEndpoint)-1]
	}

	return &KalaClient{
		apiEndpoint: apiEndpoint + api.ApiUrlPrefix,
		requester:   sling.New().Base(apiEndpoint + api.ApiUrlPrefix),
	}
}

// CreateJob is used for creating a new job within Kala. It uses a map of
// strings to strings.
// Example:
// 		c := New("http://127.0.0.1:8000")
// 		body := map[string]string{
//			"schedule": "R2/2015-06-04T19:25:16.828696-07:00/PT10S",
//			"name":		"test_job",
//			"command": 	"bash -c 'date'",
//		}
//		id, err := c.CreateJob(body)
func (kc *KalaClient) CreateJob(body map[string]string) (string, error) {
	id := &api.AddJobResponse{}
	_, err := kc.requester.New().Post(api.JobPath).BodyJSON(body).Receive(id, nil)
	if err != nil {
		return "", err
	}
	return id.Id, nil
}

// GetJob is used to retrieve a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		job, err := c.GetJob(id)
func (kc *KalaClient) GetJob(id string) (*job.Job, error) {
	j := &api.JobResponse{}
	_, err := kc.requester.New().Get(api.JobPath+id).Receive(j, nil)
	if err != nil {
		return nil, err
	}
	return j.Job, nil
}

// GetAllJobs returns a map of string (ID's) to job.Job's which contains
// all Jobs currently within Kala.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		jobs, err := c.GetAllJobs()
func (kc *KalaClient) GetAllJobs() (map[string]*job.Job, error) {
	jobs := &api.ListJobsResponse{}
	_, err := kc.requester.New().Get(api.JobPath).Receive(jobs, nil)
	if err != nil {
		return nil, err
	}
	return jobs.Jobs, nil
}

// DeleteJob is used to delete a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		ok, err := c.DeleteJob(id)
func (kc *KalaClient) DeleteJob(id string) (bool, error) {
	// nil is completely safe to use, as it is simply ignored in the sling library.
	resp, err := kc.requester.New().Delete(api.JobPath+id).Receive(nil, nil)
	if err != nil {
		return false, err
	}
	if resp.StatusCode != http.StatusNoContent {
		return false, fmt.Errorf("Delete failed with a status code of %d", resp.StatusCode)
	}
	return true, nil
}

// GetJobStats is used to retrieve stats about a Job from Kala by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		stats, err := c.GetJobStats(id)
func (kc *KalaClient) GetJobStats(id string) ([]*job.JobStat, error) {
	js := &api.ListJobStatsResponse{}
	_, err := kc.requester.New().Get(api.JobPath+"stats/"+id).Receive(js, nil)
	if err != nil {
		return nil, err
	}
	return js.JobStats, nil
}

// StartJob is used to manually start a Job by its ID.
// Example:
// 		c := New("http://127.0.0.1:8000")
//		id := "93b65499-b211-49ce-57e0-19e735cc5abd"
//		ok, err := c.StartJob(id)
func (kc *KalaClient) StartJob(id string) (bool, error) {
	resp, err := kc.requester.New().Post(api.JobPath+"start/"+id).Receive(nil, nil)
	if err != nil {
		return false, err
	}
	if resp.StatusCode != http.StatusNoContent {
		return false, nil
	}
	return true, nil
}

// GetKalaStats retrieves system-level metrics about Kala
// Example:
// 		c := New("http://127.0.0.1:8000")
//		stats, err := c.GetKalaStats()
func (kc *KalaClient) GetKalaStats() (*job.KalaStats, error) {
	ks := &api.KalaStatsResponse{}
	_, err := kc.requester.New().Get("stats").Receive(ks, nil)
	if err != nil {
		return nil, err
	}
	return ks.Stats, nil
}
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/rpcplus"
)

func New(uri string) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	return &Client{url: uri, addr: u.Host}, nil
}

type Client struct {
	url  string
	addr string
}

var ErrNotFound = errors.New("controller: not found")

func (c *Client) send(method, path string, in, out interface{}) error {
	data, err := json.Marshal(in)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(method, c.url+path, bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	if out != nil {
		return json.NewDecoder(res.Body).Decode(out)
	}
	return nil
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	res, err := http.Get(c.url + path)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 404 {
			return ErrNotFound
		}
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		*since = time.Unix(0, 0)
	}
	// TODO: handle TLS
	client, err := rpcplus.DialHTTP("tcp", c.addr)
	if err != nil {
		return nil, &err
	}
	ch := make(chan *ct.ExpandedFormation)
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.App) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller: missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put("/apps/"+appID+"/release", &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get("/apps/"+appID+"/release", release)
}
<commit_msg>Fix typo in client<commit_after>package controller

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/rpcplus"
)

func New(uri string) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	return &Client{url: uri, addr: u.Host}, nil
}

type Client struct {
	url  string
	addr string
}

var ErrNotFound = errors.New("controller: not found")

func (c *Client) send(method, path string, in, out interface{}) error {
	data, err := json.Marshal(in)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(method, c.url+path, bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	if out != nil {
		return json.NewDecoder(res.Body).Decode(out)
	}
	return nil
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	res, err := http.Get(c.url + path)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 404 {
			return ErrNotFound
		}
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		*since = time.Unix(0, 0)
	}
	// TODO: handle TLS
	client, err := rpcplus.DialHTTP("tcp", c.addr)
	if err != nil {
		return nil, &err
	}
	ch := make(chan *ct.ExpandedFormation)
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.Provider) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller: missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put("/apps/"+appID+"/release", &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get("/apps/"+appID+"/release", release)
}
<|endoftext|>


<commit_before>//Client programm is the client part of the gonetsync
package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"

	"github.com/lyckade/golib/mylogger"
)

var myLogger = mylogger.NewFileLogger("client.log", "")

type doer interface {
	Do(*http.Request) (*http.Response, error)
}

var myClient = new(http.Client)

const timestampLayout string = "20060102150405"

func main() {

	folderWalk(myConf.SyncFolder)
}

func folderWalk(baseFolder string) {
	filepath.Walk(
		baseFolder,
		func(fpath string, info os.FileInfo, err error) error {
			walkFunc(baseFolder, fpath, info)
			return err
		})
}

func walkFunc(baseFolder string, fpath string, info os.FileInfo) {

	if info.IsDir() != true {
		ts, _ := makeTimestamp(info)
		fmt.Println("Timestamp: ", ts)

		packageStr := "meinPacket"

		//Path transformation
		fpath = filepath.Clean(fpath)

		url := makeURL(myConf.ServerAdress, packageStr, baseFolder, fpath)
		fmt.Println(url.String())
		//File reading
		fileReader, err1 := os.Open(fpath)
		defer fileReader.Close()
		if err1 != nil {
			myLogger.Println("Error opening file ", fpath)
			myLogger.Println(err1)
		}

		r, err2 := http.NewRequest("PUT", url.String(), fileReader)
		if err2 != nil {
			myLogger.Println(err2)
		}

		sendClientRequest(r)
	}

}

func makeTimestamp(info os.FileInfo) (int, error) {
	ts := info.ModTime().Format(timestampLayout)
	return strconv.Atoi(ts)
}

func makeURL(schemeHost string, packageStr string, baseFolder string, filePath string) url.URL {
	url, err := url.Parse(schemeHost)
	if err != nil {
		myLogger.Print(err)
	}
	relPath, _ := filepath.Rel(baseFolder, filePath)
	url.Path = packageStr + "/" + filepath.ToSlash(relPath)
	return *url
}

func makeFileRequest(fpath string, urlStr string) *http.Request {
	//File reading
	fileReader, err1 := os.Open(fpath)
	defer fileReader.Close()
	if err1 != nil {
		myLogger.Println("Error opening file ", fpath)
		myLogger.Println(err1)
	}

	r, err2 := http.NewRequest("PUT", urlStr, fileReader)
	if err2 != nil {
		myLogger.Println(err2)
	}
	return r
}

func sendClientRequest(r *http.Request) {
	resp, err3 := myClient.Do(r)
	fmt.Println("RESPONSE: ", resp)
	//fmt.Println("RSPERR: ", err3)
	myLogger.Println(err3)
}
<commit_msg>Decorator pattern preparation<commit_after>//Client programm is the client part of the gonetsync
package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"

	"github.com/lyckade/golib/mylogger"
)

var myLogger = mylogger.NewFileLogger("client.log", "")

// A Client sends http Request and returns response and error
type Client interface {
	Do(*http.Request) (*http.Response, error)
}

// ClientFunc is a custom type for the implementation of the
// Client interface
type ClientFunc func(*http.Request) (*http.Response, error)

// Do implements the Client interface for the ClientFunc
func (f ClientFunc) Do(r *http.Request) (*http.Response, error) {
	return f(r)
}

// Decorator wraps a Client with extra behaviour
type Decorator func(Client) Client

var myClient = new(http.Client)

const timestampLayout string = "20060102150405"

func main() {

	folderWalk(myConf.SyncFolder)
}

func folderWalk(baseFolder string) {
	filepath.Walk(
		baseFolder,
		func(fpath string, info os.FileInfo, err error) error {
			walkFunc(baseFolder, fpath, info)
			return err
		})
}

func walkFunc(baseFolder string, fpath string, info os.FileInfo) {

	if info.IsDir() != true {
		ts, _ := makeTimestamp(info)
		fmt.Println("Timestamp: ", ts)

		packageStr := "meinPacket"

		//Path transformation
		fpath = filepath.Clean(fpath)

		url := makeURL(myConf.ServerAdress, packageStr, baseFolder, fpath)
		fmt.Println(url.String())
		//File reading
		fileReader, err1 := os.Open(fpath)
		defer fileReader.Close()
		if err1 != nil {
			myLogger.Println("Error opening file ", fpath)
			myLogger.Println(err1)
		}

		r, err2 := http.NewRequest("PUT", url.String(), fileReader)
		if err2 != nil {
			myLogger.Println(err2)
		}

		sendClientRequest(r)
	}

}

func makeTimestamp(info os.FileInfo) (int, error) {
	ts := info.ModTime().Format(timestampLayout)
	return strconv.Atoi(ts)
}

func makeURL(schemeHost string, packageStr string, baseFolder string, filePath string) url.URL {
	url, err := url.Parse(schemeHost)
	if err != nil {
		myLogger.Print(err)
	}
	relPath, _ := filepath.Rel(baseFolder, filePath)
	url.Path = packageStr + "/" + filepath.ToSlash(relPath)
	return *url
}

func makeFileRequest(fpath string, urlStr string) *http.Request {
	//File reading
	fileReader, err1 := os.Open(fpath)
	defer fileReader.Close()
	if err1 != nil {
		myLogger.Println("Error opening file ", fpath)
		myLogger.Println(err1)
	}

	r, err2 := http.NewRequest("PUT", urlStr, fileReader)
	if err2 != nil {
		myLogger.Println(err2)
	}
	return r
}

func sendClientRequest(r *http.Request) {
	resp, err3 := myClient.Do(r)
	fmt.Println("RESPONSE: ", resp)
	//fmt.Println("RSPERR: ", err3)
	myLogger.Println(err3)
}
<|endoftext|>