<commit_before>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)
		secondAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum,
			firstAccumulator)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum,
			secondAccumulator)

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		//accumulate forward, so the last generation has ALL cells affected in any generation on that branch
		firstAccumulator.accumulateGenerations()
		secondAccumulator.accumulateGenerations()

		if doPrint {
			log.Println("Accumulators after accumulating generations:")
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		foundOne := false

		for generation := _MAX_IMPLICATION_STEPS - 1; generation >= 0 && !foundOne; generation-- {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							foundOne = true
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

//accumulateGenerations goes through each generation (youngest to newest)
//and squaches older generation maps into each generation, so each
//generation's map represents the totality of all cells seen at that point.
func (c chainSearcherAccumulator) accumulateGenerations() {
	for i := len(c) - 2; i >= 0; i-- {
		lastGeneration := c[i+1]
		currentGeneration := c[i]
		for key, val := range lastGeneration {
			if currentVal, ok := currentGeneration[key]; ok {
				if currentVal != val {
					//No, this should be expected to happen when we get to an invalid grid state,
					//which we should expect to happen down one of the two branches (at least as explore
					//far enough.)
					log.Println("We were about to overwrite a value from an earlier generation... this shouldn't happen.")
				}
			}
			currentGeneration[key] = val
		}
	}
}

func makeChainSeacherAccumulator(size int) chainSearcherAccumulator {
	result := make(chainSearcherAccumulator, size)
	for i := 0; i < size; i++ {
		result[i] = make(map[cellRef]int)
	}
	return result
}

func chainSearcher(i int, cell *Cell, numToApply int, accumulator chainSearcherAccumulator) {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//TODO: why doesn't this just return its own chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		i,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation <= 0 {
			break
		}

		generationDetails := accumulator[step.generation-1]

		//TODO: Check here if we crossed a generation boundary. If so, accumulate last generation.
		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		//TODO: check here if we're overwriting a value; if so, don't process anymore work steps.
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation - 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

}
<commit_msg>TESTS FAIL. Did one of the TODOs in chainSearcher: when we switch to a new generation, automatically accumulate previous generation.<commit_after>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)
		secondAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum,
			firstAccumulator)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum,
			secondAccumulator)

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		if doPrint {
			log.Println("Accumulators after accumulating generations:")
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		foundOne := false

		for generation := _MAX_IMPLICATION_STEPS - 1; generation >= 0 && !foundOne; generation-- {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							foundOne = true
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

//accumulateGenerations goes through each generation (youngest to newest)
//and squaches older generation maps into each generation, so each
//generation's map represents the totality of all cells seen at that point.
func (c chainSearcherAccumulator) accumulateGenerations() {
	for i := len(c) - 2; i >= 0; i-- {
		lastGeneration := c[i+1]
		currentGeneration := c[i]
		for key, val := range lastGeneration {
			if currentVal, ok := currentGeneration[key]; ok {
				if currentVal != val {
					//No, this should be expected to happen when we get to an invalid grid state,
					//which we should expect to happen down one of the two branches (at least as explore
					//far enough.)
					log.Println("We were about to overwrite a value from an earlier generation... this shouldn't happen.")
				}
			}
			currentGeneration[key] = val
		}
	}
}

func makeChainSeacherAccumulator(size int) chainSearcherAccumulator {
	result := make(chainSearcherAccumulator, size)
	for i := 0; i < size; i++ {
		result[i] = make(map[cellRef]int)
	}
	return result
}

func chainSearcher(i int, cell *Cell, numToApply int, accumulator chainSearcherAccumulator) {

	//TODO: rename the i paramater to max generations
	//TODO: generations should count UP, not down.

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//TODO: why doesn't this just return its own chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		i,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	lastSeenGeneration := i

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation <= 0 {
			break
		}

		generationDetails := accumulator[step.generation-1]

		if step.generation != lastSeenGeneration {
			//We just crossed the generation boundary. Copy the generation above's data into ours before we start populating it.
			lastGenerationDetails := accumulator[step.generation]
			for key, val := range lastGenerationDetails {
				generationDetails[key] = val
			}
		}

		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		//TODO: check here if we're overwriting a value; if so, don't process anymore work steps.
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation - 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

}
<|endoftext|>


<commit_before>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum)

		secondAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum)

		//Quick hack to make sure that firstAccumulator and secondAccumulator have same number of generations
		//( even if they don't really)

		for len(firstAccumulator) < len(secondAccumulator) {
			firstAccumulator = firstAccumulator.addGeneration()
		}

		for len(secondAccumulator) < len(firstAccumulator) {
			secondAccumulator = secondAccumulator.addGeneration()
		}

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		for generation := 0; generation < len(firstAccumulator); generation++ {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map (length " + strconv.Itoa(len(c)) + ")\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

func (c chainSearcherAccumulator) addGeneration() chainSearcherAccumulator {
	newGeneration := make(chainSearcherGenerationDetails)
	result := append(c, newGeneration)
	if len(result) > 1 {
		oldGeneration := result[len(result)-2]
		//Accumulate forward old generation
		for key, val := range oldGeneration {
			newGeneration[key] = val
		}
	}
	return result
}

func chainSearcher(maxGeneration int, cell *Cell, numToApply int) chainSearcherAccumulator {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	var result chainSearcherAccumulator

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		0,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation > maxGeneration {
			break
		}

		for len(result) < step.generation+1 {
			result = result.addGeneration()
		}

		generationDetails := result[step.generation]

		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		if currentVal, ok := generationDetails[step.cell.ref()]; ok {
			if currentVal != step.numToApply {
				//Found a contradiction! We can bail from processing any more because this branch leads inexorably
				//to a contradiction.

				//However, this last generation--the one we found the inconsistency in--needs to be thrown out.

				return result[:len(result)-1]
			}
		}
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation + 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

	return result

}
<commit_msg>TESTS FAIL. Took note of another currently broken thing and how to fix it.<commit_after>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum)

		secondAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum)

		//Quick hack to make sure that firstAccumulator and secondAccumulator have same number of generations
		//( even if they don't really)

		for len(firstAccumulator) < len(secondAccumulator) {
			firstAccumulator = firstAccumulator.addGeneration()
		}

		for len(secondAccumulator) < len(firstAccumulator) {
			secondAccumulator = secondAccumulator.addGeneration()
		}

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		for generation := 0; generation < len(firstAccumulator); generation++ {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

		//TODO: currently we pass back duplicates, so if we find a cell in generation 5, we will also
		//return the same step in generation 6. Either keep track of cells and don't repeat, or do a
		//better job comparing generations.

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map (length " + strconv.Itoa(len(c)) + ")\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

func (c chainSearcherAccumulator) addGeneration() chainSearcherAccumulator {
	newGeneration := make(chainSearcherGenerationDetails)
	result := append(c, newGeneration)
	if len(result) > 1 {
		oldGeneration := result[len(result)-2]
		//Accumulate forward old generation
		for key, val := range oldGeneration {
			newGeneration[key] = val
		}
	}
	return result
}

func chainSearcher(maxGeneration int, cell *Cell, numToApply int) chainSearcherAccumulator {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	var result chainSearcherAccumulator

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		0,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation > maxGeneration {
			break
		}

		for len(result) < step.generation+1 {
			result = result.addGeneration()
		}

		generationDetails := result[step.generation]

		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		if currentVal, ok := generationDetails[step.cell.ref()]; ok {
			if currentVal != step.numToApply {
				//Found a contradiction! We can bail from processing any more because this branch leads inexorably
				//to a contradiction.

				//However, this last generation--the one we found the inconsistency in--needs to be thrown out.

				return result[:len(result)-1]
			}
		}
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation + 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

	return result

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Item interface {
	OrderValue(order string) string
}

type Interface interface {
	Equal(order string, i, j int) bool
	Value(order string, i int) string
	Len() int
}

type Page struct {
	items []Item
}

func (p *Page) Equal(order string, i, j int) bool {
	return p.Value(order, i) == p.Value(order, j)
}

func (p *Page) Value(order string, i int) string {
	return p.items[i].OrderValue(order)
}

func (p *Page) Len() int {
	return len(p.items)
}

type Config struct {
	count     int
	order     string
	direction int
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

func (c Cursor) DirectionString() string {
	if c.Direction == ASC {
		return "asc"
	} else {
		return "desc"
	}
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := Cursor{}
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		switch {
		case v[0] == "desc":
			c.Direction = DESC
		case v[0] == "asc":
			c.Direction = ASC
		default:
			return c, fmt.Errorf("'%s' in not a supported direction, use asc or desc", v)
		}
	}
	return c, nil
}

type Pagination struct {
	Cursor
	config Config
}

func (p *Pagination) max(items Interface) int {
	if items.Len() <= p.Count {
		return items.Len() - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items Interface, order string, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items.Equal(order, i, max) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor Cursor, config Config) *Pagination {
	if cursor.Count == 0 {
		cursor.Count = config.count
	}
	if cursor.Order == "" {
		cursor.Order = config.order
	}
	if cursor.Direction == 0 {
		cursor.Direction = config.direction
	}
	return &Pagination{cursor, config}
}

func (p *Pagination) after(items Interface, last, direction int) *Pagination {
	if items.Len() == 0 {
		return nil
	}
	value := items.Value(p.Order, last)
	offset := p.equalCount(items, p.Order, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return &Pagination{cursor, p.config}
}

func (p *Pagination) prev(items Interface) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) next(items Interface, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && items.Len() <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) toUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", p.DirectionString())
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) OrderValue(order string) string {
	switch {
	case order == "created_at":
		return strconv.Itoa(c.created_at)
	case order == "updated_at":
		return strconv.Itoa(c.updated_at)
	default:
		return ""
	}
}

func main() {
	items := &Page{[]Item{
		&Comment{"e", 3, 5},
		&Comment{"d", 3, 5},
		&Comment{"c", 2, 5},
		&Comment{"b", 1, 4},
		&Comment{"a", 0, 4},
	}}

	cursor := Cursor{Order: "created_at"}
	config := Config{count: 2, order: "updated_at", direction: DESC}
	pagination := NewPagination(cursor, config)
	next := pagination.next(items, true)
	prev := pagination.prev(items)

	fmt.Printf("next value: %i, offset: %i, direction: %i\n", next.Value, next.Offset, next.Direction)
	fmt.Printf("prev value: %i, offset: %i, direction: %i\n", prev.Value, prev.Offset, prev.Direction)

	u, _ := url.Parse("http://kajic.com?order=updated_at&direction=desc&value=5&offset=0&count=4")
	cursor, _ = NewCursorFromQuery(u.RawQuery)
	pagination = NewPagination(cursor, config)
	next = pagination.next(items, true)
	prev = pagination.prev(items)

	nexturl, _ := next.toUrl(u)
	prevurl, _ := prev.toUrl(u)
	fmt.Println("nexturl", nexturl)
	fmt.Println("prevurl", prevurl)
	fmt.Println("origurl", u)
}
<commit_msg>Add FromUrl<commit_after>package main

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Item interface {
	OrderValue(order string) string
}

type Interface interface {
	Equal(order string, i, j int) bool
	Value(order string, i int) string
	Len() int
}

type Page struct {
	items []Item
}

func (p *Page) Equal(order string, i, j int) bool {
	return p.Value(order, i) == p.Value(order, j)
}

func (p *Page) Value(order string, i int) string {
	return p.items[i].OrderValue(order)
}

func (p *Page) Len() int {
	return len(p.items)
}

type Config struct {
	count     int
	order     string
	direction int
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

func (c Cursor) DirectionString() string {
	if c.Direction == ASC {
		return "asc"
	} else {
		return "desc"
	}
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := Cursor{}
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		switch {
		case v[0] == "desc":
			c.Direction = DESC
		case v[0] == "asc":
			c.Direction = ASC
		default:
			return c, fmt.Errorf("'%s' in not a supported direction, use asc or desc", v)
		}
	}
	return c, nil
}

type Pagination struct {
	Cursor
	config Config
}

func (p *Pagination) max(items Interface) int {
	if items.Len() <= p.Count {
		return items.Len() - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items Interface, order string, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items.Equal(order, i, max) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor Cursor, config Config) *Pagination {
	if cursor.Count == 0 {
		cursor.Count = config.count
	}
	if cursor.Order == "" {
		cursor.Order = config.order
	}
	if cursor.Direction == 0 {
		cursor.Direction = config.direction
	}
	return &Pagination{cursor, config}
}

func FromUrl(rawurl *url.URL, config Config) (*Pagination, error) {
	cursor, err := NewCursorFromQuery(rawurl.RawQuery)
	if err != nil {
		return nil, err
	}
	return &Pagination{cursor, config}, nil
}

func (p *Pagination) after(items Interface, last, direction int) *Pagination {
	if items.Len() == 0 {
		return nil
	}
	value := items.Value(p.Order, last)
	offset := p.equalCount(items, p.Order, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return &Pagination{cursor, p.config}
}

func (p *Pagination) prev(items Interface) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) next(items Interface, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && items.Len() <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) toUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", p.DirectionString())
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) OrderValue(order string) string {
	switch {
	case order == "created_at":
		return strconv.Itoa(c.created_at)
	case order == "updated_at":
		return strconv.Itoa(c.updated_at)
	default:
		return ""
	}
}

func main() {
	u, _ := url.Parse("http://kajic.com?order=updated_at&direction=desc&value=5&offset=0&count=4")
	config := Config{count: 2, order: "updated_at", direction: DESC}
	pagination, _ := FromUrl(u, config)
	items := &Page{[]Item{
		&Comment{"e", 3, 5},
		&Comment{"d", 3, 5},
		&Comment{"c", 2, 5},
		&Comment{"b", 1, 4},
		&Comment{"a", 0, 4},
	}}
	next := pagination.next(items, true)
	prev := pagination.prev(items)
	nexturl, _ := next.toUrl(u)
	prevurl, _ := prev.toUrl(u)
	fmt.Println("next", nexturl)
	fmt.Println("prev", prevurl)
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"log"

	"my/itto/verify/packet/itto"
	"my/itto/verify/sim"
)

type TobLogger struct {
	lastOptionId itto.OptionId
	consumeOps   int
	curOps       int
	hasOldTob    bool
	bid          tob
	ask          tob
}
type tob struct {
	Check bool
	Side  itto.MarketSide
	Old   sim.PriceLevel
	New   sim.PriceLevel
}

func NewTobLogger() *TobLogger {
	l := &TobLogger{
		bid: tob{Side: itto.MarketSideBid},
		ask: tob{Side: itto.MarketSideAsk},
	}
	return l
}

func (l *TobLogger) MessageArrived(idm *sim.IttoDbMessage) {
	l.bid.Check, l.ask.Check = false, false
	switch idm.Pam.Layer().(type) {
	case
		*itto.IttoMessageAddOrder,
		*itto.IttoMessageSingleSideExecuted,
		*itto.IttoMessageSingleSideExecutedWithPrice,
		*itto.IttoMessageOrderCancel,
		*itto.IttoMessageSingleSideDelete,
		*itto.IttoMessageBlockSingleSideDelete:
		l.consumeOps = 1
	case
		*itto.IttoMessageSingleSideReplace,
		*itto.IttoMessageSingleSideUpdate:
		l.consumeOps = 2
	case
		*itto.IttoMessageAddQuote,
		*itto.IttoMessageQuoteDelete:
		l.consumeOps = 2
		l.bid.Check, l.ask.Check = true, true
	case
		*itto.IttoMessageQuoteReplace:
		l.consumeOps = 4
		l.bid.Check, l.ask.Check = true, true
	default:
		// silently ignore
		return
	}
	l.curOps = 0
	l.hasOldTob = false
}

func (*TobLogger) OperationAppliedToOrders(sim.IttoOperation) {}

func (l *TobLogger) BeforeBookUpdate(book sim.Book, operation sim.IttoOperation) {
	if l.hasOldTob {
		return
	}
	l.lastOptionId = operation.GetOptionId()
	if l.lastOptionId.Invalid() {
		return
	}
	switch operation.GetSide() {
	case itto.MarketSideBid:
		l.bid.Check = true
	case itto.MarketSideAsk:
		l.ask.Check = true
	default:
		log.Fatalln("wrong operation side")
	}
	l.bid.update(book, l.lastOptionId, TobUpdateOld)
	l.ask.update(book, l.lastOptionId, TobUpdateOld)
	l.hasOldTob = true
}

func (l *TobLogger) AfterBookUpdate(book sim.Book, operation sim.IttoOperation, tobUpdate TobUpdate) bool {
	l.curOps++
	if l.curOps < l.consumeOps {
		return false
	}
	l.curOps = 0
	l.hasOldTob = false
	if l.lastOptionId.Invalid() {
		return false
	}
	l.bid.update(book, l.lastOptionId, tobUpdate)
	l.ask.update(book, l.lastOptionId, tobUpdate)

	return l.bid.updated() || l.ask.updated()
}

type TobUpdate byte

const (
	TobUpdateOld TobUpdate = iota
	TobUpdateNew
	TobUpdateNewForce
)

func (tob *tob) update(book sim.Book, oid itto.OptionId, u TobUpdate) {
	pl := &tob.New
	if u == TobUpdateOld {
		pl = &tob.Old
	}
	*pl = sim.PriceLevel{}
	if tob.Check || u == TobUpdateNewForce {
		if pls := book.GetTop(oid, tob.Side, 1); len(pls) > 0 {
			*pl = pls[0]
		}
	}
}

func (tob *tob) updated() bool {
	return tob.Check && tob.Old != tob.New
}
<commit_msg>rec: prevent unneeded messages for zero price<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"log"

	"my/itto/verify/packet/itto"
	"my/itto/verify/sim"
)

type TobLogger struct {
	lastOptionId itto.OptionId
	consumeOps   int
	curOps       int
	hasOldTob    bool
	bid          tob
	ask          tob
}
type tob struct {
	Check bool
	Side  itto.MarketSide
	Old   sim.PriceLevel
	New   sim.PriceLevel
}

func NewTobLogger() *TobLogger {
	l := &TobLogger{
		bid: tob{Side: itto.MarketSideBid},
		ask: tob{Side: itto.MarketSideAsk},
	}
	return l
}

func (l *TobLogger) MessageArrived(idm *sim.IttoDbMessage) {
	l.bid.Check, l.ask.Check = false, false
	switch idm.Pam.Layer().(type) {
	case
		*itto.IttoMessageAddOrder,
		*itto.IttoMessageSingleSideExecuted,
		*itto.IttoMessageSingleSideExecutedWithPrice,
		*itto.IttoMessageOrderCancel,
		*itto.IttoMessageSingleSideDelete,
		*itto.IttoMessageBlockSingleSideDelete:
		l.consumeOps = 1
	case
		*itto.IttoMessageSingleSideReplace,
		*itto.IttoMessageSingleSideUpdate:
		l.consumeOps = 2
	case
		*itto.IttoMessageAddQuote,
		*itto.IttoMessageQuoteDelete:
		l.consumeOps = 2
		l.bid.Check, l.ask.Check = true, true
	case
		*itto.IttoMessageQuoteReplace:
		l.consumeOps = 4
		l.bid.Check, l.ask.Check = true, true
	default:
		// silently ignore
		return
	}
	l.curOps = 0
	l.hasOldTob = false
}

func (*TobLogger) OperationAppliedToOrders(sim.IttoOperation) {}

func (l *TobLogger) BeforeBookUpdate(book sim.Book, operation sim.IttoOperation) {
	if l.hasOldTob {
		return
	}
	l.lastOptionId = operation.GetOptionId()
	if l.lastOptionId.Invalid() {
		return
	}
	switch operation.GetSide() {
	case itto.MarketSideBid:
		l.bid.Check = true
	case itto.MarketSideAsk:
		l.ask.Check = true
	default:
		log.Fatalln("wrong operation side")
	}
	l.bid.update(book, l.lastOptionId, TobUpdateOld)
	l.ask.update(book, l.lastOptionId, TobUpdateOld)
	l.hasOldTob = true
}

func (l *TobLogger) AfterBookUpdate(book sim.Book, operation sim.IttoOperation, tobUpdate TobUpdate) bool {
	l.curOps++
	if l.curOps < l.consumeOps {
		return false
	}
	l.curOps = 0
	l.hasOldTob = false
	if l.lastOptionId.Invalid() {
		return false
	}
	l.bid.update(book, l.lastOptionId, tobUpdate)
	l.ask.update(book, l.lastOptionId, tobUpdate)

	return l.bid.updated() || l.ask.updated()
}

type TobUpdate byte

const (
	TobUpdateOld TobUpdate = iota
	TobUpdateNew
	TobUpdateNewForce
)

func (tob *tob) update(book sim.Book, oid itto.OptionId, u TobUpdate) {
	pl := &tob.New
	if u == TobUpdateOld {
		pl = &tob.Old
	}
	*pl = sim.PriceLevel{}
	if tob.Check || u == TobUpdateNewForce {
		if pls := book.GetTop(oid, tob.Side, 1); len(pls) > 0 {
			*pl = pls[0]
		}
	}
}

func (tob *tob) updated() bool {
	return tob.Check && tob.Old != tob.New && (tob.Old.Price != 0 || tob.New.Price != 0)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/wallix/awless/aws/spec"
	"github.com/wallix/awless/config"
	"github.com/wallix/awless/template"
)

func TestCompileAllTemplates(t *testing.T) {
	path := filepath.Join(".", "*.aws")
	files, err := filepath.Glob(path)
	if err != nil {
		t.Fatal(err)
	}

	templates := make(map[string]*template.Template)

	fileCount := len(files)
	for _, file := range files {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			t.Fatal(err)
		}

		meta, err := parseTemplateMetadata(b)
		if err != nil {
			t.Fatalf("parsing template metadata: %s", err)
		}

		if meta.MinimalVersion != "" {
			comp, err := config.CompareSemver(config.Version, meta.MinimalVersion)
			if err != nil {
				t.Errorf("version from '%s' template: %s", file, err)
			}
			if comp < 0 {
				t.Logf("ignoring %s as at least %s version required\n", file, meta.MinimalVersion)
				fileCount--
				continue
			}
		}

		if tpl, err := template.Parse(string(b)); err != nil {
			t.Error(fmt.Errorf("cannot parse template '%s'\n\n%s\n%s", file, b, err))
		} else {
			templates[fmt.Sprint(file)] = tpl
		}
	}

	if fileCount != len(templates) {
		t.Fatal("compilation run only on all templates parsed successfully")
	}

	// to get deterministic order in which test are done
	var templateNames []string
	for k := range templates {
		templateNames = append(templateNames, k)
	}
	sort.Strings(templateNames)

	awsspec.DefaultImageResolverCache.Store("suselinux::sles-12:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-1234567"},
	})
	awsspec.DefaultImageResolverCache.Store("canonical:ubuntu:xenial:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-6543210"},
	})
	awsspec.DefaultImageResolverCache.Store("amazonlinux::hvm:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-6543210"},
	})
	awsspec.DefaultImageResolverCache.Store("debian:debian:jessie:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-45637289"},
	})
	awsspec.DefaultImageResolverCache.Store("redhat:rhel:7.3:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-wuvebfoq"},
	})

	awsspec.CommandFactory = awsspec.MockAWSSessionFactory

	for _, name := range templateNames {
		tpl := templates[name]
		env := template.NewEnv()
		env.AddFillers(stubFillers)
		env.Lookuper = func(tokens ...string) interface{} {
			newCommandFunc := awsspec.MockAWSSessionFactory.Build(strings.Join(tokens, ""))
			if newCommandFunc == nil {
				return nil
			}
			return newCommandFunc()
		}
		if _, _, err := template.Compile(tpl, env, template.NewRunnerCompileMode); err != nil {
			t.Fatalf("cannot compile template '%s'\n%s", name, err)
		} else {
			log.Printf("successfully parsed and compiled '%s'\n", name)
		}
	}
}

var stubFillers = map[string]interface{}{
	"instance.keypair":               "stub",
	"instance.type":                  "stub",
	"instance.image":                 "stub",
	"instance.min-number":            1,
	"instance.subnets":               "stub",
	"instance.max-number":            1,
	"group-name":                     "stub",
	"domain.name":                    "stub",
	"dbname":                         "stub",
	"dbhost":                         "stub",
	"dbuser":                         "stub",
	"dbpassword":                     "stub",
	"wordpress.vpc":                  "stub",
	"wordpress.subnets":              "stub",
	"wordpress.keypair":              "stub",
	"instances.securitygroup":        "stub",
	"availabilityzone.1":             "stub",
	"availabilityzone.2":             "stub",
	"availabilityzone.3":             "stub",
	"my.ssh.keypair":                 "stub",
	"ubuntu.image.id":                "stub",
	"instance.count":                 1,
	"ubuntu.ami":                     "stub",
	"ssh.keypair":                    "stub",
	"role.name":                      "stub",
	"instance.subnet":                "stub",
	"instance.name":                  "stub",
	"vpc.cidr":                       "10.0.0.0/24",
	"vpc.name":                       "stub",
	"first.subnet.availabilityzone":  "stub",
	"first.subnet.cidr":              "10.0.0.0/24",
	"first.subnet.name":              "stub",
	"second.subnet.name":             "stub",
	"second.subnet.availabilityzone": "stub",
	"second.subnet.cidr":             "10.0.0.0/24",
	"dbsubnetgroup.description":      "stub",
	"dbsubnetgroup.name":             "stub",
	"elasticip.domain":               "stub",
	"keypair.name":                   "stub",
	"securitygroup.protocol":         "stub",
	"user.console-password":          "stub",
	"user.name":                      "stub",
	"awless.role-name":               "stub",
	"instance.distro":                "suselinux",
	"remoteaccess-cidr":              "10.0.0.0/24",
	"role-name":                      "stub",
	"aws-service":                    "stub",
	"aws-account-id":                 "stub",
	"assume-policy-name":             "stub",
	"scalinggroup.desired-capacity":  1,
	"instance.userdata":              "stub",
	"database.name":                  "stub",
	"office.ip":                      "127.0.0.1",
	"debian.image":                   "stub",
	"my.keypair":                     "stub",
	"database.username":              "stub",
	"database.identifier":            "stub",
	"password.minimum8chars":         "stub",
	"instance.vpc":                   "stub",
	"securitygroup.description":      "stub",
	"awless-scheduler.role-name":     "stub",
	"instance.tagvalue":              "stub",
	"instance.securitygroup":         "stub",
	"instance.tagkey":                "stub",
	"remote-access.cidr":             "10.0.0.0/24",
	"redhat-ami":                     "stub",
	"zookeeper.instance.type":        "stub",
	"broker.instance.type":           "stub",
	"subnet.cidr":                    "10.0.0.0/24",
	"subnet.vpc":                     "stub",
	"subnet.name":                    "stub",
	"vpc.internetgateway":            "stub",
	"image.description":              "stub",
	"image.bucket":                   "stub",
	"image.filepath":                 "./verifyall_test.go",
	"subnet1.zone":                   "stub",
	"subnet2.zone":                   "stub",
}
<commit_msg>Verifying new password size when creating DB<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/wallix/awless/aws/spec"
	"github.com/wallix/awless/config"
	"github.com/wallix/awless/template"
)

func TestCompileAllTemplates(t *testing.T) {
	path := filepath.Join(".", "*.aws")
	files, err := filepath.Glob(path)
	if err != nil {
		t.Fatal(err)
	}

	templates := make(map[string]*template.Template)

	fileCount := len(files)
	for _, file := range files {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			t.Fatal(err)
		}

		meta, err := parseTemplateMetadata(b)
		if err != nil {
			t.Fatalf("parsing template metadata: %s", err)
		}

		if meta.MinimalVersion != "" {
			comp, err := config.CompareSemver(config.Version, meta.MinimalVersion)
			if err != nil {
				t.Errorf("version from '%s' template: %s", file, err)
			}
			if comp < 0 {
				t.Logf("ignoring %s as at least %s version required\n", file, meta.MinimalVersion)
				fileCount--
				continue
			}
		}

		if tpl, err := template.Parse(string(b)); err != nil {
			t.Error(fmt.Errorf("cannot parse template '%s'\n\n%s\n%s", file, b, err))
		} else {
			templates[fmt.Sprint(file)] = tpl
		}
	}

	if fileCount != len(templates) {
		t.Fatal("compilation run only on all templates parsed successfully")
	}

	// to get deterministic order in which test are done
	var templateNames []string
	for k := range templates {
		templateNames = append(templateNames, k)
	}
	sort.Strings(templateNames)

	awsspec.DefaultImageResolverCache.Store("suselinux::sles-12:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-1234567"},
	})
	awsspec.DefaultImageResolverCache.Store("canonical:ubuntu:xenial:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-6543210"},
	})
	awsspec.DefaultImageResolverCache.Store("amazonlinux::hvm:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-6543210"},
	})
	awsspec.DefaultImageResolverCache.Store("debian:debian:jessie:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-45637289"},
	})
	awsspec.DefaultImageResolverCache.Store("redhat:rhel:7.3:x86_64:hvm:ebs", []*awsspec.AwsImage{
		&awsspec.AwsImage{Id: "stub-wuvebfoq"},
	})

	awsspec.CommandFactory = awsspec.MockAWSSessionFactory

	for _, name := range templateNames {
		tpl := templates[name]
		env := template.NewEnv()
		env.AddFillers(stubFillers)
		env.Lookuper = func(tokens ...string) interface{} {
			newCommandFunc := awsspec.MockAWSSessionFactory.Build(strings.Join(tokens, ""))
			if newCommandFunc == nil {
				return nil
			}
			return newCommandFunc()
		}
		if _, _, err := template.Compile(tpl, env, template.NewRunnerCompileMode); err != nil {
			t.Fatalf("cannot compile template '%s'\n%s", name, err)
		} else {
			log.Printf("successfully parsed and compiled '%s'\n", name)
		}
	}
}

var stubFillers = map[string]interface{}{
	"instance.keypair":               "stub",
	"instance.type":                  "stub",
	"instance.image":                 "stub",
	"instance.min-number":            1,
	"instance.subnets":               "stub",
	"instance.max-number":            1,
	"group-name":                     "stub",
	"domain.name":                    "stub",
	"dbname":                         "stub",
	"dbhost":                         "stub",
	"dbuser":                         "stub",
	"dbpassword":                     "stubstub",
	"wordpress.vpc":                  "stub",
	"wordpress.subnets":              "stub",
	"wordpress.keypair":              "stub",
	"instances.securitygroup":        "stub",
	"availabilityzone.1":             "stub",
	"availabilityzone.2":             "stub",
	"availabilityzone.3":             "stub",
	"my.ssh.keypair":                 "stub",
	"ubuntu.image.id":                "stub",
	"instance.count":                 1,
	"ubuntu.ami":                     "stub",
	"ssh.keypair":                    "stub",
	"role.name":                      "stub",
	"instance.subnet":                "stub",
	"instance.name":                  "stub",
	"vpc.cidr":                       "10.0.0.0/24",
	"vpc.name":                       "stub",
	"first.subnet.availabilityzone":  "stub",
	"first.subnet.cidr":              "10.0.0.0/24",
	"first.subnet.name":              "stub",
	"second.subnet.name":             "stub",
	"second.subnet.availabilityzone": "stub",
	"second.subnet.cidr":             "10.0.0.0/24",
	"dbsubnetgroup.description":      "stub",
	"dbsubnetgroup.name":             "stub",
	"elasticip.domain":               "stub",
	"keypair.name":                   "stub",
	"securitygroup.protocol":         "stub",
	"user.console-password":          "stubstub",
	"user.name":                      "stub",
	"awless.role-name":               "stub",
	"instance.distro":                "suselinux",
	"remoteaccess-cidr":              "10.0.0.0/24",
	"role-name":                      "stub",
	"aws-service":                    "stub",
	"aws-account-id":                 "stub",
	"assume-policy-name":             "stub",
	"scalinggroup.desired-capacity":  1,
	"instance.userdata":              "stub",
	"database.name":                  "stub",
	"office.ip":                      "127.0.0.1",
	"debian.image":                   "stub",
	"my.keypair":                     "stub",
	"database.username":              "stub",
	"database.identifier":            "stub",
	"password.minimum8chars":         "stubstub",
	"instance.vpc":                   "stub",
	"securitygroup.description":      "stub",
	"awless-scheduler.role-name":     "stub",
	"instance.tagvalue":              "stub",
	"instance.securitygroup":         "stub",
	"instance.tagkey":                "stub",
	"remote-access.cidr":             "10.0.0.0/24",
	"redhat-ami":                     "stub",
	"zookeeper.instance.type":        "stub",
	"broker.instance.type":           "stub",
	"subnet.cidr":                    "10.0.0.0/24",
	"subnet.vpc":                     "stub",
	"subnet.name":                    "stub",
	"vpc.internetgateway":            "stub",
	"image.description":              "stub",
	"image.bucket":                   "stub",
	"image.filepath":                 "./verifyall_test.go",
	"subnet1.zone":                   "stub",
	"subnet2.zone":                   "stub",
}
<|endoftext|>


<commit_before>package libkb

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type ClientConfig struct {
	Host       string
	Port       int
	UseTls     bool
	Url        *url.URL
	RootCAs    *x509.CertPool
	Prefix     string
	UseCookies bool
	Timeout    time.Duration
}

type Client struct {
	cli    *http.Client
	config *ClientConfig
}

func SplitHost(joined string) (host string, port int, err error) {
	re := regexp.MustCompile("^([^:]+)(:([0-9]+))?$")
	match := re.FindStringSubmatch(joined)
	if match == nil {
		err = fmt.Errorf("Invalid host/port found: %s", joined)
	} else {
		host = match[1]
		port = 0
		if len(match[3]) > 0 {
			port, err = strconv.Atoi(match[3])
			if err != nil {
				err = fmt.Errorf("Could not convert port in host %s", joined)
			}
		}
	}
	return
}

func ParseCA(raw string) (*x509.CertPool, error) {
	ret := x509.NewCertPool()
	ok := ret.AppendCertsFromPEM([]byte(raw))
	var err error
	if !ok {
		err = fmt.Errorf("Could not read CA for keybase.io")
		ret = nil
	}
	return ret, err
}

func ShortCA(raw string) string {
	parts := strings.Split(raw, "\n")
	if len(parts) >= 3 {
		parts = parts[0:3]
	}
	return strings.Join(parts, " ") + "..."
}

// Pull the information out of the environment configuration,
// and build a Client config that will be used in all API server
// requests
func (e Env) GenClientConfig() (*ClientConfig, error) {
	u := e.GetServerUri()
	if len(u) == 0 {
		err := fmt.Errorf("Cannot find a server URL")
		return nil, err
	}
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}
	useTls := (url.Scheme == "https")
	host, port, e2 := SplitHost(url.Host)
	if e2 != nil {
		return nil, e2
	}
	var rootCAs *x509.CertPool
	if raw_ca := e.GetBundledCA(host); len(raw_ca) > 0 {
		rootCAs, err = ParseCA(raw_ca)
		if err != nil {
			err = fmt.Errorf("In parsing CAs for %s: %s", host, err.Error())
			return nil, err
		}
		G.Log.Debug(fmt.Sprintf("Using special root CA for %s: %s",
			host, ShortCA(raw_ca)))
	}

	// If we're using proxies, they might have their own CAs.
	if rootCAs, err = GetProxyCAs(rootCAs, e.config); err != nil {
		return nil, err
	}

	ret := &ClientConfig{host, port, useTls, url, rootCAs, url.Path, true, HTTP_DEFAULT_TIMEOUT}
	return ret, nil
}

func NewClient(config *ClientConfig, needCookie bool) *Client {
	var jar *cookiejar.Jar
	if needCookie && (config == nil || config.UseCookies) {
		jar, _ = cookiejar.New(nil)
	}

	var xprt *http.Transport
	var timeout time.Duration

	if config != nil && config.RootCAs != nil {
		xprt = &http.Transport{
			TLSClientConfig: &tls.Config{RootCAs: config.RootCAs},
		}
		timeout = config.Timeout
	} else {
		timeout = HTTP_DEFAULT_TIMEOUT
	}

	ret := &Client{
		cli:    &http.Client{Timeout: timeout},
		config: config,
	}
	if jar != nil {
		ret.cli.Jar = jar
	}
	if xprt != nil {
		ret.cli.Transport = xprt
	}
	return ret
}
<commit_msg>read standard proxy environment variables; can't test properly yet though<commit_after>package libkb

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type ClientConfig struct {
	Host       string
	Port       int
	UseTls     bool
	Url        *url.URL
	RootCAs    *x509.CertPool
	Prefix     string
	UseCookies bool
	Timeout    time.Duration
}

type Client struct {
	cli    *http.Client
	config *ClientConfig
}

func SplitHost(joined string) (host string, port int, err error) {
	re := regexp.MustCompile("^([^:]+)(:([0-9]+))?$")
	match := re.FindStringSubmatch(joined)
	if match == nil {
		err = fmt.Errorf("Invalid host/port found: %s", joined)
	} else {
		host = match[1]
		port = 0
		if len(match[3]) > 0 {
			port, err = strconv.Atoi(match[3])
			if err != nil {
				err = fmt.Errorf("Could not convert port in host %s", joined)
			}
		}
	}
	return
}

func ParseCA(raw string) (*x509.CertPool, error) {
	ret := x509.NewCertPool()
	ok := ret.AppendCertsFromPEM([]byte(raw))
	var err error
	if !ok {
		err = fmt.Errorf("Could not read CA for keybase.io")
		ret = nil
	}
	return ret, err
}

func ShortCA(raw string) string {
	parts := strings.Split(raw, "\n")
	if len(parts) >= 3 {
		parts = parts[0:3]
	}
	return strings.Join(parts, " ") + "..."
}

// Pull the information out of the environment configuration,
// and build a Client config that will be used in all API server
// requests
func (e Env) GenClientConfig() (*ClientConfig, error) {
	u := e.GetServerUri()
	if len(u) == 0 {
		err := fmt.Errorf("Cannot find a server URL")
		return nil, err
	}
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}
	useTls := (url.Scheme == "https")
	host, port, e2 := SplitHost(url.Host)
	if e2 != nil {
		return nil, e2
	}
	var rootCAs *x509.CertPool
	if raw_ca := e.GetBundledCA(host); len(raw_ca) > 0 {
		rootCAs, err = ParseCA(raw_ca)
		if err != nil {
			err = fmt.Errorf("In parsing CAs for %s: %s", host, err.Error())
			return nil, err
		}
		G.Log.Debug(fmt.Sprintf("Using special root CA for %s: %s",
			host, ShortCA(raw_ca)))
	}

	// If we're using proxies, they might have their own CAs.
	if rootCAs, err = GetProxyCAs(rootCAs, e.config); err != nil {
		return nil, err
	}

	ret := &ClientConfig{host, port, useTls, url, rootCAs, url.Path, true, HTTP_DEFAULT_TIMEOUT}
	return ret, nil
}

func NewClient(config *ClientConfig, needCookie bool) *Client {
	var jar *cookiejar.Jar
	if needCookie && (config == nil || config.UseCookies) {
		jar, _ = cookiejar.New(nil)
	}

	var xprt *http.Transport
	var timeout time.Duration

	if config != nil && config.RootCAs != nil {
		xprt = &http.Transport{
			Proxy:           http.ProxyFromEnvironment,
			TLSClientConfig: &tls.Config{RootCAs: config.RootCAs},
		}
		timeout = config.Timeout
	} else {
		timeout = HTTP_DEFAULT_TIMEOUT
	}

	ret := &Client{
		cli:    &http.Client{Timeout: timeout},
		config: config,
	}
	if jar != nil {
		ret.cli.Jar = jar
	}
	if xprt != nil {
		ret.cli.Transport = xprt
	}
	return ret
}
<|endoftext|>


<commit_before>package libkb

import (
	"fmt"
	"github.com/keybase/go-triplesec"
)

const (
	pwhIndex   = 0
	pwhLen     = 32
	eddsaIndex = pwhLen
	eddsaLen   = 32
	dhIndex    = eddsaIndex + eddsaLen
	dhLen      = 32
	lksIndex   = dhIndex + dhLen
	lksLen     = 32
	extraLen   = pwhLen + eddsaLen + dhLen + lksLen
)

type DetKey [extraLen]byte

func NewDetKey(passphrase string, salt []byte) (dk DetKey, err error) {
	tsec, err := triplesec.NewCipher([]byte(passphrase), salt)
	if err != nil {
		return dk, err
	}
	_, extra, err := tsec.DeriveKey(extraLen)
	if err != nil {
		return dk, err
	}
	if len(extra) != extraLen {
		return dk, fmt.Errorf("DeriveKey wrong number of bytes: %d", len(extra))
	}
	copy(dk[:], extra)
	return dk, nil
}

func (d DetKey) PWHash() []byte {
	return d[pwhIndex:eddsaIndex]
}

func (d DetKey) EddsaSecretKey() []byte {
	return d[eddsaIndex:dhIndex]
}

func (d DetKey) DhSecretKey() []byte {
	return d[dhIndex:lksIndex]
}

func (d DetKey) LksClientHalf() []byte {
	return d[lksIndex:]
}

func (d DetKey) String() string {
	return fmt.Sprintf("pwh:   %x\neddsa: %x\ndh:    %x\nlks:   %x", d.PWHash(), d.EddsaSecretKey(), d.DhSecretKey(), d.LksClientHalf())
	// return hex.EncodeToString(d[:])
}
<commit_msg>made index a little clearer<commit_after>package libkb

import (
	"fmt"
	"github.com/keybase/go-triplesec"
)

const (
	pwhIndex   = 0
	pwhLen     = 32
	eddsaIndex = pwhIndex + pwhLen
	eddsaLen   = 32
	dhIndex    = eddsaIndex + eddsaLen
	dhLen      = 32
	lksIndex   = dhIndex + dhLen
	lksLen     = 32
	extraLen   = pwhLen + eddsaLen + dhLen + lksLen
)

type DetKey [extraLen]byte

func NewDetKey(passphrase string, salt []byte) (dk DetKey, err error) {
	tsec, err := triplesec.NewCipher([]byte(passphrase), salt)
	if err != nil {
		return dk, err
	}
	_, extra, err := tsec.DeriveKey(extraLen)
	if err != nil {
		return dk, err
	}
	if len(extra) != extraLen {
		return dk, fmt.Errorf("DeriveKey wrong number of bytes: %d", len(extra))
	}
	copy(dk[:], extra)
	return dk, nil
}

func (d DetKey) PWHash() []byte {
	return d[pwhIndex:eddsaIndex]
}

func (d DetKey) EddsaSecretKey() []byte {
	return d[eddsaIndex:dhIndex]
}

func (d DetKey) DhSecretKey() []byte {
	return d[dhIndex:lksIndex]
}

func (d DetKey) LksClientHalf() []byte {
	return d[lksIndex:]
}

func (d DetKey) String() string {
	return fmt.Sprintf("pwh:   %x\neddsa: %x\ndh:    %x\nlks:   %x", d.PWHash(), d.EddsaSecretKey(), d.DhSecretKey(), d.LksClientHalf())
	// return hex.EncodeToString(d[:])
}
<|endoftext|>


<commit_before>package softlayer

import (
	"os"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/softlayer/softlayer-go/session"
)

func Provider() terraform.ResourceProvider {
	defaultSoftLayerSession := session.New()
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"username": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.UserName, nil
				},
				Description: "The user name for SoftLayer API operations.",
			},
			"api_key": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.APIKey, nil
				},
				Description: "The API key for SoftLayer API operations.",
			},
			"endpoint_url": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.Endpoint, nil
				},
				Description: "The endpoint url for the SoftLayer API.",
			},
		},

		ResourcesMap: map[string]*schema.Resource{
			"softlayer_virtual_guest":          resourceSoftLayerVirtualGuest(),
			"softlayer_ssh_key":                resourceSoftLayerSSHKey(),
			"softlayer_dns_domain_record":      resourceSoftLayerDnsDomainRecord(),
			"softlayer_dns_domain":             resourceSoftLayerDnsDomain(),
			"softlayer_lb_vpx":                 resourceSoftLayerLbVpx(),
			"softlayer_lb_vpx_vip":             resourceSoftLayerLbVpxVip(),
			"softlayer_lb_vpx_service":         resourceSoftLayerLbVpxService(),
			"softlayer_lb_local":               resourceSoftLayerLbLocal(),
			"softlayer_lb_local_service_group": resourceSoftLayerLbLocalServiceGroup(),
			"softlayer_lb_local_service":       resourceSoftLayerLbLocalService(),
			"softlayer_security_certificate":   resourceSoftLayerSecurityCertificate(),
			"softlayer_user":                   resourceSoftLayerUser(),
			"softlayer_objectstorage_account":  resourceSoftLayerObjectStorageAccount(),
			"softlayer_provisioning_hook":      resourceSoftLayerProvisioningHook(),
			"softlayer_scale_policy":           resourceSoftLayerScalePolicy(),
			"softlayer_scale_group":            resourceSoftLayerScaleGroup(),
			"softlayer_basic_monitor":          resourceSoftLayerBasicMonitor(),
		},

		ConfigureFunc: providerConfigure,
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	sess := session.Session{
		UserName: d.Get("username").(string),
		APIKey:   d.Get("api_key").(string),
		Endpoint: d.Get("endpoint_url").(string),
	}

	if os.Getenv("TF_LOG") != "" {
		sess.Debug = true
	}

	return &sess, nil
}
<commit_msg>Return an error to the terraform user if credentials are missing<commit_after>package softlayer

import (
	"errors"
	"os"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/softlayer/softlayer-go/session"
)

func Provider() terraform.ResourceProvider {
	defaultSoftLayerSession := session.New()
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"username": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.UserName, nil
				},
				Description: "The user name for SoftLayer API operations.",
			},
			"api_key": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.APIKey, nil
				},
				Description: "The API key for SoftLayer API operations.",
			},
			"endpoint_url": {
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: func() (interface{}, error) {
					return defaultSoftLayerSession.Endpoint, nil
				},
				Description: "The endpoint url for the SoftLayer API.",
			},
		},

		ResourcesMap: map[string]*schema.Resource{
			"softlayer_virtual_guest":          resourceSoftLayerVirtualGuest(),
			"softlayer_ssh_key":                resourceSoftLayerSSHKey(),
			"softlayer_dns_domain_record":      resourceSoftLayerDnsDomainRecord(),
			"softlayer_dns_domain":             resourceSoftLayerDnsDomain(),
			"softlayer_lb_vpx":                 resourceSoftLayerLbVpx(),
			"softlayer_lb_vpx_vip":             resourceSoftLayerLbVpxVip(),
			"softlayer_lb_vpx_service":         resourceSoftLayerLbVpxService(),
			"softlayer_lb_local":               resourceSoftLayerLbLocal(),
			"softlayer_lb_local_service_group": resourceSoftLayerLbLocalServiceGroup(),
			"softlayer_lb_local_service":       resourceSoftLayerLbLocalService(),
			"softlayer_security_certificate":   resourceSoftLayerSecurityCertificate(),
			"softlayer_user":                   resourceSoftLayerUser(),
			"softlayer_objectstorage_account":  resourceSoftLayerObjectStorageAccount(),
			"softlayer_provisioning_hook":      resourceSoftLayerProvisioningHook(),
			"softlayer_scale_policy":           resourceSoftLayerScalePolicy(),
			"softlayer_scale_group":            resourceSoftLayerScaleGroup(),
			"softlayer_basic_monitor":          resourceSoftLayerBasicMonitor(),
		},

		ConfigureFunc: providerConfigure,
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	sess := session.Session{
		UserName: d.Get("username").(string),
		APIKey:   d.Get("api_key").(string),
		Endpoint: d.Get("endpoint_url").(string),
	}

	if sess.UserName == "" || sess.APIKey == "" {
		return nil, errors.New(
			"No SoftLayer credentials were found. Please ensure you have specified" +
				" them in the provider or in the environment (see the documentation).",
		)
	}

	if os.Getenv("TF_LOG") != "" {
		sess.Debug = true
	}

	return &sess, nil
}
<|endoftext|>


<commit_before>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

// ciao-launcher is ssntp agent that runs on compute and network nodes.
// Its primary purpose is to launch and manage containers and virtual
// machines.  For more information on installing and running ciao-launcher
// see https://github.com/01org/ciao/blob/master/ciao-launcher/README.md
// for more information.
//
// Introduction
//
// ciao-launcher tries to take advantage of Go's concurrency support as much as
// possible.  The intention here is that most of the work involved in launching
// and manipulating VMs is mostly self contained and IO bound and so should in
// theory lend itself well to a concurrent design.  As a consequence, ciao-launcher
// is highly concurrent and performant.  The concurrent nature
// of ciao-launcher can make it a little difficult to understand for new comers,
// so here are a few notes on its design.  ciao-launcher can be thought of as a
// collection of distinct go routines.  These notes explain what these go
// routines are for and how they communicate.
//
// Main
//
// Main is the go routine that starts when ciao-launcher is itself launched.  The code
// for this is in main.go.  It parses the command line parameters, initialises
// networking, ensures that no other instance of ciao-launcher are running and then
// starts the server go routine.  Having done all this, the main go routine waits
// for a signal, e.g., SIGTERM, from the OS to quit.  When this signal is retrieved
// it instructs all child go routines to quit and waits for their exit.  Note that
// it only waits for 1 second.  If all child go routines have failed to exit in 1
// second, ciao-launcher panics.  The panic is useful as it prints the stack trace of
// all the running go routines, so you can see which ones are blocked.  At least
// this was the intention.  The default behaviour of the go runtime has changed in
// this regard in 1.6 so a small code change is required, but you get the idea, I
// hope.
//
// The Server go routine
//
// Manages the connection to the SSNTP server and pre-processes all commands
// received from this server. The code for this go routine is also in main.go, at
// least for the time being.  ciao-Launcher
// establishes a connection to the ssntp server via the ssntp.Dial command.  This
// creates a separate go routine, managed by the ssntp library.  Any subsequent
// ssntp events that occur are handled by the ciao-launcher function CommandNotify.
// CommandNotify is called in the context of the ssntp go routine.  To avoid
// blocking this go routine, ciao-launcher parses the YAML associated with the command,
// sends a newly created internal command down a channel to the server go routine
// and returns.  The command is then processed further in the server go routine.
//
// Most commands are operations on instances, e.g., create a new instance, restart
// an instance, and are ultimately processed by a go routine dedicated to the
// particular instance to which they pertain.  These instance go routines are
// managed by another go routine called the overseer which will be discussed in
// more detail below.  Before the server go routine can forward a command to the
// appropriate go routine it needs to ask the overseer for a channel which can be
// used to communicate with the relevant instance go routine.  This is done by
// sending an ovsAddCmd or an ovsGetCmd to the overseer via the overseer channel,
// ovsCh.  ovsAddCmd is used when starting a new instance.  ovsGetCmd is used to
// process all other commands.
//
// The overseer go routine is started by the server go routine.  When the server
// go routine is asked to exit by the main go routine, i.e., main go routine
// closes the doneCh channel, the server go routine closes the channel it uses
// to communicate with the overseer.  This instructs the overseer to close,
// which it does after all the instance go routines it manages have in turn
// exited.  The server go routine waits for the overseer to exit before
// terminating.
//
// The Overseer
//
// The overseer is a go routine that serves three main purposes.
//
//  1.  It manages instance go routines that themselves manage individual vms.
//  2.  It collects statistics about the node and the VMs it hosts and
//      tranmits these periodically to the ssntp server via the STATS and
//      STATUS commands.
//  3.  It Rediscovers and reconnects to existing instances when ciao-launcher is started.
// Overseer launches new instances via the startInstance function from instance.go.
// This function starts a new go routine for that instance and returns a channel
// through which commands can be sent to the instance.  The overseer itself does
// not send commands down this channel.  It cannot as this would lead to deadlock.
// Instead it makes this channel available to the server go routine when it is
// requested via the ovsAddCmd or the ovsGetCmd commands.
//
// The overseer passes each instance go routine a reference to the a single
// channel, childDoneCh.  This channel is closed when the overseer starts shutting
// down.  Closing this channel serves as a broadcast notification to each instance
// go routine, indicating that they need to shutdown.  The overseer waits until all
// instance go routines have shut down before exiting.  This is achieved via a wait
// group called chilWg.
//
// The overseer maintains a map of information about each instance called
// instances.  The map is indexed by the uuid of instances.  It contains
// information about the instances, namely their running state and their resource
// usage.  This information is used when sending STATs and STATUS commands.
//
// Information about the instances ultimately comes from the instance go routines.
// However, these go routines cannot access the overseer's instance map directly.
// To update it they send commands down a channel, which the overseer passes to
// startInstance, for example, ovsStatsUpdateCmd or ovsStateChange.  The overseer
// processes these commands in the processCommand function.
//
// The Instance Go routines
//
// ciao-launcher maintains one go routine per instance it manages.  These go routines
// exist regardless of the state of the underlying instance, i.e., there is
// generally one go routine running per instance, regardless of whether that
// instance is pending, exited or running.
//
// The instance go routines serve 3 main purposes:
//
//  1. They accept and process commands from the server go routine down their
//     command channel.  These commands typically come from the ssntp server,
//     although there are some occasions where the commands originate from inside
//     ciao-launcher itself.
//  2. They monitor the running state of VMs.
//  3. They manage the collection of instance statistics, which they report
//     to the overseer.
//
// The nice thing about this design is that almost all instance related work can be
// performed in parallel.  Stats can be computed for one instance at the same time
// as a separate instance is being powered down.  ciao-Launcher can process any number
// of commands to start new instances in parallel.  There is no locking required
// apart from a synchronised access to the overseer map made by the server go
// routine when the command is first received and a small check related to the
// image from which the instance will be launched, in the case where instances are
// being started.  An additional synchronisation point is required for docker
// instances to ensure that the relevant docker network has been created
// before the container.
//
// Note that although commands submitted to different instances are in executed in
// parallel, the instance go routines serialise commands issue against a single
// instance.  This is necessary to avoid instance corruption.
//
// Right now the command channels that the server go routine uses to send command
// to instances are not buffered.  This might need to change as currently it could
// be possible for a SSNTP server to kill ciao-launcher's parallelism by repetively
// sending commands to the same instance over and over again.
//
// The code for the instance go routines is in instance.go.  However, the code that
// executes most of the commands has been placed in separate files named after the
// commands themselves, e.g., start.go, delete.go, restart.go.  It should be noted
// that the code in these files runs in the context of an instance go routine.
// Finally, some of the code used to process instance commands is in payloads.go.
// This is for legacy reasons and in the future this file will probably go away and
// its contents will be redistributed.  It should not be assumed as of the time of
// writing that all the code in payloads.go runs in an instance go routine.
// payloads.go needs cleaning up (https://github.com/01org/ciao/issues/10).
//
// The virtualizer
//
// The instance go routines need to talk to qemu and docker to manage their VMs
// and containers.  However, they do not do so directly.  Rather they do so via
// a virtualizer interface.
//
// The virtualizer interface is designed to isolate ciao-launcher, and in particular,
// functions that run in the instance go routine, from the underlying virtualisation
// technologies used to launch and manage VMs and containers.
// All the methods on the virtualizer interface will be called serially by the instance
// go routine.  Therefore there is no need to synchronised data between the virtualizers
// methods.
//
// qemu.go contains functions to start and stop a VM, to create a qcow2 image, to
// collect statistics about the instance running in the hypervisor, such as its
// memory and cpu usage, and to monitor the instance, i.e., to determine whether
// the VM is actually running or not.
//
// docker.go contains methods to manage docker containers.
//
// For more information about the virtualizer API, please see the comments
// in https://github.com/01org/ciao/blob/master/ciao-launcher/virtualizer.go
//
package main
<commit_msg>ciao-launcher: doc grammar tweek<commit_after>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

// ciao-launcher is an ssntp agent that runs on compute and network nodes.
// Its primary purpose is to launch and manage containers and virtual
// machines.  For more information on installing and running ciao-launcher
// see https://github.com/01org/ciao/blob/master/ciao-launcher/README.md
// for more information.
//
// Introduction
//
// ciao-launcher tries to take advantage of Go's concurrency support as much as
// possible.  The intention here is that most of the work involved in launching
// and manipulating VMs is mostly self contained and IO bound and so should in
// theory lend itself well to a concurrent design.  As a consequence, ciao-launcher
// is highly concurrent and performant.  The concurrent nature
// of ciao-launcher can make it a little difficult to understand for new comers,
// so here are a few notes on its design.  ciao-launcher can be thought of as a
// collection of distinct go routines.  These notes explain what these go
// routines are for and how they communicate.
//
// Main
//
// Main is the go routine that starts when ciao-launcher is itself launched.  The code
// for this is in main.go.  It parses the command line parameters, initialises
// networking, ensures that no other instance of ciao-launcher are running and then
// starts the server go routine.  Having done all this, the main go routine waits
// for a signal, e.g., SIGTERM, from the OS to quit.  When this signal is retrieved
// it instructs all child go routines to quit and waits for their exit.  Note that
// it only waits for 1 second.  If all child go routines have failed to exit in 1
// second, ciao-launcher panics.  The panic is useful as it prints the stack trace of
// all the running go routines, so you can see which ones are blocked.  At least
// this was the intention.  The default behaviour of the go runtime has changed in
// this regard in 1.6 so a small code change is required, but you get the idea, I
// hope.
//
// The Server go routine
//
// Manages the connection to the SSNTP server and pre-processes all commands
// received from this server. The code for this go routine is also in main.go, at
// least for the time being.  ciao-Launcher
// establishes a connection to the ssntp server via the ssntp.Dial command.  This
// creates a separate go routine, managed by the ssntp library.  Any subsequent
// ssntp events that occur are handled by the ciao-launcher function CommandNotify.
// CommandNotify is called in the context of the ssntp go routine.  To avoid
// blocking this go routine, ciao-launcher parses the YAML associated with the command,
// sends a newly created internal command down a channel to the server go routine
// and returns.  The command is then processed further in the server go routine.
//
// Most commands are operations on instances, e.g., create a new instance, restart
// an instance, and are ultimately processed by a go routine dedicated to the
// particular instance to which they pertain.  These instance go routines are
// managed by another go routine called the overseer which will be discussed in
// more detail below.  Before the server go routine can forward a command to the
// appropriate go routine it needs to ask the overseer for a channel which can be
// used to communicate with the relevant instance go routine.  This is done by
// sending an ovsAddCmd or an ovsGetCmd to the overseer via the overseer channel,
// ovsCh.  ovsAddCmd is used when starting a new instance.  ovsGetCmd is used to
// process all other commands.
//
// The overseer go routine is started by the server go routine.  When the server
// go routine is asked to exit by the main go routine, i.e., main go routine
// closes the doneCh channel, the server go routine closes the channel it uses
// to communicate with the overseer.  This instructs the overseer to close,
// which it does after all the instance go routines it manages have in turn
// exited.  The server go routine waits for the overseer to exit before
// terminating.
//
// The Overseer
//
// The overseer is a go routine that serves three main purposes.
//
//  1.  It manages instance go routines that themselves manage individual vms.
//  2.  It collects statistics about the node and the VMs it hosts and
//      tranmits these periodically to the ssntp server via the STATS and
//      STATUS commands.
//  3.  It Rediscovers and reconnects to existing instances when ciao-launcher is started.
// Overseer launches new instances via the startInstance function from instance.go.
// This function starts a new go routine for that instance and returns a channel
// through which commands can be sent to the instance.  The overseer itself does
// not send commands down this channel.  It cannot as this would lead to deadlock.
// Instead it makes this channel available to the server go routine when it is
// requested via the ovsAddCmd or the ovsGetCmd commands.
//
// The overseer passes each instance go routine a reference to the a single
// channel, childDoneCh.  This channel is closed when the overseer starts shutting
// down.  Closing this channel serves as a broadcast notification to each instance
// go routine, indicating that they need to shutdown.  The overseer waits until all
// instance go routines have shut down before exiting.  This is achieved via a wait
// group called chilWg.
//
// The overseer maintains a map of information about each instance called
// instances.  The map is indexed by the uuid of instances.  It contains
// information about the instances, namely their running state and their resource
// usage.  This information is used when sending STATs and STATUS commands.
//
// Information about the instances ultimately comes from the instance go routines.
// However, these go routines cannot access the overseer's instance map directly.
// To update it they send commands down a channel, which the overseer passes to
// startInstance, for example, ovsStatsUpdateCmd or ovsStateChange.  The overseer
// processes these commands in the processCommand function.
//
// The Instance Go routines
//
// ciao-launcher maintains one go routine per instance it manages.  These go routines
// exist regardless of the state of the underlying instance, i.e., there is
// generally one go routine running per instance, regardless of whether that
// instance is pending, exited or running.
//
// The instance go routines serve 3 main purposes:
//
//  1. They accept and process commands from the server go routine down their
//     command channel.  These commands typically come from the ssntp server,
//     although there are some occasions where the commands originate from inside
//     ciao-launcher itself.
//  2. They monitor the running state of VMs.
//  3. They manage the collection of instance statistics, which they report
//     to the overseer.
//
// The nice thing about this design is that almost all instance related work can be
// performed in parallel.  Stats can be computed for one instance at the same time
// as a separate instance is being powered down.  ciao-Launcher can process any number
// of commands to start new instances in parallel.  There is no locking required
// apart from a synchronised access to the overseer map made by the server go
// routine when the command is first received and a small check related to the
// image from which the instance will be launched, in the case where instances are
// being started.  An additional synchronisation point is required for docker
// instances to ensure that the relevant docker network has been created
// before the container.
//
// Note that although commands submitted to different instances are in executed in
// parallel, the instance go routines serialise commands issue against a single
// instance.  This is necessary to avoid instance corruption.
//
// Right now the command channels that the server go routine uses to send command
// to instances are not buffered.  This might need to change as currently it could
// be possible for a SSNTP server to kill ciao-launcher's parallelism by repetively
// sending commands to the same instance over and over again.
//
// The code for the instance go routines is in instance.go.  However, the code that
// executes most of the commands has been placed in separate files named after the
// commands themselves, e.g., start.go, delete.go, restart.go.  It should be noted
// that the code in these files runs in the context of an instance go routine.
// Finally, some of the code used to process instance commands is in payloads.go.
// This is for legacy reasons and in the future this file will probably go away and
// its contents will be redistributed.  It should not be assumed as of the time of
// writing that all the code in payloads.go runs in an instance go routine.
// payloads.go needs cleaning up (https://github.com/01org/ciao/issues/10).
//
// The virtualizer
//
// The instance go routines need to talk to qemu and docker to manage their VMs
// and containers.  However, they do not do so directly.  Rather they do so via
// a virtualizer interface.
//
// The virtualizer interface is designed to isolate ciao-launcher, and in particular,
// functions that run in the instance go routine, from the underlying virtualisation
// technologies used to launch and manage VMs and containers.
// All the methods on the virtualizer interface will be called serially by the instance
// go routine.  Therefore there is no need to synchronised data between the virtualizers
// methods.
//
// qemu.go contains functions to start and stop a VM, to create a qcow2 image, to
// collect statistics about the instance running in the hypervisor, such as its
// memory and cpu usage, and to monitor the instance, i.e., to determine whether
// the VM is actually running or not.
//
// docker.go contains methods to manage docker containers.
//
// For more information about the virtualizer API, please see the comments
// in https://github.com/01org/ciao/blob/master/ciao-launcher/virtualizer.go
//
package main
<|endoftext|>