<commit_before>package main

// TODO: figure out how to handle varying numbers of pixels
// when we're getting pixels via our OPC server source

import (
	"fmt"
	"github.com/davecheney/profile"
	"github.com/droundy/goopt"
	"github.com/longears/pixelslinger/beaglebone"
	"github.com/longears/pixelslinger/midi"
	"github.com/longears/pixelslinger/opc"
	"os"
	"sort"
	"strings"
	"time"
)

const ONBOARD_LED_HEARTBEAT = 0
const ONBOARD_LED_MIDI = 1

const SPI_MAGIC_WORD = "spi"
const PRINT_MAGIC_WORD = "print"
const DEVNULL_MAGIC_WORD = "/dev/null"
const LOCALHOST = "localhost"
const SPI_FN = "/dev/spidev1.0"

// these are pointers to the actual values from the command line parser
var LAYOUT_FN = goopt.String([]string{"-l", "--layout"}, "...", "layout file (required)")
var SOURCE = goopt.String([]string{"-s", "--source"}, "spatial-stripes", "pixel source (either a pattern name or "+LOCALHOST+"[:port])")
var DEST = goopt.String([]string{"-d", "--dest"}, "localhost", "destination (one of "+PRINT_MAGIC_WORD+", "+SPI_MAGIC_WORD+", "+DEVNULL_MAGIC_WORD+", or hostname[:port])")
var FPS = goopt.Int([]string{"-f", "--fps"}, 40, "max frames per second")
var SECONDS = goopt.Int([]string{"-n", "--seconds"}, 0, "quit after this many seconds")
var ONCE = goopt.Flag([]string{"-o", "--once"}, []string{}, "quit after one frame", "")

// Parse the command line flags.  If invalid, show help and quit.
// Add default ports if needed.
// Read the layout file.
// Return the number of pixels in the layout, the source and dest thread methods.
func parseFlags() (nPixels int, sourceThread, effectThread, destThread opc.ByteThread) {

	// get sorted pattern names
	patternNames := make([]string, len(opc.PATTERN_REGISTRY))
	ii := 0
	for k, _ := range opc.PATTERN_REGISTRY {
		patternNames[ii] = k
		ii++
	}
	sort.Strings(patternNames)

	goopt.Summary = "Available source patterns:\n"
	for _, patternName := range patternNames {
		goopt.Summary += "          " + patternName + "\n"
	}
	goopt.Parse(nil)

	// layout is required
	if *LAYOUT_FN == "..." {
		fmt.Println(goopt.Usage())
		fmt.Println("--------------------------------------------------------------------------------/")
		os.Exit(1)
	}

	// read locations
	locations := opc.ReadLocations(*LAYOUT_FN)
	nPixels = len(locations) / 3

	// choose source thread method
	if strings.Contains(*SOURCE, LOCALHOST) {
		// source is localhost, so we will start an OPC server.
		// add default port if needed
		if !strings.Contains(*SOURCE, ":") {
			*SOURCE += ":7890"
		}
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else if (*SOURCE)[0] == ':' {
		// source is ":4908"
		*SOURCE = "localhost" + *SOURCE
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else {
		// source is a pattern name
		sourceThreadMaker, ok := opc.PATTERN_REGISTRY[*SOURCE]
		if !ok {
			fmt.Printf("Error: unknown source or pattern \"%s\"\n", *SOURCE)
			fmt.Println("--------------------------------------------------------------------------------/")
			os.Exit(1)
		}
		sourceThread = sourceThreadMaker(locations)
	}

    // choose effect thread method
    effectThread = opc.MakeEffectFader(locations)

	// choose dest thread method
	switch *DEST {
	case DEVNULL_MAGIC_WORD:
		destThread = opc.MakeSendToDevNullThread()
	case PRINT_MAGIC_WORD:
		destThread = opc.MakeSendToScreenThread()
	case SPI_MAGIC_WORD:
		destThread = opc.MakeSendToLPD8806Thread(SPI_FN)
	default:
		// add default port if needed
		if !strings.Contains(*DEST, ":") {
			*DEST += ":7890"
		}
		destThread = opc.MakeSendToOpcThread(*DEST)
	}

	return // returns nPixels, sourceThread, destThread
}

// Launch the sourceThread and destThread methods and coordinate the transfer of bytes from one to the other.
// Run until timeToRun seconds have passed and return.  If timeToRun is 0, run forever.
// Turn on the CPU profiler if timeToRun seconds > 0.
// Limit the framerate to a max of fps unless fps is 0.
func mainLoop(nPixels int, sourceThread, effectThread, destThread opc.ByteThread, fps float64, timeToRun float64) {
	if timeToRun > 0 {
		fmt.Printf("[mainLoop] Running for %f seconds with profiling turned on, pixels and network\n", timeToRun)
		defer profile.Start(profile.CPUProfile).Stop()
	} else {
		fmt.Println("[mainLoop] Running forever")
	}

	// prepare the byte slices and channels that connect the source and dest threads
	fillingSlice := make([]byte, nPixels*3)
	sendingSlice := make([]byte, nPixels*3)

	bytesToFillChan := make(chan []byte, 0)
    toEffectChan := make(chan []byte, 0)
	bytesFilledChan := make(chan []byte, 0)
	bytesToSendChan := make(chan []byte, 0)
	bytesSentChan := make(chan []byte, 0)

	// set up midi
	midiMessageChan := midi.GetMidiMessageStream("/dev/midi1") // this launches the midi thread
	midiState := midi.MidiState{}
    // set initial values for controller knobs
    //  because the midi hardware only sends us values when the knobs move
    midiState.ControllerValues[1] = 127

	// launch the threads
	go sourceThread(bytesToFillChan, toEffectChan, &midiState)
	go effectThread(toEffectChan, bytesFilledChan, &midiState)
	go destThread(bytesToSendChan, bytesSentChan, &midiState)

	// main loop
	frame_budget_ms := 1000.0 / fps
	startTime := float64(time.Now().UnixNano()) / 1.0e9
	lastPrintTime := startTime
	frameStartTime := startTime
	frameEndTime := startTime
	framesSinceLastPrint := 0
	firstIteration := true
	flipper := 0
	beaglebone.SetOnboardLED(0, 1)
	for {
		// if we have any frame budget left from last time around, sleep to control the framerate
		if fps > 0 {
			frameEndTime = float64(time.Now().UnixNano()) / 1.0e9
			timeRemaining := float64(frame_budget_ms)/1000 - (frameEndTime - frameStartTime)
			if timeRemaining > 0 {
				time.Sleep(time.Duration(timeRemaining*1000*1000) * time.Microsecond)
			}
		}

		// fps reporting and bookkeeping
		// print framerate occasionally
		frameStartTime = float64(time.Now().UnixNano()) / 1.0e9
		framesSinceLastPrint += 1
		if frameStartTime > lastPrintTime+1 {
			lastPrintTime = frameStartTime
			fmt.Printf("[mainLoop] %f ms/frame (%d fps)\n", 1000.0/float64(framesSinceLastPrint), framesSinceLastPrint)
			framesSinceLastPrint = 0
			// toggle LED
			beaglebone.SetOnboardLED(ONBOARD_LED_HEARTBEAT, flipper)
			flipper = 1 - flipper
		}

		// if profiling, quit after a while
		if timeToRun > 0 && frameStartTime > startTime+timeToRun {
			return
		}

		// get midi
		midiState.UpdateStateFromChannel(midiMessageChan)
		if len(midiState.RecentMidiMessages) > 0 {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 1)
		} else {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 0)
		}

		// start the threads filling and sending slices in parallel.
		// if this is the first time through the loop we have to skip
		//  the sending stage or we'll send out a whole bunch of zeros.
		bytesToFillChan <- fillingSlice
		if !firstIteration {
			bytesToSendChan <- sendingSlice
		}

		// if only sending one frame, let's just get it all over with now
		//  or we'd have to compute two frames worth of pixels because of
		//  the double buffering effect of the two parallel threads
		if *ONCE {
			// get filled bytes and send them
			bytesToSendChan <- <-bytesFilledChan
			// wait for sending to complete
			<-bytesSentChan
			fmt.Println("[mainLoop] just running once.  quitting now.")
			return
		}

		// wait until both filling and sending threads are done
		<-bytesFilledChan
		if !firstIteration {
			<-bytesSentChan
		}

		// swap the slices
		sendingSlice, fillingSlice = fillingSlice, sendingSlice

		firstIteration = false
	}
}

func main() {
	fmt.Println("--------------------------------------------------------------------------------\\")
	defer fmt.Println("--------------------------------------------------------------------------------/")

	nPixels, sourceThread, effectThread, destThread := parseFlags()
	mainLoop(nPixels, sourceThread, effectThread, destThread, float64(*FPS), float64(*SECONDS))
}
<commit_msg>Fix frame stutters during MIDI in.  go fmt.<commit_after>package main

// TODO: figure out how to handle varying numbers of pixels
// when we're getting pixels via our OPC server source

import (
	"fmt"
	"github.com/davecheney/profile"
	"github.com/droundy/goopt"
	"github.com/longears/pixelslinger/beaglebone"
	"github.com/longears/pixelslinger/midi"
	"github.com/longears/pixelslinger/opc"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"
)

const ONBOARD_LED_HEARTBEAT = 0
const ONBOARD_LED_MIDI = 1

const SPI_MAGIC_WORD = "spi"
const PRINT_MAGIC_WORD = "print"
const DEVNULL_MAGIC_WORD = "/dev/null"
const LOCALHOST = "localhost"
const SPI_FN = "/dev/spidev1.0"

func init() {
	runtime.GOMAXPROCS(2)
}

// these are pointers to the actual values from the command line parser
var LAYOUT_FN = goopt.String([]string{"-l", "--layout"}, "...", "layout file (required)")
var SOURCE = goopt.String([]string{"-s", "--source"}, "spatial-stripes", "pixel source (either a pattern name or "+LOCALHOST+"[:port])")
var DEST = goopt.String([]string{"-d", "--dest"}, "localhost", "destination (one of "+PRINT_MAGIC_WORD+", "+SPI_MAGIC_WORD+", "+DEVNULL_MAGIC_WORD+", or hostname[:port])")
var FPS = goopt.Int([]string{"-f", "--fps"}, 40, "max frames per second")
var SECONDS = goopt.Int([]string{"-n", "--seconds"}, 0, "quit after this many seconds")
var ONCE = goopt.Flag([]string{"-o", "--once"}, []string{}, "quit after one frame", "")

// Parse the command line flags.  If invalid, show help and quit.
// Add default ports if needed.
// Read the layout file.
// Return the number of pixels in the layout, the source and dest thread methods.
func parseFlags() (nPixels int, sourceThread, effectThread, destThread opc.ByteThread) {

	// get sorted pattern names
	patternNames := make([]string, len(opc.PATTERN_REGISTRY))
	ii := 0
	for k, _ := range opc.PATTERN_REGISTRY {
		patternNames[ii] = k
		ii++
	}
	sort.Strings(patternNames)

	goopt.Summary = "Available source patterns:\n"
	for _, patternName := range patternNames {
		goopt.Summary += "          " + patternName + "\n"
	}
	goopt.Parse(nil)

	// layout is required
	if *LAYOUT_FN == "..." {
		fmt.Println(goopt.Usage())
		fmt.Println("--------------------------------------------------------------------------------/")
		os.Exit(1)
	}

	// read locations
	locations := opc.ReadLocations(*LAYOUT_FN)
	nPixels = len(locations) / 3

	// choose source thread method
	if strings.Contains(*SOURCE, LOCALHOST) {
		// source is localhost, so we will start an OPC server.
		// add default port if needed
		if !strings.Contains(*SOURCE, ":") {
			*SOURCE += ":7890"
		}
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else if (*SOURCE)[0] == ':' {
		// source is ":4908"
		*SOURCE = "localhost" + *SOURCE
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else {
		// source is a pattern name
		sourceThreadMaker, ok := opc.PATTERN_REGISTRY[*SOURCE]
		if !ok {
			fmt.Printf("Error: unknown source or pattern \"%s\"\n", *SOURCE)
			fmt.Println("--------------------------------------------------------------------------------/")
			os.Exit(1)
		}
		sourceThread = sourceThreadMaker(locations)
	}

	// choose effect thread method
	effectThread = opc.MakeEffectFader(locations)

	// choose dest thread method
	switch *DEST {
	case DEVNULL_MAGIC_WORD:
		destThread = opc.MakeSendToDevNullThread()
	case PRINT_MAGIC_WORD:
		destThread = opc.MakeSendToScreenThread()
	case SPI_MAGIC_WORD:
		destThread = opc.MakeSendToLPD8806Thread(SPI_FN)
	default:
		// add default port if needed
		if !strings.Contains(*DEST, ":") {
			*DEST += ":7890"
		}
		destThread = opc.MakeSendToOpcThread(*DEST)
	}

	return // returns nPixels, sourceThread, destThread
}

// Launch the sourceThread and destThread methods and coordinate the transfer of bytes from one to the other.
// Run until timeToRun seconds have passed and return.  If timeToRun is 0, run forever.
// Turn on the CPU profiler if timeToRun seconds > 0.
// Limit the framerate to a max of fps unless fps is 0.
func mainLoop(nPixels int, sourceThread, effectThread, destThread opc.ByteThread, fps float64, timeToRun float64) {
	if timeToRun > 0 {
		fmt.Printf("[mainLoop] Running for %f seconds with profiling turned on, pixels and network\n", timeToRun)
		defer profile.Start(profile.CPUProfile).Stop()
	} else {
		fmt.Println("[mainLoop] Running forever")
	}

	// prepare the byte slices and channels that connect the source and dest threads
	fillingSlice := make([]byte, nPixels*3)
	sendingSlice := make([]byte, nPixels*3)

	bytesToFillChan := make(chan []byte, 0)
	toEffectChan := make(chan []byte, 0)
	bytesFilledChan := make(chan []byte, 0)
	bytesToSendChan := make(chan []byte, 0)
	bytesSentChan := make(chan []byte, 0)

	// set up midi
	midiMessageChan := midi.GetMidiMessageStream("/dev/midi1") // this launches the midi thread
	midiState := midi.MidiState{}
	// set initial values for controller knobs
	//  because the midi hardware only sends us values when the knobs move
	midiState.ControllerValues[1] = 127

	// launch the threads
	go sourceThread(bytesToFillChan, toEffectChan, &midiState)
	go effectThread(toEffectChan, bytesFilledChan, &midiState)
	go destThread(bytesToSendChan, bytesSentChan, &midiState)

	// main loop
	frame_budget_ms := 1000.0 / fps
	startTime := float64(time.Now().UnixNano()) / 1.0e9
	lastPrintTime := startTime
	frameStartTime := startTime
	frameEndTime := startTime
	framesSinceLastPrint := 0
	firstIteration := true
	flipper := 0
	beaglebone.SetOnboardLED(0, 1)
	for {
		// if we have any frame budget left from last time around, sleep to control the framerate
		if fps > 0 {
			frameEndTime = float64(time.Now().UnixNano()) / 1.0e9
			timeRemaining := float64(frame_budget_ms)/1000 - (frameEndTime - frameStartTime)
			if timeRemaining > 0 {
				time.Sleep(time.Duration(timeRemaining*1000*1000) * time.Microsecond)
			}
		}

		// fps reporting and bookkeeping
		// print framerate occasionally
		frameStartTime = float64(time.Now().UnixNano()) / 1.0e9
		framesSinceLastPrint += 1
		if frameStartTime > lastPrintTime+1 {
			lastPrintTime = frameStartTime
			fmt.Printf("[mainLoop] %f ms/frame (%d fps)\n", 1000.0/float64(framesSinceLastPrint), framesSinceLastPrint)
			framesSinceLastPrint = 0
			// toggle LED
			beaglebone.SetOnboardLED(ONBOARD_LED_HEARTBEAT, flipper)
			flipper = 1 - flipper
		}

		// if profiling, quit after a while
		if timeToRun > 0 && frameStartTime > startTime+timeToRun {
			return
		}

		// get midi
		midiState.UpdateStateFromChannel(midiMessageChan)
		if len(midiState.RecentMidiMessages) > 0 {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 1)
		} else {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 0)
		}

		// start the threads filling and sending slices in parallel.
		// if this is the first time through the loop we have to skip
		//  the sending stage or we'll send out a whole bunch of zeros.
		bytesToFillChan <- fillingSlice
		if !firstIteration {
			bytesToSendChan <- sendingSlice
		}

		// if only sending one frame, let's just get it all over with now
		//  or we'd have to compute two frames worth of pixels because of
		//  the double buffering effect of the two parallel threads
		if *ONCE {
			// get filled bytes and send them
			bytesToSendChan <- <-bytesFilledChan
			// wait for sending to complete
			<-bytesSentChan
			fmt.Println("[mainLoop] just running once.  quitting now.")
			return
		}

		// wait until both filling and sending threads are done
		<-bytesFilledChan
		if !firstIteration {
			<-bytesSentChan
		}

		// swap the slices
		sendingSlice, fillingSlice = fillingSlice, sendingSlice

		firstIteration = false
	}
}

func main() {
	fmt.Println("--------------------------------------------------------------------------------\\")
	defer fmt.Println("--------------------------------------------------------------------------------/")

	nPixels, sourceThread, effectThread, destThread := parseFlags()
	mainLoop(nPixels, sourceThread, effectThread, destThread, float64(*FPS), float64(*SECONDS))
}
<|endoftext|>


<commit_before>package cluster

import (
	"encoding/json"
	"fmt"
	"io/ioutil"

	"github.com/openshift/installer/pkg/asset"
	"github.com/openshift/installer/pkg/terraform"
	"github.com/openshift/installer/pkg/types/config"
)

const (
	stateFileName = "terraform.state"
)

// Cluster uses the terraform executable to launch a cluster
// with the given terraform tfvar and generated templates.
type Cluster struct {
	// The root directory of the generated assets.
	rootDir    string
	tfvars     asset.Asset
	kubeconfig asset.Asset
}

var _ asset.Asset = (*Cluster)(nil)

// Name returns the human-friendly name of the asset.
func (c *Cluster) Name() string {
	return "Cluster"
}

// Dependencies returns the direct dependency for launching
// the cluster.
func (c *Cluster) Dependencies() []asset.Asset {
	return []asset.Asset{c.tfvars, c.kubeconfig}
}

// Generate launches the cluster and generates the terraform state file on disk.
func (c *Cluster) Generate(parents map[asset.Asset]*asset.State) (*asset.State, error) {
	state, ok := parents[c.tfvars]
	if !ok {
		return nil, fmt.Errorf("failed to get terraform.tfvar state in the parent asset states")
	}

	var tfvars config.Cluster
	if err := json.Unmarshal(state.Contents[0].Data, &tfvars); err != nil {
		return nil, fmt.Errorf("failed to unmarshal terraform tfvars file: %v", err)
	}

	dir, err := terraform.BaseLocation()
	if err != nil {
		return nil, err
	}

	var result asset.State

	templateDir, err := terraform.FindStepTemplates(dir, terraform.InfraStep, tfvars.Platform)
	if err != nil {
		return nil, err
	}

	// This runs the terraform in the terraform template directory.
	if err := terraform.Init(dir, templateDir); err != nil {
		return nil, err
	}

	stateFile, err := terraform.Apply(dir, terraform.InfraStep, templateDir)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, fmt.Errorf("failed to read tfstate file %q: %v", stateFile, err)
	}

	result.Contents = append(result.Contents, asset.Content{
		Name: stateFileName,
		Data: data,
	})

	// TODO(yifan): Use the kubeconfig to verify the cluster is up.

	return &result, nil
}
<commit_msg>asset/cluster: Copy the terraform.tfvars<commit_after>package cluster

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/openshift/installer/pkg/asset"
	"github.com/openshift/installer/pkg/terraform"
	"github.com/openshift/installer/pkg/types/config"
)

const (
	stateFileName = "terraform.state"
)

// Cluster uses the terraform executable to launch a cluster
// with the given terraform tfvar and generated templates.
type Cluster struct {
	// The root directory of the generated assets.
	rootDir    string
	tfvars     asset.Asset
	kubeconfig asset.Asset
}

var _ asset.Asset = (*Cluster)(nil)

// Name returns the human-friendly name of the asset.
func (c *Cluster) Name() string {
	return "Cluster"
}

// Dependencies returns the direct dependency for launching
// the cluster.
func (c *Cluster) Dependencies() []asset.Asset {
	return []asset.Asset{c.tfvars, c.kubeconfig}
}

// Generate launches the cluster and generates the terraform state file on disk.
func (c *Cluster) Generate(parents map[asset.Asset]*asset.State) (*asset.State, error) {
	dir, err := terraform.BaseLocation()
	if err != nil {
		return nil, err
	}

	state, ok := parents[c.tfvars]
	if !ok {
		return nil, fmt.Errorf("failed to get terraform.tfvar state in the parent asset states")
	}

	// Copy the terraform.tfvars to the working dir for the terraform.
	if err := ioutil.WriteFile(filepath.Join(dir, state.Contents[0].Name), state.Contents[0].Data, 0600); err != nil {
		return nil, fmt.Errorf("failed to write terraform.tfvars file: %v", err)
	}

	var tfvars config.Cluster
	if err := json.Unmarshal(state.Contents[0].Data, &tfvars); err != nil {
		return nil, fmt.Errorf("failed to unmarshal terraform tfvars file: %v", err)
	}

	templateDir, err := terraform.FindStepTemplates(dir, terraform.InfraStep, tfvars.Platform)
	if err != nil {
		return nil, err
	}

	// This runs the terraform in the terraform template directory.
	if err := terraform.Init(dir, templateDir); err != nil {
		return nil, err
	}

	stateFile, err := terraform.Apply(dir, terraform.InfraStep, templateDir)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, fmt.Errorf("failed to read tfstate file %q: %v", stateFile, err)
	}

	// TODO(yifan): Use the kubeconfig to verify the cluster is up.
	return &asset.State{
		Contents: []asset.Content{
			{
				Name: stateFileName,
				Data: data,
			},
		},
	}, nil
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"gopkg.in/yaml.v1"
	"os"
	"path/filepath"
	"strings"

	ghodssYaml "github.com/ghodss/yaml"
	"gopkg.in/oleiade/reflections.v1"

	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/dappdeps"
	"github.com/flant/dapp/pkg/util"
)

type Ansible struct {
	config *config.Ansible
	extra  *Extra
}

type Extra struct {
	ContainerDappPath string
	TmpPath           string
}

func NewAnsibleBuilder(config *config.Ansible, path *Extra) *Ansible {
	return &Ansible{config: config, extra: path}
}

func (b *Ansible) IsBeforeInstallEmpty() bool { return b.isEmptyStage("BeforeInstall") }
func (b *Ansible) IsInstallEmpty() bool       { return b.isEmptyStage("Install") }
func (b *Ansible) IsBeforeSetupEmpty() bool   { return b.isEmptyStage("BeforeSetup") }
func (b *Ansible) IsSetupEmpty() bool         { return b.isEmptyStage("Setup") }
func (b *Ansible) IsBuildArtifactEmpty() bool { return b.isEmptyStage("BuildArtifact") }

func (b *Ansible) BeforeInstall(container Container) error { return b.stage("BeforeInstall", container) }
func (b *Ansible) Install(container Container) error       { return b.stage("Install", container) }
func (b *Ansible) BeforeSetup(container Container) error   { return b.stage("BeforeSetup", container) }
func (b *Ansible) Setup(container Container) error         { return b.stage("Setup", container) }
func (b *Ansible) BuildArtifact(container Container) error { return b.stage("BuildArtifact", container) }

func (b *Ansible) BeforeInstallChecksum() string { return b.stageChecksum("BeforeInstall") }
func (b *Ansible) InstallChecksum() string       { return b.stageChecksum("Install") }
func (b *Ansible) BeforeSetupChecksum() string   { return b.stageChecksum("BeforeSetup") }
func (b *Ansible) SetupChecksum() string         { return b.stageChecksum("Setup") }
func (b *Ansible) BuildArtifactChecksum() string { return b.stageChecksum("BuildArtifact") }

func (b *Ansible) isEmptyStage(userStageName string) bool {
	return b.stageChecksum(userStageName) == ""
}

func (b *Ansible) stage(userStageName string, container Container) error {
	if len(b.stageTasks(userStageName)) == 0 {
		return nil
	}

	if err := b.createStageWorkDirStructure(userStageName); err != nil {
		return err
	}

	container.AddEnv(
		map[string]string{
			"ANSIBLE_CONFIG":              filepath.Join(b.containerWorkDir(), "ansible.cfg"),
			"DAPP_DUMP_CONFIG_DOC_PATH":   filepath.Join(b.containerWorkDir(), "dump_config.json"),
			"PYTHONPATH":                  filepath.Join(b.containerWorkDir(), "lib"),
			"PYTHONIOENCODING":            "utf-8",
			"ANSIBLE_PREPEND_SYSTEM_PATH": dappdeps.BasePath(),
		},
	)

	stageHostWorkDir, err := b.stageHostWorkDir(userStageName)
	if err != nil {
		return err
	}

	stageHostTmpDir, err := b.stageHostTmpDir(userStageName)
	if err != nil {
		return err
	}

	container.AddVolume(
		fmt.Sprintf("%s:%s:ro", stageHostWorkDir, b.containerWorkDir()),
		fmt.Sprintf("%s:%s:rw", stageHostTmpDir, b.containerTmpDir()),
	)

	containerName, err := dappdeps.AnsibleContainer()
	if err != nil {
		return err
	}
	container.AddVolumeFrom(fmt.Sprintf("%s:ro", containerName))

	commands := []string{
		strings.Join([]string{dappdeps.AnsibleBinPath("ansible-playbook"), filepath.Join(b.containerWorkDir(), "playbook.yml")}, " "),
	}

	if value, exist := os.LookupEnv("ANSIBLE_ARGS"); exist {
		commands = append(commands, value)
	}

	container.AddRunCommands(commands...)

	return nil
}

func (b *Ansible) stageChecksum(userStageName string) string {
	var checksumArgs []string

	for _, task := range b.stageTasks(userStageName) {
		output, err := yaml.Marshal(task.Config)
		if err != nil {
			panic(fmt.Sprintf("runtime err: %s", err))
		}

		jsonOutput, err := ghodssYaml.YAMLToJSON(output)
		if err != nil {
			panic(fmt.Sprintf("runtime err: %s", err))
		}
		checksumArgs = append(checksumArgs, string(jsonOutput))
	}

	if stageVersionChecksum := b.stageVersionChecksum(userStageName); stageVersionChecksum != "" {
		checksumArgs = append(checksumArgs, stageVersionChecksum)
	}

	if len(checksumArgs) != 0 {
		return util.Sha256Hash(checksumArgs...)
	} else {
		return ""
	}
}

func (b *Ansible) stageVersionChecksum(userStageName string) string {
	var stageVersionChecksumArgs []string

	cacheVersionFieldName := "CacheVersion"
	stageCacheVersionFieldName := strings.Join([]string{userStageName, cacheVersionFieldName}, "")

	stageChecksum, ok := b.configFieldValue(stageCacheVersionFieldName).(string)
	if !ok {
		panic(fmt.Sprintf("runtime error: %#v", stageChecksum))
	}

	if stageChecksum != "" {
		stageVersionChecksumArgs = append(stageVersionChecksumArgs, stageChecksum)
	}

	checksum, ok := b.configFieldValue(cacheVersionFieldName).(string)
	if !ok {
		panic(fmt.Sprintf("runtime error: %#v", checksum))
	}

	if checksum != "" {
		stageVersionChecksumArgs = append(stageVersionChecksumArgs, checksum)
	}

	if len(stageVersionChecksumArgs) != 0 {
		return util.Sha256Hash(stageVersionChecksumArgs...)
	} else {
		return ""
	}
}

func (b *Ansible) stageTasks(userStageName string) []*config.AnsibleTask {
	value := b.configFieldValue(userStageName)
	ansibleTasks, ok := value.([]*config.AnsibleTask)
	if !ok {
		panic(fmt.Sprintf("runtime error"))
	}

	return ansibleTasks
}

func (b *Ansible) configFieldValue(fieldName string) interface{} {
	value, err := reflections.GetField(b.config, fieldName)
	if err != nil {
		panic(fmt.Sprintf("runtime error: %s", err))
	}

	return value
}

func (b *Ansible) stageHostWorkDir(userStageName string) (string, error) {
	path := filepath.Join(b.extra.TmpPath, fmt.Sprintf("ansible-workdir-%s", userStageName))

	if err := mkdirP(path); err != nil {
		return "", err
	}

	return path, nil
}
<commit_msg>Fix ANSIBLE_ARGS env doesn't work properly<commit_after>package builder

import (
	"fmt"
	"gopkg.in/yaml.v1"
	"os"
	"path/filepath"
	"strings"

	ghodssYaml "github.com/ghodss/yaml"
	"gopkg.in/oleiade/reflections.v1"

	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/dappdeps"
	"github.com/flant/dapp/pkg/util"
)

type Ansible struct {
	config *config.Ansible
	extra  *Extra
}

type Extra struct {
	ContainerDappPath string
	TmpPath           string
}

func NewAnsibleBuilder(config *config.Ansible, path *Extra) *Ansible {
	return &Ansible{config: config, extra: path}
}

func (b *Ansible) IsBeforeInstallEmpty() bool { return b.isEmptyStage("BeforeInstall") }
func (b *Ansible) IsInstallEmpty() bool       { return b.isEmptyStage("Install") }
func (b *Ansible) IsBeforeSetupEmpty() bool   { return b.isEmptyStage("BeforeSetup") }
func (b *Ansible) IsSetupEmpty() bool         { return b.isEmptyStage("Setup") }
func (b *Ansible) IsBuildArtifactEmpty() bool { return b.isEmptyStage("BuildArtifact") }

func (b *Ansible) BeforeInstall(container Container) error { return b.stage("BeforeInstall", container) }
func (b *Ansible) Install(container Container) error       { return b.stage("Install", container) }
func (b *Ansible) BeforeSetup(container Container) error   { return b.stage("BeforeSetup", container) }
func (b *Ansible) Setup(container Container) error         { return b.stage("Setup", container) }
func (b *Ansible) BuildArtifact(container Container) error { return b.stage("BuildArtifact", container) }

func (b *Ansible) BeforeInstallChecksum() string { return b.stageChecksum("BeforeInstall") }
func (b *Ansible) InstallChecksum() string       { return b.stageChecksum("Install") }
func (b *Ansible) BeforeSetupChecksum() string   { return b.stageChecksum("BeforeSetup") }
func (b *Ansible) SetupChecksum() string         { return b.stageChecksum("Setup") }
func (b *Ansible) BuildArtifactChecksum() string { return b.stageChecksum("BuildArtifact") }

func (b *Ansible) isEmptyStage(userStageName string) bool {
	return b.stageChecksum(userStageName) == ""
}

func (b *Ansible) stage(userStageName string, container Container) error {
	if len(b.stageTasks(userStageName)) == 0 {
		return nil
	}

	if err := b.createStageWorkDirStructure(userStageName); err != nil {
		return err
	}

	container.AddEnv(
		map[string]string{
			"ANSIBLE_CONFIG":              filepath.Join(b.containerWorkDir(), "ansible.cfg"),
			"DAPP_DUMP_CONFIG_DOC_PATH":   filepath.Join(b.containerWorkDir(), "dump_config.json"),
			"PYTHONPATH":                  filepath.Join(b.containerWorkDir(), "lib"),
			"PYTHONIOENCODING":            "utf-8",
			"ANSIBLE_PREPEND_SYSTEM_PATH": dappdeps.BasePath(),
		},
	)

	stageHostWorkDir, err := b.stageHostWorkDir(userStageName)
	if err != nil {
		return err
	}

	stageHostTmpDir, err := b.stageHostTmpDir(userStageName)
	if err != nil {
		return err
	}

	container.AddVolume(
		fmt.Sprintf("%s:%s:ro", stageHostWorkDir, b.containerWorkDir()),
		fmt.Sprintf("%s:%s:rw", stageHostTmpDir, b.containerTmpDir()),
	)

	containerName, err := dappdeps.AnsibleContainer()
	if err != nil {
		return err
	}
	container.AddVolumeFrom(fmt.Sprintf("%s:ro", containerName))

	commandParts := []string{
		dappdeps.AnsibleBinPath("ansible-playbook"),
		filepath.Join(b.containerWorkDir(), "playbook.yml"),
	}

	if value, exist := os.LookupEnv("ANSIBLE_ARGS"); exist {
		commandParts = append(commandParts, value)
	}

	command := strings.Join(commandParts, " ")
	container.AddRunCommands(command)

	return nil
}

func (b *Ansible) stageChecksum(userStageName string) string {
	var checksumArgs []string

	for _, task := range b.stageTasks(userStageName) {
		output, err := yaml.Marshal(task.Config)
		if err != nil {
			panic(fmt.Sprintf("runtime err: %s", err))
		}

		jsonOutput, err := ghodssYaml.YAMLToJSON(output)
		if err != nil {
			panic(fmt.Sprintf("runtime err: %s", err))
		}
		checksumArgs = append(checksumArgs, string(jsonOutput))
	}

	if stageVersionChecksum := b.stageVersionChecksum(userStageName); stageVersionChecksum != "" {
		checksumArgs = append(checksumArgs, stageVersionChecksum)
	}

	if len(checksumArgs) != 0 {
		return util.Sha256Hash(checksumArgs...)
	} else {
		return ""
	}
}

func (b *Ansible) stageVersionChecksum(userStageName string) string {
	var stageVersionChecksumArgs []string

	cacheVersionFieldName := "CacheVersion"
	stageCacheVersionFieldName := strings.Join([]string{userStageName, cacheVersionFieldName}, "")

	stageChecksum, ok := b.configFieldValue(stageCacheVersionFieldName).(string)
	if !ok {
		panic(fmt.Sprintf("runtime error: %#v", stageChecksum))
	}

	if stageChecksum != "" {
		stageVersionChecksumArgs = append(stageVersionChecksumArgs, stageChecksum)
	}

	checksum, ok := b.configFieldValue(cacheVersionFieldName).(string)
	if !ok {
		panic(fmt.Sprintf("runtime error: %#v", checksum))
	}

	if checksum != "" {
		stageVersionChecksumArgs = append(stageVersionChecksumArgs, checksum)
	}

	if len(stageVersionChecksumArgs) != 0 {
		return util.Sha256Hash(stageVersionChecksumArgs...)
	} else {
		return ""
	}
}

func (b *Ansible) stageTasks(userStageName string) []*config.AnsibleTask {
	value := b.configFieldValue(userStageName)
	ansibleTasks, ok := value.([]*config.AnsibleTask)
	if !ok {
		panic(fmt.Sprintf("runtime error"))
	}

	return ansibleTasks
}

func (b *Ansible) configFieldValue(fieldName string) interface{} {
	value, err := reflections.GetField(b.config, fieldName)
	if err != nil {
		panic(fmt.Sprintf("runtime error: %s", err))
	}

	return value
}

func (b *Ansible) stageHostWorkDir(userStageName string) (string, error) {
	path := filepath.Join(b.extra.TmpPath, fmt.Sprintf("ansible-workdir-%s", userStageName))

	if err := mkdirP(path); err != nil {
		return "", err
	}

	return path, nil
}
<|endoftext|>


<commit_before>package chrootarchive

import (
	stdtar "archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/idtools"
	rsystem "github.com/opencontainers/runc/libcontainer/system"
	"github.com/pkg/errors"
)

// NewArchiver returns a new Archiver which uses chrootarchive.Untar
func NewArchiver(idMappings *idtools.IDMappings) *archive.Archiver {
	archiver := archive.NewArchiver(idMappings)
	archiver.Untar = Untar
	return archiver
}

// NewArchiverWithChown returns a new Archiver which uses chrootarchive.Untar and the provided ID mapping configuration on both ends
func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools.IDPair, untarIDMappings *idtools.IDMappings) *archive.Archiver {
	archiver := archive.NewArchiverWithChown(tarIDMappings, chownOpts, untarIDMappings)
	archiver.Untar = Untar
	return archiver
}

// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive may be compressed with one of the following algorithms:
//  identity (uncompressed), gzip, bzip2, xz.
func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
	return untarHandler(tarArchive, dest, options, true, dest)
}

// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory
// The root directory is the directory that will be chrooted to.
// `dest` must be a path within `root`, if it is not an error will be returned.
//
// `root` should set to a directory which is not controlled by any potentially
// malicious process.
//
// This should be used to prevent a potential attacker from manipulating `dest`
// such that it would provide access to files outside of `dest` through things
// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however
// sanitizing symlinks in this manner is inherrently racey:
// ref: CVE-2018-15664
func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error {
	return untarHandler(tarArchive, dest, options, true, root)
}

// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive must be an uncompressed stream.
func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
	return untarHandler(tarArchive, dest, options, false, dest)
}

// Handler for teasing out the automatic decompression
func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error {
	if tarArchive == nil {
		return fmt.Errorf("Empty archive")
	}
	if options == nil {
		options = &archive.TarOptions{}
		options.InUserNS = rsystem.RunningInUserNS()
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}

	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
	rootIDs := idMappings.RootPair()

	dest = filepath.Clean(dest)
	if _, err := os.Stat(dest); os.IsNotExist(err) {
		if err := idtools.MkdirAllAndChownNew(dest, 0755, rootIDs); err != nil {
			return err
		}
	}

	r := ioutil.NopCloser(tarArchive)
	if decompress {
		decompressedArchive, err := archive.DecompressStream(tarArchive)
		if err != nil {
			return err
		}
		defer decompressedArchive.Close()
		r = decompressedArchive
	}

	return invokeUnpack(r, dest, options, root)
}

// Tar tars the requested path while chrooted to the specified root.
func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) {
	if options == nil {
		options = &archive.TarOptions{}
	}
	return invokePack(srcPath, options, root)
}

// CopyFileWithTarAndChown returns a function which copies a single file from outside
// of any container into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			contentReader, contentWriter, err := os.Pipe()
			if err != nil {
				return errors.Wrapf(err, "error creating pipe extract data to %q", dest)
			}
			defer contentReader.Close()
			defer contentWriter.Close()
			var hashError error
			var hashWorker sync.WaitGroup
			hashWorker.Add(1)
			go func() {
				t := stdtar.NewReader(contentReader)
				_, err := t.Next()
				if err != nil {
					hashError = err
				}
				if _, err = io.Copy(hasher, t); err != nil && err != io.EOF {
					hashError = err
				}
				hashWorker.Done()
			}()
			if err = originalUntar(io.TeeReader(tarArchive, contentWriter), dest, options); err != nil {
				err = errors.Wrapf(err, "error extracting data to %q while copying", dest)
			}
			hashWorker.Wait()
			if err == nil {
				err = errors.Wrapf(hashError, "error calculating digest of data for %q while copying", dest)
			}
			return err
		}
	}
	return archiver.CopyFileWithTar
}

// CopyWithTarAndChown returns a function which copies a directory tree from outside of
// any container into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func CopyWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			return originalUntar(io.TeeReader(tarArchive, hasher), dest, options)
		}
	}
	return archiver.CopyWithTar
}

// UntarPathAndChown returns a function which extracts an archive in a specified
// location into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func UntarPathAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			return originalUntar(io.TeeReader(tarArchive, hasher), dest, options)
		}
	}
	return archiver.UntarPath
}
<commit_msg>Need to load nsswitch based config outside of container<commit_after>package chrootarchive

import (
	stdtar "archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"sync"

	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/idtools"
	rsystem "github.com/opencontainers/runc/libcontainer/system"
	"github.com/pkg/errors"
)

func init() {
	// initialize nss libraries in Glibc so that the dynamic libraries are loaded in the host
	// environment not in the chroot from untrusted files.
	_, _ = user.Lookup("storage")
	_, _ = net.LookupHost("localhost")
}

// NewArchiver returns a new Archiver which uses chrootarchive.Untar
func NewArchiver(idMappings *idtools.IDMappings) *archive.Archiver {
	archiver := archive.NewArchiver(idMappings)
	archiver.Untar = Untar
	return archiver
}

// NewArchiverWithChown returns a new Archiver which uses chrootarchive.Untar and the provided ID mapping configuration on both ends
func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools.IDPair, untarIDMappings *idtools.IDMappings) *archive.Archiver {
	archiver := archive.NewArchiverWithChown(tarIDMappings, chownOpts, untarIDMappings)
	archiver.Untar = Untar
	return archiver
}

// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive may be compressed with one of the following algorithms:
//  identity (uncompressed), gzip, bzip2, xz.
func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
	return untarHandler(tarArchive, dest, options, true, dest)
}

// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory
// The root directory is the directory that will be chrooted to.
// `dest` must be a path within `root`, if it is not an error will be returned.
//
// `root` should set to a directory which is not controlled by any potentially
// malicious process.
//
// This should be used to prevent a potential attacker from manipulating `dest`
// such that it would provide access to files outside of `dest` through things
// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however
// sanitizing symlinks in this manner is inherrently racey:
// ref: CVE-2018-15664
func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error {
	return untarHandler(tarArchive, dest, options, true, root)
}

// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive must be an uncompressed stream.
func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
	return untarHandler(tarArchive, dest, options, false, dest)
}

// Handler for teasing out the automatic decompression
func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error {
	if tarArchive == nil {
		return fmt.Errorf("Empty archive")
	}
	if options == nil {
		options = &archive.TarOptions{}
		options.InUserNS = rsystem.RunningInUserNS()
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}

	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
	rootIDs := idMappings.RootPair()

	dest = filepath.Clean(dest)
	if _, err := os.Stat(dest); os.IsNotExist(err) {
		if err := idtools.MkdirAllAndChownNew(dest, 0755, rootIDs); err != nil {
			return err
		}
	}

	r := ioutil.NopCloser(tarArchive)
	if decompress {
		decompressedArchive, err := archive.DecompressStream(tarArchive)
		if err != nil {
			return err
		}
		defer decompressedArchive.Close()
		r = decompressedArchive
	}

	return invokeUnpack(r, dest, options, root)
}

// Tar tars the requested path while chrooted to the specified root.
func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) {
	if options == nil {
		options = &archive.TarOptions{}
	}
	return invokePack(srcPath, options, root)
}

// CopyFileWithTarAndChown returns a function which copies a single file from outside
// of any container into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			contentReader, contentWriter, err := os.Pipe()
			if err != nil {
				return errors.Wrapf(err, "error creating pipe extract data to %q", dest)
			}
			defer contentReader.Close()
			defer contentWriter.Close()
			var hashError error
			var hashWorker sync.WaitGroup
			hashWorker.Add(1)
			go func() {
				t := stdtar.NewReader(contentReader)
				_, err := t.Next()
				if err != nil {
					hashError = err
				}
				if _, err = io.Copy(hasher, t); err != nil && err != io.EOF {
					hashError = err
				}
				hashWorker.Done()
			}()
			if err = originalUntar(io.TeeReader(tarArchive, contentWriter), dest, options); err != nil {
				err = errors.Wrapf(err, "error extracting data to %q while copying", dest)
			}
			hashWorker.Wait()
			if err == nil {
				err = errors.Wrapf(hashError, "error calculating digest of data for %q while copying", dest)
			}
			return err
		}
	}
	return archiver.CopyFileWithTar
}

// CopyWithTarAndChown returns a function which copies a directory tree from outside of
// any container into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func CopyWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			return originalUntar(io.TeeReader(tarArchive, hasher), dest, options)
		}
	}
	return archiver.CopyWithTar
}

// UntarPathAndChown returns a function which extracts an archive in a specified
// location into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func UntarPathAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(src, dest string) error {
	untarMappings := idtools.NewIDMappingsFromMaps(uidmap, gidmap)
	archiver := NewArchiverWithChown(nil, chownOpts, untarMappings)
	if hasher != nil {
		originalUntar := archiver.Untar
		archiver.Untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
			return originalUntar(io.TeeReader(tarArchive, hasher), dest, options)
		}
	}
	return archiver.UntarPath
}
<|endoftext|>


<commit_before>package scout

import (
	"context"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/uuid"

	"github.com/datawire/ambassador/v2/pkg/metriton"
	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/connector/userd_auth/authdata"
	"github.com/telepresenceio/telepresence/v2/pkg/filelocation"
)

// Environment variable prefix for additional metadata to be reported
const environmentMetadataPrefix = "TELEPRESENCE_REPORT_"

type bufEntry struct {
	action  string
	entries []Entry
}

// Reporter is a Metriton reported
type Reporter struct {
	index    int
	buffer   chan bufEntry
	reporter *metriton.Reporter
}

// Entry is a key/value association used when reporting
type Entry struct {
	Key   string
	Value interface{}
}

// getInstallIDFromFilesystem returns the telepresence install ID, and also sets the reporter base
// metadata to include any conflicting install IDs written by old versions of the product.
func getInstallIDFromFilesystem(ctx context.Context, reporter *metriton.Reporter) (string, error) {
	type filecacheEntry struct {
		Body string
		Err  error
	}
	filecache := make(map[string]filecacheEntry)
	readFile := func(filename string) (string, error) {
		if _, isCached := filecache[filename]; !isCached {
			bs, err := os.ReadFile(filename)
			filecache[filename] = filecacheEntry{
				Body: string(bs),
				Err:  err,
			}
		}
		return filecache[filename].Body, filecache[filename].Err
	}

	// Do these in order of precedence when there are multiple install IDs.
	var retID string
	allIDs := make(map[string]string)

	if runtime.GOOS != "windows" { // won't find any legacy on windows
		// We'll use this (and justify overriding GOOS=linux) below.
		xdgConfigHome, err := filelocation.UserConfigDir(filelocation.WithGOOS(ctx, "linux"))
		if err == nil {
			// Similarly to Telepresence-1 (below), edgectl always used the XDG filepath, but unlike
			// Telepresence-1 it did obey $XDG_CONFIG_HOME.
			if id, err := readFile(filepath.Join(xdgConfigHome, "edgectl", "id")); err == nil {
				allIDs["edgectl"] = id
				retID = id
			}
		}

		// Telepresence-1 used "$HOME/.config/telepresence/id" always, even on macOS (where ~/.config
		// isn't a thing) or when $XDG_CONFIG_HOME is something different than "$HOME/.config".
		if homeDir, err := filelocation.UserHomeDir(ctx); err == nil {
			if id, err := readFile(filepath.Join(homeDir, ".config", "telepresence", "id")); err == nil {
				allIDs["telepresence-1"] = id
				retID = id
			}
		}

		// Telepresence-2 prior to 2.1.0 did the exact same thing as edgectl, but with
		// "telepresence2" in the path instead of "edgectl".
		if id, err := readFile(filepath.Join(xdgConfigHome, "telepresence2", "id")); err == nil {
			allIDs["telepresence-2<2.1"] = id
			retID = id
		}
	}

	// Current.  Telepresence-2 now uses the most appropriate directory for the platform, and
	// uses "telepresence" instead of "telepresence2".  On GOOS=linux this is probably
	// (depending on how $XDG_CONFIG_HOME is set) the same as the Telepresence 1 location.
	telConfigDir, err := filelocation.AppUserConfigDir(ctx)
	if err != nil {
		return "", err
	}
	idFilename := filepath.Join(telConfigDir, "id")
	if id, err := readFile(idFilename); err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}
	} else {
		allIDs["telepresence-2"] = id
		retID = id
	}

	// OK, now process all of that.

	if len(allIDs) == 0 {
		retID = uuid.New().String()
	}

	if allIDs["telepresence-2"] != retID {
		if err := os.MkdirAll(filepath.Dir(idFilename), 0755); err != nil {
			return "", err
		}
		if err := os.WriteFile(idFilename, []byte(retID), 0644); err != nil {
			return "", err
		}
	}

	reporter.BaseMetadata["new_install"] = len(allIDs) == 0
	for product, id := range allIDs {
		if id != retID {
			reporter.BaseMetadata["install_id_"+product] = id
		}
	}
	return retID, nil
}

// bufferSize is the max number of entries that can be stored in the buffer
// before entries are discarded.
const bufferSize = 40

// NewReporter creates a new initialized Reporter instance that can be used to
// send telepresence reports to Metriton
func NewReporter(ctx context.Context, mode string) *Reporter {
	r := &Reporter{
		reporter: &metriton.Reporter{
			Application: "telepresence2",
			Version:     client.Version(),
			GetInstallID: func(r *metriton.Reporter) (string, error) {
				id, err := getInstallIDFromFilesystem(ctx, r)
				if err != nil {
					id = "00000000-0000-0000-0000-000000000000"
					r.BaseMetadata["new_install"] = true
					r.BaseMetadata["install_id_error"] = err.Error()
				}
				return id, nil
			},
		},
	}
	r.initialize(ctx, mode, runtime.GOOS)
	return r
}

// initialization broken out or constructor for the benefit of testing
func (r *Reporter) initialize(ctx context.Context, mode, goos string) {
	r.buffer = make(chan bufEntry, bufferSize)

	// Fixed (growing) metadata passed with every report
	baseMeta := getOsMetadata(ctx)
	baseMeta["mode"] = mode
	baseMeta["trace_id"] = uuid.New()
	baseMeta["goos"] = goos

	// Discover how Telepresence was installed based on the binary's location
	installMethod, err := client.GetInstallMechanism()
	if err != nil {
		dlog.Errorf(ctx, "scout error getting executable: %s", err)
	}
	baseMeta["install_method"] = installMethod
	for k, v := range getDefaultEnvironmentMetadata() {
		baseMeta[k] = v
	}
	r.reporter.BaseMetadata = baseMeta
}

func (r *Reporter) InstallID() string {
	return r.reporter.InstallID()
}

const setMetadatumAction = "__set_metadatum__"

// SetMetadatum associates the given key with the given value in the metadata
// of this instance.
func (r *Reporter) SetMetadatum(ctx context.Context, key string, value interface{}) {
	r.Report(ctx, setMetadatumAction, Entry{Key: key, Value: value})
}

// Start starts the instance in a goroutine
func (r *Reporter) Start(ctx context.Context) {
	go func() {
		if err := r.Run(ctx); err != nil {
			dlog.Error(ctx, err)
		}
	}()
}

// Run ensures that all reports on the send queue are sent to the endpoint
func (r *Reporter) Run(ctx context.Context) error {
	go func() {
		// Close buffer and let it drain when ctx is done.
		<-ctx.Done()
		close(r.buffer)
	}()

	hc := dcontext.HardContext(ctx)
	for be := range r.buffer {
		if be.action == setMetadatumAction {
			entry := be.entries[0]
			if entry.Value == "" {
				delete(r.reporter.BaseMetadata, entry.Key)
			} else {
				r.reporter.BaseMetadata[entry.Key] = entry.Value
			}
		} else {
			r.doReport(hc, &be)
		}
	}
	return nil
}

// Report constructs and buffers a report on the send queue. It includes the fixed (growing)
// set of metadata in the Reporter structure and the entries passed as arguments to this
// call. It also includes and increments the index, which can be used to
// determine the correct order of reported events for this installation
// attempt (correlated by the trace_id set at the start).
func (r *Reporter) Report(ctx context.Context, action string, entries ...Entry) {
	select {
	case r.buffer <- bufEntry{action, entries}:
	default:
		dlog.Infof(ctx, "scout report %q discarded. Output buffer is full (or closed)", action)
	}
}

func (r *Reporter) doReport(ctx context.Context, be *bufEntry) {
	r.index++
	metadata := make(map[string]interface{}, 4+len(be.entries))
	metadata["action"] = be.action
	metadata["index"] = r.index
	userInfo, err := authdata.LoadUserInfoFromUserCache(ctx)
	if err == nil && userInfo.Id != "" {
		metadata["user_id"] = userInfo.Id
		metadata["account_id"] = userInfo.AccountId
	}
	for _, metaItem := range be.entries {
		metadata[metaItem.Key] = metaItem.Value
	}

	_, err = r.reporter.Report(ctx, metadata)
	if err != nil && ctx.Err() == nil {
		dlog.Infof(ctx, "scout report %q failed: %v", be.action, err)
	}
}

// Returns a metadata map containing all the additional environment variables to be reported
func getDefaultEnvironmentMetadata() map[string]string {
	metadata := map[string]string{}
	for _, e := range os.Environ() {
		pair := strings.SplitN(e, "=", 2)
		if strings.HasPrefix(pair[0], environmentMetadataPrefix) {
			key := strings.ToLower(strings.TrimPrefix(pair[0], environmentMetadataPrefix))
			metadata[key] = pair[1]
		}
	}
	return metadata
}
<commit_msg>Never close the Report buffer<commit_after>package scout

import (
	"context"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/uuid"

	"github.com/datawire/ambassador/v2/pkg/metriton"
	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/connector/userd_auth/authdata"
	"github.com/telepresenceio/telepresence/v2/pkg/filelocation"
)

// Environment variable prefix for additional metadata to be reported
const environmentMetadataPrefix = "TELEPRESENCE_REPORT_"

type bufEntry struct {
	action  string
	entries []Entry
}

// Reporter is a Metriton reported
type Reporter struct {
	index    int
	buffer   chan bufEntry
	reporter *metriton.Reporter
}

// Entry is a key/value association used when reporting
type Entry struct {
	Key   string
	Value interface{}
}

// getInstallIDFromFilesystem returns the telepresence install ID, and also sets the reporter base
// metadata to include any conflicting install IDs written by old versions of the product.
func getInstallIDFromFilesystem(ctx context.Context, reporter *metriton.Reporter) (string, error) {
	type filecacheEntry struct {
		Body string
		Err  error
	}
	filecache := make(map[string]filecacheEntry)
	readFile := func(filename string) (string, error) {
		if _, isCached := filecache[filename]; !isCached {
			bs, err := os.ReadFile(filename)
			filecache[filename] = filecacheEntry{
				Body: string(bs),
				Err:  err,
			}
		}
		return filecache[filename].Body, filecache[filename].Err
	}

	// Do these in order of precedence when there are multiple install IDs.
	var retID string
	allIDs := make(map[string]string)

	if runtime.GOOS != "windows" { // won't find any legacy on windows
		// We'll use this (and justify overriding GOOS=linux) below.
		xdgConfigHome, err := filelocation.UserConfigDir(filelocation.WithGOOS(ctx, "linux"))
		if err == nil {
			// Similarly to Telepresence-1 (below), edgectl always used the XDG filepath, but unlike
			// Telepresence-1 it did obey $XDG_CONFIG_HOME.
			if id, err := readFile(filepath.Join(xdgConfigHome, "edgectl", "id")); err == nil {
				allIDs["edgectl"] = id
				retID = id
			}
		}

		// Telepresence-1 used "$HOME/.config/telepresence/id" always, even on macOS (where ~/.config
		// isn't a thing) or when $XDG_CONFIG_HOME is something different than "$HOME/.config".
		if homeDir, err := filelocation.UserHomeDir(ctx); err == nil {
			if id, err := readFile(filepath.Join(homeDir, ".config", "telepresence", "id")); err == nil {
				allIDs["telepresence-1"] = id
				retID = id
			}
		}

		// Telepresence-2 prior to 2.1.0 did the exact same thing as edgectl, but with
		// "telepresence2" in the path instead of "edgectl".
		if id, err := readFile(filepath.Join(xdgConfigHome, "telepresence2", "id")); err == nil {
			allIDs["telepresence-2<2.1"] = id
			retID = id
		}
	}

	// Current.  Telepresence-2 now uses the most appropriate directory for the platform, and
	// uses "telepresence" instead of "telepresence2".  On GOOS=linux this is probably
	// (depending on how $XDG_CONFIG_HOME is set) the same as the Telepresence 1 location.
	telConfigDir, err := filelocation.AppUserConfigDir(ctx)
	if err != nil {
		return "", err
	}
	idFilename := filepath.Join(telConfigDir, "id")
	if id, err := readFile(idFilename); err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}
	} else {
		allIDs["telepresence-2"] = id
		retID = id
	}

	// OK, now process all of that.

	if len(allIDs) == 0 {
		retID = uuid.New().String()
	}

	if allIDs["telepresence-2"] != retID {
		if err := os.MkdirAll(filepath.Dir(idFilename), 0755); err != nil {
			return "", err
		}
		if err := os.WriteFile(idFilename, []byte(retID), 0644); err != nil {
			return "", err
		}
	}

	reporter.BaseMetadata["new_install"] = len(allIDs) == 0
	for product, id := range allIDs {
		if id != retID {
			reporter.BaseMetadata["install_id_"+product] = id
		}
	}
	return retID, nil
}

// bufferSize is the max number of entries that can be stored in the buffer
// before entries are discarded.
const bufferSize = 40

// NewReporter creates a new initialized Reporter instance that can be used to
// send telepresence reports to Metriton
func NewReporter(ctx context.Context, mode string) *Reporter {
	r := &Reporter{
		reporter: &metriton.Reporter{
			Application: "telepresence2",
			Version:     client.Version(),
			GetInstallID: func(r *metriton.Reporter) (string, error) {
				id, err := getInstallIDFromFilesystem(ctx, r)
				if err != nil {
					id = "00000000-0000-0000-0000-000000000000"
					r.BaseMetadata["new_install"] = true
					r.BaseMetadata["install_id_error"] = err.Error()
				}
				return id, nil
			},
		},
	}
	r.initialize(ctx, mode, runtime.GOOS)
	return r
}

// initialization broken out or constructor for the benefit of testing
func (r *Reporter) initialize(ctx context.Context, mode, goos string) {
	r.buffer = make(chan bufEntry, bufferSize)

	// Fixed (growing) metadata passed with every report
	baseMeta := getOsMetadata(ctx)
	baseMeta["mode"] = mode
	baseMeta["trace_id"] = uuid.New()
	baseMeta["goos"] = goos

	// Discover how Telepresence was installed based on the binary's location
	installMethod, err := client.GetInstallMechanism()
	if err != nil {
		dlog.Errorf(ctx, "scout error getting executable: %s", err)
	}
	baseMeta["install_method"] = installMethod
	for k, v := range getDefaultEnvironmentMetadata() {
		baseMeta[k] = v
	}
	r.reporter.BaseMetadata = baseMeta
}

func (r *Reporter) InstallID() string {
	return r.reporter.InstallID()
}

const setMetadatumAction = "__set_metadatum__"

// SetMetadatum associates the given key with the given value in the metadata
// of this instance.
func (r *Reporter) SetMetadatum(ctx context.Context, key string, value interface{}) {
	r.Report(ctx, setMetadatumAction, Entry{Key: key, Value: value})
}

// Start starts the instance in a goroutine
func (r *Reporter) Start(ctx context.Context) {
	go func() {
		if err := r.Run(ctx); err != nil {
			dlog.Error(ctx, err)
		}
	}()
}

// Run ensures that all reports on the send queue are sent to the endpoint
func (r *Reporter) Run(ctx context.Context) error {
	go func() {
		// Close buffer and let it drain when ctx is done.
		<-ctx.Done()

		// Send a zeroed bufEntry here instead of closing the buffer so that
		// any stray Reports that arrive after the context is cancelled aren't
		// sent on a closed channel
		select {
		case r.buffer <- bufEntry{}:
		default:
		}
	}()

	hc := dcontext.HardContext(ctx)
	for be := range r.buffer {
		switch be.action {
		case "":
			break
		case setMetadatumAction:
			entry := be.entries[0]
			if entry.Value == "" {
				delete(r.reporter.BaseMetadata, entry.Key)
			} else {
				r.reporter.BaseMetadata[entry.Key] = entry.Value
			}
		default:
			r.doReport(hc, &be)
		}
	}
	return nil
}

// Report constructs and buffers a report on the send queue. It includes the fixed (growing)
// set of metadata in the Reporter structure and the entries passed as arguments to this
// call. It also includes and increments the index, which can be used to
// determine the correct order of reported events for this installation
// attempt (correlated by the trace_id set at the start).
func (r *Reporter) Report(ctx context.Context, action string, entries ...Entry) {
	select {
	case r.buffer <- bufEntry{action, entries}:
	default:
		dlog.Infof(ctx, "scout report %q discarded. Output buffer is full (or closed)", action)
	}
}

func (r *Reporter) doReport(ctx context.Context, be *bufEntry) {
	r.index++
	metadata := make(map[string]interface{}, 4+len(be.entries))
	metadata["action"] = be.action
	metadata["index"] = r.index
	userInfo, err := authdata.LoadUserInfoFromUserCache(ctx)
	if err == nil && userInfo.Id != "" {
		metadata["user_id"] = userInfo.Id
		metadata["account_id"] = userInfo.AccountId
	}
	for _, metaItem := range be.entries {
		metadata[metaItem.Key] = metaItem.Value
	}

	_, err = r.reporter.Report(ctx, metadata)
	if err != nil && ctx.Err() == nil {
		dlog.Infof(ctx, "scout report %q failed: %v", be.action, err)
	}
}

// Returns a metadata map containing all the additional environment variables to be reported
func getDefaultEnvironmentMetadata() map[string]string {
	metadata := map[string]string{}
	for _, e := range os.Environ() {
		pair := strings.SplitN(e, "=", 2)
		if strings.HasPrefix(pair[0], environmentMetadataPrefix) {
			key := strings.ToLower(strings.TrimPrefix(pair[0], environmentMetadataPrefix))
			metadata[key] = pair[1]
		}
	}
	return metadata
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"github.com/golang/glog"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/version"
	"github.com/spf13/pflag"
	"io/ioutil"
	api "k8s.io/client-go/pkg/api/v1"
	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/controller"
	"k8s.io/ingress/core/pkg/ingress/defaults"
	"net/http"
	"os/exec"
)

// HAProxyController has internal data of a HAProxyController instance
type HAProxyController struct {
	controller     *controller.GenericController
	configMap      *api.ConfigMap
	storeLister    *ingress.StoreLister
	command        string
	reloadStrategy *string
	configFile     string
	template       *template
	currentConfig  *types.ControllerConfig
	reloadRequired bool
}

// NewHAProxyController constructor
func NewHAProxyController() *HAProxyController {
	return &HAProxyController{}
}

// Info provides controller name and repository infos
func (haproxy *HAProxyController) Info() *ingress.BackendInfo {
	return &ingress.BackendInfo{
		Name:       "HAProxy",
		Release:    version.RELEASE,
		Build:      version.COMMIT,
		Repository: version.REPO,
	}
}

// Start starts the controller
func (haproxy *HAProxyController) Start() {
	haproxy.controller = controller.NewIngressController(haproxy)
	haproxy.controller.Start()
}

// Stop shutdown the controller process
func (haproxy *HAProxyController) Stop() error {
	err := haproxy.controller.Stop()
	return err
}

// Name provides the complete name of the controller
func (haproxy *HAProxyController) Name() string {
	return "HAProxy Ingress Controller"
}

// DefaultIngressClass returns the ingress class name
func (haproxy *HAProxyController) DefaultIngressClass() string {
	return "haproxy"
}

// Check health check implementation
func (haproxy *HAProxyController) Check(_ *http.Request) error {
	return nil
}

// SetListers give access to the store listers
func (haproxy *HAProxyController) SetListers(lister ingress.StoreLister) {
	haproxy.storeLister = &lister
}

// ConfigureFlags allow to configure more flags before the parsing of
// command line arguments
func (haproxy *HAProxyController) ConfigureFlags(flags *pflag.FlagSet) {
	haproxy.reloadStrategy = flags.String("reload-strategy", "native",
		`Name of the reload strategy. Options are: native (default) or multibinder`)
	ingressClass := flags.Lookup("ingress-class")
	if ingressClass != nil {
		ingressClass.Value.Set("haproxy")
		ingressClass.DefValue = "haproxy"
	}
}

// OverrideFlags allows controller to override command line parameter flags
func (haproxy *HAProxyController) OverrideFlags(flags *pflag.FlagSet) {
	if *haproxy.reloadStrategy == "native" {
		haproxy.configFile = "/etc/haproxy/haproxy.cfg"
		haproxy.template = newTemplate("haproxy.tmpl", "/etc/haproxy/template/haproxy.tmpl")
	} else if *haproxy.reloadStrategy == "multibinder" {
		haproxy.configFile = "/etc/haproxy/haproxy.cfg.erb"
		haproxy.template = newTemplate("haproxy.cfg.erb.tmpl", "/etc/haproxy/haproxy.cfg.erb.tmpl")
	} else {
		glog.Fatalf("Unsupported reload strategy: %v", *haproxy.reloadStrategy)
	}
	haproxy.command = "/haproxy-reload.sh"
}

// SetConfig receives the ConfigMap the user has configured
func (haproxy *HAProxyController) SetConfig(configMap *api.ConfigMap) {
	haproxy.configMap = configMap
}

// BackendDefaults defines default values to the ingress core
func (haproxy *HAProxyController) BackendDefaults() defaults.Backend {
	return newHAProxyConfig(haproxy).Backend
}

// OnUpdate regenerate the configuration file of the backend
func (haproxy *HAProxyController) OnUpdate(cfg ingress.Configuration) ([]byte, error) {
	updatedConfig := newControllerConfig(&cfg, haproxy)

	haproxy.reloadRequired = reconfigureBackends(haproxy.currentConfig, updatedConfig)
	haproxy.currentConfig = updatedConfig

	data, err := haproxy.template.execute(updatedConfig)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// Reload the backend if the configuration has changed
func (haproxy *HAProxyController) Reload(data []byte) ([]byte, bool, error) {
	/*
		if !haproxy.configChanged(data) {
			return nil, false, nil
		}
	*/
	// TODO missing HAProxy validation before overwrite and try to reload
	err := ioutil.WriteFile(haproxy.configFile, data, 0644)
	if err != nil {
		return nil, false, err
	}

	if !haproxy.reloadRequired {
		glog.Infoln("reload not required")
		return nil, false, nil
	}
	glog.Infoln("reload is required")

	out, err := haproxy.reloadHaproxy()
	if err == nil {
		haproxy.reloadRequired = false
	}
	if len(out) > 0 {
		glog.Infof("HAProxy output:\n%v", string(out))
	}
	return out, true, err
}

/*
func (haproxy *HAProxyController) configChanged(data []byte) bool {
	if _, err := os.Stat(haproxy.configFile); os.IsNotExist(err) {
		return true
	}
	cfg, err := ioutil.ReadFile(haproxy.configFile)
	if err != nil {
		glog.Warningf("error reading haproxy config: %v")
		return false
	}
	return !bytes.Equal(cfg, data)
}
*/

func (haproxy *HAProxyController) reloadHaproxy() ([]byte, error) {
	out, err := exec.Command(haproxy.command, *haproxy.reloadStrategy, haproxy.configFile).CombinedOutput()
	return out, err
}
<commit_msg>remove unused configChanged function<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"github.com/golang/glog"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/version"
	"github.com/spf13/pflag"
	"io/ioutil"
	api "k8s.io/client-go/pkg/api/v1"
	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/controller"
	"k8s.io/ingress/core/pkg/ingress/defaults"
	"net/http"
	"os/exec"
)

// HAProxyController has internal data of a HAProxyController instance
type HAProxyController struct {
	controller     *controller.GenericController
	configMap      *api.ConfigMap
	storeLister    *ingress.StoreLister
	command        string
	reloadStrategy *string
	configFile     string
	template       *template
	currentConfig  *types.ControllerConfig
	reloadRequired bool
}

// NewHAProxyController constructor
func NewHAProxyController() *HAProxyController {
	return &HAProxyController{}
}

// Info provides controller name and repository infos
func (haproxy *HAProxyController) Info() *ingress.BackendInfo {
	return &ingress.BackendInfo{
		Name:       "HAProxy",
		Release:    version.RELEASE,
		Build:      version.COMMIT,
		Repository: version.REPO,
	}
}

// Start starts the controller
func (haproxy *HAProxyController) Start() {
	haproxy.controller = controller.NewIngressController(haproxy)
	haproxy.controller.Start()
}

// Stop shutdown the controller process
func (haproxy *HAProxyController) Stop() error {
	err := haproxy.controller.Stop()
	return err
}

// Name provides the complete name of the controller
func (haproxy *HAProxyController) Name() string {
	return "HAProxy Ingress Controller"
}

// DefaultIngressClass returns the ingress class name
func (haproxy *HAProxyController) DefaultIngressClass() string {
	return "haproxy"
}

// Check health check implementation
func (haproxy *HAProxyController) Check(_ *http.Request) error {
	return nil
}

// SetListers give access to the store listers
func (haproxy *HAProxyController) SetListers(lister ingress.StoreLister) {
	haproxy.storeLister = &lister
}

// ConfigureFlags allow to configure more flags before the parsing of
// command line arguments
func (haproxy *HAProxyController) ConfigureFlags(flags *pflag.FlagSet) {
	haproxy.reloadStrategy = flags.String("reload-strategy", "native",
		`Name of the reload strategy. Options are: native (default) or multibinder`)
	ingressClass := flags.Lookup("ingress-class")
	if ingressClass != nil {
		ingressClass.Value.Set("haproxy")
		ingressClass.DefValue = "haproxy"
	}
}

// OverrideFlags allows controller to override command line parameter flags
func (haproxy *HAProxyController) OverrideFlags(flags *pflag.FlagSet) {
	if *haproxy.reloadStrategy == "native" {
		haproxy.configFile = "/etc/haproxy/haproxy.cfg"
		haproxy.template = newTemplate("haproxy.tmpl", "/etc/haproxy/template/haproxy.tmpl")
	} else if *haproxy.reloadStrategy == "multibinder" {
		haproxy.configFile = "/etc/haproxy/haproxy.cfg.erb"
		haproxy.template = newTemplate("haproxy.cfg.erb.tmpl", "/etc/haproxy/haproxy.cfg.erb.tmpl")
	} else {
		glog.Fatalf("Unsupported reload strategy: %v", *haproxy.reloadStrategy)
	}
	haproxy.command = "/haproxy-reload.sh"
}

// SetConfig receives the ConfigMap the user has configured
func (haproxy *HAProxyController) SetConfig(configMap *api.ConfigMap) {
	haproxy.configMap = configMap
}

// BackendDefaults defines default values to the ingress core
func (haproxy *HAProxyController) BackendDefaults() defaults.Backend {
	return newHAProxyConfig(haproxy).Backend
}

// OnUpdate regenerate the configuration file of the backend
func (haproxy *HAProxyController) OnUpdate(cfg ingress.Configuration) ([]byte, error) {
	updatedConfig := newControllerConfig(&cfg, haproxy)

	haproxy.reloadRequired = reconfigureBackends(haproxy.currentConfig, updatedConfig)
	haproxy.currentConfig = updatedConfig

	data, err := haproxy.template.execute(updatedConfig)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// Reload the backend if the configuration has changed
func (haproxy *HAProxyController) Reload(data []byte) ([]byte, bool, error) {
	// TODO missing HAProxy validation before overwrite and try to reload
	err := ioutil.WriteFile(haproxy.configFile, data, 0644)
	if err != nil {
		return nil, false, err
	}

	if !haproxy.reloadRequired {
		glog.Infoln("reload not required")
		return nil, false, nil
	}
	glog.Infoln("reload is required")

	out, err := haproxy.reloadHaproxy()
	if err == nil {
		haproxy.reloadRequired = false
	}
	if len(out) > 0 {
		glog.Infof("HAProxy output:\n%v", string(out))
	}
	return out, true, err
}

func (haproxy *HAProxyController) reloadHaproxy() ([]byte, error) {
	out, err := exec.Command(haproxy.command, *haproxy.reloadStrategy, haproxy.configFile).CombinedOutput()
	return out, err
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"

	"github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	"github.com/skygeario/skygear-server/pkg/core/auth/session"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/model"
)

// AuthnMiddleware populate auth context information
type AuthnMiddleware struct {
	AuthContextSetter auth.ContextSetter `dependency:"AuthContextSetter"`
	SessionProvider   session.Provider   `dependency:"SessionProvider"`
	SessionWriter     session.Writer     `dependency:"SessionWriter"`
	AuthInfoStore     authinfo.Store     `dependency:"AuthInfoStore"`
	TxContext         db.TxContext       `dependency:"TxContext"`
}

// AuthnMiddlewareFactory creates AuthnMiddleware per request.
type AuthnMiddlewareFactory struct{}

// NewInjectableMiddleware implements InjectableMiddlewareFactory.
func (f AuthnMiddlewareFactory) NewInjectableMiddleware() InjectableMiddleware {
	return &AuthnMiddleware{}
}

// Handle implements InjectableMiddleware.
func (m *AuthnMiddleware) Handle(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var err error
		defer func() {
			if err != nil {
				// clear session if error occurred
				m.SessionWriter.ClearSession(w)
			}

			next.ServeHTTP(w, r)
		}()

		tenantConfig := config.GetTenantConfig(r)

		key := model.CheckAccessKey(tenantConfig, model.GetAPIKey(r))
		m.AuthContextSetter.SetAccessKey(key)

		accessToken, transport, err := model.GetAccessToken(r)
		if err != nil {
			return
		}

		// No access token found. Simply proceed.
		if accessToken == "" {
			return
		}

		if err = m.TxContext.BeginTx(); err != nil {
			return
		}
		defer m.TxContext.RollbackTx()

		s, err := m.SessionProvider.GetByToken(accessToken, auth.SessionTokenKindAccessToken)
		if err != nil {
			return
		}

		if tenantConfig.UserConfig.Clients[s.ClientID].SessionTransport != transport {
			err = session.ErrSessionNotFound
			return
		}

		authInfo := authinfo.AuthInfo{}
		err = m.AuthInfoStore.GetAuth(s.UserID, &authInfo)
		if err != nil {
			return
		}

		// in case valid session is used, infer access key from session
		if !key.IsMasterKey() {
			key = model.NewAccessKey(s.ClientID)
			m.AuthContextSetter.SetAccessKey(key)
		}

		// should not use new session data in context
		sessionCopy := *s
		err = m.SessionProvider.Access(&sessionCopy)
		if err != nil {
			return
		}

		m.AuthContextSetter.SetSession(s)
		m.AuthContextSetter.SetAuthInfo(&authInfo)
	})
}
<commit_msg>Make session error handling more fine-grained<commit_after>package middleware

import (
	"net/http"

	"github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	"github.com/skygeario/skygear-server/pkg/core/auth/session"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/model"
	"github.com/skygeario/skygear-server/pkg/core/skydb"
)

// AuthnMiddleware populate auth context information
type AuthnMiddleware struct {
	AuthContextSetter auth.ContextSetter `dependency:"AuthContextSetter"`
	SessionProvider   session.Provider   `dependency:"SessionProvider"`
	SessionWriter     session.Writer     `dependency:"SessionWriter"`
	AuthInfoStore     authinfo.Store     `dependency:"AuthInfoStore"`
	TxContext         db.TxContext       `dependency:"TxContext"`
}

// AuthnMiddlewareFactory creates AuthnMiddleware per request.
type AuthnMiddlewareFactory struct{}

// NewInjectableMiddleware implements InjectableMiddlewareFactory.
func (f AuthnMiddlewareFactory) NewInjectableMiddleware() InjectableMiddleware {
	return &AuthnMiddleware{}
}

// Handle implements InjectableMiddleware.
func (m *AuthnMiddleware) Handle(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var err error
		defer func() {
			if err == session.ErrSessionNotFound {
				// clear session if session is not found
				m.SessionWriter.ClearSession(w)
			} else if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}

			next.ServeHTTP(w, r)
		}()

		tenantConfig := config.GetTenantConfig(r)

		key := model.CheckAccessKey(tenantConfig, model.GetAPIKey(r))
		m.AuthContextSetter.SetAccessKey(key)

		accessToken, transport, err := model.GetAccessToken(r)
		if err != nil {
			return
		}

		// No access token found. Simply proceed.
		if accessToken == "" {
			return
		}

		if err = m.TxContext.BeginTx(); err != nil {
			return
		}
		defer m.TxContext.RollbackTx()

		s, err := m.SessionProvider.GetByToken(accessToken, auth.SessionTokenKindAccessToken)
		if err != nil {
			return
		}

		if tenantConfig.UserConfig.Clients[s.ClientID].SessionTransport != transport {
			err = session.ErrSessionNotFound
			return
		}

		authInfo := authinfo.AuthInfo{}
		err = m.AuthInfoStore.GetAuth(s.UserID, &authInfo)
		if err != nil {
			if err == skydb.ErrUserNotFound {
				err = session.ErrSessionNotFound
			}
			return
		}

		// in case valid session is used, infer access key from session
		if !key.IsMasterKey() {
			key = model.NewAccessKey(s.ClientID)
			m.AuthContextSetter.SetAccessKey(key)
		}

		// should not use new session data in context
		sessionCopy := *s
		err = m.SessionProvider.Access(&sessionCopy)
		if err != nil {
			return
		}

		m.AuthContextSetter.SetSession(s)
		m.AuthContextSetter.SetAuthInfo(&authInfo)
	})
}
<|endoftext|>