<commit_before>package remote

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// Configuration for Terraform remote state
type RemoteState struct {
	Backend string                 `hcl:"backend"`
	Config  map[string]interface{} `hcl:"config"`
}

func (remoteState *RemoteState) String() string {
	return fmt.Sprintf("RemoteState{Backend = %v, Config = %v}", remoteState.Backend, remoteState.Config)
}

type RemoteStateInitializer interface {
	// Return true if remote state needs to be initialized
	NeedsInitialization(config map[string]interface{}, terragruntOptions *options.TerragruntOptions) (bool, error)

	// Initialize the remote state
	Initialize(config map[string]interface{}, terragruntOptions *options.TerragruntOptions) error
}

// TODO: initialization actions for other remote state backends can be added here
var remoteStateInitializers = map[string]RemoteStateInitializer{
	"s3": S3Initializer{},
}

// Fill in any default configuration for remote state
func (remoteState *RemoteState) FillDefaults() {
	// Nothing to do
}

// Validate that the remote state is configured correctly
func (remoteState *RemoteState) Validate() error {
	if remoteState.Backend == "" {
		return errors.WithStackTrace(RemoteBackendMissing)
	}

	return nil
}

// Perform any actions necessary to initialize the remote state before it's used for storage. For example, if you're
// using S3 for remote state storage, this may create the S3 bucket if it doesn't exist already.
func (remoteState *RemoteState) Initialize(terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Initializing remote state for the %s backend", remoteState.Backend)
	initializer, hasInitializer := remoteStateInitializers[remoteState.Backend]
	if hasInitializer {
		return initializer.Initialize(remoteState.Config, terragruntOptions)
	}

	return nil
}

// Returns true if remote state needs to be configured. This will be the case when:
//
// 1. Remote state has not already been configured
// 2. Remote state has been configured, but with a different configuration
// 3. The remote state initializer for this backend type, if there is one, says initialization is necessary
func (remoteState *RemoteState) NeedsInit(terragruntOptions *options.TerragruntOptions) (bool, error) {
	state, err := ParseTerraformStateFileFromLocation(terragruntOptions.WorkingDir)
	if err != nil {
		return false, err
	}

	// Remote state not configured
	if state == nil {
		return true, nil
	}

	// Remote state configured, but with a different configuration
	if remoteState.differsFrom(state.Backend, terragruntOptions) {
		return true, nil
	}

	// Remote state initializer says initialization is necessary
	initializer, hasInitializer := remoteStateInitializers[remoteState.Backend]
	if hasInitializer {
		return initializer.NeedsInitialization(remoteState.Config, terragruntOptions)
	}

	return false, nil
}

// Returns true if this remote state is different than the given remote state that is currently being used by terraform.
func (remoteState *RemoteState) differsFrom(existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) bool {
	if existingBackend.Type != remoteState.Backend {
		terragruntOptions.Logger.Printf("Backend type has changed from %s to %s", existingBackend.Type, remoteState.Backend)
		return true
	}

	// Terraform's `backend` configuration uses a boolean for the `encrypt` parameter. However, perhaps for backwards compatibility reasons,
	// Terraform stores that parameter as a string in the `terraform.tfstate` file. Therefore, we have to convert it accordingly, or `DeepEqual`
	// will fail.
	if util.KindOf(existingBackend.Config["encrypt"]) == reflect.String && util.KindOf(remoteState.Config["encrypt"]) == reflect.Bool {
		// If encrypt in remoteState is a bool and a string in existingBackend, DeepEqual will consider the maps to be different.
		// So we convert the value from string to bool to make them equivalent.
		if value, err := strconv.ParseBool(existingBackend.Config["encrypt"].(string)); err == nil {
			existingBackend.Config["encrypt"] = value
		} else {
			terragruntOptions.Logger.Printf("Remote state configuration encrypt contains invalid value %v, should be boolean.", existingBackend.Config["encrypt"])
		}
	}

	if !reflect.DeepEqual(existingBackend.Config, remoteState.Config) {
		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, remoteState.Config)
		return true
	}

	terragruntOptions.Logger.Printf("Backend %s has not changed.", existingBackend.Type)
	return false
}

// Convert the RemoteState config into the format used by the terraform init command
func (remoteState RemoteState) ToTerraformInitArgs() []string {
	backendConfigArgs := []string{}
	for key, value := range remoteState.Config {
		arg := fmt.Sprintf("-backend-config=%s=%v", key, value)
		backendConfigArgs = append(backendConfigArgs, arg)
	}

	return backendConfigArgs
}

var RemoteBackendMissing = fmt.Errorf("The remote_state.backend field cannot be empty")
<commit_msg>Restore IsRemote() check when comparing backend<commit_after>package remote

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// Configuration for Terraform remote state
type RemoteState struct {
	Backend string                 `hcl:"backend"`
	Config  map[string]interface{} `hcl:"config"`
}

func (remoteState *RemoteState) String() string {
	return fmt.Sprintf("RemoteState{Backend = %v, Config = %v}", remoteState.Backend, remoteState.Config)
}

type RemoteStateInitializer interface {
	// Return true if remote state needs to be initialized
	NeedsInitialization(config map[string]interface{}, terragruntOptions *options.TerragruntOptions) (bool, error)

	// Initialize the remote state
	Initialize(config map[string]interface{}, terragruntOptions *options.TerragruntOptions) error
}

// TODO: initialization actions for other remote state backends can be added here
var remoteStateInitializers = map[string]RemoteStateInitializer{
	"s3": S3Initializer{},
}

// Fill in any default configuration for remote state
func (remoteState *RemoteState) FillDefaults() {
	// Nothing to do
}

// Validate that the remote state is configured correctly
func (remoteState *RemoteState) Validate() error {
	if remoteState.Backend == "" {
		return errors.WithStackTrace(RemoteBackendMissing)
	}

	return nil
}

// Perform any actions necessary to initialize the remote state before it's used for storage. For example, if you're
// using S3 for remote state storage, this may create the S3 bucket if it doesn't exist already.
func (remoteState *RemoteState) Initialize(terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Initializing remote state for the %s backend", remoteState.Backend)
	initializer, hasInitializer := remoteStateInitializers[remoteState.Backend]
	if hasInitializer {
		return initializer.Initialize(remoteState.Config, terragruntOptions)
	}

	return nil
}

// Returns true if remote state needs to be configured. This will be the case when:
//
// 1. Remote state has not already been configured
// 2. Remote state has been configured, but with a different configuration
// 3. The remote state initializer for this backend type, if there is one, says initialization is necessary
func (remoteState *RemoteState) NeedsInit(terragruntOptions *options.TerragruntOptions) (bool, error) {
	state, err := ParseTerraformStateFileFromLocation(terragruntOptions.WorkingDir)
	if err != nil {
		return false, err
	}

	// Remote state not configured
	if state == nil {
		return true, nil
	}

	// Remote state configured, but with a different configuration
	if state.IsRemote() && remoteState.differsFrom(state.Backend, terragruntOptions) {
		return true, nil
	}

	// Remote state initializer says initialization is necessary
	initializer, hasInitializer := remoteStateInitializers[remoteState.Backend]
	if hasInitializer {
		return initializer.NeedsInitialization(remoteState.Config, terragruntOptions)
	}

	return false, nil
}

// Returns true if this remote state is different than the given remote state that is currently being used by terraform.
func (remoteState *RemoteState) differsFrom(existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) bool {
	if existingBackend.Type != remoteState.Backend {
		terragruntOptions.Logger.Printf("Backend type has changed from %s to %s", existingBackend.Type, remoteState.Backend)
		return true
	}

	// Terraform's `backend` configuration uses a boolean for the `encrypt` parameter. However, perhaps for backwards compatibility reasons,
	// Terraform stores that parameter as a string in the `terraform.tfstate` file. Therefore, we have to convert it accordingly, or `DeepEqual`
	// will fail.
	if util.KindOf(existingBackend.Config["encrypt"]) == reflect.String && util.KindOf(remoteState.Config["encrypt"]) == reflect.Bool {
		// If encrypt in remoteState is a bool and a string in existingBackend, DeepEqual will consider the maps to be different.
		// So we convert the value from string to bool to make them equivalent.
		if value, err := strconv.ParseBool(existingBackend.Config["encrypt"].(string)); err == nil {
			existingBackend.Config["encrypt"] = value
		} else {
			terragruntOptions.Logger.Printf("Remote state configuration encrypt contains invalid value %v, should be boolean.", existingBackend.Config["encrypt"])
		}
	}

	if !reflect.DeepEqual(existingBackend.Config, remoteState.Config) {
		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, remoteState.Config)
		return true
	}

	terragruntOptions.Logger.Printf("Backend %s has not changed.", existingBackend.Type)
	return false
}

// Convert the RemoteState config into the format used by the terraform init command
func (remoteState RemoteState) ToTerraformInitArgs() []string {
	backendConfigArgs := []string{}
	for key, value := range remoteState.Config {
		arg := fmt.Sprintf("-backend-config=%s=%v", key, value)
		backendConfigArgs = append(backendConfigArgs, arg)
	}

	return backendConfigArgs
}

var RemoteBackendMissing = fmt.Errorf("The remote_state.backend field cannot be empty")
<|endoftext|>


<commit_before>package dilithium

import (
	"log"
	"io"
	"os"
	"encoding/binary"
	"strings"
	"strconv"
	)

const recordLengthBytes = 4 // Use 32-bit numbers for writing the length in the journal.
// journal files are named journal-X where X is an integer
const journalFilenamePrefix = "journal-"
// TODO this should be set through some config
const journalDirpath = "journals/"

// Provides a persistent (journaled) job queue.
// Its goal is that each job in the queue is seen by one reader once.
// Its guarantee (in the face of failures) is that each job in the queue
// is seen by at least one reader at least once.

// Readers must eventually call a matching Done after calling Get so that
// garbage can be collected. If jobs are held (in between Get and Done)
// long enough that many other Gets or Pushes have occurred in the meantime,
// performance may suffer.

// It is not threadsafe. The user must provide some form of synchronization on top.
// The recommendation is to have one goroutine that owns it and has jobs passed in
// and out via channels. JobQueue does this..

type FileJobQueuePersister struct {
	name string // Name of the file that backs the queue
	head uint64 // id of the front of the front of the queue (last written)
	tail uint64 // id of the back of the queue (lowest id that has not been Done-ed)
	// window of outstanding elements. window[i] accounts for the element of id
	// (i + tail). true means that it has been doneted.
	window []bool
	readFile *os.File // File handle that we use to read queued jobs
	writeFile *os.File // File handle that we use to write new queued jobs
}

type Job interface {
	Id() uint64
	SetId(uint64)
	Serialize() []byte
	Deserialize([]byte)
}

type journalFile struct {
	basename string
	lastRecord uint64
}

func isJournal(name string) bool {
	return strings.HasPrefix(name, journalFilenamePrefix)
}

func journalNumber(basename string) int {
	// TODO error handle
	num, _ := strconv.Atoi(basename[len(journalFilenamePrefix):])
	return num
}

func (q *FileJobQueuePersister) journalDir() string {
	// TODO at some point should error check to make sure name doesn't have a /
	return journalDirpath + q.name + "/"
}

// Opens and reads journal and returns the highest id of any record in the file
// Also returns an error if there are problems reading the file.
func (q *FileJobQueuePersister) latestRecord(journal journalFile) (uint64, error) {
	// TODO, I don't actually have the exact type of Job that we're using.
	// For now I'm just assuming it's type job (defined in the tests), but we should
	// either make that not an abstract interface and say that your job will simply
	// have an id and a slice of bytes (probably best) or have some way to communicate
	// the exact type to the constructor.
	max := uint64(0)
	file, err := os.Open(q.journalDir() + journal.basename)
	for err != nil {
		job, e := q.nextInFile(file)
		if e == nil {
			if max < job.Id() {
				max = job.Id()
			} else {
				log.Println(job, "followed higher id", max)
			}
		} else {
			err = e
		}
	}
	return max, err
}

func NewFileJobQueuePersister(name string) *FileJobQueuePersister {
	q := new(FileJobQueuePersister)
	q.name = name
	// TODO better erorr handling
	os.Mkdir(journalDirpath, 0777)
	dirname := journalDirpath + name
	os.Mkdir(dirname, 0777)
	dirFile, _ := os.Open(dirname)
	fis, _ := dirFile.Readdir(0)
	journals := make(map[int]journalFile)
	for _, fi := range fis {
		// skip any subdirectories (there shouldn't be any)
		if !fi.IsDir() {
			if isJournal(fi.Name()) {
				journals[journalNumber(fi.Name())] = journalFile{basename: fi.Name()}
			}
		}
	}
		/*writeFile, err := os.Create(q.filename)
		if err == nil {
			// TODO
			readFile, _ := os.Open(q.filename)
			q.writeFile = writeFile
			q.readFile = readFile
		 }
		if err != nil {
			log.Println("Failure opening journal:", err)
			return nil
		} else {
			return q
		}*/
	return q
}

func (q *FileJobQueuePersister) nextInFile(readFrom *os.File) (*job, error) {
	j := &job{}

	// read from file
	lengthBuffer := make([]byte, recordLengthBytes)
	bytesRead, err := q.readFile.Read(lengthBuffer)
	if bytesRead < recordLengthBytes || err != nil {
		if err == io.EOF {
			// Don't pass back EOF
			err = nil
		}
		return nil, err
	}
	recordLength := binary.BigEndian.Uint32(lengthBuffer)
	jobBuffer := make([]byte, recordLength)
	bytesRead, err = q.readFile.Read(jobBuffer)
	if uint32(bytesRead) < recordLength || err != nil {
		return nil, err
	}

	j.Deserialize(jobBuffer)
	return j, nil
}

func (q *FileJobQueuePersister) Get() Job {
	j, _ := q.nextInFile(q.readFile)

	// Extend window to track that this job has not been done-ed.
	q.window = append(q.window, false)

	return j
}

func writeLen(f *os.File, length uint32) bool {
	bytes := make([]byte, recordLengthBytes)
	binary.BigEndian.PutUint32(bytes, length)
	return writeBytes(f, bytes)
}

func writeBytes(f *os.File, bytes []byte) bool {
	written, err := f.Write(bytes)
	return err == nil && written == len(bytes)
}

func (q *FileJobQueuePersister) Push(job Job) bool {
	// Give it the next id
	job.SetId(q.head)
	q.head++

	bytes := job.Serialize()
	success := writeLen(q.writeFile, uint32(len(bytes)))
	if success {
		success := writeBytes(q.writeFile, bytes)
		return success
	}
	return success
}

// Returns the index of job in window, or -1 if out of range
func (q *FileJobQueuePersister) windowIndex(job Job) int {
	ind := int(job.Id() - q.tail)
	if ind < 0 || ind > len(q.window) {
		ind = -1
	}
	return ind
}

func firstFalseIndex(slice []bool) int {
	for i, v := range slice {
		if !v {
			return i
		}
	}
	return len(slice)
}

func (q *FileJobQueuePersister) updateDurablyWritten() {
	// TODO write q.tail to some file.
}

func (q *FileJobQueuePersister) Done(job Job) {
	ind := q.windowIndex(job)
	if ind < 0 {
		// TODO log or return 0 or something
		return
	}
	// Try to slide the window forward
	newTail := firstFalseIndex(q.window)
	if newTail > 0 {
		// Slide window forward.
		q.window = q.window[:newTail]
		q.tail += uint64(newTail)

		q.updateDurablyWritten()
	}
}
<commit_msg>fix how get returns nil<commit_after>package dilithium

import (
	"log"
	"io"
	"os"
	"encoding/binary"
	"strings"
	"strconv"
	)

const recordLengthBytes = 4 // Use 32-bit numbers for writing the length in the journal.
// journal files are named journal-X where X is an integer
const journalFilenamePrefix = "journal-"
// TODO this should be set through some config
const journalDirpath = "journals/"

// Provides a persistent (journaled) job queue.
// Its goal is that each job in the queue is seen by one reader once.
// Its guarantee (in the face of failures) is that each job in the queue
// is seen by at least one reader at least once.

// Readers must eventually call a matching Done after calling Get so that
// garbage can be collected. If jobs are held (in between Get and Done)
// long enough that many other Gets or Pushes have occurred in the meantime,
// performance may suffer.

// It is not threadsafe. The user must provide some form of synchronization on top.
// The recommendation is to have one goroutine that owns it and has jobs passed in
// and out via channels. JobQueue does this..

type FileJobQueuePersister struct {
	name string // Name of the file that backs the queue
	head uint64 // id of the front of the front of the queue (last written)
	tail uint64 // id of the back of the queue (lowest id that has not been Done-ed)
	// window of outstanding elements. window[i] accounts for the element of id
	// (i + tail). true means that it has been doneted.
	window []bool
	journals []journalFile
	readFile *os.File // File handle that we use to read queued jobs
	writeFile *os.File // File handle that we use to write new queued jobs
}

type Job interface {
	Id() uint64
	SetId(uint64)
	Serialize() []byte
	Deserialize([]byte)
}

// Represents one journal file that we are reading from and/or
// writing to.
type journalFile struct {
	// Filename (not including directories) of the file
	basename string
	// The id of the highest record contained. Used to know when we can delete.
	lastRecord uint64
}

func isJournal(name string) bool {
	return strings.HasPrefix(name, journalFilenamePrefix)
}

func journalNumber(basename string) int {
	// TODO error handle
	num, _ := strconv.Atoi(basename[len(journalFilenamePrefix):])
	return num
}

func (q *FileJobQueuePersister) journalDir() string {
	// TODO at some point should error check to make sure name doesn't have a /
	return journalDirpath + q.name + "/"
}

// Opens and reads journal and returns the highest id of any record in the file
// Also returns an error if there are problems reading the file.
func (q *FileJobQueuePersister) latestRecord(journal journalFile) (uint64, error) {
	// TODO, I don't actually have the exact type of Job that we're using.
	// For now I'm just assuming it's type job (defined in the tests), but we should
	// either make that not an abstract interface and say that your job will simply
	// have an id and a slice of bytes (probably best) or have some way to communicate
	// the exact type to the constructor.
	max := uint64(0)
	file, err := os.Open(q.journalDir() + journal.basename)
	for err != nil {
		job, e := q.nextInFile(file)
		if e == nil {
			if max < job.Id() {
				max = job.Id()
			} else {
				log.Println(job, "followed higher id", max)
			}
		} else {
			err = e
		}
	}
	return max, err
}

func NewFileJobQueuePersister(name string) *FileJobQueuePersister {
	q := new(FileJobQueuePersister)
	q.name = name
	// TODO better erorr handling
	os.Mkdir(journalDirpath, 0777)
	dirname := journalDirpath + name
	os.Mkdir(dirname, 0777)
	dirFile, _ := os.Open(dirname)
	fis, _ := dirFile.Readdir(0)
	journals := make(map[int]journalFile)
	for _, fi := range fis {
		// skip any subdirectories (there shouldn't be any)
		if !fi.IsDir() {
			if isJournal(fi.Name()) {
				journals[journalNumber(fi.Name())] = journalFile{basename: fi.Name()}
			}
		}
	}
		/*writeFile, err := os.Create(q.filename)
		if err == nil {
			// TODO
			readFile, _ := os.Open(q.filename)
			q.writeFile = writeFile
			q.readFile = readFile
		 }
		if err != nil {
			log.Println("Failure opening journal:", err)
			return nil
		} else {
			return q
		}*/
	return q
}

func (q *FileJobQueuePersister) nextInFile(readFrom *os.File) (*job, error) {
	j := &job{}

	// read from file
	lengthBuffer := make([]byte, recordLengthBytes)
	bytesRead, err := q.readFile.Read(lengthBuffer)
	if bytesRead < recordLengthBytes || err != nil {
		if err == io.EOF {
			// Don't pass back EOF
			err = nil
		}
		return nil, err
	}
	recordLength := binary.BigEndian.Uint32(lengthBuffer)
	jobBuffer := make([]byte, recordLength)
	bytesRead, err = q.readFile.Read(jobBuffer)
	if uint32(bytesRead) < recordLength || err != nil {
		return nil, err
	}

	j.Deserialize(jobBuffer)
	return j, nil
}

func (q *FileJobQueuePersister) Get() Job {
	j, _ := q.nextInFile(q.readFile)
	if j == nil {
		return nil
	}

	// Extend window to track that this job has not been done-ed.
	q.window = append(q.window, false)

	return j
}

func writeLen(f *os.File, length uint32) bool {
	bytes := make([]byte, recordLengthBytes)
	binary.BigEndian.PutUint32(bytes, length)
	return writeBytes(f, bytes)
}

func writeBytes(f *os.File, bytes []byte) bool {
	written, err := f.Write(bytes)
	return err == nil && written == len(bytes)
}

func (q *FileJobQueuePersister) Push(job Job) bool {
	// Give it the next id
	job.SetId(q.head)
	q.head++

	bytes := job.Serialize()
	success := writeLen(q.writeFile, uint32(len(bytes)))
	if success {
		success := writeBytes(q.writeFile, bytes)
		return success
	}
	return success
}

// Returns the index of job in window, or -1 if out of range
func (q *FileJobQueuePersister) windowIndex(job Job) int {
	ind := int(job.Id() - q.tail)
	if ind < 0 || ind > len(q.window) {
		ind = -1
	}
	return ind
}

func firstFalseIndex(slice []bool) int {
	for i, v := range slice {
		if !v {
			return i
		}
	}
	return len(slice)
}

func (q *FileJobQueuePersister) updateDurablyWritten() {
	// TODO write q.tail to some file.
}

func (q *FileJobQueuePersister) Done(job Job) {
	ind := q.windowIndex(job)
	if ind < 0 {
		// TODO log or return 0 or something
		return
	}
	// Try to slide the window forward
	newTail := firstFalseIndex(q.window)
	if newTail > 0 {
		// Slide window forward.
		q.window = q.window[:newTail]
		q.tail += uint64(newTail)

		q.updateDurablyWritten()
	}
}
<|endoftext|>


<commit_before><commit_msg>Allow to get a blog post's content by slug<commit_after><|endoftext|>


<commit_before><commit_msg>Adding expvar to close #21<commit_after><|endoftext|>


<commit_before>package main

import (
	"strings"

	as "github.com/aerospike/aerospike-client-go"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// NamespaceMetrics lists the keys we report from aero's namespace statistics command.
	// See `asinfo -l -v namespace/<namespace>` for the full list.
	NamespaceMetrics = []metric{
		// allow-nonxdr-writes=true
		// allow-xdr-writes=true
		// cache_read_pct=0
		// cold-start-evict-ttl=4294967295
		// conflict-resolution-policy=generation
		// current_time=257114127
		// data-in-index=false
		// default-ttl=2592000
		// deleted_last_bin=0
		// disable-write-dup-res=false
		// disallow-null-setname=false
		// enable-benchmarks-batch-sub=false
		// enable-benchmarks-read=false
		// enable-benchmarks-udf-sub=false
		// enable-benchmarks-udf=false
		// enable-benchmarks-write=false
		// enable-hist-proxy=false
		// enable-xdr=false
		// evict_ttl=0
		// geo2dsphere-within.earth-radius-meters=6371000
		// geo2dsphere-within.level-mod=1
		// geo2dsphere-within.max-cells=12
		// geo2dsphere-within.max-level=30
		// geo2dsphere-within.min-level=1
		// geo2dsphere-within.strict=true
		// max-ttl=315360000
		// migrate-order=5
		// migrate-retransmit-ms=5000
		// migrate-sleep=1
		// ns-forward-xdr-writes=false
		// nsup_cycle_duration=0
		// nsup_cycle_sleep_pct=0
		// obj-size-hist-max=100
		// partition-tree-locks=8
		// partition-tree-sprigs=64
		// rack-id=0
		// read-consistency-level-override=off
		// sets-enable-xdr=true
		// sindex.num-partitions=32
		// single-bin=false
		// storage-engine=memory
		// tomb-raider-eligible-age=86400
		// tomb-raider-period=86400
		// truncate_lut=0
		// truncated_records=0
		// write-commit-level-override=off
		// xmem_id=0
		counter("batch_sub_proxy_complete", "batch sub proxy complete"),
		counter("batch_sub_proxy_error", "batch sub proxy error"),
		counter("batch_sub_proxy_timeout", "batch sub proxy timeout"),
		counter("batch_sub_read_error", "batch sub read error"),
		counter("batch_sub_read_not_found", "batch sub read not found"),
		counter("batch_sub_read_success", "batch sub read success"),
		counter("batch_sub_read_timeout", "batch sub read timeout"),
		counter("batch_sub_tsvc_error", "batch sub tsvc error"),
		counter("batch_sub_tsvc_timeout", "batch sub tsvc timeout"),
		counter("client_delete_error", "client delete error"),
		counter("client_delete_not_found", "client delete not found"),
		counter("client_delete_success", "client delete success"),
		counter("client_delete_timeout", "client delete timeout"),
		counter("client_lang_delete_success", "client lang delete success"),
		counter("client_lang_error", "client lang error"),
		counter("client_lang_read_success", "client lang read success"),
		counter("client_lang_write_success", "client lang write success"),
		counter("client_proxy_complete", "client proxy complete"),
		counter("client_proxy_error", "client proxy error"),
		counter("client_proxy_timeout", "client proxy timeout"),
		counter("client_read_error", "client read error"),
		counter("client_read_not_found", "client read not found"),
		counter("client_read_success", "client read success"),
		counter("client_read_timeout", "client read timeout"),
		counter("client_tsvc_error", "client tsvc error"),
		counter("client_tsvc_timeout", "client tsvc timeout"),
		counter("client_udf_complete", "client udf complete"),
		counter("client_udf_error", "client udf error"),
		counter("client_udf_timeout", "client udf timeout"),
		counter("client_write_error", "client write error"),
		counter("client_write_success", "client write success"),
		counter("client_write_timeout", "client write timeout"),
		counter("defrag_reads", "defrag reads"),
		counter("defrag_writes", "defrag writes"),
		counter("evicted_objects", "evicted objects"),
		counter("expired_objects", "expired objects"),
		counter("fail_generation", "fail generation"),
		counter("fail_key_busy", "fail key busy"),
		counter("fail_record_too_big", "fail record too big"),
		counter("fail_xdr_forbidden", "fail xdr forbidden"),
		counter("from_proxy_delete_error", "from proxy delete error"),
		counter("from_proxy_delete_not_found", "from proxy delete not found"),
		counter("from_proxy_delete_success", "from_proxy delete success"),
		counter("from_proxy_delete_timeout", "from proxy delete timeout"),
		counter("from_proxy_read_error", "from proxy read error"),
		counter("from_proxy_read_not_found", "from proxy read not found"),
		counter("from_proxy_read_success", "from proxy read success"),
		counter("from_proxy_read_timeout", "from proxy read timeout"),
		counter("from_proxy_tsvc_error", "from proxy tsvc error"),
		counter("from_proxy_tsvc_timeout", "from proxy tsvc timeout"),
		counter("from_proxy_write_error", "from proxy write error"),
		counter("from_proxy_write_success", "from proxy write success"),
		counter("from_proxy_write_timeout", "from proxy write timeout"),
		counter("geo_region_query_cells", "geo region query cells"),
		counter("geo_region_query_falsepos", "geo region query falsepos"),
		counter("geo_region_query_points", "geo region query points"),
		counter("geo_region_query_reqs", "geo region query reqs"),
		counter("query_agg_abort", "query agg abort"),
		counter("query_agg_avg_rec_count", "query agg avg rec count"),
		counter("query_agg_error", "query agg error"),
		counter("query_agg_success", "query agg success"),
		counter("query_agg", "query agg"),
		counter("query_fail", "query fail"),
		counter("query_long_queue_full", "query long queue full"),
		counter("query_long_reqs", "query long reqs"),
		counter("query_lookup_abort", "query lookup abort"),
		counter("query_lookup_avg_rec_count", "query lookup avg rec count"),
		counter("query_lookup_error", "query lookup error"),
		counter("query_lookup_success", "query lookup success"),
		counter("query_lookups", "query lookups"),
		counter("query_reqs", "query reqs"),
		counter("query_short_queue_full", "query short queue full"),
		counter("query_short_reqs", "query short reqs"),
		counter("query_udf_bg_failure", "query udf bg failure"),
		counter("query_udf_bg_success", "query udf bg success"),
		counter("retransmit_batch_sub_dup_res", "retransmit batch sub dup res"),
		counter("retransmit_client_delete_dup_res", "retransmit client delete dup res"),
		counter("retransmit_client_delete_repl_write", "retransmit client delete repl write"),
		counter("retransmit_client_read_dup_res", "retransmit client read dup res"),
		counter("retransmit_client_udf_dup_res", "retransmit client udf dup res"),
		counter("retransmit_client_udf_repl_write", "retransmit client udf repl write"),
		counter("retransmit_client_write_dup_res", "retransmit client write dup res"),
		counter("retransmit_client_write_repl_write", "retransmit client write repl write"),
		counter("retransmit_nsup_repl_write", "retransmit nsup repl write"),
		counter("retransmit_udf_sub_dup_res", "retransmit udf sub dup res"),
		counter("retransmit_udf_sub_repl_write", "retransmit udf sub repl write"),
		counter("scan_aggr_abort", "scan aggr abort"),
		counter("scan_aggr_complete", "scan aggr complete"),
		counter("scan_aggr_error", "scan aggr error"),
		counter("scan_basic_abort", "scan basic abort"),
		counter("scan_basic_complete", "scan basic complete"),
		counter("scan_basic_error", "scan basic error"),
		counter("scan_udf_bg_abort", "scan udf bg abort"),
		counter("scan_udf_bg_complete", "scan udf bg complete"),
		counter("scan_udf_bg_error", "scan udf bg error"),
		counter("udf_sub_lang_delete_success", "udf sub lang delete success"),
		counter("udf_sub_lang_error", "udf sub lang error"),
		counter("udf_sub_lang_read_success", "udf sub lang read success"),
		counter("udf_sub_lang_write_success", "udf sub lang write success"),
		counter("udf_sub_tsvc_error", "udf sub tsvc error"),
		counter("udf_sub_tsvc_timeout", "udf sub tsvc timeout"),
		counter("udf_sub_udf_complete", "udf sub udf complete"),
		counter("udf_sub_udf_error", "udf sub udf error"),
		counter("udf_sub_udf_timeout", "udf sub udf timeout"),
		counter("xdr_write_error", "xdr write error"),
		counter("xdr_write_success", "xdr write success"),
		counter("xdr_write_timeout", "xdr write timeout"),
		gauge("available_bin_names", "available bin names"),
		// broken gauge("defrag_q", "defrag queue"),
		gauge("device_available_pct", "device available pct"),
		gauge("device_compression_ratio", "device compression ratio"),
		gauge("device_free_pct", "device free pct"),
		gauge("device_total_bytes", "device total bytes"),
		gauge("device_used_bytes", "device used bytes"),
		gauge("effective_is_quiesced", "effective is quiesced"),
		gauge("effective_replication_factor", "effective replication factor"),
		gauge("evict-hist-buckets", "evict hist buckets"),
		gauge("evict-tenths-pct", "evict tenths pct"),
		gauge("high-water-disk-pct", "high water disk pct"),
		gauge("high-water-memory-pct", "high water memory pct"),
		gauge("hwm_breached", "hwm breached"),
		gauge("index_flash_used_bytes", "index flash used bytes"),
		gauge("index_flash_used_pct", "index flash used pct"),
		gauge("index-type.mounts-high-water-pct", "index type mounts high water pct"),
		gauge("index-type.mounts-size-limit", "index type mounts size limit"),
		gauge("master_objects", "master objects"),
		gauge("master_tombstones", "master tombstones"),
		gauge("memory_free_pct", "memory free pct"),
		gauge("memory_used_bytes", "memory used bytes"),
		gauge("memory_used_data_bytes", "memory used data bytes"),
		gauge("memory_used_index_bytes", "memory used index bytes"),
		gauge("memory_used_sindex_bytes", "memory used sindex bytes"),
		gauge("memory-size", "memory size"),
		gauge("migrate_record_receives", "migrate record receives"),
		gauge("migrate_record_retransmits", "migrate record retransmits"),
		gauge("migrate_records_skipped", "migrate records skipped"),
		gauge("migrate_records_transmitted", "migrate records transmitted"),
		gauge("migrate_rx_instances", "migrate rx instances"),
		gauge("migrate_rx_partitions_active", "migrate rx partitions active"),
		gauge("migrate_rx_partitions_initial", "migrate rx partitions initial"),
		gauge("migrate_rx_partitions_remaining", "migrate rx partitions remaining"),
		gauge("migrate_signals_active", "migrate signals active"),
		gauge("migrate_signals_remaining", "migrate signals remaining"),
		gauge("migrate_tx_instances", "migrate tx instances"),
		gauge("migrate_tx_partitions_active", "migrate tx partitions active"),
		gauge("migrate_tx_partitions_imbalance", "migrate tx partitions imbalance"),
		gauge("migrate_tx_partitions_initial", "migrate tx partitions initial"),
		gauge("migrate_tx_partitions_remaining", "migrate tx partitions remaining"),
		gauge("n_nodes_quiesced", "n nodes quiesced"),
		gauge("non_expirable_objects", "non expirable objects"),
		gauge("non_replica_objects", "non replica objects"),
		gauge("non_replica_tombstones", "non replica tombstones"),
		gauge("ns_cluster_size", "ns cluster size"),
		gauge("objects", "objects"),
		gauge("pending_quiesce", "pending quiesce"),
		gauge("prole_objects", "prole objects"),
		gauge("prole_tombstones", "prole tombstones"),
		gauge("replication-factor", "replication factor"),
		// broken gauge("shadow_write_q", "shadow write queue"),
		gauge("stop_writes", "stop writes"),
		gauge("stop-writes-pct", "stop writes pct"),
		gauge("tombstones", "tombstones"),
		// gauge("write_q", "write queue"),
		// device-level stats don't appear to work
		// and this plugin thinks "storage-engine.device[0].write_q" is malformed.
	}
)

type nsCollector cmetrics

func newNSCollector() nsCollector {
	ns := map[string]cmetric{}
	for _, m := range NamespaceMetrics {
		ns[m.aeroName] = cmetric{
			typ: m.typ,
			desc: prometheus.NewDesc(
				promkey(systemNamespace, m.aeroName),
				m.desc,
				[]string{"namespace"},
				nil,
			),
		}
	}
	return ns
}

func (nc nsCollector) describe(ch chan<- *prometheus.Desc) {
	for _, s := range nc {
		ch <- s.desc
	}
}

func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
	info, err := as.RequestInfo(conn, "namespaces")
	if err != nil {
		return nil, err
	}
	var metrics []prometheus.Metric
	for _, ns := range strings.Split(info["namespaces"], ";") {
		nsinfo, err := as.RequestInfo(conn, "namespace/"+ns)
		if err != nil {
			return nil, err
		}
		metrics = append(
			metrics,
			infoCollect(cmetrics(nc), nsinfo["namespace/"+ns], ns)...,
		)
	}
	return metrics, nil
}
<commit_msg>including additional key metrics as recommended by aerospike<commit_after>package main

import (
	"strings"

	as "github.com/aerospike/aerospike-client-go"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// NamespaceMetrics lists the keys we report from aero's namespace statistics command.
	// See `asinfo -l -v namespace/<namespace>` for the full list.
	NamespaceMetrics = []metric{
		// allow-nonxdr-writes=true
		// allow-xdr-writes=true
		// cache_read_pct=0
		// cold-start-evict-ttl=4294967295
		// conflict-resolution-policy=generation
		// current_time=257114127
		// data-in-index=false
		// default-ttl=2592000
		// deleted_last_bin=0
		// disable-write-dup-res=false
		// disallow-null-setname=false
		// enable-benchmarks-batch-sub=false
		// enable-benchmarks-read=false
		// enable-benchmarks-udf-sub=false
		// enable-benchmarks-udf=false
		// enable-benchmarks-write=false
		// enable-hist-proxy=false
		// enable-xdr=false
		// evict_ttl=0
		// geo2dsphere-within.earth-radius-meters=6371000
		// geo2dsphere-within.level-mod=1
		// geo2dsphere-within.max-cells=12
		// geo2dsphere-within.max-level=30
		// geo2dsphere-within.min-level=1
		// geo2dsphere-within.strict=true
		// max-ttl=315360000
		// migrate-order=5
		// migrate-retransmit-ms=5000
		// migrate-sleep=1
		// ns-forward-xdr-writes=false
		// nsup_cycle_duration=0
		// nsup_cycle_sleep_pct=0
		// obj-size-hist-max=100
		// partition-tree-locks=8
		// partition-tree-sprigs=64
		// rack-id=0
		// read-consistency-level-override=off
		// sets-enable-xdr=true
		// sindex.num-partitions=32
		// single-bin=false
		// storage-engine=memory
		// tomb-raider-eligible-age=86400
		// tomb-raider-period=86400
		// truncate_lut=0
		// truncated_records=0
		// write-commit-level-override=off
		// xmem_id=0
		counter("batch_sub_proxy_complete", "batch sub proxy complete"),
		counter("batch_sub_proxy_error", "batch sub proxy error"),
		counter("batch_sub_proxy_timeout", "batch sub proxy timeout"),
		counter("batch_sub_read_error", "batch sub read error"),
		counter("batch_sub_read_not_found", "batch sub read not found"),
		counter("batch_sub_read_success", "batch sub read success"),
		counter("batch_sub_read_timeout", "batch sub read timeout"),
		counter("batch_sub_tsvc_error", "batch sub tsvc error"),
		counter("batch_sub_tsvc_timeout", "batch sub tsvc timeout"),
		counter("client_delete_error", "client delete error"),
		counter("client_delete_not_found", "client delete not found"),
		counter("client_delete_success", "client delete success"),
		counter("client_delete_timeout", "client delete timeout"),
		counter("client_lang_delete_success", "client lang delete success"),
		counter("client_lang_error", "client lang error"),
		counter("client_lang_read_success", "client lang read success"),
		counter("client_lang_write_success", "client lang write success"),
		counter("client_proxy_complete", "client proxy complete"),
		counter("client_proxy_error", "client proxy error"),
		counter("client_proxy_timeout", "client proxy timeout"),
		counter("client_read_error", "client read error"),
		counter("client_read_not_found", "client read not found"),
		counter("client_read_success", "client read success"),
		counter("client_read_timeout", "client read timeout"),
		counter("client_tsvc_error", "client tsvc error"),
		counter("client_tsvc_timeout", "client tsvc timeout"),
		counter("client_udf_complete", "client udf complete"),
		counter("client_udf_error", "client udf error"),
		counter("client_udf_timeout", "client udf timeout"),
		counter("client_write_error", "client write error"),
		counter("client_write_success", "client write success"),
		counter("client_write_timeout", "client write timeout"),
		counter("defrag_reads", "defrag reads"),
		counter("defrag_writes", "defrag writes"),
		counter("evicted_objects", "evicted objects"),
		counter("expired_objects", "expired objects"),
		counter("fail_generation", "fail generation"),
		counter("fail_key_busy", "fail key busy"),
		counter("fail_record_too_big", "fail record too big"),
		counter("fail_xdr_forbidden", "fail xdr forbidden"),
		counter("from_proxy_delete_error", "from proxy delete error"),
		counter("from_proxy_delete_not_found", "from proxy delete not found"),
		counter("from_proxy_delete_success", "from_proxy delete success"),
		counter("from_proxy_delete_timeout", "from proxy delete timeout"),
		counter("from_proxy_read_error", "from proxy read error"),
		counter("from_proxy_read_not_found", "from proxy read not found"),
		counter("from_proxy_read_success", "from proxy read success"),
		counter("from_proxy_read_timeout", "from proxy read timeout"),
		counter("from_proxy_tsvc_error", "from proxy tsvc error"),
		counter("from_proxy_tsvc_timeout", "from proxy tsvc timeout"),
		counter("from_proxy_write_error", "from proxy write error"),
		counter("from_proxy_write_success", "from proxy write success"),
		counter("from_proxy_write_timeout", "from proxy write timeout"),
		counter("geo_region_query_cells", "geo region query cells"),
		counter("geo_region_query_falsepos", "geo region query falsepos"),
		counter("geo_region_query_points", "geo region query points"),
		counter("geo_region_query_reqs", "geo region query reqs"),
		counter("query_agg_abort", "query agg abort"),
		counter("query_agg_avg_rec_count", "query agg avg rec count"),
		counter("query_agg_error", "query agg error"),
		counter("query_agg_success", "query agg success"),
		counter("query_agg", "query agg"),
		counter("query_fail", "query fail"),
		counter("query_long_queue_full", "query long queue full"),
		counter("query_long_reqs", "query long reqs"),
		counter("query_lookup_abort", "query lookup abort"),
		counter("query_lookup_avg_rec_count", "query lookup avg rec count"),
		counter("query_lookup_error", "query lookup error"),
		counter("query_lookup_success", "query lookup success"),
		counter("query_lookups", "query lookups"),
		counter("query_reqs", "query reqs"),
		counter("query_short_queue_full", "query short queue full"),
		counter("query_short_reqs", "query short reqs"),
		counter("query_udf_bg_failure", "query udf bg failure"),
		counter("query_udf_bg_success", "query udf bg success"),
		counter("retransmit_batch_sub_dup_res", "retransmit batch sub dup res"),
		counter("retransmit_client_delete_dup_res", "retransmit client delete dup res"),
		counter("retransmit_client_delete_repl_write", "retransmit client delete repl write"),
		counter("retransmit_client_read_dup_res", "retransmit client read dup res"),
		counter("retransmit_client_udf_dup_res", "retransmit client udf dup res"),
		counter("retransmit_client_udf_repl_write", "retransmit client udf repl write"),
		counter("retransmit_client_write_dup_res", "retransmit client write dup res"),
		counter("retransmit_client_write_repl_write", "retransmit client write repl write"),
		counter("retransmit_nsup_repl_write", "retransmit nsup repl write"),
		counter("retransmit_udf_sub_dup_res", "retransmit udf sub dup res"),
		counter("retransmit_udf_sub_repl_write", "retransmit udf sub repl write"),
		counter("scan_aggr_abort", "scan aggr abort"),
		counter("scan_aggr_complete", "scan aggr complete"),
		counter("scan_aggr_error", "scan aggr error"),
		counter("scan_basic_abort", "scan basic abort"),
		counter("scan_basic_complete", "scan basic complete"),
		counter("scan_basic_error", "scan basic error"),
		counter("scan_udf_bg_abort", "scan udf bg abort"),
		counter("scan_udf_bg_complete", "scan udf bg complete"),
		counter("scan_udf_bg_error", "scan udf bg error"),
		counter("udf_sub_lang_delete_success", "udf sub lang delete success"),
		counter("udf_sub_lang_error", "udf sub lang error"),
		counter("udf_sub_lang_read_success", "udf sub lang read success"),
		counter("udf_sub_lang_write_success", "udf sub lang write success"),
		counter("udf_sub_tsvc_error", "udf sub tsvc error"),
		counter("udf_sub_tsvc_timeout", "udf sub tsvc timeout"),
		counter("udf_sub_udf_complete", "udf sub udf complete"),
		counter("udf_sub_udf_error", "udf sub udf error"),
		counter("udf_sub_udf_timeout", "udf sub udf timeout"),
		counter("xdr_write_error", "xdr write error"),
		counter("xdr_write_success", "xdr write success"),
		counter("xdr_write_timeout", "xdr write timeout"),
		gauge("available_bin_names", "available bin names"),
		// broken gauge("defrag_q", "defrag queue"),
		gauge("device_available_pct", "device available pct"),
		gauge("device_compression_ratio", "device compression ratio"),
		gauge("device_free_pct", "device free pct"),
		gauge("device_total_bytes", "device total bytes"),
		gauge("device_used_bytes", "device used bytes"),
		gauge("effective_is_quiesced", "effective is quiesced"),
		gauge("effective_replication_factor", "effective replication factor"),
		gauge("evict-hist-buckets", "evict hist buckets"),
		gauge("evict-tenths-pct", "evict tenths pct"),
		gauge("high-water-disk-pct", "high water disk pct"),
		gauge("high-water-memory-pct", "high water memory pct"),
		gauge("hwm_breached", "hwm breached"),
		gauge("index_flash_used_bytes", "index flash used bytes"),
		gauge("index_flash_used_pct", "index flash used pct"),
		gauge("index-type.mounts-high-water-pct", "index type mounts high water pct"),
		gauge("index-type.mounts-size-limit", "index type mounts size limit"),
		gauge("master_objects", "master objects"),
		gauge("master_tombstones", "master tombstones"),
		gauge("memory_free_pct", "memory free pct"),
		gauge("memory_used_bytes", "memory used bytes"),
		gauge("memory_used_data_bytes", "memory used data bytes"),
		gauge("memory_used_index_bytes", "memory used index bytes"),
		gauge("memory_used_sindex_bytes", "memory used sindex bytes"),
		gauge("memory-size", "memory size"),
		gauge("migrate_record_receives", "migrate record receives"),
		gauge("migrate_record_retransmits", "migrate record retransmits"),
		gauge("migrate_records_skipped", "migrate records skipped"),
		gauge("migrate_records_transmitted", "migrate records transmitted"),
		gauge("migrate_rx_instances", "migrate rx instances"),
		gauge("migrate_rx_partitions_active", "migrate rx partitions active"),
		gauge("migrate_rx_partitions_initial", "migrate rx partitions initial"),
		gauge("migrate_rx_partitions_remaining", "migrate rx partitions remaining"),
		gauge("migrate_signals_active", "migrate signals active"),
		gauge("migrate_signals_remaining", "migrate signals remaining"),
		gauge("migrate_tx_instances", "migrate tx instances"),
		gauge("migrate_tx_partitions_active", "migrate tx partitions active"),
		gauge("migrate_tx_partitions_imbalance", "migrate tx partitions imbalance"),
		gauge("migrate_tx_partitions_initial", "migrate tx partitions initial"),
		gauge("migrate_tx_partitions_remaining", "migrate tx partitions remaining"),
		gauge("n_nodes_quiesced", "n nodes quiesced"),
		gauge("non_expirable_objects", "non expirable objects"),
		gauge("non_replica_objects", "non replica objects"),
		gauge("non_replica_tombstones", "non replica tombstones"),
		gauge("ns_cluster_size", "ns cluster size"),
		gauge("objects", "objects"),
		gauge("pending_quiesce", "pending quiesce"),
		gauge("prole_objects", "prole objects"),
		gauge("prole_tombstones", "prole tombstones"),
		gauge("replication-factor", "replication factor"),
		// broken gauge("shadow_write_q", "shadow write queue"),
		gauge("stop_writes", "stop writes"),
		gauge("stop-writes-pct", "stop writes pct"),
		gauge("tombstones", "tombstones"),
		// including additional key metrics as recommended by aerospike  https://www.aerospike.com/docs/operations/monitor/key_metrics/index.html
		gauge("clock_skew_stop_writes", "clock skew stop writes"),
		gauge("dead_partitions", "dead partitions"),
		gauge("unavailable_partitions", "unavailable partitions"),
		// gauge("write_q", "write queue"),
		// device-level stats don't appear to work
		// and this plugin thinks "storage-engine.device[0].write_q" is malformed.
	}
)

type nsCollector cmetrics

func newNSCollector() nsCollector {
	ns := map[string]cmetric{}
	for _, m := range NamespaceMetrics {
		ns[m.aeroName] = cmetric{
			typ: m.typ,
			desc: prometheus.NewDesc(
				promkey(systemNamespace, m.aeroName),
				m.desc,
				[]string{"namespace"},
				nil,
			),
		}
	}
	return ns
}

func (nc nsCollector) describe(ch chan<- *prometheus.Desc) {
	for _, s := range nc {
		ch <- s.desc
	}
}

func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
	info, err := as.RequestInfo(conn, "namespaces")
	if err != nil {
		return nil, err
	}
	var metrics []prometheus.Metric
	for _, ns := range strings.Split(info["namespaces"], ";") {
		nsinfo, err := as.RequestInfo(conn, "namespace/"+ns)
		if err != nil {
			return nil, err
		}
		metrics = append(
			metrics,
			infoCollect(cmetrics(nc), nsinfo["namespace/"+ns], ns)...,
		)
	}
	return metrics, nil
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/labstack/echo"
)

// RegisterAPIRoutes adds API routes to the Echo's route group
func RegisterAPIRoutes(g *echo.Group) {
	// Store endpoints first since they are the most active
	//e.GET("/api/store/", storeGetView)
	//e.POST("/api/store/", storePostView)
	g = g.Group("/store")
	g.GET("/", storeGetView)
	g.POST("/", storePostView)
	// :project_id is [\w_-]+
	g = g.Group("/:project_id/store")
	g.GET("/", storeGetView)
	g.POST("/", storePostView)
	// :project_id is \d+
	g = g.Group("/:project_id/csp-report")
	// TODO is CspReportGetView needed?
	g.GET("/", cspReportGetView)
	g.POST("/", cspReportPostView)
}
<commit_msg>Add workaround for store event route<commit_after>package api

import (
	"github.com/labstack/echo"
)

// RegisterAPIRoutes adds API routes to the Echo's route group
func RegisterAPIRoutes(g *echo.Group) {
	// Store endpoints first since they are the most active
	//e.GET("/api/store/", storeGetView)
	//e.POST("/api/store/", storePostView)

	// TODO Can not register same handler for two different routes
	//g = g.Group("/store")
	//g.GET("/", storeGetView)
	//g.POST("/", storePostView)
	// :project_id is [\w_-]+
	g = g.Group("/:project_id/store")
	g.GET("/", storeGetView)
	g.POST("/", storePostView)
	// :project_id is \d+
	g = g.Group("/:project_id/csp-report")
	// TODO is CspReportGetView needed?
	g.GET("/", cspReportGetView)
	g.POST("/", cspReportPostView)
}
<|endoftext|>


<commit_before>package web

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/probs"
)

// SendError does a few things that we want for each error response:
//  - Adds both the external and the internal error to a RequestEvent.
//  - If the ProblemDetails provided is a ServerInternalProblem, audit logs the
//    internal error.
//  - Prefixes the Type field of the ProblemDetails with a namespace.
//  - Sends an HTTP response containing the error and an error code to the user.
func SendError(
	log blog.Logger,
	namespace string,
	response http.ResponseWriter,
	logEvent *RequestEvent,
	prob *probs.ProblemDetails,
	ierr error,
) {
	// Determine the HTTP status code to use for this problem
	code := probs.ProblemDetailsToStatusCode(prob)

	// Record details to the log event
	logEvent.Error = fmt.Sprintf("%d :: %s :: %s", prob.HTTPStatus, prob.Type, prob.Detail)
	if ierr != nil {
		logEvent.AddError(fmt.Sprintf("%s", ierr))
	}

	// Only audit log internal errors so users cannot purposefully cause
	// auditable events. Also, skip the audit log for deadline exceeded errors
	// since we don't need to keep those long-term. Note that they are still
	// included in the request logs.
	if prob.Type == probs.ServerInternalProblem && ierr != context.DeadlineExceeded {
		if ierr != nil {
			log.AuditErrf("Internal error - %s - %s", prob.Detail, ierr)
		} else {
			log.AuditErrf("Internal error - %s", prob.Detail)
		}
	}

	prob.Type = probs.ProblemType(namespace) + prob.Type
	problemDoc, err := json.MarshalIndent(prob, "", "  ")
	if err != nil {
		log.AuditErrf("Could not marshal error message: %s - %+v", err, prob)
		problemDoc = []byte("{\"detail\": \"Problem marshalling error message.\"}")
	}

	// Write the JSON problem response
	response.Header().Set("Content-Type", "application/problem+json")
	response.WriteHeader(code)
	response.Write(problemDoc)
}
<commit_msg>Add exception for grpc timeouts in WFE. (#4014)<commit_after>package web

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/probs"
)

// SendError does a few things that we want for each error response:
//  - Adds both the external and the internal error to a RequestEvent.
//  - If the ProblemDetails provided is a ServerInternalProblem, audit logs the
//    internal error.
//  - Prefixes the Type field of the ProblemDetails with a namespace.
//  - Sends an HTTP response containing the error and an error code to the user.
func SendError(
	log blog.Logger,
	namespace string,
	response http.ResponseWriter,
	logEvent *RequestEvent,
	prob *probs.ProblemDetails,
	ierr error,
) {
	// Determine the HTTP status code to use for this problem
	code := probs.ProblemDetailsToStatusCode(prob)

	// Record details to the log event
	logEvent.Error = fmt.Sprintf("%d :: %s :: %s", prob.HTTPStatus, prob.Type, prob.Detail)
	if ierr != nil {
		logEvent.AddError(fmt.Sprintf("%s", ierr))
	}

	// Only audit log internal errors so users cannot purposefully cause
	// auditable events. Also, skip the audit log for deadline exceeded errors
	// since we don't need to keep those long-term. Note that they are still
	// included in the request logs.
	deadlineExceeded := ierr == context.DeadlineExceeded || grpc.Code(ierr) == codes.DeadlineExceeded
	if prob.Type == probs.ServerInternalProblem && !deadlineExceeded {
		if ierr != nil {
			log.AuditErrf("Internal error - %s - %s", prob.Detail, ierr)
		} else {
			log.AuditErrf("Internal error - %s", prob.Detail)
		}
	}

	prob.Type = probs.ProblemType(namespace) + prob.Type
	problemDoc, err := json.MarshalIndent(prob, "", "  ")
	if err != nil {
		log.AuditErrf("Could not marshal error message: %s - %+v", err, prob)
		problemDoc = []byte("{\"detail\": \"Problem marshalling error message.\"}")
	}

	// Write the JSON problem response
	response.Header().Set("Content-Type", "application/problem+json")
	response.WriteHeader(code)
	response.Write(problemDoc)
}
<|endoftext|>


<commit_before><commit_msg>Extended AlmostEqual to accept two float64 values<commit_after><|endoftext|>


<commit_before><commit_msg>Fixed.<commit_after><|endoftext|>


<commit_before>package generators

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/juju/errgo"
	"github.com/sabhiram/go-git-ignore"
	"github.com/yext/edward/services"
)

type Generator interface {
	Name() string
	StartWalk(basePath string)
	StopWalk()
	VisitDir(path string, f os.FileInfo, err error) error
	Found() []*services.ServiceConfig
}

type ConfigGenerator func(path string) ([]*services.ServiceConfig, []*services.ServiceGroupConfig, error)

var Generators map[string]Generator

func RegisterGenerator(g Generator) {
	if Generators == nil {
		Generators = make(map[string]Generator)
	}
	Generators[g.Name()] = g
}

func loadIgnores(path string, currentIgnores *ignore.GitIgnore) (*ignore.GitIgnore, error) {
	ignoreFile := filepath.Join(path, ".edwardignore")
	if _, err := os.Stat(ignoreFile); err != nil {
		if os.IsNotExist(err) {
			return currentIgnores, nil
		}
		return currentIgnores, errgo.Mask(err)
	}

	ignores, err := ignore.CompileIgnoreFile(ignoreFile)
	return ignores, errgo.Mask(err)
}

func shouldIgnore(basePath, path string, ignores *ignore.GitIgnore) bool {
	if ignores == nil {
		return false
	}

	rel, err := filepath.Rel(basePath, path)
	if err != nil {
		fmt.Println("Error checking ignore:", err)
		return false
	}

	return ignores.MatchesPath(rel)
}

func GenerateServices(path string, targets []string) ([]*services.ServiceConfig, error) {
	var outServices []*services.ServiceConfig

	if info, err := os.Stat(path); err != nil || !info.IsDir() {
		if err != nil {
			return outServices, err
		}
		return outServices, errors.New(path + " is not a directory")
	}

	// TODO: Create a stack of ignore files to handle ignores in subdirs
	ignores, err := loadIgnores(path, nil)
	if err != nil {
		return outServices, errgo.Mask(err)
	}

	for name, generator := range Generators {
		generator.StartWalk(path)
		err := filepath.Walk(path, func(curPath string, f os.FileInfo, err error) error {
			if _, err := os.Stat(curPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return errgo.Mask(err)
			}

			if !f.Mode().IsDir() || shouldIgnore(path, curPath, ignores) {
				return nil
			}

			err = generator.VisitDir(curPath, f, err)
			if err == filepath.SkipDir {
				return err
			}
			return errgo.Mask(err)
		})
		generator.StopWalk()
		if err != nil {
			fmt.Println("Error in generator", name, ":", err)
		} else {
			outServices = append(outServices, generator.Found()...)
		}
	}

	if len(targets) == 0 {
		return outServices, nil
	}

	filterMap := make(map[string]struct{})
	for _, name := range targets {
		filterMap[name] = struct{}{}
	}

	var filteredServices []*services.ServiceConfig
	for _, service := range outServices {
		if _, ok := filterMap[service.Name]; ok {
			filteredServices = append(filteredServices, service)
			continue
		}
	}

	if len(filteredServices) == 0 {
		return nil, errgo.New("No matching services found")
	}

	return filteredServices, nil
}
<commit_msg>Sort the list of services returned from the generator. This should fix an unreliable test in Travis.<commit_after>package generators

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	"github.com/juju/errgo"
	"github.com/sabhiram/go-git-ignore"
	"github.com/yext/edward/services"
)

type Generator interface {
	Name() string
	StartWalk(basePath string)
	StopWalk()
	VisitDir(path string, f os.FileInfo, err error) error
	Found() []*services.ServiceConfig
}

type ConfigGenerator func(path string) ([]*services.ServiceConfig, []*services.ServiceGroupConfig, error)

var Generators map[string]Generator

func RegisterGenerator(g Generator) {
	if Generators == nil {
		Generators = make(map[string]Generator)
	}
	Generators[g.Name()] = g
}

func loadIgnores(path string, currentIgnores *ignore.GitIgnore) (*ignore.GitIgnore, error) {
	ignoreFile := filepath.Join(path, ".edwardignore")
	if _, err := os.Stat(ignoreFile); err != nil {
		if os.IsNotExist(err) {
			return currentIgnores, nil
		}
		return currentIgnores, errgo.Mask(err)
	}

	ignores, err := ignore.CompileIgnoreFile(ignoreFile)
	return ignores, errgo.Mask(err)
}

func shouldIgnore(basePath, path string, ignores *ignore.GitIgnore) bool {
	if ignores == nil {
		return false
	}

	rel, err := filepath.Rel(basePath, path)
	if err != nil {
		fmt.Println("Error checking ignore:", err)
		return false
	}

	return ignores.MatchesPath(rel)
}

func GenerateServices(path string, targets []string) ([]*services.ServiceConfig, error) {
	var outServices []*services.ServiceConfig

	if info, err := os.Stat(path); err != nil || !info.IsDir() {
		if err != nil {
			return outServices, err
		}
		return nil, errors.New(path + " is not a directory")
	}

	// TODO: Create a stack of ignore files to handle ignores in subdirs
	ignores, err := loadIgnores(path, nil)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	for name, generator := range Generators {
		generator.StartWalk(path)
		err := filepath.Walk(path, func(curPath string, f os.FileInfo, err error) error {
			if _, err := os.Stat(curPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return errgo.Mask(err)
			}

			if !f.Mode().IsDir() || shouldIgnore(path, curPath, ignores) {
				return nil
			}

			err = generator.VisitDir(curPath, f, err)
			if err == filepath.SkipDir {
				return err
			}
			return errgo.Mask(err)
		})
		generator.StopWalk()
		if err != nil {
			fmt.Println("Error in generator", name, ":", err)
		} else {
			outServices = append(outServices, generator.Found()...)
		}
	}

	if len(targets) == 0 {
		sort.Sort(ByName(outServices))
		return outServices, nil
	}

	filterMap := make(map[string]struct{})
	for _, name := range targets {
		filterMap[name] = struct{}{}
	}

	var filteredServices []*services.ServiceConfig
	for _, service := range outServices {
		if _, ok := filterMap[service.Name]; ok {
			filteredServices = append(filteredServices, service)
			continue
		}
	}

	if len(filteredServices) == 0 {
		return nil, errgo.New("No matching services found")
	}

	sort.Sort(ByName(filteredServices))
	return filteredServices, nil
}

type ByName []*services.ServiceConfig

func (s ByName) Len() int {
	return len(s)
}
func (s ByName) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s ByName) Less(i, j int) bool {
	return s[i].Name < s[j].Name
}
<|endoftext|>


<commit_before><commit_msg>Add FORWARDTSN to support extension params<commit_after><|endoftext|>


<commit_before>package asset

import (
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
)

type generationLog struct {
	log []string
}

func (l *generationLog) logGeneration(a Asset) {
	l.log = append(l.log, a.Name())
}

type testStoreAsset interface {
	Asset
	SetDependencies([]Asset)
	SetDirty(bool)
}

type testStoreAssetImpl struct {
	name          string
	dependencies  []Asset
	generationLog *generationLog
	dirty         bool
}

func (a *testStoreAssetImpl) Dependencies() []Asset {
	return a.dependencies
}

func (a *testStoreAssetImpl) Generate(Parents) error {
	a.generationLog.logGeneration(a)
	return nil
}

func (a *testStoreAssetImpl) Name() string {
	return a.name
}

func (a *testStoreAssetImpl) SetDependencies(dependencies []Asset) {
	a.dependencies = dependencies
}

func (a *testStoreAssetImpl) SetDirty(dirty bool) {
	a.dirty = dirty
}
func (a *testStoreAssetImpl) Files() []*File {
	return []*File{{Filename: a.name}}
}

func (a *testStoreAssetImpl) Load(FileFetcher) (bool, error) {
	return a.dirty, nil
}

type testStoreAssetA struct {
	testStoreAssetImpl
}

type testStoreAssetB struct {
	testStoreAssetImpl
}

type testStoreAssetC struct {
	testStoreAssetImpl
}

type testStoreAssetD struct {
	testStoreAssetImpl
}

func newTestStoreAsset(gl *generationLog, name string) testStoreAsset {
	ta := testStoreAssetImpl{
		name:          name,
		generationLog: gl,
	}
	switch name {
	case "a":
		return &testStoreAssetA{ta}
	case "b":
		return &testStoreAssetB{ta}
	case "c":
		return &testStoreAssetC{ta}
	case "d":
		return &testStoreAssetD{ta}
	default:
		return nil
	}
}

// TestStoreFetch tests the Fetch method of StoreImpl.
func TestStoreFetch(t *testing.T) {
	cases := []struct {
		name                  string
		assets                map[string][]string
		existingAssets        []string
		target                string
		expectedGenerationLog []string
	}{
		{
			name: "no dependencies",
			assets: map[string][]string{
				"a": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "single dependency",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"b", "a"},
		},
		{
			name: "multiple dependencies",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"b", "c", "a"},
		},
		{
			name: "grandchild dependency",
			assets: map[string][]string{
				"a": {"b"},
				"b": {"c"},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"c", "b", "a"},
		},
		{
			name: "intragenerational shared dependency",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"d", "b", "c", "a"},
		},
		{
			name: "intergenerational shared dependency",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"c"},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"c", "b", "a"},
		},
		{
			name: "existing asset",
			assets: map[string][]string{
				"a": {},
			},
			existingAssets:        []string{"a"},
			target:                "a",
			expectedGenerationLog: []string{},
		},
		{
			name: "existing child asset",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "absent grandchild asset",
			assets: map[string][]string{
				"a": {"b"},
				"b": {"c"},
				"c": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "absent grandchild with absent parent",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"d", "c", "a"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gl := &generationLog{
				log: []string{},
			}
			store := &StoreImpl{
				assets: make(map[reflect.Type]assetState),
			}
			assets := make(map[string]testStoreAsset, len(tc.assets))
			for name := range tc.assets {
				assets[name] = newTestStoreAsset(gl, name)
			}
			for name, deps := range tc.assets {
				dependencies := make([]Asset, len(deps))
				for i, d := range deps {
					dependencies[i] = assets[d]
				}
				assets[name].SetDependencies(dependencies)
			}
			for _, assetName := range tc.existingAssets {
				asset := assets[assetName]
				store.assets[reflect.TypeOf(asset)] = assetState{asset: asset}
			}
			err := store.Fetch(assets[tc.target])
			assert.NoError(t, err, "error fetching asset")
			assert.EqualValues(t, tc.expectedGenerationLog, gl.log)
		})
	}
}

func TestStoreFetchDirty(t *testing.T) {
	cases := []struct {
		name                  string
		assets                map[string][]string
		dirtyAssets           []string
		target                string
		expectedGenerationLog []string
		expectedDirty         bool
	}{
		{
			name: "no dirty assets",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           nil,
			target:                "a",
			expectedGenerationLog: []string{"b", "a"},
			expectedDirty:         false,
		},
		{
			name: "dirty asset causes re-generation",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"a"},
			target:                "a",
			expectedGenerationLog: []string{"b"},
			expectedDirty:         true,
		},
		{
			name: "dirty dependent asset causes re-generation",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
			expectedDirty:         true,
		},
		{
			name: "dirty dependents invalidate all its children",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			dirtyAssets:           []string{"d"},
			target:                "a",
			expectedGenerationLog: []string{"b", "c", "a"},
			expectedDirty:         true,
		},
		{
			name: "re-generate when both parents and childre are dirty",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"a", "b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
			expectedDirty:         true,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gl := &generationLog{
				log: []string{},
			}
			store := &StoreImpl{
				assets: make(map[reflect.Type]assetState),
			}
			assets := make(map[string]testStoreAsset, len(tc.assets))
			for name := range tc.assets {
				assets[name] = newTestStoreAsset(gl, name)
			}
			for name, deps := range tc.assets {
				dependencies := make([]Asset, len(deps))
				for i, d := range deps {
					dependencies[i] = assets[d]
				}
				assets[name].SetDependencies(dependencies)
			}
			for _, name := range tc.dirtyAssets {
				assets[name].SetDirty(true)
			}
			dirty, err := store.fetch(assets[tc.target], "")
			assert.NoError(t, err, "unexpected error")
			assert.EqualValues(t, tc.expectedGenerationLog, gl.log)
			assert.Equal(t, tc.expectedDirty, dirty)
		})
	}
}
<commit_msg>asset: use temporary directory for asset store in tests<commit_after>package asset

import (
	"io/ioutil"
	"os"
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
)

type generationLog struct {
	log []string
}

func (l *generationLog) logGeneration(a Asset) {
	l.log = append(l.log, a.Name())
}

type testStoreAsset interface {
	Asset
	SetDependencies([]Asset)
	SetDirty(bool)
}

type testStoreAssetImpl struct {
	name          string
	dependencies  []Asset
	generationLog *generationLog
	dirty         bool
}

func (a *testStoreAssetImpl) Dependencies() []Asset {
	return a.dependencies
}

func (a *testStoreAssetImpl) Generate(Parents) error {
	a.generationLog.logGeneration(a)
	return nil
}

func (a *testStoreAssetImpl) Name() string {
	return a.name
}

func (a *testStoreAssetImpl) SetDependencies(dependencies []Asset) {
	a.dependencies = dependencies
}

func (a *testStoreAssetImpl) SetDirty(dirty bool) {
	a.dirty = dirty
}
func (a *testStoreAssetImpl) Files() []*File {
	return []*File{{Filename: a.name}}
}

func (a *testStoreAssetImpl) Load(FileFetcher) (bool, error) {
	return a.dirty, nil
}

type testStoreAssetA struct {
	testStoreAssetImpl
}

type testStoreAssetB struct {
	testStoreAssetImpl
}

type testStoreAssetC struct {
	testStoreAssetImpl
}

type testStoreAssetD struct {
	testStoreAssetImpl
}

func newTestStoreAsset(gl *generationLog, name string) testStoreAsset {
	ta := testStoreAssetImpl{
		name:          name,
		generationLog: gl,
	}
	switch name {
	case "a":
		return &testStoreAssetA{ta}
	case "b":
		return &testStoreAssetB{ta}
	case "c":
		return &testStoreAssetC{ta}
	case "d":
		return &testStoreAssetD{ta}
	default:
		return nil
	}
}

// TestStoreFetch tests the Fetch method of StoreImpl.
func TestStoreFetch(t *testing.T) {
	cases := []struct {
		name                  string
		assets                map[string][]string
		existingAssets        []string
		target                string
		expectedGenerationLog []string
	}{
		{
			name: "no dependencies",
			assets: map[string][]string{
				"a": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "single dependency",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"b", "a"},
		},
		{
			name: "multiple dependencies",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"b", "c", "a"},
		},
		{
			name: "grandchild dependency",
			assets: map[string][]string{
				"a": {"b"},
				"b": {"c"},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"c", "b", "a"},
		},
		{
			name: "intragenerational shared dependency",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"d", "b", "c", "a"},
		},
		{
			name: "intergenerational shared dependency",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"c"},
				"c": {},
			},
			target:                "a",
			expectedGenerationLog: []string{"c", "b", "a"},
		},
		{
			name: "existing asset",
			assets: map[string][]string{
				"a": {},
			},
			existingAssets:        []string{"a"},
			target:                "a",
			expectedGenerationLog: []string{},
		},
		{
			name: "existing child asset",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "absent grandchild asset",
			assets: map[string][]string{
				"a": {"b"},
				"b": {"c"},
				"c": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
		},
		{
			name: "absent grandchild with absent parent",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			existingAssets:        []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"d", "c", "a"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gl := &generationLog{
				log: []string{},
			}
			dir, err := ioutil.TempDir("", "TestStoreFetch")
			if err != nil {
				t.Fatalf("failed to create temporary directory: %v", err)
			}
			defer os.RemoveAll(dir)
			store := &StoreImpl{
				directory: dir,
				assets:    make(map[reflect.Type]assetState),
			}
			assets := make(map[string]testStoreAsset, len(tc.assets))
			for name := range tc.assets {
				assets[name] = newTestStoreAsset(gl, name)
			}
			for name, deps := range tc.assets {
				dependencies := make([]Asset, len(deps))
				for i, d := range deps {
					dependencies[i] = assets[d]
				}
				assets[name].SetDependencies(dependencies)
			}
			for _, assetName := range tc.existingAssets {
				asset := assets[assetName]
				store.assets[reflect.TypeOf(asset)] = assetState{asset: asset}
			}
			err = store.Fetch(assets[tc.target])
			assert.NoError(t, err, "error fetching asset")
			assert.EqualValues(t, tc.expectedGenerationLog, gl.log)
		})
	}
}

func TestStoreFetchDirty(t *testing.T) {
	cases := []struct {
		name                  string
		assets                map[string][]string
		dirtyAssets           []string
		target                string
		expectedGenerationLog []string
		expectedDirty         bool
	}{
		{
			name: "no dirty assets",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           nil,
			target:                "a",
			expectedGenerationLog: []string{"b", "a"},
			expectedDirty:         false,
		},
		{
			name: "dirty asset causes re-generation",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"a"},
			target:                "a",
			expectedGenerationLog: []string{"b"},
			expectedDirty:         true,
		},
		{
			name: "dirty dependent asset causes re-generation",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
			expectedDirty:         true,
		},
		{
			name: "dirty dependents invalidate all its children",
			assets: map[string][]string{
				"a": {"b", "c"},
				"b": {"d"},
				"c": {"d"},
				"d": {},
			},
			dirtyAssets:           []string{"d"},
			target:                "a",
			expectedGenerationLog: []string{"b", "c", "a"},
			expectedDirty:         true,
		},
		{
			name: "re-generate when both parents and childre are dirty",
			assets: map[string][]string{
				"a": {"b"},
				"b": {},
			},
			dirtyAssets:           []string{"a", "b"},
			target:                "a",
			expectedGenerationLog: []string{"a"},
			expectedDirty:         true,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gl := &generationLog{
				log: []string{},
			}
			store := &StoreImpl{
				assets: make(map[reflect.Type]assetState),
			}
			assets := make(map[string]testStoreAsset, len(tc.assets))
			for name := range tc.assets {
				assets[name] = newTestStoreAsset(gl, name)
			}
			for name, deps := range tc.assets {
				dependencies := make([]Asset, len(deps))
				for i, d := range deps {
					dependencies[i] = assets[d]
				}
				assets[name].SetDependencies(dependencies)
			}
			for _, name := range tc.dirtyAssets {
				assets[name].SetDirty(true)
			}
			dirty, err := store.fetch(assets[tc.target], "")
			assert.NoError(t, err, "unexpected error")
			assert.EqualValues(t, tc.expectedGenerationLog, gl.log)
			assert.Equal(t, tc.expectedDirty, dirty)
		})
	}
}
<|endoftext|>


<commit_before>package trees

import (
	"errors"
	"sync"

	"github.com/joushou/qp"
)

// MapHandle provides the reading/writing interface to a MapFile.
type MapHandle struct {
	*DetachedHandle
	f         *MapFile
	cmdbuffer []byte
}

// Seek performs the requested seek, but updates the representation of the map
// when seeking to 0.
func (h *MapHandle) Seek(offset int64, whence int) (int64, error) {
	off, err := h.DetachedHandle.Seek(offset, whence)
	if off == 0 {
		// We update the map when you seek to 0
		h.Content = h.f.renderMap()
	}
	return off, err
}

// Write adds the written data to the internal command buffer, processing all
// complete assignments.
func (h *MapHandle) Write(p []byte) (int, error) {
	h.Lock()
	defer h.Unlock()
	h.cmdbuffer = h.f.query(append(h.cmdbuffer, p...))
	return len(p), nil
}

// MapFile provides locked map[string]string storage. Reading the file dumps
// the map, with each entry being written as "key=val", each item separated by
// newlines. Writing "key=val\n" updates "key" to "val". Writing "key=\n"
// deletes key. All characters are considered legal, with the exception that a
// key cannot contain "=", and that a value cannot contain "\n".
type MapFile struct {
	*SyntheticFile
	store     map[string]string
	storelock sync.Mutex
}

// renderMap returns the current file representation of the map.
func (f *MapFile) renderMap() []byte {
	f.storelock.Lock()
	defer f.storelock.Unlock()

	var b []byte
	for i := range f.store {
		b = append(b, []byte(i)...)
		b = append(b, '=')
		b = append(b, []byte(f.store[i])...)
		b = append(b, '\n')
	}
	return b
}

// query takes a command buffer, processes all complete and valid map
// assignments, and returns the remaining buffer.
func (f *MapFile) query(cmd []byte) []byte {
	f.storelock.Lock()
	defer f.storelock.Unlock()

	startIdx := 0
	key := ""
	gotKey := false
	consumed := 0

	for i := range cmd {
		if i < startIdx {
			continue
		}

		if gotKey {
			if cmd[i] == '\n' {
				data := string(cmd[startIdx:i])
				f.store[key] = data
				if len(data) == 0 {
					delete(f.store, key)
				}
				gotKey = false
				startIdx = i + 1
				consumed += len(key) + len(data) + 2
			}
		} else {
			if cmd[i] == '=' {
				key = string(cmd[startIdx:i])
				gotKey = true
				startIdx = i + 1
			}
		}
	}

	return cmd[consumed:]
}

// Open returns a MapHandle.
func (f *MapFile) Open(user string, mode qp.OpenMode) (ReadWriteSeekCloser, error) {
	if !f.CanOpen(user, mode) {
		return nil, errors.New("access denied")
	}

	return &MapHandle{
		DetachedHandle: &DetachedHandle{
			Readable:   true,
			Writable:   true,
			AppendOnly: true,
		},
		f: f,
	}, nil
}

// NewMapFile returns an initialised MapFile.
func NewMapFile(name string, perm qp.FileMode, user, group string) *MapFile {
	return &MapFile{
		SyntheticFile: NewSyntheticFile(name, 0666, user, group),
		store:         make(map[string]string),
	}
}
<commit_msg>Implement MapFile using CallbackFile<commit_after>package trees

import (
	"sync"

	"github.com/joushou/qp"
)

// MapFile provides locked map[string]string storage. Reading the file dumps
// the map, with each entry being written as "key=val", each item separated by
// newlines. Writing "key=val\n" updates "key" to "val". Writing "key=\n"
// deletes key. All characters are considered legal, with the exception that a
// key cannot contain "=", and that a value cannot contain "\n".
type MapFile struct {
	*CallbackFile
	store     map[string]string
	storelock sync.Mutex
}

// renderMap returns the current file representation of the map.
func (f *MapFile) renderMap() []byte {
	f.storelock.Lock()
	defer f.storelock.Unlock()

	var b []byte
	for i := range f.store {
		b = append(b, []byte(i)...)
		b = append(b, '=')
		b = append(b, []byte(f.store[i])...)
		b = append(b, '\n')
	}
	return b
}

// query takes a command buffer, processes all complete and valid map
// assignments, and returns the remaining buffer.
func (f *MapFile) query(cmd []byte) []byte {
	f.storelock.Lock()
	defer f.storelock.Unlock()

	startIdx := 0
	key := ""
	gotKey := false
	consumed := 0

	for i := range cmd {
		if i < startIdx {
			continue
		}

		if gotKey {
			if cmd[i] == '\n' {
				data := string(cmd[startIdx:i])
				f.store[key] = data
				if len(data) == 0 {
					delete(f.store, key)
				}
				gotKey = false
				startIdx = i + 1
				consumed += len(key) + len(data) + 2
			}
		} else {
			if cmd[i] == '=' {
				key = string(cmd[startIdx:i])
				gotKey = true
				startIdx = i + 1
			}
		}
	}

	return cmd[consumed:]
}

// NewMapFile returns an initialised MapFile.
func NewMapFile(name string, perm qp.FileMode, user, group string) *MapFile {
	m := &MapFile{
		store: make(map[string]string),
	}
	m.CallbackFile = NewCallbackFile(name, 0666, user, group, m.renderMap, m.query)
	return m
}
<|endoftext|>


<commit_before>package ozinit

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/subgraph/oz"
	"github.com/subgraph/oz/fs"
	"github.com/subgraph/oz/ipc"
	"github.com/subgraph/oz/xpra"
	"github.com/subgraph/oz/network"

	"github.com/kr/pty"
	"github.com/op/go-logging"
)

const SocketAddress = "/tmp/oz-init-control"

type initState struct {
	log       *logging.Logger
	profile   *oz.Profile
	config    *oz.Config
	uid       int
	gid       int
	user      *user.User
	display   int
	fs        *fs.Filesystem
	xpraReady sync.WaitGroup
	network   *network.SandboxNetwork
}

// By convention oz-init writes log messages to stderr with a single character
// prefix indicating the logging level.  These messages are read one line at a time
// over a pipe by oz-daemon and translated into appropriate log events.
func createLogger() *logging.Logger {
	l := logging.MustGetLogger("oz-init")
	be := logging.NewLogBackend(os.Stderr, "", 0)
	f := logging.MustStringFormatter("%{level:.1s} %{message}")
	fbe := logging.NewBackendFormatter(be, f)
	logging.SetBackend(fbe)
	return l
}

func Main() {
	parseArgs().runInit()
}

func parseArgs() *initState {
	log := createLogger()
	getvar := func(name string) string {
		val := os.Getenv(name)
		if val == "" {
			log.Error("Error: missing required '%s' argument", name)
			os.Exit(1)
		}
		return val
	}
	pname := getvar("INIT_PROFILE")
	uidval := getvar("INIT_UID")
	dispval := os.Getenv("INIT_DISPLAY")

	stnip := os.Getenv("INIT_ADDR")
	stnvhost := os.Getenv("INIT_VHOST")
	stnvguest := os.Getenv("INIT_VGUEST")
	stngateway := os.Getenv("INIT_GATEWAY")

	var config *oz.Config
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		log.Info("Could not load config file (%s), using default config", oz.DefaultConfigPath)
		config = oz.NewDefaultConfig()
	}

	p, err := loadProfile(config.ProfileDir, pname)
	if err != nil {
		log.Error("Could not load profile %s: %v", pname, err)
		os.Exit(1)
	}
	uid, err := strconv.Atoi(uidval)
	if err != nil {
		log.Error("Could not parse INIT_UID argument (%s) into an integer: %v", uidval, err)
		os.Exit(1)
	}
	u, err := user.LookupId(uidval)
	if err != nil {
		log.Error("Failed to look up user with uid=%s: %v", uidval, err)
		os.Exit(1)
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		log.Error("Failed to parse gid value (%s) from user struct: %v", u.Gid, err)
		os.Exit(1)
	}
	display := 0
	if dispval != "" {
		d, err := strconv.Atoi(dispval)
		if err != nil {
			log.Error("Unable to parse display (%s) into an integer: %v", dispval, err)
			os.Exit(1)
		}
		display = d
	}

	stn := new(network.SandboxNetwork)
	if stnip != "" {
		gateway, _, err := net.ParseCIDR(stngateway)
		if err != nil {
			log.Error("Unable to parse network configuration gateway (%s): %v", stngateway, err)
			os.Exit(1)
		}

		stn.Ip        = stnip
		stn.VethHost  = stnvhost
		stn.VethGuest = stnvguest
		stn.Gateway   = gateway
	}

	return &initState{
		log:     log,
		config:  config,
		profile: p,
		uid:     uid,
		gid:     gid,
		user:    u,
		display: display,
		fs:      fs.NewFromProfile(p, u, config.SandboxPath, config.UseFullDev, log),
		network: stn,
	}
}

func (st *initState) runInit() {
	st.log.Info("Starting oz-init for profile: %s", st.profile.Name)

	if st.profile.Networking.Nettype != "host" {
		err := network.NetSetup(st.network)
		if err != nil {
			st.log.Error("Unable to setup networking: %+v", err)
			os.Exit(1)
		}
	}
	network.NetPrint(st.log)

	if syscall.Sethostname([]byte(st.profile.Name)) != nil {
		st.log.Error("Failed to set hostname to (%s)", st.profile.Name)
	}
	st.log.Info("Hostname set to (%s)", st.profile.Name)

	if err := st.fs.OzInit(); err != nil {
		st.log.Error("Error: setting up filesystem failed: %v\n", err)
		os.Exit(1)
	}
	oz.ReapChildProcs(st.log, st.handleChildExit)

	if st.profile.XServer.Enabled {
		st.xpraReady.Add(1)
		st.startXpraServer()
	}
	st.xpraReady.Wait()
	st.launchApplication()

	s, err := ipc.NewServer(SocketAddress, messageFactory, st.log,
		handlePing,
		st.handleRunShell,
	)
	if err != nil {
		st.log.Error("NewServer failed: %v", err)
		os.Exit(1)
	}
	if err := os.Chown(SocketAddress, st.uid, st.gid); err != nil {
		st.log.Warning("Failed to chown oz-init control socket: %v", err)
	}
	os.Stderr.WriteString("OK\n")

	if err := s.Run(); err != nil {
		st.log.Warning("MsgServer.Run() return err: %v", err)
	}
	st.log.Info("oz-init exiting...")
}

func (st *initState) startXpraServer() {
	workdir := st.fs.Xpra()
	if workdir == "" {
		st.log.Warning("Xpra work directory not set")
		return
	}
	xpra := xpra.NewServer(&st.profile.XServer, uint64(st.display), workdir)
	p, err := xpra.Process.StderrPipe()
	if err != nil {
		st.log.Warning("Error creating stderr pipe for xpra output: %v", err)
		os.Exit(1)
	}
	go st.readXpraOutput(p)
	xpra.Process.Env = []string{
		"HOME=" + st.user.HomeDir,
	}
	xpra.Process.SysProcAttr = &syscall.SysProcAttr{}
	xpra.Process.SysProcAttr.Credential = &syscall.Credential{
		Uid: uint32(st.uid),
		Gid: uint32(st.gid),
	}
	st.log.Info("Starting xpra server")
	if err := xpra.Process.Start(); err != nil {
		st.log.Warning("Failed to start xpra server: %v", err)
	}
}

func (st *initState) readXpraOutput(r io.ReadCloser) {
	sc := bufio.NewScanner(r)
	seenReady := false
	for sc.Scan() {
		line := sc.Text()
		if len(line) > 0 {
			if strings.Contains(line, "xpra is ready.") && !seenReady {
				seenReady = true
				st.xpraReady.Done()
				if !st.config.LogXpra {
					r.Close()
					return
				}
			}
			if st.config.LogXpra {
				st.log.Debug("(xpra) %s", line)
			}
		}
	}
}

func (st *initState) launchApplication() {
	cmd := exec.Command(st.profile.Path + ".unsafe")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		st.log.Warning("Failed to create stdout pipe: %v", err)
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		st.log.Warning("Failed to create stderr pipe: %v", err)
		return
	}
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = &syscall.Credential{
		Uid: uint32(st.uid),
		Gid: uint32(st.gid),
	}
	cmd.Env = []string{
		fmt.Sprintf("DISPLAY=:%d", st.display),
	}
	if err := cmd.Start(); err != nil {
		st.log.Warning("Failed to start application (%s): %v", st.profile.Path, err)
		return
	}
	go st.readApplicationOutput(stdout, "stdout")
	go st.readApplicationOutput(stderr, "stderr")
}

func (st *initState) readApplicationOutput(r io.ReadCloser, label string) {
	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		st.log.Debug("(%s) %s", label, line)
	}
}

func loadProfile(dir, name string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(dir)
	if err != nil {
		return nil, err
	}
	for _, p := range ps {
		if name == p.Name {
			return p, nil
		}
	}
	return nil, fmt.Errorf("no profile named '%s'", name)
}

func handlePing(ping *PingMsg, msg *ipc.Message) error {
	return msg.Respond(&PingMsg{Data: ping.Data})
}

func (st *initState) handleRunShell(rs *RunShellMsg, msg *ipc.Message) error {
	if msg.Ucred == nil {
		return msg.Respond(&ErrorMsg{"No credentials received for RunShell command"})
	}
	if (msg.Ucred.Uid == 0 || msg.Ucred.Gid == 0) && st.config.AllowRootShell != true {
		return msg.Respond(&ErrorMsg{"Cannot open shell because allowRootShell is disabled"})
	}
	st.log.Info("Starting shell with uid = %d, gid = %d", msg.Ucred.Uid, msg.Ucred.Gid)
	cmd := exec.Command(st.config.ShellPath, "-i")
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = &syscall.Credential{
		Uid: msg.Ucred.Uid,
		Gid: msg.Ucred.Gid,
	}
	if rs.Term != "" {
		cmd.Env = append(cmd.Env, "TERM="+rs.Term)
	}
	if msg.Ucred.Uid != 0 && msg.Ucred.Gid != 0 {
		if homedir, _ := st.fs.GetHomeDir(); homedir != "" {
			cmd.Dir = homedir
			cmd.Env = append(cmd.Env, "HOME="+homedir)
		}
	}
	cmd.Env = append(cmd.Env, "PATH=/usr/bin:/bin")
	cmd.Env = append(cmd.Env, fmt.Sprintf("PS1=[%s] $ ", st.profile.Name))
	st.log.Info("Executing shell...")
	f, err := ptyStart(cmd)
	defer f.Close()
	if err != nil {
		return msg.Respond(&ErrorMsg{err.Error()})
	}
	err = msg.Respond(&OkMsg{}, int(f.Fd()))
	return err
}

func ptyStart(c *exec.Cmd) (ptty *os.File, err error) {
	ptty, tty, err := pty.Open()
	if err != nil {
		return nil, err
	}
	defer tty.Close()
	c.Stdin = tty
	c.Stdout = tty
	c.Stderr = tty
	if c.SysProcAttr == nil {
		c.SysProcAttr = &syscall.SysProcAttr{}
	}
	c.SysProcAttr.Setctty = true
	c.SysProcAttr.Setsid = true
	if err := c.Start(); err != nil {
		ptty.Close()
		return nil, err
	}
	return ptty, nil
}

func (is *initState) handleChildExit(pid int, wstatus syscall.WaitStatus) {
	is.log.Debug("Child process pid=%d exited with status %d", pid, wstatus.ExitStatus())
}
<commit_msg>When entering a shell, if available, set the display number in the environment<commit_after>package ozinit

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/subgraph/oz"
	"github.com/subgraph/oz/fs"
	"github.com/subgraph/oz/ipc"
	"github.com/subgraph/oz/xpra"
	"github.com/subgraph/oz/network"

	"github.com/kr/pty"
	"github.com/op/go-logging"
)

const SocketAddress = "/tmp/oz-init-control"

type initState struct {
	log       *logging.Logger
	profile   *oz.Profile
	config    *oz.Config
	uid       int
	gid       int
	user      *user.User
	display   int
	fs        *fs.Filesystem
	xpraReady sync.WaitGroup
	network   *network.SandboxNetwork
}

// By convention oz-init writes log messages to stderr with a single character
// prefix indicating the logging level.  These messages are read one line at a time
// over a pipe by oz-daemon and translated into appropriate log events.
func createLogger() *logging.Logger {
	l := logging.MustGetLogger("oz-init")
	be := logging.NewLogBackend(os.Stderr, "", 0)
	f := logging.MustStringFormatter("%{level:.1s} %{message}")
	fbe := logging.NewBackendFormatter(be, f)
	logging.SetBackend(fbe)
	return l
}

func Main() {
	parseArgs().runInit()
}

func parseArgs() *initState {
	log := createLogger()
	getvar := func(name string) string {
		val := os.Getenv(name)
		if val == "" {
			log.Error("Error: missing required '%s' argument", name)
			os.Exit(1)
		}
		return val
	}
	pname := getvar("INIT_PROFILE")
	uidval := getvar("INIT_UID")
	dispval := os.Getenv("INIT_DISPLAY")

	stnip := os.Getenv("INIT_ADDR")
	stnvhost := os.Getenv("INIT_VHOST")
	stnvguest := os.Getenv("INIT_VGUEST")
	stngateway := os.Getenv("INIT_GATEWAY")

	var config *oz.Config
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		log.Info("Could not load config file (%s), using default config", oz.DefaultConfigPath)
		config = oz.NewDefaultConfig()
	}

	p, err := loadProfile(config.ProfileDir, pname)
	if err != nil {
		log.Error("Could not load profile %s: %v", pname, err)
		os.Exit(1)
	}
	uid, err := strconv.Atoi(uidval)
	if err != nil {
		log.Error("Could not parse INIT_UID argument (%s) into an integer: %v", uidval, err)
		os.Exit(1)
	}
	u, err := user.LookupId(uidval)
	if err != nil {
		log.Error("Failed to look up user with uid=%s: %v", uidval, err)
		os.Exit(1)
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		log.Error("Failed to parse gid value (%s) from user struct: %v", u.Gid, err)
		os.Exit(1)
	}
	display := 0
	if dispval != "" {
		d, err := strconv.Atoi(dispval)
		if err != nil {
			log.Error("Unable to parse display (%s) into an integer: %v", dispval, err)
			os.Exit(1)
		}
		display = d
	}

	stn := new(network.SandboxNetwork)
	if stnip != "" {
		gateway, _, err := net.ParseCIDR(stngateway)
		if err != nil {
			log.Error("Unable to parse network configuration gateway (%s): %v", stngateway, err)
			os.Exit(1)
		}

		stn.Ip        = stnip
		stn.VethHost  = stnvhost
		stn.VethGuest = stnvguest
		stn.Gateway   = gateway
	}

	return &initState{
		log:     log,
		config:  config,
		profile: p,
		uid:     uid,
		gid:     gid,
		user:    u,
		display: display,
		fs:      fs.NewFromProfile(p, u, config.SandboxPath, config.UseFullDev, log),
		network: stn,
	}
}

func (st *initState) runInit() {
	st.log.Info("Starting oz-init for profile: %s", st.profile.Name)

	if st.profile.Networking.Nettype != "host" {
		err := network.NetSetup(st.network)
		if err != nil {
			st.log.Error("Unable to setup networking: %+v", err)
			os.Exit(1)
		}
	}
	network.NetPrint(st.log)

	if syscall.Sethostname([]byte(st.profile.Name)) != nil {
		st.log.Error("Failed to set hostname to (%s)", st.profile.Name)
	}
	st.log.Info("Hostname set to (%s)", st.profile.Name)

	if err := st.fs.OzInit(); err != nil {
		st.log.Error("Error: setting up filesystem failed: %v\n", err)
		os.Exit(1)
	}
	oz.ReapChildProcs(st.log, st.handleChildExit)

	if st.profile.XServer.Enabled {
		st.xpraReady.Add(1)
		st.startXpraServer()
	}
	st.xpraReady.Wait()
	st.launchApplication()

	s, err := ipc.NewServer(SocketAddress, messageFactory, st.log,
		handlePing,
		st.handleRunShell,
	)
	if err != nil {
		st.log.Error("NewServer failed: %v", err)
		os.Exit(1)
	}
	if err := os.Chown(SocketAddress, st.uid, st.gid); err != nil {
		st.log.Warning("Failed to chown oz-init control socket: %v", err)
	}
	os.Stderr.WriteString("OK\n")

	if err := s.Run(); err != nil {
		st.log.Warning("MsgServer.Run() return err: %v", err)
	}
	st.log.Info("oz-init exiting...")
}

func (st *initState) startXpraServer() {
	workdir := st.fs.Xpra()
	if workdir == "" {
		st.log.Warning("Xpra work directory not set")
		return
	}
	xpra := xpra.NewServer(&st.profile.XServer, uint64(st.display), workdir)
	p, err := xpra.Process.StderrPipe()
	if err != nil {
		st.log.Warning("Error creating stderr pipe for xpra output: %v", err)
		os.Exit(1)
	}
	go st.readXpraOutput(p)
	xpra.Process.Env = []string{
		"HOME=" + st.user.HomeDir,
	}
	xpra.Process.SysProcAttr = &syscall.SysProcAttr{}
	xpra.Process.SysProcAttr.Credential = &syscall.Credential{
		Uid: uint32(st.uid),
		Gid: uint32(st.gid),
	}
	st.log.Info("Starting xpra server")
	if err := xpra.Process.Start(); err != nil {
		st.log.Warning("Failed to start xpra server: %v", err)
	}
}

func (st *initState) readXpraOutput(r io.ReadCloser) {
	sc := bufio.NewScanner(r)
	seenReady := false
	for sc.Scan() {
		line := sc.Text()
		if len(line) > 0 {
			if strings.Contains(line, "xpra is ready.") && !seenReady {
				seenReady = true
				st.xpraReady.Done()
				if !st.config.LogXpra {
					r.Close()
					return
				}
			}
			if st.config.LogXpra {
				st.log.Debug("(xpra) %s", line)
			}
		}
	}
}

func (st *initState) launchApplication() {
	cmd := exec.Command(st.profile.Path + ".unsafe")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		st.log.Warning("Failed to create stdout pipe: %v", err)
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		st.log.Warning("Failed to create stderr pipe: %v", err)
		return
	}
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = &syscall.Credential{
		Uid: uint32(st.uid),
		Gid: uint32(st.gid),
	}
	cmd.Env = []string{
		fmt.Sprintf("DISPLAY=:%d", st.display),
	}
	if err := cmd.Start(); err != nil {
		st.log.Warning("Failed to start application (%s): %v", st.profile.Path, err)
		return
	}
	go st.readApplicationOutput(stdout, "stdout")
	go st.readApplicationOutput(stderr, "stderr")
}

func (st *initState) readApplicationOutput(r io.ReadCloser, label string) {
	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		st.log.Debug("(%s) %s", label, line)
	}
}

func loadProfile(dir, name string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(dir)
	if err != nil {
		return nil, err
	}
	for _, p := range ps {
		if name == p.Name {
			return p, nil
		}
	}
	return nil, fmt.Errorf("no profile named '%s'", name)
}

func handlePing(ping *PingMsg, msg *ipc.Message) error {
	return msg.Respond(&PingMsg{Data: ping.Data})
}

func (st *initState) handleRunShell(rs *RunShellMsg, msg *ipc.Message) error {
	if msg.Ucred == nil {
		return msg.Respond(&ErrorMsg{"No credentials received for RunShell command"})
	}
	if (msg.Ucred.Uid == 0 || msg.Ucred.Gid == 0) && st.config.AllowRootShell != true {
		return msg.Respond(&ErrorMsg{"Cannot open shell because allowRootShell is disabled"})
	}
	st.log.Info("Starting shell with uid = %d, gid = %d", msg.Ucred.Uid, msg.Ucred.Gid)
	cmd := exec.Command(st.config.ShellPath, "-i")
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = &syscall.Credential{
		Uid: msg.Ucred.Uid,
		Gid: msg.Ucred.Gid,
	}
	if rs.Term != "" {
		cmd.Env = append(cmd.Env, "TERM="+rs.Term)
	}
	if msg.Ucred.Uid != 0 && msg.Ucred.Gid != 0 {
		if homedir, _ := st.fs.GetHomeDir(); homedir != "" {
			cmd.Dir = homedir
			cmd.Env = append(cmd.Env, "HOME="+homedir)
		}
	}
	if st.profile.XServer.Enabled {
		cmd.Env = append(cmd.Env, "DISPLAY=:" + strconv.Itoa(st.display))
	}
	cmd.Env = append(cmd.Env, "PATH=/usr/bin:/bin")
	cmd.Env = append(cmd.Env, fmt.Sprintf("PS1=[%s] $ ", st.profile.Name))
	st.log.Info("Executing shell...")
	f, err := ptyStart(cmd)
	defer f.Close()
	if err != nil {
		return msg.Respond(&ErrorMsg{err.Error()})
	}
	err = msg.Respond(&OkMsg{}, int(f.Fd()))
	return err
}

func ptyStart(c *exec.Cmd) (ptty *os.File, err error) {
	ptty, tty, err := pty.Open()
	if err != nil {
		return nil, err
	}
	defer tty.Close()
	c.Stdin = tty
	c.Stdout = tty
	c.Stderr = tty
	if c.SysProcAttr == nil {
		c.SysProcAttr = &syscall.SysProcAttr{}
	}
	c.SysProcAttr.Setctty = true
	c.SysProcAttr.Setsid = true
	if err := c.Start(); err != nil {
		ptty.Close()
		return nil, err
	}
	return ptty, nil
}

func (is *initState) handleChildExit(pid int, wstatus syscall.WaitStatus) {
	is.log.Debug("Child process pid=%d exited with status %d", pid, wstatus.ExitStatus())
}
<|endoftext|>