<commit_before>/* A small utility to convert RIFF wave audio files into C-arrays.
 * written in 2013 by tpltnt
 * license: MIT
 */

package main

import(
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
)

// check slices for equality
func AreEqualSlices(a,b []byte) bool {
     // check for same length
     if len(a) != len(b) { return false }
     // check for same capacity
     if cap(a) != cap(b) { return false }
     // loop over a-slice & compare values
     for i := 0; i < len(b); i++ {
         if a[i] != b[i] { return false }
     }
     // same length, capacity & values
     return true
}

// Check header for given Wave-file
func IsWavefileHeaderOk(reader *bufio.Reader) bool {
     // create buffer for 4 bytes
     chunkID := make([]byte, 4)
     // try to read first 4 bytes
     bytesread, err := reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with RIFF header bytes
     riffstring := []byte{0x52, 0x49, 0x46, 0x46}
     // compare slices using equality-test function
     if !AreEqualSlices(riffstring,chunkID) { return false }

     // throw away next 4 bytes
     bytesread, err = reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }

     // read WAVE-header
     bytesread, err = reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with WAVE header bytes
     wavestring := []byte{0x57, 0x41, 0x56, 0x45}
     // compare slices using equality-test function
     if !AreEqualSlices(wavestring,chunkID) { return false }

     return true
}

// extract file format information
// assumes to be in beginning of (data) chunk
func IsWavefileFormatOk(reader *bufio.Reader) bool {
     // create buffer for 4 bytes
     chunkID := make([]byte, 4)
     //-- try to read first 4 bytes
     bytesread, err := reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF {
        log.Println("error reading chunk ID")
        return false
     }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with fmt bytes
     fmtstring := []byte{0x66, 0x6d, 0x74, 0x20}
     // compare slices using equality-test function
     if !AreEqualSlices(fmtstring,chunkID) {
        log.Println("wrong chunk ID")
        return false
     }

     fourbytes := make([]byte, 4)
     twobytes := make([]byte, 2)
     //-- read data size
     bytesread, err = reader.Read(fourbytes)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF {
        log.Println("error reading data size")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for data size")
     }
     // 16 bytes (little endian encoded)
     datachunksize := []byte{0x10, 0x00, 0x00, 0x00}
     if !AreEqualSlices(fourbytes,datachunksize) {
        log.Println("wrong data chunk size")
	return false
     }

     //-- read wFormatTag
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wFormatTag")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wFormatTag")
        return false
     }
     // PCM = 0x01 0x00 (little endian for 1)
     if !(twobytes[0] == 0x01 && twobytes [1] == 0x00) {
        log.Println("no PCM found")
        return false
     }

     //-- read wChannels
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wChannels")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wChannels")
        return false
     }
     // PCM = 0x01 0x00 (little endian for 1)
     if !(twobytes[0] == 0x01 && twobytes [1] == 0x00) {
        log.Println("no mono file found")
        return false
     }

     //-- read dwSamplesPerSec 4bytes
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading dwSamplesPerSec")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for dwSamplesPerSec")
     }
     // need to be 8 khz
     samplerate := []byte{0x40, 0x1f, 0x00, 0x00}
     if !(AreEqualSlices(fourbytes,samplerate)) {
        log.Println("not 8khz sample rate")
        return false
     }

     //-- read dwAvgBytesPerSec 4bytes (ignore)
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading dwAvgBytesPerSec")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for dwAvgBytesPerSec")
     }

     //-- read wBlockAlign (ignore, since we convert)
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wBlockAlign")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wBlockAlign")
        return false
     }

     //-- nBitsPerSample
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading nBitsPerSample")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for nBitsPerSample")
        return false
     }
     // need to be 8bit
     if !(0x08 == twobytes[0] && 0x00 == twobytes[1]) {
        log.Println("not 8 bit sample depth")
        return false
     }

     return true
}

// convert byte to integer
func Byte2Int(data []byte) uint8 {
     if 1 != len(data) { panic("more than one byte given") }
     buf := bytes.NewBuffer(data)
     var value uint8
     binary.Read(buf, binary.LittleEndian, &value)
     return value
}

// convert 4 bytes to uint32
func Bytes2Uint32(data []byte) uint32 {
     if 4 != len(data) { panic("not 4 byte given") }
     buf := bytes.NewBuffer(data)
     var value uint32
     binary.Read(buf, binary.LittleEndian, &value)
     return value
}

// save data in file with given name
func ConvertData(reader *bufio.Reader, filename string) bool {
     // check for data chunk
     fourbytes := make([]byte,4)
     bytesread, err := reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading data chunk ID")
        return false
     }
     if bytesread != 4 {
     	log.Println("not enough bytes read for data chunk ID")
	return false
     }
     datachunkid := []byte{0x64, 0x61, 0x74, 0x61}
     if !(AreEqualSlices(fourbytes,datachunkid)) {
     	log.Println("no data chunk ID found")
	return false
     }

     //-- get data size
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading data chunk size")
        return false
     }
     if bytesread != 4 {
        log.Println("not enough bytes read for data chunk size")
        return false
     }

     //-- write file
     // C-header foo
     headerfile, err := os.Create(filename)
     defer headerfile.Close()
     if err != nil { panic(err) }
     headerwriter := bufio.NewWriterSize(headerfile,4096)
     // write array length (fourbytes -> 32bits unsigned)
     writestring := "const long pcm_length = "
     byteswritten, err := headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
     	log.Println("error writing bytes to C-header file")
	return false
     }
     writestring = fmt.Sprintf("%d",Bytes2Uint32(fourbytes))
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     writestring = ";\n"
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     // actually write data to disk
     if err = headerwriter.Flush(); err != nil { panic(err) }
     // write data array
     writestring = "const unsigned char pcm_samples[] PROGMEM = {"
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     if err = headerwriter.Flush(); err != nil { panic(err) }

     databyte := make([]byte,1)
     bytesread, err = reader.Read(databyte)
     if err != nil { panic(err) }
     if 1 != bytesread {
     	log.Println("couldn't read sample data byte")
	return false
     }
     // create 8x8 blocks of samples
     samplebyte := 0	  // samples per line
     blockcounter := 0	  // sample lines per block
     // TODO: take number of samples into account
     for err != io.EOF {
     	 // write data
     	 writestring = fmt.Sprintf("%d",databyte[0])
     	 byteswritten, err = headerwriter.WriteString(writestring)
	 if err != nil { panic(err) }
    	 if len(writestring) != byteswritten {
            log.Println("error writing sample bytes to C-header file")
            return false
     	 }
	 if err = headerwriter.Flush(); err != nil { panic(err) }
	 // make break after 8 samples are written
	 samplebyte = samplebyte + 1
	 if 8 == samplebyte {
	    writestring = ",\n"
	    samplebyte = 0
	    blockcounter = blockcounter + 1
	    if 8 == blockcounter {
	       writestring = ",\n\n"
	       blockcounter = 0
	       if err = headerwriter.Flush(); err != nil { panic(err) }
	    }
	 } else {
	    writestring = ", "
	 }
         byteswritten, err = headerwriter.WriteString(writestring)
         if err != nil { panic(err) }
         if len(writestring) != byteswritten {
            log.Println("error writing sample bytes to C-header file")
            return false
         }
	 // read data for writing
	 bytesread, err = reader.Read(databyte)
	 if (err != nil) && (err != io.EOF) { panic(err) }
     	 if 1 != bytesread {
            log.Println("couldn't read sample data byte")
            return false
     	 }
     }
     // close structure
     writestring = "\n};\n"
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
     	log.Println("error writing closing structure to C-header file")
        return false
     }
     if err = headerwriter.Flush(); err != nil { panic(err) }

     return true
}

func main() {
     // check commandline-arguments
     if 2 != len(os.Args) { panic("wrong number of arguments") }
     // open file, get error
     wavefile, err := os.Open(os.Args[1])
     // if error -> panic
     if err != nil { panic(err) }
     // close file when main returns
     defer wavefile.Close()
     // create new reader from file
     reader := bufio.NewReader(wavefile)
     // check file header
     if !IsWavefileHeaderOk(reader) { panic("file header b0rked") }
     fmt.Print("file format is ")
     if !IsWavefileFormatOk(reader) {
        fmt.Print("not ")
     }
     fmt.Println("ok.")
     ConvertData(reader,"test.c")
}<commit_msg>nicer sample data output format<commit_after>/* A small utility to convert RIFF wave audio files into C-arrays.
 * written in 2013 by tpltnt
 * license: MIT
 */

package main

import(
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
)

// check slices for equality
func AreEqualSlices(a,b []byte) bool {
     // check for same length
     if len(a) != len(b) { return false }
     // check for same capacity
     if cap(a) != cap(b) { return false }
     // loop over a-slice & compare values
     for i := 0; i < len(b); i++ {
         if a[i] != b[i] { return false }
     }
     // same length, capacity & values
     return true
}

// Check header for given Wave-file
func IsWavefileHeaderOk(reader *bufio.Reader) bool {
     // create buffer for 4 bytes
     chunkID := make([]byte, 4)
     // try to read first 4 bytes
     bytesread, err := reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with RIFF header bytes
     riffstring := []byte{0x52, 0x49, 0x46, 0x46}
     // compare slices using equality-test function
     if !AreEqualSlices(riffstring,chunkID) { return false }

     // throw away next 4 bytes
     bytesread, err = reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }

     // read WAVE-header
     bytesread, err = reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF { return false }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with WAVE header bytes
     wavestring := []byte{0x57, 0x41, 0x56, 0x45}
     // compare slices using equality-test function
     if !AreEqualSlices(wavestring,chunkID) { return false }

     return true
}

// extract file format information
// assumes to be in beginning of (data) chunk
func IsWavefileFormatOk(reader *bufio.Reader) bool {
     // create buffer for 4 bytes
     chunkID := make([]byte, 4)
     //-- try to read first 4 bytes
     bytesread, err := reader.Read(chunkID)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF {
        log.Println("error reading chunk ID")
        return false
     }
     // panic if less than 4 bytes read
     if bytesread < 4 { return false }
     // create slice with fmt bytes
     fmtstring := []byte{0x66, 0x6d, 0x74, 0x20}
     // compare slices using equality-test function
     if !AreEqualSlices(fmtstring,chunkID) {
        log.Println("wrong chunk ID")
        return false
     }

     fourbytes := make([]byte, 4)
     twobytes := make([]byte, 2)
     //-- read data size
     bytesread, err = reader.Read(fourbytes)
     // panic if no error (and EOF)
     if err != nil && err != io.EOF {
        log.Println("error reading data size")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for data size")
     }
     // 16 bytes (little endian encoded)
     datachunksize := []byte{0x10, 0x00, 0x00, 0x00}
     if !AreEqualSlices(fourbytes,datachunksize) {
        log.Println("wrong data chunk size")
	return false
     }

     //-- read wFormatTag
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wFormatTag")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wFormatTag")
        return false
     }
     // PCM = 0x01 0x00 (little endian for 1)
     if !(twobytes[0] == 0x01 && twobytes [1] == 0x00) {
        log.Println("no PCM found")
        return false
     }

     //-- read wChannels
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wChannels")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wChannels")
        return false
     }
     // PCM = 0x01 0x00 (little endian for 1)
     if !(twobytes[0] == 0x01 && twobytes [1] == 0x00) {
        log.Println("no mono file found")
        return false
     }

     //-- read dwSamplesPerSec 4bytes
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading dwSamplesPerSec")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for dwSamplesPerSec")
     }
     // need to be 8 khz
     samplerate := []byte{0x40, 0x1f, 0x00, 0x00}
     if !(AreEqualSlices(fourbytes,samplerate)) {
        log.Println("not 8khz sample rate")
        return false
     }

     //-- read dwAvgBytesPerSec 4bytes (ignore)
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading dwAvgBytesPerSec")
        return false
     }
     if 4 != bytesread {
        log.Println("not enough bytes read for dwAvgBytesPerSec")
     }

     //-- read wBlockAlign (ignore, since we convert)
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading wBlockAlign")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for wBlockAlign")
        return false
     }

     //-- nBitsPerSample
     bytesread, err = reader.Read(twobytes)
     if err != nil && err != io.EOF {
        log.Println("error reading nBitsPerSample")
        return false
     }
     if 2 != bytesread {
        log.Println("not enough bytes read for nBitsPerSample")
        return false
     }
     // need to be 8bit
     if !(0x08 == twobytes[0] && 0x00 == twobytes[1]) {
        log.Println("not 8 bit sample depth")
        return false
     }

     return true
}

// convert byte to integer
func Byte2Int(data []byte) uint8 {
     if 1 != len(data) { panic("more than one byte given") }
     buf := bytes.NewBuffer(data)
     var value uint8
     binary.Read(buf, binary.LittleEndian, &value)
     return value
}

// convert 4 bytes to uint32
func Bytes2Uint32(data []byte) uint32 {
     if 4 != len(data) { panic("not 4 byte given") }
     buf := bytes.NewBuffer(data)
     var value uint32
     binary.Read(buf, binary.LittleEndian, &value)
     return value
}

// save data in file with given name
func ConvertData(reader *bufio.Reader, filename string) bool {
     // check for data chunk
     fourbytes := make([]byte,4)
     bytesread, err := reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading data chunk ID")
        return false
     }
     if bytesread != 4 {
     	log.Println("not enough bytes read for data chunk ID")
	return false
     }
     datachunkid := []byte{0x64, 0x61, 0x74, 0x61}
     if !(AreEqualSlices(fourbytes,datachunkid)) {
     	log.Println("no data chunk ID found")
	return false
     }

     //-- get data size
     bytesread, err = reader.Read(fourbytes)
     if err != nil && err != io.EOF {
        log.Println("error reading data chunk size")
        return false
     }
     if bytesread != 4 {
        log.Println("not enough bytes read for data chunk size")
        return false
     }

     //-- write file
     // C-header foo
     headerfile, err := os.Create(filename)
     defer headerfile.Close()
     if err != nil { panic(err) }
     headerwriter := bufio.NewWriterSize(headerfile,4096)
     // write array length (fourbytes -> 32bits unsigned)
     writestring := "const long pcm_length = "
     byteswritten, err := headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
     	log.Println("error writing bytes to C-header file")
	return false
     }
     writestring = fmt.Sprintf("%d",Bytes2Uint32(fourbytes))
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     writestring = ";\n"
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     // actually write data to disk
     if err = headerwriter.Flush(); err != nil { panic(err) }
     // write data array
     writestring = "const unsigned char pcm_samples[] PROGMEM = {\n    "
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
        log.Println("error writing bytes to C-header file")
        return false
     }
     if err = headerwriter.Flush(); err != nil { panic(err) }

     databyte := make([]byte,1)
     bytesread, err = reader.Read(databyte)
     if err != nil { panic(err) }
     if 1 != bytesread {
     	log.Println("couldn't read sample data byte")
	return false
     }
     // create 8x8 blocks of samples
     samplebyte := 0	  // samples per line
     blockcounter := 0	  // sample lines per block
     // TODO: take number of samples into account
     for err != io.EOF {
     	 // write data
     	 writestring = fmt.Sprintf("%d",databyte[0])
     	 byteswritten, err = headerwriter.WriteString(writestring)
	 if err != nil { panic(err) }
    	 if len(writestring) != byteswritten {
            log.Println("error writing sample bytes to C-header file")
            return false
     	 }
	 if err = headerwriter.Flush(); err != nil { panic(err) }
	 // make break after 8 samples are written
	 samplebyte = samplebyte + 1
	 if 8 == samplebyte {
	    writestring = ",\n    "
	    samplebyte = 0
	    blockcounter = blockcounter + 1
	    if 8 == blockcounter {
	       writestring = ",\n\n    "
	       blockcounter = 0
	       if err = headerwriter.Flush(); err != nil { panic(err) }
	    }
	 } else {
	    writestring = ", "
	 }
         byteswritten, err = headerwriter.WriteString(writestring)
         if err != nil { panic(err) }
         if len(writestring) != byteswritten {
            log.Println("error writing sample bytes to C-header file")
            return false
         }
	 // read data for writing
	 bytesread, err = reader.Read(databyte)
	 if (err != nil) && (err != io.EOF) { panic(err) }
     	 if 1 != bytesread {
            log.Println("couldn't read sample data byte")
            return false
     	 }
     }
     // close structure
     writestring = "\n};\n"
     byteswritten, err = headerwriter.WriteString(writestring)
     if err != nil { panic(err) }
     if len(writestring) != byteswritten {
     	log.Println("error writing closing structure to C-header file")
        return false
     }
     if err = headerwriter.Flush(); err != nil { panic(err) }

     return true
}

func main() {
     // check commandline-arguments
     if 2 != len(os.Args) { panic("wrong number of arguments") }
     // open file, get error
     wavefile, err := os.Open(os.Args[1])
     // if error -> panic
     if err != nil { panic(err) }
     // close file when main returns
     defer wavefile.Close()
     // create new reader from file
     reader := bufio.NewReader(wavefile)
     // check file header
     if !IsWavefileHeaderOk(reader) { panic("file header b0rked") }
     fmt.Print("file format is ")
     if !IsWavefileFormatOk(reader) {
        fmt.Print("not ")
     }
     fmt.Println("ok.")
     ConvertData(reader,"test.c")
}<|endoftext|>


<commit_before>package drain

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-community/splunk-firehose-nozzle/splunk"
)

//LoggingSplunk
//logger:        Logging client as implemented in lager
//client:        Splunk Client as implemented in splunk/splunk_client
//flushWindow:   time in nanoseconds representing flush window
//events:        channel implementation of map[strings] to store events for Splunk to index
type LoggingSplunk struct {
	logger      lager.Logger
	client      splunk.SplunkClient
	flushWindow time.Duration
	events      chan map[string]interface{}
}

//"constructor" for NewLoggingSplunk
func NewLoggingSplunk(logger lager.Logger, splunkClient splunk.SplunkClient, flushWindow time.Duration) *LoggingSplunk {
	return &LoggingSplunk{
		logger:      logger,
		client:      splunkClient,
		flushWindow: flushWindow,
		// FIXME, make buffer size 100 configurable
		events: make(chan map[string]interface{}, 100),
	}
}

//Connect implements "firehose-to-syslog.logging.logging" by calling LoggingSplunk.consume() on LoggingSplunk object
func (l *LoggingSplunk) Connect() bool {
	go l.consume()

	return true
}

//ShipEvents implements "firehose-to-syslog.logging.logging" by buildingEvents and sending them to LoggingSplunk.events
func (l *LoggingSplunk) ShipEvents(fields map[string]interface{}, msg string) {
	event := l.buildEvent(fields, msg)
	l.events <- event
}

//consume function will send events through to indexEvents method.
// trigger on 2 scenarios:
// 1: BatchSize count reached
// 2: flushWindow Ticker timer reached
func (l *LoggingSplunk) consume() {
	var batch []map[string]interface{}
	// FIXME, make batchSize configurable
	batchSize := 50
	tickChan := time.NewTicker(l.flushWindow).C
	// Either flush window or batch size reach limits, we flush
	for {
		select {
		case event := <-l.events:
			batch = append(batch, event)
			if len(batch) >= batchSize {
				l.logger.Info("Index Events triggered by Batch Limit")
				//reset channel timer
				tickChan = time.NewTicker(l.flushWindow).C
				batch = l.indexEvents(batch)
			}
		case <-tickChan:
			l.logger.Info("Index Events triggered by Flush Window Time Expiry")
			batch = l.indexEvents(batch)
		}
	}
}

// indexEvents indexes events to Splunk
// return nil when successful which clears all outstanding events
// return what the batch has if there is an error for next retry cycle
func (l *LoggingSplunk) indexEvents(batch []map[string]interface{}) []map[string]interface{} {
	if len(batch) == 0 {
		return batch
	}

	l.logger.Info(fmt.Sprintf("Posting %d events", len(batch)))
	err := l.client.Post(batch)
	if err != nil {
		l.logger.Error("Unable to talk to Splunk, error=%+v", err)
		// return back the batch for next retry
		return batch
	}

	return nil
}

// buildEvent constructs a splunk event from fields and msg parameter
// returns constructed event in event variable
func (l *LoggingSplunk) buildEvent(fields map[string]interface{}, msg string) map[string]interface{} {
	if len(msg) > 0 {
		fields["msg"] = msg
	}
	event := map[string]interface{}{}

	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
	if val, ok := fields["timestamp"]; ok {
		timestamp = l.nanoSecondsToSeconds(val.(int64))
	}
	event["time"] = timestamp

	event["host"] = fields["ip"]
	event["source"] = fields["job"]

	eventType := strings.ToLower(fields["event_type"].(string))
	event["sourcetype"] = fmt.Sprintf("cf:%s", eventType)

	event["event"] = fields

	return event
}

// nanoSecondsToSeconds is a simple helper function to convert nanoSeconds to Seconds
func (l *LoggingSplunk) nanoSecondsToSeconds(nanoseconds int64) string {
	seconds := float64(nanoseconds) * math.Pow(1000, -3)
	return fmt.Sprintf("%.3f", seconds)
}
<commit_msg>Cleaned up comments after review<commit_after>package drain

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-community/splunk-firehose-nozzle/splunk"
)

type LoggingSplunk struct {
	logger      lager.Logger
	client      splunk.SplunkClient
	flushWindow time.Duration               //time in nanoseconds representing flush window
	events      chan map[string]interface{} //data structure to store events for Splunk to index
}

func NewLoggingSplunk(logger lager.Logger, splunkClient splunk.SplunkClient, flushWindow time.Duration) *LoggingSplunk {
	return &LoggingSplunk{
		logger:      logger,
		client:      splunkClient,
		flushWindow: flushWindow,
		// FIXME, make buffer size 100 configurable
		events: make(chan map[string]interface{}, 100),
	}
}

func (l *LoggingSplunk) Connect() bool {
	go l.consume()

	return true
}

func (l *LoggingSplunk) ShipEvents(fields map[string]interface{}, msg string) {
	event := l.buildEvent(fields, msg)
	l.events <- event
}

func (l *LoggingSplunk) consume() {
	var batch []map[string]interface{}
	// FIXME, make batchSize configurable
	batchSize := 50
	tickChan := time.NewTicker(l.flushWindow).C
	// if flush window or batch size reaches limits, flush
	for {
		select {
		case event := <-l.events:
			batch = append(batch, event)
			if len(batch) >= batchSize {
				l.logger.Info("Index Events triggered by Batch Limit")
				tickChan = time.NewTicker(l.flushWindow).C //reset channel timer
				batch = l.indexEvents(batch)
			}
		case <-tickChan:
			l.logger.Info("Index Events triggered by Flush Window Time Expiry")
			batch = l.indexEvents(batch)
		}
	}
}

func (l *LoggingSplunk) indexEvents(batch []map[string]interface{}) []map[string]interface{} {
	if len(batch) == 0 {
		return batch
	}

	l.logger.Info(fmt.Sprintf("Posting %d events", len(batch)))
	err := l.client.Post(batch)
	if err != nil {
		l.logger.Error("Unable to talk to Splunk, error=%+v", err)
		// return back the batch for next retry
		return batch
	}

	return nil
}

func (l *LoggingSplunk) buildEvent(fields map[string]interface{}, msg string) map[string]interface{} {
	if len(msg) > 0 {
		fields["msg"] = msg
	}
	event := map[string]interface{}{}

	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
	if val, ok := fields["timestamp"]; ok {
		timestamp = l.nanoSecondsToSeconds(val.(int64))
	}
	event["time"] = timestamp

	event["host"] = fields["ip"]
	event["source"] = fields["job"]

	eventType := strings.ToLower(fields["event_type"].(string))
	event["sourcetype"] = fmt.Sprintf("cf:%s", eventType)

	event["event"] = fields

	return event
}

func (l *LoggingSplunk) nanoSecondsToSeconds(nanoseconds int64) string {
	seconds := float64(nanoseconds) * math.Pow(1000, -3)
	return fmt.Sprintf("%.3f", seconds)
}
<|endoftext|>


<commit_before>package p2p

import (
	"bytes"
	"sync"
	"testing"
	"time"

	"github.com/tendermint/tendermint/binary"
	. "github.com/tendermint/tendermint/common"
)

type PeerMessage struct {
	PeerKey string
	Bytes   []byte
	Counter int
}

type TestReactor struct {
	mtx          sync.Mutex
	channels     []*ChannelDescriptor
	peersAdded   []*Peer
	peersRemoved []*Peer
	logMessages  bool
	msgsCounter  int
	msgsReceived map[byte][]PeerMessage
}

func NewTestReactor(channels []*ChannelDescriptor, logMessages bool) *TestReactor {
	return &TestReactor{
		channels:     channels,
		logMessages:  logMessages,
		msgsReceived: make(map[byte][]PeerMessage),
	}
}

func (tr *TestReactor) Start(sw *Switch) {
}

func (tr *TestReactor) Stop() {
}

func (tr *TestReactor) GetChannels() []*ChannelDescriptor {
	return tr.channels
}

func (tr *TestReactor) AddPeer(peer *Peer) {
	tr.mtx.Lock()
	defer tr.mtx.Unlock()
	tr.peersAdded = append(tr.peersAdded, peer)
}

func (tr *TestReactor) RemovePeer(peer *Peer, reason interface{}) {
	tr.mtx.Lock()
	defer tr.mtx.Unlock()
	tr.peersRemoved = append(tr.peersRemoved, peer)
}

func (tr *TestReactor) Receive(chId byte, peer *Peer, msgBytes []byte) {
	if tr.logMessages {
		tr.mtx.Lock()
		defer tr.mtx.Unlock()
		//fmt.Printf("Received: %X, %X\n", chId, msgBytes)
		tr.msgsReceived[chId] = append(tr.msgsReceived[chId], PeerMessage{peer.Key, msgBytes, tr.msgsCounter})
		tr.msgsCounter++
	}
}

//-----------------------------------------------------------------------------

// convenience method for creating bar switches connected to each other.
func makeSwitchPair(t testing.TB, initSwitch func(*Switch) *Switch) (*Switch, *Switch) {

	// Create bar switches that will be interconnected.
	s1 := initSwitch(NewSwitch())
	s2 := initSwitch(NewSwitch())

	// Create a listener for s1
	l := NewDefaultListener("tcp", ":8001", true)

	// Dial the listener & add the connection to s2.
	lAddr := l.ExternalAddress()
	connOut, err := lAddr.Dial()
	if err != nil {
		t.Fatalf("Could not connect to listener address %v", lAddr)
	} else {
		t.Logf("Created a connection to listener address %v", lAddr)
	}
	connIn, ok := <-l.Connections()
	if !ok {
		t.Fatalf("Could not get inbound connection from listener")
	}

	s1.AddPeerWithConnection(connIn, false)
	s2.AddPeerWithConnection(connOut, true)

	// Wait for things to happen, peers to get added...
	time.Sleep(100 * time.Millisecond)

	// Close the server, no longer needed.
	l.Stop()

	return s1, s2
}

func TestSwitches(t *testing.T) {
	s1, s2 := makeSwitchPair(t, func(sw *Switch) *Switch {
		// Make bar reactors of bar channels each
		sw.AddReactor("foo", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x00), Priority: 10},
			&ChannelDescriptor{Id: byte(0x01), Priority: 10},
		}, true)).Start(sw) // Start the reactor
		sw.AddReactor("bar", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x02), Priority: 10},
			&ChannelDescriptor{Id: byte(0x03), Priority: 10},
		}, true)).Start(sw) // Start the reactor
		return sw
	})
	defer s1.Stop()
	defer s2.Stop()

	// Lets send a message from s1 to s2.
	if s1.Peers().Size() != 1 {
		t.Errorf("Expected exactly 1 peer in s1, got %v", s1.Peers().Size())
	}
	if s2.Peers().Size() != 1 {
		t.Errorf("Expected exactly 1 peer in s2, got %v", s2.Peers().Size())
	}

	ch0Msg := "channel zero"
	ch1Msg := "channel foo"
	ch2Msg := "channel bar"

	s1.Broadcast(byte(0x00), ch0Msg)
	s1.Broadcast(byte(0x01), ch1Msg)
	s1.Broadcast(byte(0x02), ch2Msg)

	// Wait for things to settle...
	time.Sleep(5000 * time.Millisecond)

	// Check message on ch0
	ch0Msgs := s2.Reactor("foo").(*TestReactor).msgsReceived[byte(0x00)]
	if len(ch0Msgs) != 2 {
		t.Errorf("Expected to have received 1 message in ch0")
	}
	if !bytes.Equal(ch0Msgs[1].Bytes, binary.BinaryBytes(ch0Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch0Msg), ch0Msgs[0].Bytes)
	}

	// Check message on ch1
	ch1Msgs := s2.Reactor("foo").(*TestReactor).msgsReceived[byte(0x01)]
	if len(ch1Msgs) != 1 {
		t.Errorf("Expected to have received 1 message in ch1")
	}
	if !bytes.Equal(ch1Msgs[0].Bytes, binary.BinaryBytes(ch1Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch1Msg), ch1Msgs[0].Bytes)
	}

	// Check message on ch2
	ch2Msgs := s2.Reactor("bar").(*TestReactor).msgsReceived[byte(0x02)]
	if len(ch2Msgs) != 1 {
		t.Errorf("Expected to have received 1 message in ch2")
	}
	if !bytes.Equal(ch2Msgs[0].Bytes, binary.BinaryBytes(ch2Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch2Msg), ch2Msgs[0].Bytes)
	}

}

func BenchmarkSwitches(b *testing.B) {

	b.StopTimer()

	s1, s2 := makeSwitchPair(b, func(sw *Switch) *Switch {
		// Make bar reactors of bar channels each
		sw.AddReactor("foo", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x00), Priority: 10},
			&ChannelDescriptor{Id: byte(0x01), Priority: 10},
		}, false))
		sw.AddReactor("bar", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x02), Priority: 10},
			&ChannelDescriptor{Id: byte(0x03), Priority: 10},
		}, false))
		return sw
	})
	defer s1.Stop()
	defer s2.Stop()

	// Allow time for goroutines to boot up
	time.Sleep(1000 * time.Millisecond)
	b.StartTimer()

	numSuccess, numFailure := 0, 0

	// Send random message from foo channel to another
	for i := 0; i < b.N; i++ {
		chId := byte(i % 4)
		successChan := s1.Broadcast(chId, "test data")
		for s := range successChan {
			if s {
				numSuccess += 1
			} else {
				numFailure += 1
			}
		}
	}

	log.Warn(Fmt("success: %v, failure: %v", numSuccess, numFailure))

	// Allow everything to flush before stopping switches & closing connections.
	b.StopTimer()
	time.Sleep(1000 * time.Millisecond)

}
<commit_msg>comment fixes<commit_after>package p2p

import (
	"bytes"
	"sync"
	"testing"
	"time"

	"github.com/tendermint/tendermint/binary"
	. "github.com/tendermint/tendermint/common"
)

type PeerMessage struct {
	PeerKey string
	Bytes   []byte
	Counter int
}

type TestReactor struct {
	mtx          sync.Mutex
	channels     []*ChannelDescriptor
	peersAdded   []*Peer
	peersRemoved []*Peer
	logMessages  bool
	msgsCounter  int
	msgsReceived map[byte][]PeerMessage
}

func NewTestReactor(channels []*ChannelDescriptor, logMessages bool) *TestReactor {
	return &TestReactor{
		channels:     channels,
		logMessages:  logMessages,
		msgsReceived: make(map[byte][]PeerMessage),
	}
}

func (tr *TestReactor) Start(sw *Switch) {
}

func (tr *TestReactor) Stop() {
}

func (tr *TestReactor) GetChannels() []*ChannelDescriptor {
	return tr.channels
}

func (tr *TestReactor) AddPeer(peer *Peer) {
	tr.mtx.Lock()
	defer tr.mtx.Unlock()
	tr.peersAdded = append(tr.peersAdded, peer)
}

func (tr *TestReactor) RemovePeer(peer *Peer, reason interface{}) {
	tr.mtx.Lock()
	defer tr.mtx.Unlock()
	tr.peersRemoved = append(tr.peersRemoved, peer)
}

func (tr *TestReactor) Receive(chId byte, peer *Peer, msgBytes []byte) {
	if tr.logMessages {
		tr.mtx.Lock()
		defer tr.mtx.Unlock()
		//fmt.Printf("Received: %X, %X\n", chId, msgBytes)
		tr.msgsReceived[chId] = append(tr.msgsReceived[chId], PeerMessage{peer.Key, msgBytes, tr.msgsCounter})
		tr.msgsCounter++
	}
}

//-----------------------------------------------------------------------------

// convenience method for creating two switches connected to each other.
func makeSwitchPair(t testing.TB, initSwitch func(*Switch) *Switch) (*Switch, *Switch) {

	// Create two switches that will be interconnected.
	s1 := initSwitch(NewSwitch())
	s2 := initSwitch(NewSwitch())

	// Create a listener for s1
	l := NewDefaultListener("tcp", ":8001", true)

	// Dial the listener & add the connection to s2.
	lAddr := l.ExternalAddress()
	connOut, err := lAddr.Dial()
	if err != nil {
		t.Fatalf("Could not connect to listener address %v", lAddr)
	} else {
		t.Logf("Created a connection to listener address %v", lAddr)
	}
	connIn, ok := <-l.Connections()
	if !ok {
		t.Fatalf("Could not get inbound connection from listener")
	}

	s1.AddPeerWithConnection(connIn, false)
	s2.AddPeerWithConnection(connOut, true)

	// Wait for things to happen, peers to get added...
	time.Sleep(100 * time.Millisecond)

	// Close the server, no longer needed.
	l.Stop()

	return s1, s2
}

func TestSwitches(t *testing.T) {
	s1, s2 := makeSwitchPair(t, func(sw *Switch) *Switch {
		// Make two reactors of two channels each
		sw.AddReactor("foo", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x00), Priority: 10},
			&ChannelDescriptor{Id: byte(0x01), Priority: 10},
		}, true)).Start(sw) // Start the reactor
		sw.AddReactor("bar", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x02), Priority: 10},
			&ChannelDescriptor{Id: byte(0x03), Priority: 10},
		}, true)).Start(sw) // Start the reactor
		return sw
	})
	defer s1.Stop()
	defer s2.Stop()

	// Lets send a message from s1 to s2.
	if s1.Peers().Size() != 1 {
		t.Errorf("Expected exactly 1 peer in s1, got %v", s1.Peers().Size())
	}
	if s2.Peers().Size() != 1 {
		t.Errorf("Expected exactly 1 peer in s2, got %v", s2.Peers().Size())
	}

	ch0Msg := "channel zero"
	ch1Msg := "channel foo"
	ch2Msg := "channel bar"

	s1.Broadcast(byte(0x00), ch0Msg)
	s1.Broadcast(byte(0x01), ch1Msg)
	s1.Broadcast(byte(0x02), ch2Msg)

	// Wait for things to settle...
	time.Sleep(5000 * time.Millisecond)

	// Check message on ch0
	ch0Msgs := s2.Reactor("foo").(*TestReactor).msgsReceived[byte(0x00)]
	if len(ch0Msgs) != 2 {
		t.Errorf("Expected to have received 1 message in ch0")
	}
	if !bytes.Equal(ch0Msgs[1].Bytes, binary.BinaryBytes(ch0Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch0Msg), ch0Msgs[0].Bytes)
	}

	// Check message on ch1
	ch1Msgs := s2.Reactor("foo").(*TestReactor).msgsReceived[byte(0x01)]
	if len(ch1Msgs) != 1 {
		t.Errorf("Expected to have received 1 message in ch1")
	}
	if !bytes.Equal(ch1Msgs[0].Bytes, binary.BinaryBytes(ch1Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch1Msg), ch1Msgs[0].Bytes)
	}

	// Check message on ch2
	ch2Msgs := s2.Reactor("bar").(*TestReactor).msgsReceived[byte(0x02)]
	if len(ch2Msgs) != 1 {
		t.Errorf("Expected to have received 1 message in ch2")
	}
	if !bytes.Equal(ch2Msgs[0].Bytes, binary.BinaryBytes(ch2Msg)) {
		t.Errorf("Unexpected message bytes. Wanted: %X, Got: %X", binary.BinaryBytes(ch2Msg), ch2Msgs[0].Bytes)
	}

}

func BenchmarkSwitches(b *testing.B) {

	b.StopTimer()

	s1, s2 := makeSwitchPair(b, func(sw *Switch) *Switch {
		// Make bar reactors of bar channels each
		sw.AddReactor("foo", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x00), Priority: 10},
			&ChannelDescriptor{Id: byte(0x01), Priority: 10},
		}, false))
		sw.AddReactor("bar", NewTestReactor([]*ChannelDescriptor{
			&ChannelDescriptor{Id: byte(0x02), Priority: 10},
			&ChannelDescriptor{Id: byte(0x03), Priority: 10},
		}, false))
		return sw
	})
	defer s1.Stop()
	defer s2.Stop()

	// Allow time for goroutines to boot up
	time.Sleep(1000 * time.Millisecond)
	b.StartTimer()

	numSuccess, numFailure := 0, 0

	// Send random message from foo channel to another
	for i := 0; i < b.N; i++ {
		chId := byte(i % 4)
		successChan := s1.Broadcast(chId, "test data")
		for s := range successChan {
			if s {
				numSuccess += 1
			} else {
				numFailure += 1
			}
		}
	}

	log.Warn(Fmt("success: %v, failure: %v", numSuccess, numFailure))

	// Allow everything to flush before stopping switches & closing connections.
	b.StopTimer()
	time.Sleep(1000 * time.Millisecond)

}
<|endoftext|>


<commit_before>package packer

import (
	"fmt"
	"github.com/mitchellh/mapstructure"
	jsonutil "github.com/mitchellh/packer/common/json"
	"io/ioutil"
	"sort"
)

// The rawTemplate struct represents the structure of a template read
// directly from a file. The builders and other components map just to
// "interface{}" pointers since we actually don't know what their contents
// are until we read the "type" field.
type rawTemplate struct {
	Variables      map[string]string
	Builders       []map[string]interface{}
	Hooks          map[string][]string
	Provisioners   []map[string]interface{}
	PostProcessors []interface{} `mapstructure:"post-processors"`
}

// The Template struct represents a parsed template, parsed into the most
// completed form it can be without additional processing by the caller.
type Template struct {
	Variables      map[string]string
	Builders       map[string]rawBuilderConfig
	Hooks          map[string][]string
	PostProcessors [][]rawPostProcessorConfig
	Provisioners   []rawProvisionerConfig
}

// The rawBuilderConfig struct represents a raw, unprocessed builder
// configuration. It contains the name of the builder as well as the
// raw configuration. If requested, this is used to compile into a full
// builder configuration at some point.
type rawBuilderConfig struct {
	Name string
	Type string

	rawConfig interface{}
}

// rawPostProcessorConfig represents a raw, unprocessed post-processor
// configuration. It contains the type of the post processor as well as the
// raw configuration that is handed to the post-processor for it to process.
type rawPostProcessorConfig struct {
	Type              string
	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
	rawConfig         interface{}
}

// rawProvisionerConfig represents a raw, unprocessed provisioner configuration.
// It contains the type of the provisioner as well as the raw configuration
// that is handed to the provisioner for it to process.
type rawProvisionerConfig struct {
	Type     string
	Override map[string]interface{}

	rawConfig interface{}
}

// ParseTemplate takes a byte slice and parses a Template from it, returning
// the template and possibly errors while loading the template. The error
// could potentially be a MultiError, representing multiple errors. Knowing
// and checking for this can be useful, if you wish to format it in a certain
// way.
func ParseTemplate(data []byte) (t *Template, err error) {
	var rawTplInterface interface{}
	err = jsonutil.Unmarshal(data, &rawTplInterface)
	if err != nil {
		return
	}

	// Decode the raw template interface into the actual rawTemplate
	// structure, checking for any extranneous keys along the way.
	var md mapstructure.Metadata
	var rawTpl rawTemplate
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &rawTpl,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return
	}

	err = decoder.Decode(rawTplInterface)
	if err != nil {
		return
	}

	errors := make([]error, 0)

	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			errors = append(
				errors, fmt.Errorf("Unknown root level key in template: '%s'", unused))
		}
	}

	t = &Template{}
	t.Variables = make(map[string]string)
	t.Builders = make(map[string]rawBuilderConfig)
	t.Hooks = rawTpl.Hooks
	t.PostProcessors = make([][]rawPostProcessorConfig, len(rawTpl.PostProcessors))
	t.Provisioners = make([]rawProvisionerConfig, len(rawTpl.Provisioners))

	// Gather all the variables
	for k, v := range rawTpl.Variables {
		t.Variables[k] = v
	}

	// Gather all the builders
	for i, v := range rawTpl.Builders {
		var raw rawBuilderConfig
		if err := mapstructure.Decode(v, &raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("builder %d: missing 'type'", i+1))
			continue
		}

		// Attempt to get the name of the builder. If the "name" key
		// missing, use the "type" field, which is guaranteed to exist
		// at this point.
		if raw.Name == "" {
			raw.Name = raw.Type
		}

		// Check if we already have a builder with this name and error if so
		if _, ok := t.Builders[raw.Name]; ok {
			errors = append(errors, fmt.Errorf("builder with name '%s' already exists", raw.Name))
			continue
		}

		// Now that we have the name, remove it from the config - as the builder
		// itself doesn't know about, and it will cause a validation error.
		delete(v, "name")

		raw.rawConfig = v

		t.Builders[raw.Name] = raw
	}

	// Gather all the post-processors. This is a complicated process since there
	// are actually three different formats that the user can use to define
	// a post-processor.
	for i, rawV := range rawTpl.PostProcessors {
		rawPP, err := parsePostProvisioner(i, rawV)
		if err != nil {
			errors = append(errors, err...)
			continue
		}

		t.PostProcessors[i] = make([]rawPostProcessorConfig, len(rawPP))
		configs := t.PostProcessors[i]
		for j, pp := range rawPP {
			config := &configs[j]
			if err := mapstructure.Decode(pp, config); err != nil {
				if merr, ok := err.(*mapstructure.Error); ok {
					for _, err := range merr.Errors {
						errors = append(errors, fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err))
					}
				} else {
					errors = append(errors, fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err))
				}

				continue
			}

			if config.Type == "" {
				errors = append(errors, fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1))
				continue
			}

			config.rawConfig = pp
		}
	}

	// Gather all the provisioners
	for i, v := range rawTpl.Provisioners {
		raw := &t.Provisioners[i]
		if err := mapstructure.Decode(v, raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("provisioner %d: missing 'type'", i+1))
			continue
		}

		// The provisioners not only don't need or want the override settings
		// (as they are processed as part of the preparation below), but will
		// actively reject them as invalid configuration.
		delete(v, "override")

		raw.rawConfig = v
	}

	if len(t.Builders) == 0 {
		errors = append(errors, fmt.Errorf("No builders are defined in the template."))
	}

	// If there were errors, we put it into a MultiError and return
	if len(errors) > 0 {
		err = &MultiError{errors}
		t = nil
		return
	}

	return
}

// ParseTemplateFile takes the given template file and parses it into
// a single template.
func ParseTemplateFile(path string) (*Template, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	return ParseTemplate(data)
}

func parsePostProvisioner(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
	switch v := rawV.(type) {
	case string:
		result = []map[string]interface{}{
			{"type": v},
		}
	case map[string]interface{}:
		result = []map[string]interface{}{v}
	case []interface{}:
		result = make([]map[string]interface{}, len(v))
		errors = make([]error, 0)
		for j, innerRawV := range v {
			switch innerV := innerRawV.(type) {
			case string:
				result[j] = map[string]interface{}{"type": innerV}
			case map[string]interface{}:
				result[j] = innerV
			case []interface{}:
				errors = append(
					errors,
					fmt.Errorf("Post-processor %d.%d: sequences not allowed to be nested in sequences", i+1, j+1))
			default:
				errors = append(errors, fmt.Errorf("Post-processor %d.%d is in a bad format.", i+1, j+1))
			}
		}

		if len(errors) == 0 {
			errors = nil
		}
	default:
		result = nil
		errors = []error{fmt.Errorf("Post-processor %d is in a bad format.", i+1)}
	}

	return
}

// BuildNames returns a slice of the available names of builds that
// this template represents.
func (t *Template) BuildNames() []string {
	names := make([]string, 0, len(t.Builders))
	for name, _ := range t.Builders {
		names = append(names, name)
	}

	return names
}

// Build returns a Build for the given name.
//
// If the build does not exist as part of this template, an error is
// returned.
func (t *Template) Build(name string, components *ComponentFinder) (b Build, err error) {
	// Setup the Builder
	builderConfig, ok := t.Builders[name]
	if !ok {
		err = fmt.Errorf("No such build found in template: %s", name)
		return
	}

	// We panic if there is no builder function because this is really
	// an internal bug that always needs to be fixed, not an error.
	if components.Builder == nil {
		panic("no builder function")
	}

	// Panic if there are provisioners on the template but no provisioner
	// component finder. This is always an internal error, so we panic.
	if len(t.Provisioners) > 0 && components.Provisioner == nil {
		panic("no provisioner function")
	}

	builder, err := components.Builder(builderConfig.Type)
	if err != nil {
		return
	}

	if builder == nil {
		err = fmt.Errorf("Builder type not found: %s", builderConfig.Type)
		return
	}

	// Gather the Hooks
	hooks := make(map[string][]Hook)
	for tplEvent, tplHooks := range t.Hooks {
		curHooks := make([]Hook, 0, len(tplHooks))

		for _, hookName := range tplHooks {
			var hook Hook
			hook, err = components.Hook(hookName)
			if err != nil {
				return
			}

			if hook == nil {
				err = fmt.Errorf("Hook not found: %s", hookName)
				return
			}

			curHooks = append(curHooks, hook)
		}

		hooks[tplEvent] = curHooks
	}

	// Prepare the post-processors
	postProcessors := make([][]coreBuildPostProcessor, 0, len(t.PostProcessors))
	for _, rawPPs := range t.PostProcessors {
		current := make([]coreBuildPostProcessor, len(rawPPs))
		for i, rawPP := range rawPPs {
			pp, err := components.PostProcessor(rawPP.Type)
			if err != nil {
				return nil, err
			}

			if pp == nil {
				return nil, fmt.Errorf("PostProcessor type not found: %s", rawPP.Type)
			}

			current[i] = coreBuildPostProcessor{
				processor:         pp,
				processorType:     rawPP.Type,
				config:            rawPP.rawConfig,
				keepInputArtifact: rawPP.KeepInputArtifact,
			}
		}

		postProcessors = append(postProcessors, current)
	}

	// Prepare the provisioners
	provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners))
	for _, rawProvisioner := range t.Provisioners {
		var provisioner Provisioner
		provisioner, err = components.Provisioner(rawProvisioner.Type)
		if err != nil {
			return
		}

		if provisioner == nil {
			err = fmt.Errorf("Provisioner type not found: %s", rawProvisioner.Type)
			return
		}

		configs := make([]interface{}, 1, 2)
		configs[0] = rawProvisioner.rawConfig

		if rawProvisioner.Override != nil {
			if override, ok := rawProvisioner.Override[name]; ok {
				configs = append(configs, override)
			}
		}

		coreProv := coreBuildProvisioner{provisioner, configs}
		provisioners = append(provisioners, coreProv)
	}

	b = &coreBuild{
		name:           name,
		builder:        builder,
		builderConfig:  builderConfig.rawConfig,
		builderType:    builderConfig.Type,
		hooks:          hooks,
		postProcessors: postProcessors,
		provisioners:   provisioners,
		variables:      t.Variables,
	}

	return
}
<commit_msg>packer: Export the raw template config structs<commit_after>package packer

import (
	"fmt"
	"github.com/mitchellh/mapstructure"
	jsonutil "github.com/mitchellh/packer/common/json"
	"io/ioutil"
	"sort"
)

// The rawTemplate struct represents the structure of a template read
// directly from a file. The builders and other components map just to
// "interface{}" pointers since we actually don't know what their contents
// are until we read the "type" field.
type rawTemplate struct {
	Variables      map[string]string
	Builders       []map[string]interface{}
	Hooks          map[string][]string
	Provisioners   []map[string]interface{}
	PostProcessors []interface{} `mapstructure:"post-processors"`
}

// The Template struct represents a parsed template, parsed into the most
// completed form it can be without additional processing by the caller.
type Template struct {
	Variables      map[string]string
	Builders       map[string]RawBuilderConfig
	Hooks          map[string][]string
	PostProcessors [][]RawPostProcessorConfig
	Provisioners   []RawProvisionerConfig
}

// The RawBuilderConfig struct represents a raw, unprocessed builder
// configuration. It contains the name of the builder as well as the
// raw configuration. If requested, this is used to compile into a full
// builder configuration at some point.
type RawBuilderConfig struct {
	Name string
	Type string

	rawConfig interface{}
}

// RawPostProcessorConfig represents a raw, unprocessed post-processor
// configuration. It contains the type of the post processor as well as the
// raw configuration that is handed to the post-processor for it to process.
type RawPostProcessorConfig struct {
	Type              string
	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
	rawConfig         interface{}
}

// RawProvisionerConfig represents a raw, unprocessed provisioner configuration.
// It contains the type of the provisioner as well as the raw configuration
// that is handed to the provisioner for it to process.
type RawProvisionerConfig struct {
	Type     string
	Override map[string]interface{}

	rawConfig interface{}
}

// ParseTemplate takes a byte slice and parses a Template from it, returning
// the template and possibly errors while loading the template. The error
// could potentially be a MultiError, representing multiple errors. Knowing
// and checking for this can be useful, if you wish to format it in a certain
// way.
func ParseTemplate(data []byte) (t *Template, err error) {
	var rawTplInterface interface{}
	err = jsonutil.Unmarshal(data, &rawTplInterface)
	if err != nil {
		return
	}

	// Decode the raw template interface into the actual rawTemplate
	// structure, checking for any extranneous keys along the way.
	var md mapstructure.Metadata
	var rawTpl rawTemplate
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &rawTpl,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return
	}

	err = decoder.Decode(rawTplInterface)
	if err != nil {
		return
	}

	errors := make([]error, 0)

	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			errors = append(
				errors, fmt.Errorf("Unknown root level key in template: '%s'", unused))
		}
	}

	t = &Template{}
	t.Variables = make(map[string]string)
	t.Builders = make(map[string]RawBuilderConfig)
	t.Hooks = rawTpl.Hooks
	t.PostProcessors = make([][]RawPostProcessorConfig, len(rawTpl.PostProcessors))
	t.Provisioners = make([]RawProvisionerConfig, len(rawTpl.Provisioners))

	// Gather all the variables
	for k, v := range rawTpl.Variables {
		t.Variables[k] = v
	}

	// Gather all the builders
	for i, v := range rawTpl.Builders {
		var raw RawBuilderConfig
		if err := mapstructure.Decode(v, &raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("builder %d: missing 'type'", i+1))
			continue
		}

		// Attempt to get the name of the builder. If the "name" key
		// missing, use the "type" field, which is guaranteed to exist
		// at this point.
		if raw.Name == "" {
			raw.Name = raw.Type
		}

		// Check if we already have a builder with this name and error if so
		if _, ok := t.Builders[raw.Name]; ok {
			errors = append(errors, fmt.Errorf("builder with name '%s' already exists", raw.Name))
			continue
		}

		// Now that we have the name, remove it from the config - as the builder
		// itself doesn't know about, and it will cause a validation error.
		delete(v, "name")

		raw.rawConfig = v

		t.Builders[raw.Name] = raw
	}

	// Gather all the post-processors. This is a complicated process since there
	// are actually three different formats that the user can use to define
	// a post-processor.
	for i, rawV := range rawTpl.PostProcessors {
		rawPP, err := parsePostProvisioner(i, rawV)
		if err != nil {
			errors = append(errors, err...)
			continue
		}

		t.PostProcessors[i] = make([]RawPostProcessorConfig, len(rawPP))
		configs := t.PostProcessors[i]
		for j, pp := range rawPP {
			config := &configs[j]
			if err := mapstructure.Decode(pp, config); err != nil {
				if merr, ok := err.(*mapstructure.Error); ok {
					for _, err := range merr.Errors {
						errors = append(errors, fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err))
					}
				} else {
					errors = append(errors, fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err))
				}

				continue
			}

			if config.Type == "" {
				errors = append(errors, fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1))
				continue
			}

			config.rawConfig = pp
		}
	}

	// Gather all the provisioners
	for i, v := range rawTpl.Provisioners {
		raw := &t.Provisioners[i]
		if err := mapstructure.Decode(v, raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("provisioner %d: missing 'type'", i+1))
			continue
		}

		// The provisioners not only don't need or want the override settings
		// (as they are processed as part of the preparation below), but will
		// actively reject them as invalid configuration.
		delete(v, "override")

		raw.rawConfig = v
	}

	if len(t.Builders) == 0 {
		errors = append(errors, fmt.Errorf("No builders are defined in the template."))
	}

	// If there were errors, we put it into a MultiError and return
	if len(errors) > 0 {
		err = &MultiError{errors}
		t = nil
		return
	}

	return
}

// ParseTemplateFile takes the given template file and parses it into
// a single template.
func ParseTemplateFile(path string) (*Template, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	return ParseTemplate(data)
}

func parsePostProvisioner(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
	switch v := rawV.(type) {
	case string:
		result = []map[string]interface{}{
			{"type": v},
		}
	case map[string]interface{}:
		result = []map[string]interface{}{v}
	case []interface{}:
		result = make([]map[string]interface{}, len(v))
		errors = make([]error, 0)
		for j, innerRawV := range v {
			switch innerV := innerRawV.(type) {
			case string:
				result[j] = map[string]interface{}{"type": innerV}
			case map[string]interface{}:
				result[j] = innerV
			case []interface{}:
				errors = append(
					errors,
					fmt.Errorf("Post-processor %d.%d: sequences not allowed to be nested in sequences", i+1, j+1))
			default:
				errors = append(errors, fmt.Errorf("Post-processor %d.%d is in a bad format.", i+1, j+1))
			}
		}

		if len(errors) == 0 {
			errors = nil
		}
	default:
		result = nil
		errors = []error{fmt.Errorf("Post-processor %d is in a bad format.", i+1)}
	}

	return
}

// BuildNames returns a slice of the available names of builds that
// this template represents.
func (t *Template) BuildNames() []string {
	names := make([]string, 0, len(t.Builders))
	for name, _ := range t.Builders {
		names = append(names, name)
	}

	return names
}

// Build returns a Build for the given name.
//
// If the build does not exist as part of this template, an error is
// returned.
func (t *Template) Build(name string, components *ComponentFinder) (b Build, err error) {
	// Setup the Builder
	builderConfig, ok := t.Builders[name]
	if !ok {
		err = fmt.Errorf("No such build found in template: %s", name)
		return
	}

	// We panic if there is no builder function because this is really
	// an internal bug that always needs to be fixed, not an error.
	if components.Builder == nil {
		panic("no builder function")
	}

	// Panic if there are provisioners on the template but no provisioner
	// component finder. This is always an internal error, so we panic.
	if len(t.Provisioners) > 0 && components.Provisioner == nil {
		panic("no provisioner function")
	}

	builder, err := components.Builder(builderConfig.Type)
	if err != nil {
		return
	}

	if builder == nil {
		err = fmt.Errorf("Builder type not found: %s", builderConfig.Type)
		return
	}

	// Gather the Hooks
	hooks := make(map[string][]Hook)
	for tplEvent, tplHooks := range t.Hooks {
		curHooks := make([]Hook, 0, len(tplHooks))

		for _, hookName := range tplHooks {
			var hook Hook
			hook, err = components.Hook(hookName)
			if err != nil {
				return
			}

			if hook == nil {
				err = fmt.Errorf("Hook not found: %s", hookName)
				return
			}

			curHooks = append(curHooks, hook)
		}

		hooks[tplEvent] = curHooks
	}

	// Prepare the post-processors
	postProcessors := make([][]coreBuildPostProcessor, 0, len(t.PostProcessors))
	for _, rawPPs := range t.PostProcessors {
		current := make([]coreBuildPostProcessor, len(rawPPs))
		for i, rawPP := range rawPPs {
			pp, err := components.PostProcessor(rawPP.Type)
			if err != nil {
				return nil, err
			}

			if pp == nil {
				return nil, fmt.Errorf("PostProcessor type not found: %s", rawPP.Type)
			}

			current[i] = coreBuildPostProcessor{
				processor:         pp,
				processorType:     rawPP.Type,
				config:            rawPP.rawConfig,
				keepInputArtifact: rawPP.KeepInputArtifact,
			}
		}

		postProcessors = append(postProcessors, current)
	}

	// Prepare the provisioners
	provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners))
	for _, rawProvisioner := range t.Provisioners {
		var provisioner Provisioner
		provisioner, err = components.Provisioner(rawProvisioner.Type)
		if err != nil {
			return
		}

		if provisioner == nil {
			err = fmt.Errorf("Provisioner type not found: %s", rawProvisioner.Type)
			return
		}

		configs := make([]interface{}, 1, 2)
		configs[0] = rawProvisioner.rawConfig

		if rawProvisioner.Override != nil {
			if override, ok := rawProvisioner.Override[name]; ok {
				configs = append(configs, override)
			}
		}

		coreProv := coreBuildProvisioner{provisioner, configs}
		provisioners = append(provisioners, coreProv)
	}

	b = &coreBuild{
		name:           name,
		builder:        builder,
		builderConfig:  builderConfig.rawConfig,
		builderType:    builderConfig.Type,
		hooks:          hooks,
		postProcessors: postProcessors,
		provisioners:   provisioners,
		variables:      t.Variables,
	}

	return
}
<|endoftext|>


<commit_before>package tui

// Color represents a color.
type Color int

// Common colors.
const (
	ColorDefault Color = iota
	ColorBlack
	ColorWhite
	ColorRed
	ColorGreen
	ColorBlue
	ColorCyan
	ColorMagenta
	ColorYellow
)

// Decoration represents a bold/underline/etc. state
type Decoration int

const (
	DecorationInherit Decoration = iota
	DecorationOn
	DecorationOff
)

// Style determines how a cell should be painted.
// The zero value uses default from
type Style struct {
	Fg Color
	Bg Color

	Reverse   Decoration
	Bold      Decoration
	Underline Decoration
}

// mergeIn returns the receiver Style, with any changes in delta applied.
func (s Style) mergeIn(delta Style) Style {
	result := s
	if delta.Fg != ColorDefault {
		result.Fg = delta.Fg
	}
	if delta.Bg != ColorDefault {
		result.Bg = delta.Bg
	}
	if delta.Reverse != DecorationInherit {
		result.Reverse = delta.Reverse
	}
	if delta.Bold != DecorationInherit {
		result.Bold = delta.Bold
	}
	if delta.Underline != DecorationInherit {
		result.Underline = delta.Underline
	}
	return result
}

// Theme defines the styles for a set of identifiers.
type Theme struct {
	styles map[string]Style
}

// DefaultTheme is a theme with reasonable defaults.
var DefaultTheme = &Theme{
	styles: map[string]Style{
		"list.item.selected":  {Reverse: DecorationOn},
		"table.cell.selected": {Reverse: DecorationOn},
		"button.focused":      {Reverse: DecorationOn},
		"box.focused":         {Reverse: DecorationOn},
	},
}

// NewTheme return an empty theme.
func NewTheme() *Theme {
	return &Theme{
		styles: make(map[string]Style),
	}
}

// SetStyle sets a style for a given identifier.
func (p *Theme) SetStyle(n string, i Style) {
	p.styles[n] = i
}

// Style returns the style associated with an identifier.
// If there is no Style associated with the name, it returns a default Style.
func (p *Theme) Style(name string) Style {
	return p.styles[name]
}

// HasStyle returns whether an identifier is associated with an identifier.
func (p *Theme) HasStyle(name string) bool {
	_, ok := p.styles[name]
	return ok
}
<commit_msg>add docstring for Decoration block<commit_after>package tui

// Color represents a color.
type Color int

// Common colors.
const (
	ColorDefault Color = iota
	ColorBlack
	ColorWhite
	ColorRed
	ColorGreen
	ColorBlue
	ColorCyan
	ColorMagenta
	ColorYellow
)

// Decoration represents a bold/underline/etc. state
type Decoration int

// Decoration modes: Inherit from parent widget, explicitly on, or explicitly off.
const (
	DecorationInherit Decoration = iota
	DecorationOn
	DecorationOff
)

// Style determines how a cell should be painted.
// The zero value uses default from
type Style struct {
	Fg Color
	Bg Color

	Reverse   Decoration
	Bold      Decoration
	Underline Decoration
}

// mergeIn returns the receiver Style, with any changes in delta applied.
func (s Style) mergeIn(delta Style) Style {
	result := s
	if delta.Fg != ColorDefault {
		result.Fg = delta.Fg
	}
	if delta.Bg != ColorDefault {
		result.Bg = delta.Bg
	}
	if delta.Reverse != DecorationInherit {
		result.Reverse = delta.Reverse
	}
	if delta.Bold != DecorationInherit {
		result.Bold = delta.Bold
	}
	if delta.Underline != DecorationInherit {
		result.Underline = delta.Underline
	}
	return result
}

// Theme defines the styles for a set of identifiers.
type Theme struct {
	styles map[string]Style
}

// DefaultTheme is a theme with reasonable defaults.
var DefaultTheme = &Theme{
	styles: map[string]Style{
		"list.item.selected":  {Reverse: DecorationOn},
		"table.cell.selected": {Reverse: DecorationOn},
		"button.focused":      {Reverse: DecorationOn},
		"box.focused":         {Reverse: DecorationOn},
	},
}

// NewTheme return an empty theme.
func NewTheme() *Theme {
	return &Theme{
		styles: make(map[string]Style),
	}
}

// SetStyle sets a style for a given identifier.
func (p *Theme) SetStyle(n string, i Style) {
	p.styles[n] = i
}

// Style returns the style associated with an identifier.
// If there is no Style associated with the name, it returns a default Style.
func (p *Theme) Style(name string) Style {
	return p.styles[name]
}

// HasStyle returns whether an identifier is associated with an identifier.
func (p *Theme) HasStyle(name string) bool {
	_, ok := p.styles[name]
	return ok
}
<|endoftext|>


<commit_before>// Copyright 2018, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Program iot uploads sensor data including temperature, humidity, sound and light strength to monitoring backend by
// using the OpenCensus framework.
package main

import (
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"time"

	"contrib.go.opencensus.io/exporter/stackdriver"
	"github.com/d2r2/go-dht"
	"github.com/d2r2/go-logger"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"gobot.io/x/gobot"
	"gobot.io/x/gobot/drivers/aio"
	"gobot.io/x/gobot/drivers/i2c"
	"gobot.io/x/gobot/platforms/raspi"
)

var (
	// view to see the sound strength distribution.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewSoundDist = &view.View{
		Name:        "opencensus.io/views/sound_strength_distribution",
		Description: "sound strength distribution over time",
		Measure:     soundStrengthMeasure,
		Aggregation: view.Distribution(0, 2, 4, 8, 16, 32, 64, 128),
	}

	// view to see the sound strength instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewSoundLast = &view.View{
		Name:        "opencensus.io/views/sound_strength_instant",
		Description: "sound strength instantly over time",
		Measure:     soundStrengthMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the light strength instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewLight = &view.View{
		Name:        "opencensus.io/views/light_strength_instant",
		Description: "voltage level on GPIO over time",
		Measure:     lightStrengthMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the humidity instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewHumidity = &view.View{
		Name:        "opencensus.io/views/humidity_instant",
		Description: "humidity_over time",
		TagKeys:     []tag.Key{sensorKey},
		Measure:     humidityMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the temperature instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewTemperature = &view.View{
		Name:        "opencensus.io/views/temperature_instant",
		Description: "temperature over time",
		TagKeys:     []tag.Key{sensorKey},
		Measure:     temperatureMeasure,
		Aggregation: view.LastValue(),
	}

	// Apply two kinds of aggregation type to the same metric in order to see the difference.
	soundStrengthMeasure = stats.Int64("opencensus.io/measure/sound_strength_svl_mp1_7c3c", "strength of sound", stats.UnitDimensionless)
	lightStrengthMeasure = stats.Int64("opencensus.io/measure/light_strength_svl_mp1_7c3c", "strength of light", stats.UnitDimensionless)
	humidityMeasure      = stats.Float64("opencensus.io/measure/humidity_svl_mp1_7c3c", "humidity", stats.UnitDimensionless)
	temperatureMeasure   = stats.Float64("opencensus.io/measure/temperature_svl_mp1_7c3c", "temperature", stats.UnitDimensionless)

	soundSamplePeriod       = 50 * time.Millisecond
	temperatureSamplePeriod = 5 * time.Second

	sensorKey tag.Key

	lg = logger.NewPackageLogger("main",
		logger.DebugLevel,
		// logger.InfoLevel,
	)
)

// The board would connect to two DHT11 temperature sensors with the GPIO4 and GPIO17.
// Communicate with ADS1015S based on the I2C and connect the sound
// and light sensor to the A0, A1 channel on the ADC module.
func main() {
	ctx := context.Background()
	projectId := os.Getenv("PROJECTID")
	if projectId == "" {
		log.Fatal("Cannot detect PROJECTID in the system environment.\n")
	} else {
		log.Printf("Project Id is set to be %s\n", projectId)
	}

	var err error
	sensorKey, err = tag.NewKey("opencensus.io/keys/sensor")
	if err != nil {
		log.Fatal(err)
	}

	initOpenCensus(projectId, 1)
	// Create a new go thread to record the temperature and humidity
	go RecordTemperatureHumidity(ctx, 4)
	go RecordTemperatureHumidity(ctx, 17)

	board := raspi.NewAdaptor()
	ads1015 := i2c.NewADS1015Driver(board)
	soundSensor := aio.NewGroveSoundSensorDriver(ads1015, "0")
	lightSensor := aio.NewGroveLightSensorDriver(ads1015, "1")

	work := func() {
		gobot.Every(soundSamplePeriod, func() {
			// Since the sample shares the same pin, it cannot be done concurrently.
			recordSound(ctx, soundSensor)
			recordLight(ctx, lightSensor)
		})
	}

	robot := gobot.NewRobot("sensorDataCollection",
		[]gobot.Connection{board},
		[]gobot.Device{ads1015},
		work,
	)
	robot.Start()
}

// Record the sound strength based on two kinds of aggregation.
// One is distribution, the other is the lastValue.
func recordSound(ctx context.Context, soundSensor *aio.GroveSoundSensorDriver) {
	soundStrength, soundErr := readSound(soundSensor)
	if soundErr != nil {
		log.Fatalf("Could not read value from sound sensors\n")
	} else {
		stats.Record(ctx, soundStrengthMeasure.M(int64(soundStrength)))
		//log.Printf("Sound Strength: %d\n", soundStrength)
	}
}

// Record the light strength.
func recordLight(ctx context.Context, lightSensor *aio.GroveLightSensorDriver) {
	lightStrength, lightErr := lightSensor.Read()
	if lightErr != nil {
		log.Fatalf("Could not read value from light sensors\n")
	} else {
		stats.Record(ctx, lightStrengthMeasure.M(int64(lightStrength)))
		//log.Printf("Light Strength: %d\n", lightStrength)
	}
}

// Sample 50 sound strength data in a period.
// Calculate the maximum and minimum value and return their difference
func readSound(sensor *aio.GroveSoundSensorDriver) (int, error) {
	min := math.MaxInt32
	max := math.MinInt32
	for i := 0; i < 50; i++ {
		strength, err := sensor.Read()
		if err != nil {
			log.Fatalf("Couldn't read data from the sensor\n")
		} else {
			if strength > max {
				max = strength
			}
			if strength < min {
				min = strength
			}
		}
	}
	return max - min, nil
}

// For every five seconds, record the temperature and humidity sensor data.
// Print the collected data on the console.
func RecordTemperatureHumidity(ctx context.Context, pin int) {
	ctx, err := tag.New(ctx,
		tag.Insert(sensorKey, fmt.Sprintf("Sensor :%d", pin)),
	)
	if err != nil {
		log.Fatal(err)
	}
	for range time.Tick(temperatureSamplePeriod) {
		defer logger.FinalizeLogger()
		// Uncomment/comment next line to suppress/increase verbosity of output
		logger.ChangePackageLogLevel("dht", logger.InfoLevel)

		sensorType := dht.DHT11
		// Read DHT11 sensor data from pin 4, retrying 50 times in case of failure.
		// You may enable "boost GPIO performance" parameter, if your device is old
		// as Raspberry PI 1 (this will require root privileges). You can switch off
		// "boost GPIO performance" parameter for old devices, but it may increase
		// retry attempts. Play with this parameter.
		temperature, humidity, retried, err :=
			dht.ReadDHTxxWithRetry(sensorType, pin, false, 50)
		if err != nil {
			lg.Fatal(err)
		}
		if temperature > 0 && humidity > 0 && err != nil && retried > 0 {
		}
		// print temperature and humidity
		lg.Infof("Sensor = %v: Temperature = %v*C, Humidity = %v%% (retried %d times)",
			sensorType, temperature, humidity, retried)
		stats.Record(ctx, temperatureMeasure.M(float64(temperature)))
		stats.Record(ctx, humidityMeasure.M(float64(humidity)))
	}
}

// Initialize the openCensus framework.
// If there is anything wrong with the registration, directly throw a fatal error.
func initOpenCensus(projectId string, reportPeriod int) {
	// Collected view data will be reported to Stackdriver Monitoring API
	// via the Stackdriver exporter.
	//
	// In order to use the Stackdriver exporter, enable Stackdriver Monitoring API
	// at https://console.cloud.google.com/apis/dashboard.
	//
	// Once API is enabled, you can use Google Application Default Credentials
	// to setup the authorization.
	// See https://developers.google.com/identity/protocols/application-default-credentials
	// for more details.
	exporter, err := stackdriver.NewExporter(stackdriver.Options{
		ProjectID: projectId, // Google Cloud Console project ID.
	})
	if err != nil {
		log.Fatal(err)
	}
	view.RegisterExporter(exporter)

	// Set reporting period to report data based on the given reportPeriod.
	view.SetReportingPeriod(time.Second * time.Duration(reportPeriod))

	viewList := []*view.View{viewSoundDist, viewSoundLast, viewLight, viewHumidity, viewTemperature}

	for _, viewToRegister := range viewList {
		if err := view.Register(viewToRegister); err != nil {
			log.Fatalf("Cannot subscribe to the view: %v", err)
		}
	}

}
<commit_msg>Comments on how to connect Sensors to Pi (#58)<commit_after>// Copyright 2018, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Program iot uploads sensor data including temperature, humidity, sound and light strength to monitoring backend by
// using the OpenCensus framework.
//
// Hardware Connections (Sensors to Raspberry Pi):
// -DHT11 Out Pin = GPIO 4
// -DHT11 Out Pin = GPIO 17
// -ADS1015 SDA = GPIO 2 (SDA)
// -ADS1015 SCL = GPIO 3 (SCL)
// -Light Sensor Out Pin = ADS1015 A1
// -Sound Sensor Out Pin = ADS1015 A0
package main

import (
	"context"
	"fmt"
	"log"
	"math"
	"os"
	"time"

	"contrib.go.opencensus.io/exporter/stackdriver"
	"github.com/d2r2/go-dht"
	"github.com/d2r2/go-logger"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"gobot.io/x/gobot"
	"gobot.io/x/gobot/drivers/aio"
	"gobot.io/x/gobot/drivers/i2c"
	"gobot.io/x/gobot/platforms/raspi"
)

var (
	// view to see the sound strength distribution.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewSoundDist = &view.View{
		Name:        "opencensus.io/views/sound_strength_distribution",
		Description: "sound strength distribution over time",
		Measure:     soundStrengthMeasure,
		Aggregation: view.Distribution(0, 2, 4, 8, 16, 32, 64, 128),
	}

	// view to see the sound strength instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewSoundLast = &view.View{
		Name:        "opencensus.io/views/sound_strength_instant",
		Description: "sound strength instantly over time",
		Measure:     soundStrengthMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the light strength instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewLight = &view.View{
		Name:        "opencensus.io/views/light_strength_instant",
		Description: "voltage level on GPIO over time",
		Measure:     lightStrengthMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the humidity instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewHumidity = &view.View{
		Name:        "opencensus.io/views/humidity_instant",
		Description: "humidity_over time",
		TagKeys:     []tag.Key{sensorKey},
		Measure:     humidityMeasure,
		Aggregation: view.LastValue(),
	}

	// view to see the temperature instantly.
	// Subscribe will allow view data to be exported.
	// Once no longer need, you can unsubscribe from the view.
	viewTemperature = &view.View{
		Name:        "opencensus.io/views/temperature_instant",
		Description: "temperature over time",
		TagKeys:     []tag.Key{sensorKey},
		Measure:     temperatureMeasure,
		Aggregation: view.LastValue(),
	}

	// Apply two kinds of aggregation type to the same metric in order to see the difference.
	soundStrengthMeasure = stats.Int64("opencensus.io/measure/sound_strength_svl_mp1_7c3c", "strength of sound", stats.UnitDimensionless)
	lightStrengthMeasure = stats.Int64("opencensus.io/measure/light_strength_svl_mp1_7c3c", "strength of light", stats.UnitDimensionless)
	humidityMeasure      = stats.Float64("opencensus.io/measure/humidity_svl_mp1_7c3c", "humidity", stats.UnitDimensionless)
	temperatureMeasure   = stats.Float64("opencensus.io/measure/temperature_svl_mp1_7c3c", "temperature", stats.UnitDimensionless)

	soundSamplePeriod       = 50 * time.Millisecond
	temperatureSamplePeriod = 5 * time.Second

	sensorKey tag.Key

	lg = logger.NewPackageLogger("main",
		logger.DebugLevel,
		// logger.InfoLevel,
	)
)

// The board would connect to two DHT11 temperature sensors with the GPIO4 and GPIO17.
// Communicate with ADS1015S based on the I2C and connect the sound
// and light sensor to the A0, A1 channel on the ADC module.
func main() {
	ctx := context.Background()
	projectId := os.Getenv("PROJECTID")
	if projectId == "" {
		log.Fatal("Cannot detect PROJECTID in the system environment.\n")
	} else {
		log.Printf("Project Id is set to be %s\n", projectId)
	}

	var err error
	sensorKey, err = tag.NewKey("opencensus.io/keys/sensor")
	if err != nil {
		log.Fatal(err)
	}

	initOpenCensus(projectId, 1)
	// Create a new go thread to record the temperature and humidity
	go RecordTemperatureHumidity(ctx, 4)
	go RecordTemperatureHumidity(ctx, 17)

	board := raspi.NewAdaptor()
	ads1015 := i2c.NewADS1015Driver(board)
	soundSensor := aio.NewGroveSoundSensorDriver(ads1015, "0")
	lightSensor := aio.NewGroveLightSensorDriver(ads1015, "1")

	work := func() {
		gobot.Every(soundSamplePeriod, func() {
			// Since the sample shares the same pin, it cannot be done concurrently.
			recordSound(ctx, soundSensor)
			recordLight(ctx, lightSensor)
		})
	}

	robot := gobot.NewRobot("sensorDataCollection",
		[]gobot.Connection{board},
		[]gobot.Device{ads1015},
		work,
	)
	robot.Start()
}

// Record the sound strength based on two kinds of aggregation.
// One is distribution, the other is the lastValue.
func recordSound(ctx context.Context, soundSensor *aio.GroveSoundSensorDriver) {
	soundStrength, soundErr := readSound(soundSensor)
	if soundErr != nil {
		log.Fatalf("Could not read value from sound sensors\n")
	} else {
		stats.Record(ctx, soundStrengthMeasure.M(int64(soundStrength)))
		//log.Printf("Sound Strength: %d\n", soundStrength)
	}
}

// Record the light strength.
func recordLight(ctx context.Context, lightSensor *aio.GroveLightSensorDriver) {
	lightStrength, lightErr := lightSensor.Read()
	if lightErr != nil {
		log.Fatalf("Could not read value from light sensors\n")
	} else {
		stats.Record(ctx, lightStrengthMeasure.M(int64(lightStrength)))
		//log.Printf("Light Strength: %d\n", lightStrength)
	}
}

// Sample 50 sound strength data in a period.
// Calculate the maximum and minimum value and return their difference
func readSound(sensor *aio.GroveSoundSensorDriver) (int, error) {
	min := math.MaxInt32
	max := math.MinInt32
	for i := 0; i < 50; i++ {
		strength, err := sensor.Read()
		if err != nil {
			log.Fatalf("Couldn't read data from the sensor\n")
		} else {
			if strength > max {
				max = strength
			}
			if strength < min {
				min = strength
			}
		}
	}
	return max - min, nil
}

// For every five seconds, record the temperature and humidity sensor data.
// Print the collected data on the console.
func RecordTemperatureHumidity(ctx context.Context, pin int) {
	ctx, err := tag.New(ctx,
		tag.Insert(sensorKey, fmt.Sprintf("Sensor :%d", pin)),
	)
	if err != nil {
		log.Fatal(err)
	}
	for range time.Tick(temperatureSamplePeriod) {
		defer logger.FinalizeLogger()
		// Uncomment/comment next line to suppress/increase verbosity of output
		logger.ChangePackageLogLevel("dht", logger.InfoLevel)

		sensorType := dht.DHT11
		// Read DHT11 sensor data from pin 4, retrying 50 times in case of failure.
		// You may enable "boost GPIO performance" parameter, if your device is old
		// as Raspberry PI 1 (this will require root privileges). You can switch off
		// "boost GPIO performance" parameter for old devices, but it may increase
		// retry attempts. Play with this parameter.
		temperature, humidity, retried, err :=
			dht.ReadDHTxxWithRetry(sensorType, pin, false, 50)
		if err != nil {
			lg.Fatal(err)
		}
		if temperature > 0 && humidity > 0 && err != nil && retried > 0 {
		}
		// print temperature and humidity
		lg.Infof("Sensor = %v: Temperature = %v*C, Humidity = %v%% (retried %d times)",
			sensorType, temperature, humidity, retried)
		stats.Record(ctx, temperatureMeasure.M(float64(temperature)))
		stats.Record(ctx, humidityMeasure.M(float64(humidity)))
	}
}

// Initialize the openCensus framework.
// If there is anything wrong with the registration, directly throw a fatal error.
func initOpenCensus(projectId string, reportPeriod int) {
	// Collected view data will be reported to Stackdriver Monitoring API
	// via the Stackdriver exporter.
	//
	// In order to use the Stackdriver exporter, enable Stackdriver Monitoring API
	// at https://console.cloud.google.com/apis/dashboard.
	//
	// Once API is enabled, you can use Google Application Default Credentials
	// to setup the authorization.
	// See https://developers.google.com/identity/protocols/application-default-credentials
	// for more details.
	exporter, err := stackdriver.NewExporter(stackdriver.Options{
		ProjectID: projectId, // Google Cloud Console project ID.
	})
	if err != nil {
		log.Fatal(err)
	}
	view.RegisterExporter(exporter)

	// Set reporting period to report data based on the given reportPeriod.
	view.SetReportingPeriod(time.Second * time.Duration(reportPeriod))

	viewList := []*view.View{viewSoundDist, viewSoundLast, viewLight, viewHumidity, viewTemperature}

	for _, viewToRegister := range viewList {
		if err := view.Register(viewToRegister); err != nil {
			log.Fatalf("Cannot subscribe to the view: %v", err)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	logpkg "log"
	"log/syslog"
	"os"
	"os/exec"
	"os/user"
	"path"
	"syscall"
	"time"

	"github.com/dotcloud/docker/pkg/mount"
)

// Must be true everywhere.
const databoxGid = 10000

var pamUser = os.Getenv("PAM_USER")

var log *logpkg.Logger

func init() {
	os.Args[0] = "PSSO"
	log, _ = syslog.NewLogger(syslog.LOG_WARNING|syslog.LOG_AUTH, 0)
}

func Fatal(first string, args ...interface{}) {
	// TODO(pwaller): send to syslog?
	log.Fatalf("pam script: "+first, args...)
}

func isDataboxUser() bool {
	u, err := user.Lookup(pamUser)
	if err != nil {
		Fatal("Failed to obtain passwd entry for %q", pamUser)
	}
	return u.Gid == fmt.Sprint(databoxGid)
}

func initMounts() {
	home := path.Join("/var/lib/cobalt/home/", pamUser)

	mounts := []struct{ src, tgt string }{
		{"/opt/basejail", "/jail"},
		{"/dev", "/jail/dev"},
		{"/dev/pts", "/jail/dev/pts"},
		{"/proc", "/jail/proc"},
		{"/var/spool/cron/crontabs", "/jail/var/spool/cron/crontabs"},
		{"/var/lib/extrausers", "/jail/var/lib/extrausers"},
		{home, "/jail/home"},
	}

	for _, m := range mounts {
		// Note the use of recursive bind mounts.
		// We could avoid some mounts by just arranging that /opt/basejail
		// already has most of the mounts.
		err := mount.Mount(m.src, m.tgt, "", "rbind")
		if err != nil {
			log.Fatalf("pamscript: Failed to mount %s -> %s: %q", m.src, m.tgt, err)
		}
	}
}

func cgcreate() error {
	args := []string{"-t", pamUser, "-g", "memory,cpu,cpuacct:" + pamUser}
	cmd := exec.Command("cgcreate", args...)
	return cmd.Run()
}

func initCgroup() {
	_, err := exec.LookPath("cgcreate")
	if err != nil {
		// cgroups isn't installed on this machine, NOOP.
		return
	}

	if _, err := os.Stat(path.Join("/sys/fs/cgroup/cpu", pamUser)); err != nil {
		err = cgcreate()
		if err != nil {
			Fatal("Failed to create cgroup")
		}
	}

	// 512MiB
	const memoryLimit = 512 * 1024 * 1024

	f := path.Join("/sys/fs/cgroup/memory", pamUser, "/memory.limit_in_bytes")
	err = ioutil.WriteFile(f, []byte(fmt.Sprint(memoryLimit)), 0)
	if err != nil {
		Fatal("Failed to write", f, ":", err)
	}

	// echo $MemoryLimit > /sys/fs/cgroup/memory/$PAM_USER/memory.limit_in_bytes

	// TODO(pwaller): do we want this? Maybe? Is there some other way we can give
	// system things priority?

	// # CPU share is form of priority. By specifying a low number here, we
	// # ensure that important system services get a higher share of the CPU
	// # and thus remain responsive.
	// Priority=12
	// echo $Priority > /sys/fs/cgroup/cpu/$PAM_USER/cpu.shares

	// Put the owning process (usually the "su -l" or cron child process)
	// into the cgroup (and therefore all of its future children)

	files := []string{
		path.Join("/sys/fs/cgroup/cpu", pamUser, "/tasks"),
		path.Join("/sys/fs/cgroup/memory", pamUser, "/tasks"),
		path.Join("/sys/fs/cgroup/cpuacct", pamUser, "/tasks"),
	}

	parentPid := []byte(fmt.Sprint(os.Getppid()))
	for _, f := range files {
		err = ioutil.WriteFile(f, parentPid, 0)
		if err != nil {
			Fatal("Failed to write", f, ":", err)
		}
	}
}

func verifyMountNamespace() {
	// if [[ "$(readlink /proc/1/ns/mnt)" == "$(readlink /proc/self/ns/mnt)" ]]; then
	initMountNS, err := os.Readlink("/proc/1/ns/mnt")
	if err != nil {
		Fatal("Unable to readlink(/proc/1/ns/mnt). Aborting.")
	}
	myMountNS, err := os.Readlink("/proc/self/ns/mnt")
	if err != nil {
		Fatal("Unable to readlink(/proc/self/ns/mnt). Aborting.")
	}
	if initMountNS == myMountNS {
		Fatal("Not in mount namespace. Abort.")
	}
}

func main() {
	me := os.Getpid()
	const HIGHEST_PRIORITY = -20
	err := syscall.Setpriority(syscall.PRIO_PROCESS, me, HIGHEST_PRIORITY)
	if err != nil {
		log.Println("Setpriority() ->", err)
	}

	start := time.Now()
	defer func() {
		// Include the time in milliseconds.
		timeMillis := time.Since(start).Seconds() * 1000
		s := fmt.Sprintf("$PAM_USER $PAM_SERVICE %f $PAM_RHOST", timeMillis)
		log.Println(os.ExpandEnv(s))
	}()

	if !isDataboxUser() {
		log.Println("Skip non-databox user")
		// skip non-databox login
		return
	}

	if pamUser == "" {
		Fatal("PAM_USER not set. Abort.")
	}

	verifyMountNamespace()

	initCgroup()
	initMounts()
}
<commit_msg>Add LockOSThread to pam script<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	logpkg "log"
	"log/syslog"
	"os"
	"os/exec"
	"os/user"
	"path"
	"runtime"
	"syscall"
	"time"

	"github.com/dotcloud/docker/pkg/mount"
)

// Must be true everywhere.
const databoxGid = 10000

var pamUser = os.Getenv("PAM_USER")

var log *logpkg.Logger

func init() {
	os.Args[0] = "PSSO"
	log, _ = syslog.NewLogger(syslog.LOG_WARNING|syslog.LOG_AUTH, 0)
}

func Fatal(first string, args ...interface{}) {
	// TODO(pwaller): send to syslog?
	log.Fatalf("pam script: "+first, args...)
}

func isDataboxUser() bool {
	u, err := user.Lookup(pamUser)
	if err != nil {
		Fatal("Failed to obtain passwd entry for %q", pamUser)
	}
	return u.Gid == fmt.Sprint(databoxGid)
}

func initMounts() {
	home := path.Join("/var/lib/cobalt/home/", pamUser)

	mounts := []struct{ src, tgt string }{
		{"/opt/basejail", "/jail"},
		{"/dev", "/jail/dev"},
		{"/dev/pts", "/jail/dev/pts"},
		{"/proc", "/jail/proc"},
		{"/var/spool/cron/crontabs", "/jail/var/spool/cron/crontabs"},
		{"/var/lib/extrausers", "/jail/var/lib/extrausers"},
		{home, "/jail/home"},
	}

	for _, m := range mounts {
		// Note the use of recursive bind mounts.
		// We could avoid some mounts by just arranging that /opt/basejail
		// already has most of the mounts.
		err := mount.Mount(m.src, m.tgt, "", "rbind")
		if err != nil {
			log.Fatalf("pamscript: Failed to mount %s -> %s: %q", m.src, m.tgt, err)
		}
	}
}

func cgcreate() error {
	args := []string{"-t", pamUser, "-g", "memory,cpu,cpuacct:" + pamUser}
	cmd := exec.Command("cgcreate", args...)
	return cmd.Run()
}

func initCgroup() {
	_, err := exec.LookPath("cgcreate")
	if err != nil {
		// cgroups isn't installed on this machine, NOOP.
		return
	}

	if _, err := os.Stat(path.Join("/sys/fs/cgroup/cpu", pamUser)); err != nil {
		err = cgcreate()
		if err != nil {
			Fatal("Failed to create cgroup")
		}
	}

	// 512MiB
	const memoryLimit = 512 * 1024 * 1024

	f := path.Join("/sys/fs/cgroup/memory", pamUser, "/memory.limit_in_bytes")
	err = ioutil.WriteFile(f, []byte(fmt.Sprint(memoryLimit)), 0)
	if err != nil {
		Fatal("Failed to write", f, ":", err)
	}

	// echo $MemoryLimit > /sys/fs/cgroup/memory/$PAM_USER/memory.limit_in_bytes

	// TODO(pwaller): do we want this? Maybe? Is there some other way we can give
	// system things priority?

	// # CPU share is form of priority. By specifying a low number here, we
	// # ensure that important system services get a higher share of the CPU
	// # and thus remain responsive.
	// Priority=12
	// echo $Priority > /sys/fs/cgroup/cpu/$PAM_USER/cpu.shares

	// Put the owning process (usually the "su -l" or cron child process)
	// into the cgroup (and therefore all of its future children)

	files := []string{
		path.Join("/sys/fs/cgroup/cpu", pamUser, "/tasks"),
		path.Join("/sys/fs/cgroup/memory", pamUser, "/tasks"),
		path.Join("/sys/fs/cgroup/cpuacct", pamUser, "/tasks"),
	}

	parentPid := []byte(fmt.Sprint(os.Getppid()))
	for _, f := range files {
		err = ioutil.WriteFile(f, parentPid, 0)
		if err != nil {
			Fatal("Failed to write", f, ":", err)
		}
	}
}

func verifyMountNamespace() {
	// if [[ "$(readlink /proc/1/ns/mnt)" == "$(readlink /proc/self/ns/mnt)" ]]; then
	initMountNS, err := os.Readlink("/proc/1/ns/mnt")
	if err != nil {
		Fatal("Unable to readlink(/proc/1/ns/mnt). Aborting.")
	}
	myMountNS, err := os.Readlink("/proc/self/ns/mnt")
	if err != nil {
		Fatal("Unable to readlink(/proc/self/ns/mnt). Aborting.")
	}
	if initMountNS == myMountNS {
		Fatal("Not in mount namespace. Abort.")
	}
}

func main() {

	// Voodoo: Ensure that code runs in the same thread with the high priority.
	// <pwaller> I did this because you can see threads that don't have the
	// highest priority. Hopefully this helps?
	runtime.LockOSThread()

	me := os.Getpid()
	const HIGHEST_PRIORITY = -20
	err := syscall.Setpriority(syscall.PRIO_PROCESS, me, HIGHEST_PRIORITY)
	if err != nil {
		log.Println("Setpriority() ->", err)
	}

	start := time.Now()
	defer func() {
		// Include the time in milliseconds.
		timeMillis := time.Since(start).Seconds() * 1000
		s := fmt.Sprintf("$PAM_USER $PAM_SERVICE %f $PAM_RHOST", timeMillis)
		log.Println(os.ExpandEnv(s))
	}()

	if !isDataboxUser() {
		log.Println("Skip non-databox user")
		// skip non-databox login
		return
	}

	if pamUser == "" {
		Fatal("PAM_USER not set. Abort.")
	}

	verifyMountNamespace()

	initCgroup()
	initMounts()
}
<|endoftext|>